Skip to content
Commits on Source (175)
before_script:
- "pip install -r .ci/requirements.txt"
- "REQUIREMENTS=$(mktemp) && python setup.py requirements > ${REQUIREMENTS} && pip install -r ${REQUIREMENTS} && rm ${REQUIREMENTS}"
- "pip install -r .ci/requirements_modules.txt"
- "pip3 install -r .ci/requirements.txt"
- "REQUIREMENTS=$(mktemp) && python3 setup.py requirements > ${REQUIREMENTS} && pip3 install -r ${REQUIREMENTS} && rm ${REQUIREMENTS}"
- "pip3 install -r .ci/requirements_modules.txt"
build:2.7:
image: "python:2.7"
stage: "build"
script:
- "./tools/local_install.sh ~/bin"
- "./tools/local_install.sh -2 ~/bin"
pyflakes:2.7:
image: "python:2.7"
stage: "test"
script:
- "./tools/pyflakes.sh"
- "./tools/pyflakes.sh -2"
lint:2.7:
image: "python:2.7"
stage: "test"
allow_failure: true
script:
- "./tools/weboob_lint.sh"
- "./tools/weboob_lint.sh -2"
unittests:2.7:
image: "python:2.7"
stage: "test"
script:
- "./tools/run_tests.sh --no-modules"
- "./tools/run_tests.sh -2 --no-modules"
unittests-modules:2.7:
image: "python:2.7"
stage: "test"
allow_failure: true
script:
- "./tools/run_tests.sh --no-core"
- "./tools/run_tests.sh -2 --no-core"
doc:2.7:
image: "python:2.7"
......@@ -45,33 +45,33 @@ build:3:
image: "python:3"
stage: "build"
script:
- "./tools/local_install.sh -3 ~/bin"
- "./tools/local_install.sh ~/bin"
pyflakes:3:
image: "python:3"
stage: "test"
script:
- "./tools/pyflakes.sh -3"
- "./tools/pyflakes.sh"
lint:3:
image: "python:3"
stage: "test"
allow_failure: true
script:
- "./tools/weboob_lint.sh -3"
- "./tools/weboob_lint.sh"
unittests:3:
image: "python:3"
stage: "test"
script:
- "./tools/run_tests.sh -3 --no-modules"
- "./tools/run_tests.sh --no-modules"
unittests-modules:3:
image: "python:3"
stage: "test"
allow_failure: true
script:
- "./tools/run_tests.sh -3 --no-core"
- "./tools/run_tests.sh --no-core"
doc:3:
image: "python:3"
......
......@@ -10,7 +10,7 @@ you have to handle them by hand, according to your distribution.
The requirements are provided in ``setup.py``, except for:
* gpgv (for secure updates). If not packaged alone, it should be in ``gnupg`` or ``gpg``.
* PyQt5 (python-pyqt5, pyqt5-dev-tools, python-pyqt5.qtmultimedia) for graphical applications.
* PyQt5 (python3-pyqt5, pyqt5-dev-tools, python3-pyqt5.qtmultimedia) for graphical applications.
* For more performance, ensure you have ``libyaml`` and ``simplejson`` installed.
Some modules may have more dependencies.
......@@ -36,8 +36,7 @@ System installation (discouraged)
---------------------------------
The install mode copies files to the Python system-wide packages directory (for
example ``/usr/lib/python2.5/site-packages`` for Python 2.5, or
``/usr/local/lib/python2.6/dist-packages`` for Python 2.6). ::
example ``/usr/lib/python3.7/dist-packages`` for Python 3.7. ::
# ./setup.py install
......
#!/usr/bin/env python
#!/usr/bin/env python3
import logging
import os
......
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright(C) 2012 Alexandre Flament
......
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
......
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
......
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
......
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright(C) 2017 Matthieu Weber
......
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
......
#!/usr/bin/env python
#!/usr/bin/env python3
import os
......
......@@ -21,27 +21,28 @@
from random import randint
from weboob.browser import URL, LoginBrowser, need_login
from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable
from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired
from weboob.tools.compat import basestring
from .pages import LoginPage, IndexPage, BadLogin, AccountDetailPage, AccountHistoryPage
from .pages import (
LoginPage, IndexPage, WrongPasswordPage, WrongWebsitePage,
AccountDetailPage, AccountHistoryPage, MigrationPage,
)
class AferBrowser(LoginBrowser):
BASEURL = 'https://adherent.gie-afer.fr'
login = URL('/web/ega.nsf/listeAdhesions\?OpenForm', LoginPage)
bad_login = URL('/names.nsf\?Login', BadLogin)
login = URL(r'/espaceadherent/MonCompte/Connexion$', LoginPage)
wrong_password = URL(r'/espaceadherent/MonCompte/Connexion\?err=6001', WrongPasswordPage)
wrong_website = URL(r'/espaceadherent/MonCompte/Connexion\?err=6008', WrongWebsitePage)
migration = URL(r'/espaceadherent/MonCompte/Migration', MigrationPage)
index = URL('/web/ega.nsf/listeAdhesions\?OpenForm', IndexPage)
account_detail = URL('/web/ega.nsf/soldeEpargne\?openForm', AccountDetailPage)
account_history = URL('/web/ega.nsf/generationSearchModule\?OpenAgent', AccountHistoryPage)
history_detail = URL('/web/ega.nsf/WOpendetailOperation\?OpenAgent', AccountHistoryPage)
def do_login(self):
"""
Attempt to log in.
Note: this method does nothing if we are already logged in.
"""
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
self.login.go()
......@@ -51,13 +52,14 @@ def do_login(self):
except BrowserUnavailable:
raise BrowserIncorrectPassword()
if self.bad_login.is_here():
if self.migration.is_here():
raise BrowserPasswordExpired(self.page.get_error())
if self.wrong_password.is_here():
error = self.page.get_error()
if "La saisie de l’identifiant ou du code confidentiel est incorrecte" in error or \
"Veuillez-vous identifier" in error:
if error:
raise BrowserIncorrectPassword(error)
else:
assert False, "Message d'erreur inconnu: %s" % error
assert False, 'We landed on WrongPasswordPage but no error message was fetched.'
@need_login
......
......@@ -39,8 +39,11 @@ class AferModule(Module, CapBankWealth):
VERSION = '1.6'
BROWSER = AferBrowser
CONFIG = BackendConfig(ValueBackendPassword('login', label='Username', regexp='[A-z]\d+', masked=False),
ValueBackendPassword('password', label=u"mdp", regexp='\d{1,8}'))
CONFIG = BackendConfig(
ValueBackendPassword('login', label='Identifiant', regexp=r'.+', masked=False),
ValueBackendPassword('password', label="Mot de passe", regexp=r'\d{1,8}|[a-zA-Z0-9]{7,30}')
# TODO lose previous regex (and in backend) once users credentials migration is complete
)
def create_default_browser(self):
return self.create_browser(self.config['login'].get(),
......
......@@ -33,18 +33,31 @@
class LoginPage(HTMLPage):
def login(self, login, passwd):
form = self.get_form(name='_DominoForm')
form['Username'] = login
form = self.get_form(id='loginForm')
form['username'] = login
form['password'] = passwd
form.submit()
def is_here(self):
return bool(self.doc.xpath('//form[@name="_DominoForm"]'))
class WrongPasswordPage(HTMLPage):
def get_error(self):
return CleanText('//p[contains(text(), "Votre saisie est erronée")]')(self.doc)
class WrongWebsitePage(HTMLPage):
# We land on this page when the website indicates that
# an account is already created on the 'Aviva et moi' space,
# So we check the message and raise ActionNeeded with it
def on_load(self):
message = CleanText('//p[contains(text(), "Vous êtes déjà inscrit")]')(self.doc)
if message:
raise ActionNeeded(message)
assert False, 'We landed on WrongWebsitePage but no message was fetched.'
class BadLogin(HTMLPage):
class MigrationPage(HTMLPage):
def get_error(self):
return CleanText('//div[@id="idDivErrorLogin"]')(self.doc)
return CleanText('//h1[contains(text(), "Votre nouvel identifiant et mot de passe")]')(self.doc)
class IndexPage(LoggedPage, HTMLPage):
......
......@@ -42,7 +42,7 @@ def do_login(self):
@need_login
def iter_subscription(self):
self.subscription_page.go()
return self.page.iter_subscriptions()
yield self.page.get_subscription()
@need_login
def iter_documents(self, subscription):
......@@ -75,6 +75,6 @@ def iter_documents(self, subscription):
# then we set Rechercher to actionEvt to filter for this subscription, within last 6 months
# without first request we would have filter for this subscription but within last 2 months
params['actionEvt'] = 'Rechercher'
params['Beneficiaire'] = subscription._param
params['Beneficiaire'] = 'tout_selectionner'
self.documents_page.go(params=params)
return self.page.iter_documents(subid=subscription.id)
......@@ -19,15 +19,15 @@
from __future__ import unicode_literals
import re
from weboob.browser.elements import method, ListElement, ItemElement
from weboob.browser.filters.html import Attr, Link
from weboob.browser.filters.html import Link
from weboob.browser.filters.standard import CleanText, Regexp, CleanDecimal, Currency, Field, Format, Env
from weboob.browser.pages import LoggedPage, HTMLPage, PartialHTMLPage
from weboob.capabilities.bill import Subscription, Bill
from weboob.exceptions import BrowserUnavailable
from weboob.tools.date import parse_french_date
from weboob.tools.json import json
class LoginPage(HTMLPage):
......@@ -45,32 +45,22 @@ def on_load(self):
class SubscriptionPage(LoggedPage, HTMLPage):
@method
class iter_subscriptions(ListElement):
item_xpath = '//div[@id="corps-de-la-page"]//div[@class="tableau"]/div'
class item(ItemElement):
klass = Subscription
obj__labelid = Attr('.', 'aria-labelledby')
def get_subscription(self):
sub = Subscription()
# DON'T TAKE social security number for id because it's a very confidential data, take birth date instead
sub.id = CleanText('//button[@id="idAssure"]//td[@class="dateNaissance"]')(self.doc).replace('/', '')
sub.label = sub.subscriber = CleanText('//div[@id="pageAssure"]//span[@class="NomEtPrenomLabel"]')(self.doc)
def obj__birthdate(self):
return CleanText('//button[@id="%s"]//td[@class="dateNaissance"]' % Field('_labelid')(self))(self)
return sub
def obj_id(self):
# DON'T TAKE social security number for id because it's a very confidential data, take birth date instead
return ''.join(re.findall(r'\d+', Field('_birthdate')(self)))
def obj__param(self):
reversed_date = ''.join(reversed(re.findall(r'\d+', Field('_birthdate')(self))))
name = CleanText('//button[@id="%s"]//td[@class="nom"]' % Field('_labelid')(self))(self)
return '%s!-!%s!-!1' % (reversed_date, name)
obj_subscriber = CleanText('.//span[@class="NomEtPrenomLabel"]')
obj_label = obj_subscriber
class DocumentsPage(LoggedPage, PartialHTMLPage):
ENCODING = 'utf-8'
def build_doc(self, content):
res = json.loads(content)
return super(DocumentsPage, self).build_doc(res['tableauPaiement'].encode('utf-8'))
class DocumentsPage(LoggedPage, PartialHTMLPage):
@method
class iter_documents(ListElement):
item_xpath = '//ul[@id="unordered_list"]//li[has-class("rowitem")]'
......@@ -82,7 +72,7 @@ class item(ItemElement):
obj_label = CleanText('.//div[has-class("col-label")]')
obj_price = CleanDecimal.French('.//div[has-class("col-montant")]/span')
obj_currency = Currency('.//div[has-class("col-montant")]/span')
obj_url = Link('.//a[@class="downdetail"]')
obj_url = Link('.//div[@class="col-download"]/a')
obj_format = 'pdf'
def obj_date(self):
......
......@@ -26,6 +26,7 @@
__all__ = ['AmeliProBrowser']
class AmeliProBrowser(LoginBrowser):
BASEURL = 'https://espacepro.ameli.fr'
......@@ -63,7 +64,6 @@ def get_subscription_list(self):
@need_login
def get_subscription(self, id):
assert isinstance(id, basestring)
return self.get_subscription_list()
@need_login
......@@ -101,7 +101,6 @@ def iter_documents(self):
@need_login
def get_document(self, id):
assert isinstance(id, basestring)
for b in self.iter_documents():
if id == b.id:
return b
......
......@@ -21,8 +21,10 @@
from datetime import datetime
import re
from decimal import Decimal
from weboob.browser.pages import HTMLPage
from weboob.capabilities.bill import DocumentTypes, Subscription, Detail, Bill
from weboob.tools.compat import unicode
# Ugly array to avoid the use of french locale
......@@ -36,14 +38,17 @@ def login(self, login, password):
form['vp_connexion_portlet_1password'] = password.encode('utf8')
form.submit()
class HomePage(HTMLPage):
def on_loaded(self):
pass
class SearchPage(HTMLPage):
def on_loaded(self):
pass
class AccountPage(HTMLPage):
def iter_subscription_list(self):
ident = self.doc.xpath('//div[@id="identification"]')[0]
......
......@@ -17,17 +17,20 @@
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import datetime
from weboob.exceptions import BrowserIncorrectPassword
from weboob.browser.browsers import LoginBrowser, need_login
from weboob.browser.exceptions import HTTPNotFound, ServerError
from weboob.browser.url import URL
from dateutil.parser import parse as parse_date
from .pages import (
AccountsPage, JsonBalances, JsonPeriods, JsonHistory,
JsonBalances2, CurrencyPage, LoginPage, WrongLoginPage, AccountSuspendedPage,
NoCardPage, NotFoundPage
NoCardPage, NotFoundPage,
)
......@@ -49,16 +52,16 @@ class AmericanExpressBrowser(LoginBrowser):
js_posted = URL(r'/account-data/v1/financials/transactions\?limit=1000&offset=(?P<offset>\d+)&statement_end_date=(?P<end>[0-9-]+)&status=posted',
JsonHistory)
js_periods = URL(r'/account-data/v1/financials/statement_periods', JsonPeriods)
currency_page = URL(r'https://www.aexp-static.com/cdaas/axp-app/modules/axp-offers/1.11.1/fr-fr/axp-offers.json', CurrencyPage)
currency_page = URL(r'https://www.aexp-static.com/cdaas/axp-app/modules/axp-balance-summary/4.7.0/(?P<locale>\w\w-\w\w)/axp-balance-summary.json', CurrencyPage)
no_card = URL('https://www.americanexpress.com/us/content/no-card/',
'https://www.americanexpress.com/us/no-card/', NoCardPage)
no_card = URL(r'https://www.americanexpress.com/us/content/no-card/',
r'https://www.americanexpress.com/us/no-card/', NoCardPage)
not_found = URL(r'/accounts/error', NotFoundPage)
SUMMARY_CARD_LABEL = [
u'PAYMENT RECEIVED - THANK YOU',
u'PRELEVEMENT AUTOMATIQUE ENREGISTRE-MERCI'
'PAYMENT RECEIVED - THANK YOU',
'PRELEVEMENT AUTOMATIQUE ENREGISTRE-MERCI',
]
def __init__(self, *args, **kwargs):
......@@ -73,7 +76,6 @@ def do_login(self):
if self.wrong_login.is_here() or self.login.is_here() or self.account_suspended.is_here():
raise BrowserIncorrectPassword()
@need_login
def get_accounts(self):
self.accounts.go()
......@@ -91,7 +93,8 @@ def get_accounts(self):
self.page.set_balances(accounts)
# get currency
self.currency_page.go()
loc = self.session.cookies.get_dict(domain=".americanexpress.com")['axplocale'].lower()
self.currency_page.go(locale=loc)
currency = self.page.get_currency()
for acc in accounts:
......@@ -120,29 +123,30 @@ def iter_history(self, account):
@need_login
def iter_coming(self, account):
"""
Coming transactions can be found in a'pending' JSON if it exists (corresponding a 'Transactions en attente' tab on the website),
as well as in a 'posted' JSON (corresponding a 'Transactions enregistrées' tab on the website for futur transactions)
"""
# Coming transactions can be found in a 'pending' JSON if it exists
# ('En attente' tab on the website), as well as in a 'posted' JSON
# ('Enregistrées' tab on the website)
# "pending" have no vdate and debit date is in future
self.js_periods.go(headers={'account_token': account._token})
date = datetime.datetime.strptime(self.page.get_periods()[0], '%Y-%m-%d').date()
periods = self.page.get_periods()
date = parse_date(periods[0]).date()
today = datetime.date.today()
try:
self.js_pending.go(offset=0, headers={'account_token': account._token})
# when the latest period ends today we can't know the coming debit date
if date != today:
# when the latest period ends today we can't know the coming debit date
if date != today:
try:
self.js_pending.go(offset=0, headers={'account_token': account._token})
except ServerError as exc:
# At certain times of the month a connection might not have pendings;
# in that case, `js_pending.go` would throw a 502 error Bad Gateway
error_code = exc.response.json().get('code')
error_message = exc.response.json().get('message')
self.logger.warning('No pendings page to access to, got error %s and message "%s" instead.', error_code, error_message)
else:
for tr in self.page.iter_history(periods=periods):
if tr._owner == account._idforJSON:
tr.date = date
yield tr
except ServerError as exc:
# At certain time of the month a connection might not have pendings;
# in that case, `js_pending.go` would throw a 502 error Bad Gateway
error_code = exc.response.json().get('code')
error_message = exc.response.json().get('message')
self.logger.warning('No pendings page to access to, got error %s and message "%s" instead.' % (error_code, error_message))
# "posted" have a vdate but debit date can be future or past
for p in periods:
......
......@@ -22,7 +22,6 @@
from ast import literal_eval
from decimal import Decimal
import re
from dateutil.parser import parse as parse_date
from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage
from weboob.browser.elements import ItemElement, DictElement, method
......@@ -33,11 +32,13 @@
from weboob.tools.json import json
from weboob.tools.compat import basestring
from weboob.exceptions import ActionNeeded, BrowserUnavailable
from dateutil.parser import parse as parse_date
def float_to_decimal(f):
return Decimal(str(f))
def parse_decimal(s):
# we might get 1,399,680 in rupie indonésienne
if s.count(',') > 1 and not s.count('.'):
......@@ -87,13 +88,13 @@ def iter_accounts(self):
else:
assert False, "data was not found"
assert data[13] == 'core'
assert len(data[14]) == 3
assert data[15] == 'core'
assert len(data[16]) == 3
# search for products to get products list
for index, el in enumerate(data[14][2]):
for index, el in enumerate(data[16][2]):
if 'products' in el:
accounts_data = data[14][2][index+1]
accounts_data = data[16][2][index + 1]
assert len(accounts_data) == 2
assert accounts_data[1][4] == 'productsList'
......@@ -105,14 +106,14 @@ def iter_accounts(self):
if isinstance(account_data, basestring):
balances_token = account_data
elif isinstance(account_data, list) and not account_data[4][2][0]=="Canceled":
elif isinstance(account_data, list) and not account_data[4][2][0] == "Canceled":
acc = Account()
if len(account_data) > 15:
token.append(account_data[-11])
acc._idforJSON = account_data[10][-1]
acc._idforJSON = account_data[10][-1]
else:
acc._idforJSON = account_data[-5][-1]
acc._idforJSON = re.sub('\s+', ' ', acc._idforJSON)
acc._idforJSON = re.sub(r'\s+', ' ', acc._idforJSON)
acc.number = '-%s' % account_data[2][2]
acc.label = '%s %s' % (account_data[6][4], account_data[10][-1])
acc._balances_token = acc.id = balances_token
......@@ -142,7 +143,7 @@ def set_balances(self, accounts):
class CurrencyPage(LoggedPage, JsonPage):
def get_currency(self):
return self.doc['currency']
return self.doc['localeSettings']['currency_code']
class JsonPeriods(LoggedPage, JsonPage):
......@@ -172,7 +173,8 @@ def obj_type(self):
obj_raw = CleanText(Dict('description', default=''))
def obj_date(self):
""" 'statement_end_date' might be absent from this json, we must match the rdate with the right date period """
# 'statement_end_date' might be absent from this json,
# we must match the rdate with the right date period
_date = Date(Dict('statement_end_date', default=None), default=NotAvailable)(self)
if not _date:
periods = Env('periods')(self)
......@@ -207,6 +209,4 @@ def obj_original_amount(self):
else:
return original_amount
# obj__ref = Dict('reference_id')
obj__ref = Dict('identifier')
......@@ -56,6 +56,9 @@ def iter_accounts(self):
@need_login
def iter_investment(self, account):
if account.balance == 0:
self.logger.info('Account %s has a null balance, no investment available.', account.label)
return
headers = {'X-noee-authorization': 'noeprd %s' % self.token}
self.accounts.go(headers=headers)
for inv in self.page.iter_investments(account_id=account.id):
......