.*)', AvJPage)
def __init__(self, *args, **kwargs):
super(GroupamaBrowser, self).__init__(*args, **kwargs)
@@ -68,6 +71,13 @@ def get_accounts_list(self, balance=True, need_iban=False):
if self.av_account_form.is_here():
self.page.av_account_form()
account.balance, account.currency = self.page.get_av_balance()
+ # New page where some AV are stored
+ elif "front/vie/" in account._link:
+ link = re.search('contrat\/(.+)-Groupama', account._link)
+ if link:
+ self.av_secondary.go(id_contrat=link.group(1))
+ account.balance, account.currency = self.page.get_av_balance()
+
self.accounts.stay_or_go()
if account.balance or not balance:
if account.type != Account.TYPE_LIFE_INSURANCE and need_iban:
diff --git a/modules/groupama/pages.py b/modules/groupama/pages.py
index fbf99ad762bf25138517d8e8f88f597a4bf93df0..b04b426bb90bd04e04951388bcd64f14cfa61b13 100644
--- a/modules/groupama/pages.py
+++ b/modules/groupama/pages.py
@@ -25,7 +25,7 @@
from decimal import Decimal
-from weboob.browser.pages import HTMLPage, pagination, LoggedPage, FormNotFound
+from weboob.browser.pages import HTMLPage, pagination, LoggedPage, FormNotFound, JsonPage
from weboob.browser.elements import method, TableElement, ItemElement
from weboob.browser.filters.standard import Env, CleanDecimal, CleanText, Date, Regexp, Eval
from weboob.browser.filters.html import Attr, Link, TableCell
@@ -33,6 +33,7 @@
from weboob.capabilities.bank import Account, Investment
from weboob.capabilities.base import NotAvailable
from weboob.tools.capabilities.bank.transactions import FrenchTransaction
+from weboob.browser.filters.json import Dict
class LoginPage(HTMLPage):
@@ -217,6 +218,13 @@ def condition(self):
obj_code_type = Investment.CODE_TYPE_ISIN
+class AvJPage(LoggedPage, JsonPage):
+ def get_av_balance(self):
+ balance = CleanDecimal(Dict('montant'))(self.doc)
+ currency = "EUR"
+ return balance, currency
+
+
class AVHistoryPage(LoggedPage, HTMLPage):
@method
class get_av_history(TableElement):
diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py
index 4f3255192e3be74b93562e64a0c5932cdd59745a..b3e8d1a3ffd1309cd00641abd7943e5b945004c8 100644
--- a/modules/hsbc/browser.py
+++ b/modules/hsbc/browser.py
@@ -23,6 +23,7 @@
import ssl
from datetime import timedelta, date
from lxml.etree import XMLSyntaxError
+from collections import OrderedDict
from weboob.tools.date import LinearDateGuesser
from weboob.capabilities.bank import Account, AccountNotFound
@@ -34,8 +35,8 @@
from weboob.capabilities.base import find_object
from .pages.account_pages import (
- AccountsPage, CBOperationPage, CPTOperationPage, LoginPage, AppGonePage, RibPage,
- UnavailablePage, OtherPage, FrameContainer, ProfilePage,
+ AccountsPage, OwnersListPage, CBOperationPage, CPTOperationPage, LoginPage,
+ AppGonePage, RibPage, UnavailablePage, OtherPage, FrameContainer, ProfilePage,
)
from .pages.life_insurances import (
LifeInsurancesPage, LifeInsurancePortal, LifeInsuranceMain, LifeInsuranceUseless,
@@ -78,6 +79,7 @@ class HSBC(LoginBrowser):
AppGonePage)
rib = URL(r'/cgi-bin/emcgi', RibPage)
accounts = URL(r'/cgi-bin/emcgi', AccountsPage)
+ owners_list = URL(r'/cgi-bin/emcgi', OwnersListPage)
life_insurance_useless = URL(r'/cgi-bin/emcgi', LifeInsuranceUseless)
profile = URL(r'/cgi-bin/emcgi', ProfilePage)
unavailable = URL(r'/cgi-bin/emcgi', UnavailablePage)
@@ -115,15 +117,16 @@ class HSBC(LoginBrowser):
RetrieveUselessPage
)
-
# catch-all
other_page = URL(r'/cgi-bin/emcgi', OtherPage)
def __init__(self, username, password, secret, *args, **kwargs):
super(HSBC, self).__init__(username, password, *args, **kwargs)
- self.accounts_list = dict()
+ self.accounts_list = OrderedDict()
+ self.unique_accounts_list = dict()
self.secret = secret
self.PEA_LISTING = {}
+ self.owners = []
def load_state(self, state):
return
@@ -159,8 +162,9 @@ def do_login(self):
if new_base_url in self.url:
self.BASEURL = new_base_url
- self.js_url = self.page.get_js_url()
- home_url = self.page.get_frame()
+ if self.frame_page.is_here():
+ home_url = self.page.get_frame()
+ self.js_url = self.page.get_js_url()
if not home_url or not self.page.logged:
raise BrowserIncorrectPassword()
@@ -177,51 +181,99 @@ def go_post(self, url, data=None):
url = url[:url.find('?')]
self.location(url, data=q)
- @need_login
- def get_accounts_list(self):
- if not self.accounts_list:
- self.update_accounts_list()
-
- # go on cards page if there are cards accounts
- for a in self.accounts_list.values():
- if a.type == Account.TYPE_CARD:
- self.location(a.url)
- break
-
- # get all couples (card, parent) on cards page
- all_card_and_parent = []
- if self.cbPage.is_here():
- all_card_and_parent = self.page.get_all_parent_id()
- self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'})
-
- # update cards parent and currency
- for a in self.accounts_list.values():
- if a.type == Account.TYPE_CARD:
- for card in all_card_and_parent:
- if a.id in card[0].replace(' ', ''):
- a.parent = find_object(self.accounts_list.values(), id=card[1])
- if a.parent and not a.currency:
- a.currency = a.parent.currency
- yield a
+ def go_to_owner_accounts(self, owner):
+ """
+ The owners URLs change all the time so we must refresh them.
+ If we try to go to a person's accounts page while we are already
+ on this page, the website returns an empty page with the message
+ "Pas de TIERS", so we must always go to the owners list before
+ going to the owner's account page.
+ """
+ if not self.owners_list.is_here():
+ self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'})
+
+ if not self.owners_list.is_here():
+ # Sometimes when we fetch info from a PEA account, the first POST
+ # fails and we are blocked on some owner's AccountsPage.
+ self.logger.warning('The owners list redirection failed, we must try again.')
+ self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'})
+
+ # Refresh owners URLs in case they changed:
+ self.owners = self.page.get_owners_urls()
+ self.go_post(self.owners[owner])
@need_login
- def update_accounts_list(self, iban=True):
- if self.accounts.is_here():
- self.go_post(self.js_url)
+ def iter_account_owners(self):
+ """
+ Some connections have a "Compte de Tiers" section with several
+ people each having their own accounts. We must fetch the account
+ for each person and store the owner of each account.
+ """
+ if self.unique_accounts_list:
+ for account in self.unique_accounts_list.values():
+ yield account
else:
- data = {'debr': 'COMPTES_PAN'}
- self.go_post(self.js_url, data=data)
+ self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'})
+ if self.owners_list.is_here():
+ self.owners = self.page.get_owners_urls()
+
+ # self.accounts_list will be a dictionary of owners each
+ # containing a dictionary of the owner's accounts.
+ for owner in range(len(self.owners)):
+ self.accounts_list[owner] = {}
+ self.update_accounts_list(owner, True)
+
+ # We must set an "_owner" attribute to each account.
+ for a in self.accounts_list[owner].values():
+ a._owner = owner
+
+ # go on cards page if there are cards accounts
+ for a in self.accounts_list[owner].values():
+ if a.type == Account.TYPE_CARD:
+ self.location(a.url)
+ break
+
+ # get all couples (card, parent) on cards page
+ all_card_and_parent = []
+ if self.cbPage.is_here():
+ all_card_and_parent = self.page.get_all_parent_id()
+ self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'})
+
+ # update cards parent and currency
+ for a in self.accounts_list[owner].values():
+ if a.type == Account.TYPE_CARD:
+ for card in all_card_and_parent:
+ if a.id in card[0].replace(' ', ''):
+ a.parent = find_object(self.accounts_list[owner].values(), id=card[1])
+ if a.parent and not a.currency:
+ a.currency = a.parent.currency
+
+ # We must get back to the owners list before moving to the next owner:
+ self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'})
+
+ # Fill a dictionary will all accounts without duplicating common accounts:
+ for owner in self.accounts_list.values():
+ for account in owner.values():
+ if account.id not in self.unique_accounts_list.keys():
+ self.unique_accounts_list[account.id] = account
+
+ for account in self.unique_accounts_list.values():
+ yield account
+ @need_login
+ def update_accounts_list(self, owner, iban=True):
+ # Go to the owner's account page in case we are not there already:
+ self.go_to_owner_accounts(owner)
for a in self.page.iter_spaces_account():
try:
- self.accounts_list[a.id].url = a.url
+ self.accounts_list[owner][a.id].url = a.url
except KeyError:
- self.accounts_list[a.id] = a
+ self.accounts_list[owner][a.id] = a
if iban:
self.location(self.js_url, params={'debr': 'COMPTES_RIB'})
if self.rib.is_here():
- self.page.get_rib(self.accounts_list)
+ self.page.get_rib(self.accounts_list[owner])
@need_login
def _quit_li_space(self):
@@ -248,10 +300,9 @@ def _quit_li_space(self):
@need_login
def _go_to_life_insurance(self, account):
self._quit_li_space()
-
self.go_post(account.url)
- if self.frame_page.is_here() or self.life_insurance_useless.is_here() or self.life_not_found.is_here():
+ if self.accounts.is_here() or self.frame_page.is_here() or self.life_insurance_useless.is_here() or self.life_not_found.is_here():
self.logger.warning('cannot go to life insurance %r', account)
return False
@@ -264,9 +315,8 @@ def _go_to_life_insurance(self, account):
@need_login
def get_history(self, account, coming=False, retry_li=True):
self._quit_li_space()
-
- self.update_accounts_list(False)
- account = self.accounts_list[account.id]
+ self.update_accounts_list(account._owner, False)
+ account = self.accounts_list[account._owner][account.id]
if account.url is None:
return []
@@ -307,18 +357,18 @@ def get_history(self, account, coming=False, retry_li=True):
return history
try:
- self.go_post(self.accounts_list[account.id].url)
+ self.go_post(self.accounts_list[account._owner][account.id].url)
# sometime go to hsbc life insurance space do logout
except HTTPNotFound:
self.app_gone = True
self.do_logout()
self.do_login()
- # If we relogin on hsbc, all link have change
+ # If we relogin on hsbc, all links have changed
if self.app_gone:
self.app_gone = False
- self.update_accounts_list()
- self.location(self.accounts_list[account.id].url)
+ self.update_accounts_list(account._owner, False)
+ self.location(self.accounts_list[account._owner][account.id].url)
if self.page is None:
return []
@@ -326,6 +376,7 @@ def get_history(self, account, coming=False, retry_li=True):
# for 'fusion' space
if hasattr(account, '_is_form') and account._is_form:
# go on accounts page to get account form
+ self.go_to_owner_accounts(account._owner)
self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'})
self.page.go_history_page(account)
@@ -356,6 +407,8 @@ def _get_history(self):
yield tr
def get_investments(self, account, retry_li=True):
+ if not account.url:
+ raise NotImplementedError()
if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION):
return self.get_life_investments(account, retry_li=retry_li)
elif account.type == Account.TYPE_PEA:
@@ -369,6 +422,8 @@ def get_investments(self, account, retry_li=True):
raise NotImplementedError()
def get_scpi_investments(self, account):
+ if not account.url:
+ raise NotImplementedError()
# Clean account url
m = re.search(r"'(.*)'", account.url)
if m:
@@ -377,6 +432,7 @@ def get_scpi_investments(self, account):
account_url = account.url
# Need to be on accounts page to go on scpi page
+ self.go_to_owner_accounts(account._owner)
self.accounts.go()
# Go on scpi page
self.location(account_url)
@@ -387,6 +443,7 @@ def get_scpi_investments(self, account):
return self.page.iter_scpi_investment()
def get_pea_investments(self, account):
+ self.go_to_owner_accounts(account._owner)
assert account.type in (Account.TYPE_PEA, Account.TYPE_MARKET)
# When invest balance is 0, there is not link to go on market page
@@ -394,7 +451,10 @@ def get_pea_investments(self, account):
return []
if not self.PEA_LISTING:
- self._go_to_wealth_accounts()
+ # _go_to_wealth_accounts returns True if everything went well.
+ if not self._go_to_wealth_accounts(account):
+ self.logger.warning('Unable to connect to wealth accounts.')
+ return []
# Get account number without "EUR"
account_id = re.search(r'\d{4,}', account.id).group(0)
@@ -421,12 +481,9 @@ def get_pea_investments(self, account):
return pea_invests
def get_life_investments(self, account, retry_li=True):
-
self._quit_li_space()
-
- self.update_accounts_list(False)
- account = self.accounts_list[account.id]
-
+ self.update_accounts_list(account._owner, False)
+ account = self.accounts_list[account._owner][account.id]
try:
if not self._go_to_life_insurance(account):
self._quit_li_space()
@@ -452,16 +509,28 @@ def get_life_investments(self, account, retry_li=True):
return investments
- def _go_to_wealth_accounts(self):
+ def _go_to_wealth_accounts(self, account):
if not hasattr(self.page, 'get_middle_frame_url'):
# if we can catch the URL, we go directly, else we need to browse
# the website
- self.update_accounts_list()
+ self.update_accounts_list(account._owner, False)
self.location(self.page.get_middle_frame_url())
+
if self.page.get_patrimoine_url():
self.location(self.page.get_patrimoine_url())
self.page.go_next()
+
+ if self.login.is_here():
+ self.logger.warning('Connection to the Logon page failed, we must try again.')
+ self.do_login()
+ self.update_accounts_list(account._owner, False)
+ self.investment_form_page.go()
+ # If reloggin did not help accessing the wealth space,
+ # there is nothing more we can do to get there.
+ if not self.investment_form_page.is_here():
+ return False
+
self.page.go_to_logon()
helper = ProductViewHelper(self)
# we need to go there to initialize the session
@@ -469,9 +538,17 @@ def _go_to_wealth_accounts(self):
self.PEA_LISTING['liquidities'] = list(helper.retrieve_liquidity())
self.PEA_LISTING['investments'] = list(helper.retrieve_invests())
self.connection.go()
+ return True
@need_login
def get_profile(self):
+ if not self.owners:
+ self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'})
+ if self.owners_list.is_here():
+ self.owners = self.page.get_owners_urls()
+
+ # The main owner of the connection is always the first of the list:
+ self.go_to_owner_accounts(0)
data = {'debr': 'PARAM'}
self.go_post(self.js_url, data=data)
return self.page.get_profile()
diff --git a/modules/hsbc/module.py b/modules/hsbc/module.py
index 1124ddcf0c52028c03f91d4e91f16360d7e98165..35fec581505ac782d168608a935d837a9e0017de 100644
--- a/modules/hsbc/module.py
+++ b/modules/hsbc/module.py
@@ -47,11 +47,11 @@ def create_default_browser(self):
self.config['secret'].get())
def iter_accounts(self):
- for account in self.browser.get_accounts_list():
+ for account in self.browser.iter_account_owners():
yield account
def get_account(self, _id):
- return find_object(self.browser.get_accounts_list(), id=_id, error=AccountNotFound)
+ return find_object(self.browser.iter_account_owners(), id=_id, error=AccountNotFound)
def iter_history(self, account):
for tr in self.browser.get_history(account):
diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py
index f129080835bafea840c50e2ffe834c8e58482f1b..089ba1ce941a859ed3478eba4e9255043cdc6a8f 100644
--- a/modules/hsbc/pages/account_pages.py
+++ b/modules/hsbc/pages/account_pages.py
@@ -82,40 +82,42 @@ def on_load(self):
class AccountsType(Filter):
PATTERNS = [
- ('c.aff', Account.TYPE_CHECKING),
- ('pea', Account.TYPE_PEA),
- ('invest', Account.TYPE_MARKET),
- ('ptf', Account.TYPE_MARKET),
- ('ldd', Account.TYPE_SAVINGS),
- ('cel', Account.TYPE_SAVINGS),
- ('pel', Account.TYPE_SAVINGS),
- ('livret', Account.TYPE_SAVINGS),
- ('livjeu', Account.TYPE_SAVINGS),
- ('compte', Account.TYPE_CHECKING),
- ('cpte', Account.TYPE_CHECKING),
- ('scpi', Account.TYPE_MARKET),
- ('account', Account.TYPE_CHECKING),
- ('pret', Account.TYPE_LOAN),
- ('vie', Account.TYPE_LIFE_INSURANCE),
- ('strategie patr.', Account.TYPE_LIFE_INSURANCE),
- ('essentiel', Account.TYPE_LIFE_INSURANCE),
- ('elysee', Account.TYPE_LIFE_INSURANCE),
- ('abondance', Account.TYPE_LIFE_INSURANCE),
- ('ely. retraite', Account.TYPE_LIFE_INSURANCE),
- ('lae option assurance', Account.TYPE_LIFE_INSURANCE),
- ('carte ', Account.TYPE_CARD),
- ('business ', Account.TYPE_CARD),
- ('plan assur. innovat.', Account.TYPE_LIFE_INSURANCE),
- ('hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE),
- ('hsbc evol pat capi', Account.TYPE_CAPITALISATION),
- ('bourse libre', Account.TYPE_MARKET),
- ('plurival', Account.TYPE_LIFE_INSURANCE),
+ (r'c\.aff', Account.TYPE_CHECKING),
+ (r'\bssmouv\b', Account.TYPE_CHECKING),
+ (r'\bpea\b', Account.TYPE_PEA),
+ (r'invest', Account.TYPE_MARKET),
+ (r'\bptf\b', Account.TYPE_MARKET),
+ (r'\bldd\b', Account.TYPE_SAVINGS),
+ (r'\bcel\b', Account.TYPE_SAVINGS),
+ (r'\bpel\b', Account.TYPE_SAVINGS),
+ (r'livret', Account.TYPE_SAVINGS),
+ (r'livjeu', Account.TYPE_SAVINGS),
+ (r'csljun', Account.TYPE_SAVINGS),
+ (r'compte', Account.TYPE_CHECKING),
+ (r'cpte', Account.TYPE_CHECKING),
+ (r'scpi', Account.TYPE_MARKET),
+ (r'account', Account.TYPE_CHECKING),
+ (r'\bpret\b', Account.TYPE_LOAN),
+ (r'\bvie\b', Account.TYPE_LIFE_INSURANCE),
+ (r'strategie patr.', Account.TYPE_LIFE_INSURANCE),
+ (r'essentiel', Account.TYPE_LIFE_INSURANCE),
+ (r'elysee', Account.TYPE_LIFE_INSURANCE),
+ (r'abondance', Account.TYPE_LIFE_INSURANCE),
+ (r'ely\. retraite', Account.TYPE_LIFE_INSURANCE),
+ (r'lae option assurance', Account.TYPE_LIFE_INSURANCE),
+ (r'carte ', Account.TYPE_CARD),
+ (r'business ', Account.TYPE_CARD),
+ (r'plan assur\. innovat\.', Account.TYPE_LIFE_INSURANCE),
+ (r'hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE),
+ (r'hsbc evol pat capi', Account.TYPE_CAPITALISATION),
+ (r'bourse libre', Account.TYPE_MARKET),
+ (r'plurival', Account.TYPE_LIFE_INSURANCE),
]
def filter(self, label):
label = label.lower()
for pattern, type in self.PATTERNS:
- if pattern in label:
+ if re.search(pattern, label):
return type
return Account.TYPE_UNKNOWN
@@ -158,11 +160,20 @@ class item(ItemElement):
def condition(self):
return len(self.el.xpath('./td')) > 2
- obj_label = Label(CleanText('./td[1]/a'))
+ # Some accounts have no in the first
+ def obj_label(self):
+ if self.el.xpath('./td[1]/a'):
+ return Label(CleanText('./td[1]/a'))(self) or 'Compte sans libellé'
+ return Label(CleanText('./td[1]'))(self) or 'Compte sans libellé'
+
obj_coming = Env('coming')
obj_currency = FrenchTransaction.Currency('./td[2]')
- obj_url = CleanText(AbsoluteLink('./td[1]/a'), replace=[('\n', '')])
+ def obj_url(self):
+ # Accounts without an in the | have no link
+ if self.el.xpath('./td[1]/a'):
+ return CleanText(AbsoluteLink('./td[1]/a'), default=None, replace=[('\n', '')])(self)
+ return None
obj_type = AccountsType(Field('label'))
obj_coming = NotAvailable
@@ -222,6 +233,13 @@ def obj_id(self):
return account_id
+class OwnersListPage(AccountsPage):
+ is_here = '//h1[text()="Comptes de tiers"]'
+
+ def get_owners_urls(self):
+ return self.doc.xpath('//div[@class="GoBack"]/a/@href')
+
+
class RibPage(GenericLandingPage):
def is_here(self):
return bool(self.doc.xpath('//h1[contains(text(), "RIB/IBAN")]'))
@@ -240,7 +258,8 @@ def get_rib(self, accounts):
form = self.get_form(name="FORM_RIB")
form['index_rib'] = str(nb+1)
form.submit()
- self.browser.page.link_rib(accounts)
+ if self.browser.rib.is_here():
+ self.browser.page.link_rib(accounts)
class Pagination(object):
diff --git a/modules/hsbc/pages/investments.py b/modules/hsbc/pages/investments.py
index 67a0e81a7a42e0fe0b338552c4dd03ce390962be..b1e382d17db842970e295bd17a800cd1a49a43fa 100644
--- a/modules/hsbc/pages/investments.py
+++ b/modules/hsbc/pages/investments.py
@@ -9,6 +9,7 @@
from weboob.capabilities import NotAvailable
from weboob.capabilities.bank import Account, Investment
+from weboob.tools.capabilities.bank.investments import is_isin_valid
from weboob.browser.elements import ItemElement, TableElement, DictElement, method
from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage
@@ -359,9 +360,13 @@ class item(ItemElement):
klass = Investment
obj_label = CleanText(Dict('productName'))
- obj_code = CleanText(Dict('productIdInformation/0/productAlternativeNumber'))
- obj_code_type = Investment.CODE_TYPE_ISIN
obj_quantity = CleanDecimal(Dict('holdingDetailInformation/0/productHoldingQuantityCount'))
+ obj_code = CleanText(Dict('productIdInformation/0/productAlternativeNumber'), replace=[('-FR', '')])
+
+ def obj_code_type(self):
+ if is_isin_valid(Field('code')(self)):
+ return Investment.CODE_TYPE_ISIN
+ return NotAvailable
def obj_vdate(self):
vdate = Dict('holdingDetailInformation/0/productPriceUpdateDate')(self)
@@ -467,7 +472,9 @@ class item(ItemElement):
def condition(self):
return Dict('productTypeCode')(self) == 'INVCASH'
- obj_label = CleanText(Dict('productShortName'))
+ obj_label = "Liquidités"
+ obj_code = "XX-liquidity"
+ obj_code_type = NotAvailable
obj_valuation = CleanDecimal(
Dict(
'holdingDetailInformation/0/holdingDetailMultipleCurrencyInformation/1'
diff --git a/modules/ing/browser.py b/modules/ing/browser.py
index 08085654446a54601f11edf9ac60068711f9ce74..89a9c0a07b4c7d3f57f0db2a8a9bfc2644d50ea8 100644
--- a/modules/ing/browser.py
+++ b/modules/ing/browser.py
@@ -219,11 +219,6 @@ def get_accounts_on_space(self, space, get_iban=True):
accounts_list.append(loan)
yield loan
- def get_coming_balance(self, account):
- if account.type == Account.TYPE_CHECKING:
- self.go_account_page(account)
- return self.page.get_coming_balance()
- return NotAvailable
@need_login
@start_with_main_site
@@ -235,20 +230,17 @@ def get_accounts_list(self, space=None, get_iban=True):
if space:
for acc in self.get_accounts_on_space(space, get_iban=get_iban):
- acc.coming = self.get_coming_balance(acc)
yield acc
elif self.multispace:
for space in self.multispace:
for acc in self.get_accounts_on_space(space, get_iban=get_iban):
- acc.coming = self.get_coming_balance(acc)
yield acc
else:
for acc in self.page.get_list():
acc._space = None
if get_iban:
self.get_iban(acc)
- acc.coming = self.get_coming_balance(acc)
yield acc
for loan in self.iter_detailed_loans():
diff --git a/modules/ing/pages/accounts_list.py b/modules/ing/pages/accounts_list.py
index 665160fc0cccee027d13d70901d0c83114ef7870..687de82fc552a76fafe9e01112897106bedcc0c7 100644
--- a/modules/ing/pages/accounts_list.py
+++ b/modules/ing/pages/accounts_list.py
@@ -323,11 +323,6 @@ def load_space_page(self):
self.fillup_form(form, r"\),\{(.*)\},'", on_click)
form.submit()
- def get_coming_balance(self):
- return CleanDecimal('//div[@class="previsionnel"]/div[@class="solde_value"]//label',
- replace_dots=True,
- default=NotAvailable)(self.doc)
-
class IbanPage(LoggedPage, HTMLPage):
def get_iban(self):
diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py
index 637a8f2bcf5c9bdce64ad6e68a46242802c47a8d..4a97874cc3b9fc8ff15d5ca1dd58e48c1b6fc517 100644
--- a/modules/lcl/pages.py
+++ b/modules/lcl/pages.py
@@ -581,7 +581,11 @@ def obj_label(self):
return "%s Bourse" % CleanText((TableCell('label')(self)[0]).xpath('./div[b]'))(self)
def obj_type(self):
- return self.page.TYPES.get(' '.join(Field('label')(self).split()[:-1]).lower(), Account.TYPE_MARKET)
+ _label = ' '.join(Field('label')(self).split()[:-1]).lower()
+ for key in self.page.TYPES:
+ if key in _label:
+ return self.page.TYPES.get(key)
+ return Account.TYPE_MARKET
def get_logout_link(self):
return Link('//a[@class="link-underline" and contains(text(), "espace client")]')(self.doc)
diff --git a/modules/materielnet/browser.py b/modules/materielnet/browser.py
index c81e142c5a6c378c0894d5e4e74ae357988f1444..0b0704d5f1487218d5c1ee8e03359369d1edef1e 100644
--- a/modules/materielnet/browser.py
+++ b/modules/materielnet/browser.py
@@ -27,12 +27,24 @@
class MaterielnetBrowser(LoginBrowser):
BASEURL = 'https://secure.materiel.net'
- login = URL(r'https://www.materiel.net/form/login',
- r'/Login/PartialPublicLogin', LoginPage)
+ login = URL(r'/Login/Login', LoginPage)
captcha = URL('/pm/client/captcha.html', CaptchaPage)
- profil = URL(r'/Account/InformationsSection', ProfilPage)
- documents = URL(r'/Orders/PartialCompletedOrdersHeader', DocumentsPage)
- document_details = URL(r'/Orders/PartialCompletedOrderContent', DocumentsDetailsPage)
+ profil = URL(r'/Account/InformationsSection',
+ r'/pro/Account/InformationsSection', ProfilPage)
+ documents = URL(r'/Orders/PartialCompletedOrdersHeader',
+ r'/pro/Orders/PartialCompletedOrdersHeader', DocumentsPage)
+ document_details = URL(r'/Orders/PartialCompletedOrderContent',
+ r'/pro/Orders/PartialCompletedOrderContent', DocumentsDetailsPage)
+
+ def __init__(self, *args, **kwargs):
+ super(MaterielnetBrowser, self).__init__(*args, **kwargs)
+ self.is_pro = None
+
+ def par_or_pro_location(self, url, *args, **kwargs):
+ if self.is_pro:
+ url = '/pro' + url
+
+ return super(MaterielnetBrowser, self).location(url, *args, **kwargs)
def do_login(self):
self.login.go()
@@ -47,13 +59,16 @@ def do_login(self):
if error:
raise BrowserIncorrectPassword(error)
+ self.is_pro = 'pro' in self.url
+
@need_login
def get_subscription_list(self):
- return self.profil.stay_or_go().get_list()
+ return self.par_or_pro_location('/Account/InformationsSection').page.get_list()
@need_login
def iter_documents(self, subscription):
- json_response = self.location('/Orders/CompletedOrdersPeriodSelection').json()
+ json_response = self.par_or_pro_location('/Orders/CompletedOrdersPeriodSelection').json()
for data in json_response:
- return self.documents.go(data=data).get_documents()
+ for doc in self.par_or_pro_location('/Orders/PartialCompletedOrdersHeader', data=data).page.get_documents():
+ yield doc
diff --git a/modules/materielnet/pages.py b/modules/materielnet/pages.py
index ffe08a87e7128d89866704833ced69f8eb0b2149..ace5a75527ad1405914cbcb51b162e6d872b04d3 100644
--- a/modules/materielnet/pages.py
+++ b/modules/materielnet/pages.py
@@ -44,7 +44,7 @@ def login(self, login, password):
if not re.match(regex, login):
raise BrowserIncorrectPassword(Attr('//input[@id="Email"]', 'data-val-regex')(self.doc))
- form = self.get_form(id='loginForm')
+ form = self.get_form(xpath='//form[contains(@action, "/Login/Login")]')
form['Email'] = login
form['Password'] = password
form.submit()
diff --git a/modules/paypal/browser.py b/modules/paypal/browser.py
index 79f27bea90b763715e516d8b2a2730df8ce2514c..4a6a00d042ee6d49c1f17fb57f110c514ab17122 100644
--- a/modules/paypal/browser.py
+++ b/modules/paypal/browser.py
@@ -79,7 +79,6 @@ def __init__(self, *args, **kwargs):
super(Paypal, self).__init__(*args, **kwargs)
def do_login(self):
- raise BrowserUnavailable()
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
@@ -88,6 +87,7 @@ def do_login(self):
response = self.open(self.page.get_script_url())
token, csrf, key, value, sessionID, cookie = self.page.get_token_and_csrf(response.text)
+
self.session.cookies.update({'xppcts': cookie})
data = {}
data['ads_token_js'] = token
@@ -114,7 +114,8 @@ def do_login(self):
self.detect_account_type()
def detect_account_type(self):
- self.page.detect_account_type()
+ if self.page:
+ self.page.detect_account_type()
@need_login
def get_accounts(self):
diff --git a/modules/paypal/pages.py b/modules/paypal/pages.py
index 3bd10a1d5ea1a63c46cfedae858617f942b923af..458066b7b64022f68b60b06bcfbbabe73ca5b911 100644
--- a/modules/paypal/pages.py
+++ b/modules/paypal/pages.py
@@ -96,6 +96,9 @@ def exec_decoder(mtc):
csrf = re.search(r"%s'([^']+)'" % re.escape("'&_csrf='+encodeURIComponent("), cleaner_code).group(1)
key, value = re.findall(r"'(\w+)','(\w+)'", cleaner_code)[-1]
+ # Remove setCookie function content
+ cleaner_code = re.sub(r"'setCookie'.*(?=,'removeCookie')", "'setCookie':function(){}", cleaner_code)
+
# Detect the name of the function that computes the token, detect the
# variable that stores the result and store it as a global.
get_token_func_name = re.search(r"ads_token_js='\+encodeURIComponent\((\w+)\)", cleaner_code).group(1)
@@ -107,64 +110,6 @@ def exec_decoder(mtc):
cleaner_code = cleaner_code.replace(loop_func_name + "();", "")
cleaner_code = cleaner_code.replace("data;", "return;")
- # Simulate a browser environment
- simulate_browser_code = """
- if (!document.createAttribute) {
- document.createAttribute = null;
- }
-
- if (!document.domain) {
- document.domain = "paypal.com";
- }
-
- if (!document.styleSheets) {
- document.styleSheets = null;
- }
-
- if (!document.characterSet) {
- document.characterSet = "UTF-8";
- }
-
- if (!document.documentElement) {
- document.documentElement = {};
- }
-
- if (!window.innerWidth || !window.innerHeight) {
- window.innerWidth = 1280;
- window.innerHeight = 800;
- }
-
- if (typeof(screen) === "undefined") {
- var screen = window.screen = {
- width: 1280,
- height: 800
- };
- }
-
- if (typeof(history) === "undefined") {
- var history = window.history = {};
- }
-
- if (typeof(location) === "undefined") {
- var location = window.location = {
- host: "paypal.com"
- };
- }
-
- var XMLHttpRequest = function() {};
- XMLHttpRequest.prototype.onreadystatechange = function(){};
- XMLHttpRequest.prototype.open = function(){};
- XMLHttpRequest.prototype.setRequestHeader = function(){};
- XMLHttpRequest.prototype.send = function(){};
- window.XMLHttpRequest = XMLHttpRequest;
-
- if (!navigator.appName) {
- navigator.appName = "Netscape";
- }
-
- navigator.userAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0";
- """
-
# Add a function that returns the token
cleaner_code += """
function GET_ADS_JS_TOKEN()
@@ -173,7 +118,11 @@ def exec_decoder(mtc):
}
"""
- token = str(Javascript(simulate_browser_code + cleaner_code).call("GET_ADS_JS_TOKEN"))
+ try:
+ token = str(Javascript(cleaner_code, None, "paypal.com").call("GET_ADS_JS_TOKEN"))
+ except:
+ raise BrowserUnavailable()
+
return token, csrf, key, value, sessionID, cookie
def login(self, login, password, ):
diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py
index 81e26a10c64587e04aa833d9016481b0f6de20be..fbc43cfd01de91b6a5561df098b895e8431b3aa1 100644
--- a/modules/societegenerale/sgpe/browser.py
+++ b/modules/societegenerale/sgpe/browser.py
@@ -25,7 +25,7 @@
from weboob.browser.browsers import LoginBrowser, need_login, StatesMixin
from weboob.browser.url import URL
from weboob.browser.exceptions import ClientError
-from weboob.exceptions import BrowserIncorrectPassword
+from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, NoAccountsException
from weboob.capabilities.base import find_object
from .compat.weboob_capabilities_bank import (
AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientBankError,
@@ -35,10 +35,12 @@
from .pages import (
LoginPage, CardsPage, CardHistoryPage, IncorrectLoginPage,
- ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage,
+ ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage, InscriptionPage,
ErrorPage,
)
-from .json_pages import AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage
+from .json_pages import (
+ AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage,
+)
from .transfer_pages import (
EasyTransferPage, RecipientsJsonPage, TransferPage, SignTransferPage, TransferDatesPage,
AddRecipientPage, AddRecipientStepPage, ConfirmRecipientPage,
@@ -59,6 +61,7 @@ class SGPEBrowser(LoginBrowser):
'/gae/afficherInscriptionUtilisateur.html',
'/gae/afficherChangementCodeSecretExpire.html',
ChangePassPage)
+ inscription_page = URL('/icd-web/gax/gax-inscription.html', InscriptionPage)
def check_logged_status(self):
if not self.page or self.login.is_here():
@@ -82,6 +85,9 @@ def do_login(self):
except ClientError:
raise BrowserIncorrectPassword()
+ if self.inscription_page.is_here():
+ raise ActionNeeded(self.page.get_error())
+
# force page change
if not self.accounts.is_here():
self.go_accounts()
@@ -123,23 +129,47 @@ class SGEnterpriseBrowser(SGPEBrowser):
CERTHASH = '2231d5ddb97d2950d5e6fc4d986c23be4cd231c31ad530942343a8fdcc44bb99'
accounts = URL('/icd/syd-front/data/syd-comptes-accederDepuisMenu.json', AccountsJsonPage)
+ intraday_accounts = URL('/icd/syd-front/data/syd-intraday-accederDepuisMenu.json', AccountsJsonPage)
+
balances = URL('/icd/syd-front/data/syd-comptes-chargerSoldes.json', BalancesJsonPage)
+ intraday_balances = URL('/icd/syd-front/data/syd-intraday-chargerSoldes.json', BalancesJsonPage)
+
history = URL('/icd/syd-front/data/syd-comptes-chargerReleve.json',
'/icd/syd-front/data/syd-intraday-chargerDetail.json', HistoryJsonPage)
history_next = URL('/icd/syd-front/data/syd-comptes-chargerProchainLotEcriture.json', HistoryJsonPage)
+
profile = URL('/gae/afficherModificationMesDonnees.html', ProfileEntPage)
subscription = URL(r'/Pgn/NavigationServlet\?MenuID=BANRELRIE&PageID=ReleveRIE&NumeroPage=1&Origine=Menu', SubscriptionPage)
subscription_form = URL(r'Pgn/NavigationServlet', SubscriptionPage)
def go_accounts(self):
- self.accounts.go()
+ try:
+ # get standard accounts
+ self.accounts.go()
+ except NoAccountsException:
+ # get intraday accounts
+ self.intraday_accounts.go()
@need_login
def get_accounts_list(self):
- accounts = []
- accounts.extend(self.accounts.stay_or_go().iter_accounts())
- for acc in self.balances.go().populate_balances(accounts):
+ # 'Comptes' are standard accounts on sge website
+ # 'Opérations du jour' are intraday accounts on sge website
+ # Standard and Intraday accounts are same accounts with different detail
+ # User could have standard accounts with no intraday accounts or the contrary
+ # They also could have both, in that case, retrieve only standard accounts
+ try:
+ # get standard accounts
+ self.accounts.go()
+ accounts = list(self.page.iter_class_accounts())
+ self.balances.go()
+ except NoAccountsException:
+ # get intraday accounts
+ self.intraday_accounts.go()
+ accounts = list(self.page.iter_class_accounts())
+ self.intraday_balances.go()
+
+ for acc in self.page.populate_balances(accounts):
yield acc
@need_login
diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py
index 992e67110a97a295778e3c84b8467cff50cd39a0..b06b284c909617bf9565675fbd1678d714371f21 100644
--- a/modules/societegenerale/sgpe/json_pages.py
+++ b/modules/societegenerale/sgpe/json_pages.py
@@ -22,13 +22,18 @@
from weboob.browser.pages import LoggedPage, JsonPage, pagination
from weboob.browser.elements import ItemElement, method, DictElement
-from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Format, BrowserURL, Env
+from weboob.browser.filters.standard import (
+ CleanDecimal, CleanText, Date, Format, BrowserURL, Env,
+ Field,
+)
from weboob.browser.filters.json import Dict
from weboob.capabilities.base import Currency
from weboob.capabilities import NotAvailable
from weboob.capabilities.bank import Account
from weboob.capabilities.bill import Document, Subscription
-from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword
+from weboob.exceptions import (
+ BrowserUnavailable, NoAccountsException, BrowserIncorrectPassword, BrowserPasswordExpired,
+)
from weboob.tools.capabilities.bank.iban import is_iban_valid
from weboob.tools.capabilities.bank.transactions import FrenchTransaction
from weboob.tools.compat import quote_plus
@@ -52,27 +57,55 @@ class AccountsJsonPage(LoggedPage, JsonPage):
u'Prêt': Account.TYPE_LOAN,
}
- def iter_accounts(self):
- for classeur in self.doc.get('donnees', {}).get('classeurs', {}):
- title = classeur['title']
- for compte in classeur.get('comptes', []):
- a = Account()
- a.label = CleanText().filter(compte['libelle'])
- a._id = compte['id']
- a.type = self.obj_type(a.label)
- a.number = compte['iban'].replace(' ', '')
- # for some account that don't have Iban the account number is store under this variable in the Json
- if not is_iban_valid(a.number):
- a.iban = NotAvailable
- else:
- a.iban = a.number
- # id based on iban to match ids in database.
- a.id = a.number[4:-2] if len(a.number) == 27 else a.number
- a._agency = compte['agenceGestionnaire']
- a._title = title
- yield a
-
- def obj_type(self, label):
+ def on_load(self):
+ if self.doc['commun']['statut'].lower() == 'nok':
+ reason = self.doc['commun']['raison']
+ if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS':
+ raise NoAccountsException("Vous n'avez pas l'autorisation de consulter : {}".format(reason))
+ elif reason == 'niv_auth_insuff':
+ raise BrowserIncorrectPassword('Vos identifiants sont incorrects')
+ elif reason == 'chgt_mdp_oblig':
+ raise BrowserPasswordExpired('Veuillez renouveler votre mot de passe')
+ raise BrowserUnavailable(reason)
+
+ @method
+ class iter_class_accounts(DictElement):
+ item_xpath = 'donnees/classeurs'
+
+ class iter_accounts(DictElement):
+ @property
+ def item_xpath(self):
+ if 'intradayComptes' in self.el:
+ return 'intradayComptes'
+ return 'comptes'
+
+ class item(ItemElement):
+ klass = Account
+
+ obj__id = Dict('id')
+ obj_number = CleanText(Dict('iban'), replace=[(' ', '')])
+ obj_iban = Field('number')
+ obj_label = CleanText(Dict('libelle'))
+ obj__agency = Dict('agenceGestionnaire')
+
+ def obj_id(self):
+ number = Field('number')(self)
+ if len(number) == 27:
+ # id based on iban to match ids in database.
+ return number[4:-2]
+ return number
+
+ def obj_iban(self):
+ # for some account that don't have Iban the account number is store under this variable in the Json
+ number = Field('number')(self)
+ if not is_iban_valid(number):
+ return NotAvailable
+ return number
+
+ def obj_type(self):
+ return self.page.acc_type(Field('label')(self))
+
+ def acc_type(self, label):
for wording, acc_type in self.TYPES.items():
if wording.lower() in label.lower():
return acc_type
@@ -91,14 +124,15 @@ def on_load(self):
if self.doc['commun']['statut'] == 'NOK':
reason = self.doc['commun']['raison']
if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS':
- raise BrowserIncorrectPassword("Vous n'avez pas l'autorisation de consulter : {}".format(reason))
+ raise NoAccountsException("Vous n'avez pas l'autorisation de consulter : {}".format(reason))
raise BrowserUnavailable(reason)
def populate_balances(self, accounts):
for account in accounts:
acc_dict = self.doc['donnees']['compteSoldesMap'][account._id]
- account.balance = CleanDecimal(replace_dots=True).filter(acc_dict['soldeComptable'])
- account.currency = Currency.get_currency(acc_dict['deviseSoldeComptable'])
+ account.balance = CleanDecimal(replace_dots=True).filter(acc_dict.get('soldeComptable', acc_dict.get('soldeInstantane')))
+ account.currency = Currency.get_currency(acc_dict.get('deviseSoldeComptable', acc_dict.get('deviseSoldeInstantane')))
+ account.coming = CleanDecimal(replace_dots=True, default=NotAvailable).filter(acc_dict.get('montantOperationJour'))
yield account
diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py
index 82f8b6d8970ce9fb74876a06f785fc9179ebacc5..5fc8d3276586ad97be4890f4e0d312c6d0410bfc 100644
--- a/modules/societegenerale/sgpe/pages.py
+++ b/modules/societegenerale/sgpe/pages.py
@@ -259,3 +259,9 @@ def on_load(self):
if self.doc.xpath('//div[@class="ngo_mu_message" and contains(text(), "momentanément indisponible")]'):
# Warning: it could occurs because of wrongpass, user have to change password
raise BrowserUnavailable(CleanText('//div[@class="ngo_mu_message"]')(self.doc))
+
+
+class InscriptionPage(SGPEPage):
+ def get_error(self):
+ message = CleanText('//head/title')(self.doc)
+ return message
diff --git a/modules/societegenerale/sgpe/transfer_pages.py b/modules/societegenerale/sgpe/transfer_pages.py
index 8794a7c6c4075d273f42ce249812c7f23d68d022..799c627ba4ef6af114a34c3a1521acf1d668bfff 100644
--- a/modules/societegenerale/sgpe/transfer_pages.py
+++ b/modules/societegenerale/sgpe/transfer_pages.py
@@ -29,7 +29,7 @@
from weboob.browser.filters.html import Attr
from weboob.browser.filters.json import Dict
from weboob.browser.filters.standard import Date, Eval
-from weboob.capabilities.bank import Recipient, Transfer, Account
+from weboob.capabilities.bank import Recipient, Transfer
from .pages import LoginPage
@@ -102,14 +102,8 @@ def update_origin_account(self, origin_account):
origin_account._underproduct_code = json_data['codeSousProduit']
break
else:
- assumptions = (
- not origin_account.balance,
- not origin_account.iban,
- origin_account.currency != 'EUR',
- origin_account.type == Account.TYPE_PEA,
- )
- if not any(assumptions):
- assert False, 'Account %s not found on transfer page' % (origin_account.label)
+ # some accounts are not able to do transfer
+ self.logger.warning('Account %s not found on transfer page', origin_account.label)
def iter_internal_recipients(self):
if self.doc.xpath('//ul[@id="idCmptToInterne"]'):
diff --git a/modules/yomoni/browser.py b/modules/yomoni/browser.py
index c9e86972d9ae81d9553c8b983254a46ae1b420d1..b405e840e4ad54e7a30b99e2c57386e1f24aeba3 100644
--- a/modules/yomoni/browser.py
+++ b/modules/yomoni/browser.py
@@ -143,7 +143,9 @@ def iter_investment(self, account, invs=None):
i.unitvalue = CleanDecimal().filter(inv['valeurCotation'])
i.valuation = CleanDecimal().filter(inv['montantEuro'])
i.vdate = Date().filter(inv['datePosition'])
- i.diff = CleanDecimal().filter(inv['performanceEuro'])
+ # performanceEuro is null sometimes in the JSON we retrieve.
+ if inv['performanceEuro']:
+ i.diff = CleanDecimal().filter(inv['performanceEuro'])
self.investments[account.id].append(i)
return self.investments[account.id]
|