diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index bc8821c9c3be22b6497d14606143d9d79798ed47..fef4a367e510cbbbdb88ccaca7d2b494d50e0057 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -124,11 +124,12 @@ class HSBC(LoginBrowser): def __init__(self, username, password, secret, *args, **kwargs): super(HSBC, self).__init__(username, password, *args, **kwargs) - self.accounts_list = OrderedDict() - self.unique_accounts_list = dict() + self.accounts_dict = OrderedDict() + self.unique_accounts_dict = dict() self.secret = secret self.PEA_LISTING = {} self.owners = [] + self.web_space = None def load_state(self, state): return @@ -184,6 +185,10 @@ def go_to_owner_accounts(self, owner): "Pas de TIERS", so we must always go to the owners list before going to the owner's account page. """ + # In case of only one owner, do nothing and exit + if len(self.owners) == 1: + return + if not self.owners_list.is_here(): self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) @@ -204,71 +209,85 @@ def iter_account_owners(self): people each having their own accounts. We must fetch the account for each person and store the owner of each account. """ - if self.unique_accounts_list: - for account in self.unique_accounts_list.values(): + self.web_space = self.page.get_web_space() + if not self.unique_accounts_dict and self.web_space == 'new_space': + """ + With the new space the "Mes comptes de tiers" service is not activated by default, so this page is empty. + We must declare here the only owner in 'self.owners' + This could change in the future with more people migrating. + """ + self.owners = [0] + self.accounts_dict[self.owners[0]] = {} + self.update_accounts_dict(self.owners[0]) + for a in self.accounts_dict[self.owners[0]].values(): + a._owner = self.owners[0] + self.unique_accounts_dict = self.accounts_dict[self.owners[0]] + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + + if self.unique_accounts_dict: + for account in self.unique_accounts_dict.values(): yield account else: self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) - if self.owners_list.is_here(): - self.owners = self.page.get_owners_urls() - - # self.accounts_list will be a dictionary of owners each - # containing a dictionary of the owner's accounts. - for owner in range(len(self.owners)): - self.accounts_list[owner] = {} - self.update_accounts_list(owner, True) - - # We must set an "_owner" attribute to each account. - for a in self.accounts_list[owner].values(): - a._owner = owner - - # go on cards page if there are cards accounts - for a in self.accounts_list[owner].values(): - if a.type == Account.TYPE_CARD: - self.location(a.url) - break - - # get all couples (card, parent) on cards page - all_card_and_parent = [] - if self.cbPage.is_here(): - all_card_and_parent = self.page.get_all_parent_id() - self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) - - # update cards parent and currency - for a in self.accounts_list[owner].values(): - if a.type == Account.TYPE_CARD: - for card in all_card_and_parent: - if a.id in card[0].replace(' ', ''): - a.parent = find_object(self.accounts_list[owner].values(), id=card[1]) - if a.parent and not a.currency: - a.currency = a.parent.currency - - # We must get back to the owners list before moving to the next owner: - self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) - - # Fill a dictionary will all accounts without duplicating common accounts: - for owner in self.accounts_list.values(): - for account in owner.values(): - if account.id not in self.unique_accounts_list.keys(): - self.unique_accounts_list[account.id] = account - - for account in self.unique_accounts_list.values(): - yield account + self.owners = self.page.get_owners_urls() + + # self.accounts_dict will be a dictionary of owners each + # containing a dictionary of the owner's accounts. + for owner in range(len(self.owners)): + self.accounts_dict[owner] = {} + self.update_accounts_dict(owner) + + # We must set an "_owner" attribute to each account. + for a in self.accounts_dict[owner].values(): + a._owner = owner + + # go on cards page if there are cards accounts + for a in self.accounts_dict[owner].values(): + if a.type == Account.TYPE_CARD: + self.location(a.url) + break + + # get all couples (card, parent) on cards page + all_card_and_parent = [] + if self.cbPage.is_here(): + all_card_and_parent = self.page.get_all_parent_id() + self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) + + # update cards parent and currency + for a in self.accounts_dict[owner].values(): + if a.type == Account.TYPE_CARD: + for card in all_card_and_parent: + if a.id in card[0].replace(' ', ''): + a.parent = find_object(self.accounts_dict[owner].values(), id=card[1]) + if a.parent and not a.currency: + a.currency = a.parent.currency + + # We must get back to the owners list before moving to the next owner: + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + + # Fill a dictionary will all accounts without duplicating common accounts: + for owner in self.accounts_dict.values(): + for account in owner.values(): + if account.id not in self.unique_accounts_dict.keys(): + self.unique_accounts_dict[account.id] = account + for account in self.unique_accounts_dict.values(): + yield account @need_login - def update_accounts_list(self, owner, iban=True): + def update_accounts_dict(self, owner, iban=True): # Go to the owner's account page in case we are not there already: self.go_to_owner_accounts(owner) - for a in self.page.iter_spaces_account(): + + for a in self.page.iter_spaces_account(self.web_space): try: - self.accounts_list[owner][a.id].url = a.url + self.accounts_dict[owner][a.id].url = a.url except KeyError: - self.accounts_list[owner][a.id] = a + self.accounts_dict[owner][a.id] = a if iban: self.location(self.js_url, params={'debr': 'COMPTES_RIB'}) if self.rib.is_here(): - self.page.get_rib(self.accounts_list[owner]) + self.page.get_rib(self.accounts_dict[owner]) @need_login def _quit_li_space(self): @@ -310,8 +329,10 @@ def _go_to_life_insurance(self, account): @need_login def get_history(self, account, coming=False, retry_li=True): self._quit_li_space() - self.update_accounts_list(account._owner, False) - account = self.accounts_list[account._owner][account.id] + # Update accounts list only in case of several owners + if len(self.owners) > 1: + self.update_accounts_dict(account._owner, iban=False) + account = self.accounts_dict[account._owner][account.id] if account.url is None: return [] @@ -367,23 +388,22 @@ def get_history(self, account, coming=False, retry_li=True): return history try: - self.go_post(self.accounts_list[account._owner][account.id].url) + self.go_post(account.url) # sometime go to hsbc life insurance space do logout except HTTPNotFound: self.app_gone = True self.do_logout() self.do_login() - # If we relogin on hsbc, all links have changed if self.app_gone: self.app_gone = False - self.update_accounts_list(account._owner, False) - self.location(self.accounts_list[account._owner][account.id].url) + self.update_accounts_dict(account._owner, iban=False) + self.location(self.accounts_dict[account._owner][account.id].url) if self.page is None: return [] - # for 'fusion' space + # for 'fusion' and 'new' space there is a form to submit on the page to go the account's history if hasattr(account, '_is_form') and account._is_form: # go on accounts page to get account form self.go_to_owner_accounts(account._owner) @@ -492,8 +512,8 @@ def get_pea_investments(self, account): def get_life_investments(self, account, retry_li=True): self._quit_li_space() - self.update_accounts_list(account._owner, False) - account = self.accounts_list[account._owner][account.id] + self.update_accounts_dict(account._owner, False) + account = self.accounts_dict[account._owner][account.id] try: if not self._go_to_life_insurance(account): self._quit_li_space() @@ -523,7 +543,7 @@ def _go_to_wealth_accounts(self, account): if not hasattr(self.page, 'get_middle_frame_url'): # if we can catch the URL, we go directly, else we need to browse # the website - self.update_accounts_list(account._owner, False) + self.update_accounts_dict(account._owner, False) self.location(self.page.get_middle_frame_url()) @@ -534,7 +554,7 @@ def _go_to_wealth_accounts(self, account): if self.login.is_here(): self.logger.warning('Connection to the Logon page failed, we must try again.') self.do_login() - self.update_accounts_list(account._owner, False) + self.update_accounts_dict(account._owner, False) self.investment_form_page.go() # If reloggin did not help accessing the wealth space, # there is nothing more we can do to get there. diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index f923b66f69761e37735cdad0ea1d3df961a370c3..ca0b932b00febc7b193b3ae3aa2338449476c616 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -31,24 +31,23 @@ from weboob.browser.filters.standard import ( Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp, Currency, Format, Date ) -from weboob.browser.filters.html import AbsoluteLink, TableCell +from weboob.browser.filters.html import AbsoluteLink, Attr, TableCell from weboob.browser.filters.javascript import JSVar from weboob.capabilities.profile import Person from .landing_pages import GenericLandingPage class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile(r'^PRLV (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s+(?P.*)'), - FrenchTransaction.TYPE_CARD), - (re.compile(r'^DAB (?P
\d{2})/(?P\d{2}) ((?P\d{2})H(?P\d{2}) )?(?P.*?)( CB N°.*)?$'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^CHEQUE( \d+)?$'), FrenchTransaction.TYPE_CHECK), - (re.compile(r'^COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile(r'^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(r'^FACTURES CB (?P.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), - ] + PATTERNS = [ + (re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^PRLV (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s+(?P.*)'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^DAB (?P
\d{2})/(?P\d{2}) ((?P\d{2})H(?P\d{2}) )?(?P.*?)( CB N°.*)?$'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^CHEQUE( \d+)?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^FACTURES CB (?P.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + ] class FrameContainer(GenericLandingPage): @@ -57,7 +56,7 @@ class FrameContainer(GenericLandingPage): # main page, a frameset def on_load(self): txt = CleanText('//p[@class="debit"]', default='')(self.doc) - if u"Vos données d'identification (identifiant - code secret) sont incorrectes" in txt: + if "Vos données d'identification (identifiant - code secret) sont incorrectes" in txt: raise BrowserIncorrectPassword() def get_js_url(self): @@ -66,7 +65,7 @@ def get_js_url(self): def get_frame(self): try: - a = self.doc.xpath(u'//frame["@name=FrameWork"]')[0] + a = self.doc.xpath('//frame["@name=FrameWork"]')[0] except IndexError: return None else: @@ -130,26 +129,56 @@ def filter(self, text): class AccountsPage(GenericLandingPage): - is_here = '//h1[text()="Synthèse"]' + def is_here(self): + return CleanText('//h1[text()="Synthèse"]')(self.doc) or CleanText('//span[@class="hsbc-pib-title-text"][text()="Tous mes comptes au "]')(self.doc) - def iter_spaces_account(self): + def get_web_space(self): + """ Several spaces on HSBC, need to get which one we are on to adapt parsing to owners""" if self.doc.xpath('//p[text()="HSBC Fusion"]'): - space = 'fusion' + return 'fusion' + elif self.doc.xpath('//span[contains(@class, "screen-reader-text") and text()="Aller vers hsbc.fr"]'): + return 'new_space' else: - space = 'default' + return 'default' + def iter_spaces_account(self, space): accounts = { 'fusion': self.iter_fusion_accounts, 'default': self.iter_accounts, + 'new_space': self.iter_new_space_accounts, } return accounts[space]() def go_history_page(self, account): - for acc in self.doc.xpath('//div[@onclick]'): - # label contains account number, it's enough to check if it's the right account - if account.label == Label(CleanText('.//p[@class="title"]'))(acc): - form_id = CleanText('.//form/@id')(acc) - return self.get_form(id=form_id).submit() + if self.browser.web_space == 'new_space': + # Must iterate through forms and find a match between accound number and the 'value' tag to know which form to submit + for form in self.doc.xpath('//form[@id]'): + value = Attr('.//input[@name="CPT_IdPrestation"]', 'value')(form) + if account.id in value: + form_id = Attr('.', 'id')(form) + return self.get_form(id=form_id).submit() + else: + for acc in self.doc.xpath('//div[@onclick]'): + # label contains account number, it's enough to check if it's the right account + if account.label == Label(CleanText('.//p[@class="title"]'))(acc): + form_id = CleanText('.//form/@id')(acc) + return self.get_form(id=form_id).submit() + + @method + class iter_new_space_accounts(ListElement): + item_xpath = '//div[@class="hsbc-pib-bloc-row-container"]' + + class item(ItemElement): + klass = Account + + # TODO: 'obj_id' will need redefinition when we find connections using the new space and Investment account and main account that have the same id + obj_id = CleanText('./p/span[@class="hsbc-pib-text--xsmall uk-text-gray"]', replace=[('.', ''), (' ', '')]) + obj_label = Label(CleanText('./p/span[@class="hsbc-pib-text hsbc-pib-bloc-account-name"]')) + obj_type = AccountsType(Field('label')) + obj_balance = CleanDecimal('./p/span[@class="hsbc-pib-text uk-text-bold"]', replace_dots=True) + obj_currency = Currency('./p/span[@class="hsbc-pib-text uk-text-bold"]') + obj_url = CleanText('.//form/@action') + obj__is_form = bool(CleanText('.//form/@id')) @method class iter_accounts(ListElement): @@ -182,7 +211,7 @@ def obj_url(self): @property def obj_balance(self): - if self.el.xpath('./parent::*/tr/th') and self.el.xpath('./parent::*/tr/th')[0].text in [u'Credits', u'Crédits']: + if self.el.xpath('./parent::*/tr/th') and self.el.xpath('./parent::*/tr/th')[0].text in ['Credits', 'Crédits']: return CleanDecimal(replace_dots=True, sign=lambda x: -1).filter(self.el.xpath('./td[3]')) return CleanDecimal(replace_dots=True).filter(self.el.xpath('./td[3]')) @@ -236,7 +265,8 @@ def obj_id(self): class OwnersListPage(AccountsPage): - is_here = '//h1[text()="Comptes de tiers"]' + def is_here(self): + return CleanText('//h1[text()="Comptes de tiers"]')(self.doc) or CleanText('//h1[text()="Gérer les comptes de mes tiers"]')(self.doc) def get_owners_urls(self): return self.doc.xpath('//div[@class="GoBack"]/a/@href') @@ -250,9 +280,9 @@ def link_rib(self, accounts): for id, acc in accounts.items(): if acc.iban or acc.type is not Account.TYPE_CHECKING: continue - digit_id = ''.join(re.findall('\d', id)) + digit_id = ''.join(re.findall(r'\d', id)) if digit_id in CleanText('//div[@class="RIB_content"]')(self.doc): - acc.iban = re.search('(FR\d{25})', CleanText('//div[strong[contains(text(), "IBAN")]]', replace=[(' ', '')])(self.doc)).group(1) + acc.iban = re.search(r'(FR\d{25})', CleanText('//div[strong[contains(text(), "IBAN")]]', replace=[(' ', '')])(self.doc)).group(1) def get_rib(self, accounts): self.link_rib(accounts) @@ -316,11 +346,16 @@ def get_all_parent_id(self): class CPTOperationPage(GenericLandingPage): - is_here = '''//h1[text()="Historique des opérations"] and //h2[text()="Recherche d'opération"]''' + def is_here(self): + return (CleanText('//h1[text()="Historique des opérations"]')(self.doc) + and (CleanText('''//h2[text()="Recherche d'opération"]''')(self.doc) + or CleanText('//div[@class="hsbc-datatable-search"]/label[text()="Rechercher"]') + ) + ) def get_history(self): if self.doc.xpath('//form[@name="FORM_SUITE"]'): - m = re.search('suite[\s]+=[\s]+([\w]+)', CleanText().filter(self.doc.xpath('//script[contains(text(), "var suite")]'))) + m = re.search(r'suite[\s]+=[\s]+([\w]+)', CleanText().filter(self.doc.xpath('//script[contains(text(), "var suite")]'))) if m and m.group(1) == "true": form = self.get_form(name="FORM_SUITE") self.doc = self.browser.location("%s" % form.url, params=dict(form)).page.doc @@ -332,7 +367,7 @@ def get_history(self): first_history = None for m in re.finditer(r"CL\((\d+),'(.+)','(.+)','(.+)','([\d -\.,]+)',('([\d -\.,]+)',)?'\d+','\d+','[\w\s]+'\);", script.text, flags=re.MULTILINE | re.UNICODE): op = Transaction() - raw = re.sub(u'[ ]+', u' ', m.group(4).replace(u'\n', u' ').replace(r"\'", "'")) + raw = re.sub(r'\s+', ' ', m.group(4).replace('\n', ' ').replace("\'", "'")) op.parse(date=m.group(3), raw=raw) op.set_amount(m.group(5)) op._coming = (re.match(r'\d+/\d+/\d+', m.group(2)) is None) @@ -355,7 +390,7 @@ def on_load(self): class LoginPage(HTMLPage): @property def logged(self): - if self.doc.xpath(u'//p[contains(text(), "You are now being redirected to your Personal Internet Banking.")]'): + if self.doc.xpath('//p[contains(text(), "You are now being redirected to your Personal Internet Banking.")]'): return True return False @@ -378,7 +413,7 @@ def login(self, login): def get_no_secure_key(self): try: - a = self.doc.xpath(u'//a[contains(text(), "Without HSBC Secure Key")]')[0] + a = self.doc.xpath('//a[contains(text(), "Without HSBC Secure Key")]')[0] except IndexError: return None else: @@ -387,8 +422,8 @@ def get_no_secure_key(self): def login_w_secure(self, password, secret): form = self.get_form(nr=0) form['memorableAnswer'] = secret - inputs = self.doc.xpath(u'//input[starts-with(@id, "keyrcc_password_first")]') - split_pass = u'' + inputs = self.doc.xpath('//input[starts-with(@id, "keyrcc_password_first")]') + split_pass = '' if len(password) < len(inputs): raise BrowserIncorrectPassword('The password must be at least %d characters' % len(inputs)) elif len(password) > len(inputs):