diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index 172a9847ea6b4e011e5c7f514750c344bbe807f3..a6342afead9bae1e167a18a55d256076f32a2e4f 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -368,7 +368,8 @@ def get_history_by_receipt(account, coming, sel_tbl1=None): self.location('/cyber/internet/Page.do', params=next_params) if coming and account._coming_count: - for i in range(account._coming_count): + for i in range(account._coming_start, + account._coming_start + account._coming_count): for tr in get_history_by_receipt(account, coming, sel_tbl1=i): yield tr else: diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index f98ca17e33b12eaeba483843e659572189e30822..bb88da9e4776739da17066f46fce8093a9d71b86 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -310,26 +310,27 @@ def login(self, login, passwd): class MyVirtKeyboard(SplitKeyboard): char_to_hash = { - '0': '6a2cb38bcfc27781faaec727ad304ce2', - '1': '296140f37a22b5e2b4871272aed22444', - '2': 'c1318fd381665a97e1052f85213867a7', - '3': 'fe19d2cc8f8d09b818b05c2a10218233', - '4': 'd5a03e69857bf01fc373cedbe2530ca9', - '5': '289ae90e4adfa58ef4767d9151c96348', - '6': '88938bbbb6b81ee2a32568f7081be488', - '7': '96499777fb95974ee651f19181de6c01', - '8': '6e2e052c9301d1f381155912ad4d3874', - '9': '5958d54d88bfaa172305575164b39a8d', + '0': 'cce0f72c47c74a3dde57c4fdbcda1db4', + '1': 'f5d22afb3ece4dec73bd8a2a4c2844da', + '2': '6d3e5db2ccac3f2c13c1f0ba22571857', + '3': ('c8e4f6addac4d322f0f9668d472a146c', '34d0566ea3f2330c675365da3178f6ab'), + '4': '8a8c769418ec829c208ed442fbf5fe77', + '5': '2c3ae480bc91f73b431b048b584026c7', + '6': 'a80d639443818e838b434c36dd518df5', + '7': '8e59048702e4c5f89bbbc1a598d06d1e', + '8': '46bc59a5b288c63477ff52811a3961c5', + '9': 'a7bf34568154ef91e990aa5bade3e946', } + codesep = ' ' def convert(self, buffer): im = Image.open(BytesIO(buffer)) - im = im.resize((5, 8), Image.BILINEAR) + im = im.resize((5, 8), Image.BICUBIC) im = im.filter(ImageFilter.UnsharpMask(radius=2, percent=110, threshold=3)) - im = im.convert("P", dither=Image.NONE) + im = im.convert("L", dither=Image.NONE) im = Image.eval(im, lambda x: 0 if x < 160 else 255) s = BytesIO() im.save(s, 'png') @@ -682,6 +683,11 @@ def iter_accounts(self, next_pages): self.logger.debug('there are no cards on this page') continue + # We are processing another card, so reset account + if CleanText('.')(cols[0]) and account is not None: + yield account + account = None + id = CleanText(None).filter(cols[self.COL_ID]) if len(id) > 0: if account is not None: @@ -700,7 +706,15 @@ def iter_accounts(self, next_pages): account._coming_params = params.copy() account._coming_params['dialogActionPerformed'] = 'SELECTION_ENCOURS_CARTE' account._coming_params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1] - account._coming_count = len(self.doc.xpath('//table[@id="tbl1"]/tbody/tr/td[5]/span[not(contains(text(), "(1)"))]')) + + # select current row and next rows till parent name is empty + account._coming_start = int(tr.attrib['id'].split('_', 1)[1]) + account._coming_count = 1 + for row in tr.xpath('./following-sibling::tr[./td[5]/span[not(contains(text(), "(1)"))]]'): + if CleanText('./td[2]')(row): + break + account._coming_count += 1 + elif account is None: raise BrokenPageError('Unable to find accounts on cards page') else: diff --git a/modules/binck/browser.py b/modules/binck/browser.py index 7dba65057d4199d4cb278fbd29265ab97f36554c..ebcebe0633ed2747550b1271dd80da759e849b13 100644 --- a/modules/binck/browser.py +++ b/modules/binck/browser.py @@ -30,6 +30,7 @@ from .pages import ( LoginPage, HomePage, AccountsPage, OldAccountsPage, HistoryPage, InvestmentPage, InvestDetailPage, InvestmentListPage, QuestionPage, ChangePassPage, LogonFlowPage, ViewPage, SwitchPage, + HandlePasswordsPage, PostponePasswords, ) @@ -61,6 +62,8 @@ class BinckBrowser(LoginBrowser): r'FsmaMandatoryQuestionnairesOverview', QuestionPage) change_pass = URL(r'/ChangePassword/Index', r'/EditSetting/GetSetting\?code=MutationPassword', ChangePassPage) + handle_passwords = URL(r'/PersonalCredentials/Index', HandlePasswordsPage) + postpone_passwords = URL(r'/PersonalCredentials/PostPone', PostponePasswords) def deinit(self): if self.page and self.page.logged: @@ -94,6 +97,13 @@ def switch_account(self, account_id): @need_login def iter_accounts(self): + # If we already know that it is an old website connection, + # we can call old_website_connection() right away. + if self.old_website_connection: + for account in self.iter_old_accounts(): + yield account + return + if self.unique_account: self.account.stay_or_go() else: @@ -121,41 +131,46 @@ def iter_accounts(self): # so we need to fetch them on the OldAccountsPage for now: else: self.old_website_connection = True - self.old_accounts.go() - for a in self.page.iter_accounts(): - try: - self.old_accounts.stay_or_go().go_to_account(a.id) - except ServerError as exception: - # get html error to parse - parser = etree.HTMLParser() - html_error = etree.parse(StringIO(exception.response.text), parser) - account_error = html_error.xpath('//p[contains(text(), "Votre compte est")]/text()') - if account_error: - raise ActionNeeded(account_error[0]) - else: - raise - - a.iban = self.page.get_iban() - # Get token - token = self.page.get_token() - # Get investment page - data = {'grouping': "SecurityCategory"} - try: - a._invpage = self.investment.go(data=data, headers=token) \ - if self.page.is_investment() else None - except HTTPNotFound: - # if it's not an invest account, the portfolio link may be present but hidden and return a 404 - a._invpage = None + for account in self.iter_old_accounts(): + yield account + + @need_login + def iter_old_accounts(self): + self.old_accounts.go() + for a in self.page.iter_accounts(): + try: + self.old_accounts.stay_or_go().go_to_account(a.id) + except ServerError as exception: + # get html error to parse + parser = etree.HTMLParser() + html_error = etree.parse(StringIO(exception.response.text), parser) + account_error = html_error.xpath('//p[contains(text(), "Votre compte est")]/text()') + if account_error: + raise ActionNeeded(account_error[0]) + else: + raise + + a.iban = self.page.get_iban() + # Get token + token = self.page.get_token() + # Get investment page + data = {'grouping': "SecurityCategory"} + try: + a._invpage = self.investment.go(data=data, headers=token) \ + if self.page.is_investment() else None + except HTTPNotFound: + # if it's not an invest account, the portfolio link may be present but hidden and return a 404 + a._invpage = None - if a._invpage: - a.valuation_diff = a._invpage.get_valuation_diff() - # Get history page - data = [('currencyCode', a.currency), ('startDate', ""), ('endDate', "")] - a._histpages = [self.history.go(data=data, headers=token)] - while self.page.doc['EndOfData'] is False: - a._histpages.append(self.history.go(data=self.page.get_nextpage_data(data[:]), headers=token)) + if a._invpage: + a.valuation_diff = a._invpage.get_valuation_diff() + # Get history page + data = [('currencyCode', a.currency), ('startDate', ""), ('endDate', "")] + a._histpages = [self.history.go(data=data, headers=token)] + while self.page.doc['EndOfData'] is False: + a._histpages.append(self.history.go(data=self.page.get_nextpage_data(data[:]), headers=token)) - yield a + yield a @need_login def iter_investment(self, account): diff --git a/modules/binck/pages.py b/modules/binck/pages.py index 46962a33d9c486ef1acee005b674d59dde279d19..3f57077ad45aae8f45fc0829c064c5300a452ce0 100644 --- a/modules/binck/pages.py +++ b/modules/binck/pages.py @@ -47,6 +47,13 @@ def on_load(self): form.submit() +class BinckPage(LoggedPage, HTMLPage): + # Used to factorize the get_token() method + def get_token(self): + return [{Attr('.', 'name')(input): Attr('.', 'value')(input)} + for input in self.doc.xpath('//input[contains(@name, "Token")]')][0] + + class ViewPage(LoggedPage, HTMLPage): # We automatically skip the new website tutorial def on_load(self): @@ -75,6 +82,16 @@ def on_load(self): raise BrowserPasswordExpired(message) +class HandlePasswordsPage(BinckPage): + def on_load(self): + token = self.get_token() + self.browser.postpone_passwords.go(headers=token, method='POST') + self.browser.home_page.go() + + +class PostponePasswords(LoggedPage, HTMLPage): + pass + class LogonFlowPage(HTMLPage): def on_load(self): raise ActionNeeded(CleanText('//article//h1 | //article//h3')(self.doc)) @@ -91,7 +108,7 @@ def get_error(self): return CleanText('//div[contains(@class, "errors")]')(self.doc) -class AccountsPage(LoggedPage, HTMLPage): +class AccountsPage(BinckPage): TYPES = {'L': Account.TYPE_SAVINGS, 'CT': Account.TYPE_MARKET, 'PEA': Account.TYPE_PEA, @@ -103,10 +120,6 @@ class AccountsPage(LoggedPage, HTMLPage): def has_accounts_table(self): return self.doc.xpath('//table[contains(@class, "accountoverview-table")]') - def get_token(self): - return [{Attr('.', 'name')(input): Attr('.', 'value')(input)} \ - for input in self.doc.xpath('//input[contains(@name, "Token")]')][0] - @method class iter_accounts(ListElement): # Tables have no headers so we must use ListElement. @@ -135,7 +148,7 @@ def obj_currency(self): return Account.get_currency(CleanText('.//div[contains(text(), "Total des avoirs")]/following::strong[1]')(self)) -class OldAccountsPage(LoggedPage, HTMLPage): +class OldAccountsPage(BinckPage): ''' Old website accounts page. We can get rid of this class when all users have access to the new website. @@ -154,10 +167,6 @@ def go_to_account(self, number): def get_iban(self): return CleanText('//div[@class="iban"]/text()', replace=[(' ', '')], default=NotAvailable)(self.doc) - def get_token(self): - return [{Attr('.', 'name')(input): Attr('.', 'value')(input)} \ - for input in self.doc.xpath('//input[contains(@name, "Token")]')][0] - def is_investment(self): # warning: the link can be present even in case of non-investement account return CleanText('//a[contains(@href, "Portfolio")]', default=False)(self.doc) diff --git a/modules/bnporc/enterprise/browser.py b/modules/bnporc/enterprise/browser.py index 16d78bf0e79a46585b6aae655404a1a0d34fc043..058446e6e9868849e91a2ad189a5d36e094561a5 100644 --- a/modules/bnporc/enterprise/browser.py +++ b/modules/bnporc/enterprise/browser.py @@ -146,8 +146,8 @@ def _iter_history_base(self, account): self.logger.debug('skipping coming %r', transaction.to_dict()) continue history.append(transaction) - for transaction in sorted_transactions(history): - yield transaction + for transaction in sorted_transactions(history): + yield transaction @need_login def iter_coming_operations(self, account): diff --git a/modules/bnppere/pages.py b/modules/bnppere/pages.py index 23b87bfa552ad9e18042685e73afc6c522378d5c..25be10d89398e25eecbd65d6cd1be6f80b1c1734 100644 --- a/modules/bnppere/pages.py +++ b/modules/bnppere/pages.py @@ -116,11 +116,6 @@ class iter_history(ListElement): class item(ItemElement): klass = Transaction - def obj_id(self): - label = CleanText(Attr('./a[contains(@class, "accordion_collapse")]', "id"))(self) - label = ''.join(i for i in label if i.isdigit()) - return label - obj_date = Date(CleanText('./div[contains(@class, "accordion_header")]/div[1]/p')) obj_category = CleanText('./div[contains(@class, "accordion_header")]/div[2]/p[1]') obj_label = CleanText('./div[contains(@class, "accordion_header")]/div[3]/p[1]') diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 1b924322d23030b0e8b272443e8cd9a60c4aa5ab..4edd2fd620fc7b77022a20a4a9bfba78771eb2ba 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -65,7 +65,7 @@ def get_iban(self): if self.doc.xpath('//div[has-class("alert")]/p[contains(text(), "Une erreur est survenue")]') or \ self.doc.xpath('//div[has-class("alert")]/p[contains(text(), "Le compte est introuvable")]'): return NotAvailable - return CleanText('//table[thead[tr[th[contains(text(), "Code I.B.A.N")]]]]/tbody/tr/td[2]', replace=[(' ', '')])(self.doc) + return CleanText('//div[strong[contains(text(),"IBAN")]]/div[contains(@class, "definition")]', replace=[(' ', '')])(self.doc) class AuthenticationPage(HTMLPage): diff --git a/modules/bred/bred/pages.py b/modules/bred/bred/pages.py index 35b1082b8f6a15a847b9972eca1c1f9e94c257a4..d46823f3da4fce33b0a683af23740511e37af394 100644 --- a/modules/bred/bred/pages.py +++ b/modules/bred/bred/pages.py @@ -331,9 +331,13 @@ class ErrorCodePage(HTMLPage): def on_load(self): code = re.search(r'\/\?errorCode=(\d+)', self.url).group(1) page = self.browser.open('/particuliers/compte-bancaire/comptes-en-ligne/bredconnect-compte-ligne?errorCode=%s' % code).page - # invalid login/password - if code == '20100': - msg = CleanText('//label[contains(@class, "error")]')(page.doc) + msg = CleanText('//label[contains(@class, "error")]', default=None)(page.doc) + # 20100: invalid login/password + # 139: dispobank user trying to connect to Bred + if code in ('20100', '139'): raise BrowserIncorrectPassword(msg) + # 20104 & 1000: unknown error during login + elif code in ('20104', '1000'): + raise BrowserUnavailable(msg) - assert False, 'The % error is not handled.' % code + assert False, 'Error %s is not handled yet.' % code diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index b3f7e74d0e2d62edd4f96e644a60a7da4aeb5b20..c5b6bd1796b2fa3d30a516108e636dcabb006bc3 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -30,13 +30,16 @@ from weboob.browser.switch import SiteSwitch from weboob.browser.url import URL from .compat.weboob_capabilities_bank import Account, AddRecipientStep, Recipient, TransferBankError, Transaction, TransferStep -from weboob.capabilities.base import NotAvailable +from weboob.capabilities.base import NotAvailable, find_object from weboob.capabilities.profile import Profile from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError, ServerError from weboob.exceptions import ( BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPError, BrowserPasswordExpired, ActionNeeded ) -from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction +from weboob.tools.capabilities.bank.transactions import ( + sorted_transactions, FrenchTransaction, keep_only_card_transactions, + omit_deferred_transactions, +) from weboob.tools.capabilities.bank.investments import create_french_liquidity from weboob.tools.compat import urljoin, urlparse from weboob.tools.value import Value @@ -49,7 +52,7 @@ ProTransferSummaryPage, ProAddRecipientOtpPage, ProAddRecipientPage, SmsPage, SmsPageOption, SmsRequest, AuthentPage, RecipientPage, CanceledAuth, CaissedepargneKeyboard, TransactionsDetailsPage, LoadingPage, ConsLoanPage, MeasurePage, NatixisLIHis, NatixisLIInv, NatixisRedirectPage, - SubscriptionPage, CreditCooperatifMarketPage, UnavailablePage, + SubscriptionPage, CreditCooperatifMarketPage, UnavailablePage, CardsPage, CardsComingPage, CardsOldWebsitePage, ) from .linebourse_browser import LinebourseAPIBrowser @@ -81,6 +84,9 @@ class CaisseEpargne(LoginBrowser, StatesMixin): pro_add_recipient_otp = URL('https://.*/Portail.aspx.*', ProAddRecipientOtpPage) pro_add_recipient = URL('https://.*/Portail.aspx.*', ProAddRecipientPage) measure_page = URL('https://.*/Portail.aspx.*', MeasurePage) + cards_old = URL('https://.*/Portail.aspx.*', CardsOldWebsitePage) + cards = URL('https://.*/Portail.aspx.*', CardsPage) + cards_coming = URL('https://.*/Portail.aspx.*', CardsComingPage) authent = URL('https://.*/Portail.aspx.*', AuthentPage) subscription = URL('https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) home = URL('https://.*/Portail.aspx.*', IndexPage) @@ -430,6 +436,30 @@ def get_accounts_list(self): else: assert False, "new domain that hasn't been seen so far ?" + self.home.go() + self.page.go_list() + self.page.go_cards() + + if self.cards.is_here() or self.cards_old.is_here(): + cards = list(self.page.iter_cards()) + for card in cards: + card.parent = find_object(self.accounts, number=card._parent_id) + assert card.parent, 'card account %r parent was not found' % card + + # If we are in the new site, we have to get each card coming transaction link. + if self.cards.is_here(): + for card in cards: + info = card.parent._card_links + + # If info is filled, that mean there are comings transaction + card._coming_info = None + if info: + self.page.go_list() + self.page.go_history(info) + card._coming_info = self.page.get_card_coming_info(card.number, info.copy()) + + self.accounts.extend(cards) + # Some accounts have no available balance or label and cause issues # in the backend so we must exclude them from the accounts list: self.accounts = [account for account in self.accounts if account.label and account.balance != NotAvailable] @@ -445,7 +475,6 @@ def get_loans_list(self): if self.page.check_no_accounts() or self.page.check_no_loans(): return [] - access_to_loans = False for trial in range(5): for _ in range(3): self.home_tache.go(tache='CRESYNT0') @@ -461,12 +490,8 @@ def get_loans_list(self): self.logger.warning('Access to loans failed, we try again') else: # We managed to reach the Loans JSON - access_to_loans = True break - if not access_to_loans: - raise BrowserUnavailable() - for _ in range(3): try: self.home_tache.go(tache='CPTSYNT0') @@ -480,8 +505,13 @@ def get_loans_list(self): return iter(self.loans) + # For all account, we fill up the history with transaction. For checking account, there will have + # also deferred_card transaction too. + # From this logic, if we send "account_card", that mean we recover all transactions from the parent + # checking account of the account_card, then we filter later the deferred transaction. @need_login - def _get_history(self, info): + def _get_history(self, info, account_card=None): + # Only fetch deferred debit card transactions if `account_card` is not None if isinstance(info['link'], list): info['link'] = info['link'][0] if not info['link'].startswith('HISTORIQUE'): @@ -500,6 +530,11 @@ def _get_history(self, info): if 'netpro' in self.page.url and not self.page.is_history_of(info['id']): self.page.go_history_netpro(info) + # In this case, we want the coming transaction for the new website + # (old website return coming directly in `get_coming()` ) + if account_card and info and info['type'] == 'HISTORIQUE_CB': + self.page.go_coming(account_card._coming_info['link']) + info['link'] = [info['link']] for i in range(self.HISTORY_MAX_PAGE): @@ -508,14 +543,25 @@ def _get_history(self, info): # list of transactions on account page transactions_list = [] - list_form = [] + card_and_forms = [] for tr in self.page.get_history(): transactions_list.append(tr) if tr.type == tr.TYPE_CARD_SUMMARY: - list_form.append(self.page.get_form_to_detail(tr)) + if account_card: + if self.card_matches(tr.card, account_card.number): + card_and_forms.append((tr.card, self.page.get_form_to_detail(tr))) + else: + self.logger.debug('will skip summary detail (%r) for different card %r', tr, account_card.number) + + # For deferred card history only : + # + # Now that we find transactions that have TYPE_CARD_SUMMARY on the checking account AND the account_card number we want, + # we browse deferred card transactions that are resume by that list of TYPE_CARD_SUMMARY transaction. - # add detail card to list of transactions - for form in list_form: + # Checking account transaction: + # - 01/01 - Summary 5134XXXXXX103 - 900.00€ - TYPE_CARD_SUMMARY <-- We have to go in the form of this tr to get + # cards details transactions. + for card, form in card_and_forms: form.submit() if self.home.is_here() and self.page.is_access_error(): self.logger.warning('Access to card details is unavailable for this user') @@ -523,6 +569,8 @@ def _get_history(self, info): assert self.transaction_detail.is_here() for tr in self.page.get_detail(): tr.type = Transaction.TYPE_DEFERRED_CARD + if account_card: + tr.card = card transactions_list.append(tr) if self.new_website: self.page.go_newsite_back_to_summary() @@ -592,6 +640,15 @@ def _get_history_invests(self, account): def get_history(self, account): self.home.go() self.deleteCTX() + + if account.type == account.TYPE_CARD: + def match_cb(tr): + return self.card_matches(tr.card, account.number) + + hist = self._get_history(account.parent._info, account) + hist = keep_only_card_transactions(hist, match_cb) + return hist + if not hasattr(account, '_info'): raise NotImplementedError if account.type is Account.TYPE_LIFE_INSURANCE and 'measure_id' not in account._info: @@ -608,18 +665,37 @@ def get_history(self, account): self.linebourse.session.cookies.update(self.session.cookies) self.update_linebourse_token() return self.linebourse.iter_history(account.id) - return self._get_history(account._info) + + hist = self._get_history(account._info, False) + return omit_deferred_transactions(hist) @need_login def get_coming(self, account): + if account.type != account.TYPE_CARD: + return [] + trs = [] - if not hasattr(account, '_info'): + if not hasattr(account.parent, '_info'): raise NotImplementedError() - for info in account._card_links: - for tr in self._get_history(info.copy()): + + # We are on the old website + if hasattr(account, '_coming_eventargument'): + + if not self.cards_old.is_here(): + self.home.go() + self.page.go_list() + self.page.go_cards() + self.page.go_card_coming(account._coming_eventargument) + + return sorted_transactions(self.page.iter_coming()) + + # We are on the new website. + info = account.parent._card_links + # if info is empty, that mean there are no coming yet + if info: + for tr in self._get_history(info.copy(), account): tr.type = tr.TYPE_DEFERRED_CARD - tr.nopurge = True trs.append(tr) return sorted_transactions(trs) @@ -706,7 +782,7 @@ def get_profile(self): @need_login def iter_recipients(self, origin_account): - if origin_account.type == Account.TYPE_LOAN: + if origin_account.type in [Account.TYPE_LOAN, Account.TYPE_CARD]: return [] if 'pro' in self.url: @@ -933,3 +1009,9 @@ def download_document(self, document): self.page.go_document_list(sub_id=sub_id) return self.page.download_document(document).content + + def card_matches(self, a, b): + # For the same card, depending where we scrape it, we have + # more or less visible number. `X` are visible number, `*` hidden one's. + # tr.card: XXXX******XXXXXX, account.number: XXXXXX******XXXX + return (a[:4], a[-4:]) == (b[:4], b[-4:]) diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index f92080714c37e33adee7806607d2a8e2daf3ffd3..817fe405f9bc25e3b2eee4ce7fff2f0092f1f3e7 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -208,7 +208,7 @@ def on_load(self): if 'OIC_QCF' in self.browser.url: # QCF is a mandatory test to make sure you know the basics about financials products # however, you can still choose to postpone it. hence the continue link - link = Link('//span[@id="lea-prdvel-lien"]/p/b/a[contains(text(), "Continuer")]')(self.doc) + link = Link('//span[@id="lea-prdvel-lien"]//b/a[contains(text(), "Continuer")]')(self.doc) if link: self.logger.warning("By-passing QCF") self.browser.location(link) @@ -284,7 +284,7 @@ def _get_account_info(self, a, accounts): def is_account_inactive(self, account_id): return self.doc.xpath('//tr[td[contains(text(), $id)]][@class="Inactive"]', id=account_id) - def _add_account(self, accounts, link, label, account_type, balance): + def _add_account(self, accounts, link, label, account_type, balance, number=None): info = self._get_account_info(link, accounts) if info is None: self.logger.warning('Unable to parse account %r: %r' % (label, link)) @@ -295,6 +295,7 @@ def _add_account(self, accounts, link, label, account_type, balance): if is_rib_valid(info['id']): account.iban = rib2iban(info['id']) account._info = info + account.number = number account.label = label account.type = self.ACCOUNT_TYPES.get(label, info['acc_type'] if 'acc_type' in info else account_type) if 'PERP' in account.label: @@ -309,16 +310,15 @@ def _add_account(self, accounts, link, label, account_type, balance): account.currency = account.get_currency(balance) if balance and balance is not NotAvailable else NotAvailable account._card_links = [] + # Set coming history link to the parent account. At this point, we don't have card account yet. if account._info['type'] == 'HISTORIQUE_CB' and account.id in accounts: a = accounts[account.id] - if not a.coming: - a.coming = Decimal('0.0') - if account.balance and account.balance is not NotAvailable: - a.coming += account.balance - a._card_links.append(account._info) + a.coming = Decimal('0.0') + a._card_links = account._info return accounts[account.id] = account + return account def get_balance(self, account): if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP, Account.TYPE_CAPITALISATION): @@ -368,20 +368,27 @@ def get_list(self): self.ACCOUNT_TYPES.get(CleanText('.')(tds[2])) or\ self.ACCOUNT_TYPES.get(CleanText('.')(tds[3]), Account.TYPE_UNKNOWN) else: - # On the same row, there are many accounts (for example a - # check accound and a card one). + # On the same row, there could have many accounts (check account and a card one). + # For the card line, the number will be the same than the checking account, so we skip it. if len(tds) > 4: for i, a in enumerate(tds[2].xpath('./a')): label = CleanText('.')(a) balance = CleanText('.')(tds[-2].xpath('./a')[i]) - self._add_account(accounts, a, label, account_type, balance) - # Only 4 tds on banque de la reunion website. + number = None + # if i > 0, that mean it's a card account. The number will be the same than it's + # checking parent account, we have to skip it. + if i == 0: + number = CleanText('.')(tds[-4].xpath('./a')[0]) + self._add_account(accounts, a, label, account_type, balance, number) + # Only 4 tds on "banque de la reunion" website. elif len(tds) == 4: for i, a in enumerate(tds[1].xpath('./a')): label = CleanText('.')(a) balance = CleanText('.')(tds[-1].xpath('./a')[i]) self._add_account(accounts, a, label, account_type, balance) + self.logger.debug('we are on the %s website', 'old' if accounts else 'new') + if len(accounts) == 0: # New website self.browser.new_website = True @@ -405,7 +412,9 @@ def get_list(self): label = CleanText('./strong')(tds[0]) balance = CleanText('.')(tds[-1]) - self._add_account(accounts, a, label, account_type, balance) + account = self._add_account(accounts, a, label, account_type, balance) + if account: + account.number = CleanText('.')(tds[1]) return accounts.values() @@ -444,6 +453,8 @@ def get_loan_list(self): account.currency = account.get_currency(CleanText('./a')(tds[4])) accounts[account.id] = account + self.logger.debug('we are on the %s website', 'old' if accounts else 'new') + if len(accounts) == 0: # New website for table in self.doc.xpath('//div[@class="panel"]'): @@ -522,23 +533,58 @@ class item(ItemElement): obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date", default=''), default=NotAvailable), default=NotAvailable) + def submit_form(self, form, eventargument, eventtarget, scriptmanager): + form['__EVENTARGUMENT'] = eventargument + form['__EVENTTARGET'] = eventtarget + form['m_ScriptManager'] = scriptmanager + fix_form(form) + form.submit() + def go_list(self): + form = self.get_form(id='main') + eventargument = "CPTSYNT0" - form['__EVENTARGUMENT'] = "CPTSYNT0" + if "MM$m_CH$IsMsgInit" in form: + # Old website + eventtarget = "Menu_AJAX" + scriptmanager = "m_ScriptManager|Menu_AJAX" + else: + # New website + eventtarget = "MM$m_PostBack" + scriptmanager = "MM$m_UpdatePanel|MM$m_PostBack" + + self.submit_form(form, eventargument, eventtarget, scriptmanager) + + def go_cards(self): + form = self.get_form(id='main') + eventargument = "" if "MM$m_CH$IsMsgInit" in form: # Old website - form['__EVENTTARGET'] = "Menu_AJAX" - form['m_ScriptManager'] = "m_ScriptManager|Menu_AJAX" + eventtarget = "Menu_AJAX" + eventargument = "HISENCB0" + scriptmanager = "m_ScriptManager|Menu_AJAX" else: # New website - form['__EVENTTARGET'] = "MM$m_PostBack" - form['m_ScriptManager'] = "MM$m_UpdatePanel|MM$m_PostBack" + eventtarget = "MM$SYNTHESE$btnSyntheseCarte" + scriptmanager = "MM$m_UpdatePanel|MM$SYNTHESE$btnSyntheseCarte" - fix_form(form) + self.submit_form(form, eventargument, eventtarget, scriptmanager) - form.submit() + # only for old website + def go_card_coming(self, eventargument): + form = self.get_form(id='main') + eventtarget = "MM$HISTORIQUE_CB" + scriptmanager = "m_ScriptManager|Menu_AJAX" + self.submit_form(form, eventargument, eventtarget, scriptmanager) + + # only for new website + def go_coming(self, eventargument): + form = self.get_form(id='main') + eventtarget = "MM$HISTORIQUE_CB" + scriptmanager = "MM$m_UpdatePanel|MM$HISTORIQUE_CB" + self.submit_form(form, eventargument, eventtarget, scriptmanager) # On some pages, navigate to indexPage does not lead to the list of measures, so we need this form ... def go_measure_list(self): @@ -679,7 +725,11 @@ def get_history(self): continue if 'tot dif' in t.raw.lower(): t._link = Link(tr.xpath('./td/a'))(self.doc) - t.deleted = True + + # "Cb" for new site, "CB" for old one + mtc = re.match(r'(Cb|CB) (\d{4}\*+\d{6}) ', raw) + if mtc is not None: + t.card = mtc.group(2) t.set_amount(credit, debit) yield t @@ -786,6 +836,154 @@ def is_transfer_allowed(self): return not self.doc.xpath('//ul/li[contains(text(), "Aucun compte tiers n\'est disponible")]') +class CardsPage(IndexPage): + def is_here(self): + return CleanText('//h3[normalize-space(text())="Mes cartes (cartes dont je suis le titulaire)"]')(self.doc) + + @method + class iter_cards(TableElement): + head_xpath = '//table[@class="cartes"]/tbody/tr/th' + + col_label = 'Carte' + col_number = 'N°' + col_parent = 'Compte dépot associé' + col_coming = 'Encours' + + item_xpath = '//table[@class="cartes"]/tbody/tr[not(th)]' + + class item(ItemElement): + klass = Account + + obj_type = Account.TYPE_CARD + obj_label = Format('%s %s', CleanText(TableCell('label')), Field('id')) + obj_number = CleanText(TableCell('number')) + obj_id = CleanText(TableCell('number'), replace=[('*', 'X')]) + obj__parent_id = CleanText(TableCell('parent')) + obj_balance = 0 + obj_currency = Currency(TableCell('coming')) + + def obj_coming(self): + if CleanText(TableCell('coming'))(self) == '-': + raise SkipItem('immediate debit card?') + return CleanDecimal.French(TableCell('coming'), sign=lambda x: -1)(self) + + def condition(self): + immediate_str = '' + # There are some card without any information. To exclude them, we keep only account + # with extra "option" (ex: coming transaction link, block bank card...) + if 'Faire opposition' in CleanText("./td[5]")(self): + # Only deferred card have this option to see coming transaction, even when + # there is 0 coming (Table element have no thead for the 5th column). + if 'Consulter mon encours carte' in CleanText("./td[5]")(self): + return True + + # Card without 'Consulter mon encours carte' are immediate card. There are logged + # for now to make the debug easier + immediate_str = '[Immediate card]' + + self.logger.warning('Skip card %s (no history/coming information) %s', Field('number')(self), immediate_str) + return False + + +class CardsComingPage(IndexPage): + def is_here(self): + return CleanText('//h2[text()="Encours de carte à débit différé"]')(self.doc) + + def get_card_coming_info(self, number, info): + + # If the xpath match, that mean there are only one card + # We have enought information in `info` to get its coming transaction + if CleanText('//tr[@id="MM_HISTORIQUE_CB_rptMois0_ctl01_trItem"]')(self.doc): + return info + + # If the xpath match, that mean there are at least 2 cards + xpath = '//tr[@id="MM_HISTORIQUE_CB_rptMois0_trItem_0"]' + + # In case of multiple card, first card coming's transactions are reachable + # with information in `info`. + if Regexp(CleanText(xpath), r'(\d{6}\*{6}\d{4})')(self.doc) == number: + return info + + # For all card except the first one for the same check account, we have to get info through their href info + link = CleanText(Attr('//a[contains(text(),"%s")]' % number, 'href'))(self.doc) + infos = re.match(r'.*(DETAIL_OP_M0&[^\"]+).*', link) + info['link'] = infos.group(1) + + return info + + +class CardsOldWebsitePage(IndexPage): + def is_here(self): + return CleanText('//span[@id="MM_m_CH_lblTitle" and contains(text(), "Historique de vos encours CB")]')(self.doc) + + def get_account(self): + infos = CleanText('.//span[@id="MM_HISTORIQUE_CB"]/table[position()=1]//td')(self.doc) + result = re.search(r'.*(\d{11}).*', infos) + return result.group(1) + + def get_date(self): + title = CleanText('//span[@id="MM_HISTORIQUE_CB_m_TableTitle3_lblTitle"]')(self.doc) + title_date = re.match('.*le (.*) sur .*', title) + return Date(dayfirst=True).filter(title_date.group(1)) + + @method + class iter_cards(TableElement): + head_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGOpeM0"]//tr[@class="DataGridHeader"]/td' + item_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGOpeM0"]//tr[not(contains(@class, "DataGridHeader")) and position() < last()]' + + col_label = 'Libellé' + col_coming = 'Solde' + + class item(ItemElement): + klass = Account + + obj_type = Account.TYPE_CARD + obj_label = Format('%s %s', CleanText(TableCell('label')), CleanText(Field('number'))) + obj_balance = 0 + obj_coming = CleanDecimal.French(TableCell('coming')) + obj_currency = Currency(TableCell('coming')) + + def obj__parent_id(self): + return self.page.get_account() + + def obj_number(self): + return CleanText(TableCell('number'))(self).replace('*', 'X') + + def obj_id(self): + number = Field('number')(self).replace('X', '') + account_id = '%s-%s' % (self.obj__parent_id(), number) + return account_id + + def obj__coming_eventargument(self): + url = Attr('.//a', 'href')(self) + res = re.match(r'.*(DETAIL_OP_M0\&.*;\d{8})", .*', url) + return res.group(1) + + def parse(self, obj): + # There are no thead name for this column. + self._cols['number'] = 3 + + @method + class iter_coming(TableElement): + head_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGDetailOpe"]//tr[@class="DataGridHeader"]/td' + item_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGDetailOpe"]//tr[not(contains(@class, "DataGridHeader"))]' + + col_label = 'Libellé' + col_coming = 'Débit' + col_date = 'Date' + + class item(ItemElement): + klass = Transaction + + obj_type = Transaction.TYPE_DEFERRED_CARD + obj_label = CleanText(TableCell('label')) + obj_amount = CleanDecimal.French(TableCell('coming')) + obj_rdate = Date(CleanText(TableCell('date')), dayfirst=True) + + def obj_date(self): + return self.page.get_date() + + class ConsLoanPage(JsonPage): def get_conso(self): return self.doc @@ -992,7 +1190,9 @@ def parse(self, el): # TODO use after 'I'? _id = Regexp(CleanText('.'), r'- (\w+\d\w+)')(self) # at least one digit accounts = list(self.page.browser.get_accounts_list()) + list(self.page.browser.get_loans_list()) - match = [acc for acc in accounts if _id in acc.id] + # If it's an internal account, we should always find only one account with _id in it's id. + # Type card account contains their parent account id, and should not be listed in recipient account. + match = [acc for acc in accounts if _id in acc.id and acc.type != Account.TYPE_CARD] assert len(match) == 1 match = match[0] self.env['id'] = match.id diff --git a/modules/cmes/pages.py b/modules/cmes/pages.py index 468d18e9ea4feeccb9fa3b6297c8362f408abac5..5631ea1b95b8ccd7dc72a94be76e38762ab22be7 100644 --- a/modules/cmes/pages.py +++ b/modules/cmes/pages.py @@ -57,7 +57,7 @@ def on_load(self): # Need to update mail. Try to skip msg = "Merci de renseigner votre adresse e-mail" if CleanText('//p[@role="heading" and contains(text(), "%s")]' % msg)(self.doc): - url = Link('//a[contains(., "PASSER CETTE ETAPE")]')(self.doc) + url = Link('//a[contains(., "PASSER CETTE ETAPE")]', default=None)(self.doc) if url: self.browser.location(url) else: diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index b39ec4751de77e891cdab47d322aa56b52d86e26..acaad93e1ecbccbad0029ed77eb2e630899553f1 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -33,7 +33,7 @@ from weboob.tools.capabilities.bank.transactions import sorted_transactions from .pages import ( - LogoutPage, InfosPage, AccountsPage, HistoryPage, LifeinsurancePage, MarketPage, + LogoutPage, AccountsPage, HistoryPage, LifeinsurancePage, MarketPage, AdvisorPage, LoginPage, ProfilePage, ) from .transfer_pages import TransferInfoPage, RecipientsListPage, TransferPage @@ -84,7 +84,6 @@ class CmsoParBrowser(LoginBrowser, StatesMixin): logout = URL('/securityapi/revoke', '/auth/errorauthn', '/\/auth/errorauthn', LogoutPage) - infos = URL('/comptes/', InfosPage) accounts = URL('/domiapi/oauth/json/accounts/synthese(?P.*)', AccountsPage) history = URL('/domiapi/oauth/json/accounts/(?P.*)', HistoryPage) loans = URL('/creditapi/rest/oauth/v1/synthese', AccountsPage) @@ -171,8 +170,7 @@ def iter_accounts(self): accounts_eligibilite_debit = self.page.get_eligibilite_debit() # First get all checking accounts... - data = dict(self.infos.stay_or_go().get_typelist()) - self.accounts.go(data=json.dumps(data), type='comptes', headers=self.json_headers) + self.accounts.go(json={'typeListeCompte': 'COMPTE_SOLDE_COMPTES_CHEQUES'}, type='comptes') self.page.check_response() for key in self.page.get_keys(): for a in self.page.iter_accounts(key=key): diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index 6bf100715fdb4db73ffd3fd88d9fd4e5e7834759..43b24bfc053b820cbfe920e6c862ade96ccd78f3 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -55,15 +55,6 @@ class LogoutPage(RawPage): pass -class InfosPage(LoggedPage, HTMLPage): - def get_typelist(self): - url = Attr(None, 'src').filter(self.doc.xpath('//script[contains(@src, "comptes/scripts")]')) - m = re.findall(r'synthesecomptes[^\w]+([^:]+)[^\w]+([^"]+)', self.browser.open(url).text) - for data in m: - if data[0] != 'method': - return {data[0]: data[1]} - - class AccountsPage(LoggedPage, JsonPage): TYPES = OrderedDict([('courant', Account.TYPE_CHECKING), ('pee', Account.TYPE_PEE), diff --git a/modules/cmso/pro/pages.py b/modules/cmso/pro/pages.py index 13e18270c6c9d6120193ca7dbc1c011c675b7448..4d41aac7d8edd7dc9177b70cb12b45e8386b4651 100644 --- a/modules/cmso/pro/pages.py +++ b/modules/cmso/pro/pages.py @@ -29,7 +29,7 @@ from .compat.weboob_capabilities_bank import Account, Investment from weboob.capabilities.base import NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction -from weboob.tools.compat import urljoin +from weboob.tools.compat import urljoin, parse_qsl from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -284,9 +284,13 @@ def get_sso_url(self): class TokenPage(CMSOPage, UpdateTokenMixin): def on_load(self): - d = re.search(r'id_token=(?P[^&]+)&access_token=(?P[^&]+)', self.text).groupdict() - self.browser.token = d['id_token'] - self.browser.csrf = d['access_token'] + auth_query_params = re.search(r'parent\.location = ".*#(.*)";', self.text) + assert auth_query_params, 'Url query parameter with token for authentication was not found' + auth_query_params = auth_query_params.group(1) + + params = dict(parse_qsl(auth_query_params)) + self.browser.token = params['id_token'] + self.browser.csrf = params['access_token'] class AuthCheckUser(HTMLPage): diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 45a3ab700668bc61f59b0d001e7a6f608ded48bb..b070d97b2b9357a4bcbdb02190a20d2051b8ae0b 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -139,6 +139,7 @@ class item_account_generic(ItemElement): ('C/C', Account.TYPE_CHECKING), ('Start', Account.TYPE_CHECKING), ('Comptes courants', Account.TYPE_CHECKING), + ('Service Accueil', Account.TYPE_CHECKING), ('Catip', Account.TYPE_DEPOSIT), ('Cic Immo', Account.TYPE_LOAN), ('Credit', Account.TYPE_LOAN), diff --git a/modules/googletranslate/module.py b/modules/googletranslate/module.py index 952a202bb5891c423e93027dea9c65637fa4436e..367548d7b79ec27d84ee1f254a0226d9579e32bc 100644 --- a/modules/googletranslate/module.py +++ b/modules/googletranslate/module.py @@ -38,84 +38,84 @@ class GoogleTranslateModule(Module, CapTranslate): NAME = 'googletranslate' DESCRIPTION = u'Google translation web service' BROWSER = GoogleTranslateBrowser - GOOGLELANGUAGE = { - 'Arabic': 'ar', - 'Afrikaans': 'af', - 'Albanian': 'sq', - 'Armenian': 'hy', - 'Azerbaijani': 'az', - 'Basque': 'eu', - 'Belarusian': 'be', - 'Bengali': 'bn', - 'Bulgarian': 'bg', - 'Catalan': 'ca', - 'Chinese': 'zh-CN', - 'Croatian': 'hr', - 'Czech': 'cs', - 'Danish': 'da', - 'Dutch': 'nl', - 'English': 'en', - 'Esperanto': 'eo', - 'Estonian': 'et', - 'Filipino': 'tl', - 'Finnish': 'fi', - 'French': 'fr', - 'Galician': 'gl', - 'Georgian': 'ka', - 'German': 'de', - 'Greek': 'el', - 'Gujarati': 'gu', - 'Haitian': 'ht', - 'Hebrew': 'iw', - 'Hindi': 'hi', - 'Hungaric': 'hu', - 'Icelandic': 'is', - 'Indonesian': 'id', - 'Irish': 'ga', - 'Italian': 'it', - 'Japanese': 'ja', - 'Kannada': 'kn', - 'Korean': 'ko', - 'Latin': 'la', - 'Latvian': 'lv', - 'Lithuanian': 'lt', - 'Macedonian': 'mk', - 'Malay': 'ms', - 'Maltese': 'mt', - 'Norwegian': 'no', - 'Persian': 'fa', - 'Polish': 'pl', - 'Portuguese': 'pt', - 'Romanian': 'ro', - 'Russian': 'ru', - 'Serbian': 'sr', - 'Slovak': 'sk', - 'Slovenian': 'sl', - 'Spanish': 'es', - 'Swahili': 'sw', - 'Swedish': 'sv', - 'Tamil': 'ta', - 'Telugu': 'te', - 'Thai': 'th', - 'Turkish': 'tr', - 'Ukrainian': 'uk', - 'Urdu': 'ur', - 'Vietnamese': 'vi', - 'Welsh': 'cy', - 'Yiddish': 'yi', - } + GOOGLELANGUAGE = [ + 'ar', + 'af', + 'sq', + 'hy', + 'az', + 'eu', + 'be', + 'bn', + 'bg', + 'ca', + 'zh-CN', + 'hr', + 'cs', + 'da', + 'nl', + 'en', + 'eo', + 'et', + 'tl', + 'fi', + 'fr', + 'gl', + 'ka', + 'de', + 'el', + 'gu', + 'ht', + 'iw', + 'hi', + 'hu', + 'is', + 'id', + 'ga', + 'it', + 'ja', + 'kn', + 'ko', + 'la', + 'lv', + 'lt', + 'mk', + 'ms', + 'mt', + 'no', + 'fa', + 'pl', + 'pt', + 'ro', + 'ru', + 'sr', + 'sk', + 'sl', + 'es', + 'sw', + 'sv', + 'ta', + 'te', + 'th', + 'tr', + 'uk', + 'ur', + 'vi', + 'cy', + 'yi', + ] def translate(self, lan_from, lan_to, text): - if lan_from not in self.GOOGLELANGUAGE.keys(): + if lan_from not in self.GOOGLELANGUAGE: raise LanguageNotSupported() - if lan_to not in self.GOOGLELANGUAGE.keys(): + if lan_to not in self.GOOGLELANGUAGE: raise LanguageNotSupported() translation = Translation(0) - translation.lang_src = self.GOOGLELANGUAGE[lan_from] - translation.lang_dst = self.GOOGLELANGUAGE[lan_to] - translation.text = self.browser.translate(self.GOOGLELANGUAGE[lan_from], self.GOOGLELANGUAGE[lan_to], text) + translation.lang_src = lan_from + translation.lang_dst = lan_to + translation.text = self.browser.translate(lan_from, lan_to, text) if empty(translation.text): raise TranslationFail() diff --git a/modules/larousse/browser.py b/modules/larousse/browser.py index ec7fc5d59bc52cb3424e2c58f4391f071a880eb5..5feae87b927fa8a6a1db8cd308c1a66db80a321a 100644 --- a/modules/larousse/browser.py +++ b/modules/larousse/browser.py @@ -25,13 +25,10 @@ class LarousseBrowser(PagesBrowser): - BASEURL = 'http://www.larousse.fr' + BASEURL = 'https://www.larousse.fr' langlist = URL('/dictionnaires/bilingues$', LangList) - # warning: the order of params is important... - word = URL(r'/dictionnaires/rechercher\?q=(?P.*)&l=(?P\w+)-(?P\w+)&culture=', - r'/dictionnaires/(?P\w+)-(?P\w+)/(?P[^/]+)(?:/(?P\d+))?', - WordPage) + word = URL(r'/dictionnaires/(?P\w+)-(?P\w+)/(?P.*)', WordPage) LANGS = None diff --git a/modules/larousse/pages.py b/modules/larousse/pages.py index fbf47ba683b99d478c13c71c17fe467f89672b80..0fe11a2c28068a9028271ba73d5cd9b791bb04b3 100644 --- a/modules/larousse/pages.py +++ b/modules/larousse/pages.py @@ -27,13 +27,13 @@ from weboob.browser.pages import HTMLPage CODES = { - 'allemand': 'German', - 'anglais': 'English', - 'arabe': 'Arabic', - 'chinois': 'Chinese', - 'espagnol': 'Spanish', - 'francais': 'French', - 'italien': 'Italian', + 'allemand': 'de', + 'anglais': 'en', + 'arabe': 'ar', + 'chinois': 'zh', + 'espagnol': 'es', + 'francais': 'fr', + 'italien': 'it', } RCODES = {v: k for k, v in CODES.items()} @@ -41,14 +41,12 @@ class LangList(HTMLPage): def get_langs(self): res = {} - for a in self.doc.xpath('//ul[@class="menu-items"]/li//a'): + for a in self.doc.xpath('//a[@class="item-dico-bil"]'): url = a.attrib['href'] mtc = re.search(r'/dictionnaires/(\w+)-(\w+)', url) if not mtc: continue src, dst = mtc.groups() - if dst == 'monolingue': - continue res[CODES[src], CODES[dst]] = (src, dst) return res diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 9e4051579b1ab436897437c49d369607f552ad2d..ace73dc527e66f51e71a9ba5fdabb85beba88a87 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -145,7 +145,7 @@ def execute_transfer(self, transfer, **params): return self.browser.execute_transfer(transfer) def transfer_check_label(self, old, new): - old = re.sub(r"[/<\?='!\+:]", '', old).strip() + old = re.sub(r"[/<\?='!\+:#]", '', old).strip() old = old.encode('latin-1', errors='replace').decode('latin-1') # if no reason given, the site changes the label if not old and ("INTERNET-FAVEUR" in new): diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index 544124d61280d048e8f25f3cbf1b906f0f4eac2f..7744660ee318032fbaae722e2b8d8331d2ce4afb 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -191,7 +191,8 @@ class AMFAmundiPage(HTMLPage): CODE_TYPE = Investment.CODE_TYPE_AMF def get_code(self): - return Regexp(CleanText('//td[@class="bannerColumn"]//li[contains(., "(C)")]', default=NotAvailable), r'(\d+)')(self.doc) + return Regexp(CleanText('//td[@class="bannerColumn"]//li[contains(., "(C)")]', default=NotAvailable), + r'(\d+)', default=NotAvailable)(self.doc) class AMFSGPage(HTMLPage): @@ -384,6 +385,8 @@ def condition(self): obj_label = Env('label') def obj_type(self): + if Field('label')(self).startswith('ETOILE'): + return self.page.TYPES.get(Field('label')(self).split()[1].upper(), Account.TYPE_UNKNOWN) return self.page.TYPES.get(Field('label')(self).split()[0].upper(), Account.TYPE_UNKNOWN) def obj_balance(self): diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index 6a07e3d4a8e0ac8637e0fe1fc45db62fb27fe3bb..fae35d223db5e1c09117c36d449be76fe06ecc77 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -26,7 +26,7 @@ from weboob.browser.url import URL from weboob.browser.exceptions import ClientError from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, NoAccountsException -from weboob.capabilities.base import find_object +from weboob.capabilities.base import find_object, NotAvailable from .compat.weboob_capabilities_bank import ( AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientBankError, Recipient, TransferBankError, AccountOwnerType, @@ -36,7 +36,7 @@ from .pages import ( LoginPage, CardsPage, CardHistoryPage, IncorrectLoginPage, ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage, InscriptionPage, - ErrorPage, UselessPage, + ErrorPage, UselessPage, MainPage, MarketAccountPage, MarketInvestmentPage, ) from .json_pages import ( AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage, @@ -107,6 +107,10 @@ def card_history(self, account, coming): @need_login def get_cb_operations(self, account): + if account.type in (account.TYPE_MARKET, ): + # market account transactions are in checking account + return + self.location('/Pgn/NavigationServlet?PageID=Cartes&MenuID=%sOPF&Classeur=1&NumeroPage=1&Rib=%s&Devise=%s' % (self.MENUID, account.id, account.currency)) if self.inscription_page.is_here(): @@ -132,6 +136,8 @@ class SGEnterpriseBrowser(SGPEBrowser): MENUID = 'BANREL' CERTHASH = '2231d5ddb97d2950d5e6fc4d986c23be4cd231c31ad530942343a8fdcc44bb99' + main_page = URL('/icd-web/syd-front/index-comptes.html', MainPage) + accounts = URL('/icd/syd-front/data/syd-comptes-accederDepuisMenu.json', AccountsJsonPage) intraday_accounts = URL('/icd/syd-front/data/syd-intraday-accederDepuisMenu.json', AccountsJsonPage) @@ -142,6 +148,13 @@ class SGEnterpriseBrowser(SGPEBrowser): '/icd/syd-front/data/syd-intraday-chargerDetail.json', HistoryJsonPage) history_next = URL('/icd/syd-front/data/syd-comptes-chargerProchainLotEcriture.json', HistoryJsonPage) + market_investment = URL(r'/Pgn/NavigationServlet\?.*PageID=CompteTitreDetailFrame', + r'/Pgn/NavigationServlet\?.*PageID=CompteTitreDetail', + MarketInvestmentPage) + market_accounts = URL(r'/Pgn/NavigationServlet\?.*PageID=CompteTitreFrame', + r'/Pgn/NavigationServlet\?.*PageID=CompteTitre', + MarketAccountPage) + profile = URL('/gae/afficherModificationMesDonnees.html', ProfileEntPage) subscription = URL(r'/Pgn/NavigationServlet\?MenuID=BANRELRIE&PageID=ReleveRIE&NumeroPage=1&Origine=Menu', SubscriptionPage) @@ -177,14 +190,37 @@ def get_accounts_list(self): acc.owner_type = AccountOwnerType.ORGANIZATION yield acc + # retrieve market accounts if exist + for market_account in self.iter_market_accounts(): + yield market_account + @need_login def iter_history(self, account): + if account.type in (account.TYPE_MARKET, ): + # market account transactions are in checking account + return + value = self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': 'valeur'}).get_value() for tr in self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': value}).iter_history(value=value): yield tr for tr in self.location('/icd/syd-front/data/syd-intraday-chargerDetail.json', data={'cl500_compte': account._id}).page.iter_history(): yield tr + @need_login + def iter_market_accounts(self): + self.main_page.go() + # retrieve market accounts if exist + market_accounts_link = self.page.get_market_accounts_link() + + # there are no examples of entreprise space with market accounts yet + assert not market_accounts_link, 'There are market accounts, retrieve them.' + return [] + + @need_login + def iter_investment(self, account): + # there are no examples of entreprise space with market accounts yet + return [] + @need_login def iter_subscription(self): subscriber = self.get_profile() @@ -257,6 +293,39 @@ def load_state(self, state): self.need_reload_state = None super(SGProfessionalBrowser, self).load_state(state) + @need_login + def iter_market_accounts(self): + self.main_page.go() + # retrieve market accounts if exist + market_accounts_link = self.page.get_market_accounts_link() + if market_accounts_link is NotAvailable: + return [] + assert market_accounts_link, 'Market accounts link xpath may have changed' + + # need to be on market accounts page to get the accounts iframe + self.location(market_accounts_link) + market_accounts_list_link = self.page.get_table_iframe_link() + if market_accounts_list_link is NotAvailable: + return [] + assert market_accounts_link, 'Market accounts iframe link xpath may have changed' + + self.location(market_accounts_list_link) + return self.page.iter_market_accounts() + + @need_login + def iter_investment(self, account): + if account.type not in (account.TYPE_MARKET, ): + return [] + + assert account._url_data, 'This account has no url to retrieve investments' + # need to be on market accounts investment page to get the invetment iframe + self.location('/Pgn/NavigationServlet?%s' % account._url_data) + + invests_list_link = self.page.get_table_iframe_link() + assert invests_list_link, 'It seems that this market account has no investment' + self.location(invests_list_link) + return self.page.iter_investment() + def copy_recipient_obj(self, recipient): rcpt = Recipient() rcpt.id = recipient.iban diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index 86ef8d841dedc83669335fd4c3d178e3119536b1..da47fbd18433113b702f33403f7065b169291517 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -24,15 +24,17 @@ from io import BytesIO from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.elements import ListElement, ItemElement, method +from weboob.browser.elements import ListElement, ItemElement, method, TableElement from .compat.weboob_browser_filters_standard import ( CleanText, CleanDecimal, Date, - Env, Regexp, Field, Format, + Env, Regexp, Field, Format, TableCell, ) -from weboob.browser.filters.html import Attr +from weboob.browser.filters.html import Attr, Link +from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.profile import Profile, Person from weboob.capabilities.bill import Document, Subscription +from .compat.weboob_capabilities_bank import Account, Investment from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.json import json @@ -269,3 +271,83 @@ def get_error(self): class UselessPage(LoggedPage, SGPEPage): pass + + +class MainPage(LoggedPage, SGPEPage): + def get_market_accounts_link(self): + market_accounts_link = Link('//li/a[@title="Comptes titres"]', default=None)(self.doc) + + if market_accounts_link: + return market_accounts_link + elif self.doc.xpath('//span[contains(text(), "Comptes titres") and contains(@title, "pas habilité à utiliser ce service")]'): + return NotAvailable + # return None when we don't know if there are market accounts or not + # it will be handled in `browser.py` + + +class MarketAccountPage(LoggedPage, SGPEPage): + def get_table_iframe_link(self): + if self.doc.xpath('//div[contains(text(), "Aucun compte-titres")]'): + return NotAvailable + return Attr('//iframe[@id="frameTableau"]', 'src')(self.doc) + + @method + class iter_market_accounts(TableElement): + item_xpath = '//table[@id="tab-corps"]//tr' + head_xpath = '//table[@id="tab-entete"]//td' + + col_id = 'COMPTE' + col_label = 'INTITULE' + col_balance = 'EVALUATION' + + class item(ItemElement): + def condition(self): + # table with empty row filled by empty `td` + return Field('number')(self) + + klass = Account + + obj_id = Format('%s_TITRE', CleanText(TableCell('id'), replace=[(' ', '')])) + obj_number = CleanText(TableCell('id'), replace=[(' ', '')]) + obj_label = CleanText(TableCell('label')) + obj_balance = CleanDecimal.French(CleanText(TableCell('balance'))) + obj_type = Account.TYPE_MARKET + + # all `a` balises have same `href` + obj__url_data = Regexp(Link('(.//a)[1]'), r"lienParent\('(.*)'\)", default=NotAvailable) + + +class MarketInvestmentPage(LoggedPage, SGPEPage): + def get_table_iframe_link(self): + return Attr('//iframe[@id="frameTableau"]', 'src')(self.doc) + + @method + class iter_investment(TableElement): + item_xpath = '//table[@id="tab-corps"]//tr' + head_xpath = '//table[@id="tab-entete"]//td' + + col_code = 'CODE' + col_label = 'VALEUR' + col_valuation = 'MONTANT' + col_quantity = 'QUANTITE' + col_unitvalue = 'COURS' + + class item(ItemElement): + def condition(self): + # table with empty row filled by empty `td` + return Field('valuation')(self) + + klass = Investment + + obj_code_type = Investment.CODE_TYPE_ISIN + obj_label = CleanText(TableCell('label')) + obj_valuation = CleanDecimal.French(CleanText(TableCell('valuation'))) + obj_quantity = CleanDecimal.French(CleanText(TableCell('quantity'))) + obj_unitvalue = CleanDecimal.French(CleanText(TableCell('unitvalue'))) + + def obj_code(self): + code = CleanText(TableCell('code'))(self) + # there is no example of invests without valid ISIN code + # wait for it to retrieve them corretly + assert is_isin_valid(code), 'This code is not a valid ISIN, please check what invest is it.' + return code diff --git a/modules/spirica/browser.py b/modules/spirica/browser.py index 351c1d53e22c30f44c8a787375e364eaa8b696d1..cdd06a6e3649266fe83045081c6960c9e1c20883 100644 --- a/modules/spirica/browser.py +++ b/modules/spirica/browser.py @@ -21,7 +21,6 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword -from weboob.browser.exceptions import ClientError, ServerError from .pages import LoginPage, AccountsPage, DetailsPage, MaintenancePage @@ -76,28 +75,6 @@ def check_if_logged_in(self, url): # Store new transaction_page after login: self.transaction_page = self.page - @need_login - def get_transactions_with_investments(self, max_count, url): - transactions = [] - for index, transaction in enumerate(self.page.iter_history()): - self.check_if_logged_in(url) - if index < max_count: - try: - self.transaction_page.go_investments_form(transaction._index) - except (ClientError, ServerError) as e: - self.logger.warning(e) - # Check if we are logged out - if self.login.is_here(): - self.check_if_logged_in(url) - if self.details.is_here(): - transaction.investments = [] - for inv in self.page.iter_transactions_investments(): - # Only keep investments that have at least a label and a valuation: - if inv.label and inv.valuation: - transaction.investments.append(inv) - transactions.append(transaction) - return transactions - @need_login def iter_history(self, account): self.location(account.url) @@ -106,23 +83,7 @@ def iter_history(self, account): # Determining the number of transaction pages: total_pages = int(self.page.count_transactions()) // 100 - - # Scraping transactions with their investments for the 20 first transactions. - # Sometimes go_historyall fails so we go back to the accounts page and retry. - if self.transaction_page.go_historyall(page_number=0): - for tr in self.get_transactions_with_investments(20, account.url): - yield tr - else: - self.logger.warning('The first go_historyall() failed, go back to account details and retry.') - self.location(account.url) - self.page.go_historytab() - self.transaction_page = self.page - if self.transaction_page.go_historyall(page_number=0): - for tr in self.get_transactions_with_investments(20, account.url): - yield tr - - # Scraping other transaction pages without their investments: - for page_number in range(1, total_pages + 1): + for page_number in range(total_pages + 1): self.check_if_logged_in(account.url) if not self.transaction_page.go_historyall(page_number): self.logger.warning('The first go_historyall() failed, go back to account details and retry.') diff --git a/modules/spirica/pages.py b/modules/spirica/pages.py index 9e38476f714591bb1fe5989d8ec6a04f08b524ca..79efc7e2708cceb09bf36d27f3af5f37c3dd4641 100644 --- a/modules/spirica/pages.py +++ b/modules/spirica/pages.py @@ -44,6 +44,11 @@ def on_load(self): class LoginPage(HTMLPage): + def on_load(self): + error_msg = CleanText('//li[@class="globalErreurMessage"]')(self.doc) + if error_msg: + raise BrowserUnavailable(error_msg) + def login(self, login, password): form = self.get_form('//form[@id="loginForm"]') form['loginForm:name'] = login @@ -186,7 +191,6 @@ def obj_portfolio_share(self): profile_share = MyDecimal(path)(self) assert profile_share - #raise Exception('dtc') profile_share = Eval(lambda x: x / 100, profile_share)(self) return inv_share * profile_share else: @@ -244,20 +248,6 @@ def go_historyall(self, page_number): form.submit() return True - def go_investments_form(self, index): - form = self.get_form(xpath='//form[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")]') - form['javax.faces.behavior.event'] = 'rowToggle' - form['javax.faces.partial.event'] = 'rowToggle' - id_ = Attr('//div[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")][has-class("listeAvecDetail")]', 'id')(self.doc) - form['javax.faces.source'] = id_ - form['javax.faces.partial.execute'] = id_ - form['javax.faces.partial.render'] = id_ + ':detail ' + id_ - form[id_ + '_rowExpansion'] = 'true' - form[id_ + '_encodeFeature'] = 'true' - form[id_ + '_expandedRowIndex'] = index - form.submit() - - @method class iter_history(ListElement): item_xpath = '//tr[@role="row"]' @@ -287,31 +277,3 @@ def condition(self): and "Arrêté annuel" not in Field('label')(self) and "Fusion-absorption" not in Field('label')(self) ) - - @method - class iter_transactions_investments(TableInvestment): - item_xpath = '//table[thead[.//span[text()="ISIN"]]]/tbody/tr' - head_xpath = '//thead[.//span[text()="ISIN"]]//th' - - col_isin = 'ISIN' - col_valuation = 'Montant net' - col_portfolio_share = '%' - - class item(ItemElement): - klass = Investment - - # Columns do not always appear depending on transactions so we need - # to precise "default=NotAvailable" for all TableCell filters. - obj_label = CleanText(TableCell('label', default=NotAvailable), default=NotAvailable) - obj_vdate = Date(CleanText(TableCell('vdate', default="")), dayfirst=True, default=NotAvailable) - obj_unitvalue = MyDecimal(TableCell('unitvalue', default=NotAvailable), default=NotAvailable) - obj_quantity = MyDecimal(TableCell('quantity', default=NotAvailable), default=NotAvailable) - obj_valuation = MyDecimal(TableCell('valuation', default=NotAvailable), default=NotAvailable) - obj_portfolio_share = MyDecimal(TableCell('portfolio_share', default=NotAvailable), default=NotAvailable) - - def obj_code(self): - code = CleanText(TableCell('isin', default=NotAvailable), default=NotAvailable)(self) - return code if code != '-' else NotAvailable - - def obj_code_type(self): - return Investment.CODE_TYPE_ISIN if Field('code')(self) else NotAvailable diff --git a/modules/wordreference/module.py b/modules/wordreference/module.py index 05a48347810e18b4c7182ad0b015f78d754b451f..645de93bea0c6855a534097f61c78dccf082c81d 100644 --- a/modules/wordreference/module.py +++ b/modules/wordreference/module.py @@ -36,20 +36,31 @@ class WordReferenceModule(Module, CapTranslate): NAME = 'wordreference' DESCRIPTION = u'Free online translator' BROWSER = WordReferenceBrowser - WRLANGUAGE = { - 'Arabic': 'ar', 'Chinese': 'zh', 'Czech': 'cz', 'English': 'en', 'French': 'fr', 'Greek': 'gr', - 'Italian': 'it', 'Japanese': 'ja', 'Korean': 'ko', 'Polish': 'pl', 'Portuguese': 'pt', - 'Romanian': 'ro', 'Spanish': 'es', 'Turkish': 'tr', - } + WRLANGUAGE = [ + 'ar', + 'zh', + 'cz', + 'en', + 'fr', + 'gr', + 'it', + 'ja', + 'ko', + 'pl', + 'pt', + 'ro', + 'es', + 'tr', + ] def translate(self, lan_from, lan_to, text): - if lan_from not in self.WRLANGUAGE.keys(): + if lan_from not in self.WRLANGUAGE: raise LanguageNotSupported() - if lan_to not in self.WRLANGUAGE.keys(): + if lan_to not in self.WRLANGUAGE: raise LanguageNotSupported() - translations = self.browser.translate(self.WRLANGUAGE[lan_from], self.WRLANGUAGE[lan_to], text) + translations = self.browser.translate(lan_from, lan_to, text) has_translation = False for translation in translations: