diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index c91e4691b11bba3a3b6bf985ba3c6df60a64c67e..73ebe78e8ab56aecf9f735f23b9912f1b7737531 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -129,6 +129,7 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): self.config = config self.auth_token = None self.accounts_list = None + self.cards_list = None self.deferred_card_calendar = None kwargs['username'] = self.config['login'].get() kwargs['password'] = self.config['password'].get() @@ -245,13 +246,13 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): self.accounts_list.remove(account) self.accounts_list.extend(self.loans_list) - cards = [acc for acc in self.accounts_list if acc.type == Account.TYPE_CARD] - if cards: - self.go_cards_number(cards[0].url) + self.cards_list = [acc for acc in self.accounts_list if acc.type == Account.TYPE_CARD] + if self.cards_list: + self.go_cards_number(self.cards_list[0].url) if self.cards.is_here(): - self.page.populate_cards_number(cards) + self.page.populate_cards_number(self.cards_list) # Cards without a number are not activated yet: - for card in cards: + for card in self.cards_list: if not card.number: self.accounts_list.remove(card) @@ -259,7 +260,7 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): if account.type not in (Account.TYPE_CARD, Account.TYPE_LOAN, Account.TYPE_CONSUMER_CREDIT, Account.TYPE_MORTGAGE, Account.TYPE_REVOLVING_CREDIT, Account.TYPE_LIFE_INSURANCE): account.iban = self.iban.go(webid=account._webid).get_iban() - for card in cards: + for card in self.cards_list: checking, = [account for account in self.accounts_list if account.type == Account.TYPE_CHECKING and account.url in card.url] card.parent = checking @@ -281,6 +282,33 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): if i[0] < debit_date <= j[0]: return j[1] + @retry_on_logout() + @need_login + def get_history(self, account, coming=False): + if account.type in (Account.TYPE_LOAN, Account.TYPE_CONSUMER_CREDIT) or '/compte/derive' in account.url: + return [] + if account.type is Account.TYPE_SAVINGS and u"PLAN D'ÉPARGNE POPULAIRE" in account.label: + return [] + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET): + return self.get_invest_transactions(account, coming) + elif account.type == Account.TYPE_CARD: + return self.get_card_transactions(account, coming) + return self.get_regular_transactions(account, coming) + + def get_regular_transactions(self, account, coming): + # We look for 3 years of history. + params = {} + params['movementSearch[toDate]'] = (date.today() + relativedelta(days=40)).strftime('%d/%m/%Y') + params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=3)).strftime('%d/%m/%Y') + params['movementSearch[selectedAccounts][]'] = account._webid + self.location('%s/mouvements' % account.url.rstrip('/'), params=params) + for t in self.page.iter_history(): + yield t + if coming and account.type == Account.TYPE_CHECKING: + self.location('%s/mouvements-a-venir' % account.url.rstrip('/'), params=params) + for t in self.page.iter_history(coming=True): + yield t + def get_card_transactions(self, account, coming): # All card transactions can be found in the CSV (history and coming), # however the CSV shows a maximum of 1000 transactions from all accounts. @@ -289,7 +317,6 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): # for some cards, the site redirects us to '/'... return - self.location(account.url) if self.deferred_card_calendar is None: self.location(self.page.get_calendar_link()) params = {} @@ -321,33 +348,6 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): for t in sorted(transactions, key=lambda tr: tr.date, reverse=True): yield t - def get_regular_transactions(self, account, coming): - # We look for 3 years of history. - params = {} - params['movementSearch[toDate]'] = (date.today() + relativedelta(days=40)).strftime('%d/%m/%Y') - params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=3)).strftime('%d/%m/%Y') - params['movementSearch[selectedAccounts][]'] = account._webid - self.location('%s/mouvements' % account.url.rstrip('/'), params=params) - for t in self.page.iter_history(): - yield t - if coming and account.type == Account.TYPE_CHECKING: - self.location('%s/mouvements-a-venir' % account.url.rstrip('/'), params=params) - for t in self.page.iter_history(coming=True): - yield t - - @retry_on_logout() - @need_login - def get_history(self, account, coming=False): - if account.type in (Account.TYPE_LOAN, Account.TYPE_CONSUMER_CREDIT) or '/compte/derive' in account.url: - return [] - if account.type is Account.TYPE_SAVINGS and u"PLAN D'\xc9PARGNE POPULAIRE" in account.label: - return [] - if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET): - return self.get_invest_transactions(account, coming) - elif account.type == Account.TYPE_CARD: - return self.get_card_transactions(account, coming) - return self.get_regular_transactions(account, coming) - @need_login def get_investment(self, account): if '/compte/derive' in account.url: diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index a52ce3e8f6652ad8b7efe4daf124a6b71fe7c82b..6f32f5b275bf88950c25364182cd60c5c82cb18c 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -113,17 +113,20 @@ class Transaction(FrenchTransaction): (re.compile(r'^(?P.+)?(ACHAT|PAIEMENT) CARTE (?P
\d{2})(?P\d{2})(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_CARD), (re.compile(r'^(?P.+)?((ACHAT|PAIEMENT)\s)?CARTE (?P
\d{2})(?P\d{2})(?P\d{4}) (?P.*)'), - FrenchTransaction.TYPE_DEFERRED_CARD), + FrenchTransaction.TYPE_CARD), (re.compile('^(PRLV SEPA |PRLV |TIP )(?P.*)'), FrenchTransaction.TYPE_ORDER), (re.compile('^RETRAIT DAB (?P
\d{2})(?P\d{2})(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(r'^([A-Z][\sa-z]* )?RETRAIT DAB (?P
\d{2})(?P\d{2})(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^([A-Z][\sa-z]* )?Retrait dab (?P
\d{2})(?P\d{2})(?P\d{4}) (?P.*)'), + FrenchTransaction.TYPE_WITHDRAWAL), (re.compile('^AVOIR (?P
\d{2})(?P\d{2})(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_PAYBACK), (re.compile(r'^(?P[A-Z][\sa-z]* )?AVOIR (?P
\d{2})(?P\d{2})(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_PAYBACK), (re.compile('^REM CHQ (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), (re.compile(u'^([*]{3} solde des operations cb [*]{3} )?Relevé différé Carte (.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^Ech pret'), FrenchTransaction.TYPE_LOAN_PAYMENT), ] @@ -417,18 +420,27 @@ class HistoryPage(LoggedPage, HTMLPage): obj_raw = Transaction.Raw(CleanText('.//div[has-class("list__movement__line--label__name")]')) obj_date = Date(Attr('.//time', 'datetime')) obj_amount = CleanDecimal('.//div[has-class("list__movement__line--amount")]', replace_dots=True) - obj_category = CleanText('.//div[has-class("category")]') + obj_category = CleanText('.//span[has-class("category")]') + obj__account_name = CleanText('.//span[contains(@class, "account__name-xs")]', default=None) def obj_id(self): return Attr('.', 'data-id', default=NotAvailable)(self) or Attr('.', 'data-custom-id', default=NotAvailable)(self) def obj_type(self): + # In order to set TYPE_DEFERRED_CARD transactions correctly, + # we must check if the transaction's account_name is in the list + # of deferred cards, but summary transactions must escape this rule. + if self.obj.type == Transaction.TYPE_CARD_SUMMARY: + return self.obj.type + deferred_card_labels = [card.label for card in self.page.browser.cards_list] + if 'cartes débit différé' in Field('category')(self) or Field('_account_name')(self).upper() in deferred_card_labels: + return Transaction.TYPE_DEFERRED_CARD if not Env('is_card', default=False)(self): if Env('coming', default=False)(self) and Field('raw')(self).startswith('CARTE '): return Transaction.TYPE_CARD_SUMMARY # keep the value previously set by Transaction.Raw return self.obj.type - return Transaction.TYPE_DEFERRED_CARD + return Transaction.TYPE_UNKNOWN def obj_rdate(self): if self.obj.rdate: @@ -459,9 +471,14 @@ class HistoryPage(LoggedPage, HTMLPage): # TYPE_DEFERRED_CARD transactions are already present in the card history # so we only return TYPE_DEFERRED_CARD for the coming: if not Env('coming', default=False)(self): - return not len(self.xpath(u'.//span[has-class("icon-carte-bancaire")]')) and \ - not len(self.xpath(u'.//a[contains(@href, "/carte")]')) \ + return not len(self.xpath(u'.//span[has-class("icon-carte-bancaire")]')) \ + and not len(self.xpath(u'.//a[contains(@href, "/carte")]')) \ and obj.type != Transaction.TYPE_DEFERRED_CARD + elif Env('coming', default=False)(self): + # Do not return coming from deferred cards if their + # summary does not have a fixed amount yet: + if obj.type == Transaction.TYPE_CARD_SUMMARY: + return False return True def get_cards_number_link(self): @@ -529,8 +546,8 @@ class CardHistoryPage(LoggedPage, CsvPage): class Myiter_investment(TableElement): # We do not scrape the investments contained in the "Engagements en liquidation" table # so we must check that the

before the
does not contain this title. - item_xpath = '//div[preceding-sibling::h3[text()!="Engagements en liquidation"]]//table[contains(@class, "operations")]/tbody/tr' - head_xpath = '//div[preceding-sibling::h3[text()!="Engagements en liquidation"]]//table[contains(@class, "operations")]/thead/tr/th' + item_xpath = '//div[preceding-sibling::h3[1][text()!="Engagements en liquidation"]]//table[contains(@class, "operations")]/tbody/tr' + head_xpath = '//div[preceding-sibling::h3[1][text()!="Engagements en liquidation"]]//table[contains(@class, "operations")]/thead/tr/th' col_value = u'Valeur' col_quantity = u'Quantité' @@ -622,7 +639,9 @@ class MarketPage(LoggedPage, HTMLPage): return CleanDecimal(replace_dots=True, default=NotAvailable).filter((TableCell('unitvalue')(self)[0]).xpath('./span[not(@class)]')) def iter_investment(self): - valuation = CleanDecimal('//li/span[contains(text(), "Solde Espèces")]/following-sibling::span', replace_dots=True, default=None)(self.doc) + # Xpath can be h3/h4 or div/span; in both cases + # the first node contains "Solde Espèces": + valuation = CleanDecimal('//li/*[contains(text(), "Solde Espèces")]/following-sibling::*', replace_dots=True, default=None)(self.doc) if valuation: yield create_french_liquidity(valuation) diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py index e42aee2dfced7f526c399d5935be3f88c69b8ad4..2c1c1b8f6ef0f2223ccd462cd38bc4dc96b80212 100644 --- a/modules/caissedepargne/cenet/browser.py +++ b/modules/caissedepargne/cenet/browser.py @@ -140,8 +140,12 @@ class CenetBrowser(LoginBrowser, StatesMixin): for account in self.accounts: try: - account._cards = [card for card in self.cenet_cards.go(data=json.dumps(data), headers=headers).get_cards() \ - if card['Compte']['Numero'] == account.id] + account._cards = [] + self.cenet_cards.go(data=json.dumps(data), headers=headers) + + for card in self.page.get_cards(): + if card['Compte']['Numero'] == account.id: + account._cards.append(card) except BrowserUnavailable: # for some accounts, the site can throw us an error, during weeks self.logger.warning('ignoring cards because site is unavailable...') @@ -218,15 +222,16 @@ class CenetBrowser(LoginBrowser, StatesMixin): } for card in account._cards: - data = { - 'contexte': '', - 'dateEntree': None, - 'donneesEntree': json.dumps(card), - 'filtreEntree': None - } - - for tr in self.cenet_account_coming.go(data=json.dumps(data), headers=headers).get_history(): - trs.append(tr) + if card['CumulEnCours']['Montant']['Valeur'] != 0: + data = { + 'contexte': '', + 'dateEntree': None, + 'donneesEntree': json.dumps(card), + 'filtreEntree': None + } + + for tr in self.cenet_account_coming.go(data=json.dumps(data), headers=headers).get_history(): + trs.append(tr) return sorted_transactions(trs) diff --git a/modules/caissedepargne/cenet/pages.py b/modules/caissedepargne/cenet/pages.py index 62bc5a7590864c12563637f45263a94da13708b0..038b5c37240cf81992cdf02aa191cc082a58eb1f 100644 --- a/modules/caissedepargne/cenet/pages.py +++ b/modules/caissedepargne/cenet/pages.py @@ -79,9 +79,9 @@ class CenetJsonPage(JsonPage): # Why you are so ugly.... self.doc = json.loads(self.doc['d']) - if self.doc['Erreur'] and self.doc['Erreur']['Titre']: - self.logger.warning('error on %r: %s', self.url, self.doc['Erreur']['Titre']) - raise BrowserUnavailable(self.doc['Erreur']['Titre']) + if self.doc['Erreur'] and (self.doc['Erreur']['Titre'] or self.doc['Erreur']['Code']): + self.logger.warning('error on %r: %s', self.url, self.doc['Erreur']['Titre'] or self.doc['Erreur']['Code']) + raise BrowserUnavailable(self.doc['Erreur']['Titre'] or self.doc['Erreur']['Description']) self.doc['DonneesSortie'] = json.loads(self.doc['DonneesSortie']) diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index f369d87d731aacf9597105588d40be354553a051..b864c231a585dcffb9fff95a783e5684d700ea4a 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -762,13 +762,11 @@ class NatixisRedirectPage(LoggedPage, HTMLPage): class MarketPage(LoggedPage, HTMLPage): - def is_error(self): - try: - return self.doc.xpath('//caption')[0].text == "Erreur" - except IndexError: - return False - except AssertionError: - return True + def on_load(self): + error = CleanText('//caption[contains(text(),"Erreur")]')(self.doc) + if error: + message = CleanText('//td[contains(@class,"donneeLongIdent")]')(self.doc) + raise BrowserUnavailable(message) def parse_decimal(self, td, percentage=False): value = CleanText('.')(td) diff --git a/modules/cragr/web/browser.py b/modules/cragr/web/browser.py index 89f9bad8aaf780e22af766b8ea23792d9399116a..a73de43f5c04b0886361794b3de37a97deed6933 100644 --- a/modules/cragr/web/browser.py +++ b/modules/cragr/web/browser.py @@ -450,7 +450,10 @@ class Cragr(LoginBrowser, StatesMixin): url = self.page.get_next_url() elif self.page and not self.no_fixed_deposit_page.is_here(): - date_guesser = LinearDateGuesser() + if account.type == Account.TYPE_SAVINGS: + date_guesser = LinearDateGuesser(date_max_bump=timedelta(2)) + else: + date_guesser = LinearDateGuesser() self.page.order_transactions() while True: assert self.transactions.is_here() diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index e1a321d98440b2c6b9e5a178af10b44bbd19cc6f..0a8d8b63922583fb4207d9a94dc4d76e88400376 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -19,6 +19,7 @@ from __future__ import unicode_literals +from dateutil.relativedelta import relativedelta from datetime import date as ddate, datetime from decimal import Decimal import re @@ -874,6 +875,12 @@ class TransactionsPage(MyLoggedPage, BasePage): t.date = MyDate().filter(date) t.rdate = t.date t.raw = raw + # Transactions DO NOT have a year. its only DD/MM. In some cases, if there is not enough + # transactions in the account, a transaction dating from last year has it date incorrectly + # guessed (with the date_guesser/ to avoid this we had to reduce de data_guesser.date_max_bump + # to 2 in order to prevenmt such problems, but only for savings accounts. It is not perfect, + # hence the assertion to make sure we aren't returning a future date that is more than 2 days forward. + assert datetime(year=t.date.year, month=t.date.month, day=t.date.day) + relativedelta(date_guesser.date_max_bump.days) < datetime.today() # On some accounts' history page, there is a tag in columns. if col_text.find('font') is not None: diff --git a/modules/creditdunord/browser.py b/modules/creditdunord/browser.py index f3d470bf5dcf6fa67af0e117f1d10a3e3915c24e..cde5c547bfc24264ae882c637f6333eadd3c6d4f 100644 --- a/modules/creditdunord/browser.py +++ b/modules/creditdunord/browser.py @@ -133,13 +133,14 @@ class CreditDuNordBrowser(LoginBrowser): return find_object(account_list, id=id) @need_login - def iter_transactions(self, link, args, acc_type): + def iter_transactions(self, account): + args = account._args if args is None: return while args is not None: - self.location(link, data=args) + self.location(account._link, data=args) assert (self.transactions.is_here() or self.protransactions.is_here()) - for tr in self.page.get_history(acc_type): + for tr in self.page.get_history(account): yield tr args = self.page.get_next_args(args) @@ -148,7 +149,7 @@ class CreditDuNordBrowser(LoginBrowser): def get_history(self, account, coming=False): if coming and account.type != Account.TYPE_CARD or account.type == Account.TYPE_LOAN: return - for tr in self.iter_transactions(account._link, account._args, account.type): + for tr in self.iter_transactions(account): yield tr @need_login diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index 8bcb03087843fa999e9115ba253a3e3ad8648e43..129e6d5cbad3e1ecfd080ce170081506e1ef7a8d 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -602,7 +602,7 @@ class TransactionsPage(LoggedPage, CDNBasePage): t.type = t.TYPE_DEFERRED_CARD return False - def get_history(self, acc_type): + def get_history(self, account): txt = self.get_from_js('ListeMvts_data = new Array(', ');\n') if txt is None: no_trans = self.get_from_js('js_noMvts = new Ext.Panel(', ')') @@ -618,7 +618,7 @@ class TransactionsPage(LoggedPage, CDNBasePage): for line in data: t = self.TRANSACTION() - if acc_type is Account.TYPE_CARD and MyStrip(line[self.COL_DEBIT_DATE]): + if account.type is Account.TYPE_CARD and MyStrip(line[self.COL_DEBIT_DATE]): date = vdate = Date(dayfirst=True).filter(MyStrip(line[self.COL_DEBIT_DATE])) else: date = Date(dayfirst=True, default=NotAvailable).filter(MyStrip(line[self.COL_DATE])) @@ -638,7 +638,7 @@ class TransactionsPage(LoggedPage, CDNBasePage): t.amount = -CleanDecimal(replace_dots=True).filter(m.group(1)) self.logger.info('parsing amount in transaction label: %r', t) - if self.condition(t, acc_type): + if self.condition(t, account.type): continue yield t @@ -720,6 +720,7 @@ class TransactionsPage(LoggedPage, CDNBasePage): class ProTransactionsPage(TransactionsPage): TRANSACTION = Transaction + def get_next_args(self, args): if len(self.doc.xpath('//a[contains(text(), "Suivant")]')) > 0: args['PageDemandee'] = int(args.get('PageDemandee', 1)) + 1 @@ -742,10 +743,12 @@ class ProTransactionsPage(TransactionsPage): return sorted(transactions.items()) - def detect_currency(self, t, raw): + # We don't want detect the account_devise as an original_currency, since it's + # already the main currency + def detect_currency(self, t, raw, account_devise): matches = [] for currency in Currency.CURRENCIES: - if ' ' + currency + ' ' in raw: + if currency != account_devise and ' ' + currency + ' ' in raw: m = re.search(r'(\d+[,.]\d{1,2}? ' + currency + r')', raw) if m: matches.append((m, currency)) @@ -758,11 +761,11 @@ class ProTransactionsPage(TransactionsPage): if (t.amount < 0): t.original_amount = -t.original_amount - def get_history(self, acc_type): + def get_history(self, account): for i, tr in self.parse_transactions(): t = self.TRANSACTION() - if acc_type is Account.TYPE_CARD: + if account.type is Account.TYPE_CARD: date = vdate = Date(dayfirst=True, default=None).filter(tr['dateval']) else: date = Date(dayfirst=True, default=None).filter(tr['date']) @@ -770,9 +773,9 @@ class ProTransactionsPage(TransactionsPage): raw = MyStrip(' '.join([tr['typeope'], tr['LibComp']])) t.parse(date, raw, vdate) t.set_amount(tr['mont']) - self.detect_currency(t, raw) + self.detect_currency(t, raw, account.currency) - if self.condition(t, acc_type): + if self.condition(t, account.type): continue yield t diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 45233dc6ef9dd83f0cf8fb2a8285c33380509c8f..8b7ba663f1bee2a0f6354182848a1136ac49b975 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -188,6 +188,7 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): self.unavailablecards = [] self.cards_histo_available = [] self.cards_list =[] + self.cards_list2 =[] # For some cards the validity information is only availaible on these 2 links self.cards_hist_available.go(subbank=self.currentSubBank) @@ -218,6 +219,8 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): companies = self.page.companies_link() if self.cards_activity.is_here() else \ [self.page] if self.is_new_website else [] for company in companies: + # We need to return to the main page to avoid navigation error + self.cards_activity.go(subbank=self.currentSubBank) page = self.open(company).page if isinstance(company, basestring) else company for card in page.iter_cards(): card2 = find_object(self.cards_list, id=card.id[:16]) @@ -230,7 +233,8 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): card._secondpage = card2._secondpage self.accounts_list.remove(card2) self.accounts_list.append(card) - self.cards_list.append(card) + self.cards_list2.append(card) + self.cards_list.extend(self.cards_list2) # Populate accounts from old website if not self.is_new_website: diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index ed2a5426c84f5b9b0b82e40971e1e2e5f64fa532..153fea68ec58455c03169d59dd744fc9382d277d 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -551,8 +551,6 @@ class CardsListPage(LoggedPage, HTMLPage): class item(ItemElement): klass = Account - load_details = Field('_link_id') & AsyncLoad - obj_number = Field('_link_id') & Regexp(pattern=r'ctr=(\d+)') obj__card_number = Env('id', default="") obj_id = Format('%s%s', Env('id', default=""), Field('number')) @@ -574,7 +572,7 @@ class CardsListPage(LoggedPage, HTMLPage): return Link(TableCell('card')(self)[0].xpath('./a'))(self) def parse(self, el): - page = Async('details').loaded_page(self) + page = self.page.browser.open(Field('_link_id')(self)).page self.env['page'] = [page] if len(page.doc.xpath('//caption[contains(text(), "débits immédiats")]')): diff --git a/modules/ing/browser.py b/modules/ing/browser.py index 89a9c0a07b4c7d3f57f0db2a8a9bfc2644d50ea8..56b83fb5cde484d23115e8dd4c5569d6cd478730 100644 --- a/modules/ing/browser.py +++ b/modules/ing/browser.py @@ -54,11 +54,11 @@ def start_with_main_site(f): else: break browser.where = 'start' - elif browser.url and browser.url.startswith('https://ingdirectvie.ingdirect.fr/'): + elif browser.url and browser.url.startswith('https://ingdirectvie.ing.fr/'): browser.lifeback.go() browser.where = 'start' - elif browser.url and browser.url.startswith('https://subscribe.ingdirect.fr/'): + elif browser.url and browser.url.startswith('https://subscribe.ing.fr/'): browser.return_from_loan_site() return f(*args, **kwargs) @@ -66,12 +66,12 @@ def start_with_main_site(f): class IngBrowser(LoginBrowser): - BASEURL = 'https://secure.ingdirect.fr' + BASEURL = 'https://secure.ing.fr' TIMEOUT = 60.0 DEFERRED_CB = 'deferred' IMMEDIATE_CB = 'immediate' # avoid relogin every time - lifeback = URL(r'https://ingdirectvie.ingdirect.fr/b2b2c/entreesite/EntAccExit', ReturnPage) + lifeback = URL(r'https://ingdirectvie.ing.fr/b2b2c/entreesite/EntAccExit', ReturnPage) # Login and error loginpage = URL('/public/displayLogin.jsf.*', LoginPage) @@ -85,7 +85,7 @@ class IngBrowser(LoginBrowser): titredetails = URL('/general\?command=display.*', TitreDetails) ibanpage = URL('/protected/pages/common/rib/initialRib.jsf', IbanPage) loantokenpage = URL('general\?command=goToConsumerLoanCommand&redirectUrl=account-details', LoanTokenPage) - loandetailpage = URL('https://subscribe.ingdirect.fr/consumerloan/consumerloan-v1/consumer/details', LoanDetailPage) + loandetailpage = URL('https://subscribe.ing.fr/consumerloan/consumerloan-v1/consumer/details', LoanDetailPage) # CapBank-Market netissima = URL('/data/asv/fiches-fonds/fonds-netissima.html', NetissimaPage) starttitre = URL('/general\?command=goToAccount&zone=COMPTE', TitrePage) @@ -93,10 +93,10 @@ class IngBrowser(LoginBrowser): titrehistory = URL('https://bourse.ingdirect.fr/priv/compte.php\?ong=3', TitreHistory) titrerealtime = URL('https://bourse.ingdirect.fr/streaming/compteTempsReelCK.php', TitrePage) titrevalue = URL('https://bourse.ingdirect.fr/priv/fiche-valeur.php\?val=(?P.*)&pl=(?P.*)&popup=1', TitreValuePage) - asv_history = URL('https://ingdirectvie.ingdirect.fr/b2b2c/epargne/CoeLisMvt', - 'https://ingdirectvie.ingdirect.fr/b2b2c/epargne/CoeDetMvt', ASVHistory) - asv_invest = URL('https://ingdirectvie.ingdirect.fr/b2b2c/epargne/CoeDetCon', ASVInvest) - detailfonds = URL('https://ingdirectvie.ingdirect.fr/b2b2c/fonds/PerDesFac\?codeFonds=(.*)', DetailFondsPage) + asv_history = URL('https://ingdirectvie.ing.fr/b2b2c/epargne/CoeLisMvt', + 'https://ingdirectvie.ing.fr/b2b2c/epargne/CoeDetMvt', ASVHistory) + asv_invest = URL('https://ingdirectvie.ing.fr/b2b2c/epargne/CoeDetCon', ASVInvest) + detailfonds = URL('https://ingdirectvie.ing.fr/b2b2c/fonds/PerDesFac\?codeFonds=(.*)', DetailFondsPage) # CapDocument billpage = URL('/protected/pages/common/estatement/eStatement.jsf', BillsPage) # CapProfile @@ -279,8 +279,8 @@ class IngBrowser(LoginBrowser): def return_from_loan_site(self): data = {'context': '{"originatingApplication":"SECUREUI"}', 'targetSystem': 'INTERNET'} - self.location('https://subscribe.ingdirect.fr/consumerloan/consumerloan-v1/sso/exit', data=data) - self.location('https://secure.ingdirect.fr/', data={'token': self.response.text}) + self.location('https://subscribe.ing.fr/consumerloan/consumerloan-v1/sso/exit', data=data) + self.location('https://secure.ing.fr/', data={'token': self.response.text}) def get_account(self, _id, space=None): return find_object(self.get_accounts_list(get_iban=False, space=space), id=_id, error=AccountNotFound) diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index 4281ae109024ec4326a2fe187b14d992c2d01fe2..10a4132cbcae546c2a44d46ed1f84bead3bd1d8b 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -109,8 +109,8 @@ class LCLBrowser(LoginBrowser, StatesMixin): send_sms = URL('/outil/UWBE/Otp/envoiCodeOtp\?telChoisi=MOBILE', '/outil/UWBE/Otp/getValidationCodeOtp\?codeOtp', SmsPage) recip_recap = URL('/outil/UWBE/Creation/executeCreation', RecipRecapPage) documents = URL('/outil/UWDM/ConsultationDocument/derniersReleves', - '/outil/UWDM/Recherche/afficherPlus', '/outil/UWDM/Recherche/rechercherAll', DocumentsPage) + documents_plus = URL('/outil/UWDM/Recherche/afficherPlus', DocumentsPage) profile = URL('/outil/UWIP/Accueil/rafraichir', ProfilePage) @@ -144,6 +144,9 @@ class LCLBrowser(LoginBrowser, StatesMixin): if not self.password.isdigit(): raise BrowserIncorrectPassword() + # Since a while the virtual keyboard accepts only the first 6 digits of the password + self.password = self.password[:6] + self.contracts = [] self.current_contract = None @@ -513,7 +516,7 @@ class LCLBrowser(LoginBrowser, StatesMixin): def iter_documents(self, subscription): documents = [] self.documents.go() - self.location('https://particuliers.secure.lcl.fr/outil/UWDM/Recherche/afficherPlus') + self.documents_plus.go() self.page.do_search_request() for document in self.page.get_list(): documents.append(document) diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 34d3a9f0daaba201927b2b10362d7a45b295f795..897a13967b0a5c31c4603db98abe886575cb9572 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -149,7 +149,7 @@ class LCLModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, old = old.encode('latin-1', errors='replace').decode('latin-1') return super(LCLModule, self).transfer_check_label(old, new) - @only_for_websites('par') + @only_for_websites('par', 'elcl', 'pro') def iter_contacts(self): return self.browser.get_advisor() @@ -162,30 +162,30 @@ class LCLModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, return profile raise NotImplementedError() - @only_for_websites('par') + @only_for_websites('par', 'elcl', 'pro') def get_document(self, _id): return find_object(self.iter_documents(None), id=_id, error=DocumentNotFound) - @only_for_websites('par') + @only_for_websites('par', 'elcl', 'pro') def get_subscription(self, _id): return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) - @only_for_websites('par') + @only_for_websites('par', 'elcl', 'pro') def iter_bills(self, subscription): return self.iter_documents(None) - @only_for_websites('par') + @only_for_websites('par', 'elcl', 'pro') def iter_documents(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) return self.browser.iter_documents(subscription) - @only_for_websites('par') + @only_for_websites('par', 'elcl', 'pro') def iter_subscription(self): return self.browser.iter_subscriptions() - @only_for_websites('par') + @only_for_websites('par', 'elcl', 'pro') def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index f8fcc5218393b600ce4a8b354e854f67d43144af..b615c2a07f1476ac59cdaa3db623e4cad759385c 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -1109,6 +1109,8 @@ class DocumentsPage(LoggedPage, HTMLPage): head_xpath = '//table[@class="dematTab"]/thead/tr/th' item_xpath = u'//table[@class="dematTab"]/tbody/tr[./td[@class="dematTab-firstCell"]]' + ignore_duplicate = True + col_label = 'Nature de document' col_id = 'Type de document' col_url = 'Visualiser' diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 219094a666c365908a935568d8995e3cf24999e1..5206127c7f1fddf6ef7f76937b87e77aaef3abaf 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -232,6 +232,12 @@ class SocieteGenerale(LoginBrowser, StatesMixin): # Other Life Insurance pages: self.life_insurance_invest.go() + if self.life_insurance.is_here(): + # check that investements are here + error_msg = self.page.get_error_msg() + if error_msg and 'Le service est momentanément indisponible' in error_msg: + raise BrowserUnavailable(error_msg) + # Yield investments from the first page: for invest in self.page.iter_investment(): yield invest @@ -310,9 +316,19 @@ class SocieteGenerale(LoginBrowser, StatesMixin): if r.page.doc['donnees']['transaction_status'] == 'in_progress': raise ActionNeeded('Veuillez valider le bénéficiaire sur votre application bancaire.') - data = [('context', self.context), ('b64_jeton_transaction', self.context), - ('dup', self.dup), ('n10_id_transaction', self.id_transaction), ('oob_op', 'sign')] - self.add_recipient.go(data=data, headers={'Referer': 'https://particuliers.secure.societegenerale.fr/lgn/url.html'}) + data = [ + ('context', self.context), + ('b64_jeton_transaction', self.context), + ('dup', self.dup), + ('n10_id_transaction', self.id_transaction), + ('oob_op', 'sign') + ] + add_recipient_url = self.absurl('/lgn/url.html', base=True) + self.location( + add_recipient_url, + data=data, + headers={'Referer': add_recipient_url} + ) return self.page.get_recipient_object(recipient) @need_login diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 65ebb813216b5d63d93d9401c4b79b07195491da..9256d2907ba45dfa8bcd7fec2b667cfcf1870849 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -502,6 +502,9 @@ class LifeInsurance(LoggedPage, BasePage): def has_link(self): return Link('//a[@href="asvcns20a.html"]', default=NotAvailable)(self.doc) + def get_error_msg(self): + # to check page errors + return CleanText('//span[@class="error_msg"]')(self.doc) class LifeInsuranceInvest(LifeInsurance, Invest): COL_LABEL = 0 diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index e48c2ace3b954a73983b60faa68a5fdbccea34da..a7163d5f59f4d651fcae849535adee5f5ca8fc95 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -33,6 +33,7 @@ from weboob.browser.filters.html import Link from weboob.browser.filters.json import Dict from weboob.tools.value import Value, ValueBool from weboob.tools.json import json +from weboob.exceptions import BrowserUnavailable from .base import BasePage from .login import LoginPage @@ -43,6 +44,9 @@ class TransferJson(LoggedPage, JsonPage): if Dict('commun/statut')(self.doc).upper() == 'NOK': if self.doc['commun'].get('action'): raise TransferBankError(message=Dict('commun/action')(self.doc)) + elif self.doc['commun'].get('raison') == 'err_tech': + # on SG website, there is unavalaible message 'Le service est momentanément indisponible.' + raise BrowserUnavailable() else: assert False, 'Something went wrong, transfer is not created: %s' % self.doc['commun'].get('raison') @@ -167,7 +171,7 @@ class AddRecipientPage(LoggedPage, BasePage): return bool(CleanText(u'//h3[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ bool(CleanText(u'//h1[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ bool(CleanText(u'//h3[contains(text(), "Veuillez vérifier les informations du compte à ajouter")]')(self.doc)) or \ - bool(Link('//a[contains(@href, "per_cptBen_ajouterFrBic")]', default=NotAvailable)(self.doc)) + bool(Link('//a[contains(@href, "per_cptBen_ajouter")]', default=NotAvailable)(self.doc)) def post_iban(self, recipient): form = self.get_form(name='persoAjoutCompteBeneficiaire')