From 3e8da1445280eb6b28b3e3d99a2e688ce7d04e90 Mon Sep 17 00:00:00 2001 From: Vincent Ardisson Date: Wed, 19 Apr 2017 17:36:25 +0200 Subject: [PATCH] use url field instead of _link for Accounts in a few modules bforbank, boursorama, bp, cragr, spirica --- modules/bforbank/browser.py | 12 ++++++------ modules/bforbank/pages.py | 11 ++++++++--- modules/boursorama/browser.py | 32 +++++++++++++++---------------- modules/boursorama/pages.py | 15 ++++++++------- modules/bp/browser.py | 10 +++++----- modules/bp/pages/accountlist.py | 2 +- modules/bp/pages/pro.py | 2 +- modules/cragr/web/browser.py | 14 +++++++------- modules/cragr/web/pages.py | 34 ++++++++++++++++----------------- modules/spirica/browser.py | 4 ++-- modules/spirica/pages.py | 8 +++++--- 11 files changed, 76 insertions(+), 68 deletions(-) diff --git a/modules/bforbank/browser.py b/modules/bforbank/browser.py index 64ab2afb6a..bd8cb7be52 100644 --- a/modules/bforbank/browser.py +++ b/modules/bforbank/browser.py @@ -95,7 +95,7 @@ def iter_accounts(self): card = self.page.get_card(account.id) if card is not None: # if there's a credit card (not debit), create a separate, virtual account - card._link = account._link + card.url = account.url card.balance = account._card_balance assert not empty(card.balance) account._card_account = card @@ -106,7 +106,7 @@ def iter_accounts(self): return iter(self.accounts) def _get_card_transactions(self, account): - self.location(account._link.replace('tableauDeBord', 'encoursCarte') + '/0?month=1') + self.location(account.url.replace('tableauDeBord', 'encoursCarte') + '/0?month=1') assert self.card_history.is_here() return list(self.page.get_operations()) @@ -127,7 +127,7 @@ def get_history(self, account): return self.spirica.iter_history(account) if account.type != Account.TYPE_CARD: - self.location(account._link.replace('tableauDeBord', 'operations')) + self.location(account.url.replace('tableauDeBord', 'operations')) assert self.history.is_here() or self.loan_history.is_here() return self.page.get_operations() @@ -157,7 +157,7 @@ def get_coming(self, account): return self.page.get_operations() elif account.type == Account.TYPE_CARD: # TODO there could be multiple cards, how to find the number of cards? - self.location(account._link.replace('tableauDeBord', 'encoursCarte') + '/0') + self.location(account.url.replace('tableauDeBord', 'encoursCarte') + '/0') return self.page.get_operations() else: raise NotImplementedError() @@ -181,7 +181,7 @@ def goto_spirica(self, account): for insurance_account in self.page.iter_accounts(): self.logger.debug('testing %r', account) if insurance_account.id == account.id: - self.location(insurance_account._link) + self.location(insurance_account.url) assert self.lifeinsurance_iframe.is_here() break else: @@ -199,7 +199,7 @@ def goto_spirica(self, account): redir = self.page.get_redir() assert redir - account._link = self.absurl(redir) + account.url = self.absurl(redir) self.spirica.session.cookies.update(self.session.cookies) self.spirica.logged = True return True diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index 06decb79b4..34eec6bb28 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -22,7 +22,7 @@ from io import BytesIO from PIL import Image from urllib import urlencode -from urlparse import urlparse, urlunparse, parse_qsl +from urlparse import urlparse, urlunparse, parse_qsl, urljoin from weboob.browser.pages import LoggedPage, HTMLPage, pagination, AbstractPage from weboob.browser.elements import method, ListElement, ItemElement, TableElement @@ -147,7 +147,10 @@ class item(ItemElement): obj_balance = MyDecimal('./td//div[contains(@class, "-synthese-num")]', replace_dots=True) obj_currency = FrenchTransaction.Currency('./td//div[contains(@class, "-synthese-num")]') obj_type = Map(Regexp(Field('label'), r'^([^ ]*)'), TYPE, default=Account.TYPE_UNKNOWN) - obj__link = CleanText('./@data-href') + + def obj_url(self): + return urljoin(self.page.url, CleanText('./@data-href')(self)) + obj__card_balance = CleanDecimal('./td//div[@class="synthese-encours"]/div[2]', default=None) def condition(self): @@ -285,7 +288,9 @@ class item(ItemElement): klass = Account obj_id = CleanText('./td/a') - obj__link = Link('./td/a') + + def obj_url(self): + return urljoin(self.page.url, Link('./td/a')(self)) class LifeInsuranceIframe(LoggedPage, HTMLPage): diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index 5a134b33f2..c22eb7244c 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -196,7 +196,7 @@ def get_debit_date(self, debit_date): return j[1].date() def get_card_transactions(self, account): - self.location('%s' % account._link, params={'movementSearch[period]': 'currentPeriod'}) + self.location(account.url, params={'movementSearch[period]': 'currentPeriod'}) if self.home.is_here(): # for some cards, the site redirects us to '/'... return @@ -204,7 +204,7 @@ def get_card_transactions(self, account): for t in self.page.iter_history(is_card=True): yield t - self.location('%s' % account._link, params={'movementSearch[period]': 'previousPeriod'}) + self.location(account.url, params={'movementSearch[period]': 'previousPeriod'}) for t in self.page.iter_history(is_card=True, is_previous=True): yield t @@ -212,7 +212,7 @@ def get_invest_transactions(self, account, coming): if coming: return transactions = [] - self.location('%s/mouvements' % account._link.rstrip('/')) + self.location('%s/mouvements' % account.url.rstrip('/')) account._history_pages = [] for t in self.page.iter_history(account=account): transactions.append(t) @@ -227,17 +227,17 @@ def get_regular_transactions(self, account, coming): params['movementSearch[toDate]'] = (date.today() + relativedelta(days=40)).strftime('%d/%m/%Y') params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=1)).strftime('%d/%m/%Y') params['movementSearch[selectedAccounts][]'] = account._webid - self.location('%s/mouvements' % account._link.rstrip('/'), params=params) + self.location('%s/mouvements' % account.url.rstrip('/'), params=params) for t in self.page.iter_history(): yield t if coming and account.type == Account.TYPE_CHECKING: - self.location('%s/mouvements-a-venir' % account._link.rstrip('/'), params=params) + self.location('%s/mouvements-a-venir' % account.url.rstrip('/'), params=params) for t in self.page.iter_history(coming=True): yield t @need_login def get_history(self, account, coming=False): - if account.type is Account.TYPE_LOAN or '/compte/derive' in account._link: + if account.type is Account.TYPE_LOAN or '/compte/derive' in account.url: return [] if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET): return self.get_invest_transactions(account, coming) @@ -247,11 +247,11 @@ def get_history(self, account, coming=False): @need_login def get_investment(self, account): - if '/compte/derive' in account._link: + if '/compte/derive' in account.url: return iter([]) if not account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET, Account.TYPE_PEA): raise NotImplementedError() - self.location(account._link) + self.location(account.url) # We might deconnect at this point. if self.login.is_here(): return self.get_investment(account) @@ -271,11 +271,11 @@ def get_advisor(self): @need_login def iter_transfer_recipients(self, account): - assert account._link - if account._link.endswith('/'): - target = account._link + 'virements' + assert account.url + if account.url.endswith('/'): + target = account.url + 'virements' else: - target = account._link + '/virements' + target = account.url + '/virements' try: self.location(target) @@ -372,14 +372,14 @@ def new_recipient(self, recipient, **kwargs): account = None for account in self.get_accounts_list(): - if account._link: + if account.url: break suffix = 'virements/comptes-externes/nouveau' - if account._link.endswith('/'): - target = account._link + suffix + if account.url.endswith('/'): + target = account.url + suffix else: - target = account._link + '/' + suffix + target = account.url + '/' + suffix self.location(target) assert self.page.is_charac() diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 77553f5d0c..f2facd2c57 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -24,6 +24,7 @@ import re from io import BytesIO from datetime import date +from urlparse import urljoin from weboob.browser.pages import HTMLPage, LoggedPage, pagination, NextPage, FormNotFound from weboob.browser.elements import ListElement, ItemElement, method, TableElement, SkipItem @@ -191,7 +192,7 @@ class iter_accounts(ListElement): class item(ItemElement): klass = Account - load_details = Field('_link') & AsyncLoad + load_details = Field('url') & AsyncLoad obj_label = CleanText('.//a[@class="account--name"] | .//div[@class="account--name"]') obj_balance = CleanDecimal('.//a[has-class("account--balance")]', replace_dots=True) @@ -231,14 +232,14 @@ def obj_type(self): return Account.TYPE_UNKNOWN - def obj__link(self): + def obj_url(self): link = Attr('.//a[@class="account--name"] | .//a[2]', 'href', default=NotAvailable)(self) if not self.page.browser.webid: self.page.browser.webid = re.search('\/([^\/|?|$]{32})(\/|\?|$)', link).group(1) - return link + return urljoin(self.page.url, link) def obj__webid(self): - m = re.search('([a-z\d]{32})', self.obj__link()) + m = re.search('([a-z\d]{32})', Field('url')(self)) if m: return m.group(1) return None @@ -500,8 +501,8 @@ def populate(self, accounts): acc.balance = balance acc.label = label acc.currency = FrenchTransaction.Currency().filter(balance_el) - acc._link = Link().filter(a.xpath('.')) - acc._history_page = acc._link + acc.url = urljoin(self.url, Link().filter(a.xpath('.'))) + acc._history_page = acc.url try: acc.id = acc._webid = Regexp(pattern='carte/(.*)$').filter(Link().filter(a.xpath('.'))) except RegexpError: @@ -515,7 +516,7 @@ def populate(self, accounts): account.type = AccountsPage.ACCOUNT_TYPES.get(title, Account.TYPE_UNKNOWN) account._webid = Attr(None, 'data-account-label').filter(a.xpath('.//span[@class="nav-category__name"]')) if cards: - self.browser.go_cards_number(cards[0]._link) + self.browser.go_cards_number(cards[0].url) if self.browser.cards.is_here(): self.browser.page.populate_cards_number(cards) accounts.extend(cards) diff --git a/modules/bp/browser.py b/modules/bp/browser.py index d819697e08..359d9c1fa3 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -186,7 +186,7 @@ def get_history(self, account): transactions = [] if account.type is not Account.TYPE_LOAN: - self.location(account._link_id) + self.location(account.url) history = {Account.TYPE_CHECKING: self.par_account_checking_history, Account.TYPE_SAVINGS: self.par_account_savings_and_invests_history, @@ -209,14 +209,14 @@ def get_history(self, account): @need_login def go_linebourse(self, account): - self.location(account._link_id) + self.location(account.url) self.market_login.go() self.linebourse.session.cookies.update(self.session.cookies) self.par_accounts_checking.go() def _get_coming_transactions(self, account): if account.type == Account.TYPE_CHECKING: - self.location(account._link_id) + self.location(account.url) self.par_account_checking_coming.go(accountId=account.id) if self.par_account_checking_coming.is_here() and self.page.has_coming(): @@ -361,7 +361,7 @@ def set_variables(self): self.accounts_url = self.base_url + '/voscomptes/synthese/synthese.ea' def go_linebourse(self, account): - self.location(account._link_id) + self.location(account.url) self.location('../bourseenligne/oicformautopost.jsp') self.linebourse.session.cookies.update(self.session.cookies) self.location(self.accounts_url) @@ -373,7 +373,7 @@ def get_history(self, account): return self.linebourse.iter_history(account.id) transactions = [] - v = urlsplit(account._link_id) + v = urlsplit(account.url) args = dict(parse_qsl(v.query)) args['typeRecherche'] = 10 diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index 98189bbcc6..8ad5f8ab96 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -60,7 +60,7 @@ def condition(self): obj_id = CleanText('.//abbr/following-sibling::text()') obj_currency = Currency('.//span[@class="number"]') - def obj__link_id(self): + def obj_url(self): url = Link(u'./a', default=NotAvailable)(self) if url: return urljoin(self.page.url, url) diff --git a/modules/bp/pages/pro.py b/modules/bp/pages/pro.py index 205774e946..79e8e51a8c 100644 --- a/modules/bp/pages/pro.py +++ b/modules/bp/pages/pro.py @@ -68,7 +68,7 @@ def get_accounts_list(self): a.currency = u'EUR' a.balance = Decimal(Transaction.clean_amount(tmp_balance)) a._has_cards = False - a._link_id = urljoin(self.url, link.attrib['href']) + a.url = urljoin(self.url, link.attrib['href']) yield a diff --git a/modules/cragr/web/browser.py b/modules/cragr/web/browser.py index 4472df3718..f53f17d144 100644 --- a/modules/cragr/web/browser.py +++ b/modules/cragr/web/browser.py @@ -308,7 +308,7 @@ def get_list(self): try: new_location = self.moveto_market_website(account, home=True) except WebsiteNotSupported: - account._link = None + account.url = None self.update_sag() else: self.location(new_location) @@ -340,7 +340,7 @@ def get_history(self, account): raise NotImplementedError() # some accounts may exist without a link to any history page - if account._link is None or 'CATITRES' in account._link: + if account.url is None or 'CATITRES' in account.url: return if account._perimeter != self.current_perimeter: @@ -351,7 +351,7 @@ def get_history(self, account): account = self.get_cards_or_card(account.number) if account.type != Account.TYPE_CARD or not self.page.is_on_right_detail(account): - self.location(account._link.format(self.sag)) + self.location(account.url.format(self.sag)) if self.cards.is_here(): date_guesser = ChaoticDateGuesser(date.today()-timedelta(weeks=36)) @@ -385,7 +385,7 @@ def get_history(self, account): @need_login def iter_investment(self, account): - if not account._link or account.type not in (Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE): + if not account.url or account.type not in (Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE): return if account._perimeter != self.current_perimeter: @@ -442,7 +442,7 @@ def get_profile(self): @need_login def moveto_market_website(self, account, home=False): - response = self.open(account._link % self.sag).text + response = self.open(account.url % self.sag).text self._sag = None # https://www.cabourse.credit-agricole.fr/netfinca-titres/servlet/com.netfinca.frontcr.navigation.AccueilBridge?TOKEN_ID= m = re.search('document.location="([^"]+)"', response) @@ -479,14 +479,14 @@ def quit_market_website(self): @need_login def moveto_insurance_website(self, account): - page = self.open(account._link % self.sag).page + page = self.open(account.url % self.sag).page self._sag = None # POST to https://assurance-personnes.credit-agricole.fr/filiale/ServletReroutageCookie try: form = page.get_form(name='formulaire') except FormNotFound: # bgpi-gestionprivee. - body = self.open(account._link % self.sag).text + body = self.open(account.url % self.sag).text return re.search('location="([^"]+)"', body, flags=re.MULTILINE).group(1) data = { diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index 2dbee1a590..8f5ee2fd6c 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -21,7 +21,7 @@ from datetime import date as ddate, datetime from decimal import Decimal -from urlparse import urlparse +from urlparse import urlparse, urljoin import re from weboob.browser.pages import HTMLPage, FormNotFound @@ -225,7 +225,7 @@ def get_list(self): account.balance = Decimal(Transaction.clean_amount(balance)) account.currency = account.get_currency(cleaner(cols[self.COL_CURRENCY])) - account._link = None + account.url = None self.set_link(account, cols) @@ -371,13 +371,13 @@ def get_list(self): if 'link' in xpaths: try: - account._link = table.xpath(xpaths['link'])[-1] + account.url = table.xpath(xpaths['link'])[-1] except IndexError: - account._link = None + account.url = None else: - account._link = re.sub('[\n\r\t]+', '', account._link) + account.url = urljoin(self.url, re.sub('[\n\r\t]+', '', account.url)) else: - account._link = self.url + account.url = self.url account._idelco = re.search('IDELCO=(\d+)&', self.url).group(1) account._perimeter = self.browser.current_perimeter yield account @@ -456,9 +456,9 @@ class AccountsPage(_AccountsPage): def set_link(self, account, cols): a = cols[0].find('a') if a is not None: - account._link = a.attrib['href'].replace(' ', '%20') - page = self.browser.open(account._link).page - account._link = re.sub('sessionSAG=[^&]+', 'sessionSAG={0}', account._link) + account.url = urljoin(self.url, a.attrib['href'].replace(' ', '%20')) + page = self.browser.open(account.url).page + account.url = re.sub('sessionSAG=[^&]+', 'sessionSAG={0}', account.url) url = page.get_iban_url() if url: page = self.browser.open(url).page @@ -483,30 +483,30 @@ class SavingsPage(_AccountsPage): def set_link(self, account, cols): origin = urlparse(self.url) - if not account._link: + if not account.url: a = cols[0].xpath('descendant::a[contains(@href, "CATITRES")]') # Sometimes there is no link. if a or account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): url = 'https://%s/stb/entreeBam?sessionSAG=%%s&stbpg=pagePU&site=CATITRES&typeaction=reroutage_aller' - account._link = url % origin.netloc + account.url = url % origin.netloc a = cols[0].xpath("descendant::a[contains(@href, \"'PREDICA','CONTRAT'\")]") if a: account.type = Account.TYPE_LIFE_INSURANCE url = 'https://%s/stb/entreeBam?sessionSAG=%%s&stbpg=pagePU&site=PREDICA&' \ 'typeaction=reroutage_aller&sdt=CONTRAT¶mpartenaire=%s' - account._link = url % (origin.netloc, account.id) + account.url = url % (origin.netloc, account.id) a = cols[0].xpath('descendant::a[not(contains(@href, "javascript"))]') - if len(a) == 1 and not account._link: - account._link = a[0].attrib['href'].replace(' ', '%20') - account._link = re.sub('sessionSAG=[^&]+', 'sessionSAG={0}', account._link) + if len(a) == 1 and not account.url: + account.url = urljoin(self.url, a[0].attrib['href'].replace(' ', '%20')) + account.url = re.sub('sessionSAG=[^&]+', 'sessionSAG={0}', account.url) a = cols[0].xpath('descendant::a[(contains(@href, "javascript"))]') # This aims to handle bgpi-gestionprivee. - if len(a) == 1 and not account._link: + if len(a) == 1 and not account.url: m = re.findall("'([^']*)'", a[0].attrib['href']) if len(m) == 3: url = 'https://%s/stb/entreeBam?sessionSAG=%%s&stbpg=pagePU&typeaction=reroutage_aller&site=%s&sdt=%s¶mpartenaire=%s' - account._link = url % (origin.netloc, m[0], m[1], m[2]) + account.url = url % (origin.netloc, m[0], m[1], m[2]) class TransactionsPage(MyLoggedPage, BasePage): diff --git a/modules/spirica/browser.py b/modules/spirica/browser.py index 17b24664d4..71607ade84 100644 --- a/modules/spirica/browser.py +++ b/modules/spirica/browser.py @@ -59,7 +59,7 @@ def iter_accounts(self): def iter_investment(self, account): if account.id not in self.cache['invs']: # Get form to show PRM - form = self.location(account._link).page.get_investment_form() + form = self.location(account.url).page.get_investment_form() invs = [i for i in self.location(form.url, data=dict(form)).page.iter_investment()] self.cache['invs'][account.id] = invs return self.cache['invs'][account.id] @@ -68,7 +68,7 @@ def iter_investment(self, account): def iter_history(self, account): if account.id not in self.cache['trs']: # Get form to go to History's tab - form = self.location(account._link).page.get_historytab_form() + form = self.location(account.url).page.get_historytab_form() # Get form to show all transactions form = self.location(form.url, data=dict(form)).page.get_historyallpages_form() if form: diff --git a/modules/spirica/pages.py b/modules/spirica/pages.py index b8b56492a5..d51c2d5e0a 100644 --- a/modules/spirica/pages.py +++ b/modules/spirica/pages.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . - +from urlparse import urljoin import re from weboob.browser.pages import HTMLPage, LoggedPage @@ -71,14 +71,16 @@ class iter_accounts(TableElement): class item(ItemElement): klass = Account - load_details = Field('_link') & AsyncLoad + load_details = Field('url') & AsyncLoad obj_id = CleanText(TableCell('id'), replace=[(' ', '')]) obj_label = CleanText(TableCell('label')) obj_balance = MyDecimal(TableCell('balance')) obj_valuation_diff = Async('details') & MyDecimal('//tr[1]/td[contains(text(), \ "value du contrat")]/following-sibling::td') - obj__link = Link('.//a') + + def obj_url(self): + return urljoin(self.page.url, Link('.//a')(self)) def obj_type(self): return self.page.TYPES[Async('details', CleanText('//td[contains(text(), \ -- GitLab