Skip to content
Commits on Source (58)
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
paths:
- .cache/pip
before_script:
- "pip install -r .ci/requirements.txt"
- "REQUIREMENTS=$(mktemp) && python setup.py requirements > ${REQUIREMENTS} && pip install -r ${REQUIREMENTS} && rm ${REQUIREMENTS}"
......
......@@ -75,3 +75,4 @@ Matthieu Weber <mweber+weboob@free.fr> <mweber@free.fr>
Théo Dorée <theo.doree@budget-insight.com> <tdoree@budget-insight.com>
Tenma <nicolas.gattolin@budget-insight.com>
Damien Ramelet <damien.ramelet@budget-insight.com>
Yasmine Idwy <yasmine.idwy@budget-insight.com>
......@@ -37,8 +37,10 @@ class SevenFiftyGramsBrowser(PagesBrowser):
def iter_recipes(self, pattern):
return self.search.go(pattern=quote_plus(pattern.encode('utf-8')), page=1).iter_recipes()
def get_recipe(self, id, recipe=None):
self.recipe.go(id=id)
@recipe.id2url
def get_recipe(self, url, recipe=None):
self.location(url)
assert self.recipe.is_here()
return self.get_recipe_content(recipe)
def get_comments(self, id):
......
......@@ -31,4 +31,3 @@ def test_recipe(self):
self.assertTrue(full_recipe.instructions, 'No instructions for %s' % recipe.id)
self.assertTrue(full_recipe.ingredients, 'No ingredients for %s' % recipe.id)
self.assertTrue(full_recipe.title, 'No title for %s' % recipe.id)
self.assertTrue(full_recipe.preparation_time, 'No preparation time for %s' % recipe.id)
......@@ -61,9 +61,9 @@ class AmazonModule(Module, CapDocument):
Value('website', label=u'Website', choices=website_choices, default='www.amazon.com'),
ValueBackendPassword('email', label='Username', masked=False),
ValueBackendPassword('password', label='Password'),
Value('captcha_response', label='Captcha Response', required=False, default=''),
Value('pin_code', label='OTP response', required=False, default=''),
Value('request_information', label='request_information', default=None, required=False, noprompt=True),
ValueTransient('captcha_response', label='Captcha Response'),
ValueTransient('pin_code', label='OTP response'),
ValueTransient('request_information'),
ValueTransient('resume'),
)
......
......@@ -96,7 +96,8 @@ def do_login(self):
error_message = self.page.get_error_message()
if error_message:
is_website_unavailable = re.search(
"Veuillez nous excuser pour la gêne occasionnée",
"Veuillez nous excuser pour la gêne occasionnée"
+ "|votre espace client est temporairement indisponible",
error_message
)
......
......@@ -117,6 +117,7 @@ def wrapper(browser, *args, **kwargs):
class BanquePopulaire(LoginBrowser):
first_login_page = URL(r'/$')
first_cm_login_page = URL(r'/cyber/ibp/ate/portal/internet89C3Portal.jsp')
login_page = URL(r'https://[^/]+/auth/UI/Login.*', LoginPage)
new_login = URL(r'https://[^/]+/.*se-connecter/sso', NewLoginPage)
js_file = URL(r'https://[^/]+/.*se-connecter/main-.*.js$', JsFilePage)
......@@ -292,8 +293,9 @@ class BanquePopulaire(LoginBrowser):
def __init__(self, website, *args, **kwargs):
self.website = website
self.BASEURL = 'https://%s' % website
# this url is required because the creditmaritime abstract uses an other url
if 'cmgo.creditmaritime' in self.BASEURL:
self.is_creditmaritime = 'cmgo.creditmaritime' in self.BASEURL
if self.is_creditmaritime:
# this url is required because the creditmaritime abstract uses an other url
self.redirect_url = 'https://www.icgauth.creditmaritime.groupe.banquepopulaire.fr/dacsrest/api/v1u0/transaction/'
else:
self.redirect_url = 'https://www.icgauth.banquepopulaire.fr/dacsrest/api/v1u0/transaction/'
......@@ -334,7 +336,10 @@ def follow_back_button_if_any(self, params=None, actions=None):
@no_need_login
def do_login(self):
try:
self.first_login_page.go()
if self.is_creditmaritime:
self.first_cm_login_page.go()
else:
self.first_login_page.go()
except (ClientError, HTTPNotFound) as e:
if e.response.status_code in (403, 404):
# Sometimes the website makes some redirections that leads
......@@ -377,6 +382,7 @@ def do_old_login(self):
self.location('/cyber/internet/Login.do', data=data)
def get_bpcesta(self, cdetab):
# Don't add term_id parameter
return {
'csid': str(uuid4()),
'typ_app': 'rest',
......@@ -471,13 +477,14 @@ def do_new_login(self):
self.authorize.go(params=params)
self.page.send_form()
validation_id = self.page.get_validation_id()
self.get_current_subbank()
self.check_for_fallback(validation_id)
if self.authorize_error.is_here():
raise BrowserUnavailable(self.page.get_error_message())
self.page.check_errors(feature='login')
self.get_current_subbank()
validation_id = self.page.get_validation_id()
validation_unit_id = self.page.validation_unit_id
vk_info = self.page.get_authentication_method_info()
......@@ -544,6 +551,28 @@ def do_new_login(self):
# Need to do the redirect a second time to finish login
self.do_redirect(headers)
if self.is_creditmaritime:
data = {
'integrationMode': 'INTERNET_89C3',
'realOrigin': self.BASEURL,
}
# Supplementary request needed to get token
self.location('/cyber/internet/Login.do', data=data)
def check_for_fallback(self, validation_id):
for _ in range(2):
auth_method = self.page.get_authentication_method_type()
if self.page.is_other_authentication_method() and auth_method != 'PASSWORD':
self.authentication_step.go(
subbank=self.current_subbank,
validation_id=validation_id,
json={'fallback': {}},
)
else:
break
else:
raise AssertionError("Failure of the fallback authentication method, never end up on the PASSWORD method")
ACCOUNT_URLS = ['mesComptes', 'mesComptesPRO', 'maSyntheseGratuite', 'accueilSynthese', 'equipementComplet']
def do_redirect(self, headers):
......@@ -1013,11 +1042,27 @@ def get_invest_history(self, account):
@need_login
def get_profile(self):
if self.is_creditmaritime:
# Supplementary request needed to reach profile
self.creditmaritime_start_profile()
self.location(self.absurl('/cyber/internet/StartTask.do?taskInfoOID=accueil&token=%s' % self.token, base=True))
# For some user this page is not accessible
if not self.page.is_profile_unavailable():
return self.page.get_profile()
def creditmaritime_start_profile(self):
data = {
'integrationMode': 'INTERNET_89C3',
'realOrigin': self.BASEURL,
}
if not self.home_page.is_here():
self.location('/cyber/internet/Login.do', data=data)
data['taskId'] = self.page.get_profile_type()
self.location('/cyber/internet/Login.do', data=data) # It's not a real login request
@retry(LoggedOut)
@need_login
def get_advisor(self):
......@@ -1114,7 +1159,10 @@ def get_current_subbank(self):
@need_login
def get_owner_type(self):
self.first_login_page.go()
if self.is_creditmaritime:
self.first_cm_login_page.go()
else:
self.first_login_page.go()
if self.home_page.is_here():
return self.page.get_owner_type()
......
......@@ -33,7 +33,7 @@
from woob.browser.elements import method, DictElement, ItemElement
from woob.browser.filters.standard import (
CleanText, CleanDecimal, Regexp, Eval,
Date, Field, MapIn, Coalesce,
Date, Field, MapIn, Coalesce, QueryValue,
)
from woob.browser.filters.html import Attr, Link, AttributeNotFound
from woob.browser.filters.json import Dict
......@@ -687,10 +687,22 @@ def build_doc(self, data, *args, **kwargs):
return None
return super(MyHTMLPage, self).build_doc(data, *args, **kwargs)
def get_profile_type(self):
# Can be aUniversMonProfil or aUniversProfil
return Regexp(
Attr('//li/a[text()="Profil"]', 'onclick'),
r"selectUniverse\('(\w+)'"
)(self.doc)
@retry(KeyError)
# sometime the server redirects to a bad url, not containing token.
# therefore "return args['token']" crashes with a KeyError
def get_token(self):
if self.browser.is_creditmaritime:
# The request done in banquepopulaire does not work for CM
# We get token directly in the url we were redirected
return QueryValue(None, 'token', default=None).filter(self.url)
vary = None
if self.params.get('vary', None) is not None:
vary = self.params['vary']
......
......@@ -62,14 +62,14 @@ class BforbankBrowser(TwoFactorBrowser):
home = URL('/espace-client/$', AccountsPage)
rib = URL(
'/espace-client/rib',
r'/espace-client/rib/(?P<id>\d+)',
r'/espace-client/rib/(?P<id>[^/]+)$',
RibPage
)
loan_history = URL('/espace-client/livret/consultation.*', LoanHistoryPage)
history = URL('/espace-client/consultation/operations/.*', HistoryPage)
coming = URL(r'/espace-client/consultation/operationsAVenir/(?P<account>[^/]+)$', HistoryPage)
card_history = URL('espace-client/consultation/encoursCarte/.*', CardHistoryPage)
card_page = URL(r'/espace-client/carte/(?P<account>\d+)$', CardPage)
card_page = URL(r'/espace-client/carte/(?P<account>[^/]+)$', CardPage)
lifeinsurance = URL(r'/espace-client/assuranceVie/(?P<account_id>\d+)')
lifeinsurance_list = URL(r'/client/accounts/lifeInsurance/lifeInsuranceSummary.action', LifeInsuranceList)
......@@ -275,14 +275,16 @@ def iter_accounts(self):
self.home.go()
accounts = list(self.page.iter_accounts(name=owner_name))
if self.page.RIB_AVAILABLE:
self.rib.go().populate_rib(accounts)
for account in accounts:
self.rib.go(id=account._url_code)
self.page.populate_rib(account)
self.accounts = []
for account in accounts:
self.accounts.append(account)
if account.type == Account.TYPE_CHECKING:
self.card_page.go(account=account.id)
self.card_page.go(account=account._url_code)
if self.page.has_no_card():
continue
cards = self.page.get_cards(account.id)
......
......@@ -119,21 +119,11 @@ def filter(self, text):
class RibPage(LoggedPage, HTMLPage):
def populate_rib(self, accounts):
for option in self.doc.xpath('//select[@id="compte-select"]/option'):
if 'selected' in option.attrib:
self.get_iban(accounts)
else:
page = self.browser.rib.go(id=Regexp(Attr('.', 'value'), r'/(.+)')(option))
page.get_iban(accounts)
def get_iban(self, accounts):
for account in accounts:
if self.doc.xpath('//option[@selected and contains(@value, $id)]', id=account.id):
account.iban = CleanText(
'//td[contains(text(), "IBAN")]/following-sibling::td[1]',
replace=[(' ', '')]
)(self.doc)
def populate_rib(self, account):
account.iban = CleanText(
'//td[contains(text(), "IBAN")]/following-sibling::td[1]',
replace=[(' ', '')]
)(self.doc)
class AccountsPage(LoggedPage, HTMLPage):
......@@ -175,8 +165,9 @@ def obj_url(self):
path = Attr('.', 'data-urlcatitre')(self)
return urljoin(self.page.url, path)
# Looks like a variant of base64: ASKHJLHWF272jhk22kjhHJQ1_ufad892hjjj122j348=
obj__url_code = Regexp(Field('url'), r'/espace-client/consultation/operations/(.*)', default=None)
# Looks like a variant of base64: 'ASKHJLHWF272jhk22kjhHJQ1_ufad892hjjj122j348=' at the end of the URL.
# Must match '/espace-client/consultation/operations/(.*)' and '/espace-client/livret/consultation/(.*)'.
obj__url_code = Regexp(Field('url'), r'/espace-client/.+/(.+)', default=None)
obj__card_balance = CleanDecimal('./td//div[@class="synthese-encours"][last()]/div[2]', default=None)
def obj_balance(self):
......
......@@ -269,12 +269,17 @@ def load_state(self, state):
def handle_authentication(self):
if self.authentication.is_here():
self.check_interactive()
confirmation_link = self.page.get_confirmation_link()
if confirmation_link:
self.location(confirmation_link)
if self.page.has_skippable_2fa():
# The 2FA can be done before the end of the 90d
# We skip it
return
self.check_interactive()
self.page.sms_first_step()
self.page.sms_second_step()
......
......@@ -108,6 +108,11 @@ def authenticate(self):
def get_confirmation_link(self):
return Link('//a[contains(@href, "validation")]', default=None)(self.doc)
def has_skippable_2fa(self):
return self.doc.xpath(
'//form[@name="form"]/div[contains(@data-strong-authentication-payload, "Ignorer")]'
)
def sms_first_step(self):
"""
This function simulates the registration of a device on
......
......@@ -24,7 +24,7 @@
from woob.browser import LoginBrowser, URL, need_login
from woob.browser.exceptions import HTTPNotFound, ClientError
from woob.exceptions import BrowserIncorrectPassword
from woob.exceptions import BrowserIncorrectPassword, ScrapingBlocked
from woob.tools.compat import urlparse, parse_qsl
from .pages import (
......@@ -69,7 +69,12 @@ def __init__(self, username, password, lastname, *args, **kwargs):
self.headers = None
def do_login(self):
self.login_page.go()
try:
self.login_page.go()
except ClientError as e:
if e.response.status_code == 407:
raise ScrapingBlocked()
raise
try:
self.page.login(self.username, self.password, self.lastname)
......
......@@ -95,11 +95,12 @@ class Transaction(FrenchTransaction):
class AccountHistory(LoggedPage, MyHTMLPage):
def on_load(self):
super(AccountHistory, self).on_load()
if bool(CleanText('//h2[contains(text(), "ERREUR")]')(self.doc)):
raise BrowserUnavailable()
def is_here(self):
return not bool(CleanText('//h1[contains(text(), "tail de vos cartes")]')(self.doc))
return not bool(CleanText('//h2[contains(text(), "tail de vos cartes")]')(self.doc))
def get_next_link(self):
for a in self.doc.xpath('//a[@class="btn_crt"]'):
......@@ -246,16 +247,16 @@ def get_single_card(self, parent_id):
class CardsList(LoggedPage, MyHTMLPage):
def is_here(self):
return bool(
CleanText('//h1[contains(text(), "tail de vos cartes")]')(self.doc)
and not CleanText('//h1[contains(text(), "tail de vos op")]')(self.doc)
CleanText('//h2[contains(text(), "tail de vos cartes")]')(self.doc)
and not CleanText('//h2[contains(text(), "tail de vos op")]')(self.doc)
)
@method
class get_cards(TableElement):
item_xpath = '//table[@class="dataNum"]/tbody/tr'
head_xpath = '//table[@class="dataNum"]/thead/tr/th'
item_xpath = '//table[has-class("dataNum") or has-class("dataCarte")]/tbody/tr'
head_xpath = '//table[has-class("dataNum") or has-class("dataCarte")]/thead/tr/th'
col_label = re.compile('Vos cartes Encours actuel prélevé au')
col_label = re.compile('Vos cartes,? [Ee]ncours actuel prélevé au')
col_balance = 'Euros'
col_number = 'Numéro'
col__credit = 'Crédit (euro)'
......@@ -291,6 +292,7 @@ def obj_url(self):
class SavingAccountSummary(LoggedPage, MyHTMLPage):
def on_load(self):
super(SavingAccountSummary, self).on_load()
link = Link('//ul[has-class("tabs")]//a[@title="Historique des mouvements"]', default=NotAvailable)(self.doc)
if link:
self.browser.location(link)
......
......@@ -260,7 +260,7 @@ def obj_type(self):
class AccountList(LoggedPage, MyHTMLPage):
def on_load(self):
MyHTMLPage.on_load(self)
super(AccountList, self).on_load()
# website sometimes crash
if CleanText('//h2[text()="ERREUR"]')(self.doc):
......
......@@ -272,6 +272,10 @@ def handle_response(self, transfer):
error_msg = CleanText('//div[@id="blocErreur"]')(self.doc)
if error_msg:
raise TransferBankError(message=error_msg)
# handle 'Opération engageante - Code personnel périmé' error
response_title = CleanText('//h1[@class="title-level1"]')(self.doc)
if 'Code personnel périmé' in response_title:
raise TransferBankError(message=response_title)
account_txt = CleanText(
'//form//h3[contains(text(), "débiter")]//following::span[1]', replace=[(' ', '')]
......@@ -372,6 +376,7 @@ def handle_response(self, transfer):
class CreateRecipient(LoggedPage, MyHTMLPage):
def on_load(self):
super(CreateRecipient, self).on_load()
if self.doc.xpath('//h1[contains(text(), "Service Désactivé")]'):
raise BrowserUnavailable(CleanText('//p[img[@title="attention"]]/text()')(self.doc))
......
......@@ -96,10 +96,10 @@ def fill_account(self, account, fields):
def iter_transfer_recipients(self, account):
if not isinstance(account, Account):
account = find_object(self.iter_accounts(), id=account)
account = find_object(self.iter_accounts(), id=account, error=AccountNotFound)
elif not hasattr(account, '_univers'):
# We need a Bred filled Account to know the "univers" associated with the account
account = find_object(self.iter_accounts(), id=account.id)
account = find_object(self.iter_accounts(), id=account.id, error=AccountNotFound)
return self.browser.iter_transfer_recipients(account)
......
......@@ -1290,8 +1290,10 @@ def add_owner_accounts(self):
if self.home.is_here():
for account in self.page.get_list(owner_name):
if account.id not in [acc.id for acc in self.accounts]:
if account.type == Account.TYPE_LIFE_INSURANCE:
# For life insurance accounts, we check if the contract is still open
if account.type == Account.TYPE_LIFE_INSURANCE and "MILLEVIE" not in account.label:
# For life insurance accounts, we check if the contract is still open,
# Except for MILLEVIE insurances, because the flow is different
# and we can't check at that point.
if not self.go_life_insurance_investments(account):
return
if self.page.is_contract_closed():
......@@ -1300,8 +1302,15 @@ def add_owner_accounts(self):
wealth_not_accessible = False
except ServerError:
self.logger.warning("Could not access wealth accounts page")
self.logger.warning("Could not access wealth accounts page (ServerError)")
wealth_not_accessible = True
except ClientError as e:
resp = e.response
if resp.status_code == 403 and "Ce contenu n'existe pas." in resp.text:
self.logger.warning("Could not access wealth accounts page (ClientError)")
wealth_not_accessible = True
else:
raise
if wealth_not_accessible:
# The navigation can be broken here
......@@ -1697,7 +1706,10 @@ def go_life_insurance_investments(self, account):
# life insurance website is not always available
raise BrowserUnavailable()
self.page.submit()
self.life_insurance_investments.go()
try:
self.life_insurance_investments.go()
except ServerError:
raise BrowserUnavailable()
return True
@need_login
......
......@@ -61,7 +61,7 @@
BrowserPasswordExpired,
)
from woob.browser.filters.json import Dict
from woob.browser.exceptions import ClientError
from woob.browser.exceptions import ClientError, ServerError
from .base_pages import fix_form, BasePage
......@@ -236,10 +236,18 @@ def check_errors(self, feature):
# error will be handle in `if` case.
# If there is no error, it will retrive 'AUTHENTICATION' as result value.
result = self.doc['step']['phase']['state']
elif 'phase' in self.doc and self.get_authentication_method_type() == 'PASSWORD_ENROLL':
elif 'phase' in self.doc and (
self.get_authentication_method_type() == 'PASSWORD_ENROLL'
or self.get_authentication_method_type() == 'PASSWORD'
):
result = self.doc['phase']['state']
else:
result = self.doc['phase']['previousResult']
# A failed authentication (e.g. wrongpass) could match the self.doc['phase']['state'] structure
# of the JSON object returned is case of a fallback authentication
# So we could mistake a failed authentication with an authentication fallback step
# Double checking with the presence of previousResult key
previous_result = Dict('phase/previousResult', default=None)(self.doc)
if previous_result:
result = previous_result
if result in ('AUTHENTICATION', 'AUTHENTICATION_SUCCESS'):
return
......@@ -1056,8 +1064,12 @@ def submit_form(self, form, eventargument, eventtarget, scriptmanager):
# For Pro users, after several redirections, leading to GarbagePage,
# baseurl can be back to Par users URL, when this form must be submitted.
self.browser.url = urljoin(self.browser.BASEURL, form.url)
form.submit()
try:
form.submit()
except ServerError as err:
if err.response.status_code in (500, 503):
raise BrowserUnavailable()
raise
def go_levies(self, account_id=None):
form = self.get_form(id='main')
......@@ -1380,10 +1392,6 @@ def go_life_insurance(self, account):
form['__EVENTTARGET'] = m.group(1)
form['__EVENTARGUMENT'] = m.group(2)
if "MM$m_CH$IsMsgInit" not in form:
# Not available on new website
pass
form['MM$m_CH$IsMsgInit'] = "0"
form['m_ScriptManager'] = "MM$m_UpdatePanel|MM$SYNTHESE"
......
......@@ -33,7 +33,7 @@
from woob.browser.elements import DictElement, ItemElement, TableElement, SkipItem, method
from woob.browser.filters.standard import (
CleanText, Upper, Date, Regexp, Format, CleanDecimal, Filter, Env, Slugify,
Field, Currency, Map, Base, MapIn, Coalesce, DateTime,
Field, Currency, Map, Base, MapIn, Coalesce, DateTime, MultiJoin,
)
from woob.browser.filters.json import Dict
from woob.browser.filters.html import Attr, Link, TableCell, AbsoluteLink
......@@ -159,7 +159,9 @@ def find_elements(self):
selector = self.item_xpath.split('/')
for sub_element in selector:
if isinstance(self.el, dict) and self.el and sub_element == '*':
self.el = next(iter(self.el.values())) # replace self.el with its first value
# In this case we have to flatten the dict to make it into a list
# Ex: {1: ['a', 'b'], 2: ['c', 'd']} -> ['a', 'b', 'c', 'd']
self.el = sum(self.el.values(), [])
if sub_element == '*':
continue
self.el = self.el[sub_element]
......@@ -173,7 +175,7 @@ def condition(self):
return "LIVRET" not in Dict('accountType')(self.el)
obj_id = Dict('numeroContratSouscrit')
obj_label = Upper(Dict('lib'))
obj_label = Upper(MultiJoin(Dict('lib'), Field('_owner_name'), pattern=' '))
obj_currency = Dict('deviseCompteCode')
obj_coming = CleanDecimal(Dict('AVenir', default=None), default=NotAvailable)
# Iban is available without last 5 numbers, or by sms
......@@ -274,7 +276,7 @@ def store(self, obj):
class item(ItemElement):
klass = Account
obj_label = Upper(Dict('libelleContrat'))
obj_label = Upper(MultiJoin(Dict('libelleContrat'), Field('_owner_name'), pattern=' '))
obj_balance = CleanDecimal(Dict('solde', default="0"))
obj_currency = 'EUR'
obj_coming = CleanDecimal(Dict('AVenir', default=None), default=NotAvailable)
......