From 39d0b5eb9d3a36e9ea78276dc9ac9c1fe6d5514f Mon Sep 17 00:00:00 2001 From: Quentin Defenouillere Date: Tue, 2 Jun 2020 10:55:04 +0200 Subject: [PATCH] [societegenerale] Implemented Market Orders --- modules/societegenerale/browser.py | 86 ++++++++++++-- modules/societegenerale/module.py | 27 +++-- .../societegenerale/pages/accounts_list.py | 112 +++++++++++++++++- 3 files changed, 208 insertions(+), 17 deletions(-) diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index b54bff6d60..91fc5761f3 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -42,7 +42,7 @@ from .pages.accounts_list import ( AccountsMainPage, AccountDetailsPage, AccountsPage, LoansPage, HistoryPage, - CardHistoryPage, PeaLiquidityPage, + CardHistoryPage, PeaLiquidityPage, MarketOrderPage, MarketOrderDetailPage, AdvisorPage, HTMLProfilePage, CreditPage, CreditHistoryPage, OldHistoryPage, MarketPage, LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, UnavailableServicePage, LoanDetailsPage, TemporaryBrowserUnavailable, @@ -94,12 +94,17 @@ class SocieteGenerale(TwoFactorBrowser): # Wealth market = URL(r'/brs/cct/comti20.html', MarketPage) pea_liquidity = URL(r'/restitution/cns_detailPea.html', PeaLiquidityPage) - life_insurance = URL(r'/asv/asvcns10.html', - r'/asv/AVI/asvcns10a.html', - r'/brs/fisc/fisca10a.html', LifeInsurance) + life_insurance = URL( + r'/asv/asvcns10.html', + r'/asv/AVI/asvcns10a.html', + r'/brs/fisc/fisca10a.html', + LifeInsurance + ) life_insurance_invest = URL(r'/asv/AVI/asvcns20a.html', LifeInsuranceInvest) life_insurance_invest_2 = URL(r'/asv/PRV/asvcns10priv.html', LifeInsuranceInvest2) life_insurance_history = URL(r'/asv/AVI/asvcns2(?P[0-9])c.html', LifeInsuranceHistory) + market_orders = URL(r'/brs/suo/suivor20.html', MarketOrderPage) + market_orders_details = URL(r'/brs/suo/suivor30.html', MarketOrderDetailPage) # Profile advisor = URL(r'/icd/pon/data/get-contacts.xml', AdvisorPage) @@ -497,8 +502,10 @@ def iter_coming(self, account): @need_login def iter_investment(self, account): - if account.type not in (Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, - Account.TYPE_PEA, Account.TYPE_PERP, ): + if account.type not in ( + Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, + Account.TYPE_PEA, Account.TYPE_PERP, + ): self.logger.debug('This account is not supported') return @@ -509,13 +516,78 @@ def iter_investment(self, account): for invest in self.page.iter_investments(account=account): yield invest - if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP, ): + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): if self.page.has_link(): self.life_insurance_invest.go() for invest in self.page.iter_investment(): yield invest + @need_login + def access_market_orders(self, account): + account_dropdown_id = self.page.get_dropdown_menu() + link = self.page.get_market_order_link() + if not link: + self.logger.warning('Could not find Market Order link for account %s.', account.label) + return + self.location(link) + # Once we reached the Market Orders page, we must select the right market account: + params = { + 'action': '10', + 'numPage': '1', + 'idCptSelect': account_dropdown_id, + } + self.market_orders.go(params=params) + + @need_login + def iter_market_orders(self, account): + if account.type not in (Account.TYPE_MARKET, Account.TYPE_PEA): + return + + # Market Orders page sometimes bugs so we try accessing them twice + for trial in range(2): + self.account_details_page.go(params={'idprest': account._prestation_id}) + if self.pea_liquidity.is_here(): + self.logger.debug('Liquidity PEA have no market orders') + return + + self.access_market_orders(account) + if not self.market_orders.is_here(): + self.logger.warning('Landed on unknown page when trying to fetch market orders for account %s', account.label) + return + + if self.page.orders_unavailable(): + if trial == 0: + self.logger.warning('Market Orders page is unavailable for account %s, retrying now.', account.label) + continue + self.logger.warning('Market Orders are unavailable for account %s.', account.label) + return + + if self.page.has_no_market_order(): + self.logger.debug('Account %s has no market orders.', account.label) + return + + # Handle pagination + total_pages = self.page.get_pages() + account_dropdown_id = self.page.get_dropdown_menu() + for page in range(1, total_pages + 1): + if page > 1: + # Select the right page + params = { + 'action': '12', + 'numPage': page, + 'idCptSelect': account_dropdown_id, + } + self.market_orders.go(params=params) + for order in self.page.iter_market_orders(): + if order.url: + self.location(order.url) + if self.market_orders_details.is_here(): + self.page.fill_market_order(obj=order) + else: + self.logger.warning('Landed on unknown Market Order detail page for order %s', order.label) + yield order + @need_login def iter_recipients(self, account): try: diff --git a/modules/societegenerale/module.py b/modules/societegenerale/module.py index 4bced7e618..81180730bd 100644 --- a/modules/societegenerale/module.py +++ b/modules/societegenerale/module.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +# flake8: compatible + import re from decimal import Decimal from datetime import timedelta @@ -53,10 +55,14 @@ class SocieteGeneraleModule(Module, CapBankWealth, CapBankTransferAddRecipient, LICENSE = 'LGPLv3+' DESCRIPTION = u'Société Générale' CONFIG = BackendConfig( - ValueBackendPassword('login', label='Code client', masked=False), - ValueBackendPassword('password', label='Code secret'), - Value('website', label='Type de compte', default='par', - choices={'par': 'Particuliers', 'pro': 'Professionnels', 'ent': 'Entreprises'}), + ValueBackendPassword('login', label='Code client', masked=False), + ValueBackendPassword('password', label='Code secret'), + Value( + 'website', + label='Type de compte', + default='par', + choices={'par': 'Particuliers', 'pro': 'Professionnels', 'ent': 'Entreprises'} + ), # SCA ValueTransient('code'), ValueTransient('resume'), @@ -101,6 +107,9 @@ def iter_history(self, account): def iter_investment(self, account): return self.browser.iter_investment(account) + def iter_market_orders(self, account): + return self.browser.iter_market_orders(account) + def iter_contacts(self): if not hasattr(self.browser, 'get_advisor'): raise NotImplementedError() @@ -121,13 +130,13 @@ def iter_transfer_recipients(self, origin_account): def new_recipient(self, recipient, **params): if self.config['website'].get() not in ('par', 'pro'): raise NotImplementedError() - recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) + recipient.label = ' '.join(w for w in re.sub(r'[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) def init_transfer(self, transfer, **params): if self.config['website'].get() not in ('par', 'pro'): raise NotImplementedError() - transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z ]+', '', transfer.label).split()) + transfer.label = ' '.join(w for w in re.sub(r'[^0-9a-zA-Z ]+', '', transfer.label).split()) self.logger.info('Going to do a new transfer') account = strict_find_object(self.iter_accounts(), iban=transfer.account_iban) @@ -136,7 +145,11 @@ def init_transfer(self, transfer, **params): recipient = strict_find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id) if not recipient: - recipient = strict_find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) + recipient = strict_find_object( + self.iter_transfer_recipients(account.id), + iban=transfer.recipient_iban, + error=RecipientNotFound + ) transfer.amount = transfer.amount.quantize(Decimal('.01')) return self.browser.init_transfer(account, recipient, transfer) diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 99ca093bea..e68aec4397 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -26,7 +26,9 @@ from dateutil.relativedelta import relativedelta from weboob.capabilities.base import NotAvailable from weboob.capabilities.bank import Account, Loan, AccountOwnership -from weboob.capabilities.wealth import Investment +from weboob.capabilities.wealth import ( + Investment, MarketOrder, MarketOrderDirection, MarketOrderType, +) from weboob.capabilities.bill import Subscription from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Person, ProfileMissing @@ -36,8 +38,8 @@ from weboob.browser.elements import DictElement, ItemElement, TableElement, method, ListElement from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import ( - CleanText, CleanDecimal, Regexp, Currency, Eval, Field, Format, Date, Env, Map, Coalesce, - empty, + CleanText, CleanDecimal, Lower, Regexp, Currency, Eval, Field, + Format, Date, Env, Map, MapIn, Coalesce, Base, empty, ) from weboob.browser.filters.html import Link, TableCell, Attr from weboob.browser.pages import HTMLPage, XMLPage, JsonPage, LoggedPage, pagination @@ -835,6 +837,9 @@ def get_pages(self): # "several_pages" value is "1/5" for example return re.search(r'(\d+)/(\d+)', several_pages).group(1, 2) + def get_market_order_link(self): + return Link('//a[contains(text(), "Suivi des ordres")]', default=None)(self.doc) + def market_pagination(self): # Next page is handled by js. Need to build the right url by changing params in current url several_pages = self.get_pages() @@ -907,6 +912,107 @@ def iter_investments(self, account): yield (create_french_liquidity(account.balance)) +MARKET_ORDER_DIRECTIONS = { + 'Achat': MarketOrderDirection.BUY, + 'Vente': MarketOrderDirection.SALE, +} + +MARKET_ORDER_TYPES = { + 'marché': MarketOrderType.MARKET, + 'limit': MarketOrderType.LIMIT, + 'déclenchement': MarketOrderType.TRIGGER, +} + + +class MarketOrderPage(LoggedPage, HTMLPage): + def has_no_market_order(self): + return CleanText('//div[@class="Error" and contains(text(), "Vous n\'avez aucun ordre en cours")]')(self.doc) + + def orders_unavailable(self): + return CleanText('//div[@class="Error" and contains(text(), "Liste des ordres indisponible")]')(self.doc) + + def get_dropdown_menu(self): + # Get the 'idCptSelect' in a drop-down menu that corresponds the current account + return Attr('//select[@id="idCptSelect"]//option[@value and @selected="selected"]', 'value')(self.doc) + + def get_pages(self): + several_pages = CleanText('//td[@class="TabTit1lActif"]')(self.doc) + if several_pages: + # "several_pages" value is "1/5" for example + return int(re.search(r'(\d+)/(\d+)', several_pages).group(2)) + return 1 + + @method + class iter_market_orders(TableElement): + table_xpath = '//tr[td[contains(@class,"TabTit1l")]]/following-sibling::tr//table' + head_xpath = table_xpath + '//tr[1]/td' + item_xpath = table_xpath + '//tr[position()>1]' + + col_label = 'Valeur' + col_code = 'Code' + col_direction = 'Sens' + col_date = 'Date' + col_state = 'Etat' + + class item(ItemElement): + klass = MarketOrder + + obj_label = CleanText(TableCell('label')) + obj_url = Base(TableCell('label'), Link('.//a', default=None)) + obj_code = IsinCode(CleanText(TableCell('code')), default=NotAvailable) + obj_state = CleanText(TableCell('state')) + obj_date = Date(CleanText(TableCell('date')), dayfirst=True) + obj_direction = MapIn( + CleanText(TableCell('direction')), + MARKET_ORDER_DIRECTIONS, + MarketOrderDirection.UNKNOWN + ) + + +class MarketOrderDetailPage(LoggedPage, HTMLPage): + @method + class fill_market_order(ItemElement): + obj_order_type = MapIn( + Lower('//td[contains(text(), "Type de l\'ordre")]//following-sibling::td[1]'), + MARKET_ORDER_TYPES, + MarketOrderType.UNKNOWN + ) + obj_execution_date = Date( + CleanText('//td[contains(text(), "Date d\'exécution")]//following-sibling::td[1]'), + dayfirst=True, + default=NotAvailable + ) + obj_quantity = CleanDecimal.French( + '//td[contains(text(), "Quantité demandée")]//following-sibling::td[1]' + ) + obj_ordervalue = CleanDecimal.French( + '//td[contains(text(), "Cours limite")]//following-sibling::td[1]', + default=NotAvailable + ) + obj_amount = CleanDecimal.French( + '//td[contains(text(), "Montant net")]//following-sibling::td[1]', + default=NotAvailable + ) + obj_unitprice = CleanDecimal.French( + '//td[contains(text(), "Cours d\'exécution")]//following-sibling::td[1]', + default=NotAvailable + ) + # Extract currency & stock_market from string like 'Achat en USD sur NYSE' + obj_currency = Currency( + Regexp( + CleanText('//td[contains(@class, "TabTit1l")][contains(text(), "Achat") or contains(text(), "Vente")]'), + r'en (\w+) sur', + default='' + ), + default=NotAvailable + ) + obj_stock_market = Regexp( + CleanText('//td[contains(@class, "TabTit1l")][contains(text(), "Achat") or contains(text(), "Vente")]'), + r'en .* sur (\w+)$', + default=NotAvailable + ) + + class AdvisorPage(LoggedPage, XMLPage): ENCODING = 'ISO-8859-15' -- GitLab