From c2504a84174e408f618a84e5d4197a52de202ba7 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Wed, 1 Jan 2020 13:40:38 +0100 Subject: [PATCH] Update of modules --- modules/{wellsfargo => airparif}/__init__.py | 9 +- .../test.py => airparif/browser.py} | 31 +- modules/airparif/favicon.png | Bin 0 -> 6715 bytes modules/airparif/favicon.xcf | Bin 0 -> 52704 bytes modules/airparif/module.py | 81 +++ modules/airparif/pages.py | 111 ++++ modules/airparif/test.py | 62 +++ modules/allocine/browser.py | 2 +- modules/amazonstorecard/pages.py | 25 +- modules/americanexpress/browser.py | 72 ++- modules/americanexpress/module.py | 2 +- modules/americanexpress/pages.py | 137 +++-- modules/amundi/browser.py | 34 +- modules/amundi/module.py | 18 +- modules/amundi/pages.py | 32 +- modules/apivie/browser.py | 12 +- modules/apivie/pages.py | 5 + modules/axabanque/browser.py | 155 ++++-- modules/axabanque/module.py | 17 +- modules/axabanque/pages/bank.py | 103 ++-- modules/axabanque/pages/document.py | 42 +- modules/axabanque/pages/login.py | 61 +-- modules/axabanque/pages/transfer.py | 33 +- modules/banquepopulaire/browser.py | 188 +++---- modules/banquepopulaire/module.py | 27 +- modules/banquepopulaire/pages.py | 234 +++++---- modules/bforbank/browser.py | 16 +- modules/bforbank/pages.py | 21 +- modules/binck/pages.py | 16 +- modules/bnporc/enterprise/pages.py | 18 +- modules/bnporc/pp/pages.py | 27 +- modules/bnppere/pages.py | 4 +- modules/boursorama/browser.py | 148 +++--- modules/boursorama/pages.py | 161 +++--- modules/bp/browser.py | 24 +- modules/bp/pages/__init__.py | 4 +- modules/bp/pages/accounthistory.py | 2 +- modules/bp/pages/accountlist.py | 39 +- modules/bp/pages/transfer.py | 18 +- modules/caissedepargne/browser.py | 112 ++-- modules/caissedepargne/cenet/browser.py | 70 +-- modules/caissedepargne/cenet/pages.py | 59 +-- modules/caissedepargne/module.py | 20 +- modules/caissedepargne/pages.py | 201 +++++--- modules/cmes/browser.py | 6 +- modules/cmes/pages.py | 44 +- modules/cmso/par/browser.py | 8 +- modules/cmso/par/pages.py | 12 +- modules/cragr/api/browser.py | 65 +-- modules/cragr/api/pages.py | 45 +- modules/cragr/regions/browser.py | 37 +- modules/cragr/regions/pages.py | 6 +- modules/creditdunord/browser.py | 16 +- modules/creditdunord/pages.py | 8 +- modules/creditmutuel/browser.py | 2 +- modules/creditmutuel/pages.py | 101 ++-- modules/fortuneo/browser.py | 1 + modules/fortuneo/pages/accounts_list.py | 1 + modules/fortuneo/pages/login.py | 8 +- modules/ganassurances/browser.py | 131 +---- modules/ganassurances/module.py | 34 +- modules/ganassurances/pages.py | 304 ----------- modules/groupamaes/module.py | 21 +- modules/hsbchk/browser.py | 0 modules/hsbchk/module.py | 0 modules/hsbchk/pages/account_pages.py | 0 modules/hsbchk/pages/login.py | 0 modules/hsbchk/sbrowser.py | 0 modules/humanis/module.py | 19 +- modules/ing/api/accounts_page.py | 14 +- modules/ing/api/login.py | 83 +-- modules/ing/api_browser.py | 1 + modules/ing/web/titre.py | 12 +- modules/lcl/browser.py | 6 +- modules/lcl/module.py | 2 +- modules/lcl/pages.py | 12 +- modules/linebourse/api/pages.py | 4 +- modules/n26/browser.py | 12 +- modules/netfinca/pages.py | 51 +- modules/oney/browser.py | 30 +- modules/oney/module.py | 16 +- modules/oney/pages.py | 19 +- modules/opensubtitles/test.py | 2 +- modules/pixabay/test.py | 4 +- modules/pradoepargne/module.py | 10 +- modules/residentadvisor/test.py | 4 +- modules/s2e/browser.py | 20 +- modules/s2e/pages.py | 120 +++-- modules/senscritique/browser.py | 2 +- modules/societegenerale/browser.py | 90 ++-- .../societegenerale/pages/accounts_list.py | 22 +- modules/societegenerale/pages/login.py | 26 +- modules/societegenerale/sgpe/pages.py | 25 +- modules/trainline/pages.py | 76 ++- modules/transilien/browser.py | 4 +- modules/vimeo/browser.py | 215 ++------ modules/vimeo/module.py | 75 +-- modules/vimeo/pages.py | 148 +----- modules/wellsfargo/browser.py | 300 ----------- modules/wellsfargo/favicon.png | Bin 285 -> 0 bytes modules/wellsfargo/module.py | 71 --- modules/wellsfargo/pages.py | 486 ------------------ modules/wellsfargo/parsers.py | 321 ------------ 103 files changed, 2260 insertions(+), 3345 deletions(-) rename modules/{wellsfargo => airparif}/__init__.py (84%) rename modules/{wellsfargo/test.py => airparif/browser.py} (55%) create mode 100644 modules/airparif/favicon.png create mode 100644 modules/airparif/favicon.xcf create mode 100644 modules/airparif/module.py create mode 100644 modules/airparif/pages.py create mode 100644 modules/airparif/test.py delete mode 100644 modules/ganassurances/pages.py mode change 100755 => 100644 modules/hsbchk/browser.py mode change 100755 => 100644 modules/hsbchk/module.py mode change 100755 => 100644 modules/hsbchk/pages/account_pages.py mode change 100755 => 100644 modules/hsbchk/pages/login.py mode change 100755 => 100644 modules/hsbchk/sbrowser.py delete mode 100644 modules/wellsfargo/browser.py delete mode 100644 modules/wellsfargo/favicon.png delete mode 100644 modules/wellsfargo/module.py delete mode 100644 modules/wellsfargo/pages.py delete mode 100644 modules/wellsfargo/parsers.py diff --git a/modules/wellsfargo/__init__.py b/modules/airparif/__init__.py similarity index 84% rename from modules/wellsfargo/__init__.py rename to modules/airparif/__init__.py index 6f54e99b2..5d0e746a9 100644 --- a/modules/wellsfargo/__init__.py +++ b/modules/airparif/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2014 Oleg Plakhotniuk +# Copyright(C) 2019 Vincent A # # This file is part of a weboob module. # @@ -17,7 +17,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals -from .module import WellsFargoModule -__all__ = ['WellsFargoModule'] +from .module import AirparifModule + + +__all__ = ['AirparifModule'] diff --git a/modules/wellsfargo/test.py b/modules/airparif/browser.py similarity index 55% rename from modules/wellsfargo/test.py rename to modules/airparif/browser.py index aadc5b32d..51370a6f9 100644 --- a/modules/wellsfargo/test.py +++ b/modules/airparif/browser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2014 Oleg Plakhotniuk +# Copyright(C) 2019 Vincent A # # This file is part of a weboob module. # @@ -17,18 +17,23 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -from weboob.tools.test import BackendTest -from itertools import chain +from __future__ import unicode_literals -class WellsFargoTest(BackendTest): - MODULE = 'wellsfargo' +from weboob.browser import PagesBrowser, URL - def test_history(self): - """ - Test that there's at least one transaction in the whole history. - """ - b = self.backend - ts = chain(*[b.iter_history(a) for a in b.iter_accounts()]) - t = next(ts, None) - self.assertNotEqual(t, None) +from .pages import AllPage + + +class AirparifBrowser(PagesBrowser): + BASEURL = 'https://airparif.asso.fr' + + all_page = URL(r'/stations/indicepolluant/', AllPage) + + def iter_gauges(self): + self.all_page.go(method='POST', headers={ + # don't remove the following headers, site returns 404 else... + 'X-Requested-With': 'XMLHttpRequest', + 'Referer': 'https://airparif.asso.fr/stations/index/', + }) + return self.page.iter_gauges() diff --git a/modules/airparif/favicon.png b/modules/airparif/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..9f4946015fc5b9da9c47196ba32b20e59f3718dd GIT binary patch literal 6715 zcmcI}XH-*Nu=a_e1PB63M+8KWUV;>8_x&QzHzd`^Q z5%^*5Rb~(Vh;mX;&{kDYVD2_?M_YuTaY#S@!Ll+ydALm+tPwzKX~vFKaH>M6ez~mY07O{kXw#F3A1r@y?fK zmaR>h%>~H0a&)=d4{mxUt*P^a&FKYte=QHnW}7I|R%akrQ1 z0dBh6i}+;#wlUt7e=3SQ}{`DREkEJP2~RTO~> z{Hx$~X(~8E;)ycx1^}|F_%{T|ent-t68orXC=q{!(~xpO#kY4?z##@7C1W224>vbE zcOO8(%kI99oh_@slaC{-imHaTQTSy#0HD9Bs(4pFU}huNEa{q2$lK2GaT$?EpGIo@ zj7&EMEB_3?Il44$OvJ==X&_R;;2MkqNm%i&_{#MF)OA#$K1U5wTmDk|4H`HJO$2$p zaASNl`84X%fF--S<>$(^U#%aVx4iT7@-|araxFIOHru4eGq-q$#(&AmjGsw`g`CNi zwvAHhs*%JMN^?iUxZ;SAI@ILoLM}noY9AFg#qN!DQ&uU&D>(%n_gmJ&+f@Yh>oO<#a!3G1)JIt{s} z)fQ8{M5GWtoC;EJlK3LFWDgOV|DG~}M2t#L@XrNn@9N4PR3EtP)e2GqF5zezoMbCQ>i zAqDqU1pQsDlN92=mhRG`XN2&P>&a&7@@n&t{o)RvPH*)}#`)7%_+9GMKipT7^5sye zW>fGG)^{P#y*P3i#ll|?Y5QIcZnAt)F#!JPt1Afsc$Zo&TSOaDat&;MFk>p1x*n;5 zVHql~kS5ly{&Tus%&CBQl?9ph4Y0dKBmf#}lMQ@&YTaFJRO76vc^9)_UuFnBS3~=+ zeJyXeb|nG+x?lStBDl$-#W|}c1t-o>!AqF8!4HN%lL^Uxy~fkKD+LVO8vvGfR~@ry zT_!*m9B~Uo{knp0`L+vWA-2h4eNvf20b#Dof~@XpTbeYet~InpVMO4}qI}J>F(o$? z5LJ!WWCQ`?;+vpYwwn7T{jbraO$|-!Z?NzIA(oyv1lqZ&b8fG_mH9;TT@u`Zo+n+B zc-CyB#VeVOVa28_t<8r2kyUP<*XJcmQHGQybByyHY*TF{sUPlpd++XY1&XAziACe}{$Vc4?!CwQIg_&^5(#t6*%6{fLk|af+Bt`!b1* zsLDXCX#uT#P*+XbQ^@w966QKMzSCQ|)Zrc0!w-hu_J=XFY~}e=cQvztBsjANK!&Te zrKmD~#pnC0O~^1$)WD!|in*h={Tch)ZS?j}eC8xhdK;%Pk%@NroX z10pg5{(40Fp~#-DwHh}=v+WNP4E7NntCd;YZdWQBL+b?K)$PEe2CBtzBFlw92^2=) zJrH+C2%I@E1_jeMjW`FOR{8*N#T7!dsgqmrMi>bqZYp>%0xbF6~8gEiIS` zB0yZBBd9Cz9q_RgtN$QHR0&*96q=;Fyeb$>Vbzvy9UPo2WX5HtW_JI_huk66Xr zXD~ZBS@YLE*bjL1gwSTc@0YOG=3wa%bp9PxbE{fiD(7cg)J-X%=sT9atgvy%ItEdMQ{ z!2RMgiI8QbqqLjyIL+JD&XIE36zpy{@=!Lr8x;2sbGIC#K(7RJDwruMw#J&MNE=wG ze<9V^oTKEKVGVu-924OfjQuqUPjok)BX4B^3$5-k_Ax!xA3;}wcpw&9tm$gVsZ=3; zve4luyX<&v5b-7k^eaB&j6kx3_57b|s5R9iYL=T4gM=VY*5JtTQ*CONk3r7ZHO|$R z+WTr;q*gYp3J9NE$dNux+%lA)U~RzUhe&$C8VNlH!baZ?;-a3;F9nI*->7^4!)KP4 zOeS4TUW+4nGL=#dO@$|sDYVQ*i6Dc~l{V?|Dsqg{MI|ybe zSs1nRB070V_+Ek)QZiSbPcnAr#agql&-Lz=jpoB}l@G6YdSM`ae9Tw?z;&jTNrON4 zPE$jp|MN-*D1Mo-YH)Xx;q=YiqsUj86ZB@5oHbQ-6_Woz9|W;=0f?XbG7h%{5jbQ* zXN?B#6Fg7Bb5sNEBRNn8mWDl7qOLD^4V@5r<%qwCnn1^F0=xRz?ks|7k82}mtyUQ_ z)dCZ>wSt1xKQ3MKA<6|@aEb|Sx6(11EnEl?*W=M@f}pa($~aOLOz^T(K%7Q%(g1z@ zJv{u3=;YM}mvR~f@R8&+fWUND;)}II4xlt+*vEqb>y=L(#Mp3YVo>NKo-7Je_5efb zPhPyxA0Q6>{jnGbx}_LUQVV`=ilQHk46;>&bxYX(R&+)7$B7B?9Ob zGyhnq)Nmqp6%q2MXTW#1wRA{sG|9=C1HoX!{A>`D@3|7XG31vwri#j-X=m z2-hhR`X~1?ChuOo?ofnxcNi!vj>umQamDk40U3*x6nz+}gJ-W86sDDsrtcp7?aK2s z(0Ml^AyHZV2TU81rA_!)r9x2L!gu^|MfC&Il*El7k-#Ba1ksfbl^mH`ztauzE7BvS zX}Mnpr+vV$Xaaq#Dg2*}FA#T?+5fU~lo5IJJ+TQGM++O^1A?Uzt~F=-3bZZg6Yy3= z@PC~8`rK%cN6ujRwW(k%XiYK^Yq6rOS6Nla-UNZ}vo8_g*fEi{Jv^$PcyOj4p@-s> z=*H*vS8~?mvUR5ludUCmCLo+xQMB=G^QF-CL=DiS0O(`PEuNXgNYf+`j4t2?O+@~; zzhJB~b@c|=u~)_?n%|!Qt&yAuI8|y5UM&v0oW+}hzLhd{{n-8-cko1|205h{Z;l^g zL`qh(BkmY~=+Mo6yptdXj;%5x`4};FhMRfmesj&L|15Yn@sZOAwkrnK`TI&0pC_!R zYAqBmb`;PA%I=rt2k)>~4rV2`>}ITF_viz_30C^a67C6rKEZHGZ*TXrMR|sF;Z2$Z z7#WS0R?^w0Uj^PW6>GFvZ4^Wrh?Z4^o#>4hl!qQo{M0hYSy<5%Jqnfu&~_$5B1DL+ z+Xx0|>Qx*1JnB3c-f9)5u8IspE>o#DdNwb$KM^LC@^*z)%6nB$oo*BV0=OTp>`6`g z9`?(m*8fuqSdU|6B>aD?lvTQLk(%T*g+64+uefi^7a_CZBK^r7UrVYWuG7I%*t6xu zvFVAFo`+t6e`C7@(>$Y3T9( zTZN3FM;jj^VFbUDg~{nLmr=-_V|N32wtkB*Bi+`3$F*}S*ZDxY2;coY5`1}6VeFdd z59YWd`E8Lfguna(CZ*07(oUTHs|2Yfu*S@Mt8tpsPb2K!zf zEr?r1rqR}minh6VtzQyz|NK3+3!j^pKdr-TJ_wEEpQtWuSje1Lam;*N8?-PILq zAN=^m+I|CcvY)Z*ON$<1lYf;(zITY89@%NP_QKgn$NXy6f6C$QzC~XSJ~6L()f|@d zNKS1Gwpm>;jdtJG&>|276M~(bl`3(0-ArqB>*-kb*vFYXIexSJ3t1PwU@CEpK#Qy= zeyimI^nI*3sPSi~gsm(teAe>}6D+FxN8SfcM*dry5wi+zxs-1*{r?P1wUQLO@wgB$8Vo@uA`!uCQi_p^^CDMVQn*aA20KQ$aCkk{5VqJ+j_uHf#%mW z9aO_Oa>Ai!OcI+j|M}1!vh3NtQ85{%9gd{~w5bVQTK#GGc2_=})$t9b|0M?sisJk# zF?(U&jUXSMcXTy1*lU~g^Wbilp}pQ)N2X`|53KKGP$d-$i_V`i93fM`Mm7uWM)E_) zegET^gzxfL3bU_rwOgNx`JIRB{Vzv-+LZ8a-bU*q!e4sGSiLfTM1my+hWXjbO!c}E zOR*v~YWM#klqr}Oo)t?cOK(;8CDq8A1~!3fuyG*yIYn|&EMZeb^#ns# zy(M?9_VB5%8<>J6oXYAa)ftVx^b9=aE4GWf(;=&8PbOW?!F0}XGvNlOeo9Finv)o? zg-h@33%;p+yR_>x4&{YG*r;J_^05p63O=A02yevCIuys@s!Y>IyNq7QjB zBA=$Z+hqznn1D&Odm&pJU+-weMs165dgU`0K83y3d$O=GT^Cr&4<0P=l(9CU)qH&( z;#0lWH@|iEE+W+mCfMI*gOFV-z zhMZPaHJgs!_PZA>y#32GM`*y|qwC$X9;d9Meo8R`<7)w`zA(e^9-JyZRX014=LEUDl;|d=w0uZ9QrPK`Q9v!9ils7 ztsb3`3;^j^iL(o%v$j2Ayv%Y+XGZ+?8%i$pY|y(=M)q%IMB8Nt+`kaNZ{jp6O7}+8))!2U>mNJjAUw!lx`m88t3cJ+MBRI1X z0JiDMYP&Up;GZp9tTu|7r!xB*GRB=hEvq)$$%pwXb&RH9!z}1RuDW|yfBKNVaHLQD zD(-nUI7{DpRLOC+P$?KpW9Gk*a4_O4N!H-M z{)>ru3vRH6#?cM|lQ`XSf*MQ77?Dyeu5&Af#vt_YEydD!*h%8{aj+;E*u;8Fh0N>Kw1ZGa*wpaWSpW4$ z*AC|nQ5u&(mem=;GhG3>kQI{CgLh#W-(MZd{LXB?=vJYss=BhNw)4?pzlSE}`n-SB$d&SLQBSeF6k6+`=&_NOkMGu`|u zuFg|?Y_yb`e<3O6_WHGR_zOP^!E7^eo9*|)#wC`oPQU#yiJI{*eCZNmfK3FaPr`XS z$OHC_h=qi`>c9D{*(9Fv2Z?W#1vv-w&p)bL^*_H9-uh9uf-XPwTJpB70XE^^Bm_D~ zyk>d3yEfhY{NON{bgR{c!nycx^h=kZp}kucb|C$pD^nT&?Sf$`fr zZ_Yiab4`4eK@tRPv*%Yi=XmPOlpgPdC!XpIY)W~as=}WI+4Ny!{;F{}N_PWpomlpb z^%<$F3kKh9Ka9*52uz7icZ-?F7vyE{~K0$G^R^#H~iDykNQK zV4fnQVKkiOTtv6LxBm41%cnqYF6*=(?HLW8zYtv$d*lA==e(ie@w;A=Yv;!z>gN`C zI*PpSt1W7RgavLB+sWCGgZ-oW&Z?5&-rkPd`{%M#XJ)3exMQPkRE+w)tb2RRL6>Gd z!dQZWn)HL=V6r#ZKYe1Q18u$?z_@iSjMkUDx~>WyrCm_^U`$``b{MoMjWX_%3`6)Y ziZ^4aWtI8Ye|UNYQ@c#LaWVbYU0PK?jG_e3^{@$1c6wKNgfLOQR%8@G2(s^ zZcUW{TCHs$cd0w*r~$Zn)7?Dr`EwiQx5_bN4K>67WkP&UebqDMkncA&^yx6!_5&tkV31K4ucZj{uCHzkafG@wJ^~yXPyN|;miY0HxUJ)6W5OH$^S73~# zFRLqC+ofnX(8gnLC8?E+jfJx4^P@{LL<76&Djv(zXv5ucXGLis9ytFg8-U;~Hk3tm zo|tR6Sy)X`4_dMcRO!iH*_16y#v7J9zE2zvNbeBiFSHz5BMt-V@BVK0|sf9o!cH=%j8BL*yq zLLSzU=0XnEX_thi4j!y|@z-wry8Ec#6o&h-aplM&5MP!N@bSTV6Z_~8(Wl<&%x)n%7H$`jHo)O_wg5I!KkT!7$b7BU)Q8ey!@7_H=%~aaZf%d**yV==U9M_QS_pX%O zz@LjY8jG!<*)tt0*) DEoU*{ literal 0 HcmV?d00001 diff --git a/modules/airparif/favicon.xcf b/modules/airparif/favicon.xcf new file mode 100644 index 0000000000000000000000000000000000000000..4bc6bad44ab11a9b82ff32aec5020c24e5eb0085 GIT binary patch literal 52704 zcmeHQcYGApzCSa&Np{lh+;t$yYF7?ND)c0bKdvNObT4C@4fra_1(`|_|5nCJ8jR-e9z2pW_~$m zQtC8EWX|}Bk=Z?BVhACEApk;_Lc$;dOO1e71|W$EKFuIzNHC;@p|f($A-_R_CZXxB z<1?mBOG(d!tu-A!z?m{OGi5^L^wiABk%NW|AIWk;wW6uh#!gDv5;-8lk$QM*WIKtK6piA(Kz0NuHuJo9K~E^nfOMU=uy4i5>#oLMeR#$Uc4$_T8-gy9^yb+A|&HSYO*U z(VMae$@L#`eZAI%lB`{P#FIKke{A44IT)&@rF7&_iS zmw|2rD|Ni}5d$Y1m}6j>jvYSJF?z3oZ|O)#zE2-QJ-rR-MFbC%o<#H{5+6@I@$=Y; z#bYD>o;VVqNh}HUFo_{Sn)D#So*Sh*3GviRHxlaULc%?rNrWbyNHfokQlH1lG3iRe zG|}r>Jp%gCiC8@*;;W&AJ_Yc;|GW0P8JwZu2?b9yNhT&*FA2n#f;UtmRtnBgaD{>^ znt&&i3AmvNxIuBOZz?$U6uh9|iYC#dIgKJMXh+iWro!@=uyRaTy?<6GU-%qMvV&2b zNOO+}Y(qMDT9PPFbD)6H(45@mX+=7E!Y~p9672~l-61_ZK_tc#NMa3%f!#w(XZazT zbn}FgE}n4IX^!?pE%W z=|GsY^mHUGJW<-nv3eN$*{HN*RIm{mkKcm54)MXLHG7hX$&-vxoKIB3f9ZeXF>R}j zFMsg8B`#M1v=xb>%}FO3LAuZ|(v5QAH-#scGi zzL-tCZbiY_=?3u@vecVwwbFVWYzuo5L80Zzq*f_?NhwMs*WAjEPxmSD@#JceGJns> z?=~ypc5+oIT2S@Ir{@>DnLf9C^`3YCwp)SjahJ?nw|AT3me9`CIV-DQdTsw&#cd;3 z@=8{1fA!=Szg(*O-QjX5{B!${L3Aj;U&4w-G8+E&w9>Ww*?eM1>Rrc<&1<>}Q9 z7xuqHUia=cxfAxkOU~c&#@Xev8T;Ak%3rzr-Or0<=<>`Q&z3hn+rsozIcwI!sx`6` z`(M7AJ7;zEi`y5-PV7f#_S`kkA3Xi-FYXLja%R!eE!d4et(RdroU`Vwec{lD|FxM} z8JXE7%eTIA;)~VHO39v{zhJ|zBWLEiSiPKCIpr&>54<DXY3z3d6C8kp|CLwO^Llv5?{ir&d%S zd`M#YfZmVhu6o7I^xjhP-A~VU*=!QIBH8;t>Tr9?C)(}UZ%N4yPb*rv^Q{j@#lf;C zj+l_UVDo{~GbLD(ec)q`k~O>DQJ7^*>^XActjetivzgVpXU{=TxXRb=eS+1KdJpV9 zI<;WwbAz>dy#^)T^T?E03zA@2B^rl)k6zFOw)MmBz}{oFz0pRhT~BS)SQ))EjbkMn z?Z!4(+CP^t(h+_R#d$0W{tr9Hv2$BI&S&|;UZRS}dJ)z16*lHC8#Z4+^J(0C&P=xP zJ=b*OZTY>BOzKSgX-P7nib&~wi9{Wyid4ODS3FVM{c5J_X@C64JRhohhuDcKEH_bA zorugeKbxpm{U!^!HNKRd22msYsCwBC%xwwf4=WUZC_kAc^qqczg1@8S3E&UFm(kjj zXYd6_2FDY65VuPk69_#5d=7XVxEJ_7@E{Nc=`r9r;0M63fu8}-qk(S(G;jgR4HCM z5=#P*hP>hc=**^t^%}Eui6JyOP@!&k)k+T%q|jNq#;oh5JE0Z9DA|pKD0Fr=LT802 z@V_^R@{M+TLSFc?B<%hFwWPPv9 z-LwC|8*d$Z_w>h~e(}xUe)#3$?|=MpScxLvw<%n|ZO4myUw-Y-TgOg)aOSfwzy14< zzbq+?CO<0kR<7N+rTY0@`(And@Y}~(!^f0P+;Y#tB`emff3~W6`vxViBk@s8 zZb1>;-Xgc#2S2tWJ49}ib^(mSL8{j*ikZ*?i_$^6DAGb$xI|KS2knr;n$7Z1H{VCk zDQZvXIE&&7%c8VLOZ6I~0CYES_MJMy_E+vi89Lh!TTLtU(`w#<5>~~c>C_#Ga!j`> zk*su~uG5xqsaUTe&B#ZJ`I!5595e@Zsf3Gq@XPmQ`)7es0Ik7k2I0`_jv=9C-cUp~FYsK6>oL$y4v2{_x{7 zXFomn#g|`ybEd@IoSZ3g&+*J#u&{E;(&Z~wt$AkM`i;+SuG&`p-19qM+`VVtOE15A z;Pp2Sy?JD*BH-zWeqpo9AUgyWE`IbjD=vEeGAQrbxMcaV%6Szf%4}siD4z3QU!OUC zaQD_Vi%JW!VP|e!x@c}mer{%FPW}v#+3Amq87QySr%xfp6(EE3Xcu}q(*q&YQLiye z7m&V3AZqtOD?kT`uUn2F8EM)!7+Rx%{COcVsE=}>eiqY1HR(c{%TdHDJ2Fg;z>K=| z3|R*E<;s1pFebkL(b>=% z<5;w6Eocm)g8s)(eelV-uf8jh!F_r5ocUn!`e(O3hp_?rpLp+p4DQRBdHHTIoN;*b zwjCH62e->kG$lLpaW)>8I;Y(LOvjGEVcQL-?yqsH7q~zyK&&tl3RajgxZ_%=q ztJf@+UEscKatgBGtaUk^vcrs@!5j|i0OuWksLGC5qYjQEl{zs3rdecWS!6ICWuT`S zdWH{6$pH6@EHYS*GT?rL#!4eK1DFo(GuH>UzYN_8eJb)EwZih3!G7f40Sz2yVTG8T;J@sgzo>HYlBLU*uUNTi^_pkau3KL}8X5)z8w>j? zWbj`;Q?_E&>Sxw%*!b+0s%_hMJl`-5K;*aHe&_hfQ}3Pr@S`)wvz*|+%b7QGc7CC| zsHCi7u4n$jMT@n8uy);sP0w!GTD^V8&KGxK9F)o$|0UWLyobAh5g;*P*Gky&nyX>M z7)YEo%_UD6H|C*{Lx%SP=ldT$ysv8IoLT8lj~dbwcG8R~6CNKmq+e2EpTT{>ZCikZ z5rB65CE5wxhl7GtJ1m9?ItEJM{XYoCKuC}TjzfYZ8l!D(y3NuW1>Elwj26LuZA)We z*};E_3HUEX5mk!wjTK|eJ;WHHhbq}I1f&V`Dpx$SY3q*N`(J!$v*)#JI`RTsgDdfb9VxzE6R?|a~}r&3bW(kHM^ z`$#4`yj8mxUuTzWHW4bW01{0G?`?jlYn5~jV?$uBoEa$?0kOKy%1}E-ftEv)EXaf2 z8P@=o6Nhmy&?14=Hj9LDaJ?3XlCj`AQsH=N8qWR_&i-f<{6`(|Ujol zX`A%my>NP;`GkO#7=N}TTeUqA8YSF@Sef<4jVCY z^n(vS`uLMikDD-Y^3-Xlo}H6B69c2Tv|{eO`435q|G3UkS_P+RK5!p!8Sri37T~AA z*L|saUW7~eF_bG7j0b7%<=U$?YE5k&T{s$ye%hbXWf%tokE&{Ity=p>jraYjafFV* z^Z7K4rZnIjU>UFqxCwX!_&V?#;8_b*FIi&YKVP&`+7c}Wqs1P;F2G^Hdx4XIlYm7) z1-Ksg4DdDJKHxdv$Nunog&uIie}q=o0+)@TKuWsa$99|q3sOzH+zSHBlbYtL0zuN1+&=w@&_qY3>i zf>H@MFPy3$3Q+ooQ<~igBm?FE_afuBmQjRuLydJ&q^|pZs+#DvYwsLCapIjXeW>^B zI5V=3!&g3KChGkvbEBnilo^{y1f?45Ktro_Y5PD z;PX8a)*#_c)@nP@|1gwaeeNV`>pEyZ3P@Uv!h>+Fm;!voLg+-`O4x^h6|j@oeTXTN zsI6*|^0i`;H_Vjn51aZQ4Qrg>>AMOIgB2%$gB9>B!P$Cx7BGGMTxrNb61YEC9xXTEK)maH6 zbWRs+yDFh@5xOcNBtl_N*L;XKP;rOA4pr2kal!Dl0~NN#4M+)^aQnl;&5FAv{N5l< zYfXegUnP*~MYj;OsL%%nyR&Iae3bxH2~gBRp)FBE3(bl@q2Oi3$iF9+T(oq}#tr4`=2le|A#3LHHFHd)c4lEgVWD5awZek@!c6B>WK4EC3rysv z*>iNKrQT~<@Km>s$n6;0z7&_BSw8528Mi6QB#cFrzcg&<(0hjr89aEJ4@Jhfe7}Qu-7U&BuB@y~mrHbtuJ1Clva+Y= z%$SjzH*@B!+4%(pg^IhVxVS`tKM2yu9(MnrQAN9h*sb3|XtxkexpwST2P^Gf-{5C$ zyV-3={e8y{)^>W1!LUX*t zk7m&faFDsz;3RHMR0rc^AZ&1QGP5&pCxXKyYm`K*pq*}I;!Y()w=B3bp_z9OGyLCR z83c5Q%tJ<(r~}qiW~s7E%(>#s%E_7G!oOS;tD8D~x(jriBIkO_3YC(1m6bCV&Rns= zg{-u4Po9ZfOLJs6@ZX#vIE0Kk$HbJUkT)hJ1&;qu>3Lv-vB%jCFL=;kWOTNPsMGKh zOiq)_o9TQU1VaU3xG+o@`UhA!2&BAQ97sDOW7z*hN$Hb>p$S`dj<_2a_`XLBkmh{|LEcNc+I(af)Q2YsnIibBCS7H!QVGdR9lwYz%uDR zrz?|j91Lgm7`;)Aoq$<2eZaAsgc7ao-cLOF)YD_fO-Pvt?`cZfwDb&z?9}Sd$kpl> zDq8&vl>S_@-!pi~&|$;TxlyAZeCXjv9)0X_z5e+6`abFEgHm(iD~Wa?UPurU#UwG= zr>BrCBzY6#<3+m(4Rw;hsen!rO$;YpBrvik+jf<}NGXKS_y|d``M~Y4hr$e%RB{){ z7#}EM8^io^)K2u5f(RAi)yIL8u@+c?l4_H1Yt+e4!YNXsUBF1Y%?Cyx{79T4r2r(~ zhIVac2~6x8fC_e;9ph~#*sKt!2OYG;m(f_=vJe`F*1Cg=a7=?GkPl8XsAxl~4OR@Z zR4GC1eZ`*Cr*GeczKQ+%C4f_Ly?Q5rim_6^#~v6u^!~@kjqUsN-KnX`EbD>C27n}S zCYvC^4VUabk~c9a9y#%eN$|gavh`;i7&ZEV2iaa8^Td!CgNSjxP|t3P7ZXJ6TOnCZ z=nBT!LB;!pk-~k~MhqW5Z0NoBnugGBNFRy9e|B>8Mpaa`r```%J%Y1HmngVm@CJgf z!5f$WZ{Rz_D|ihU+nlJ6swk`O3r5z;4)9~cQMi{End>5m>i0=BrGwcuuM?$Tg%P!L zO=lVk@8d`)y?pV=S|xqlljG%z=ROTVDoqQbwd&CsF(xai6I)NBwXxz`t{|XVW4Ar*iIdz1P5CMDq z5fN__Rbe-r-w~oHm`rBT$Jf`-&&OBvsq1!)BKM~U|ETrWsot8}8n0KasmZojtX9!q z3=jiDg2X^Epw@r*mEXNMtU{d_@#^tFdTnt~aBxUys2C=MixJjlVuTPbgbATSh#EXu zq@SUrcza&n{;l8se&N#P3zx23yL6#$+xoPy@bHLc&6>Ap(Xv&m)?yps7NL#MT4?oS z%N8w~H)|FV9v&7NdjGn$mjzKY^$DSu67RSrEIfi$K^?KBxs}jTZ1Dpt8WA2A78()~ z9MrsRn~Uu0dYLZzU#h**5gijl%%PSrQ>Yjs1Pehzpb#MV3s$ei_irXr7o3ooXu^3w z=xOTZ4YpWJ)?j~=RkR3xqOag1m_?H+;x^My^f3u$lNe_55q+5RE3zwizStMuNpC#1 zLo|g#JLzvMO$uTuG@(UF#H=LYu_mH%@Hi1~`QfqPoxj3_J;`Qo;`HawG><)dwh^_o zSS(kSQf!Lz;>hL7vUTg1E?K;M!{$mnVfNx|>%-iYLbwNo-N^;`CwQm=ZxRm}tG2FJ zy2B3>;RFgrVzIZRD8D-qhXxFzL%WmUF?4Xg-vzzXa&KIrve0W;>$Ub)k2=tAnF)2E z-!l3wClX&d0q?)iUrid{Us1fB!vmj8_V^IJ&dwBbR5@Fmp=RQa@UlB+!J?AlqVmdR zZbuyMD@0t+es^SK;Npgj1083#zBubI!maPJ6)P9X=qMh3i7p{i%<^VuWppRz;nop! z_zfMG&~Ygq9hZ7(uf$9eI;;t`qtg;PEuqsAIxV5on$URtRgeDEhxTC4YtZFhiDDmB z>LvD7lj6|j5q+L~deq4K9vnAu1fJ@cCA?;{i&CvEF3x5{e{oTHyk zaMX9ri21-hBipK1JK`lcRSya5?Kz#ww(4(f*{c+FuYmVjl#XFr^~VSkzX%fxq9`t3%_5uck(UfvM5XrOQH6K_dtG#1eKaIPJa**~k^dOY$)gH+ zR3Tz0dHm$RD^iq472Yh?qiIYVk1FI*g*>Xzh_K~Rg*>W|M-}p@LLOC!$D)WHM1&QO zD&$dxJgSgK74oP;9#zPr3VBo^q62wUA&)A&*<+>pG#*vRqY8OcVZ$FY@u)%`Rd};N zR~}W!qY8OcA&)BLQH4CJkVh5rsKS2|U5Mzj0z+6JT37}}zJI+9-j)c$mcU?pwi!kI-g<(b{ zD-O>6bmUQmJgTrWiO}XI>_WmlT?vmW?BEdy^)x4Ud0JsEx-cTbj`jqT?vNgy zAQEFl6vhH$VE53{S$>End|lvw*9?B26JL)V>veY`%{?Np4e8)%NuqEp|I>~sUjey( zS(>KUbf!GM0y19#nXiD%S3t(RkdmEHz5+5|0hzCW%vV6>DDHbA^#0xkhA6h literal 0 HcmV?d00001 diff --git a/modules/airparif/module.py b/modules/airparif/module.py new file mode 100644 index 000000000..a216e3f52 --- /dev/null +++ b/modules/airparif/module.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.backend import Module +from weboob.capabilities.base import find_object +from weboob.capabilities.gauge import ( + CapGauge, SensorNotFound, Gauge, GaugeSensor, +) + +from .browser import AirparifBrowser + + +__all__ = ['AirparifModule'] + + +class AirparifModule(Module, CapGauge): + NAME = 'airparif' + DESCRIPTION = 'airparif website' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'LGPLv3+' + VERSION = '1.6' + + BROWSER = AirparifBrowser + + def iter_gauges(self, pattern=None): + if pattern: + pattern = pattern.lower() + + for gauge in self.browser.iter_gauges(): + if not pattern or pattern in gauge._searching: + yield gauge + + def _get_gauge_by_id(self, id): + return find_object(self.browser.iter_gauges(), id=id) + + def iter_sensors(self, gauge, pattern=None): + if pattern: + pattern = pattern.lower() + + if not isinstance(gauge, Gauge): + gauge = self._get_gauge_by_id(gauge) + if gauge is None: + raise SensorNotFound() + + if pattern is None: + for sensor in gauge.sensors: + yield sensor + else: + for sensor in gauge.sensors: + if pattern in sensor.name.lower(): + yield sensor + + def _get_sensor_by_id(self, id): + gid = id.partition('.')[0] + return find_object(self.iter_sensors(gid), id=id) + + def get_last_measure(self, sensor): + if not isinstance(sensor, GaugeSensor): + sensor = self._get_sensor_by_id(sensor) + if sensor is None: + raise SensorNotFound() + return sensor.lastvalue diff --git a/modules/airparif/pages.py b/modules/airparif/pages.py new file mode 100644 index 000000000..3097756fe --- /dev/null +++ b/modules/airparif/pages.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.pages import JsonPage +from weboob.browser.elements import ItemElement, DictElement, method +from weboob.browser.filters.standard import ( + Env, Format, Regexp, DateTime, CleanDecimal, Lower, Map, +) +from weboob.browser.filters.json import Dict +from weboob.capabilities.address import GeoCoordinates, PostalAddress +from weboob.capabilities.gauge import Gauge, GaugeSensor, GaugeMeasure + + +SENSOR_NAMES = { + 'PM25': 'PM 2.5', + 'PM10': 'PM 10', + 'O3': 'O₃', + 'NO3': 'NO₃', + 'NO2': 'NO₂', +} + + +class AllPage(JsonPage): + @method + class iter_gauges(DictElement): + def find_elements(self): + return self.el.values() + + class item(ItemElement): + klass = Gauge + + def condition(self): + # sometimes the "date" field (which contains the hour) is empty + # and no measure is present in it, so we discard it + return bool(self.el['date']) + + def parse(self, el): + for k in el: + self.env[k] = el[k] + + self.env['city'] = Regexp(Dict('commune'), r'^(\D+)')(self) + + obj_id = Dict('nom_court_sit') + obj_name = Dict('isit_long') + obj_city = Env('city') + obj_object = 'Pollution' + + obj__searching = Lower(Format( + '%s %s %s %s', + Dict('isit_long'), + Dict('commune'), + Dict('ninsee'), + Dict('adresse'), + )) + + class obj_sensors(DictElement): + def find_elements(self): + return [dict(zip(('key', 'value'), tup)) for tup in self.el['indices'].items()] + + class item(ItemElement): + klass = GaugeSensor + + obj_name = Map(Dict('key'), SENSOR_NAMES) + obj_gaugeid = Env('nom_court_sit') + obj_id = Format('%s.%s', obj_gaugeid, Dict('key')) + obj_unit = 'µg/m³' + + class obj_lastvalue(ItemElement): + klass = GaugeMeasure + + obj_date = DateTime( + Format( + '%s %s', + Env('min_donnees'), + Env('date'), # "date" contains the time... + ) + ) + obj_level = CleanDecimal(Dict('value')) + + class obj_geo(ItemElement): + klass = GeoCoordinates + + obj_latitude = CleanDecimal(Env('latitude')) + obj_longitude = CleanDecimal(Env('longitude')) + + class obj_location(ItemElement): + klass = PostalAddress + + obj_street = Env('adresse') + obj_postal_code = Env('ninsee') + obj_city = Env('city') + obj_region = 'Ile-de-France' + obj_country = 'France' diff --git a/modules/airparif/test.py b/modules/airparif/test.py new file mode 100644 index 000000000..6ee2ed9cc --- /dev/null +++ b/modules/airparif/test.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.test import BackendTest + + +class AirparifTest(BackendTest): + MODULE = 'airparif' + + def test_gauges(self): + all_gauges = list(self.backend.iter_gauges()) + paris_gauges = list(self.backend.iter_gauges(pattern='paris')) + self.assertTrue(all_gauges) + self.assertTrue(paris_gauges) + + self._check_gauge(all_gauges[0]) + + def _check_gauge(self, g): + self.assertTrue(g.id) + self.assertTrue(g.name) + self.assertTrue(g.city) + self.assertTrue(g.object) + self.assertTrue(g.sensors) + + self._check_sensor(g.sensors[0], g) + + def _check_sensor(self, s, g): + self.assertTrue(s.id) + self.assertTrue(s.name) + self.assertTrue(s.unit) + self.assertTrue(s.gaugeid == g.id) + + self.assertTrue(s.lastvalue.date) + self.assertTrue(s.lastvalue.level) + + self.assertTrue(s.geo.latitude) + self.assertTrue(s.geo.longitude) + + self.assertTrue(s.location.street) + self.assertTrue(s.location.city) + self.assertTrue(s.location.region) + self.assertTrue(s.location.country) + self.assertTrue(s.location.postal_code) diff --git a/modules/allocine/browser.py b/modules/allocine/browser.py index e785277d2..5e0545fca 100644 --- a/modules/allocine/browser.py +++ b/modules/allocine/browser.py @@ -566,7 +566,7 @@ class AllocineBrowser(APIBrowser): ] if query.summary: - movie = self.iter_movies(query.summary).next() + movie = next(self.iter_movies(query.summary)) params.append(('movie', movie.id)) result = self.__do_request('showtimelist', params) diff --git a/modules/amazonstorecard/pages.py b/modules/amazonstorecard/pages.py index 5f5acc2ba..26a8908e3 100644 --- a/modules/amazonstorecard/pages.py +++ b/modules/amazonstorecard/pages.py @@ -31,13 +31,6 @@ import re import json -try: - cmp = cmp -except NameError: - def cmp(x, y): - return (x > y) - (x < y) - - class SomePage(HTMLPage): @property def logged(self): @@ -86,7 +79,7 @@ class ActivityPage(SomePage): records = json.loads(self.doc.xpath( '//div[@id="completedActivityRecords"]//input[1]/@value')[0]) recent = [x for x in records if x['PDF_LOC'] is None] - for rec in sorted(recent, ActivityPage.cmp_records, reverse=True): + for rec in sorted(recent, key=lambda rec: ActivityPage.parse_date(rec['TRANS_DATE']), reverse=True): desc = u' '.join(rec['TRANS_DESC'].split()) trans = Transaction((rec['REF_NUM'] or u'').strip()) trans.date = ActivityPage.parse_date(rec['TRANS_DATE']) @@ -97,11 +90,6 @@ class ActivityPage(SomePage): trans.amount = -AmTr.decimal_amount(rec['TRANS_AMOUNT']) yield trans - @staticmethod - def cmp_records(rec1, rec2): - return cmp(ActivityPage.parse_date(rec1['TRANS_DATE']), - ActivityPage.parse_date(rec2['TRANS_DATE'])) - @staticmethod def parse_date(recdate): return datetime.strptime(recdate, u'%B %d, %Y') @@ -141,10 +129,13 @@ class StatementPage(RawPage): self._tok = ReTokenizer(self._pdf, '\n', self.LEX) def iter_transactions(self): - return sorted(self.read_transactions(), - cmp=lambda t1, t2: cmp(t2.date, t1.date) or - cmp(t1.label, t2.label) or - cmp(t1.amount, t2.amount)) + trs = self.read_transactions() + # since the sorts are not in the same direction, we can't do in one pass + # python sorting is stable, so sorting in 2 passes can achieve a multisort + # the official docs give this way + trs = sorted(trs, key=lambda tr: (tr.label, tr.amount)) + trs = sorted(trs, key=lambda tr: tr.date, reverse=True) + return trs def read_transactions(self): # Statement typically cover one month. diff --git a/modules/americanexpress/browser.py b/modules/americanexpress/browser.py index 74bd7b64d..28ea130c8 100644 --- a/modules/americanexpress/browser.py +++ b/modules/americanexpress/browser.py @@ -21,7 +21,7 @@ from __future__ import unicode_literals import datetime -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from weboob.browser.browsers import LoginBrowser, need_login from weboob.browser.exceptions import HTTPNotFound, ServerError from weboob.browser.url import URL @@ -29,8 +29,8 @@ from dateutil.parser import parse as parse_date from .pages import ( AccountsPage, JsonBalances, JsonPeriods, JsonHistory, - JsonBalances2, CurrencyPage, LoginPage, WrongLoginPage, AccountSuspendedPage, - NoCardPage, NotFoundPage, + JsonBalances2, CurrencyPage, LoginPage, NoCardPage, + NotFoundPage, ) @@ -40,11 +40,9 @@ __all__ = ['AmericanExpressBrowser'] class AmericanExpressBrowser(LoginBrowser): BASEURL = 'https://global.americanexpress.com' - login = URL('/myca/logon/.*', LoginPage) - wrong_login = URL('/myca/fuidfyp/emea/.*', WrongLoginPage) - account_suspended = URL('/myca/onlinepayments/', AccountSuspendedPage) + login = URL(r'/myca/logon/emea/action/login', LoginPage) - accounts = URL(r'/accounts', AccountsPage) + accounts = URL(r'/api/servicing/v1/member', AccountsPage) js_balances = URL(r'/account-data/v1/financials/balances', JsonBalances) js_balances2 = URL(r'/api/servicing/v1/financials/transaction_summary\?type=split_by_cardmember&statement_end_date=(?P[\d-]+)', JsonBalances2) js_pending = URL(r'/account-data/v1/financials/transactions\?limit=1000&offset=(?P\d+)&status=pending', @@ -69,51 +67,49 @@ class AmericanExpressBrowser(LoginBrowser): self.cache = {} def do_login(self): - if not self.login.is_here(): - self.location('/myca/logon/emea/action?request_type=LogonHandler&DestPage=https%3A%2F%2Fglobal.americanexpress.com%2Fmyca%2Fintl%2Facctsumm%2Femea%2FaccountSummary.do%3Frequest_type%3D%26Face%3Dfr_FR%26intlink%3Dtopnavvotrecompteneligne-HPmyca&Face=fr_FR&Info=CUExpired') - - self.page.login(self.username, self.password) - if self.wrong_login.is_here() or self.login.is_here() or self.account_suspended.is_here(): + self.login.go(data={ + 'request_type': 'login', + 'UserID': self.username, + 'Password': self.password, + 'Logon': 'Logon', + }) + + if self.page.get_status_code() != 0: + if self.page.get_error_code() == 'LGON004': + # This error happens when the website needs that the user + # enter his card information and reset his password. + # There is no message returned when this error happens. + raise ActionNeeded() raise BrowserIncorrectPassword() @need_login - def get_accounts(self): - self.accounts.go() - accounts = list(self.page.iter_accounts()) + def iter_accounts(self): + loc = self.session.cookies.get_dict(domain=".americanexpress.com")['axplocale'].lower() + self.currency_page.go(locale=loc) + currency = self.page.get_currency() - for account in accounts: + self.accounts.go() + account_list = list(self.page.iter_accounts(currency=currency)) + for account in account_list: try: # for the main account - self.js_balances.go(headers={'account_tokens': account._balances_token}) + self.js_balances.go(headers={'account_tokens': account.id}) except HTTPNotFound: # for secondary accounts - self.js_periods.go(headers={'account_token': account._balances_token}) + self.js_periods.go(headers={'account_token': account._history_token}) periods = self.page.get_periods() - self.js_balances2.go(date=periods[1], headers={'account_tokens': account._balances_token}) - self.page.set_balances(accounts) - - # get currency - loc = self.session.cookies.get_dict(domain=".americanexpress.com")['axplocale'].lower() - self.currency_page.go(locale=loc) - currency = self.page.get_currency() - - for acc in accounts: - acc.currency = currency - yield acc - - @need_login - def get_accounts_list(self): - for account in self.get_accounts(): + self.js_balances2.go(date=periods[1], headers={'account_tokens': account.id}) + self.page.fill_balances(obj=account) yield account @need_login def iter_history(self, account): - self.js_periods.go(headers={'account_token': account._token}) + self.js_periods.go(headers={'account_token': account._history_token}) periods = self.page.get_periods() today = datetime.date.today() # TODO handle pagination for p in periods: - self.js_posted.go(offset=0, end=p, headers={'account_token': account._token}) + self.js_posted.go(offset=0, end=p, headers={'account_token': account._history_token}) for tr in self.page.iter_history(periods=periods): # As the website is very handy, passing account_token is not enough: # it will return every transactions of each account, so we @@ -128,14 +124,14 @@ class AmericanExpressBrowser(LoginBrowser): # ('Enregistrées' tab on the website) # "pending" have no vdate and debit date is in future - self.js_periods.go(headers={'account_token': account._token}) + self.js_periods.go(headers={'account_token': account._history_token}) periods = self.page.get_periods() date = parse_date(periods[0]).date() today = datetime.date.today() # when the latest period ends today we can't know the coming debit date if date != today: try: - self.js_pending.go(offset=0, headers={'account_token': account._token}) + self.js_pending.go(offset=0, headers={'account_token': account._history_token}) except ServerError as exc: # At certain times of the month a connection might not have pendings; # in that case, `js_pending.go` would throw a 502 error Bad Gateway @@ -150,7 +146,7 @@ class AmericanExpressBrowser(LoginBrowser): # "posted" have a vdate but debit date can be future or past for p in periods: - self.js_posted.go(offset=0, end=p, headers={'account_token': account._token}) + self.js_posted.go(offset=0, end=p, headers={'account_token': account._history_token}) for tr in self.page.iter_history(periods=periods): if tr.date > today or not tr.date: if tr._owner == account._idforJSON: diff --git a/modules/americanexpress/module.py b/modules/americanexpress/module.py index 1c80742cf..c62231a6f 100644 --- a/modules/americanexpress/module.py +++ b/modules/americanexpress/module.py @@ -44,7 +44,7 @@ class AmericanExpressModule(Module, CapBank): self.config['password'].get()) def iter_accounts(self): - return self.browser.get_accounts_list() + return self.browser.iter_accounts() def iter_history(self, account): return self.browser.iter_history(account) diff --git a/modules/americanexpress/pages.py b/modules/americanexpress/pages.py index 6b512a228..25cf438a3 100644 --- a/modules/americanexpress/pages.py +++ b/modules/americanexpress/pages.py @@ -19,18 +19,14 @@ from __future__ import unicode_literals -from ast import literal_eval from decimal import Decimal -import re from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage from weboob.browser.elements import ItemElement, DictElement, method -from weboob.browser.filters.standard import Date, Eval, Env, CleanText, Field, CleanDecimal +from weboob.browser.filters.standard import Date, Eval, Env, CleanText, Field, CleanDecimal, Format, Currency from weboob.browser.filters.json import Dict from weboob.capabilities.bank import Account, Transaction from weboob.capabilities.base import NotAvailable -from weboob.tools.json import json -from weboob.tools.compat import basestring from weboob.exceptions import ActionNeeded, BrowserUnavailable from dateutil.parser import parse as parse_date @@ -48,14 +44,6 @@ def parse_decimal(s): return CleanDecimal(replace_dots=comma).filter(s) -class WrongLoginPage(HTMLPage): - pass - - -class AccountSuspendedPage(HTMLPage): - pass - - class NoCardPage(HTMLPage): def on_load(self): raise ActionNeeded() @@ -68,77 +56,70 @@ class NotFoundPage(HTMLPage): raise BrowserUnavailable(alert_header, alert_content) -class LoginPage(HTMLPage): - def login(self, username, password): - form = self.get_form(name='ssoform') - form['UserID'] = username - form['USERID'] = username - form['Password'] = password - form['PWD'] = password - form.submit() - - -class AccountsPage(LoggedPage, HTMLPage): - def iter_accounts(self): - for line in self.doc.xpath('//script[@id="initial-state"]')[0].text.split('\n'): - m = re.search('window.__INITIAL_STATE__ = (.*);', line) - if m: - data = json.loads(literal_eval(m.group(1))) - break - else: - assert False, "data was not found" - - assert data[15] == 'core' - assert len(data[16]) == 3 - - # search for products to get products list - for index, el in enumerate(data[16][2]): - if 'products' in el: - accounts_data = data[16][2][index + 1] - - assert len(accounts_data) == 2 - assert accounts_data[1][4] == 'productsList' - - accounts_data = accounts_data[1][5] - token = [] - - for account_data in accounts_data: - if isinstance(account_data, basestring): - balances_token = account_data - - elif isinstance(account_data, list) and not account_data[4][2][0] == "Canceled": - acc = Account() - if len(account_data) > 15: - token.append(account_data[-11]) - acc._idforJSON = account_data[10][-1] - else: - acc._idforJSON = account_data[-5][-1] - acc._idforJSON = re.sub(r'\s+', ' ', acc._idforJSON) - acc.number = '-%s' % account_data[2][2] - acc.label = '%s %s' % (account_data[6][4], account_data[10][-1]) - acc._balances_token = acc.id = balances_token - acc._token = token[-1] - acc.type = Account.TYPE_CARD - yield acc +class LoginPage(JsonPage): + def get_status_code(self): + # - 0 = OK + # - 1 = Incorrect login/password + return CleanDecimal(Dict('statusCode'))(self.doc) + + def get_error_code(self): + # - LGON004 = ActionNeeded + return CleanText(Dict('errorCode'))(self.doc) + + +class AccountsPage(LoggedPage, JsonPage): + @method + class iter_accounts(DictElement): + def find_elements(self): + for obj in self.page.doc.get('accounts', []): + obj['_history_token'] = obj['account_token'] + yield obj + for secondary_acc in obj.get('supplementary_accounts', []): + # Secondary accounts use the id of the parrent account + # when searching history/coming. History/coming are filtered + # on the owner name (_idforJSON). + secondary_acc['_history_token'] = obj['account_token'] + yield secondary_acc + + class item(ItemElement): + klass = Account + + def condition(self): + return any(status == 'Active' for status in Dict('status/account_status')(self)) + + obj_id = Dict('account_token') + obj__history_token = Dict('_history_token') + obj__account_type = Dict('account/relationship') + obj_number = Format('-%s', Dict('account/display_account_number')) + obj_type = Account.TYPE_CARD + obj_currency = Currency(Env('currency')) + obj__idforJSON = Dict('profile/embossed_name') + + def obj_label(self): + if Dict('account/relationship')(self) == 'SUPP': + return Format( + '%s %s', + Dict('platform/amex_region'), + Dict('profile/embossed_name'), + )(self) + return CleanText(Dict('product/description'))(self) class JsonBalances(LoggedPage, JsonPage): - def set_balances(self, accounts): - by_token = {a._balances_token: a for a in accounts} - for d in self.doc: - # coming is what should be refunded at a futur deadline - by_token[d['account_token']].coming = -float_to_decimal(d['total_debits_balance_amount']) - # balance is what is currently due - by_token[d['account_token']].balance = -float_to_decimal(d['remaining_statement_balance_amount']) + @method + class fill_balances(ItemElement): + # coming is what should be refunded at a future deadline + obj_coming = CleanDecimal.US(Dict('0/total_debits_balance_amount'), sign=lambda x: -1) + # balance is what is currently due + obj_balance = CleanDecimal.US(Dict('0/remaining_statement_balance_amount'), sign=lambda x: -1) class JsonBalances2(LoggedPage, JsonPage): - def set_balances(self, accounts): - by_token = {a._balances_token: a for a in accounts} - for d in self.doc: - by_token[d['account_token']].balance = -float_to_decimal(d['total']['payments_credits_total_amount']) - by_token[d['account_token']].coming = -float_to_decimal(d['total']['debits_total_amount']) - # warning: payments_credits_total_amount is not the coming value here + @method + class fill_balances(ItemElement): + obj_coming = CleanDecimal.US(Dict('0/total/debits_total_amount'), sign=lambda x: -1) + obj_balance = CleanDecimal.US(Dict('0/total/payments_credits_total_amount'), sign=lambda x: -1) + # warning: payments_credits_total_amount is not the coming value here class CurrencyPage(LoggedPage, JsonPage): diff --git a/modules/amundi/browser.py b/modules/amundi/browser.py index 7eb43d1b8..2d463682d 100644 --- a/modules/amundi/browser.py +++ b/modules/amundi/browser.py @@ -26,9 +26,9 @@ from weboob.capabilities.base import empty, NotAvailable from .pages import ( LoginPage, AccountsPage, AccountHistoryPage, AmundiInvestmentsPage, AllianzInvestmentPage, - EEInvestmentPage, EEInvestmentDetailPage, EEProductInvestmentPage, EresInvestmentPage, - CprInvestmentPage, BNPInvestmentPage, BNPInvestmentApiPage, AxaInvestmentPage, - EpsensInvestmentPage, EcofiInvestmentPage, + EEInvestmentPage, EEInvestmentPerformancePage, EEInvestmentDetailPage, EEProductInvestmentPage, + EresInvestmentPage, CprInvestmentPage, BNPInvestmentPage, BNPInvestmentApiPage, AxaInvestmentPage, + EpsensInvestmentPage, EcofiInvestmentPage, SGGestionInvestmentPage, SGGestionPerformancePage, ) @@ -44,7 +44,8 @@ class AmundiBrowser(LoginBrowser): amundi_investments = URL(r'https://www.amundi.fr/fr_part/product/view', AmundiInvestmentsPage) # EEAmundi browser investments ee_investments = URL(r'https://www.amundi-ee.com/part/home_fp&partner=PACTEO_SYS', EEInvestmentPage) - ee_investment_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call', EEInvestmentDetailPage) + ee_performance_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call(.*)_tab_2', EEInvestmentPerformancePage) + ee_investment_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call(.*)_tab_5', EEInvestmentDetailPage) # EEAmundi product investments ee_product_investments = URL(r'https://www.amundi-ee.com/product', EEProductInvestmentPage) # Allianz GI investments @@ -62,6 +63,9 @@ class AmundiBrowser(LoginBrowser): epsens_investments = URL(r'https://www.epsens.com/information-financiere', EpsensInvestmentPage) # Ecofi investments ecofi_investments = URL(r'http://www.ecofi.fr/fr/fonds/dynamis-solidaire', EcofiInvestmentPage) + # Société Générale gestion investments + sg_gestion_investments = URL(r'https://www.societegeneralegestion.fr/psSGGestionEntr/productsheet/view/idvm', SGGestionInvestmentPage) + sg_gestion_performance = URL(r'https://www.societegeneralegestion.fr/psSGGestionEntr/ezjscore/call', SGGestionPerformancePage) def do_login(self): data = { @@ -101,7 +105,7 @@ class AmundiBrowser(LoginBrowser): handled_urls = ( 'www.amundi.fr/fr_part', # AmundiInvestmentsPage - 'www.amundi-ee.com/part/home_fp', # EEInvestmentPage + 'www.amundi-ee.com/part/home_fp', # EEInvestmentDetailPage & EEInvestmentPerformancePage 'www.amundi-ee.com/product', # EEProductInvestmentPage 'fr.allianzgi.com/fr-fr', # AllianzInvestmentPage 'www.eres-group.com/eres', # EresInvestmentPage @@ -110,6 +114,7 @@ class AmundiBrowser(LoginBrowser): 'axa-im.fr/fr/fund-page', # AxaInvestmentPage 'www.epsens.com/information-financiere', # EpsensInvestmentPage 'www.ecofi.fr/fr/fonds/dynamis-solidaire', # EcofiInvestmentPage + 'www.societegeneralegestion.fr', # SGGestionInvestmentPage ) for inv in self.page.iter_investments(account_id=account.id): @@ -156,10 +161,29 @@ class AmundiBrowser(LoginBrowser): elif self.ee_investments.is_here(): inv.recommended_period = self.page.get_recommended_period() details_url = self.page.get_details_url() + performance_url = self.page.get_performance_url() if details_url: self.location(details_url) if self.ee_investment_details.is_here(): inv.asset_category = self.page.get_asset_category() + if performance_url: + self.location(performance_url) + if self.ee_performance_details.is_here(): + # The investments JSON only contains 1 & 5 years performances + # If we can access EEInvestmentPerformancePage, we can fetch all three + # values (1, 3 and 5 years), in addition the values are more accurate here. + complete_performance_history = self.page.get_performance_history() + if complete_performance_history: + inv.performance_history = complete_performance_history + + elif self.sg_gestion_investments.is_here(): + # Fetch asset category & recommended period + self.page.fill_investment(obj=inv) + # Fetch all performances on the details page + performance_url = self.page.get_performance_url() + if performance_url: + self.location(performance_url) + inv.performance_history = self.page.get_performance_history() elif self.bnp_investments.is_here(): # We fetch the fund ID and get the attributes directly from the BNP-ERE API diff --git a/modules/amundi/module.py b/modules/amundi/module.py index 78b14bb41..f4dbb22cb 100644 --- a/modules/amundi/module.py +++ b/modules/amundi/module.py @@ -34,11 +34,19 @@ class AmundiModule(Module, CapBankWealth): EMAIL = 'james.galt.bi@gmail.com' LICENSE = 'LGPLv3+' VERSION = '1.6' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp=r'\d+', masked=False), - ValueBackendPassword('password', label=u"Mot de passe", regexp=r'\d+'), - Value('website', label='Type de compte', default='ee', - choices={'ee': 'Amundi Epargne Entreprise', - 'tc': 'Amundi Tenue de Compte'})) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', regexp=r'\d+', masked=False), + ValueBackendPassword('password', label='Mot de passe'), + Value( + 'website', + label='Type de compte', + default='ee', + choices={ + 'ee': 'Amundi Epargne Entreprise', + 'tc': 'Amundi Tenue de Compte' + } + ) + ) def create_default_browser(self): b = {'ee': EEAmundi, 'tc': TCAmundi} diff --git a/modules/amundi/pages.py b/modules/amundi/pages.py index fc6e72cfd..a34993952 100644 --- a/modules/amundi/pages.py +++ b/modules/amundi/pages.py @@ -48,6 +48,7 @@ ACCOUNT_TYPES = { 'PERCO': Account.TYPE_PERCO, 'PERCOI': Account.TYPE_PERCO, 'RSP': Account.TYPE_RSP, + 'ART 83': Account.TYPE_ARTICLE_83, } class AccountsPage(LoggedPage, JsonPage): @@ -84,7 +85,7 @@ class AccountsPage(LoggedPage, JsonPage): except UnicodeError: try: return Dict('libelleDispositif')(self).encode('latin1').decode('utf8') - except UnicodeDecodeError: + except UnicodeError: return Dict('libelleDispositif')(self) @method @@ -185,6 +186,21 @@ class EEInvestmentPage(LoggedPage, HTMLPage): def get_details_url(self): return Attr('//a[contains(text(), "Caractéristiques")]', 'data-href', default=None)(self.doc) + def get_performance_url(self): + return Attr('//a[contains(text(), "Performances")]', 'data-href', default=None)(self.doc) + + +class EEInvestmentPerformancePage(LoggedPage, HTMLPage): + def get_performance_history(self): + perfs = {} + if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-2]', default=None)(self.doc): + perfs[1] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-2]'))(self.doc) + if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-1]', default=None)(self.doc): + perfs[3] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-1]'))(self.doc) + if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()]', default=None)(self.doc): + perfs[5] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()]'))(self.doc) + return perfs + class EEInvestmentDetailPage(LoggedPage, HTMLPage): def get_asset_category(self): @@ -272,3 +288,17 @@ class EcofiInvestmentPage(LoggedPage, HTMLPage): r'\/Horizon (.*)\.png' ) obj_asset_category = CleanText('//div[contains(text(), "Classification")]/following-sibling::div[1]', default=NotAvailable) + + +class SGGestionInvestmentPage(LoggedPage, HTMLPage): + @method + class fill_investment(ItemElement): + obj_asset_category = CleanText('//label[contains(text(), "Classe d\'actifs")]/following-sibling::span', default=NotAvailable) + obj_recommended_period = CleanText('//label[contains(text(), "Durée minimum")]/following-sibling::span', default=NotAvailable) + + def get_performance_url(self): + return Attr('(//li[@role="presentation"])[1]//a', 'data-href', default=None)(self.doc) + + +class SGGestionPerformancePage(EEInvestmentPerformancePage): + pass diff --git a/modules/apivie/browser.py b/modules/apivie/browser.py index 650d284f0..6f599735a 100644 --- a/modules/apivie/browser.py +++ b/modules/apivie/browser.py @@ -20,10 +20,9 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.capabilities.base import find_object -from weboob.exceptions import BrowserIncorrectPassword - -from .pages import LoginPage, AccountsPage, InvestmentsPage, OperationsPage +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded +from .pages import LoginPage, AccountsPage, InvestmentsPage, OperationsPage, InfoPage __all__ = ['ApivieBrowser'] @@ -33,6 +32,7 @@ class ApivieBrowser(LoginBrowser): r'/accueil$', r'/perte.*', LoginPage) + info = URL(r'/coordonnees.*', InfoPage) accounts = URL(r'/accueil-connect', AccountsPage) investments = URL(r'/synthese-contrat.*', InvestmentsPage) operations = URL(r'/historique-contrat.*', OperationsPage) @@ -50,6 +50,12 @@ class ApivieBrowser(LoginBrowser): self.page.login(self.username, self.password) + # If the user's contact info is too old the website asks to verify them. We're logged but we can't go further. + if self.info.is_here(): + error_message = self.page.get_error_message() + assert error_message, 'Error message location has changed on info page' + raise ActionNeeded(error_message) + if self.login.is_here() or self.page is None: raise BrowserIncorrectPassword() diff --git a/modules/apivie/pages.py b/modules/apivie/pages.py index a09ee0104..521aa734b 100644 --- a/modules/apivie/pages.py +++ b/modules/apivie/pages.py @@ -36,6 +36,11 @@ class LoginPage(HTMLPage): return form.submit() +class InfoPage(LoggedPage, HTMLPage): + def get_error_message(self): + return CleanText('//span[@class="ui-messages-fatal-detail"]')(self.doc) + + class AccountsPage(LoggedPage, HTMLPage): TYPES = {u'APIVIE': Account.TYPE_LIFE_INSURANCE, u'LINXEA ZEN CLIENT': Account.TYPE_LIFE_INSURANCE, diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index 2fae88026..2eacaf6c2 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -56,17 +56,22 @@ from .pages.document import DocumentsPage, DownloadPage class AXABrowser(LoginBrowser): # Login - keyboard = URL('https://connect.axa.fr/keyboard/password', KeyboardPage) - login = URL('https://connect.axa.fr/api/identity/auth', LoginPage) - password = URL('https://connect.axa.fr/#/changebankpassword', ChangepasswordPage) - predisconnected = URL('https://www.axa.fr/axa-predisconnect.html', - 'https://www.axa.fr/axa-postmaw-predisconnect.html', PredisconnectedPage) - - denied = URL('https://connect.axa.fr/Account/AccessDenied', DeniedPage) - account_space_login = URL('https://connect.axa.fr/api/accountspace', AccountSpaceLogin) - errors = URL('https://espaceclient.axa.fr/content/ecc-public/accueil-axa-connect/_jcr_content/par/text.html', - 'https://espaceclient.axa.fr/content/ecc-public/errors/500.html', - ErrorPage) + keyboard = URL(r'https://connect.axa.fr/keyboard/password', KeyboardPage) + login = URL(r'https://connect.axa.fr/api/identity/auth', LoginPage) + password = URL(r'https://connect.axa.fr/#/changebankpassword', ChangepasswordPage) + predisconnected = URL( + r'https://www.axa.fr/axa-predisconnect.html', + r'https://www.axa.fr/axa-postmaw-predisconnect.html', + PredisconnectedPage + ) + + denied = URL(r'https://connect.axa.fr/Account/AccessDenied', DeniedPage) + account_space_login = URL(r'https://connect.axa.fr/api/accountspace', AccountSpaceLogin) + errors = URL( + r'https://espaceclient.axa.fr/content/ecc-public/accueil-axa-connect/_jcr_content/par/text.html', + r'https://espaceclient.axa.fr/content/ecc-public/errors/500.html', + ErrorPage + ) def do_login(self): # due to the website change, login changed too, this is for don't try to login with the wrong login @@ -98,61 +103,74 @@ class AXABrowser(LoginBrowser): class AXABanque(AXABrowser, StatesMixin): - BASEURL = 'https://www.axabanque.fr/' + BASEURL = 'https://www.axabanque.fr' STATE_DURATION = 5 # Bank - bank_accounts = URL(r'transactionnel/client/liste-comptes.html', - r'transactionnel/client/liste-(?P.*).html', - r'webapp/axabanque/jsp/visionpatrimoniale/liste_panorama_.*\.faces', - r'/webapp/axabanque/page\?code=(?P\d+)', - r'webapp/axabanque/client/sso/connexion\?token=(?P.*)', BankAccountsPage) + bank_accounts = URL( + r'/transactionnel/client/liste-comptes.html', + r'/transactionnel/client/liste-(?P.*).html', + r'/webapp/axabanque/jsp/visionpatrimoniale/liste_panorama_.*\.faces', + r'/webapp/axabanque/page\?code=(?P\d+)', + r'/webapp/axabanque/client/sso/connexion\?token=(?P.*)', + BankAccountsPage + ) iban_pdf = URL(r'http://www.axabanque.fr/webapp/axabanque/formulaire_AXA_Banque/.*\.pdf.*', IbanPage) - cbttransactions = URL(r'webapp/axabanque/jsp/detailCarteBleu.*.faces', CBTransactionsPage) - transactions = URL(r'webapp/axabanque/jsp/panorama.faces', - r'webapp/axabanque/jsp/visionpatrimoniale/panorama_.*\.faces', - r'webapp/axabanque/jsp/detail.*.faces', - r'webapp/axabanque/jsp/.*/detail.*.faces', TransactionsPage) - unavailable = URL(r'login_errors/indisponibilite.*', - r'.*page-indisponible.html.*', - r'.*erreur/erreurBanque.faces', - r'http://www.axabanque.fr/message/maintenance.htm', UnavailablePage) + cbttransactions = URL(r'/webapp/axabanque/jsp/detailCarteBleu.*.faces', CBTransactionsPage) + transactions = URL( + r'/webapp/axabanque/jsp/panorama.faces', + r'/webapp/axabanque/jsp/visionpatrimoniale/panorama_.*\.faces', + r'/webapp/axabanque/jsp/detail.*.faces', + r'/webapp/axabanque/jsp/.*/detail.*.faces', + TransactionsPage + ) + unavailable = URL( + r'/login_errors/indisponibilite.*', + r'.*page-indisponible.html.*', + r'.*erreur/erreurBanque.faces', + r'http://www.axabanque.fr/message/maintenance.htm', + UnavailablePage + ) # Wealth wealth_accounts = URL( - 'https://espaceclient.axa.fr/$', - 'https://espaceclient.axa.fr/accueil.html', - 'https://connexion.adis-assurances.com', + r'https://espaceclient.axa.fr/$', + r'https://espaceclient.axa.fr/accueil.html', + r'https://connexion.adis-assurances.com', WealthAccountsPage ) - investment = URL('https://espaceclient.axa.fr/.*content/ecc-popin-cards/savings/(\w+)/repartition', InvestmentPage) + investment = URL(r'https://espaceclient.axa.fr/.*content/ecc-popin-cards/savings/(\w+)/repartition', InvestmentPage) history = URL(r'https://espaceclient.axa.fr/accueil/savings/savings/contract/_jcr_content.eccGetSavingsOperations.json', HistoryPage) history_investments = URL(r'https://espaceclient.axa.fr/accueil/savings/savings/contract/_jcr_content.eccGetSavingOperationDetail.json', HistoryInvestmentsPage) details = URL( - 'https://espaceclient.axa.fr/.*accueil/savings/(\w+)/contract', - 'https://espaceclient.axa.fr/#', + r'https://espaceclient.axa.fr/.*accueil/savings/(\w+)/contract', + r'https://espaceclient.axa.fr/#', AccountDetailsPage ) lifeinsurance_iframe = URL( - 'https://assurance-vie.axabanque.fr/Consultation/SituationContrat.aspx', - 'https://assurance-vie.axabanque.fr/Consultation/HistoriqueOperations.aspx', + r'https://assurance-vie.axabanque.fr/Consultation/SituationContrat.aspx', + r'https://assurance-vie.axabanque.fr/Consultation/HistoriqueOperations.aspx', LifeInsuranceIframe ) # netfinca bourse - bourse = URL(r'/transactionnel/client/homepage_bourseCAT.html', - r'https://bourse.axabanque.fr/netfinca-titres/servlet/com.netfinca.*', - BoursePage) + bourse = URL( + r'/transactionnel/client/homepage_bourseCAT.html', + r'https://bourse.axabanque.fr/netfinca-titres/servlet/com.netfinca.*', + BoursePage + ) bourse_history = URL(r'https://bourse.axabanque.fr/netfinca-titres/servlet/com.netfinca.frontcr.account.AccountHistory', BoursePage) # Transfer - recipients = URL('/transactionnel/client/enregistrer-nouveau-beneficiaire.html', RecipientsPage) - add_recipient = URL('/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', AddRecipientPage) - recipient_confirmation_page = URL('/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', RecipientConfirmationPage) - validate_transfer = URL('/webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', ValidateTransferPage) - register_transfer = URL('/transactionnel/client/virement.html', - 'webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', - RegisterTransferPage) + recipients = URL(r'/transactionnel/client/enregistrer-nouveau-beneficiaire.html', RecipientsPage) + add_recipient = URL(r'/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', AddRecipientPage) + recipient_confirmation_page = URL(r'/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', RecipientConfirmationPage) + validate_transfer = URL(r'/webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', ValidateTransferPage) + register_transfer = URL( + r'/transactionnel/client/virement.html', + r'/webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', + RegisterTransferPage + ) confirm_transfer = URL('/webapp/axabanque/jsp/virementSepa/confirmationVirementSepa.faces', ConfirmTransferPage) profile_page = URL('/transactionnel/client/coordonnees.html', BankProfilePage) @@ -282,11 +300,12 @@ class AXABanque(AXABrowser, StatesMixin): def get_netfinca_account(self, account): # Important: this part is controlled by modules/lcl/pages.py + owner_name = self.get_profile().name.upper().split(' ', 1)[1] self.go_account_pages(account, None) self.page.open_market() self.page.open_market_next() self.page.open_iframe() - for bourse_account in self.page.get_list(): + for bourse_account in self.page.get_list(name=owner_name): self.logger.debug('iterating account %r', bourse_account) bourse_id = bourse_account.id.replace('bourse', '') if account.id.startswith(bourse_id): @@ -388,7 +407,7 @@ class AXABanque(AXABrowser, StatesMixin): self.go_account_pages(account, 'history') if self.page.more_history(): - for tr in self.page.get_history(): + for tr in sorted_transactions(self.page.get_history()): yield tr # Get deferred card history elif account._acctype == 'bank' and account.type == Account.TYPE_CARD: @@ -580,10 +599,28 @@ class AXAAssurance(AXABrowser): AccountDetailsPage ) investment = URL(r'/content/ecc-popin-cards/savings/[^/]+/repartition', InvestmentPage) - documents = URL(r'/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', DocumentsPage) - download = URL(r'/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', - r'/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', - DownloadPage) + + documents_life_insurance = URL( + r'/content/espace-client/accueil/mes-documents/situations-de-contrats-assurance-vie.content-inner.din_SAVINGS_STATEMENT.html', + DocumentsPage + ) + documents_certificates = URL( + r'/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', + DocumentsPage + ) + documents_tax_area = URL( + r'https://espaceclient.axa.fr/content/espace-client/accueil/mes-documents/espace-fiscal.content-inner.din_TAX.html', + DocumentsPage + ) + documents_membership_fee = URL( + r'/content/espace-client/accueil/mes-documents/avis-d-echeance.content-inner.din_PREMIUM_STATEMENT.html', + DocumentsPage + ) + + download = URL( + r'/content/ecc-popin-cards/technical/detailed/download-document.downloadPdf.html', + DownloadPage + ) profile = URL(r'/content/ecc-popin-cards/transverse/userprofile.content-inner.html\?_=\d+', ProfilePage) def __init__(self, *args, **kwargs): @@ -621,7 +658,7 @@ class AXAAssurance(AXABrowser): else: self.cache['invs'][account.id] = [] for inv in portfolio_page.iter_investment(currency=account.currency): - i = [i for i in self.cache['invs'][account.id] if (i.valuation == inv.valuation and i.label == inv.label)] + i = [i2 for i2 in self.cache['invs'][account.id] if (i2.valuation == inv.valuation and i2.label == inv.label)] assert len(i) in (0, 1) if i: i[0].portfolio_share = inv.portfolio_share @@ -685,12 +722,20 @@ class AXAAssurance(AXABrowser): @need_login def iter_documents(self, subscription): - return self.documents.go().get_documents(subid=subscription.id) + document_urls = [ + self.documents_life_insurance, + self.documents_certificates, + self.documents_tax_area, + self.documents_membership_fee, + ] + for url in document_urls: + url.go() + for doc in self.page.get_documents(subid=subscription.id): + yield doc @need_login - def download_document(self, url): - self.location(url) - self.page.create_document() + def download_document(self, download_id): + self.download.go(data={'documentId': download_id}) return self.page.content @need_login diff --git a/modules/axabanque/module.py b/modules/axabanque/module.py index 620d37c81..29a61c4ec 100644 --- a/modules/axabanque/module.py +++ b/modules/axabanque/module.py @@ -17,9 +17,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.capabilities.bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound -from weboob.capabilities.base import find_object, NotAvailable, empty +from weboob.capabilities.base import find_object, empty from weboob.capabilities.bank import Account, TransferInvalidLabel from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import CapDocument, Subscription, Document, DocumentNotFound, SubscriptionNotFound @@ -34,13 +35,15 @@ __all__ = ['AXABanqueModule'] class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapProfile): NAME = 'axabanque' - MAINTAINER = u'Romain Bignon' + MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '1.6' - DESCRIPTION = u'AXA Banque' + DESCRIPTION = 'AXA Banque' LICENSE = 'LGPLv3+' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Code', regexp='\d+')) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', masked=False), + ValueBackendPassword('password', label='Code', regexp='\d+'), + ) BROWSER = AXABanque def create_default_browser(self): @@ -140,9 +143,7 @@ class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDoc def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) - if document.url is NotAvailable: - return - return self.browser.download_document(document.url) + return self.browser.download_document(document._download_id) def iter_resources(self, objs, split_path): if Account in objs: diff --git a/modules/axabanque/pages/bank.py b/modules/axabanque/pages/bank.py index 3d24a3a37..2585a59ae 100644 --- a/modules/axabanque/pages/bank.py +++ b/modules/axabanque/pages/bank.py @@ -42,6 +42,7 @@ def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) + class UnavailablePage(HTMLPage): def on_load(self): raise BrowserUnavailable() @@ -71,23 +72,23 @@ class MyHTMLPage(HTMLPage): sub = re.search('oamSubmitForm.+?,\'([^:]+).([^\']+)', s) args['%s:_idcl' % sub.group(1)] = "%s:%s" % (sub.group(1), sub.group(2)) args['%s_SUBMIT' % sub.group(1)] = 1 - args['_form_name'] = sub.group(1) # for weboob only + args['_form_name'] = sub.group(1) # for weboob only return args class AccountsPage(LoggedPage, MyHTMLPage): ACCOUNT_TYPES = OrderedDict(( - ('visa', Account.TYPE_CARD), - ('pea', Account.TYPE_PEA), - ('valorisation', Account.TYPE_MARKET), - ('courant-titre', Account.TYPE_CHECKING), - ('courant', Account.TYPE_CHECKING), - ('livret', Account.TYPE_SAVINGS), - ('ldd', Account.TYPE_SAVINGS), - ('pel', Account.TYPE_SAVINGS), - ('cel', Account.TYPE_SAVINGS), - ('titres', Account.TYPE_MARKET), + ('visa', Account.TYPE_CARD), + ('pea', Account.TYPE_PEA), + ('valorisation', Account.TYPE_MARKET), + ('courant-titre', Account.TYPE_CHECKING), + ('courant', Account.TYPE_CHECKING), + ('livret', Account.TYPE_SAVINGS), + ('ldd', Account.TYPE_SAVINGS), + ('pel', Account.TYPE_SAVINGS), + ('cel', Account.TYPE_SAVINGS), + ('titres', Account.TYPE_MARKET), )) def get_tabs(self): @@ -169,11 +170,11 @@ class AccountsPage(LoggedPage, MyHTMLPage): self.logger.debug('Args: %r' % args) if 'paramNumCompte' not in args: - #The displaying of life insurances is very different from the other + # The displaying of life insurances is very different from the other if args.get('idPanorama:_idcl').split(":")[1] == 'tableaux-direct-solution-vie': account_details = self.browser.open("#", data=args) scripts = account_details.page.doc.xpath('//script[@type="text/javascript"]/text()') - script = filter(lambda x: "src" in x, scripts)[0] + script = list(filter(lambda x: "src" in x, scripts))[0] iframe_url = re.search("src:(.*),", script).group()[6:-2] account_details_iframe = self.browser.open(iframe_url, data=args) account.id = CleanText('//span[contains(@id,"NumeroContrat")]/text()')(account_details_iframe.page.doc) @@ -233,7 +234,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): account.balance = Decimal(0) except InvalidOperation: - #The account doesn't have a amount + # The account doesn't have a amount pass account._url = self.doc.xpath('//form[contains(@action, "panorama")]/@action')[0] @@ -259,6 +260,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): def get_profile_name(self): return Regexp(CleanText('//div[@id="bloc_identite"]/h5'), r'Bonjour (.*)')(self.doc) + class IbanPage(PDFPage): def get_iban(self): iban = '' @@ -269,7 +271,7 @@ class IbanPage(PDFPage): # findall will find something like # ['FRXX', '1234', ... , '9012', 'FRXX', '1234', ... , '9012'] iban += part - iban = iban[:len(iban)//2] + iban = iban[:len(iban) // 2] # we suppose that all iban are French iban iban_last_part = re.findall(r'([A-Z0-9]{3})\1\1Titulaire', extract_text(self.data), flags=re.MULTILINE) @@ -283,26 +285,21 @@ class IbanPage(PDFPage): class BankTransaction(FrenchTransaction): - PATTERNS = [(re.compile('^RET(RAIT) DAB (?P
\d{2})/(?P\d{2}) (?P.*)'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^(CARTE|CB ETRANGER|CB) (?P
\d{2})/(?P\d{2}) (?P.*)'), - FrenchTransaction.TYPE_CARD), - (re.compile('^(?PVIR(EMEN)?T? (SEPA)?(RECU|FAVEUR)?)( /FRM)?(?P.*)'), - FrenchTransaction.TYPE_TRANSFER), - (re.compile('^PRLV (?P.*)( \d+)?$'), FrenchTransaction.TYPE_ORDER), - (re.compile('^(CHQ|CHEQUE) .*$'), FrenchTransaction.TYPE_CHECK), - (re.compile('^(AGIOS /|FRAIS) (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^(CONVENTION \d+ |F )?COTIS(ATION)? (?P.*)'), - FrenchTransaction.TYPE_BANK), - (re.compile('^(F|R)-(?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^(?P.*)( \d+)? QUITTANCE .*'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), - FrenchTransaction.TYPE_UNKNOWN), - (re.compile('^ACHATS (CARTE|CB)'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile('^ANNUL (?P.*)'), FrenchTransaction.TYPE_PAYBACK) - ] + PATTERNS = [ + (re.compile(r'^RET(RAIT) DAB (?P
\d{2})/(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(CARTE|CB ETRANGER|CB) (?P
\d{2})/(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^(?PVIR(EMEN)?T? (SEPA)?(RECU|FAVEUR)?)( /FRM)?(?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^PRLV (?P.*)( \d+)?$'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(CHQ|CHEQUE) .*$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(AGIOS /|FRAIS) (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(CONVENTION \d+ |F )?COTIS(ATION)? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(F|R)-(?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P.*)( \d+)? QUITTANCE .*'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), FrenchTransaction.TYPE_UNKNOWN), + (re.compile(r'^ACHATS (CARTE|CB)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^ANNUL (?P.*)'), FrenchTransaction.TYPE_PAYBACK) + ] class TransactionsPage(LoggedPage, MyHTMLPage): @@ -384,24 +381,26 @@ class TransactionsPage(LoggedPage, MyHTMLPage): if link is None: # this is a check account - args = {'categorieMouvementSelectionnePagination': 'afficherTout', - 'nbLigneParPageSelectionneHautPagination': -1, - 'nbLigneParPageSelectionneBasPagination': -1, - 'nbLigneParPageSelectionneComponent': -1, - 'idDetail:btnRechercherParNbLigneParPage': '', - 'idDetail_SUBMIT': 1, - 'javax.faces.ViewState': self.get_view_state(), - } + args = { + 'categorieMouvementSelectionnePagination': 'afficherTout', + 'nbLigneParPageSelectionneHautPagination': -1, + 'nbLigneParPageSelectionneBasPagination': -1, + 'nbLigneParPageSelectionneComponent': -1, + 'idDetail:btnRechercherParNbLigneParPage': '', + 'idDetail_SUBMIT': 1, + 'javax.faces.ViewState': self.get_view_state(), + } else: # something like a PEA or so value = link.attrib['id'] id = value.split(':')[0] - args = {'%s:_idcl' % id: value, - '%s:_link_hidden_' % id: '', - '%s_SUBMIT' % id: 1, - 'javax.faces.ViewState': self.get_view_state(), - 'paramNumCompte': '', - } + args = { + '%s:_idcl' % id: value, + '%s:_link_hidden_' % id: '', + '%s_SUBMIT' % id: 1, + 'javax.faces.ViewState': self.get_view_state(), + 'paramNumCompte': '', + } self.browser.location(form.attrib['action'], data=args) return True @@ -427,10 +426,10 @@ class TransactionsPage(LoggedPage, MyHTMLPage): return True def get_history(self): - #DAT account can't have transaction + # DAT account can't have transaction if self.doc.xpath('//table[@id="table-dat"]'): return - #These accounts have investments, no transactions + # These accounts have investments, no transactions if self.doc.xpath('//table[@id="InfosPortefeuille"]'): return tables = self.doc.xpath('//table[@id="table-detail-operation"]') @@ -534,7 +533,7 @@ class LifeInsuranceIframe(LoggedPage, HTMLPage): def obj_diff_ratio(self): diff_percent = MyDecimal(TableCell('diff')(self)[0])(self) - return diff_percent/100 if diff_percent != NotAvailable else diff_percent + return diff_percent / 100 if diff_percent != NotAvailable else diff_percent @method class iter_history(TableElement): diff --git a/modules/axabanque/pages/document.py b/modules/axabanque/pages/document.py index 3f2228c8c..b338151e6 100644 --- a/modules/axabanque/pages/document.py +++ b/modules/axabanque/pages/document.py @@ -17,35 +17,45 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals + from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.filters.standard import CleanText, Env, Regexp, Format -from weboob.browser.elements import ListElement, ItemElement, method, SkipItem +from weboob.browser.filters.standard import CleanText, Env, Regexp, Format, Date +from weboob.browser.elements import ListElement, ItemElement, method from weboob.capabilities.bill import Document -from weboob.tools.compat import urljoin +from weboob.tools.date import parse_french_date class DocumentsPage(LoggedPage, HTMLPage): @method class get_documents(ListElement): - item_xpath = '//article' + item_xpath = '//div[has-class("mawa-cards-item dashboard-item")]' class item(ItemElement): klass = Document - obj_id = Format('%s_%s', Env('subid'), Regexp(CleanText('./@data-route'), '#/details/(.*)')) - obj_format = u"pdf" - obj_label = CleanText('.//h2') - obj_type = u"document" - - def obj_url(self): - url = urljoin(self.page.browser.BASEURL, CleanText('./@data-url')(self)) - self.page.browser.location(url) - if self.page.doc.xpath('//form[contains(., "Afficher")]'): - return url - raise SkipItem() + obj_id = Format( + '%s_%s', + Env('subid'), + Regexp(CleanText('./@data-module-open-link--link'), '#/details/(.*)'), + ) + obj_format = 'pdf' + # eg when formatted (not complete list): + # - Situation de contrat suite à réajustement automatique Assurance Vie N° XXXXXXXXXX + # - Lettre d'information client Assurance Vie N° XXXXXXXXXX + # - Attestation de rachat partiel Assurance Vie N° XXXXXXXXXXXXXX + obj_label = Format( + '%s %s %s', + CleanText('.//h3[@class="card-title"]'), + CleanText('.//div[@class="sticker-content"]//strong'), + CleanText('.//p[@class="contract-info"]'), + ) + obj_date = Date(CleanText('.//p[@class="card-date"]'), parse_func=parse_french_date) + obj_type = 'document' + obj__download_id = Regexp(CleanText('./@data-url'), r'.did_(.*?)\.') class DownloadPage(LoggedPage, HTMLPage): def create_document(self): - form = self.get_form(xpath='//form[contains(., "Afficher")]') + form = self.get_form(xpath='//form[has-class("form-download-pdf")]') form.submit() diff --git a/modules/axabanque/pages/login.py b/modules/axabanque/pages/login.py index 70890144e..9a8964a26 100644 --- a/modules/axabanque/pages/login.py +++ b/modules/axabanque/pages/login.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from io import BytesIO @@ -31,35 +32,37 @@ class MyVirtKeyboard(VirtKeyboard): margin = 5, 5, 5, 5 color = (255, 255, 255) - symbols = {'0': '6959163af44cc50b3863e7e306d6e571', - '1': '98b32dff471e903b6fa8e3a0f1544b17', - '2': '32722d5b6572f9d46350aca7fb66263a', - '3': '835a9c8bf66e28f3ffa2b12994bc3f9a', - '4': 'e7457342c434da4fb0fd974f7dc37002', - '5': 'c8b74429a805e12a08c5ed87fd9730ce', - '6': '70a84c766bc323343c0c291146f652db', - '7': 'e4e7fb4f8cc90c8ad472906b5eceeb99', - '8': 'ffb78dbea5a171990e14d707d4772ba2', - '9': '063dcb4179beaeff60fb73c80cbd429d' - } - - coords = {'0': (0, 0, 40, 40), - '1': (40, 0, 80, 40), - '2': (80, 0, 120, 40), - '3': (120, 0, 160, 40), - '4': (0, 40, 40, 80), - '5': (40, 40, 80, 80), - '6': (80, 40, 120, 80), - '7': (120, 40, 160, 80), - '8': (0, 80, 40, 120), - '9': (40, 80, 80, 120), - '10': (80, 80, 120, 120), - '11': (120, 80, 160, 120), - '12': (0, 120, 40, 160), - '13': (40, 120, 80, 160), - '14': (80, 120, 120, 160), - '15': (120, 120, 160, 160) - } + symbols = { + '0': '6959163af44cc50b3863e7e306d6e571', + '1': '98b32dff471e903b6fa8e3a0f1544b17', + '2': '32722d5b6572f9d46350aca7fb66263a', + '3': '835a9c8bf66e28f3ffa2b12994bc3f9a', + '4': 'e7457342c434da4fb0fd974f7dc37002', + '5': 'c8b74429a805e12a08c5ed87fd9730ce', + '6': '70a84c766bc323343c0c291146f652db', + '7': 'e4e7fb4f8cc90c8ad472906b5eceeb99', + '8': 'ffb78dbea5a171990e14d707d4772ba2', + '9': '063dcb4179beaeff60fb73c80cbd429d', + } + + coords = { + '0': (0, 0, 40, 40), + '1': (40, 0, 80, 40), + '2': (80, 0, 120, 40), + '3': (120, 0, 160, 40), + '4': (0, 40, 40, 80), + '5': (40, 40, 80, 80), + '6': (80, 40, 120, 80), + '7': (120, 40, 160, 80), + '8': (0, 80, 40, 120), + '9': (40, 80, 80, 120), + '10': (80, 80, 120, 120), + '11': (120, 80, 160, 120), + '12': (0, 120, 40, 160), + '13': (40, 120, 80, 160), + '14': (80, 120, 120, 160), + '15': (120, 120, 160, 160), + } def __init__(self, page): VirtKeyboard.__init__(self, BytesIO(page.content), self.coords, self.color, convert='RGB') diff --git a/modules/axabanque/pages/transfer.py b/modules/axabanque/pages/transfer.py index 6afef8024..d62972aaa 100644 --- a/modules/axabanque/pages/transfer.py +++ b/modules/axabanque/pages/transfer.py @@ -50,17 +50,18 @@ class TransferVirtualKeyboard(SimpleVirtualKeyboard): margin = 1 tile_margin = 10 - symbols = {'0': '715df9c139fc7b46829526229c415a67', - '1': '12d398f7f389711c5f8298ee68a8af28', - '2': 'f43ca3a5dd649d30bf02060ab65c4eff', - '3': 'b6dd7864cfd941badb0784be37f7eeb3', - '4': ('7138d0a663eef56c699d85dc6c3ac639', '0faced58777f371097a7a70bb9570dd7', ), - '5': 'b71bd38e71ce0b611642a01b6900218f', - '6': 'f71f7249413c189165da7b588c2f0493', - '7': '81fc65230d7df341e80d02e414f183d4', - '8': '8106671a6b24aee3475d6f12a650f59b', - '9': 'e8c4567eb46dba5e2a92619076441a8a' - } + symbols = { + '0': '715df9c139fc7b46829526229c415a67', + '1': '12d398f7f389711c5f8298ee68a8af28', + '2': 'f43ca3a5dd649d30bf02060ab65c4eff', + '3': 'b6dd7864cfd941badb0784be37f7eeb3', + '4': ('7138d0a663eef56c699d85dc6c3ac639', '0faced58777f371097a7a70bb9570dd7', ), + '5': 'b71bd38e71ce0b611642a01b6900218f', + '6': 'f71f7249413c189165da7b588c2f0493', + '7': '81fc65230d7df341e80d02e414f183d4', + '8': '8106671a6b24aee3475d6f12a650f59b', + '9': 'e8c4567eb46dba5e2a92619076441a8a', + } # Clean image def alter_image(self): @@ -174,9 +175,9 @@ class AddRecipientPage(LoggedPage, HTMLPage): # fill iban part _iban_rcpt_part = 4 - for i in range(3,10): + for i in range(3, 10): form_key = 'ibanContenuZone{}Hidden'.format(i) - form[form_key] = rcpt_iban[_iban_rcpt_part: _iban_rcpt_part+4] + form[form_key] = rcpt_iban[_iban_rcpt_part: _iban_rcpt_part + 4] if form[form_key]: form['ibanContenuZone{}'.format(i)] = form[form_key] _iban_rcpt_part += 4 @@ -339,7 +340,7 @@ class ValidateTransferPage(LoggedPage, HTMLPage): date = Regexp(pattern=r'(\d+/\d+/\d+)').filter(self.get_element_by_name('Date du virement')) transfer.exec_date = Date(dayfirst=True).filter(date) - account_label_id = self.get_element_by_name(u'Compte à débiter') + account_label_id = self.get_element_by_name('Compte à débiter') transfer.account_id = (Regexp(pattern=r'(\d+)').filter(account_label_id)) transfer.account_label = Regexp(pattern=r'([\w \.]+)').filter(account_label_id) # account iban is not in the summary page @@ -347,7 +348,7 @@ class ValidateTransferPage(LoggedPage, HTMLPage): transfer.recipient_id = recipient.id transfer.recipient_iban = self.get_element_by_name('IBAN').replace(' ', '') - transfer.recipient_label = self.get_element_by_name(u'Nom du bénéficiaire') + transfer.recipient_label = self.get_element_by_name('Nom du bénéficiaire') transfer.label = CleanText('//table[@id="table-confLibelle"]//p')(self.doc) return transfer @@ -357,7 +358,7 @@ class ValidateTransferPage(LoggedPage, HTMLPage): f = BytesIO(self.browser.open(img_src).content) vk = TransferVirtualKeyboard(file=f, cols=8, rows=3, - matching_symbols=string.ascii_lowercase[:8*3], browser=self.browser) + matching_symbols=string.ascii_lowercase[:8 * 3], browser=self.browser) return vk.get_string_code(password) diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index 774963a46..d807d2596 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -40,7 +40,7 @@ from .pages import ( IbanPage, AdvisorPage, TransactionDetailPage, TransactionsBackPage, NatixisPage, EtnaPage, NatixisInvestPage, NatixisHistoryPage, NatixisErrorPage, NatixisDetailsPage, NatixisChoicePage, NatixisRedirect, - LineboursePage, AlreadyLoginPage, + LineboursePage, AlreadyLoginPage, InvestmentPage, ) from .document_pages import BasicTokenPage, SubscriberPage, SubscriptionsPage, DocumentsPage @@ -128,6 +128,8 @@ class BanquePopulaire(LoginBrowser): r'https://[^/]+/cyber/internet/Sort.do\?.*', TransactionsPage) + investment_page = URL(r'https://[^/]+/cyber/ibp/ate/skin/internet/pages/webAppReroutingAutoSubmit.jsp', InvestmentPage) + transactions_back_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*ActionPerformed=BACK.*', TransactionsBackPage) transaction_detail_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=DETAIL_ECRITURE.*', TransactionDetailPage) @@ -168,7 +170,7 @@ class BanquePopulaire(LoginBrowser): natixis_history = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load-operation/(?P\w+)/(?P\w+)/(?P\w+)', NatixisHistoryPage) natixis_pdf = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load-releve/(?P\w+)/(?P\w+)/(?P\w+)/(?P\d+)', NatixisDetailsPage) - linebourse_home = URL(r'https://www.linebourse.fr/ReroutageSJR', LineboursePage) + linebourse_home = URL(r'https://www.linebourse.fr', LineboursePage) advisor = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=accueil.*', r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=contacter.*', AdvisorPage) @@ -194,7 +196,6 @@ class BanquePopulaire(LoginBrowser): dirname += '/bourse' self.linebourse = LinebourseBrowser('https://www.linebourse.fr', logger=self.logger, responses_dirname=dirname, weboob=self.weboob, proxy=self.PROXIES) - self.investments = {} self.documents_headers = None def deinit(self): @@ -248,7 +249,7 @@ class BanquePopulaire(LoginBrowser): raise BrowserIncorrectPassword() if 'internetRescuePortal' in self.url: # 1 more request is necessary - data = {'integrationMode': 'INTERNET_RESCUE'} + data = {'integrationMode': 'INTERNET_RESCUE'} self.location('/cyber/internet/Login.do', data=data) ACCOUNT_URLS = ['mesComptes', 'mesComptesPRO', 'maSyntheseGratuite', 'accueilSynthese', 'equipementComplet'] @@ -257,19 +258,26 @@ class BanquePopulaire(LoginBrowser): @need_login def go_on_accounts_list(self): for taskInfoOID in self.ACCOUNT_URLS: + # 4 possible URLs but we stop as soon as one of them works data = OrderedDict([('taskInfoOID', taskInfoOID), ('token', self.token)]) + + # Go from AdvisorPage to AccountsPage self.location(self.absurl('/cyber/internet/StartTask.do', base=True), params=data) + if not self.page.is_error(): if self.page.pop_up(): self.logger.debug('Popup displayed, retry') data = OrderedDict([('taskInfoOID', taskInfoOID), ('token', self.token)]) self.location('/cyber/internet/StartTask.do', params=data) + + # Set the valid ACCOUNT_URL and break the loop self.ACCOUNT_URLS = [taskInfoOID] break else: raise BrokenPageError('Unable to go on the accounts list page') if self.page.is_short_list(): + # Go from AccountsPage to AccountsFullPage to get the full accounts list form = self.page.get_form(nr=0) form['dialogActionPerformed'] = 'EQUIPEMENT_COMPLET' form['token'] = self.page.build_token(form['token']) @@ -280,16 +288,23 @@ class BanquePopulaire(LoginBrowser): @retry(LoggedOut) @need_login - def get_accounts_list(self, get_iban=True): - # We have to parse account list in 2 different way depending if we want the iban number or not - # thanks to stateful website + def iter_accounts(self, get_iban=True): + # We have to parse account list in 2 different way depending if + # we want the iban number or not thanks to stateful website next_pages = [] accounts = [] profile = self.get_profile() + if profile.name: - owner_name = re.search(r' (.+)', profile.name).group(1).upper() + name = profile.name + else: + name = profile.company_name + + # Handle names/company names without spaces + if ' ' in name: + owner_name = re.search(r' (.+)', name).group(1).upper() else: - owner_name = re.search(r' (.+)', profile.company_name).group(1).upper() + owner_name = name.upper() self.go_on_accounts_list() @@ -311,6 +326,7 @@ class BanquePopulaire(LoginBrowser): params['token'] = self.page.build_token(self.token) self.location('/cyber/internet/ContinueTask.do', data=params) + # Go to next_page with params and token next_page['token'] = self.page.build_token(self.token) self.location('/cyber/internet/ContinueTask.do', data=next_page) @@ -323,8 +339,6 @@ class BanquePopulaire(LoginBrowser): if get_iban: for a in accounts: a.iban = self.get_iban_number(a) - for a in accounts: - self.get_investment(a) yield a # TODO: see if there's other type of account with a label without name which @@ -358,7 +372,7 @@ class BanquePopulaire(LoginBrowser): @retry(LoggedOut) @need_login def get_account(self, id): - return find_object(self.get_accounts_list(False), id=id) + return find_object(self.iter_accounts(get_iban=False), id=id) def set_gocardless_transaction_details(self, transaction): # Setting references for a GoCardless transaction @@ -378,7 +392,7 @@ class BanquePopulaire(LoginBrowser): @retry(LoggedOut) @need_login - def get_history(self, account, coming=False): + def iter_history(self, account, coming=False): def get_history_by_receipt(account, coming, sel_tbl1=None): account = self.get_account(account.id) @@ -400,7 +414,7 @@ class BanquePopulaire(LoginBrowser): return params['token'] = self.page.build_token(params['token']) - if sel_tbl1 != None: + if sel_tbl1 is not None: params['attribute($SEL_$tbl1)'] = str(sel_tbl1) self.location(self.absurl('/cyber/internet/ContinueTask.do', base=True), data=params) @@ -408,12 +422,17 @@ class BanquePopulaire(LoginBrowser): if not self.page or self.error_page.is_here() or self.page.no_operations(): return - # Sort by values dates (see comment in TransactionsPage.get_history) + # Sort by operation date if len(self.page.doc.xpath('//a[@id="tcl4_srt"]')) > 0: - form = self.page.get_form(id='myForm') - form.url = self.absurl('/cyber/internet/Sort.do?property=tbl1&sortBlocId=blc2&columnName=dateValeur') - params['token'] = self.page.build_token(params['token']) - form.submit() + # The first request sort might transaction by oldest. If this is the case, + # we need to do the request a second time for the transactions to be sorted by newest. + for _ in range(2): + form = self.page.get_form(id='myForm') + form.url = self.absurl('/cyber/internet/Sort.do?property=tbl1&sortBlocId=blc2&columnName=dateOperation') + params['token'] = self.page.build_token(params['token']) + form.submit() + if self.page.is_sorted_by_most_recent(): + break transactions_next_page = True @@ -421,6 +440,7 @@ class BanquePopulaire(LoginBrowser): assert self.transactions_page.is_here() transaction_list = self.page.get_history(account, coming) + for tr in transaction_list: # Add information about GoCardless if 'GoCardless' in tr.label and tr._has_link: @@ -445,7 +465,6 @@ class BanquePopulaire(LoginBrowser): @need_login def go_investments(self, account, get_account=False): - if not account._invest_params and not (account.id.startswith('TIT') or account.id.startswith('PRV')): raise NotImplementedError() @@ -453,10 +472,11 @@ class BanquePopulaire(LoginBrowser): account = self.get_account(account.id) if account._params: - params = {'taskInfoOID': "ordreBourseCTJ", - 'controlPanelTaskAction': "true", - 'token': self.page.build_token(account._params['token']) - } + params = { + 'taskInfoOID': 'ordreBourseCTJ', + 'controlPanelTaskAction': 'true', + 'token': self.page.build_token(account._params['token']), + } self.location(self.absurl('/cyber/internet/StartTask.do', base=True), params=params) else: params = account._invest_params @@ -469,78 +489,74 @@ class BanquePopulaire(LoginBrowser): if self.error_page.is_here(): raise NotImplementedError() - url, params = self.page.get_investment_page_params() - if params: - try: - self.location(url, data=params) - except BrowserUnavailable: - return False - - if "linebourse" in self.url: - self.linebourse.session.cookies.update(self.session.cookies) - self.linebourse.invest.go() + if self.page.go_investment(): + url, params = self.page.get_investment_page_params() + if params: + try: + self.location(url, data=params) + except BrowserUnavailable: + return False - if self.natixis_error_page.is_here(): - self.logger.warning("natixis site doesn't work") - return False + if 'linebourse' in self.url: + self.linebourse.session.cookies.update(self.session.cookies) + self.linebourse.invest.go() - if self.natixis_redirect.is_here(): - url = self.page.get_redirect() - if re.match(r'https://www.assurances.natixis.fr/etna-ihs-bp/#/equipement;codeEtab=\d+\?windowId=[a-f0-9]+$', url): - self.logger.warning('there may be no contract associated with %s, skipping', url) + if self.natixis_error_page.is_here(): + self.logger.warning('Natixis site does not work.') return False + + if self.natixis_redirect.is_here(): + url = self.page.get_redirect() + if re.match(r'https://www.assurances.natixis.fr/etna-ihs-bp/#/equipement;codeEtab=\d+\?windowId=[a-f0-9]+$', url): + self.logger.warning('There may be no contract associated with %s, skipping', url) + return False return True @need_login - def get_investment(self, account): - if account.type in (Account.TYPE_LOAN,): - self.investments[account.id] = [] - return [] + def iter_investments(self, account): + if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PEA, Account.TYPE_MARKET, Account.TYPE_PERP): + return # Add "Liquidities" investment if the account is a "Compte titres PEA": if account.type == Account.TYPE_PEA and account.id.startswith('CPT'): - self.investments[account.id] = [create_french_liquidity(account.balance)] - return self.investments[account.id] + yield create_french_liquidity(account.balance) + return - if account.id in self.investments.keys() and self.investments[account.id] is False: - raise NotImplementedError() + if self.go_investments(account, get_account=True): + # Redirection URL is https://www.linebourse.fr/ReroutageSJR + if 'linebourse' in self.url: + self.logger.warning('Going to Linebourse space to fetch investments.') + # Eliminating the 3 letters prefix to match IDs on Linebourse: + linebourse_id = account.id[3:] + for inv in self.linebourse.iter_investment(linebourse_id): + yield inv + return - if account.id not in self.investments.keys(): - self.investments[account.id] = [] - try: - if self.go_investments(account, get_account=True): - # Redirection URL is https://www.linebourse.fr/ReroutageSJR - if "linebourse" in self.url: - # Eliminating the 3 letters prefix to match IDs on Linebourse: - linebourse_id = account.id[3:] - for inv in self.linebourse.iter_investment(linebourse_id): - self.investments[account.id].append(inv) - - if self.etna.is_here(): - params = self.page.params - elif self.natixis_redirect.is_here(): - # the url may contain a "#", so we cannot make a request to it, the params after "#" would be dropped - url = self.page.get_redirect() - self.logger.debug('using redirect url %s', url) - m = self.etna.match(url) - if not m: - # url can be contratPrev which is not investments - self.logger.debug('Unable to handle this kind of contract') - raise NotImplementedError() - - params = m.groupdict() - - if self.natixis_redirect.is_here() or self.etna.is_here(): - try: - self.natixis_invest.go(**params) - except ServerError: - # Broken website .. nothing to do. - self.investments[account.id] = iter([]) - return self.investments[account.id] - self.investments[account.id] = list(self.page.get_investments()) - except NotImplementedError: - self.investments[account.id] = [] - return self.investments[account.id] + if self.etna.is_here(): + self.logger.warning('Going to Etna space to fetch investments.') + params = self.page.params + + elif self.natixis_redirect.is_here(): + self.logger.warning('Going to Natixis space to fetch investments.') + # the url may contain a "#", so we cannot make a request to it, the params after "#" would be dropped + url = self.page.get_redirect() + self.logger.debug('using redirect url %s', url) + m = self.etna.match(url) + if not m: + # URL can be contratPrev which is not investments + self.logger.warning('Unable to handle this kind of contract.') + return + + params = m.groupdict() + + if self.natixis_redirect.is_here() or self.etna.is_here(): + try: + self.natixis_invest.go(**params) + except ServerError: + # Broken website... nothing to do. + return + for inv in self.page.iter_investments(): + yield inv @need_login def get_invest_history(self, account): @@ -591,7 +607,7 @@ class BanquePopulaire(LoginBrowser): @need_login def get_profile(self): - self.location('/cyber/internet/StartTask.do?taskInfoOID=accueil&token=%s' % self.token) + self.location(self.absurl('/cyber/internet/StartTask.do?taskInfoOID=accueil&token=%s' % self.token, base=True)) return self.page.get_profile() @retry(LoggedOut) diff --git a/modules/banquepopulaire/module.py b/modules/banquepopulaire/module.py index b569d4521..6523bfc72 100644 --- a/modules/banquepopulaire/module.py +++ b/modules/banquepopulaire/module.py @@ -69,9 +69,13 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo 'www.ibps.sud.banquepopulaire.fr': 'Sud', 'www.ibps.valdefrance.banquepopulaire.fr': 'Val de France', }.items(), key=lambda k_v: (k_v[1], k_v[0]))]) - CONFIG = BackendConfig(Value('website', label='Région', choices=website_choices), - ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + + CONFIG = BackendConfig( + Value('website', label='Région', choices=website_choices), + ValueBackendPassword('login', label='Identifiant', masked=False), + ValueBackendPassword('password', label='Mot de passe') + ) + BROWSER = BanquePopulaire accepted_document_types = (DocumentTypes.STATEMENT,) @@ -89,10 +93,13 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo ('ouest.banquepopulaire', 'bpgo.banquepopulaire'), ] website = reduce(lambda a, kv: a.replace(*kv), repls, self.config['website'].get()) - return self.create_browser(website, - self.config['login'].get(), - self.config['password'].get(), - weboob=self.weboob) + + return self.create_browser( + website, + self.config['login'].get(), + self.config['password'].get(), + weboob=self.weboob + ) def iter_accounts(self): return self.browser.get_accounts_list() @@ -105,13 +112,13 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo raise AccountNotFound() def iter_history(self, account): - return self.browser.get_history(account) + return self.browser.iter_history(account) def iter_coming(self, account): - return self.browser.get_history(account, coming=True) + return self.browser.iter_history(account, coming=True) def iter_investment(self, account): - return self.browser.get_investment(account) + return self.browser.iter_investments(account) def iter_contacts(self): return self.browser.get_advisor() diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index a17d468ac..da36b0591 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -84,7 +84,7 @@ class WikipediaARC4(object): self.x = 0 def crypt(self, input): - output = [None]*len(input) + output = [None] * len(input) for i in range(len(input)): self.x = (self.x + 1) & 0xFF self.y = (self.state[self.x] + self.y) & 0xFF @@ -113,11 +113,12 @@ class BasePage(object): def is_error(self): for script in self.doc.xpath('//script'): - if script.text is not None and \ - (u"Le service est momentanément indisponible" in script.text or - u"Le service est temporairement indisponible" in script.text or - u"Votre abonnement ne vous permet pas d'accéder à ces services" in script.text or - u'Merci de bien vouloir nous en excuser' in script.text): + if script.text is not None and ( + "Le service est momentanément indisponible" in script.text + or "Le service est temporairement indisponible" in script.text + or "Votre abonnement ne vous permet pas d'accéder à ces services" in script.text + or 'Merci de bien vouloir nous en excuser' in script.text + ): return True return False @@ -159,9 +160,10 @@ class BasePage(object): continue for id, action, strategy in re.findall(r'''attEvt\(window,"(?P[^"]+)","click","sab\('(?P[^']+)','(?P[^']+)'\);"''', script.text, re.MULTILINE): - actions[id] = {'dialogActionPerformed': action, - 'validationStrategy': strategy, - } + actions[id] = { + 'dialogActionPerformed': action, + 'validationStrategy': strategy, + } return actions def get_back_button_params(self, params=None, actions=None): @@ -272,7 +274,7 @@ class RedirectPage(LoggedPage, MyHTMLPage): class ErrorPage(LoggedPage, MyHTMLPage): def on_load(self): if CleanText('//script[contains(text(), "momentanément indisponible")]')(self.doc): - raise BrowserUnavailable(u"Le service est momentanément indisponible") + raise BrowserUnavailable("Le service est momentanément indisponible") elif CleanText('//h1[contains(text(), "Cette page est indisponible")]')(self.doc): raise BrowserUnavailable('Cette page est indisponible') return super(ErrorPage, self).on_load() @@ -381,12 +383,12 @@ class Login2Page(LoginPage): def login(self, login, password): payload = { 'validate': { - self.form_id[0]: [ { + self.form_id[0]: [{ 'id': self.form_id[1], 'login': login.upper(), 'password': password, 'type': 'PASSWORD_LOOKUP', - } ] + }] } } url = self.request_url + '/step' @@ -436,8 +438,8 @@ class Login2Page(LoginPage): if 'phase' in doc and doc['phase']['state'] == "ENROLLMENT": raise ActionNeeded() - if (('phase' in doc and doc['phase']['previousResult'] == 'FAILED_AUTHENTICATION') or - doc['response']['status'] != 'AUTHENTICATION_SUCCESS'): + if (('phase' in doc and doc['phase']['previousResult'] == 'FAILED_AUTHENTICATION') + or doc['response']['status'] != 'AUTHENTICATION_SUCCESS'): raise BrowserIncorrectPassword() data = {'SAMLResponse': doc['response']['saml2_post']['samlResponse']} @@ -523,22 +525,23 @@ class HomePage(LoggedPage, MyHTMLPage): class AccountsPage(LoggedPage, MyHTMLPage): - ACCOUNT_TYPES = {u'Mes comptes d\'épargne': Account.TYPE_SAVINGS, - u'Mon épargne': Account.TYPE_SAVINGS, - u'Placements': Account.TYPE_SAVINGS, - u'Liste complète de mon épargne': Account.TYPE_SAVINGS, - u'Mes comptes': Account.TYPE_CHECKING, - u'Comptes en euros': Account.TYPE_CHECKING, - u'Mes comptes en devises': Account.TYPE_CHECKING, - u'Liste complète de mes comptes': Account.TYPE_CHECKING, - u'Mes emprunts': Account.TYPE_LOAN, - u'Liste complète de mes emprunts': Account.TYPE_LOAN, - u'Financements': Account.TYPE_LOAN, - u'Liste complète de mes engagements': Account.TYPE_LOAN, - u'Mes services': None, # ignore this kind of accounts (no bank ones) - u'Équipements': None, # ignore this kind of accounts (no bank ones) - u'Synthèse': None, # ignore this title - } + ACCOUNT_TYPES = { + 'Mes comptes d\'épargne': Account.TYPE_SAVINGS, + 'Mon épargne': Account.TYPE_SAVINGS, + 'Placements': Account.TYPE_SAVINGS, + 'Liste complète de mon épargne': Account.TYPE_SAVINGS, + 'Mes comptes': Account.TYPE_CHECKING, + 'Comptes en euros': Account.TYPE_CHECKING, + 'Mes comptes en devises': Account.TYPE_CHECKING, + 'Liste complète de mes comptes': Account.TYPE_CHECKING, + 'Mes emprunts': Account.TYPE_LOAN, + 'Liste complète de mes emprunts': Account.TYPE_LOAN, + 'Financements': Account.TYPE_LOAN, + 'Liste complète de mes engagements': Account.TYPE_LOAN, + 'Mes services': None, # ignore this kind of accounts (no bank ones) + 'Équipements': None, # ignore this kind of accounts (no bank ones) + 'Synthèse': None, # ignore this title + } PATTERN = [ (re.compile(r'.*Titres Pea.*'), Account.TYPE_PEA), @@ -548,7 +551,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): (re.compile(r'.*Titres.*'), Account.TYPE_MARKET), (re.compile(r'.*Selection Vie.*'), Account.TYPE_LIFE_INSURANCE), (re.compile(r'^Fructi Pulse.*'), Account.TYPE_MARKET), - (re.compile(r'^(Quintessa|Solevia).*'), Account.TYPE_MARKET), + (re.compile(r'^(Quintessa|Solevia).*'), Account.TYPE_LIFE_INSURANCE), (re.compile(r'^Plan Epargne Enfant Mul.*'), Account.TYPE_MARKET), (re.compile(r'^Alc Premium'), Account.TYPE_MARKET), (re.compile(r'^Plan Epargne Enfant Msu.*'), Account.TYPE_LIFE_INSURANCE), @@ -576,7 +579,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): actions = self.get_button_actions() for div in self.doc.xpath('//div[has-class("btit")]'): - if div.text in (None, u'Synthèse'): + if div.text in (None, 'Synthèse'): continue account_type = self.ACCOUNT_TYPES.get(div.text.strip(), Account.TYPE_UNKNOWN) @@ -612,8 +615,9 @@ class AccountsPage(LoggedPage, MyHTMLPage): account = Account() account.id = args['identifiant'].replace(' ', '') - account.label = u' '.join([u''.join([txt.strip() for txt in tds[1].itertext()]), - u''.join([txt.strip() for txt in tds[2].itertext()])]).strip() + account.number = account.id + account.label = ' '.join([''.join([txt.strip() for txt in tds[1].itertext()]), + ''.join([txt.strip() for txt in tds[2].itertext()])]).strip() for pattern, _type in self.PATTERN: match = pattern.match(account.label) @@ -623,7 +627,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): else: account.type = account_type - balance_text = u''.join([txt.strip() for txt in tds[3].itertext()]) + balance_text = ''.join([txt.strip() for txt in tds[3].itertext()]) balance = FrenchTransaction.clean_amount(balance_text) account.balance = Decimal(balance or '0.0') account.currency = currency or Account.get_currency(balance_text) @@ -637,7 +641,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): account._coming_params = None account._coming_count = None account._invest_params = None - if balance != u'' and len(tds[3].xpath('.//a')) > 0: + if balance != '' and len(tds[3].xpath('.//a')) > 0: account._params = params.copy() account._params['dialogActionPerformed'] = 'SOLDE' account._params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1] @@ -706,18 +710,19 @@ class CardsPage(LoggedPage, MyHTMLPage): yield account account = Account() account.id = id.replace(' ', '') + account.number = account.id account.type = Account.TYPE_CARD account.balance = account.coming = Decimal('0') account._next_debit = datetime.date.today() - account._prev_debit = datetime.date(2000,1,1) - account.label = u' '.join([CleanText(None).filter(cols[self.COL_TYPE]), + account._prev_debit = datetime.date(2000, 1, 1) + account.label = ' '.join([CleanText(None).filter(cols[self.COL_TYPE]), CleanText(None).filter(cols[self.COL_LABEL])]) account.currency = currency if accounts_parsed is not None: for account_parsed in accounts_parsed: - if (account_parsed.type == Account.TYPE_CHECKING and - account_parsed.id.replace('CPT', '') == Regexp(CleanText('//div[@class="btit"]'), r'(\d+)$')(self.doc)): + if (account_parsed.type == Account.TYPE_CHECKING + and account_parsed.id.replace('CPT', '') == Regexp(CleanText('//div[@class="btit"]'), r'(\d+)$')(self.doc)): account.parent = account_parsed account._params = None @@ -749,7 +754,7 @@ class CardsPage(LoggedPage, MyHTMLPage): date = datetime.date(*[int(c) for c in m.groups()][::-1]) if date.year < 100: - date = date.replace(year=date.year+2000) + date = date.replace(year=date.year + 2000) amount = Decimal(FrenchTransaction.clean_amount(CleanText(None).filter(cols[self.COL_AMOUNT]))) @@ -769,37 +774,51 @@ class CardsPage(LoggedPage, MyHTMLPage): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^RET DAB (?P.*?) RETRAIT (DU|LE) (?P
\d{2})(?P\d{2})(?P\d+).*'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^RET DAB (?P.*?) CARTE ?:.*'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^(?P.*) RETRAIT DU (?P
\d{2})(?P\d{2})(?P\d{2}) .*'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^(RETRAIT CARTE )?RET(RAIT)? DAB (?P.*)'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('((\w+) )?(?P
\d{2})(?P\d{2})(?P\d{2}) CB[:\*][^ ]+ (?P.*)'), - FrenchTransaction.TYPE_CARD), - (re.compile('^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile('^(PRLV|PRELEVEMENT) (?P.*)'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^(?PCHEQUE .*)'), FrenchTransaction.TYPE_CHECK), - (re.compile('^(AGIOS /|FRAIS) (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile('^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile('^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^(?PECHEANCE PRET .*)'), FrenchTransaction.TYPE_LOAN_PAYMENT), - (re.compile('^(?P.*)( \d+)? QUITTANCE .*'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), - FrenchTransaction.TYPE_UNKNOWN), - (re.compile(r'^RELEVE CARTE'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile(r'^RET GAB .*'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^RETRAIT CARTE AGENCE \d+$'), FrenchTransaction.TYPE_WITHDRAWAL), - ] + PATTERNS = [ + (re.compile('^RET DAB (?P.*?) RETRAIT (DU|LE) (?P
\d{2})(?P\d{2})(?P\d+).*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('^RET DAB (?P.*?) CARTE ?:.*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('^(?P.*) RETRAIT DU (?P
\d{2})(?P\d{2})(?P\d{2}) .*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('^(RETRAIT CARTE )?RET(RAIT)? DAB (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('((\w+) )?(?P
\d{2})(?P\d{2})(?P\d{2}) CB[:\*][^ ]+ (?P.*)'), FrenchTransaction.TYPE_CARD), + (re.compile('^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile('^(PRLV|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile('^(?PCHEQUE .*)'), FrenchTransaction.TYPE_CHECK), + (re.compile('^(AGIOS /|FRAIS) (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile('^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile('^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile('^(?PECHEANCE PRET .*)'), FrenchTransaction.TYPE_LOAN_PAYMENT), + (re.compile('^(?P.*)( \d+)? QUITTANCE .*'), FrenchTransaction.TYPE_ORDER), + (re.compile('^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), FrenchTransaction.TYPE_UNKNOWN), + (re.compile(r'^RELEVE CARTE'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^RET GAB .*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^RETRAIT CARTE AGENCE \d+$'), FrenchTransaction.TYPE_WITHDRAWAL), + ] + + +class InvestmentPage(LoggedPage, HTMLPage): + def get_investment_page_params(self): + script = self.doc.xpath('//body')[0].attrib['onload'] + url = None + m = re.search(r"','([^']+?)',\[", script, re.MULTILINE) + if m: + url = m.group(1) + + params = {} + for key, value in re.findall(r"key:'(?PSJRToken)'\,value:'(?P.*?)'}", script, re.MULTILINE): + params[key] = value + + if url and params: + return url, params + return None class TransactionsPage(LoggedPage, MyHTMLPage): + def is_sorted_by_most_recent(self): + # If the transactions are not sorted correctly, the class of this + # 'a' tag changes ('tcth' if sorted the other way, 'tctm' if not sorted + # by operation date) + return CleanText('//a[@class="tctb" and contains(text(), "Date opé")]')(self.doc) + def get_next_params(self): nxt = self.doc.xpath('//li[contains(@id, "_nxt")]') if len(nxt) == 0 or nxt[0].attrib.get('class', '') == 'nxt-dis': @@ -825,7 +844,7 @@ class TransactionsPage(LoggedPage, MyHTMLPage): COL_COMPTA_DATE = 0 COL_LABEL = 1 - COL_REF = 2 # optional + COL_REF = 2 # optional COL_OP_DATE = -4 COL_VALUE_DATE = -3 COL_DEBIT = -2 @@ -847,10 +866,8 @@ class TransactionsPage(LoggedPage, MyHTMLPage): # (only used for GoCardLess transactions so far) t._has_link = bool(tds[self.COL_DEBIT].findall('a') or tds[self.COL_CREDIT].findall('a')) - # XXX We currently take the *value* date, but it will probably - # necessary to use the *operation* one. - # Default sort on website is by compta date, so in browser.py we - # change the sort on value date. + # Default sort on website is by compta date, in browser.py we + # change the sort on operation date. cleaner = CleanText(None).filter date = cleaner(tds[self.COL_OP_DATE]) vdate = cleaner(tds[self.COL_VALUE_DATE]) @@ -897,10 +914,10 @@ class TransactionsPage(LoggedPage, MyHTMLPage): t.amount = -account._prev_balance yield t - currency = Account.get_currency(self.doc\ - .xpath('//table[@id="TabFact"]/thead//th')[self.COL_CARD_AMOUNT]\ - .text\ - .replace('(', ' ')\ + currency = Account.get_currency(self.doc + .xpath('//table[@id="TabFact"]/thead//th')[self.COL_CARD_AMOUNT] + .text + .replace('(', ' ') .replace(')', ' ')) for i, tr in enumerate(self.doc.xpath('//table[@id="TabFact"]/tbody/tr')): tds = tr.findall('td') @@ -927,22 +944,11 @@ class TransactionsPage(LoggedPage, MyHTMLPage): def no_operations(self): if len(self.doc.xpath('//table[@id="tbl1" or @id="TabFact"]//td[@colspan]')) > 0: return True - if len(self.doc.xpath(u'//div[contains(text(), "Accès à LineBourse")]')) > 0: + if len(self.doc.xpath('//div[contains(text(), "Accès à LineBourse")]')) > 0: return True return False - def get_investment_page_params(self): - script = self.doc.xpath('//body')[0].attrib['onload'] - url = None - m = re.search(r"','([^']+?)',\[", script, re.MULTILINE) - if m: - url = m.group(1) - params = {} - for key, value in re.findall(r"key:'(?PSJRToken)'\,value:'(?P.*?)'}", script, re.MULTILINE): - params[key] = value - return url, params if url and params else None - def get_transaction_table_id(self, ref): tr = self.doc.xpath('//table[@id="tbl1"]/tbody/tr[.//span[contains(text(), "%s")]]' % ref)[0] @@ -957,12 +963,12 @@ class TransactionsPage(LoggedPage, MyHTMLPage): # index in which the link lies # # To get more details about how things are done, see the following javascript functions: - #- attachTableRowEvents (atre) - #- attachActiveSelectionEventsOnRow - #- astr - #- updateSelection (uds) - #- selectActionButton (sab) - #- a script element embedded in the html page (search for "tcl5", "tcl6") + # - attachTableRowEvents (atre) + # - attachActiveSelectionEventsOnRow + # - astr + # - updateSelection (uds) + # - selectActionButton (sab) + # - a script element embedded in the html page (search for "tcl5", "tcl6") assert transaction._has_link @@ -971,6 +977,14 @@ class TransactionsPage(LoggedPage, MyHTMLPage): elif transaction._amount_type == 'credit': return 'NV' + def go_investment(self): + script = self.doc.xpath('//body')[0].attrib['onload'] + if re.search(r'startWebAppTask\(', script) is None: + return False + params = {'oid': re.search(r"'urlReturn',\w+?,'(\w+)'\)", script).group(1)} + self.browser.location(self.browser.absurl('/cyber/ibp/ate/skin/internet/pages/webAppReroutingAutoSubmit.jsp'), params=params) + return True + class NatixisChoicePage(LoggedPage, HTMLPage): def on_load(self): @@ -1006,7 +1020,7 @@ class TransactionsBackPage(TransactionsPage): class NatixisRedirect(LoggedPage, XMLPage): def get_redirect(self): url = self.doc.xpath('/partial-response/redirect/@url')[0] - return url.replace('http://', 'https://') # why do they use http on a bank site??? + return url.replace('http://', 'https://') # why do they use http on a bank site??? class NatixisErrorPage(LoggedPage, HTMLPage): @@ -1029,7 +1043,7 @@ class IbanPage(LoggedPage, MyHTMLPage): form['token'] = self.build_token(form['token']) form['dialogActionPerformed'] = "DETAIL_IBAN_RIB" tr_id = Attr(None, 'id').filter(tr.xpath('.')).split('_') - form[u'attribute($SEL_$%s)' % tr_id[0]] = tr_id[1] + form['attribute($SEL_$%s)' % tr_id[0]] = tr_id[1] form.submit() return True return False @@ -1057,7 +1071,7 @@ def float_to_decimal(f): class NatixisInvestPage(LoggedPage, JsonPage): @method - class get_investments(DictElement): + class iter_investments(DictElement): item_xpath = 'detailContratVie/valorisation/supports' class item(ItemElement): @@ -1159,7 +1173,7 @@ class NatixisDetailsPage(LoggedPage, RawPage): tr.amount = -abs(tr.amount) else: assert False, 'unhandled line %s' % label - assert not any(len(cell) for cell in row[self.COL_LABEL+1:]), 'there should be only the label' + assert not any(len(cell) for cell in row[self.COL_LABEL + 1:]), 'there should be only the label' else: if not tr: continue @@ -1188,24 +1202,26 @@ class AdvisorPage(LoggedPage, MyHTMLPage): class get_advisor(ItemElement): klass = Advisor - condition = lambda self: Field('name')(self) + def condition(self): + return Field('name')(self) - obj_name = CleanText(u'//div[label[contains(text(), "Votre conseiller")]]/span') - obj_agency = CleanText(u'//div[label[contains(text(), "Votre agence")]]/span') + obj_name = CleanText('//div[label[contains(text(), "Votre conseiller")]]/span') + obj_agency = CleanText('//div[label[contains(text(), "Votre agence")]]/span') obj_email = obj_mobile = NotAvailable @method class update_agency(ItemElement): - obj_phone = CleanText(u'//div[label[contains(text(), "Téléphone")]]/span', replace=[('.', '')]) - obj_fax = CleanText(u'//div[label[contains(text(), "Fax")]]/span', replace=[('.', '')]) - obj_address = CleanText(u'//div[div[contains(text(), "Votre agence")]]/following-sibling::div[1]//div[not(label)]/span') + obj_phone = CleanText('//div[label[contains(text(), "Téléphone")]]/span', replace=[('.', '')]) + obj_fax = CleanText('//div[label[contains(text(), "Fax")]]/span', replace=[('.', '')]) + obj_address = CleanText('//div[div[contains(text(), "Votre agence")]]/following-sibling::div[1]//div[not(label)]/span') def get_profile(self): profile = Person() - # the name is only available in a welcome message. Sometimes, the message will look like that : - # "Bienvenue M - " and sometimes just "Bienvenue M " - # Or even "Bienvenue " + # the name is only available in a welcome message. The messages can look like : + # - Bienvenue M - + # - Bienvenue M + # - Bienvenue # We need to detect wether the company name is there, and where it begins. # relying on the dash only is dangerous as people may have dashes in their name and so may companies. # but we can detect company name from a dash between space diff --git a/modules/bforbank/browser.py b/modules/bforbank/browser.py index 536b17681..28aa6c141 100644 --- a/modules/bforbank/browser.py +++ b/modules/bforbank/browser.py @@ -67,7 +67,7 @@ class BforbankBrowser(LoginBrowser): BoursePage) bourse_titre = URL(r'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.navigation.Titre', BoursePage) # to get logout link - bourse_disco = URL(r'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.login.ContextTransferDisconnect', BourseDisconnectPage) + bourse_disco = URL(r'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.login.Logout', BourseDisconnectPage) profile = URL(r'/espace-client/profil/informations', ProfilePage) def __init__(self, birthdate, username, password, *args, **kwargs): @@ -96,8 +96,9 @@ class BforbankBrowser(LoginBrowser): @need_login def iter_accounts(self): if self.accounts is None: - self.home.stay_or_go() - accounts = list(self.page.iter_accounts()) + owner_name = self.get_profile().name.upper().split(' ', 1)[1] + self.home.go() + accounts = list(self.page.iter_accounts(name=owner_name)) if self.page.RIB_AVAILABLE: self.rib.go().populate_rib(accounts) @@ -144,7 +145,9 @@ class BforbankBrowser(LoginBrowser): if not bourse_account: return iter([]) - self.location(bourse_account._link_id) + self.location(bourse_account._link_id, params={ + 'nump': bourse_account._market_id, + }) assert self.bourse.is_here() history = list(self.page.iter_history()) self.leave_espace_bourse() @@ -245,6 +248,7 @@ class BforbankBrowser(LoginBrowser): return True def get_bourse_account(self, account): + owner_name = self.get_profile().name.upper().split(' ', 1)[1] self.bourse_login.go(id=account.id) # "login" to bourse page self.bourse.go() @@ -253,7 +257,7 @@ class BforbankBrowser(LoginBrowser): if self.page.password_required(): return self.logger.debug('searching account matching %r', account) - for bourse_account in self.page.get_list(): + for bourse_account in self.page.get_list(name=owner_name): self.logger.debug('iterating account %r', bourse_account) if bourse_account.id.startswith(account.id[3:]): return bourse_account @@ -295,7 +299,7 @@ class BforbankBrowser(LoginBrowser): if self.bourse.is_here(): self.location(self.bourse_titre.build()) self.location(self.page.get_logout_link()) - self.location(self.page.get_relocation()) + self.login.go() @need_login def get_profile(self): diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index e4dbc4bbd..d8f9179f1 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -29,7 +29,7 @@ from PIL import Image from weboob.exceptions import ActionNeeded from weboob.browser.pages import LoggedPage, HTMLPage, pagination, AbstractPage from weboob.browser.elements import method, ListElement, ItemElement, TableElement -from weboob.capabilities.bank import Account +from weboob.capabilities.bank import Account, AccountOwnership from weboob.capabilities.profile import Person from weboob.browser.filters.html import Link, Attr, TableCell from weboob.browser.filters.standard import ( @@ -170,6 +170,16 @@ class AccountsPage(LoggedPage, HTMLPage): def condition(self): return not len(self.el.xpath('./td[@class="chart"]')) + def obj_ownership(self): + owner = CleanText('./td//div[contains(@class, "-synthese-text") and not(starts-with(., "N°"))]', default=None)(self) + + if owner: + if re.search(r'(m|mr|me|mme|mlle|mle|ml)\.? (.*)\bou (m|mr|me|mme|mlle|mle|ml)\b(.*)', owner, re.IGNORECASE): + return AccountOwnership.CO_OWNER + elif all(n in owner.upper() for n in self.env['name'].split()): + return AccountOwnership.OWNER + return AccountOwnership.ATTORNEY + class Transaction(FrenchTransaction): PATTERNS = [(re.compile('^(?PVIREMENT)'), FrenchTransaction.TYPE_TRANSFER), @@ -385,13 +395,12 @@ class BoursePage(AbstractPage): PARENT = 'lcl' PARENT_URL = 'bourse' + def get_logout_link(self): + return Link('//a[@title="Retour à l\'accueil"]')(self.doc) + class BourseDisconnectPage(LoggedPage, HTMLPage): - def get_relocation(self): - link = re.search(r"window\.location= \'(.+)\';", self.content) - if link: - m = link.group(1) - return m + pass class ProfilePage(LoggedPage, HTMLPage): diff --git a/modules/binck/pages.py b/modules/binck/pages.py index 3a9b3a54f..8350da918 100644 --- a/modules/binck/pages.py +++ b/modules/binck/pages.py @@ -23,7 +23,7 @@ import re from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage from weboob.browser.elements import ItemElement, ListElement, DictElement, TableElement, method -from weboob.browser.filters.standard import CleanText, Date, DateTime, Format, CleanDecimal, Eval, Env, Field +from weboob.browser.filters.standard import CleanText, Date, Format, CleanDecimal, Eval, Env, Field from weboob.browser.filters.html import Attr, Link, TableCell from weboob.browser.filters.json import Dict from weboob.exceptions import BrowserPasswordExpired, ActionNeeded @@ -224,20 +224,6 @@ class InvestmentPage(LoggedPage, JsonPage): obj_original_diff = Env('o_diff', default=NotAvailable) obj__security_id = Dict('SecurityId') - def obj_vdate(self): - raw_date = CleanText(Dict('Time'))(self) - if raw_date == '---': - return NotAvailable - elif re.match(r'\d{2}\/\d{2}\/\d{4}', raw_date): - # during stocks closing hours only date (dd/mm/yyyy) is given - return Date(CleanText(Dict('Time')), dayfirst=True)(self) - elif re.match(r'\d{2}:\d{2}:\d{2}', raw_date): - # during stocks opening hours only time (hh:mm:ss) is given, - # can even be realtime, depending on user settings, - # can be given in foreign markets time, - # e.g. 10:00 is displayed at 9:00 for an action in NASDAQ Helsinki market - return DateTime(CleanText(Dict('Time')), dayfirst=True, strict=False)(self) - def obj_code(self): if is_isin_valid(Dict('IsinCode')(self)): return Dict('IsinCode')(self) diff --git a/modules/bnporc/enterprise/pages.py b/modules/bnporc/enterprise/pages.py index 0d27d8fd6..aa04b302c 100644 --- a/modules/bnporc/enterprise/pages.py +++ b/modules/bnporc/enterprise/pages.py @@ -24,6 +24,7 @@ import re from datetime import datetime from io import BytesIO +import dateutil.parser from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage from weboob.browser.filters.json import Dict from weboob.browser.filters.html import TableCell, Attr @@ -35,7 +36,6 @@ from weboob.browser.filters.standard import ( from weboob.capabilities.bank import Transaction, Account, Investment from weboob.capabilities.profile import Person from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard, VirtKeyboardError -from weboob.tools.date import parse_french_date from weboob.capabilities import NotAvailable from weboob.exceptions import ActionNeeded, BrowserForbidden @@ -178,7 +178,7 @@ class BnpHistoryItem(ItemElement): mtc = re.search(r'\bDU (\d{2})\.?(\d{2})\.?(\d{2})\b', raw) if mtc: date = '%s/%s/%s' % (mtc.group(1), mtc.group(2), mtc.group(3)) - return parse_french_date(date) + return dateutil.parser.parse(date, dayfirst=True) # The date can be truncated, so it is not retrieved if 'dateCreation' in self.el: @@ -270,10 +270,18 @@ class AccountHistoryPage(LoggedPage, JsonPage): def obj_rdate(self): raw = self.obj_raw() mtc = re.search(r'\bDU (\d{6}|\d{8})\b', raw) + if mtc: - date = mtc.group(1) - date = '%s/%s/%s' % (date[0:2], date[2:4], date[4:]) - return parse_french_date(date) + numbers = mtc.group(1) + # we need to create this string because dateutil crashes + # with dates in the ddmmyyyy format + # dd/mm/yy and dd/mm/yyyy + date = '%s/%s/%s' % (numbers[0:2], numbers[2:4], numbers[4:]) + try: + return dateutil.parser.parse(date, dayfirst=True) + except ValueError: + # parsing failed assuming yyyymmdd format + return dateutil.parser.parse(numbers) return fromtimestamp(Dict('dateCreation')(self)) diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index c7e884766..0faca2aa7 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -38,14 +38,14 @@ from weboob.browser.pages import JsonPage, LoggedPage, HTMLPage from weboob.capabilities import NotAvailable from weboob.capabilities.bank import ( Account, Investment, Recipient, Transfer, TransferBankError, - AddRecipientBankError, AddRecipientTimeout, + AddRecipientBankError, AddRecipientTimeout, AccountOwnership, ) from weboob.capabilities.base import empty from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Person, ProfileMissing from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired, ActionNeeded from weboob.tools.capabilities.bank.iban import rib2iban, rebuild_rib, is_iban_valid -from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.capabilities.bank.transactions import FrenchTransaction, parse_with_patterns from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard from weboob.tools.date import parse_french_date from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -310,6 +310,9 @@ class ProfilePage(LoggedPage, JsonPage): class AccountsPage(BNPPage): + def get_user_ikpi(self): + return self.doc['data']['infoUdc']['titulaireConnecte']['ikpi'] + @method class iter_accounts(DictElement): item_xpath = 'data/infoUdc/familleCompte' @@ -375,6 +378,17 @@ class AccountsPage(BNPPage): return iban return None + def obj_ownership(self): + indic = Dict('titulaire/indicTitulaireCollectif', default=None)(self) + # The boolean is in the form of a string ('true' or 'false') + if indic == 'true': + return AccountOwnership.CO_OWNER + elif indic == 'false': + if self.page.get_user_ikpi() == Dict('titulaire/ikpi')(self): + return AccountOwnership.OWNER + return AccountOwnership.ATTORNEY + return NotAvailable + # softcap not used TODO don't pass this key when backend is ready # deferred cb can disappear the day after the appear, so 0 as day_for_softcap obj__bisoftcap = {'deferred_cb': {'softcap_day': 1000, 'day_for_softcap': 1}} @@ -636,9 +650,12 @@ class HistoryPage(BNPPage): 'amount': op.get('montant'), 'card': op.get('numeroPorteurCarte'), }) - tr.parse(date=parse_french_date(op.get('dateOperation')), - vdate=parse_french_date(op.get('valueDate')), - raw=CleanText().filter(op.get('libelle'))) + + tr.date = parse_french_date(op.get('dateOperation')) + tr.vdate = parse_french_date(op.get('valueDate')) + tr.rdate = NotAvailable + tr.raw = CleanText().filter(op.get('libelle')) + parse_with_patterns(tr.raw, tr, Transaction.PATTERNS) if tr.type == Transaction.TYPE_CARD: tr.type = self.browser.card_to_transaction_type.get(op.get('keyCarte'), diff --git a/modules/bnppere/pages.py b/modules/bnppere/pages.py index a83d37924..2ad1fe25c 100644 --- a/modules/bnppere/pages.py +++ b/modules/bnppere/pages.py @@ -125,9 +125,9 @@ class HistoryPage(LoggedPage, HTMLPage): # This wonderful website randomly displays separators as '.' or ',' # For example, numbers can look like "€12,345.67" or "12 345,67 €" try: - return CleanDecimal.French('./div[contains(@class, "accordion_header")]/div[6]')(self) + return CleanDecimal.French('./div[contains(@class, "accordion_header")]/div[position()=last()]')(self) except NumberFormatError: - return CleanDecimal.US('./div[contains(@class, "accordion_header")]/div[6]')(self) + return CleanDecimal.US('./div[contains(@class, "accordion_header")]/div[position()=last()]')(self) class InvestmentPage(LoggedPage, HTMLPage): diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index b3311aa25..05356bb41 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -21,7 +21,6 @@ import requests from datetime import date, datetime from dateutil.relativedelta import relativedelta -from dateutil import parser from weboob.browser.retry import login_method, retry_on_logout, RetryLoginBrowser from weboob.browser.browsers import need_login, StatesMixin @@ -31,11 +30,11 @@ from weboob.browser.exceptions import LoggedOut, ClientError from weboob.capabilities.bank import ( Account, AccountNotFound, TransferError, TransferInvalidAmount, TransferInvalidEmitter, TransferInvalidLabel, TransferInvalidRecipient, - AddRecipientStep, Recipient, Rate, TransferBankError, AccountOwnership, + AddRecipientStep, Rate, TransferBankError, AccountOwnership, RecipientNotFound, + AddRecipientTimeout, ) -from weboob.capabilities.base import empty +from weboob.capabilities.base import empty, find_object from weboob.capabilities.contact import Advisor -from weboob.tools.captcha.virtkeyboard import VirtKeyboardError from weboob.tools.value import Value from weboob.tools.compat import basestring, urlsplit from weboob.tools.capabilities.bank.transactions import sorted_transactions @@ -126,7 +125,7 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): currencylist = URL('https://www.boursorama.com/bourse/devises/parite/_detail-parite', CurrencyListPage) currencyconvert = URL('https://www.boursorama.com/bourse/devises/convertisseur-devises/convertir', CurrencyConvertPage) - __states__ = ('auth_token',) + __states__ = ('auth_token', 'recipient_form',) def __init__(self, config=None, *args, **kwargs): self.config = config @@ -134,6 +133,7 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): self.accounts_list = None self.cards_list = None self.deferred_card_calendar = None + self.recipient_form = None kwargs['username'] = self.config['login'].get() kwargs['password'] = self.config['password'].get() super(BoursoramaBrowser, self).__init__(*args, **kwargs) @@ -145,8 +145,14 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): pass def load_state(self, state): - if ('expire' in state and parser.parse(state['expire']) > datetime.now()) or state.get('auth_token'): - return super(BoursoramaBrowser, self).load_state(state) + # needed to continue the session while adding recipient with otp + # it keeps the form to continue to submit the otp + if state.get('recipient_form'): + state.pop('url', None) + super(BoursoramaBrowser, self).load_state(state) + + elif state.get('auth_token'): + super(BoursoramaBrowser, self).load_state(state) def handle_authentication(self): if self.authentication.is_here(): @@ -169,16 +175,8 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): if self.auth_token and self.config['pin_code'].get(): self.page.authenticate() else: - for _ in range(3): - self.login.go() - try: - self.page.login(self.username, self.password) - except VirtKeyboardError: - self.logger.error('Failed to process VirtualKeyboard') - else: - break - else: - raise VirtKeyboardError() + self.login.go() + self.page.login(self.username, self.password) if self.login.is_here() or self.error.is_here(): raise BrowserIncorrectPassword() @@ -410,34 +408,35 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): advisor.phone = u"0146094949" return iter([advisor]) - @need_login - def iter_transfer_recipients(self, account): - if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): - return [] - assert account.url - + def go_recipients_list(self, account_url, account_id): # url transfer preparation - url = urlsplit(account.url) + url = urlsplit(account_url) parts = [part for part in url.path.split('/') if part] assert len(parts) > 2, 'Account url missing some important part to iter recipient' account_type = parts[1] # cav, ord, epargne ... account_webid = parts[-1] - try: - self.transfer_main_page.go(acc_type=account_type, webid=account_webid) - except BrowserHTTPNotFound: - return [] + self.transfer_main_page.go(acc_type=account_type, webid=account_webid) # may raise a BrowserHTTPNotFound # can check all account available transfer option if self.transfer_main_page.is_here(): self.transfer_accounts.go(acc_type=account_type, webid=account_webid) if self.transfer_accounts.is_here(): - try: - self.page.submit_account(account.id) - except AccountNotFound: - return [] + self.page.submit_account(account_id) # may raise AccountNotFound + + + @need_login + def iter_transfer_recipients(self, account): + if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): + return [] + assert account.url + + try: + self.go_recipients_list(account.url, account.id) + except (BrowserHTTPNotFound, AccountNotFound): + return [] assert self.recipients_page.is_here() return self.page.iter_recipients() @@ -481,10 +480,11 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): # at this stage, the site doesn't show the real ids/ibans, we can only guess if recipients[0].label != ret.recipient_label: - if not recipients[0].label.startswith('%s - ' % ret.recipient_label): - # the label displayed here is just "" - # but in the recipients list it is " - "... - raise TransferError('Recipient label changed during transfer') + self.logger.info('Recipients from iter_recipient and from the transfer are diffent: "%s" and "%s"' % (recipients[0].label, ret.recipient_label)) + if not ret.recipient_label.startswith('%s - ' % recipients[0].label): + # the label displayed here is " - " + # but in the recipients list it is ""... + assert False, 'Recipient label changed during transfer (from "%s" to "%s")' % (recipients[0].label, ret.recipient_label) ret.recipient_id = recipients[0].id ret.recipient_iban = recipients[0].iban @@ -509,26 +509,11 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): # the last page contains no info, return the last transfer object from init_transfer return transfer - def build_recipient(self, recipient): - r = Recipient() - r.iban = recipient.iban - r.id = recipient.iban - r.label = recipient.label - r.category = recipient.category - r.enabled_at = date.today() - r.currency = u'EUR' - r.bank_name = recipient.bank_name - return r - @need_login - def new_recipient(self, recipient, **kwargs): - if 'code' in kwargs: - assert self.rcpt_page.is_here() - assert self.page.is_confirm_sms() - - self.page.confirm_sms(kwargs['code']) - return self.rcpt_after_sms() + def init_new_recipient(self, recipient): + self.recipient_form = None # so it is reset when a new recipient is added + # get url account = None for account in self.get_accounts_list(): if account.url: @@ -541,26 +526,57 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): target = account.url + '/' + suffix self.location(target) - assert self.page.is_charac() + assert self.page.is_charac(), 'Not on the page to add recipients.' + # fill recipient form self.page.submit_recipient(recipient) + recipient.origin_account_id = account.id + + # confirm sending sms + assert self.page.is_confirm_send_sms(), 'Cannot reach the page asking to send a sms.' + self.page.confirm_send_sms() if self.page.is_send_sms(): + # send sms self.page.send_sms() - assert self.page.is_confirm_sms() - raise AddRecipientStep(self.build_recipient(recipient), Value('code', label='Veuillez saisir le code')) - # if the add recipient is restarted after the sms has been confirmed recently, the sms step is not presented again + assert self.page.is_confirm_sms(), 'The sms was not send.' - return self.rcpt_after_sms() + self.recipient_form = self.page.get_confirm_sms_form() + self.recipient_form['account_url'] = account.url + raise AddRecipientStep(recipient, Value('code', label='Veuillez saisir le code')) - def rcpt_after_sms(self): - assert self.page.is_confirm() - - ret = self.page.get_recipient() - self.page.confirm() + # if the add recipient is restarted after the sms has been confirmed recently, the sms step is not presented again + return self.rcpt_after_sms() - assert self.page.is_created() - return ret + def new_recipient(self, recipient, **kwargs): + # step 2 of new_recipient + if 'code' in kwargs: + # there is no confirmation to check the recipient + # validating the sms code directly adds the recipient + if not self.recipient_form: # the session expired + raise AddRecipientTimeout() + + url = self.recipient_form.pop('url') + account_url = self.recipient_form.pop('account_url') + self.recipient_form['strong_authentication_confirm[code]'] = kwargs['code'] + self.location(url, data=self.recipient_form) + + self.recipient_form = None + return self.rcpt_after_sms(recipient, account_url) + + # step 1 of new recipient + return self.init_new_recipient(recipient) + + def rcpt_after_sms(self, recipient, account_url): + assert self.page.is_created(), 'The recipient was not added.' + + # at this point, the recipient was added to the webiste + # we just want here to return the right Recipient object + # we are taking it from the recipient list page + # because there is no summary of the adding + self.go_recipients_list(account_url, recipient.origin_account_id) + rec = find_object(self.page.iter_recipients(), id=recipient.id, error=RecipientNotFound) + return rec def iter_currencies(self): return self.currencylist.go().get_currency_list() diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 56ef045fd..b5e0b1943 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -25,6 +25,7 @@ from decimal import Decimal import re from io import BytesIO from datetime import date +from PIL import Image from weboob.browser.pages import HTMLPage, LoggedPage, pagination, NextPage, FormNotFound, PartialHTMLPage, LoginPage, CsvPage, RawPage, JsonPage from weboob.browser.elements import ListElement, ItemElement, method, TableElement, SkipItem, DictElement @@ -47,8 +48,7 @@ from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.value import Value from weboob.tools.date import parse_french_date -from weboob.tools.captcha.virtkeyboard import VirtKeyboard, VirtKeyboardError -from weboob.tools.compat import urljoin, urlencode, urlparse +from weboob.tools.compat import urljoin, urlencode, urlparse, range from weboob.exceptions import BrowserQuestion, BrowserIncorrectPassword, BrowserHTTPNotFound, BrowserUnavailable, ActionNeeded @@ -142,45 +142,72 @@ class VirtKeyboardPage(HTMLPage): pass -class BoursoramaVirtKeyboard(VirtKeyboard): - symbols = {'0': (17, 7, 24, 17), - '1': (18, 6, 21, 18), - '2': (9, 7, 32, 34), - '3': (10, 7, 31, 34), - '4': (11, 6, 29, 34), - '5': (14, 6, 28, 34), - '6': (7, 7, 34, 34), - '7': (5, 6, 36, 34), - '8': (8, 7, 32, 34), - '9': (4, 7, 38, 34)} +class BoursoramaVirtKeyboard(object): + symbols = { + '0': '0000110000001111110001110011100110000110111000011011100001101100000110111000011011100001100110000110011100111000111111000000110000', + '1': '0000110000000111000000111100000111110000011011000000001100000000110000000011000000001100000000110000000011000000001100000000110000', + '2': '0001111000011111110001100011100000000110000000011000000001100000001110000001110000001110000001110000001110000001111111110111111111', + '3': '0001111000011111111001100011100000000110000000011000000011100001111000000111110000000001100000000110111000111001111111100001110000', + '4': '0000011100000011110000001111000001101100000100110000110011000110001100011000110011111111101111111110000000111000000011000000001100', + '5': '1111111100111111110011100000001100000000110000000011111111001111111110010000011000000001100000000110111000111011111111000001110000', + '6': '0000111000001111111001110001000110000000011000000011101111001111111110111000011011100001100110000110011100111000111111000000111000', + '7': '0111111111011111111100000001100000000110000000111000000011000000011100000001100000000110000000110000000011000000011100000001100000', + '8': '0001111000011111111011100011101110000110011000011001111111000011111000011100111011100001101110000110111000111001111111100001111000', + '9': '0001110000011111110001100011101110000110110000011011100001100111111110001111011000000001100000000110001000110001111111000001110000', + } + + def __init__(self, browser, page): + self.browser = browser + self.fingerprints = {} + col = 0 - color = (255,255,255) + keys = page.doc.xpath('//ul[@class="password-input"]//button/@data-matrix-key') - def __init__(self, page): - self.md5 = {} for button in page.doc.xpath('//ul[@class="password-input"]//button'): - c = button.attrib['data-matrix-key'] txt = button.attrib['style'].replace('background-image:url(data:image/png;base64,', '').rstrip(');') - img = BytesIO(b64decode(txt.encode('ascii'))) - self.load_image(img, self.color, convert='RGB') - self.load_symbols((0, 0, 42, 42), c) - - def load_symbols(self, coords, c): - coord = self.get_symbol_coords(coords) - if coord == (-1, -1, -1, -1): - return - self.md5[coord] = c - - def get_code(self, password): - code = '' - for i, d in enumerate(password): - if i > 0: - code += '|' - try: - code += self.md5[self.symbols[d]] - except KeyError: - raise VirtKeyboardError() - return code + + img = Image.open(BytesIO(b64decode(txt.encode('ascii')))) + width, height = img.size + + img = img.crop((16, 6, width - 16, height - 23)) + width, height = img.size + + matrix = img.load() + s = "" + for y in range(height): + for x in range(width): + (r, g, b, a) = matrix[x, y] + # If the pixel is white and opaque enough + if a > 200 and r + g + b > 740: + s += "1" + else: + s += "0" + self.fingerprints[keys[col]] = s + col += 1 + + def get_symbol_code(self, char): + fingerprint = self.symbols[char] + for code, string in self.fingerprints.items(): + if fingerprint == string: + return code + # Image contains some noise, and the match is not always perfect + # (this is why we can't use md5 hashs) + # But if we can't find the perfect one, we can take the best one + best = 0 + result = None + for code, string in self.fingerprints.items(): + match = 0 + for j, bit in enumerate(string): + if bit == fingerprint[j]: + match += 1 + if match > best: + best = match + result = code + self.browser.logger.info(self.fingerprints[result] + "(" + result + ") match " + char) + return result + + def get_string_code(self, string): + return '|'.join(self.get_symbol_code(c) for c in string) class LoginPage(LoginPage, HTMLPage): @@ -199,8 +226,8 @@ class LoginPage(LoginPage, HTMLPage): password = ''.join([c if c.isdigit() else [k for k, v in self.TO_DIGIT.items() if c in v][0] for c in password.lower()]) form = self.get_form() keyboard_page = self.browser.keyboard.open() - vk = BoursoramaVirtKeyboard(keyboard_page) - code = vk.get_code(password) + vk = BoursoramaVirtKeyboard(self.browser, keyboard_page) + code = vk.get_string_code(password) form['form[login]'] = login form['form[fakePassword]'] = len(password) * '•' form['form[password]'] = code @@ -852,6 +879,8 @@ class ProfilePage(LoggedPage, HTMLPage): klass = Person obj_name = Format('%s %s %s', MySelect('genderTitle'), MyInput('firstName'), MyInput('lastName')) + obj_firstname = MyInput('firstName') + obj_lastname = MyInput('lastName') obj_nationality = CleanText(u'//span[contains(text(), "Nationalité")]/span') obj_spouse_name = MyInput('spouseFirstName') obj_children = CleanDecimal(MyInput('dependentChildren'), default=NotAvailable) @@ -993,10 +1022,7 @@ class TransferCharac(LoggedPage, HTMLPage): else: assert self.get_option(form.el.xpath('//select[@id="Characteristics_schedulingType"]')[0], 'Différé') == '2' form['Characteristics[schedulingType]'] = '2' - # If we let the 0 in the front of the month or the day like 02, the website will not interpret the good date - form['Characteristics[scheduledDate][day]'] = exec_date.strftime('%d').lstrip("0") - form['Characteristics[scheduledDate][month]'] = exec_date.strftime('%m').lstrip("0") - form['Characteristics[scheduledDate][year]'] = exec_date.strftime('%Y') + form['Characteristics[scheduledDate]'] = exec_date.strftime('%d/%m/%Y') form['Characteristics[notice]'] = 'none' form.submit() @@ -1009,25 +1035,25 @@ class TransferConfirm(LoggedPage, HTMLPage): raise TransferInvalidAmount(message=errors) def need_refresh(self): - return not self.doc.xpath('//form[@name="Confirm"]//button[contains(text(), "Je valide")]') + return not self.doc.xpath('//form[@name="Confirm"]//button[contains(text(), "Valider")]') @method class get_transfer(ItemElement): klass = Transfer - obj_label = CleanText('//div[@id="transfer-label"]/span[@class="transfer__account-value"]') - obj_amount = CleanDecimal('//div[@id="transfer-amount"]/span[@class="transfer__account-value"]', replace_dots=True) - obj_currency = CleanCurrency('//div[@id="transfer-amount"]/span[@class="transfer__account-value"]') + obj_label = CleanText('//span[@id="transfer-label"]/span[@class="transfer__account-value"]') + obj_amount = CleanDecimal.French('//span[@id="transfer-amount"]/span[@class="transfer__account-value"]') + obj_currency = CleanCurrency('//span[@id="transfer-amount"]/span[@class="transfer__account-value"]') obj_account_label = CleanText('//span[@id="transfer-origin-account"]') obj_recipient_label = CleanText('//span[@id="transfer-destination-account"]') def obj_exec_date(self): - type_ = CleanText('//div[@id="transfer-type"]/span[@class="transfer__account-value"]')(self) + type_ = CleanText('//span[@id="transfer-type"]/span[@class="transfer__account-value"]')(self) if type_ == 'Ponctuel': return datetime.date.today() elif type_ == 'Différé': - return Date(CleanText('//div[@id="transfer-date"]/span[@class="transfer__account-value"]'), dayfirst=True)(self) + return Date(CleanText('//span[@id="transfer-date"]/span[@class="transfer__account-value"]'), dayfirst=True)(self) def submit(self): form = self.get_form(name='Confirm') @@ -1066,42 +1092,31 @@ class AddRecipientPage(LoggedPage, HTMLPage): form['externalAccountsPrepareType[beneficiaryFirstname]'] = recipient.label form['externalAccountsPrepareType[bank]'] = recipient.bank_name or 'Autre' form['externalAccountsPrepareType[iban]'] = recipient.iban + form['submit'] = '' form.submit() def is_send_sms(self): - return self._is_form(name='otp_prepare') + return self._is_form(name='strong_authentication_prepare') def send_sms(self): - form = self.get_form(name='otp_prepare') - form['otp_prepare[receiveCode]'] = '' + form = self.get_form(name='strong_authentication_prepare') form.submit() def is_confirm_sms(self): - return self._is_form(name='otp_confirm') + return self._is_form(name='strong_authentication_confirm') - def confirm_sms(self, code): - form = self.get_form(name='otp_confirm') - form['otp_confirm[otpCode]'] = code - form.submit() + def get_confirm_sms_form(self): + form = self.get_form(name='strong_authentication_confirm') + recipient_form = {k: v for k, v in form.items()} + recipient_form['url'] = form.url + return recipient_form - def is_confirm(self): + def is_confirm_send_sms(self): return self._is_form(name='externalAccountsConfirmType') - def confirm(self): - self.get_form(name='externalAccountsConfirmType').submit() - - def get_recipient(self): - div = self.doc.xpath('//div[@class="confirmation__text"]')[0] - - ret = Recipient() - ret.label = CleanText('//p[b[contains(text(),"Libellé du compte :")]]/text()')(div) - ret.iban = ret.id = CleanText('//p[b[contains(text(),"Iban :")]]/text()')(div) - ret.bank_name = CleanText(u'//p[b[contains(text(),"Établissement bancaire :")]]/text()')(div) - ret.currency = u'EUR' - ret.category = u'Externe' - ret.enabled_at = datetime.date.today() - assert ret.label - return ret + def confirm_send_sms(self): + form = self.get_form(name='externalAccountsConfirmType') + form.submit() def is_created(self): return CleanText('//p[contains(text(), "Le bénéficiaire a bien été ajouté.")]')(self.doc) != "" diff --git a/modules/bp/browser.py b/modules/bp/browser.py index d2970cd61..cb3c47a19 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -26,12 +26,13 @@ from weboob.browser.exceptions import ServerError from weboob.capabilities.base import NotAvailable from weboob.exceptions import BrowserIncorrectPassword, BrowserBanned, NoAccountsException, BrowserUnavailable from weboob.tools.compat import urlsplit, urlunsplit, parse_qsl +from weboob.tools.decorators import retry from .pages import ( LoginPage, Initident, CheckPassword, repositionnerCheminCourant, BadLoginPage, AccountDesactivate, AccountList, AccountHistory, CardsList, UnavailablePage, AccountRIB, Advisor, TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary, CreateRecipient, ValidateRecipient, - ValidateCountry, ConfirmPage, RcptSummary, SubscriptionPage, DownloadPage, ProSubscriptionPage, + ValidateCountry, ConfirmPage, RcptSummary, SubscriptionPage, DownloadPage, ProSubscriptionPage, RevolvingAttributesPage, ) from .pages.accounthistory import ( LifeInsuranceInvest, LifeInsuranceHistory, LifeInsuranceHistoryInv, RetirementHistory, @@ -76,9 +77,11 @@ class BPBrowser(LoginBrowser, StatesMixin): '/voscomptes/canalXHTML/pret/creditRenouvelable/init-consulterCreditRenouvelable.ea', '/voscomptes/canalXHTML/pret/encours/rechercherPret-encoursPrets.ea', '/voscomptes/canalXHTML/sso/commun/init-integration.ea\?partenaire=cristalCEC', - '/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp', AccountList) - par_accounts_revolving = URL('https://espaceclientcreditconso.labanquepostale.fr/sav/accueil.do', AccountList) + + revolving_start = URL(r'/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp', AccountList) + par_accounts_revolving = URL(r'https://espaceclientcreditconso.labanquepostale.fr/sav/loginlbpcrypt.do', RevolvingAttributesPage) + accounts_rib = URL(r'.*voscomptes/canalXHTML/comptesCommun/imprimerRIB/init-imprimer_rib.ea.*', '/voscomptes/canalXHTML/comptesCommun/imprimerRIB/init-selection_rib.ea', AccountRIB) @@ -141,7 +144,11 @@ class BPBrowser(LoginBrowser, StatesMixin): r'/voscomptes/canalXHTML/virement/virementSafran_sepa/valider-virementSepa.ea', r'/voscomptes/canalXHTML/virement/virementSafran_sepa/confirmerInformations-virementSepa.ea', r'/voscomptes/canalXHTML/virement/virementSafran_national/valider-creerVirementNational.ea', - r'/voscomptes/canalXHTML/virement/virementSafran_national/validerVirementNational-virementNational.ea', TransferConfirm) + r'/voscomptes/canalXHTML/virement/virementSafran_national/validerVirementNational-virementNational.ea', + # the following url is already used in transfer_summary + # but we need it to detect the case where the website displaies the list of devices + # when a transfer is made with an otp or decoupled + r'/voscomptes/canalXHTML/virement/virementSafran_sepa/confirmer-creerVirementSepa.ea', TransferConfirm) transfer_summary = URL(r'/voscomptes/canalXHTML/virement/virementSafran_national/confirmerVirementNational-virementNational.ea', r'/voscomptes/canalXHTML/virement/virementSafran_pea/confirmerInformations-virementPea.ea', r'/voscomptes/canalXHTML/virement/virementSafran_sepa/confirmer-creerVirementSepa.ea', @@ -271,8 +278,7 @@ class BPBrowser(LoginBrowser, StatesMixin): accounts.append(student_loan) else: # The main revolving page is not accessible, we can reach it by this new way - self.location(self.absurl('/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp')) - self.page.go_revolving() + self.go_revolving() revolving_loan = self.page.get_revolving_attributes(account) accounts.append(revolving_loan) page.go() @@ -308,6 +314,11 @@ class BPBrowser(LoginBrowser, StatesMixin): return self.accounts + @retry(BrowserUnavailable, delay=5) + def go_revolving(self): + self.revolving_start.go() + self.page.go_revolving() + def iter_cards(self, account): self.deferred_card_history.go(accountId=account.id, monthIndex=0, cardIndex=0) if self.cards_list.is_here(): @@ -482,6 +493,7 @@ class BPBrowser(LoginBrowser, StatesMixin): self.page.confirm() # Should only happen if double auth. if self.transfer_confirm.is_here(): + self.page.choose_device() self.page.double_auth(transfer) return self.page.handle_response(transfer) diff --git a/modules/bp/pages/__init__.py b/modules/bp/pages/__init__.py index 939cdd8fd..b5b566aa3 100644 --- a/modules/bp/pages/__init__.py +++ b/modules/bp/pages/__init__.py @@ -19,7 +19,7 @@ from .login import LoginPage, Initident, CheckPassword,repositionnerCheminCourant, BadLoginPage, AccountDesactivate, UnavailablePage -from .accountlist import AccountList, AccountRIB, Advisor +from .accountlist import AccountList, AccountRIB, Advisor, RevolvingAttributesPage from .accounthistory import AccountHistory, CardsList from .transfer import TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary, CreateRecipient, ValidateRecipient,\ ValidateCountry, ConfirmPage, RcptSummary @@ -29,4 +29,4 @@ from .subscription import SubscriptionPage, DownloadPage, ProSubscriptionPage __all__ = ['LoginPage', 'Initident', 'CheckPassword', 'repositionnerCheminCourant', "AccountList", 'AccountHistory', 'BadLoginPage', 'AccountDesactivate', 'TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary', 'UnavailablePage', 'CardsList', 'AccountRIB', 'Advisor', 'CreateRecipient', 'ValidateRecipient', 'ValidateCountry', 'ConfirmPage', 'RcptSummary', - 'SubscriptionPage', 'DownloadPage', 'ProSubscriptionPage'] + 'SubscriptionPage', 'DownloadPage', 'ProSubscriptionPage', 'RevolvingAttributesPage'] diff --git a/modules/bp/pages/accounthistory.py b/modules/bp/pages/accounthistory.py index a92056fdb..88dc76eab 100644 --- a/modules/bp/pages/accounthistory.py +++ b/modules/bp/pages/accounthistory.py @@ -76,7 +76,7 @@ class AccountHistory(LoggedPage, MyHTMLPage): def get_next_link(self): for a in self.doc.xpath('//a[@class="btn_crt"]'): - txt = u''.join([txt.strip() for txt in a.itertext()]) + txt = u''.join([txt2.strip() for txt2 in a.itertext()]) if u'mois précédent' in txt: return a.attrib['href'] diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index 2239d7627..d142eca3b 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -189,7 +189,8 @@ class AccountList(LoggedPage, MyHTMLPage): def on_load(self): MyHTMLPage.on_load(self) - if self.doc.xpath('//h2[text()="ERREUR"]'): # website sometime crash + # website sometimes crash + if CleanText('//h2[text()="ERREUR"]')(self.doc): self.browser.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/initialiser-identif.ea') raise BrowserUnavailable() @@ -217,22 +218,6 @@ class AccountList(LoggedPage, MyHTMLPage): def condition(self): return item_account_generic.condition(self) - - def get_revolving_attributes(self, account): - loan = Loan() - loan.id = account.id - loan.label = '%s - %s' %(account.label, account.id) - loan.currency = account.currency - loan.url = account.url - - loan.used_amount = CleanDecimal.US('//tr[td[contains(text(), "Montant Utilisé") or contains(text(), "Montant utilisé")]]/td[2]')(self.doc) - loan.available_amount = CleanDecimal.US(Regexp(CleanText('//tr[td[contains(text(), "Montant Disponible") or contains(text(), "Montant disponible")]]/td[2]'), r'(.*) au'))(self.doc) - loan.balance = -loan.used_amount - loan._has_cards = False - loan.type = Account.TYPE_REVOLVING_CREDIT - return loan - - @method class iter_revolving_loans(ListElement): item_xpath = '//div[@class="bloc Tmargin"]//dl' @@ -463,3 +448,23 @@ class ProfilePage(LoggedPage, HTMLPage): profile.job = CleanText('//div[@id="persoIdentiteDetail"]//dd[4]')(self.doc) return profile + + +class RevolvingAttributesPage(LoggedPage, HTMLPage): + def on_load(self): + if CleanText('//h1[contains(text(), "Erreur")]')(self.doc): + raise BrowserUnavailable() + + def get_revolving_attributes(self, account): + loan = Loan() + loan.id = account.id + loan.label = '%s - %s' % (account.label, account.id) + loan.currency = account.currency + loan.url = account.url + + loan.used_amount = CleanDecimal.US('//tr[td[contains(text(), "Montant Utilisé") or contains(text(), "Montant utilisé")]]/td[2]')(self.doc) + loan.available_amount = CleanDecimal.US(Regexp(CleanText('//tr[td[contains(text(), "Montant Disponible") or contains(text(), "Montant disponible")]]/td[2]'), r'(.*) au'))(self.doc) + loan.balance = -loan.used_amount + loan._has_cards = False + loan.type = Account.TYPE_REVOLVING_CREDIT + return loan diff --git a/modules/bp/pages/transfer.py b/modules/bp/pages/transfer.py index d4e0b69ed..043e4f763 100644 --- a/modules/bp/pages/transfer.py +++ b/modules/bp/pages/transfer.py @@ -33,7 +33,7 @@ from weboob.browser.elements import ListElement, ItemElement, method, SkipItem from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.value import Value -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, AuthMethodNotImplemented from .base import MyHTMLPage @@ -140,9 +140,21 @@ class TransferConfirm(LoggedPage, CheckTransferError): def is_here(self): return ( not CleanText('//p[contains(text(), "Vous pouvez le consulter dans le menu")]')(self.doc) - or self.doc.xpath('//input[@title="Confirmer la demande de virement"]') + or self.doc.xpath('//input[@title="Confirmer la demande de virement"]') # appears when there is no need for otp/polling + or self.doc.xpath("//span[contains(text(), 'cliquant sur le bouton \"CONFIRMER\"')]") # appears on the page when there is a 'Confirmer' button or not ) + def choose_device(self): + # When there is no "Confirmer" button, + # it means that the device pop up appeared (it is called by js) + if ( + not self.doc.xpath('//input[@value="Confirmer"]') + or self.doc.xpath('//input[@name="codeOTPSaisi"]') + ): + # transfer validation form with sms cannot be tested yet + raise AuthMethodNotImplemented() + assert False, 'Should not be on confirmation page after posting the form.' + def double_auth(self, transfer): code_needed = CleanText('//label[@for="code_securite"]')(self.doc) if code_needed: @@ -185,6 +197,8 @@ class TransferConfirm(LoggedPage, CheckTransferError): class TransferSummary(LoggedPage, CheckTransferError): + is_here = '//h3[contains(text(), "Récapitulatif")]' + def handle_response(self, transfer): summary_filter = CleanText( '//div[contains(@class, "bloc-recapitulatif")]//p' diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 8b5ed8985..282e9d94f 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -73,60 +73,77 @@ class CaisseEpargne(LoginBrowser, StatesMixin): LINEBOURSE_BROWSER = LinebourseAPIBrowser - login = URL('/authentification/manage\?step=identification&identifiant=(?P.*)', - 'https://.*/login.aspx', LoginPage) - account_login = URL('/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) - loading = URL('https://.*/CreditConso/ReroutageCreditConso.aspx', LoadingPage) - cons_loan = URL('https://www.credit-conso-cr.caisse-epargne.fr/websavcr-web/rest/contrat/getContrat\?datePourIe=(?P)', ConsLoanPage) - transaction_detail = URL('https://.*/Portail.aspx.*', TransactionsDetailsPage) - recipient = URL('https://.*/Portail.aspx.*', RecipientPage) - transfer = URL('https://.*/Portail.aspx.*', TransferPage) - transfer_summary = URL('https://.*/Portail.aspx.*', TransferSummaryPage) - transfer_confirm = URL('https://.*/Portail.aspx.*', TransferConfirmPage) - pro_transfer = URL('https://.*/Portail.aspx.*', ProTransferPage) - pro_transfer_confirm = URL('https://.*/Portail.aspx.*', ProTransferConfirmPage) - pro_transfer_summary = URL('https://.*/Portail.aspx.*', ProTransferSummaryPage) - pro_add_recipient_otp = URL('https://.*/Portail.aspx.*', ProAddRecipientOtpPage) - pro_add_recipient = URL('https://.*/Portail.aspx.*', ProAddRecipientPage) - measure_page = URL('https://.*/Portail.aspx.*', MeasurePage) - cards_old = URL('https://.*/Portail.aspx.*', CardsOldWebsitePage) - cards = URL('https://.*/Portail.aspx.*', CardsPage) - cards_coming = URL('https://.*/Portail.aspx.*', CardsComingPage) + login = URL( + r'/authentification/manage\?step=identification&identifiant=(?P.*)', + r'https://.*/login.aspx', + LoginPage + ) + account_login = URL(r'/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) + loading = URL(r'https://.*/CreditConso/ReroutageCreditConso.aspx', LoadingPage) + cons_loan = URL(r'https://www.credit-conso-cr.caisse-epargne.fr/websavcr-web/rest/contrat/getContrat\?datePourIe=(?P)', ConsLoanPage) + transaction_detail = URL(r'https://.*/Portail.aspx.*', TransactionsDetailsPage) + recipient = URL(r'https://.*/Portail.aspx.*', RecipientPage) + transfer = URL(r'https://.*/Portail.aspx.*', TransferPage) + transfer_summary = URL(r'https://.*/Portail.aspx.*', TransferSummaryPage) + transfer_confirm = URL(r'https://.*/Portail.aspx.*', TransferConfirmPage) + pro_transfer = URL(r'https://.*/Portail.aspx.*', ProTransferPage) + pro_transfer_confirm = URL(r'https://.*/Portail.aspx.*', ProTransferConfirmPage) + pro_transfer_summary = URL(r'https://.*/Portail.aspx.*', ProTransferSummaryPage) + pro_add_recipient_otp = URL(r'https://.*/Portail.aspx.*', ProAddRecipientOtpPage) + pro_add_recipient = URL(r'https://.*/Portail.aspx.*', ProAddRecipientPage) + measure_page = URL(r'https://.*/Portail.aspx.*', MeasurePage) + cards_old = URL(r'https://.*/Portail.aspx.*', CardsOldWebsitePage) + cards = URL(r'https://.*/Portail.aspx.*', CardsPage) + cards_coming = URL(r'https://.*/Portail.aspx.*', CardsComingPage) old_checkings_levies = URL(r'https://.*/Portail.aspx.*', OldLeviesPage) new_checkings_levies = URL(r'https://.*/Portail.aspx.*', NewLeviesPage) - authent = URL('https://.*/Portail.aspx.*', AuthentPage) - subscription = URL('https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) + authent = URL(r'https://.*/Portail.aspx.*', AuthentPage) + subscription = URL(r'https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) transaction_popup = URL(r'https://.*/Portail.aspx.*', TransactionPopupPage) - home = URL('https://.*/Portail.aspx.*', IndexPage) - home_tache = URL('https://.*/Portail.aspx\?tache=(?P).*', IndexPage) - error = URL('https://.*/login.aspx', - 'https://.*/Pages/logout.aspx.*', - 'https://.*/particuliers/Page_erreur_technique.aspx.*', ErrorPage) - market = URL('https://.*/Pages/Bourse.*', - 'https://www.caisse-epargne.offrebourse.com/ReroutageSJR', - r'https://www.caisse-epargne.offrebourse.com/fr/6CE.*', MarketPage) - unavailable_page = URL('https://www.caisse-epargne.fr/.*/au-quotidien', UnavailablePage) - - creditcooperatif_market = URL('https://www.offrebourse.com/.*', CreditCooperatifMarketPage) # just to catch the landing page of the Credit Cooperatif's Linebourse - natixis_redirect = URL(r'/NaAssuranceRedirect/NaAssuranceRedirect.aspx', - r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/views/common/routage-itce.xhtml\?windowId=automatedEntryPoint', - NatixisRedirectPage) + home = URL(r'https://.*/Portail.aspx.*', IndexPage) + home_tache = URL(r'https://.*/Portail.aspx\?tache=(?P).*', IndexPage) + error = URL( + r'https://.*/login.aspx', + r'https://.*/Pages/logout.aspx.*', + r'https://.*/particuliers/Page_erreur_technique.aspx.*', + ErrorPage + ) + market = URL( + r'https://.*/Pages/Bourse.*', + r'https://www.caisse-epargne.offrebourse.com/ReroutageSJR', + r'https://www.caisse-epargne.offrebourse.com/fr/6CE.*', + MarketPage + ) + unavailable_page = URL(r'https://www.caisse-epargne.fr/.*/au-quotidien', UnavailablePage) + + creditcooperatif_market = URL(r'https://www.offrebourse.com/.*', CreditCooperatifMarketPage) # just to catch the landing page of the Credit Cooperatif's Linebourse + natixis_redirect = URL( + r'/NaAssuranceRedirect/NaAssuranceRedirect.aspx', + r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/views/common/routage-itce.xhtml\?windowId=automatedEntryPoint', + NatixisRedirectPage + ) life_insurance_history = URL(r'https://www.extranet2.caisse-epargne.fr/cin-front/contrats/evenements', LifeInsuranceHistory) life_insurance_investments = URL(r'https://www.extranet2.caisse-epargne.fr/cin-front/contrats/details', LifeInsuranceInvestments) - life_insurance = URL(r'https://.*/Assurance/Pages/Assurance.aspx', - r'https://www.extranet2.caisse-epargne.fr.*', LifeInsurance) + life_insurance = URL( + r'https://.*/Assurance/Pages/Assurance.aspx', + r'https://www.extranet2.caisse-epargne.fr.*', + LifeInsurance + ) natixis_life_ins_his = URL(r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load-operation/(?P\w+)/(?P\w+)/(?P)', NatixisLIHis) natixis_life_ins_inv = URL(r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load/(?P\w+)/(?P\w+)/(?P)', NatixisLIInv) message = URL(r'https://www.caisse-epargne.offrebourse.com/DetailMessage\?refresh=O', MessagePage) - garbage = URL(r'https://www.caisse-epargne.offrebourse.com/Portefeuille', - r'https://www.caisse-epargne.fr/particuliers/.*/emprunter.aspx', - r'https://.*/particuliers/emprunter.*', - r'https://.*/particuliers/epargner.*', GarbagePage) + garbage = URL( + r'https://www.caisse-epargne.offrebourse.com/Portefeuille', + r'https://www.caisse-epargne.fr/particuliers/.*/emprunter.aspx', + r'https://.*/particuliers/emprunter.*', + r'https://.*/particuliers/epargner.*', + GarbagePage + ) sms = URL(r'https://www.icgauth.caisse-epargne.fr/dacswebssoissuer/AuthnRequestServlet', SmsPage) sms_option = URL(r'https://www.icgauth.caisse-epargne.fr/dacstemplate-SOL/index.html\?transactionID=.*', SmsPageOption) request_sms = URL( r'https://(?Pwww.icgauth.[^/]+)/dacsrest/api/v1u0/transaction/(?P)', - SmsRequest, + SmsRequest ) __states__ = ( @@ -421,13 +438,14 @@ class CaisseEpargne(LoginBrowser, StatesMixin): if self.accounts is None: self.accounts = self.get_measure_accounts_list() if self.accounts is None: + owner_name = self.get_profile().name.upper().split(' ', 1)[1] if self.home.is_here(): self.page.check_no_accounts() self.page.go_list() else: self.home.go() - self.accounts = list(self.page.get_list()) + self.accounts = list(self.page.get_list(owner_name)) for account in self.accounts: self.deleteCTX() if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): @@ -485,6 +503,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): for card in self.page.iter_cards(): card.parent = account card._coming_info = self.page.get_card_coming_info(card.number, card.parent._card_links.copy()) + card.ownership = account.ownership self.accounts.append(card) self.home.go() @@ -506,6 +525,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): # If card.parent._card_links is not filled, it mean this checking account # has no coming transactions. card._coming_info = None + card.ownership = card.parent.ownership if info: self.page.go_list() self.page.go_history(info) @@ -715,6 +735,10 @@ class CaisseEpargne(LoginBrowser, StatesMixin): except (IndexError, AttributeError) as e: self.logger.error(e) return [] + except ServerError as e: + if e.response.status_code == 500: + raise BrowserUnavailable() + raise return self.page.iter_history() @need_login @@ -882,7 +906,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): profile = Profile() if len([k for k in self.session.cookies.keys() if k == 'CTX']) > 1: del self.session.cookies['CTX'] - elif 'username=' in self.session.cookies.get('CTX', ''): + if 'username=' in self.session.cookies.get('CTX', ''): profile.name = to_unicode(re.search('username=([^&]+)', self.session.cookies['CTX']).group(1)) elif 'nomusager=' in self.session.cookies.get('headerdei'): profile.name = to_unicode(re.search('nomusager=(?:[^&]+/ )?([^&]+)', self.session.cookies['headerdei']).group(1)) diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py index c4f83d8b9..52d75fb59 100644 --- a/modules/caissedepargne/cenet/browser.py +++ b/modules/caissedepargne/cenet/browser.py @@ -19,6 +19,7 @@ from __future__ import unicode_literals + import json from weboob.browser import LoginBrowser, need_login, StatesMixin @@ -47,26 +48,35 @@ class CenetBrowser(LoginBrowser, StatesMixin): STATE_DURATION = 5 - login = URL(r'https://(?P[^/]+)/authentification/manage\?step=identification&identifiant=(?P.*)', - r'https://.*/authentification/manage\?step=identification&identifiant=.*', - r'https://.*/login.aspx', LoginPage) - account_login = URL('https://(?P[^/]+)/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) - cenet_vk = URL('https://www.cenet.caisse-epargne.fr/Web/Api/ApiAuthentification.asmx/ChargerClavierVirtuel') - cenet_home = URL('/Default.aspx$', CenetHomePage) - cenet_accounts = URL('/Web/Api/ApiComptes.asmx/ChargerSyntheseComptes', CenetAccountsPage) - cenet_loans = URL('/Web/Api/ApiFinancements.asmx/ChargerListeFinancementsMLT', CenetLoanPage) - cenet_account_history = URL('/Web/Api/ApiComptes.asmx/ChargerHistoriqueCompte', CenetAccountHistoryPage) - cenet_account_coming = URL('/Web/Api/ApiCartesBanquaires.asmx/ChargerEnCoursCarte', CenetAccountHistoryPage) - cenet_tr_detail = URL('/Web/Api/ApiComptes.asmx/ChargerDetailOperation', CenetCardSummaryPage) - cenet_cards = URL('/Web/Api/ApiCartesBanquaires.asmx/ChargerCartes', CenetCardsPage) - error = URL(r'https://.*/login.aspx', - r'https://.*/Pages/logout.aspx.*', - r'https://.*/particuliers/Page_erreur_technique.aspx.*', ErrorPage) - cenet_login = URL(r'https://.*/$', - r'https://.*/default.aspx', CenetLoginPage) - - subscription = URL('/Web/Api/ApiReleves.asmx/ChargerListeEtablissements', SubscriptionPage) - documents = URL('/Web/Api/ApiReleves.asmx/ChargerListeReleves', SubscriptionPage) + login = URL( + r'https://(?P[^/]+)/authentification/manage\?step=identification&identifiant=(?P.*)', + r'https://.*/authentification/manage\?step=identification&identifiant=.*', + r'https://.*/login.aspx', + LoginPage, + ) + account_login = URL(r'https://(?P[^/]+)/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) + cenet_vk = URL(r'https://www.cenet.caisse-epargne.fr/Web/Api/ApiAuthentification.asmx/ChargerClavierVirtuel') + cenet_home = URL(r'/Default.aspx$', CenetHomePage) + cenet_accounts = URL(r'/Web/Api/ApiComptes.asmx/ChargerSyntheseComptes', CenetAccountsPage) + cenet_loans = URL(r'/Web/Api/ApiFinancements.asmx/ChargerListeFinancementsMLT', CenetLoanPage) + cenet_account_history = URL(r'/Web/Api/ApiComptes.asmx/ChargerHistoriqueCompte', CenetAccountHistoryPage) + cenet_account_coming = URL(r'/Web/Api/ApiCartesBanquaires.asmx/ChargerEnCoursCarte', CenetAccountHistoryPage) + cenet_tr_detail = URL(r'/Web/Api/ApiComptes.asmx/ChargerDetailOperation', CenetCardSummaryPage) + cenet_cards = URL(r'/Web/Api/ApiCartesBanquaires.asmx/ChargerCartes', CenetCardsPage) + error = URL( + r'https://.*/login.aspx', + r'https://.*/Pages/logout.aspx.*', + r'https://.*/particuliers/Page_erreur_technique.aspx.*', + ErrorPage, + ) + cenet_login = URL( + r'https://.*/$', + r'https://.*/default.aspx', + CenetLoginPage, + ) + + subscription = URL(r'/Web/Api/ApiReleves.asmx/ChargerListeEtablissements', SubscriptionPage) + documents = URL(r'/Web/Api/ApiReleves.asmx/ChargerListeReleves', SubscriptionPage) download = URL(r'/Default.aspx\?dashboard=ComptesReleves&lien=SuiviReleves', DownloadDocumentPage) __states__ = ('BASEURL',) @@ -192,7 +202,7 @@ class CenetBrowser(LoginBrowser, StatesMixin): card_tr_list.append(tr) tr.deleted = True - tr_dict = [tr_dict for tr_dict in data_out if tr_dict['Libelle'] == tr.label] + tr_dict = [tr_dict2 for tr_dict2 in data_out if tr_dict2['Libelle'] == tr.label] donneesEntree = {} donneesEntree['Compte'] = account._formated donneesEntree['ListeOperations'] = [tr_dict[0]] @@ -280,15 +290,15 @@ class CenetBrowser(LoginBrowser, StatesMixin): def iter_documents(self, subscription): sub_id = subscription.id input_filter = { - 'Page':0, - 'NombreParPage':0, - 'Tris':[], - 'Criteres':[ - {'Champ': 'Etablissement','TypeCritere': 'Equals','Value': sub_id}, - {'Champ': 'DateDebut','TypeCritere': 'Equals','Value': None}, - {'Champ': 'DateFin','TypeCritere': 'Equals','Value': None}, - {'Champ': 'MaxRelevesAffichesParNumero','TypeCritere': 'Equals','Value': '100'} - ] + 'Page': 0, + 'NombreParPage': 0, + 'Tris': [], + 'Criteres': [ + {'Champ': 'Etablissement', 'TypeCritere': 'Equals', 'Value': sub_id}, + {'Champ': 'DateDebut', 'TypeCritere': 'Equals', 'Value': None}, + {'Champ': 'DateFin', 'TypeCritere': 'Equals', 'Value': None}, + {'Champ': 'MaxRelevesAffichesParNumero', 'TypeCritere': 'Equals', 'Value': '100'}, + ], } json_data = { 'contexte': '', diff --git a/modules/caissedepargne/cenet/pages.py b/modules/caissedepargne/cenet/pages.py index 028a004fa..833356eb9 100644 --- a/modules/caissedepargne/cenet/pages.py +++ b/modules/caissedepargne/cenet/pages.py @@ -35,36 +35,23 @@ from weboob.exceptions import BrowserUnavailable class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^CB (?P.*?) FACT (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('^RET(RAIT)? DAB (?P
\d+)-(?P\d+)-.*', re.IGNORECASE), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^RET(RAIT)? DAB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2}) (?P\d{2})H(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^VIR(EMENT)?(\.PERIODIQUE)? (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_TRANSFER), - (re.compile('^PRLV (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_ORDER), - (re.compile('^CHEQUE.*', re.IGNORECASE), FrenchTransaction.TYPE_CHECK), - (re.compile('^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile(r'^\* (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile('^REMISE (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^(?P.*)( \d+)? QUITTANCE .*', re.IGNORECASE), - FrenchTransaction.TYPE_ORDER), - (re.compile('^CB [\d\*]+ TOT DIF .*', re.IGNORECASE), - FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile('^CB [\d\*]+ (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('^CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('\*CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('^FAC CB (?P.*?) (?P
\d{2})/(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - ] + PATTERNS = [ + (re.compile(r'^CB (?P.*?) FACT (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'^RET(RAIT)? DAB (?P
\d+)-(?P\d+)-.*', re.IGNORECASE), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^RET(RAIT)? DAB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2}) (?P\d{2})H(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^VIR(EMENT)?(\.PERIODIQUE)? (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^PRLV (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^CHEQUE.*', re.IGNORECASE), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile(r'^\* (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile(r'^REMISE (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P.*)( \d+)? QUITTANCE .*', re.IGNORECASE), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^CB [\d\*]+ TOT DIF .*', re.IGNORECASE), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^CB [\d\*]+ (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'^CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'\*CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'^FAC CB (?P.*?) (?P
\d{2})/(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + ] class LoginPage(JsonPage): @@ -120,7 +107,7 @@ class CenetJsonPage(JsonPage): class CenetAccountsPage(LoggedPage, CenetJsonPage): - ACCOUNT_TYPES = {u'CCP': Account.TYPE_CHECKING} + ACCOUNT_TYPES = {'CCP': Account.TYPE_CHECKING} @method class get_accounts(DictElement): @@ -139,7 +126,6 @@ class CenetAccountsPage(LoggedPage, CenetJsonPage): return -absolut_amount return absolut_amount - def obj_currency(self): return CleanText(Dict('Devise'))(self).upper() @@ -214,6 +200,7 @@ class CenetCardsPage(LoggedPage, CenetJsonPage): return cards + class CenetAccountHistoryPage(LoggedPage, CenetJsonPage): TR_TYPES_LABEL = { 'VIR': Transaction.TYPE_TRANSFER, @@ -224,10 +211,10 @@ class CenetAccountHistoryPage(LoggedPage, CenetJsonPage): TR_TYPES_API = { 'VIR': Transaction.TYPE_TRANSFER, - 'PE': Transaction.TYPE_ORDER, # PRLV - 'CE': Transaction.TYPE_CHECK, # CHEQUE - 'DE': Transaction.TYPE_CASH_DEPOSIT, # APPRO - 'PI': Transaction.TYPE_CASH_DEPOSIT, # REMISE CHEQUE + 'PE': Transaction.TYPE_ORDER, # PRLV + 'CE': Transaction.TYPE_CHECK, # CHEQUE + 'DE': Transaction.TYPE_CASH_DEPOSIT, # APPRO + 'PI': Transaction.TYPE_CASH_DEPOSIT, # REMISE CHEQUE } @method diff --git a/modules/caissedepargne/module.py b/modules/caissedepargne/module.py index f3ec10e52..299478f0a 100644 --- a/modules/caissedepargne/module.py +++ b/modules/caissedepargne/module.py @@ -17,9 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -from collections import OrderedDict +from __future__ import unicode_literals + import re from decimal import Decimal +from collections import OrderedDict from weboob.capabilities.bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, Account, RecipientNotFound from weboob.capabilities.bill import ( @@ -40,20 +42,22 @@ __all__ = ['CaisseEpargneModule'] class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapContact, CapProfile): NAME = 'caissedepargne' - MAINTAINER = u'Romain Bignon' + MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '1.6' - DESCRIPTION = u'Caisse d\'Épargne' + DESCRIPTION = 'Caisse d\'Épargne' LICENSE = 'LGPLv3+' BROWSER = ProxyBrowser website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'www.caisse-epargne.fr': u'Caisse d\'Épargne', 'www.banquebcp.fr': u'Banque BCP', }.items(), key=lambda k_v: (k_v[1], k_v[0]))]) - CONFIG = BackendConfig(Value('website', label='Banque', choices=website_choices, default='www.caisse-epargne.fr'), - ValueBackendPassword('login', label='Identifiant client', masked=False), - ValueBackendPassword('password', label='Code personnel', regexp='\d+'), - Value('nuser', label='User ID (optional)', default='', regexp='[A-Z\d]{0,8}')) + CONFIG = BackendConfig( + Value('website', label='Banque', choices=website_choices, default='www.caisse-epargne.fr'), + ValueBackendPassword('login', label='Identifiant client', masked=False), + ValueBackendPassword('password', label='Code personnel', regexp='\d+'), + Value('nuser', label='User ID (optional)', default='', regexp='[A-Z\d]{0,8}'), + ) accepted_document_types = (DocumentTypes.OTHER,) @@ -117,7 +121,7 @@ class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, Ca return self.browser.execute_transfer(transfer) def new_recipient(self, recipient, **params): - #recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) + # recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) def iter_resources(self, objs, split_path): diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 6a9ef5f71..375b8d40b 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import unicode_literals + from base64 import b64decode from collections import OrderedDict import re @@ -38,7 +39,7 @@ from weboob.browser.filters.html import Link, Attr, TableCell from weboob.capabilities import NotAvailable from weboob.capabilities.bank import ( Account, Investment, Recipient, TransferBankError, Transfer, - AddRecipientBankError, Loan, + AddRecipientBankError, Loan, AccountOwnership, ) from weboob.capabilities.bill import DocumentTypes, Subscription, Document from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -49,15 +50,18 @@ from weboob.tools.compat import unicode from weboob.exceptions import NoAccountsException, BrowserUnavailable, ActionNeeded from weboob.browser.filters.json import Dict + def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True) return CleanDecimal(*args, **kwargs) + class MyTableCell(TableCell): def __init__(self, *names, **kwargs): super(MyTableCell, self).__init__(*names, **kwargs) self.td = './tr[%s]/td' + def fix_form(form): keys = ['MM$HISTORIQUE_COMPTE$btnCumul', 'Cartridge$imgbtnMessagerie', 'MM$m_CH$ButtonImageFondMessagerie', 'MM$m_CH$ButtonImageMessagerie'] @@ -105,7 +109,7 @@ class CaissedepargneKeyboard(GridVirtKeyboard): class GarbagePage(LoggedPage, HTMLPage): def on_load(self): - go_back_link = Link('//a[@class="btn"]', default=NotAvailable)(self.doc) + go_back_link = Link('//a[@class="btn" or @class="cta_stroke back"]', default=NotAvailable)(self.doc) if go_back_link is not NotAvailable: assert len(go_back_link) != 1 @@ -161,6 +165,7 @@ class Transaction(FrenchTransaction): (re.compile(r'^RACHAT PARTIEL', re.IGNORECASE), FrenchTransaction.TYPE_BANK), ] + class IndexPage(LoggedPage, HTMLPage): ACCOUNT_TYPES = { 'Epargne liquide': Account.TYPE_SAVINGS, @@ -218,23 +223,25 @@ class IndexPage(LoggedPage, HTMLPage): raise BrowserUnavailable(mess) # This page is sometimes an useless step to the market website. - bourse_link = Link(u'//div[@id="MM_COMPTE_TITRE_pnlbourseoic"]//a[contains(text(), "Accédez à la consultation")]', default=None)(self.doc) + bourse_link = Link('//div[@id="MM_COMPTE_TITRE_pnlbourseoic"]//a[contains(text(), "Accédez à la consultation")]', default=None)(self.doc) if bourse_link: self.browser.location(bourse_link) def need_auth(self): - return bool(CleanText(u'//span[contains(text(), "Authentification non rejouable")]')(self.doc)) + return bool(CleanText('//span[contains(text(), "Authentification non rejouable")]')(self.doc)) def check_no_loans(self): - return not bool(CleanText(u'//table[@class="menu"]//div[contains(., "Crédits")]')(self.doc)) and \ - not bool(CleanText(u'//table[@class="header-navigation_main"]//a[contains(., "Crédits")]')(self.doc)) + return ( + not bool(CleanText('//table[@class="menu"]//div[contains(., "Crédits")]')(self.doc)) + and not bool(CleanText('//table[@class="header-navigation_main"]//a[contains(., "Crédits")]')(self.doc)) + ) def check_measure_accounts(self): - return not CleanText(u'//div[@class="MessageErreur"]/ul/li[contains(text(), "Aucun compte disponible")]')(self.doc) + return not CleanText('//div[@class="MessageErreur"]/ul/li[contains(text(), "Aucun compte disponible")]')(self.doc) def check_no_accounts(self): - no_account_message = CleanText(u'//span[@id="MM_LblMessagePopinError"]/p[contains(text(), "Aucun compte disponible")]')(self.doc) + no_account_message = CleanText('//span[@id="MM_LblMessagePopinError"]/p[contains(text(), "Aucun compte disponible")]')(self.doc) if no_account_message: raise NoAccountsException(no_account_message) @@ -243,7 +250,7 @@ class IndexPage(LoggedPage, HTMLPage): # The site might be broken: id in js: 4097800039137N418S00197, id in title: 1379418S001 (N instead of 9) # So we seek for a 1 letter difference and replace if found .... (so sad) for i in range(len(info['id']) - len(acc_id) + 1): - sub_part = info['id'][i:i+len(acc_id)] + sub_part = info['id'][i:i + len(acc_id)] z = zip(sub_part, acc_id) if len([tuple_letter for tuple_letter in z if len(set(tuple_letter)) > 1]) == 1: info['link'] = info['link'].replace(sub_part, acc_id) @@ -280,7 +287,7 @@ class IndexPage(LoggedPage, HTMLPage): def is_account_inactive(self, account_id): return self.doc.xpath('//tr[td[contains(text(), $id)]][@class="Inactive"]', id=account_id) - def _add_account(self, accounts, link, label, account_type, balance, number=None): + def _add_account(self, accounts, link, label, account_type, balance, number=None, ownership=NotAvailable): info = self._get_account_info(link, accounts) if info is None: self.logger.warning('Unable to parse account %r: %r' % (label, link)) @@ -294,11 +301,14 @@ class IndexPage(LoggedPage, HTMLPage): account._info = info account.number = number account.label = label + account.ownership = ownership account.type = self.ACCOUNT_TYPES.get(label, info['acc_type'] if 'acc_type' in info else account_type) if 'PERP' in account.label: account.type = Account.TYPE_PERP if 'NUANCES CAPITALISATI' in account.label: account.type = Account.TYPE_CAPITALISATION + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): + account.ownership = AccountOwnership.OWNER balance = balance or self.get_balance(account) @@ -324,13 +334,13 @@ class IndexPage(LoggedPage, HTMLPage): balance = page.doc.xpath('.//tr[td[contains(@id,"NumContrat")]]/td[@class="somme"]/a[contains(@href, $id)]', id=account.id) if len(balance) > 0: balance = CleanText('.')(balance[0]) - balance = balance if balance != u'' else NotAvailable + balance = balance if balance != '' else NotAvailable else: # Specific xpath for some Life Insurances: balance = page.doc.xpath('//tr[td[contains(text(), $id)]]/td/div[contains(@id, "Solde")]', id=account.id) if len(balance) > 0: balance = CleanText('.')(balance[0]) - balance = balance if balance != u'' else NotAvailable + balance = balance if balance != '' else NotAvailable else: # sometimes the accounts are attached but no info is available balance = NotAvailable @@ -351,7 +361,7 @@ class IndexPage(LoggedPage, HTMLPage): accounts_id.append(re.search("(\d{6,})", Attr('.', 'href')(a)).group(1)) return accounts_id - def get_list(self): + def get_list(self, owner_name): accounts = OrderedDict() # Old website @@ -361,9 +371,11 @@ class IndexPage(LoggedPage, HTMLPage): for tr in table.xpath('./tr'): tds = tr.findall('td') if tr.attrib.get('class', '') == 'DataGridHeader': - account_type = self.ACCOUNT_TYPES.get(tds[1].text.strip()) or\ - self.ACCOUNT_TYPES.get(CleanText('.')(tds[2])) or\ - self.ACCOUNT_TYPES.get(CleanText('.')(tds[3]), Account.TYPE_UNKNOWN) + account_type = ( + self.ACCOUNT_TYPES.get(tds[1].text.strip()) + or self.ACCOUNT_TYPES.get(CleanText('.')(tds[2])) + or self.ACCOUNT_TYPES.get(CleanText('.')(tds[3]), Account.TYPE_UNKNOWN) + ) else: # On the same row, there could have many accounts (check account and a card one). # For the card line, the number will be the same than the checking account, so we skip it. @@ -384,7 +396,7 @@ class IndexPage(LoggedPage, HTMLPage): balance = CleanText('.')(tds[-1].xpath('./a')[i]) self._add_account(accounts, a, label, account_type, balance) - self.logger.debug('we are on the %s website', 'old' if accounts else 'new') + self.logger.warning('we are on the %s website', 'old' if accounts else 'new') if len(accounts) == 0: # New website @@ -408,12 +420,24 @@ class IndexPage(LoggedPage, HTMLPage): # (perhaps only on creditcooperatif) label = CleanText('./strong')(tds[0]) balance = CleanText('.')(tds[-1]) + ownership = self.get_ownership(tds, owner_name) - account = self._add_account(accounts, a, label, account_type, balance) + account = self._add_account(accounts, a, label, account_type, balance, ownership=ownership) if account: account.number = CleanText('.')(tds[1]) - return accounts.values() + return list(accounts.values()) + + def get_ownership(self, tds, owner_name): + if len(tds) > 2: + account_owner = CleanText('.', default=None)(tds[2]).upper() + if account_owner and any(title in account_owner for title in ('M', 'MR', 'MLLE', 'MLE', 'MME')): + if re.search(r'(m|mr|me|mme|mlle|mle|ml)\.? ?(.*)\bou (m|mr|me|mme|mlle|mle|ml)\b(.*)', account_owner, re.IGNORECASE): + return AccountOwnership.CO_OWNER + elif all(n in account_owner for n in owner_name.split()): + return AccountOwnership.OWNER + return AccountOwnership.ATTORNEY + return NotAvailable def is_access_error(self): error_message = u"Vous n'êtes pas autorisé à accéder à cette fonction" @@ -463,12 +487,12 @@ class IndexPage(LoggedPage, HTMLPage): account_type = self.ACCOUNT_TYPES.get(CleanText('.')(title), Account.TYPE_UNKNOWN) for tr in table.xpath('./table/tbody/tr[contains(@id,"MM_SYNTHESE_CREDITS") and contains(@id,"IdTrGlobal")]'): tds = tr.findall('td') - if len(tds) == 0 : + if len(tds) == 0: continue for i in tds[0].xpath('.//a/strong'): label = i.text.strip() break - if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) == False: + if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) is False: # in case of Consumer credit or revolving credit, we substract avalaible amount with max amout # to get what was spend balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) - Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) @@ -481,6 +505,9 @@ class IndexPage(LoggedPage, HTMLPage): account.balance = -abs(balance) account.currency = account.get_currency(CleanText('.')(tds[-1])) account._card_links = [] + # The website doesn't show any information relative to the loan + # owner, we can then assume they all belong to the credentials owner. + account.ownership = AccountOwnership.OWNER if "renouvelables" in CleanText('.')(title): if 'JSESSIONID' in self.browser.session.cookies: @@ -493,7 +520,7 @@ class IndexPage(LoggedPage, HTMLPage): account.available_amount = float_to_decimal(d['situationCredit']['disponible']) account.next_payment_amount = float_to_decimal(d['situationCredit']['mensualiteEnCours']) accounts[account.id] = account - return accounts.values() + return list(accounts.values()) @method class get_real_estate_loans(ListElement): @@ -505,12 +532,12 @@ class IndexPage(LoggedPage, HTMLPage): item_xpath = './table[@class="static"][1]/tbody' head_xpath = './table[@class="static"][1]/tbody/tr/th' - col_total_amount = u'Capital Emprunté' - col_rate = u'Taux d’intérêt nominal' - col_balance = u'Capital Restant Dû' - col_last_payment_date = u'Dernière échéance' - col_next_payment_amount = u'Montant prochaine échéance' - col_next_payment_date = u'Prochaine échéance' + col_total_amount = 'Capital Emprunté' + col_rate = 'Taux d’intérêt nominal' + col_balance = 'Capital Restant Dû' + col_last_payment_date = 'Dernière échéance' + col_next_payment_amount = 'Montant prochaine échéance' + col_next_payment_date = 'Prochaine échéance' def parse(self, el): self.env['id'] = CleanText("./h2")(el).split()[-1] @@ -530,6 +557,9 @@ class IndexPage(LoggedPage, HTMLPage): obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date", default=''), default=NotAvailable), default=NotAvailable) obj_rate = MyDecimal(MyTableCell("rate", default=NotAvailable), default=NotAvailable) + # The website doesn't show any information relative to the loan + # owner, we can then assume they all belong to the credentials owner. + obj_ownership = AccountOwnership.OWNER def submit_form(self, form, eventargument, eventtarget, scriptmanager): form['__EVENTARGUMENT'] = eventargument @@ -726,7 +756,7 @@ class IndexPage(LoggedPage, HTMLPage): i = min(len(tds) - 4, 1) if tr.attrib.get('class', '') == 'DataGridHeader': - if tds[2].text == u'Titulaire': + if tds[2].text == 'Titulaire': ignore = True else: ignore = False @@ -742,14 +772,14 @@ class IndexPage(LoggedPage, HTMLPage): t = Transaction() - date = u''.join([txt.strip() for txt in tds[i+0].itertext()]) - raw = u' '.join([txt.strip() for txt in tds[i+1].itertext()]) - debit = u''.join([txt.strip() for txt in tds[-2].itertext()]) - credit = u''.join([txt.strip() for txt in tds[-1].itertext()]) + date = ''.join([txt.strip() for txt in tds[i + 0].itertext()]) + raw = ' '.join([txt.strip() for txt in tds[i + 1].itertext()]) + debit = ''.join([txt.strip() for txt in tds[-2].itertext()]) + credit = ''.join([txt.strip() for txt in tds[-1].itertext()]) t.parse(date, re.sub(r'[ ]+', ' ', raw)) - card_debit_date = self.doc.xpath(u'//span[@id="MM_HISTORIQUE_CB_m_TableTitle3_lblTitle"] | //label[contains(text(), "débiter le")]') + card_debit_date = self.doc.xpath('//span[@id="MM_HISTORIQUE_CB_m_TableTitle3_lblTitle"] | //label[contains(text(), "débiter le")]') if card_debit_date: t.rdate = t.bdate = Date(dayfirst=True).filter(date) m = re.search(r'\b(\d{2}/\d{2}/\d{4})\b', card_debit_date[0].text) @@ -819,7 +849,7 @@ class IndexPage(LoggedPage, HTMLPage): form.submit() def transfer_link(self): - return self.doc.xpath(u'//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') + return self.doc.xpath('//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') def go_transfer_via_history(self, account): self.go_history(account._info) @@ -844,7 +874,7 @@ class IndexPage(LoggedPage, HTMLPage): form.submit() def transfer_unavailable(self): - return CleanText(u'//li[contains(text(), "Pour accéder à cette fonctionnalité, vous devez disposer d’un moyen d’authentification renforcée")]')(self.doc) + return CleanText('//li[contains(text(), "Pour accéder à cette fonctionnalité, vous devez disposer d’un moyen d’authentification renforcée")]')(self.doc) def loan_unavailable_msg(self): msg = CleanText('//span[@id="MM_LblMessagePopinError"] | //p[@id="MM_ERREUR_PAGE_BLANCHE_pAlert"]')(self.doc) @@ -875,8 +905,8 @@ class IndexPage(LoggedPage, HTMLPage): def levies_page_enabled(self): """ Levies page does not exist in the nav bar for every connections """ return ( - CleanText('//a/span[contains(text(), "Suivre mes prélèvements reçus")]')(self.doc) or # new website - CleanText('//a[contains(text(), "Suivre les prélèvements reçus")]')(self.doc) # old website + CleanText('//a/span[contains(text(), "Suivre mes prélèvements reçus")]')(self.doc) # new website + or CleanText('//a[contains(text(), "Suivre les prélèvements reçus")]')(self.doc) # old website ) @@ -1021,8 +1051,8 @@ class CardsComingPage(IndexPage): # We must handle two kinds of Regexp because the 'X' are not # located at the same level for sub-modules such as palatine return Coalesce( - Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{6}\X{6}\d{4})', default=NotAvailable), - Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{4}\X{6}\d{6})', default=NotAvailable), + Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{6}X{6}\d{4})', default=NotAvailable), + Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{4}X{6}\d{6})', default=NotAvailable), )(self) def obj_number(self): @@ -1181,7 +1211,7 @@ class MarketPage(LoggedPage, HTMLPage): form.submit() def iter_investment(self): - for tbody in self.doc.xpath(u'//table[@summary="Contenu du portefeuille valorisé"]/tbody'): + for tbody in self.doc.xpath('//table[@summary="Contenu du portefeuille valorisé"]/tbody'): inv = Investment() inv.label = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a/span')[0]) inv.code = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a')[0]).split(' - ')[1] @@ -1195,7 +1225,7 @@ class MarketPage(LoggedPage, HTMLPage): yield inv def get_valuation_diff(self, account): - val = CleanText(self.doc.xpath(u'//td[contains(text(), "values latentes")]/following-sibling::*[1]')) + val = CleanText(self.doc.xpath('//td[contains(text(), "values latentes")]/following-sibling::*[1]')) account.valuation_diff = CleanDecimal(Regexp(val, '([^\(\)]+)'), replace_dots=True)(self) def is_on_right_portfolio(self, account): @@ -1205,7 +1235,7 @@ class MarketPage(LoggedPage, HTMLPage): return self.doc.xpath('//option[contains(text(), $id)]/@value', id=account._info['id'])[0] def come_back(self): - link = Link(u'//div/a[contains(text(), "Accueil accès client")]', default=NotAvailable)(self.doc) + link = Link('//div/a[contains(text(), "Accueil accès client")]', default=NotAvailable)(self.doc) if link: self.browser.location(link) @@ -1215,9 +1245,14 @@ class LifeInsurance(MarketPage): class LifeInsuranceHistory(LoggedPage, JsonPage): + def build_doc(self, text): + # If history is empty, there is no text + if not text: + return {} + return super(LifeInsuranceHistory, self).build_doc(text) + @method class iter_history(DictElement): - def find_elements(self): return self.el or [] # JSON contains 'null' if no transaction @@ -1234,7 +1269,7 @@ class LifeInsuranceHistory(LoggedPage, JsonPage): def obj_date(self): date = Dict('dateTraitement')(self) if date: - return datetime.fromtimestamp(date/1000) + return datetime.fromtimestamp(date / 1000) return NotAvailable obj_rdate = obj_date @@ -1242,7 +1277,7 @@ class LifeInsuranceHistory(LoggedPage, JsonPage): def obj_vdate(self): vdate = Dict('dateEffet')(self) if vdate: - return datetime.fromtimestamp(vdate/1000) + return datetime.fromtimestamp(vdate / 1000) return NotAvailable @@ -1270,7 +1305,7 @@ class LifeInsuranceInvestments(LoggedPage, JsonPage): def obj_vdate(self): vdate = Dict('cotation/date')(self) if vdate: - return datetime.fromtimestamp(vdate/1000) + return datetime.fromtimestamp(vdate / 1000) return NotAvailable def obj_quantity(self): @@ -1356,7 +1391,7 @@ class MyRecipient(ItemElement): klass = Recipient # Assume all recipients currency is euros. - obj_currency = u'EUR' + obj_currency = 'EUR' def obj_enabled_at(self): return datetime.now().replace(microsecond=0) @@ -1364,9 +1399,10 @@ class MyRecipient(ItemElement): class TransferErrorPage(object): def on_load(self): - errors_xpaths = ['//div[h2[text()="Information"]]/p[contains(text(), "Il ne pourra pas être crédité avant")]', - '//span[@id="MM_LblMessagePopinError"]/p | //div[h2[contains(text(), "Erreur de saisie")]]/p[1] | //span[@class="error"]/strong', - '//div[@id="MM_m_CH_ValidationSummary" and @class="MessageErreur"]', + errors_xpaths = [ + '//div[h2[text()="Information"]]/p[contains(text(), "Il ne pourra pas être crédité avant")]', + '//span[@id="MM_LblMessagePopinError"]/p | //div[h2[contains(text(), "Erreur de saisie")]]/p[1] | //span[@class="error"]/strong', + '//div[@id="MM_m_CH_ValidationSummary" and @class="MessageErreur"]', ] for error_xpath in errors_xpaths: @@ -1399,10 +1435,10 @@ class MyRecipients(ListElement): # Autres comptes if value == 'AC': raise SkipItem() - self.env['category'] = u'Interne' if value[0] == 'I' else u'Externe' - if self.env['category'] == u'Interne': + self.env['category'] = 'Interne' if value[0] == 'I' else 'Externe' + if self.env['category'] == 'Interne': # TODO use after 'I'? - _id = Regexp(CleanText('.'), r'- (\w+\d\w+)')(self) # at least one digit + _id = Regexp(CleanText('.'), r'- (\w+\d\w+)')(self) # at least one digit accounts = list(self.page.browser.get_accounts_list()) + list(self.page.browser.get_loans_list()) # If it's an internal account, we should always find only one account with _id in it's id. # Type card account contains their parent account id, and should not be listed in recipient account. @@ -1443,7 +1479,7 @@ class TransferPage(TransferErrorPage, IndexPage): RECIPIENT_XPATH = '//select[@id="MM_VIREMENT_SAISIE_VIREMENT_ddlCompteCrediter"]/option' def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Effectuer un virement")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Effectuer un virement")]')(self.doc)) def can_transfer(self, account): for o in self.doc.xpath('//select[@id="MM_VIREMENT_SAISIE_VIREMENT_ddlCompteDebiter"]/option'): @@ -1457,10 +1493,10 @@ class TransferPage(TransferErrorPage, IndexPage): return origin_value[0] def get_recipient_value(self, recipient): - if recipient.category == u'Externe': + if recipient.category == 'Externe': recipient_value = [Attr('.', 'value')(o) for o in self.doc.xpath(self.RECIPIENT_XPATH) if Regexp(CleanText('.'), '.* - ([A-Za-z0-9]*) -', default=NotAvailable)(o) == recipient.iban] - elif recipient.category == u'Interne': + elif recipient.category == 'Interne': recipient_value = [Attr('.', 'value')(o) for o in self.doc.xpath(self.RECIPIENT_XPATH) if Regexp(CleanText('.'), '- (\d+)', default=NotAvailable)(o) and Regexp(CleanText('.'), '- (\d+)', default=NotAvailable)(o) in recipient.id] assert len(recipient_value) == 1, 'error during recipient matching' @@ -1510,7 +1546,7 @@ class TransferPage(TransferErrorPage, IndexPage): def go_add_recipient(self): form = self.get_form(id='main') - link = self.doc.xpath(u'//a[span[contains(text(), "Ajouter un compte bénéficiaire")]]')[0] + link = self.doc.xpath('//a[span[contains(text(), "Ajouter un compte bénéficiaire")]]')[0] m = re.search("PostBackOptions?\([\"']([^\"']+)[\"'],\s*['\"]([^\"']+)?['\"]", link.attrib.get('href', '')) form['__EVENTTARGET'] = m.group(1) form['__EVENTARGUMENT'] = m.group(2) @@ -1531,7 +1567,7 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): return super(TransferErrorPage, self).build_doc(content) def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Confirmer mon virement")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Confirmer mon virement")]')(self.doc)) def confirm(self): form = self.get_form(id='main') @@ -1543,9 +1579,9 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): # transfer informations transfer.label = ( - CleanText(u'.//tr[td[contains(text(), "Motif de l\'opération")]]/td[not(@class)]')(self.doc) or - CleanText(u'.//tr[td[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) or - CleanText(u'.//tr[th[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) + CleanText('.//tr[td[contains(text(), "Motif de l\'opération")]]/td[not(@class)]')(self.doc) + or CleanText('.//tr[td[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) + or CleanText('.//tr[th[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) ) transfer.exec_date = Date(CleanText('.//tr[th[contains(text(), "En date du")]]/td[not(@class)]'), dayfirst=True)(self.doc) transfer.amount = CleanDecimal('.//tr[td[contains(text(), "Montant")]]/td[not(@class)] | \ @@ -1558,8 +1594,8 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): transfer.recipient_label = recipient.label transfer.recipient_id = recipient.id - if recipient.category == u'Externe': - for word in Upper(CleanText(u'.//tr[th[contains(text(), "Compte à créditer")]]/td[not(@class)]'))(self.doc).split(): + if recipient.category == 'Externe': + for word in Upper(CleanText('.//tr[th[contains(text(), "Compte à créditer")]]/td[not(@class)]'))(self.doc).split(): if is_iban_valid(word): transfer.recipient_iban = word break @@ -1580,7 +1616,7 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): class ProTransferConfirmPage(TransferConfirmPage): def is_here(self): - return bool(CleanText(u'//span[@id="MM_m_CH_lblTitle" and contains(text(), "Confirmez votre virement")]')(self.doc)) + return bool(CleanText('//span[@id="MM_m_CH_lblTitle" and contains(text(), "Confirmez votre virement")]')(self.doc)) def continue_transfer(self, origin_label, recipient, label): # Pro internal transfer initiation doesn't need a second step. @@ -1593,7 +1629,7 @@ class ProTransferConfirmPage(TransferConfirmPage): t.amount = CleanDecimal('//span[@id="MM_VIREMENT_CONF_VIREMENT_MontantVir"] | \ //span[@id="MM_VIREMENT_CONF_VIREMENT_lblMontantSelect"]', replace_dots=True)(self.doc) t.account_iban = account.iban - if recipient.category == u'Externe': + if recipient.category == 'Externe': for word in Upper(CleanText('//span[@id="MM_VIREMENT_CONF_VIREMENT_lblCptCrediterResult"]'))(self.doc).split(): if is_iban_valid(word): t.recipient_iban = word @@ -1618,10 +1654,10 @@ class ProTransferConfirmPage(TransferConfirmPage): class TransferSummaryPage(TransferErrorPage, IndexPage): def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Accusé de réception")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Accusé de réception")]')(self.doc)) def populate_reference(self, transfer): - transfer.id = Regexp(CleanText(u'//p[contains(text(), "a bien été enregistré")]'), '(\d+)')(self.doc) + transfer.id = Regexp(CleanText('//p[contains(text(), "a bien été enregistré")]'), '(\d+)')(self.doc) return transfer @@ -1638,7 +1674,7 @@ class ProTransferPage(TransferPage): RECIPIENT_XPATH = '//select[@id="MM_VIREMENT_SAISIE_VIREMENT_ddlCompteCrediterPro"]/option' def is_here(self): - return CleanText(u'//span[contains(text(), "Créer une liste de virements")] | //span[contains(text(), "Réalisez un virement")]')(self.doc) + return CleanText('//span[contains(text(), "Créer une liste de virements")] | //span[contains(text(), "Réalisez un virement")]')(self.doc) @method class iter_recipients(MyRecipients): @@ -1684,7 +1720,7 @@ class SmsRequest(LoggedPage, JsonPage): return self.doc['validationUnits'][0] def get_saml(self, otp_exception): - if not 'response' in self.doc: + if 'response' not in self.doc: error = self.doc['phase']['previousResult'] if error == 'FAILED_AUTHENTICATION': @@ -1704,7 +1740,7 @@ class SmsPage(LoggedPage, HTMLPage): raise AddRecipientBankError(message='Wrongcode, ' + error) def get_prompt_text(self): - return CleanText(u'//td[@class="auth_info_prompt"]')(self.doc) + return CleanText('//td[@class="auth_info_prompt"]')(self.doc) def post_form(self): form = self.get_form(name='downloadAuthForm') @@ -1724,7 +1760,7 @@ class SmsPage(LoggedPage, HTMLPage): class AuthentPage(LoggedPage, HTMLPage): def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Authentification réussie")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Authentification réussie")]')(self.doc)) def go_on(self): form = self.get_form(id='main') @@ -1742,7 +1778,7 @@ class RecipientPage(LoggedPage, HTMLPage): raise AddRecipientBankError(message=error) def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Ajouter un compte bénéficiaire")] |\ + return bool(CleanText('//h2[contains(text(), "Ajouter un compte bénéficiaire")] |\ //h2[contains(text(), "Confirmer l\'ajout d\'un compte bénéficiaire")]')(self.doc)) def post_recipient(self, recipient): @@ -1750,7 +1786,7 @@ class RecipientPage(LoggedPage, HTMLPage): form['__EVENTTARGET'] = '%s$m_WizardBar$m_lnkNext$m_lnkButton' % self.EVENTTARGET form['%s$m_RibIban$txtTitulaireCompte' % self.FORM_FIELD_ADD] = recipient.label for i in range(len(recipient.iban) // 4 + 1): - form['%s$m_RibIban$txtIban%s' % (self.FORM_FIELD_ADD, str(i + 1))] = recipient.iban[4*i:4*i+4] + form['%s$m_RibIban$txtIban%s' % (self.FORM_FIELD_ADD, str(i + 1))] = recipient.iban[4 * i:4 * i + 4] form.submit() def confirm_recipient(self): @@ -1775,7 +1811,7 @@ class ProAddRecipientOtpPage(IndexPage): self.browser.recipient_form['url'] = form.url def get_prompt_text(self): - return CleanText(u'////span[@id="MM_ANR_WS_AUTHENT_ANR_WS_AUTHENT_SAISIE_lblProcedure1"]')(self.doc) + return CleanText('////span[@id="MM_ANR_WS_AUTHENT_ANR_WS_AUTHENT_SAISIE_lblProcedure1"]')(self.doc) class ProAddRecipientPage(RecipientPage): @@ -1790,7 +1826,7 @@ class ProAddRecipientPage(RecipientPage): class TransactionsDetailsPage(LoggedPage, HTMLPage): def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Débits différés imputés")] | //span[@id="MM_m_CH_lblTitle" and contains(text(), "Débit différé imputé")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Débits différés imputés")] | //span[@id="MM_m_CH_lblTitle" and contains(text(), "Débit différé imputé")]')(self.doc)) @pagination @method @@ -1798,10 +1834,10 @@ class TransactionsDetailsPage(LoggedPage, HTMLPage): item_xpath = '//table[@id="MM_ECRITURE_GLOBALE_m_ExDGEcriture"]/tr[not(@class)] | //table[has-class("small special")]//tbody/tr[@class="rowClick"]' head_xpath = '//table[@id="MM_ECRITURE_GLOBALE_m_ExDGEcriture"]/tr[@class="DataGridHeader"]/td | //table[has-class("small special")]//thead/tr/th' - col_date = u'Date' - col_label = [u'Opération', u'Libellé'] - col_debit = u'Débit' - col_credit = u'Crédit' + col_date = 'Date' + col_label = ['Opération', 'Libellé'] + col_debit = 'Débit' + col_credit = 'Crédit' def next_page(self): # only for new website, don't have any accounts with enough deferred card transactions on old webiste @@ -1827,7 +1863,7 @@ class TransactionsDetailsPage(LoggedPage, HTMLPage): def go_form_to_summary(self): # return to first page - to_history = Link(self.doc.xpath(u'//a[contains(text(), "Retour à l\'historique")]'))(self.doc) + to_history = Link(self.doc.xpath('//a[contains(text(), "Retour à l\'historique")]'))(self.doc) n = re.match('.*\([\'\"](MM\$.*?)[\'\"],.*\)$', to_history) form = self.get_form(id='main') form['__EVENTTARGET'] = n.group(1) @@ -1865,6 +1901,7 @@ class SubscriptionPage(LoggedPage, HTMLPage): class iter_documents(ListElement): # sometimes there is several documents with same label at same date and with same content ignore_duplicate = True + @property def item_xpath(self): if Env('has_subscription')(self): @@ -1876,7 +1913,7 @@ class SubscriptionPage(LoggedPage, HTMLPage): obj_format = 'pdf' obj_url = Regexp(Link('.//td[@class="telecharger"]//a'), r'WebForm_PostBackOptions\("(\S*)"') - obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./td[2]', symbols='/', replace=[(' ', '_')]), Regexp(CleanText('./td[3]'), r'([\wé]*)')) + obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./td[2]', symbols='/', replace=[(' ', '_')]), Regexp(CleanText('./td[3]'), r'([\wé]*)')) obj_label = Format('%s %s', CleanText('./td[3]'), CleanText('./td[2]')) obj_date = Date(CleanText('./td[2]'), dayfirst=True) diff --git a/modules/cmes/browser.py b/modules/cmes/browser.py index f75fba03b..fa1533570 100644 --- a/modules/cmes/browser.py +++ b/modules/cmes/browser.py @@ -85,8 +85,12 @@ class CmesBrowser(LoginBrowser): self.accounts.stay_or_go(subsite=self.subsite, client_space=self.client_space) for inv in self.page.iter_investments(account=account): if inv._url: - # Go to the investment details to get performances + # Go to the investment details to get employee savings attributes self.location(inv._url) + + # Fetch SRRI, asset category & recommended period + self.page.fill_investment(obj=inv) + performances = {} # Get 1-year performance diff --git a/modules/cmes/pages.py b/modules/cmes/pages.py index c22962d7c..c13c3b0b0 100644 --- a/modules/cmes/pages.py +++ b/modules/cmes/pages.py @@ -20,11 +20,12 @@ from __future__ import unicode_literals import re + from weboob.browser.pages import HTMLPage, LoggedPage from weboob.browser.elements import ListElement, ItemElement, method from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Date, Regexp, Field, Currency, - Upper, MapIn, Eval, + Upper, MapIn, Eval, Title, ) from weboob.browser.filters.html import Link from weboob.capabilities.bank import Account, Investment, Pocket, NotAvailable @@ -33,11 +34,12 @@ from weboob.exceptions import ActionNeeded class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^(?P.*[Vv]ersement.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(u'^(?P([Aa]rbitrage|[Pp]rélèvements.*))'), FrenchTransaction.TYPE_ORDER), - (re.compile(u'^(?P([Rr]etrait|[Pp]aiement.*))'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(u'^(?P.*)'), FrenchTransaction.TYPE_BANK), - ] + PATTERNS = [ + (re.compile(r'^(?P.*[Vv]ersement.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P([Aa]rbitrage|[Pp]rélèvements.*))'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?P([Rr]etrait|[Pp]aiement.*))'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(?P.*)'), FrenchTransaction.TYPE_BANK), + ] def MyDecimal(*args, **kwargs): @@ -47,7 +49,7 @@ def MyDecimal(*args, **kwargs): class LoginPage(HTMLPage): def login(self, login, password): - form = self.get_form(name="bloc_ident") + form = self.get_form(name='bloc_ident') form['_cm_user'] = login form['_cm_pwd'] = password form.submit() @@ -100,10 +102,14 @@ class AccountsPage(LoggedPage, HTMLPage): def obj_id(self): # Use customer number + label to build account id - number = Regexp(CleanText('//div[@id="ei_tpl_fullSite"]//div[contains(@class, "ei_tpl_profil_content")]/p'), - r'(\d+)$', '\\1')(self) + number = Regexp( + CleanText('//div[@id="ei_tpl_fullSite"]//div[contains(@class, "ei_tpl_profil_content")]/p'), + r'(\d+)$', '\\1' + )(self) return Field('label')(self) + number + obj_number = obj_id + def iter_invest_rows(self, account): """ Process each invest row, extract elements needed to get @@ -161,7 +167,7 @@ class AccountsPage(LoggedPage, HTMLPage): pocket.condition = Pocket.CONDITION_AVAILABLE else: pocket.condition = Pocket.CONDITION_DATE - pocket.availability_date = Date(Regexp(Upper(CleanText('./td[1]')), 'AU[\s]+(.*)'), dayfirst=True)(row) + pocket.availability_date = Date(Regexp(Upper(CleanText('./td[1]')), r'AU[\s]+(.*)'), dayfirst=True)(row) yield pocket @@ -183,6 +189,24 @@ class AccountsPage(LoggedPage, HTMLPage): class InvestmentPage(LoggedPage, HTMLPage): + @method + class fill_investment(ItemElement): + # Sometimes there is a 'LIBELLES EN EURO' string joined with the category so we remove it + obj_asset_category = Title(CleanText('//tr[th[text()="Classification AMF"]]/td', replace=[('LIBELLES EN EURO', '')])) + + def obj_srri(self): + # Extract the value from '1/7' or '6/7' for instance + srri = Regexp(CleanText('//tr[th[text()="Niveau de risque"]]/td'), r'(\d+)/7', default=None)(self) + if srri: + return int(srri) + return NotAvailable + + def obj_recommended_period(self): + period = CleanText('//tr[th[text()="Durée de placement recommandée"]]/td')(self) + if period != 'NC': + return period + return NotAvailable + def get_form_url(self): form = self.get_form(id='C:P:F') return form.url diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index 94bcae26c..a8e64f0ec 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -260,9 +260,13 @@ class CmsoParBrowser(LoginBrowser, StatesMixin): self.history.go(data=json.dumps({'index': account._index}), page="detailcompte", headers=self.json_headers) - self.trs = {'lastdate': None, 'list': []} + self.trs = set() for tr in self.page.iter_history(index=account._index, nbs=nbs): + # Check for duplicates + if tr.id in self.trs: + continue + self.trs.add(tr.id) if has_deferred_cards and tr.type == Transaction.TYPE_CARD: tr.type = Transaction.TYPE_DEFERRED_CARD tr.bdate = tr.rdate @@ -283,8 +287,8 @@ class CmsoParBrowser(LoginBrowser, StatesMixin): self.history.go(data=json.dumps({"index": account._index}), page="pendingListOperations", headers=self.json_headers) + # There is no ids for comings, so no check for duplicates for key in self.page.get_keys(): - self.trs = {'lastdate': None, 'list': []} for c in self.page.iter_history(key=key): if hasattr(c, '_deferred_date'): c.bdate = c.rdate diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index e0c7ff0ef..39608ab8b 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -360,6 +360,9 @@ class HistoryPage(LoggedPage, JsonPage): obj_raw = Transaction.Raw(Dict('libelleCourt')) obj_vdate = Date(Dict('dateValeur', NotAvailable), dayfirst=True, default=NotAvailable) obj_amount = CleanDecimal(Dict('montantEnEuro'), default=NotAvailable) + # DO NOT USE `OperationID` the ids aren't constant after 1 month. `clefDomirama` seems + # to be constant forever. Must be kept under watch though + obj_id = Dict('clefDomirama', default='') def parse(self, el): key = Env('key', default=None)(self) @@ -370,15 +373,6 @@ class HistoryPage(LoggedPage, JsonPage): break setattr(self.obj, '_deferred_date', self.FromTimestamp().filter(deferred_date)) - # Skip duplicate transactions - amount = Dict('montantEnEuro', default=None)(self) - tr = Dict('libelleCourt')(self) + Dict('dateOperation', '')(self) + str(amount) - if amount is None or (tr in self.page.browser.trs['list'] and self.page.browser.trs['lastdate'] <= Field('date')(self)): - raise SkipItem() - - self.page.browser.trs['lastdate'] = Field('date')(self) - self.page.browser.trs['list'].append(tr) - class LifeinsurancePage(LoggedPage, HTMLPage): def get_account_id(self): diff --git a/modules/cragr/api/browser.py b/modules/cragr/api/browser.py index b2c330b19..22af0db50 100644 --- a/modules/cragr/api/browser.py +++ b/modules/cragr/api/browser.py @@ -31,6 +31,7 @@ from weboob.browser.exceptions import ServerError, ClientError, BrowserHTTPNotFo from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, ActionNeeded from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import sorted_transactions +from weboob.tools.decorators import retry from .pages import ( LoginPage, LoggedOutPage, KeypadPage, SecurityPage, ContractsPage, FirstConnectionPage, AccountsPage, AccountDetailsPage, @@ -99,7 +100,6 @@ class CragrAPI(LoginBrowser): TransferPage) old_website = URL(r'https://.*particuliers.html$', OldWebsitePage) - def __init__(self, website, *args, **kwargs): super(CragrAPI, self).__init__(*args, **kwargs) self.website = website @@ -118,6 +118,36 @@ class CragrAPI(LoginBrowser): super(CragrAPI, self).deinit() self.netfinca.deinit() + @retry(BrowserUnavailable) + def do_security_check(self): + try: + form = self.get_security_form() + self.security_check.go(data=form) + except ServerError as exc: + # Wrongpass returns a 500 server error... + exc_json = exc.response.json() + error = exc_json.get('error') + error_type = exc_json.get('erreurType') + if error: + message = error.get('message', '') + wrongpass_messages = ("Votre identification est incorrecte", "Vous n'avez plus droit") + if any(value in message for value in wrongpass_messages): + raise BrowserIncorrectPassword() + if 'obtenir un nouveau code' in message: + raise ActionNeeded(message) + + code = error.get('code', '') + technical_error_messages = ('Un incident technique', 'identifiant et votre code personnel') + # Sometimes there is no error message, so we try to use the code as well + technical_error_codes = ('technical_error',) + if any(value in message for value in technical_error_messages) or \ + any(value in code for value in technical_error_codes): + raise BrowserUnavailable(message) + elif error_type and 'UNAUTHORIZED_ERREUR_TYPE' in error_type: + # Usually appears when doing retries after a BrowserUnavailable + raise BrowserUnavailable() + raise + def do_login(self): if not self.username or not self.password: raise BrowserIncorrectPassword() @@ -133,31 +163,7 @@ class CragrAPI(LoginBrowser): self.logger.warning('This is a regional connection, switching to old website with URL %s', self.BASEURL) raise SiteSwitch('region') - form = self.get_security_form() - try: - self.security_check.go(data=form) - except ServerError as exc: - # Wrongpass returns a 500 server error... - error = exc.response.json().get('error') - if error: - message = error.get('message', '') - wrongpass_messages = ("Votre identification est incorrecte", "Vous n'avez plus droit") - if any(value in message for value in wrongpass_messages): - raise BrowserIncorrectPassword() - if 'obtenir un nouveau code' in message: - raise ActionNeeded(message) - technical_errors = ('Un incident technique', 'identifiant et votre code personnel') - if any(value in message for value in technical_errors): - # If it is a technical error, we try login again - form = self.get_security_form() - try: - self.security_check.go(data=form) - except ServerError as exc: - error = exc.response.json().get('error') - if error: - message = error.get('message', '') - if 'Un incident technique' in message: - raise BrowserUnavailable(message) + self.do_security_check() # accounts_url may contain '/particulier', '/professionnel', '/entreprise', '/agriculteur' or '/association' self.accounts_url = self.page.get_accounts_url() @@ -402,9 +408,10 @@ class CragrAPI(LoginBrowser): self.do_login() try: self.contracts_page.go(space=self.space, id_contract=contract) - except ServerError: - self.logger.warning('Space switch returned a 500 error, try again.') - self.contracts_page.go(space=self.space, id_contract=contract) + except ServerError as e: + if e.response.status_code == 500: + raise BrowserUnavailable() + raise assert self.accounts_page.is_here() @need_login diff --git a/modules/cragr/api/pages.py b/modules/cragr/api/pages.py index a47d5b9a7..57d0d410c 100644 --- a/modules/cragr/api/pages.py +++ b/modules/cragr/api/pages.py @@ -29,13 +29,15 @@ from weboob.exceptions import ActionNeeded from weboob.capabilities import NotAvailable from weboob.capabilities.base import empty from weboob.capabilities.bank import ( - Account, AccountOwnerType, Transaction, Investment, + Account, AccountOwnerType, Investment, ) +from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.profile import Person, Company from weboob.capabilities.contact import Advisor from weboob.browser.elements import DictElement, ItemElement, method from weboob.browser.filters.standard import ( - CleanText, CleanDecimal, Currency as CleanCurrency, Format, Field, Map, Eval, Env, Regexp, Date, Coalesce, + CleanText, CleanDecimal, Currency as CleanCurrency, Format, Field, Map, Eval, Env, + Regexp, Date, Coalesce, DateTime, ) from weboob.browser.filters.html import Attr from weboob.browser.filters.json import Dict @@ -43,10 +45,26 @@ from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.exceptions import BrowserPasswordExpired + def float_to_decimal(f): return Decimal(str(f)) +class Transaction(FrenchTransaction): + # this is only used to to find the rdate + PATTERNS = [ + (re.compile(r'^(?PPAIEMENT PAR CARTE) (?P.*) (?P
\d{2})/(?P\d{2})$'), None), + (re.compile(r'^(?PPRELEVEMENT) (?P.*) (?P
\d{2})/(?P\d{2})/(?P\d{4}) .*'), None), + (re.compile(r'^(?PPRELEVEMENT) (?P.*) (?P
\d{2})\s(?P\d{2})\s(?P\d{4}) .*'), None), + (re.compile(r'^(?PVIREMENT EN VOTRE FAVEUR) (?P.*) (?P
\d{2})\.(?P\d{2})\.(?P\d{4})$'), None), + (re.compile(r'^(?PREMBOURSEMENT DE PRET) (?P.*) (?P
\d{2})/(?P\d{2})/(?P\d{2,4})$'), None), + (re.compile(r'^(?PRETRAIT AU DISTRIBUTEUR) (?P.*) (?P
\d{2})/(?P\d{2}) .*'), None), + (re.compile(r'^(?PPRELEVEMENT URSSAF) (?P.*) (du)? (?P
\d{2})/(?P\d{2})/(?P\d{2,4})$'), None), + (re.compile(r"^(?PVERSEMENT D'ESPECES) (?P.*) (?P
\d{2})/(?P\d{2})/(?P\d{4}) .*"), None), + (re.compile(r'^(?PPRELEVEMENT) (?P.*) (?P
\d{2})-(?P\d{2})$'), None), + ] + + class KeypadPage(JsonPage): def build_password(self, password): # Fake Virtual Keyboard: just get the positions of each digit. @@ -425,20 +443,29 @@ class HistoryPage(LoggedPage, JsonPage): klass = Transaction + obj_date = DateTime(Dict('dateValeur')) + # There is a key in the json called dateOperation but most of the time it is the + # same as the dateValeur, meanwhile a different rdate is available in many labels. + # So we force all rdate to NotAvailable and only fill it when we find something to + # extract from the label + obj_rdate = NotAvailable # Transactions in foreign currencies have no 'libelleTypeOperation' # and 'libelleComplementaire' keys, hence the default values. # The CleanText() gets rid of additional spaces. - obj_raw = CleanText(Format('%s %s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')), CleanText(Dict('libelleComplementaire', default='')))) + obj_raw = Transaction.Raw( + CleanText( + Format( + '%s %s %s', + CleanText(Dict('libelleTypeOperation', default='')), + CleanText(Dict('libelleOperation')), + CleanText(Dict('libelleComplementaire', default='')) + ) + ) + ) obj_label = CleanText(Format('%s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')))) obj_amount = Eval(float_to_decimal, Dict('montant')) obj_type = Map(CleanText(Dict('libelleTypeOperation', default='')), TRANSACTION_TYPES, Transaction.TYPE_UNKNOWN) - def obj_date(self): - return dateutil.parser.parse(Dict('dateValeur')(self)) - - def obj_rdate(self): - return dateutil.parser.parse(Dict('dateOperation')(self)) - class CardsPage(LoggedPage, JsonPage): @method diff --git a/modules/cragr/regions/browser.py b/modules/cragr/regions/browser.py index aec2054c4..ce4e4ce02 100644 --- a/modules/cragr/regions/browser.py +++ b/modules/cragr/regions/browser.py @@ -324,17 +324,8 @@ class CragrRegion(LoginBrowser): return valid_accounts @need_login - def iter_perimeter_accounts(self, iban, all_accounts): - ''' - In order to use this method, we must pass the 3 accounts URLs: Regular, Wealth and Loans. - Accounts may appear on several URLs: we must check for duplicates before adding to cragr_accounts. - Once we fetched all cragr accounts, we go to the Netfinca space to get Netfinca accounts. - If there are account duplicates, we preferably yield the Netfinca version because it is more - complete ; in addition, Netfinca may contain accounts that do not appear on the cragr website. - ''' - cragr_accounts = [] - - # Regular accounts (Checking and Savings) + def iter_perimeter_regular_accounts(self, iban): + unique_ids = set() self.accounts.stay_or_go() self.page.set_cragr_code() for account in self.page.iter_accounts(): @@ -343,8 +334,23 @@ class CragrRegion(LoginBrowser): # Refresh account form in case it expired refreshed_account = find_object(self.page.iter_accounts(), id=account.id) account.iban = self.get_account_iban(refreshed_account._form) - if account.id not in [a.id for a in cragr_accounts]: - cragr_accounts.append(account) + + if account.id not in unique_ids: + # Do not yield accounts with duplicate IDs + unique_ids.add(account.id) + yield account + + @need_login + def iter_perimeter_accounts(self, iban, all_accounts): + ''' + In order to use this method, we must pass the 3 accounts URLs: Regular, Wealth and Loans. + Accounts may appear on several URLs: we must check for duplicates before adding to cragr_accounts. + Once we fetched all cragr accounts, we go to the Netfinca space to get Netfinca accounts. + If there are account duplicates, we preferably yield the Netfinca version because it is more + complete ; in addition, Netfinca may contain accounts that do not appear on the cragr website. + ''' + # Regular accounts (Checking & Savings) + cragr_accounts = list(self.iter_perimeter_regular_accounts(iban)) # Wealth accounts (PEA, Market, Life Insurances, PERP...) self.wealth.go() @@ -606,8 +612,9 @@ class CragrRegion(LoginBrowser): self.accounts.stay_or_go() self.go_to_perimeter(account._perimeter) - # No need to fetch IBANs and Netfinca accounts just to fetch an account form - refreshed_account = find_object(self.iter_perimeter_accounts(iban=False, all_accounts=False), AccountNotFound, id=account.id) + # Only fetch the perimeter's regular accounts (Checking & Savings) + # No need to go to Wealth, Loans or Netfinca for transactions + refreshed_account = find_object(self.iter_perimeter_regular_accounts(iban=False), AccountNotFound, id=account.id) refreshed_account._form.submit() if self.failed_history.is_here(): self.logger.warning('Form submission failed to reach the account history, we try again.') diff --git a/modules/cragr/regions/pages.py b/modules/cragr/regions/pages.py index 8f49ed811..5782e17b6 100644 --- a/modules/cragr/regions/pages.py +++ b/modules/cragr/regions/pages.py @@ -263,6 +263,7 @@ ACCOUNT_TYPES = { 'FLORIANE 2': Account.TYPE_LIFE_INSURANCE, 'CAP DECOUV': Account.TYPE_LIFE_INSURANCE, 'ESPACE LIB': Account.TYPE_LIFE_INSURANCE, + 'ESPACELIB3': Account.TYPE_LIFE_INSURANCE, 'ESP LIB 2': Account.TYPE_LIFE_INSURANCE, 'AST SELEC': Account.TYPE_LIFE_INSURANCE, 'PRGE': Account.TYPE_LIFE_INSURANCE, @@ -276,11 +277,6 @@ ACCOUNT_TYPES = { class AccountsPage(LoggedPage, CragrPage): - def on_load(self): - # Verify that all accounts page have the text 'Synthèse comptes' - if not CleanText('//h1[contains(text(), "Synthèse comptes")]')(self.doc): - self.logger.warning('We found an AccountsPage without the "Synthèse comptes" text.') - def no_other_perimeter(self): return not CleanText('//a[@title="Espace Autres Comptes"]')(self.doc) diff --git a/modules/creditdunord/browser.py b/modules/creditdunord/browser.py index e88e63efe..120a3936c 100644 --- a/modules/creditdunord/browser.py +++ b/modules/creditdunord/browser.py @@ -69,7 +69,8 @@ class CreditDuNordBrowser(LoginBrowser): not self.page.doc.xpath(u'//b[contains(text(), "vous devez modifier votre code confidentiel")]') def do_login(self): - self.login.go().login(self.username, self.password) + self.login.go() + self.page.login(self.username, self.password) if self.accounts.is_here(): expired_error = self.page.get_password_expired() if expired_error: @@ -134,10 +135,15 @@ class CreditDuNordBrowser(LoginBrowser): self.multitype_iban.go() link = self.page.iban_go() - for a in [a for a in accounts if a._acc_nb]: - if a.type != Account.TYPE_CARD: - self.location(link + a._acc_nb) - a.iban = self.page.get_iban() + if link: + # For some accounts, the IBAN is displayed somewhere else behind + # an OTP validation (icd/zco/public-index.html#zco/transac/impression_rib), + # the link is None if this is the case. + # TODO when we will be able to test this OTP + for a in accounts: + if a._acc_nb and a.type != Account.TYPE_CARD: + self.location(link + a._acc_nb) + a.iban = self.page.get_iban() return accounts diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index a8ae60202..ecc8dc6d9 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -79,6 +79,7 @@ class CDNVirtKeyboard(GridVirtKeyboard): def __init__(self, browser, crypto, grid): f = BytesIO(browser.open('/sec/vk/gen_ui?modeClavier=0&cryptogramme=%s' % crypto).content) + super(CDNVirtKeyboard, self).__init__(range(16), self.ncol, self.nrow, f, self.color) self.check_symbols(self.symbols, browser.responses_dirname) self.codes = grid @@ -95,7 +96,7 @@ class CDNVirtKeyboard(GridVirtKeyboard): for nbchar, c in enumerate(string): index = self.get_symbol_code(self.symbols[c]) res.append(self.codes[(nbchar * ndata) + index]) - return ','.join(res) + return ','.join(map(str, res)) class HTMLErrorPage(HTMLPage): @@ -228,7 +229,10 @@ class CDNBasePage(HTMLPage): return self.get_from_js("name: 'execution', value: '", "'") def iban_go(self): - return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) + value_from_js = self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"') + if not value_from_js: + return None + return '/vos-comptes/IPT/cdnProxyResource%s' % value_from_js class ProIbanPage(CDNBasePage): diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index d4c954956..69ef6673b 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -300,7 +300,7 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): elif acc.type in (Account.TYPE_MORTGAGE, Account.TYPE_LOAN) and acc._parent_id: acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable) - self.accounts_list = accounts_by_id.values() + self.accounts_list = list(accounts_by_id.values()) if has_no_account and not self.accounts_list: raise NoAccountsException(has_no_account) diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 4d9138c65..fcbbd1341 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -20,7 +20,6 @@ from __future__ import unicode_literals import re -import hashlib import time from decimal import Decimal, InvalidOperation @@ -90,6 +89,7 @@ class LoginPage(HTMLPage): # format login/password like login/password sent by firefox or chromium browser form['_cm_user'] = login.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252') form['_cm_pwd'] = passwd.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252') + form['_charset_'] = 'cp1252' form.submit() @property @@ -193,6 +193,10 @@ class UserSpacePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//form[@id="GoValider"]'): raise ActionNeeded("Le site du contrat Banque à Distance a besoin d'informations supplémentaires") + personal_infos = CleanText('//form[@class="_devb_act ___Form"]//div[contains(@class, "bloctxt")]/p[1]')(self.doc) + if 'Afin de compléter vos informations personnelles, renseignez le formulaire ci-dessous' in personal_infos: + raise ActionNeeded("Le site nécessite la saisie des informations personnelles de l'utilisateur.") + super(UserSpacePage, self).on_load() @@ -241,7 +245,7 @@ class item_account_generic(ItemElement): (re.compile(r'Ldd'), Account.TYPE_SAVINGS), (re.compile(r'Livret'), Account.TYPE_SAVINGS), (re.compile(r"Plan D'Epargne"), Account.TYPE_SAVINGS), - (re.compile(r'Tonic Croissance'), Account.TYPE_SAVINGS), + (re.compile(r'Tonic Crois'), Account.TYPE_SAVINGS), # eg: 'Tonic Croissance', 'Tonic Crois Pro' (re.compile(r'Tonic Societaire'), Account.TYPE_SAVINGS), (re.compile(r'Capital Expansion'), Account.TYPE_SAVINGS), (re.compile(r'Épargne'), Account.TYPE_SAVINGS), @@ -277,7 +281,9 @@ class item_account_generic(ItemElement): def loan_condition(self, check_no_details=False): _type = Field('type')(self) label = Field('label')(self) - details_link = Link('.//a', default=None)(self) + # The 'lien_inter_sites' link leads to a 404 and is not a link to loans details. + # The link name on the website is : Vos encours mobilisation de créances + details_link = Link('.//a[not(contains(@href, "lien_inter_sites"))]', default=None)(self) # mobile accounts are leading to a 404 error when parsing history # furthermore this is not exactly a loan account @@ -957,7 +963,7 @@ class CardPage(OperationsPage, LoggedPage): return not CleanText('//td[contains(., "Aucun mouvement")]', default=False)(self) def parse(self, el): - label = CleanText('//*[contains(text(), "Achats")]')(el) + label = CleanText('//span[contains(text(), "Achats")]/following-sibling::span[2]')(el) if not label: return try: @@ -988,16 +994,24 @@ class CardPage(OperationsPage, LoggedPage): def parse(self, el): try: - self.env['raw'] = "%s %s" % (CleanText().filter(TableCell('commerce')(self)[0].text), - CleanText().filter(TableCell('ville')(self)[0].text)) - except (ColumnNotFound, AttributeError): - self.env['raw'] = "%s" % (CleanText().filter(TableCell('commerce')(self)[0].text)) - - self.env['type'] = (Transaction.TYPE_DEFERRED_CARD - if CleanText('//a[contains(text(), "Prélevé fin")]', default=None) - else Transaction.TYPE_CARD) - self.env['differed_date'] = parse_french_date(Regexp(CleanText('//*[contains(text(), "Achats")]'), - r'au[\s]+(.*)')(self)).date() + self.env['raw'] = Format( + '%s %s', + CleanText(TableCell('commerce'), children=False), + CleanText(TableCell('ville')), + )(self) + except ColumnNotFound: + self.env['raw'] = CleanText(TableCell('commerce'), chilren=False)(self) + + if CleanText('//span[contains(text(), "Prélevé fin")]', default=None)(self): + self.env['type'] = Transaction.TYPE_DEFERRED_CARD + else: + self.env['type'] = Transaction.TYPE_CARD + + self.env['differed_date'] = Date( + CleanText('//span[contains(text(), "Achats")]/following-sibling::span[2]'), + parse_func=parse_french_date, + )(self) + amount = TableCell('credit')(self)[0] if self.page.browser.is_new_website: if not len(amount.xpath('./div')): @@ -1567,18 +1581,22 @@ class InternalTransferPage(LoggedPage, HTMLPage): def check_errors(self): # look for known errors content = self.text - messages = ['Le montant du virement doit être positif, veuillez le modifier', - 'Montant maximum autorisé au débit pour ce compte', - 'Dépassement du montant journalier autorisé', - 'Le solde de votre compte est insuffisant', - 'Nom prénom du bénéficiaire différent du titulaire. Utilisez un compte courant', - "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", - 'Montant maximum autorisé au crédit pour ce compte', - 'Débit interdit sur ce compte', - 'Virement interdit sur compte clos', - "L'intitulé du virement ne peut contenir le ou les caractères suivants", - 'La date ne peut être inférieure à la date du jour. Veuillez la corriger', - ] + messages = [ + 'Le montant du virement doit être positif, veuillez le modifier', + 'Montant maximum autorisé au débit pour ce compte', + 'Dépassement du montant journalier autorisé', + 'Le solde de votre compte est insuffisant', + 'Nom prénom du bénéficiaire différent du titulaire. Utilisez un compte courant', + "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", + 'Montant maximum autorisé au crédit pour ce compte', + 'Débit interdit sur ce compte', + 'Virement interdit sur compte clos', + 'Virement interdit sur compte inexistant', + "L'intitulé du virement ne peut contenir le ou les caractères suivants", + 'La date ne peut être inférieure à la date du jour. Veuillez la corriger', + 'Opération non conforme, les virements entre comptes Epargne ne sont pas autorisés.', + 'Opération non conforme: entre titulaires différents, seul un compte courant peut être débité.', + ] for message in messages: if message in content: @@ -1803,6 +1821,8 @@ class VerifCodePage(LoggedPage, HTMLPage): ) for error in errors: if error: + # don't reload state + self.browser.need_clear_storage = True raise AddRecipientBankError(message=error) action_needed = CleanText('//p[contains(text(), "Carte de CLÉS PERSONNELLES révoquée")]')(self.doc) @@ -1815,23 +1835,32 @@ class VerifCodePage(LoggedPage, HTMLPage): return v def get_question(self): - s = CleanText('//label[@for="txtCle"]')(self.doc) - img_hash_md5 = hashlib.md5(self.browser.open(Attr('//label[@for="txtCle"]/img', 'src')(self.doc)).content).hexdigest() - key_case = self.get_key_case(img_hash_md5) - assert key_case, "Hash %s not found, matching key case is available on session folder." % img_hash_md5 - return s[:25] + ' %s' % key_case + s[25:] + question = Regexp(CleanText('//div/p[input]'), r'(Veuillez .*):')(self.doc) + return question def post_code(self, key): - form = self.get_form(id='frm') - form['code'] = key - form['valChx.x'] = '1' - form['valChx.y'] = '1' + form = self.get_form('//form[contains(@action, "verif_code")]') + form['[t:xsd%3astring;]Data_KeyInput'] = key + + # we don't know the card id + # by default all users have only one card + # but to be sure, let's get it dynamically + do_validate = [k for k in form.keys() if '_FID_DoValidate_cardId' in k] + assert len(do_validate) == 1, 'There should be only one card.' + form[do_validate[0]] = '' + + activate = [k for k in form.keys() if '_FID_GoCardAction_action' in k] + assert len(activate) == 1, 'There should be only one card.' + del form[activate[0]] + form.submit() def handle_error(self): error_msg = CleanText('//div[@class="blocmsg info"]/p')(self.doc) # the card was not activated yet if 'veuillez activer votre carte' in error_msg: + # don't reload state + self.browser.need_clear_storage = True raise AddRecipientBankError(message=error_msg) @@ -1974,7 +2003,7 @@ class RevolvingLoansList(LoggedPage, HTMLPage): def obj_rate(self): if not self.async_load: - return MyDecimal(Regexp(CleanText('.//td[2]'), r'.* (\d*,\d*)%'))(self) + return MyDecimal(Regexp(CleanText('.//td[2]'), r'.* (\d*,\d*)%', default=NotAvailable))(self) class ErrorPage(HTMLPage): diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index 876f92ed0..4fe0209b4 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -105,6 +105,7 @@ class Fortuneo(LoginBrowser, StatesMixin): self.page.login(self.username, self.password) if self.login_page.is_here(): + self.page.check_is_blocked() raise BrowserIncorrectPassword() self.location('/fr/prive/default.jsp?ANav=1') diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 30dd0002c..0dfc2ff23 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -127,6 +127,7 @@ class PeaHistoryPage(LoggedPage, HTMLPage): return False form['dateDebut'] = (date.today() - relativedelta(years=2)).strftime('%d/%m/%Y') form['nbResultats'] = '100' + form['typeOperation'] = '01' form.submit() return True diff --git a/modules/fortuneo/pages/login.py b/modules/fortuneo/pages/login.py index 3b177a59b..ec10ef688 100644 --- a/modules/fortuneo/pages/login.py +++ b/modules/fortuneo/pages/login.py @@ -17,10 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.browser.pages import HTMLPage from weboob.browser.filters.standard import CleanText -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, ActionNeeded class LoginPage(HTMLPage): @@ -35,6 +36,11 @@ class LoginPage(HTMLPage): form['passwd'] = passwd form.submit() + def check_is_blocked(self): + error_message = CleanText('//div[@id="acces_client"]//p[@class="container error"]/label')(self.doc) + if 'Votre accès est désormais bloqué' in error_message: + raise ActionNeeded(error_message) + class UnavailablePage(HTMLPage): def on_load(self): diff --git a/modules/ganassurances/browser.py b/modules/ganassurances/browser.py index 25638009e..9ed56ec21 100644 --- a/modules/ganassurances/browser.py +++ b/modules/ganassurances/browser.py @@ -17,129 +17,16 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -import re +from __future__ import unicode_literals -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword -from weboob.capabilities.bank import Account -from weboob.capabilities.base import empty +from weboob.browser import AbstractBrowser -from .pages import LoginPage, AccountsPage, TransactionsPage, AVAccountPage, AVHistoryPage, FormPage, IbanPage, AvJPage +class GanAssurancesBrowser(AbstractBrowser): + PARENT = 'ganpatrimoine' + PARENT_ATTR = 'package.browser.GanPatrimoineBrowser' -__all__ = ['GanAssurancesBrowser'] - - -class GanAssurancesBrowser(LoginBrowser): - BASEURL = 'https://espaceclient.ganassurances.fr' - - login = URL(r'https://authentification.(?P.*).fr/cas/login', LoginPage) - iban = URL(r'/wps/myportal/!ut/(.*)/\?paramNumCpt=(.*)', IbanPage) - accounts = URL(r'/wps/myportal/TableauDeBord', AccountsPage) - transactions = URL(r'/wps/myportal/!ut', TransactionsPage) - av_account_form = URL(r'/wps/myportal/assurancevie/', FormPage) - av_account = URL(r'https://secure-rivage.ganassurances.fr/contratVie.rivage.syntheseContratEparUc.gsi', - r'/front/vie/epargne/contrat/(.*)', AVAccountPage) - av_history = URL(r'https://secure-rivage.(?P.*).fr/contratVie.rivage.mesOperations.gsi', AVHistoryPage) - av_secondary = URL(r'/api/ecli/vie/contrats/(?P.*)', AvJPage) - - def __init__(self, *args, **kwargs): - super(GanAssurancesBrowser, self).__init__(*args, **kwargs) - self.website = 'ganassurances' - self.domain = 'ganass' - - def do_login(self): - login_url = 'https://espaceclient.%s.fr/login-%s' % (self.website, self.domain) - self.login.go(website=self.website, params={'service': login_url}) - self.page.login(self.username, self.password) - - if self.login.is_here(): - error_msg = self.page.get_error() - if error_msg and "LOGIN_ERREUR_MOT_PASSE_INVALIDE" in error_msg: - raise BrowserIncorrectPassword() - assert False, 'Unhandled error at login: %s' % error_msg - - # For life asssurance accounts, to get balance we use the link from the account. - # And to get history (or other) we need to use the link again but the link works only once. - # So we get balance only for iter_account to not use the new link each time. - @need_login - def get_accounts_list(self, balance=True, need_iban=False): - accounts = [] - self.accounts.stay_or_go() - for account in self.page.get_list(): - if account.type == Account.TYPE_LIFE_INSURANCE and balance: - assert empty(account.balance) - self.location(account._link) - if self.av_account_form.is_here(): - self.page.av_account_form() - account.balance, account.currency = self.page.get_av_balance() - # New page where some AV are stored - elif "front/vie/" in account._link: - link = re.search(r'contrat\/(.+)-Groupama', account._link) - if link: - self.av_secondary.go(id_contrat=link.group(1)) - account.balance, account.currency = self.page.get_av_balance() - - self.accounts.stay_or_go() - if account.balance or not balance: - if account.type != Account.TYPE_LIFE_INSURANCE and need_iban: - self.location(account._link) - if self.transactions.is_here() and self.page.has_iban(): - self.page.go_iban() - account.iban = self.page.get_iban() - accounts.append(account) - return accounts - - def _get_history(self, account): - if "front/vie" in account._link: - return [] - accounts = self.get_accounts_list(balance=False) - for a in accounts: - if a.id == account.id: - self.location(a._link) - if a.type == Account.TYPE_LIFE_INSURANCE: - if not self.page.av_account_form(): - self.logger.warning('history form not found for %s', account) - return [] - self.av_history.go(website=self.website) - return self.page.get_av_history() - assert self.transactions.is_here() - return self.page.get_history(accid=account.id) - return [] - - # Duplicate line in case of arbitration because the site has only one line for the 2 transactions (debit and credit on the same line) - def get_history(self, account): - for tr in self._get_history(account): - yield tr - if getattr(tr, '_arbitration', False): - tr = tr.copy() - tr.amount = -tr.amount - yield tr - - def get_coming(self, account): - if account.type == Account.TYPE_LIFE_INSURANCE: - return [] - for a in self.get_accounts_list(): - if a.id == account.id: - self.location(a._link) - assert self.transactions.is_here() - link = self.page.get_coming_link() - if link is not None: - self.location(self.page.get_coming_link()) - assert self.transactions.is_here() - return self.page.get_history(accid=account.id) - return [] - - def get_investment(self, account): - if account.type != Account.TYPE_LIFE_INSURANCE: - return [] - for a in self.get_accounts_list(balance=False): - if a.id == account.id: - # There isn't any invest on AV having front/vie - # in theirs url - if "front/vie/" not in account._link: - self.location(a._link) - self.page.av_account_form() - if self.av_account.is_here(): - return self.page.get_av_investments() - return [] + def __init__(self, website, *args, **kwargs): + super(GanAssurancesBrowser, self).__init__(website, *args, **kwargs) + self.website = website + self.BASEURL = 'https://espaceclient.%s.fr' % website diff --git a/modules/ganassurances/module.py b/modules/ganassurances/module.py index 8b2bf530b..9a3f2f93e 100644 --- a/modules/ganassurances/module.py +++ b/modules/ganassurances/module.py @@ -17,13 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . - from __future__ import unicode_literals -from weboob.capabilities.bank import CapBankWealth, AccountNotFound -from weboob.capabilities.base import find_object -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import Value, ValueBackendPassword +from weboob.capabilities.bank import CapBank +from weboob.tools.backend import AbstractModule, BackendConfig +from weboob.tools.value import ValueBackendPassword from .browser import GanAssurancesBrowser @@ -31,7 +29,7 @@ from .browser import GanAssurancesBrowser __all__ = ['GanAssurancesModule'] -class GanAssurancesModule(Module, CapBankWealth): +class GanAssurancesModule(AbstractModule, CapBank): NAME = 'ganassurances' MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -39,28 +37,18 @@ class GanAssurancesModule(Module, CapBankWealth): DESCRIPTION = 'Gan Assurances' LICENSE = 'LGPLv3+' CONFIG = BackendConfig( - Value('login', label='Numéro client'), + ValueBackendPassword('login', label='Numéro client', masked=False), ValueBackendPassword('password', label="Code d'accès") ) + + PARENT = 'ganpatrimoine' BROWSER = GanAssurancesBrowser + def create_default_browser(self): return self.create_browser( + 'ganassurances', self.config['login'].get(), - self.config['password'].get() + self.config['password'].get(), + weboob=self.weboob ) - - def iter_accounts(self): - return self.browser.get_accounts_list(need_iban=True) - - def get_account(self, _id): - return find_object(self.browser.get_accounts_list(need_iban=True), id=_id, error=AccountNotFound) - - def iter_history(self, account): - return self.browser.get_history(account) - - def iter_coming(self, account): - return self.browser.get_coming(account) - - def iter_investment(self, account): - return self.browser.get_investment(account) diff --git a/modules/ganassurances/pages.py b/modules/ganassurances/pages.py deleted file mode 100644 index edd8b6f7d..000000000 --- a/modules/ganassurances/pages.py +++ /dev/null @@ -1,304 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2012 Romain Bignon -# -# This file is part of a weboob module. -# -# This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This weboob module is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this weboob module. If not, see . - - -from __future__ import unicode_literals - -import re -import requests -import ast - -from decimal import Decimal - -from weboob.browser.pages import HTMLPage, pagination, LoggedPage, FormNotFound, JsonPage -from weboob.browser.elements import method, TableElement, ItemElement -from weboob.browser.filters.standard import Env, CleanDecimal, CleanText, Date, Regexp, Eval, Field -from weboob.browser.filters.html import Attr, Link, TableCell -from weboob.browser.filters.javascript import JSVar -from weboob.capabilities.bank import Account, Investment -from weboob.capabilities.base import NotAvailable -from weboob.tools.capabilities.bank.transactions import FrenchTransaction -from weboob.tools.capabilities.bank.investments import is_isin_valid -from weboob.browser.filters.json import Dict - - -class LoginPage(HTMLPage): - def login(self, login, passwd): - tab = re.search(r'clavierAChristian = (\[[\d,\s]*\])', self.text).group(1) - number_list = ast.literal_eval(tab) - key_map = {} - for i, number in enumerate(number_list): - if number < 10: - key_map[number] = chr(ord('A') + i) - pass_string = ''.join(key_map[int(n)] for n in passwd) - form = self.get_form(name='loginForm') - form['username'] = login - form['password'] = pass_string - form.submit() - - def get_error(self): - return CleanText('//div[@id="msg"]')(self.doc) - - -class AccountsPage(LoggedPage, HTMLPage): - ACCOUNT_TYPES = { - 'Solde des comptes bancaires - Groupama Banque': Account.TYPE_CHECKING, - 'Solde des comptes bancaires': Account.TYPE_CHECKING, - 'Epargne bancaire constituée - Groupama Banque': Account.TYPE_SAVINGS, - 'Epargne bancaire constituée': Account.TYPE_SAVINGS, - 'Mes crédits': Account.TYPE_LOAN, - 'Assurance Vie': Account.TYPE_LIFE_INSURANCE, - 'Certificats Mutualistes': Account.TYPE_SAVINGS, - } - - ACCOUNT_TYPES2 = { - 'plan epargne actions': Account.TYPE_PEA, - } - - def get_list(self): - account_type = Account.TYPE_UNKNOWN - accounts = [] - - for tr in self.doc.xpath('//div[@class="finance"]/form/table[@class="ecli"]/tr'): - if tr.attrib.get('class', '') == 'entete': - account_type = self.ACCOUNT_TYPES.get(tr.find('th').text.strip(), Account.TYPE_UNKNOWN) - continue - - tds = tr.findall('td') - a = tds[0].find('a') - - # Skip accounts that can't be accessed - if a is None: - continue - - balance = tds[-1].text.strip() - - account = Account() - account.label = ' '.join([txt.strip() for txt in tds[0].itertext()]) - account.label = re.sub(r'[\s\xa0\u2022]+', ' ', account.label).strip() - - # take "N° (FOO123 456)" but "N° (FOO123) MR. BAR" - account.id = re.search(r'N° (\w+( \d+)*)', account.label).group(1).replace(' ', '') - account.number = account.id - account.type = account_type - - for patt, type in self.ACCOUNT_TYPES2.items(): - if patt in account.label.lower(): - account.type = type - break - - if balance: - account.balance = Decimal(FrenchTransaction.clean_amount(balance)) - account.currency = account.get_currency(balance) - - if 'onclick' in a.attrib: - m = re.search(r"javascript:submitForm\(([\w]+),'([^']+)'\);", a.attrib['onclick']) - if not m: - self.logger.warning('Unable to find link for %r', account.label) - account._link = None - else: - account._link = m.group(2) - else: - account._link = a.attrib['href'].strip() - - if accounts and accounts[-1].label == account.label and account.type == Account.TYPE_PEA: - self.logger.warning('%s seems to be a duplicate of %s, skipping', account, accounts[-1]) - continue - accounts.append(account) - return accounts - - -class Transaction(FrenchTransaction): - PATTERNS = [ - (re.compile(r'^Facture (?P
\d{2})/(?P\d{2})-(?P.*) carte .*'), FrenchTransaction.TYPE_CARD), - (re.compile(r'^(Prlv( de)?|Ech(éance|\.)) (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(r'^(Vir|VIR)( de)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile(r'^CHEQUE.*? (N° \w+)?$'), FrenchTransaction.TYPE_CHECK), - (re.compile(r'^Cotis(ation)? (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile(r'(?PInt .*)'), FrenchTransaction.TYPE_BANK), - (re.compile(r'^SOUSCRIPTION|REINVESTISSEMENT'), FrenchTransaction.TYPE_DEPOSIT), - ] - - -class TransactionsPage(HTMLPage): - logged = True - - @pagination - @method - class get_history(Transaction.TransactionsElement): - head_xpath = '//table[@id="releve_operation"]//tr/th' - item_xpath = '//table[@id="releve_operation"]//tr' - - col_date = ['Date opé', 'Date', 'Date d\'opé', 'Date opération'] - col_vdate = ['Date valeur'] - col_credit = ['Crédit', 'Montant', 'Valeur'] - col_debit = ['Débit'] - - def next_page(self): - url = Attr('//a[contains(text(), "Page suivante")]', 'onclick', default=None)(self) - if url: - m = re.search(r'\'([^\']+).*([\d]+)', url) - return requests.Request("POST", m.group(1), - data={ - 'numCompte': Env('accid')(self), - 'vue': "ReleveOperations", - 'tri': "DateOperation", - 'sens': "DESC", - 'page': m.group(2), - 'nb_element': "25"} - ) - - class item(Transaction.TransactionElement): - def condition(self): - return len(self.el.xpath('./td')) > 3 - - def get_coming_link(self): - try: - a = self.doc.getroot().cssselect('div#sous_nav ul li a.bt_sans_off')[0] - except IndexError: - return None - return re.sub(r'[\s]+', '', a.attrib['href']) - - def has_iban(self): - return self.doc.xpath('//a[@class="rib"]') - - def go_iban(self): - js_event = Attr("//a[@class='rib']", 'onclick')(self.doc) - m = re.search(r'envoyer(.*);', js_event) - iban_params = ast.literal_eval(m.group(1)) - self.browser.location("{}?paramNumCpt={}".format(iban_params[1], iban_params[0])) - - -class IbanPage(LoggedPage, HTMLPage): - - def get_iban(self): - return CleanText('(//b[contains(text(), "IBAN")])[1]/../text()')(self.doc) - - -class AVAccountPage(LoggedPage, HTMLPage): - """ - Get balance - - :return: decimal balance, currency - :rtype: tuple - """ - def get_av_balance(self): - balance_xpath = '//p[contains(text(), "Épargne constituée")]/span' - balance = CleanDecimal(balance_xpath)(self.doc) - currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) - return balance, currency - - @method - class get_av_investments(TableElement): - item_xpath = '//table[@id="repartition_epargne3"]/tr[position() > 1]' - head_xpath = '//table[@id="repartition_epargne3"]/tr/th[position() > 1]' - - col_quantity = 'Nombre d’unités de compte' - col_unitvalue = "Valeur de l’unité de compte" - col_valuation = 'Épargne constituée en euros' - col_portfolio_share = 'Répartition %' - - class item(ItemElement): - klass = Investment - - def condition(self): - return (CleanText('./th')(self) != 'Total épargne constituée') and ('Détail' not in Field('label')(self)) - - obj_label = CleanText('./th') - obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable) - obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable) - obj_valuation = CleanDecimal(TableCell('valuation'), default=NotAvailable) - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'))) - - def obj_code(self): - code = Regexp(Link('./th/a', default=''), r'isin=(\w+)|/(\w+)\.pdf', default=NotAvailable)(self) - return code if is_isin_valid(code) else NotAvailable - - def obj_code_type(self): - return Investment.CODE_TYPE_ISIN if is_isin_valid(Field('code')(self)) else NotAvailable - - -class AvJPage(LoggedPage, JsonPage): - def get_av_balance(self): - balance = CleanDecimal(Dict('montant'))(self.doc) - currency = "EUR" - return balance, currency - - -class AVHistoryPage(LoggedPage, HTMLPage): - @method - class get_av_history(TableElement): - item_xpath = '//table[@id="enteteTableSupports"]/tbody/tr' - head_xpath = '//table[@id="enteteTableSupports"]/thead/tr/th' - - col_date = 'Date' - col_label = 'Type de mouvement' - col_debit = 'Montant Désinvesti' - col_credit = ['Montant investi', 'Montant Net Perçu'] - # There is several types of life insurances, so multiple columns - col_credit2 = ['Montant Brut Versé'] - - class item(ItemElement): - klass = Transaction - - def condition(self): - return CleanText(TableCell('date'))(self) != 'en cours' - - obj_label = CleanText(TableCell('label')) - obj_type = Transaction.TYPE_BANK - obj_date = Date(CleanText(TableCell('date')), dayfirst=True) - obj__arbitration = False - - def obj_amount(self): - credit = CleanDecimal(TableCell('credit'), default=Decimal(0))(self) - # Different types of life insurances, use different columns. - if TableCell('debit', default=None)(self): - debit = CleanDecimal(TableCell('debit'), default=Decimal(0))(self) - # In case of financial arbitration, both columns are equal - if credit and debit: - assert credit == debit - self.obj._arbitration = True - return credit - else: - return credit - abs(debit) - else: - credit2 = CleanDecimal(TableCell('credit2'), default=Decimal(0))(self) - assert not (credit and credit2) - return credit + credit2 - - -class FormPage(LoggedPage, HTMLPage): - def get_av_balance(self): - balance_xpath = '//p[contains(text(), "montant de votre épargne")]' - balance = CleanDecimal(Regexp(CleanText(balance_xpath), r'est de ([\s\d,]+)', default=NotAvailable), - replace_dots=True, default=NotAvailable)(self.doc) - currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) - return balance, currency - - def av_account_form(self): - try: - form = self.get_form(id="formGoToRivage") - form['gfr_numeroContrat'] = JSVar(var='numContrat').filter(CleanText('//script[contains(text(), "var numContrat")]')(self.doc)) - form['gfr_data'] = JSVar(var='pCryptage').filter(CleanText('//script[contains(text(), "var pCryptage")]')(self.doc)) - form['gfr_adrSite'] = 'https://espaceclient.%s.fr' % self.browser.website - form.url = 'https://secure-rivage.%s.fr/contratVie.rivage.syntheseContratEparUc.gsi' % self.browser.website - form.submit() - return True - except FormNotFound: - return False diff --git a/modules/groupamaes/module.py b/modules/groupamaes/module.py index 260e5fa69..75954d242 100644 --- a/modules/groupamaes/module.py +++ b/modules/groupamaes/module.py @@ -17,10 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -from weboob.capabilities.bank import CapBankPockets, AccountNotFound +from __future__ import unicode_literals + +from weboob.capabilities.bank import CapBankPockets from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword -from weboob.capabilities.base import find_object from .browser import GroupamaesBrowser @@ -30,16 +31,18 @@ __all__ = ['GroupamaesModule'] class GroupamaesModule(Module, CapBankPockets): NAME = 'groupamaes' - DESCRIPTION = u"Groupama Épargne Salariale" - MAINTAINER = u'Bezleputh' + DESCRIPTION = 'Groupama Épargne Salariale' + MAINTAINER = 'Bezleputh' EMAIL = 'carton_ben@yahoo.fr' LICENSE = 'LGPLv3+' VERSION = '1.6' BROWSER = GroupamaesBrowser - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp='\d{8,}', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', regexp=r'\d{8,}', masked=False), + ValueBackendPassword('password', label='Mot de passe') + ) def create_default_browser(self): return self.create_browser( @@ -47,10 +50,8 @@ class GroupamaesModule(Module, CapBankPockets): self.config['password'].get(), 'https://www.gestion-epargne-salariale.fr', 'groupama-es/', - weboob=self.weboob) - - def get_account(self, _id): - return find_object(self.browser.iter_accounts(), id=_id, error=AccountNotFound) + weboob=self.weboob + ) def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/hsbchk/browser.py b/modules/hsbchk/browser.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/module.py b/modules/hsbchk/module.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/pages/account_pages.py b/modules/hsbchk/pages/account_pages.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/pages/login.py b/modules/hsbchk/pages/login.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/sbrowser.py b/modules/hsbchk/sbrowser.py old mode 100755 new mode 100644 diff --git a/modules/humanis/module.py b/modules/humanis/module.py index 0cf4af0c7..2954b160b 100644 --- a/modules/humanis/module.py +++ b/modules/humanis/module.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword -from weboob.capabilities.bank import CapBankPockets, AccountNotFound -from weboob.capabilities.base import find_object +from weboob.capabilities.bank import CapBankPockets from .browser import HumanisBrowser @@ -31,13 +31,15 @@ __all__ = ['HumanisModule'] class HumanisModule(Module, CapBankPockets): NAME = 'humanis' - DESCRIPTION = u'Humanis Épargne Salariale' - MAINTAINER = u'Jean Walrave' - EMAIL = 'jwalrave@budget-insight.com' + DESCRIPTION = 'Humanis Épargne Salariale' + MAINTAINER = 'Quentin Defenouillère' + EMAIL = 'quentin.defenouillere@budget-insight.com' LICENSE = 'LGPLv3+' VERSION = '1.6' - CONFIG = BackendConfig(ValueBackendPassword('login', label=u'Code d\'accès', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Code d\'accès', masked=False), + ValueBackendPassword('password', label='Mot de passe') + ) BROWSER = HumanisBrowser @@ -50,9 +52,6 @@ class HumanisModule(Module, CapBankPockets): weboob=self.weboob ) - def get_account(self, _id): - return find_object(self.browser.iter_accounts(), id=_id, error=AccountNotFound) - def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/ing/api/accounts_page.py b/modules/ing/api/accounts_page.py index adacb451b..8aa2180cd 100644 --- a/modules/ing/api/accounts_page.py +++ b/modules/ing/api/accounts_page.py @@ -27,7 +27,7 @@ from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Date, Eval, Lower, Format, Field, Map, Upper, ) -from weboob.capabilities.bank import Account +from weboob.capabilities.bank import Account, AccountOwnership from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.base import NotAvailable @@ -75,6 +75,18 @@ class AccountsPage(LoggedPage, JsonPage): return -CleanDecimal(Dict('ledgerBalance'))(self) return CleanDecimal(Dict('ledgerBalance'))(self) + def obj_ownership(self): + ownership = Dict('ownership/code', default=None)(self) + role = Dict('role/label', default=None)(self) + + if ownership == 'JOINT': + return AccountOwnership.CO_OWNER + elif ownership == 'SINGLE': + if role == 'Titulaire': + return AccountOwnership.OWNER + elif role == 'Procuration': + return AccountOwnership.ATTORNEY + class HistoryPage(LoggedPage, JsonPage): def is_empty_page(self): diff --git a/modules/ing/api/login.py b/modules/ing/api/login.py index b186e4b14..64540a868 100644 --- a/modules/ing/api/login.py +++ b/modules/ing/api/login.py @@ -43,41 +43,9 @@ class INGVirtKeyboard(SimpleVirtualKeyboard): 'limit_pixel': 200 } - # for matching_symbols_coords, indexes are cases place like this - # --- --- --- --- --- - # |0| |1| |2| |3| |4| - # --- --- --- --- --- - # --- --- --- --- --- - # |5| |6| |7| |8| |9| - # --- --- --- --- --- - matching_symbols_coords = { - '0': (3, 3, 93, 91), - '1': (99, 3, 189, 91), - '2': (196, 3, 286, 91), - '3': (293, 3, 383, 91), - '4': (390, 3, 480, 91), - '5': (3, 98, 93, 186), - '6': (99, 98, 189, 186), - '7': (196, 98, 286, 186), - '8': (293, 98, 383, 186), - '9': (390, 98, 480, 186), - } - - symbols = { - '0': ('7b4989b431e631ec79df5d71aecb1a47', 'e2522e1f7476ad6430219a73b10799b0', 'f7db285c5c742c3a348e332c0e9f7f3e',), - '1': ('9f1b03aa9a6f9789714c38eb90a43a11', '86bc0e7e1173472928e746db874b38c3',), - '2': ('3a7d1ba32f4326a02f717f71262ba02b', 'afc2a00289ba9e362c4e9333c14a574a',), - '3': ('203bfd122f474eb9c5c278eeda01bed4', 'c1daa556a1eff1fd18817dbef39792f8',), - '4': ('c09b323e5a80a195d9cb0c3000f3d7ec', 'f020eaf7cdffefec065d3b2801ed73e2', '5e194b0aae3b8f02ebbf9cdec5c37239',), - '5': ('1749dc3f2e302cd3562a0558755ab030', 'b64163e3f5f7d83ff1baad8c4d1bc37b',), - '6': ('0888a7dc9085fcf09d56363ac253a54a', 'e269686d10f95678caf995de6834f74b', '8c505dad47cf6029921fca5fb4b0bc8d',), - '7': ('75aaa903b8277b82c458c3540208a009', 'e97b0c0e01d77dd480b8a5f5c138a268',), - '8': ('f5fa36d16f55b72ba988eb87fa1ed753', '118a52a6a480b5db5eabb0ea26196db3',), - '9': ('62f91d10650583cb6146d25bb9ac161d', 'fd81675aa1c26cbf5bb6c9f1bcdbbdf9',), - } - - def __init__(self, file, cols, rows, browser): - # use matching_symbols_coords because margins between tiles are not equals + def __init__(self, file, cols, rows, browser, matching_symbols_coords=None, symbols=None): + self.symbols = symbols or {} + self.matching_symbols_coords = matching_symbols_coords or {} super(INGVirtKeyboard, self).__init__(file=file, cols=cols, rows=rows, matching_symbols_coords=self.matching_symbols_coords, browser=browser) def process_tiles(self): @@ -131,18 +99,53 @@ class INGVirtKeyboard(SimpleVirtualKeyboard): class LoginPage(JsonPage): + # for matching_symbols_coords, indexes are cases place like this + # --- --- --- --- --- + # |0| |1| |2| |3| |4| + # --- --- --- --- --- + # --- --- --- --- --- + # |5| |6| |7| |8| |9| + # --- --- --- --- --- + matching_symbols_coords = { + '0': (3, 3, 93, 91), + '1': (99, 3, 189, 91), + '2': (196, 3, 286, 91), + '3': (293, 3, 383, 91), + '4': (390, 3, 480, 91), + '5': (3, 98, 93, 186), + '6': (99, 98, 189, 186), + '7': (196, 98, 286, 186), + '8': (293, 98, 383, 186), + '9': (390, 98, 480, 186), + } + + vk_symbols = { + '0': ('7b4989b431e631ec79df5d71aecb1a47', 'e2522e1f7476ad6430219a73b10799b0', 'f7db285c5c742c3a348e332c0e9f7f3e',), + '1': ('9f1b03aa9a6f9789714c38eb90a43a11', '86bc0e7e1173472928e746db874b38c3',), + '2': ('3a7d1ba32f4326a02f717f71262ba02b', 'afc2a00289ba9e362c4e9333c14a574a',), + '3': ('203bfd122f474eb9c5c278eeda01bed4', 'c1daa556a1eff1fd18817dbef39792f8',), + '4': ('c09b323e5a80a195d9cb0c3000f3d7ec', 'f020eaf7cdffefec065d3b2801ed73e2', '5e194b0aae3b8f02ebbf9cdec5c37239',), + '5': ('1749dc3f2e302cd3562a0558755ab030', 'b64163e3f5f7d83ff1baad8c4d1bc37b',), + '6': ('0888a7dc9085fcf09d56363ac253a54a', 'e269686d10f95678caf995de6834f74b', '8c505dad47cf6029921fca5fb4b0bc8d',), + '7': ('75aaa903b8277b82c458c3540208a009', 'e97b0c0e01d77dd480b8a5f5c138a268',), + '8': ('f5fa36d16f55b72ba988eb87fa1ed753', '118a52a6a480b5db5eabb0ea26196db3',), + '9': ('62f91d10650583cb6146d25bb9ac161d', 'fd81675aa1c26cbf5bb6c9f1bcdbbdf9',), + } + @property def is_logged(self): return 'firstName' in self.doc - def get_password_coord(self, img, password): - assert 'pinPositions' in self.doc, 'Virtualkeyboard position has failed' - assert 'keyPadUrl' in self.doc, 'Virtualkeyboard image url is missing' - + def init_vk(self, img, password): pin_position = Dict('pinPositions')(self.doc) image = BytesIO(img) - vk = INGVirtKeyboard(image, cols=5, rows=2, browser=self.browser) + vk = INGVirtKeyboard(image, cols=5, rows=2, browser=self.browser, matching_symbols_coords=self.matching_symbols_coords, symbols=self.vk_symbols) password_random_coords = vk.password_tiles_coord(password) # pin positions (website side) start at 1, our positions start at 0 return [password_random_coords[index - 1] for index in pin_position] + + def get_password_coord(self, img, password): + assert 'pinPositions' in self.doc, 'Virtualkeyboard position has failed' + assert 'keyPadUrl' in self.doc, 'Virtualkeyboard image url is missing' + return self.init_vk(img, password) diff --git a/modules/ing/api_browser.py b/modules/ing/api_browser.py index 3c7a32480..4776f17f0 100644 --- a/modules/ing/api_browser.py +++ b/modules/ing/api_browser.py @@ -263,6 +263,7 @@ class IngAPIBrowser(LoginBrowser, StatesMixin): if web_acc.id[-4:] == api_acc.number[-4:]: web_acc._uid = api_acc.id web_acc.coming = api_acc.coming + web_acc.ownership = api_acc.ownership yield web_acc break else: diff --git a/modules/ing/web/titre.py b/modules/ing/web/titre.py index c0792b5d7..0db033e66 100644 --- a/modules/ing/web/titre.py +++ b/modules/ing/web/titre.py @@ -29,8 +29,7 @@ from weboob.browser.elements import ListElement, TableElement, ItemElement, meth from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Env from weboob.browser.filters.html import Link, Attr, TableCell from weboob.tools.capabilities.bank.transactions import FrenchTransaction -from weboob.tools.capabilities.bank.investments import create_french_liquidity -from weboob.tools.compat import unicode +from weboob.tools.capabilities.bank.investments import create_french_liquidity, IsinCode class NetissimaPage(HTMLPage): @@ -43,8 +42,9 @@ class Transaction(FrenchTransaction): class TitreValuePage(LoggedPage, HTMLPage): def get_isin(self): - isin = self.doc.xpath('//div[@id="headFiche"]//span[@id="test3"]/text()') - return unicode(isin[0].split(' - ')[0].strip()) if isin else NotAvailable + # redirection page with a url which contains the ISIN + # example: https://bourse.ing.fr/fr/marche/euronext-paris/ div. If we didn't find code in url, # we try to find it in the cell text - - tablecell = TableCell('label', colspan=True)(self)[0] + tablecell = TableCell('label')(self)[0] # url find try code_match = Regexp( Link(tablecell.xpath('./following-sibling::td[position()=1]/div/a')), r'sico=([A-Z0-9]*)', - default=None)(self) + default=None + )(self) if is_isin_valid(code_match): return code_match @@ -155,11 +162,7 @@ class InvestmentsPage(LoggedPage, HTMLPage): return code return NotAvailable - def obj_code_type(self): - if is_isin_valid(self.obj_code()): - return Investment.CODE_TYPE_ISIN - return NotAvailable - + obj_code_type = IsinType(Field('code')) def obj_unitvalue(self): currency, unitvalue = self.original_unitvalue() @@ -181,26 +184,26 @@ class InvestmentsPage(LoggedPage, HTMLPage): return unitvalue def obj_vdate(self): - tablecell = TableCell('vdate', colspan=True)(self)[0] - vdate_scrapped = tablecell.xpath('./preceding-sibling::td[position()=1]//span/text()')[0] + tablecell = TableCell('vdate')(self)[0] + vdate_scraped = tablecell.xpath('./preceding-sibling::td[position()=1]//span/text()')[0] # Scrapped date could be a schedule time (00:00) or a date (01/01/1970) vdate = NotAvailable - if ':' in vdate_scrapped: + if ':' in vdate_scraped: today = datetime.date.today() - h, m = [int(x) for x in vdate_scrapped.split(':')] + h, m = [int(x) for x in vdate_scraped.split(':')] hour = datetime.time(hour=h, minute=m) vdate = datetime.datetime.combine(today, hour) - elif '/' in vdate_scrapped: - vdate = datetime.datetime.strptime(vdate_scrapped, '%d/%m/%y') + elif '/' in vdate_scraped: + vdate = datetime.datetime.strptime(vdate_scraped, '%d/%m/%y') return vdate # extract unitvalue and currency def original_unitvalue(self): - tablecell = TableCell('unitvalue', colspan=True)(self)[0] + tablecell = TableCell('unitvalue')(self)[0] text = tablecell.xpath('./text()') return Currency(text, default=NotAvailable)(self), CleanDecimal.French(text, default=NotAvailable)(self) diff --git a/modules/oney/browser.py b/modules/oney/browser.py index 4003e7e7c..5c71c5e93 100644 --- a/modules/oney/browser.py +++ b/modules/oney/browser.py @@ -40,11 +40,15 @@ __all__ = ['OneyBrowser'] class OneyBrowser(LoginBrowser): BASEURL = 'https://www.oney.fr' - home_login = URL(r'/site/s/login/login.html', - LoginPage) - login = URL(r'https://login.oney.fr/login', - r'https://login.oney.fr/context', - LoginPage) + home_login = URL( + r'/site/s/login/login.html', + LoginPage + ) + login = URL( + r'https://login.oney.fr/login', + r'https://login.oney.fr/context', + LoginPage + ) send_username = URL(r'https://login.oney.fr/middle/authenticationflowinit', SendUsernamePage) send_password = URL(r'https://login.oney.fr/middle/completeauthflowstep', SendPasswordPage) @@ -83,11 +87,18 @@ class OneyBrowser(LoginBrowser): # There is a VK on the website but it does not encode the password self.login.go() + if '@' in self.username: + auth_type = 'EML' + step_type = 'EMAIL_PASSWORD' + else: + auth_type = 'IAD' + step_type = 'IAD_ACCESS_CODE' + self.send_username.go(json={ 'authentication_type': 'LIGHT', 'authentication_factor': { 'public_value': self.username, - 'type': 'IAD', + 'type': auth_type, } }) @@ -95,11 +106,14 @@ class OneyBrowser(LoginBrowser): self.send_password.go(json={ 'flow_id': flow_id, - 'step_type': 'IAD_ACCESS_CODE', + 'step_type': step_type, 'value': self.password, }) - self.page.check_error() + error = self.page.get_error() + if error: + raise BrowserIncorrectPassword(error) + token = self.page.get_token() self.check_token.go(params={'token': token}) diff --git a/modules/oney/module.py b/modules/oney/module.py index 5050d2d13..2745ac583 100644 --- a/modules/oney/module.py +++ b/modules/oney/module.py @@ -18,6 +18,8 @@ # along with this weboob module. If not, see . +from __future__ import unicode_literals + from weboob.capabilities.bank import CapBank, AccountNotFound from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig @@ -31,18 +33,22 @@ __all__ = ['OneyModule'] class OneyModule(Module, CapBank): NAME = 'oney' - MAINTAINER = u'Vincent Paredes' + MAINTAINER = 'Vincent Paredes' EMAIL = 'vparedes@budget-insight.com' VERSION = '1.6' LICENSE = 'LGPLv3+' DESCRIPTION = 'Oney' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', masked=False, regexp=r'([0-9]{9}|.+@.+\..+)'), + ValueBackendPassword('password', label='Mot de passe'), + ) BROWSER = OneyBrowser def create_default_browser(self): - return self.create_browser(self.config['login'].get(), - self.config['password'].get()) + return self.create_browser( + self.config['login'].get(), + self.config['password'].get(), + ) def iter_accounts(self): for account in self.browser.get_accounts_list(): diff --git a/modules/oney/pages.py b/modules/oney/pages.py index 86637c741..523075d85 100644 --- a/modules/oney/pages.py +++ b/modules/oney/pages.py @@ -20,8 +20,6 @@ from __future__ import unicode_literals import re -from decimal import Decimal - import requests from weboob.capabilities.bank import Account @@ -31,9 +29,9 @@ from weboob.browser.elements import ListElement, ItemElement, method, DictElemen from weboob.browser.filters.standard import Env, CleanDecimal, CleanText, Field, Format, Currency, Date from weboob.browser.filters.html import Attr from weboob.browser.filters.json import Dict -from weboob.exceptions import BrowserIncorrectPassword from weboob.tools.compat import urlparse, parse_qsl + class Transaction(FrenchTransaction): PATTERNS = [(re.compile(r'^(?PRetrait .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(r'^(?P(Prélèvement|Cotisation|C R C A M) .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_ORDER), # C R C A M is a bank it is hardcoded here because some client want it typed and it would be a mess to scrap it @@ -64,10 +62,10 @@ class SendPasswordPage(JsonPage): def get_token(self): return self.doc['completeAuthFlowStep']['token'] - def check_error(self): + def get_error(self): errors = self.doc['completeAuthFlowStep']['errors'] if errors: - raise BrowserIncorrectPassword(errors[0]['label']) + return errors[0]['label'] class CheckTokenPage(JsonPage): @@ -103,7 +101,7 @@ class ClientPage(LoggedPage, HTMLPage): class item(ItemElement): klass = Account - obj_currency = u'EUR' + obj_currency = 'EUR' obj_type = Account.TYPE_REVOLVING_CREDIT obj_label = Env('label') obj__num = Env('_num') @@ -112,8 +110,8 @@ class ClientPage(LoggedPage, HTMLPage): obj__site = 'oney' def parse(self, el): - self.env['label'] = CleanText('./h3/a')(self) or u'Carte Oney' - self.env['_num'] = Attr('%s%s%s' % ('//option[contains(text(), "', Field('label')(self).replace('Ma ', ''), '")]'), 'value', default=u'')(self) + self.env['label'] = CleanText('./h3/a')(self) or 'Carte Oney' + self.env['_num'] = Attr('%s%s%s' % ('//option[contains(text(), "', Field('label')(self).replace('Ma ', ''), '")]'), 'value', default='')(self) self.env['id'] = Format('%s%s' % (self.page.browser.username, Field('_num')(self)))(self) # On the multiple accounts page, decimals are separated with dots, and separated with commas on single account page. @@ -239,7 +237,10 @@ class CreditHistory(LoggedPage, XLSPage): def obj_amount(self): assert not (Dict('Débit')(self) and Dict('Credit')(self)), "cannot have both debit and credit" - return Decimal(Dict('Credit')(self) or 0) - abs(Decimal(Dict('Débit')(self) or 0)) + + if Dict('Credit')(self): + return CleanDecimal.US(Dict('Credit'))(self) + return -CleanDecimal.US(Dict('Débit'))(self) obj_date = Date(Dict('Date'), dayfirst=True) diff --git a/modules/opensubtitles/test.py b/modules/opensubtitles/test.py index aa5f9d5a7..e45f0b174 100644 --- a/modules/opensubtitles/test.py +++ b/modules/opensubtitles/test.py @@ -29,7 +29,7 @@ class OpensubtitlesTest(BackendTest): lsub = [] subtitles = self.backend.iter_subtitles('fr', 'spiderman') for i in range(5): - subtitle = subtitles.next() + subtitle = next(subtitles) lsub.append(subtitle) assert subtitle.url.startswith('https') assert (len(lsub) > 0) diff --git a/modules/pixabay/test.py b/modules/pixabay/test.py index 6addf29f9..b90d2beff 100644 --- a/modules/pixabay/test.py +++ b/modules/pixabay/test.py @@ -26,13 +26,13 @@ class PixabayTest(BackendTest): def test_search(self): it = self.backend.search_image('flower') - img = it.next() + img = next(it) assert img assert img.title assert self.backend.fillobj(img, ['data']) assert len(img.data) - img = it.next() + img = next(it) assert img assert img.title diff --git a/modules/pradoepargne/module.py b/modules/pradoepargne/module.py index ef3c2af1d..47473339b 100644 --- a/modules/pradoepargne/module.py +++ b/modules/pradoepargne/module.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.tools.backend import AbstractModule, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -28,13 +29,14 @@ __all__ = ['PradoepargneModule'] class PradoepargneModule(AbstractModule, CapBankPockets): NAME = 'pradoepargne' - DESCRIPTION = u'Prado Épargne Salariale' - MAINTAINER = u'Edouard Lambert' + DESCRIPTION = 'Prado Épargne Salariale' + MAINTAINER = 'Edouard Lambert' EMAIL = 'elambert@budget-insight.com' LICENSE = 'LGPLv3+' VERSION = '1.6' CONFIG = BackendConfig( - ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + ValueBackendPassword('login', label='Identifiant', masked=False), + ValueBackendPassword('password', label='Mot de passe') + ) PARENT = 'humanis' diff --git a/modules/residentadvisor/test.py b/modules/residentadvisor/test.py index abc4428e1..db922b487 100644 --- a/modules/residentadvisor/test.py +++ b/modules/residentadvisor/test.py @@ -33,7 +33,7 @@ class ResidentadvisorTest(BackendTest): self.assertTrue(len(list(self.backend.search_events(query))) > 0) - event = self.backend.search_events(query).next() + event = next(self.backend.search_events(query)) self.assertTrue(self.backend.get_event(event.id)) def test_datefrom(self): @@ -41,7 +41,7 @@ class ResidentadvisorTest(BackendTest): later = (datetime.now() + timedelta(days=31)) query.start_date = later - event = self.backend.search_events(query).next() + event = next(self.backend.search_events(query)) self.assertTrue(later.date() <= event.start_date.date()) event = self.backend.get_event(event.id) diff --git a/modules/s2e/browser.py b/modules/s2e/browser.py index c3b0c3bdd..11b08bba4 100644 --- a/modules/s2e/browser.py +++ b/modules/s2e/browser.py @@ -31,7 +31,7 @@ from .pages import ( LoginPage, AccountsPage, AMFHSBCPage, AMFAmundiPage, AMFSGPage, HistoryPage, ErrorPage, LyxorfcpePage, EcofiPage, EcofiDummyPage, LandingPage, SwissLifePage, LoginErrorPage, EtoileGestionPage, EtoileGestionCharacteristicsPage, ProfilePage, APIInvestmentDetailsPage, - LyxorFundsPage, + LyxorFundsPage, EsaliaDetailsPage, ) @@ -58,6 +58,7 @@ class S2eBrowser(LoginBrowser, StatesMixin): profile = URL(r'/portal/salarie-(?P\w+)/mesdonnees/coordperso\?scenario=ConsulterCP', ProfilePage) bnp_investments = URL(r'https://optimisermon.epargne-retraite-entreprises.bnpparibas.com') api_investment_details = URL(r'https://funds-api.bnpparibas.com/api/performances/FromIsinCode/', APIInvestmentDetailsPage) + esalia_details = URL(r'https://www.societegeneralegestion.fr/psSGGestionEntr/productsheet/view', EsaliaDetailsPage) STATE_DURATION = 10 @@ -161,13 +162,22 @@ class S2eBrowser(LoginBrowser, StatesMixin): inv.code = m.group(2) inv.code_type = Investment.CODE_TYPE_ISIN self.location('https://funds-api.bnpparibas.com/api/performances/FromIsinCode/' + inv.code) - inv.performance_history = self.page.get_investment_performances() + self.page.fill_investment(obj=inv) elif self.amfcode_sg.match(inv._link) or self.lyxorfunds.match(inv._link): - # Esalia (Société Générale Épargne Salariale) or Lyxor investments - # Not all sggestion-ede.com or lyxorfunds.com have available performances. + # SGgestion-ede or Lyxor investments: not all of them have available attributes. # For those requests to work in every case we need the headers from AccountsPage self.location(inv._link, headers={'Referer': self.accounts.build(slug=self.SLUG)}) - inv.performance_history = self.page.get_investment_performances() + self.page.fill_investment(obj=inv) + elif self.esalia_details.match(inv._link): + # Esalia (Société Générale Épargne Salariale) details page: + # Fetch code, code_type & asset_category here + m = re.search(r'idvm\/(.*)\/lg', inv._link) + if m: + if is_isin_valid(m.group(1)): + inv.code = m.group(1) + inv.code_type = Investment.CODE_TYPE_ISIN + self.location(inv._link) + inv.asset_category = self.page.get_asset_category() return investments @need_login diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index da1659d7f..dc68404d5 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -117,7 +117,7 @@ class LoginPage(HTMLPage): def get_error(self): cgu = CleanText('//h1[contains(text(), "Conditions")]', default=None)(self.doc) if cgu: - cgu = u"Veuillez accepter les conditions générales d'utilisation." if self.browser.LANG == "fr" \ + cgu = "Veuillez accepter les conditions générales d'utilisation." if self.browser.LANG == "fr" \ else "Please accept the general conditions of use." if self.browser.LANG == 'en' \ else cgu return cgu or CleanText('//div[contains(text(), "Erreur")]', default='')(self.doc) @@ -181,7 +181,9 @@ class CodePage(object): This class is used as a parent class to include all classes that contain a get_code() method. ''' - pass + def get_asset_category(self): + # Overriden for pages containing the asset category. + return NotAvailable # AMF codes @@ -204,6 +206,9 @@ class AMFHSBCPage(XMLPage, CodePage): def get_code(self): return CleanText('//AMF_Code', default=NotAvailable)(self.doc) + def get_asset_category(self): + return CleanText('//Asset_Class')(self.doc) + class AMFAmundiPage(HTMLPage, CodePage): CODE_TYPE = Investment.CODE_TYPE_AMF @@ -226,6 +231,9 @@ class AMFSGPage(LoggedPage, HTMLPage, CodePage): return Regexp(CleanText('//div[@id="header_code"]'), r'(\d+)', default=NotAvailable)(self.doc) def get_investment_performances(self): + # TODO: Handle supplementary attributes for AMFSGPage + self.logger.warning('This investment leads to AMFSGPage, please handle SRRI, asset_category and recommended_period.') + # Fetching the performance history (1 year, 3 years & 5 years) perfs = {} if not self.doc.xpath('//table[tr[th[contains(text(), "Performances glissantes")]]]'): @@ -249,20 +257,24 @@ class LyxorfcpePage(LoggedPage, HTMLPage, CodePage): class LyxorFundsPage(LoggedPage, HTMLPage): - def get_investment_performances(self): - # Fetching the performance history (1 year, 3 years & 5 years) - perfs = {} - if not self.doc.xpath('//table[tr[td[text()="Performance"]]]'): - return - # Available performance history: 1 month, 3 months, 6 months, 1 year, 2 years, 3 years, 4years & 5 years. - # We need to match the durations with their respective values. - durations = [CleanText('.')(el) for el in self.doc.xpath('//table[tr[td[text()="Performance"]]]//tr//th')] - values = [CleanText('.')(el) for el in self.doc.xpath('//table[tr[td[text()="Performance"]]]//tr//td')] - matches = dict(zip(durations, values)) - perfs[1] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['1A'])) - perfs[3] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['3A'])) - perfs[5] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['5A'])) - return perfs + @method + class fill_investment(ItemElement): + obj_asset_category = CleanText('//div[contains(@class, "asset-class-list")]//div[contains(@class, "assetClass")][2]/span') + + def obj_performance_history(self): + # Fetching the performance history (1 year, 3 years & 5 years) + perfs = {} + if not self.xpath('//table[tr[td[text()="Performance"]]]'): + return + # Available performance history: 1 month, 3 months, 6 months, 1 year, 2 years, 3 years, 4 years & 5 years. + # We need to match the durations with their respective values. + durations = [CleanText('.')(el) for el in self.xpath('//table[tr[td[text()="Performance"]]]//tr//th')] + values = [CleanText('.')(el) for el in self.xpath('//table[tr[td[text()="Performance"]]]//tr//td')] + matches = dict(zip(durations, values)) + perfs[1] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['1A'])) + perfs[3] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['3A'])) + perfs[5] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['5A'])) + return perfs class EcofiPage(LoggedPage, HTMLPage, CodePage): @@ -284,14 +296,30 @@ class ItemInvestment(ItemElement): obj_code = Env('code') obj_code_type = Env('code_type') obj__link = Env('_link') + obj_asset_category = Env('asset_category') def obj_label(self): - return CleanText(TableCell('label')(self)[0].xpath('.//div[contains(@style, \ - "text-align")][1]'))(self) + return CleanText( + TableCell('label')(self)[0].xpath('.//div[contains(@style, "text-align")][1]') + )(self) def obj_valuation(self): return MyDecimal(TableCell('valuation')(self)[0].xpath('.//div[not(.//div)]'))(self) + def obj_srri(self): + # We search "isque" because it can be "Risque" or "Echelle de risque" + srri = CleanText( + TableCell('label')(self)[0].xpath('.//div[contains(text(), "isque")]//span[1]'), + )(self) + if srri: + return int(srri) + return NotAvailable + + def obj_recommended_period(self): + return CleanText( + TableCell('label')(self)[0].xpath('.//div[contains(text(), "isque")]//span[2]'), + )(self) + def parse(self, el): # Trying to find vdate and unitvalue unitvalue, vdate = None, None @@ -299,11 +327,12 @@ class ItemInvestment(ItemElement): if unitvalue is None: unitvalue = Regexp(CleanText('.'), '^([\d,]+)$', default=None)(span) if vdate is None: - vdate = None if any(x in CleanText('./parent::div')(span) for x in [u"échéance", "Maturity"]) else \ + vdate = None if any(x in CleanText('./parent::div')(span) for x in ["échéance", "Maturity"]) else \ Regexp(CleanText('.'), '^([\d\/]+)$', default=None)(span) self.env['unitvalue'] = MyDecimal().filter(unitvalue) if unitvalue else NotAvailable self.env['vdate'] = Date(dayfirst=True).filter(vdate) if vdate else NotAvailable self.env['_link'] = None + self.env['asset_category'] = NotAvailable page = None link_id = Attr(u'.//a[contains(@title, "détail du fonds")]', 'id', default=None)(self) @@ -318,7 +347,7 @@ class ItemInvestment(ItemElement): if 'hsbc.fr' in self.page.browser.BASEURL: # Special space for HSBC, does not contain any information related to performances. m = re.search(r'fundid=(\w+).+SH=(\w+)', CleanText('//complete', default='')(page.doc)) - if m: # had to put full url to skip redirections. + if m: # had to put full url to skip redirections. page = page.browser.open('https://www.assetmanagement.hsbc.com/feedRequest?feed_data=gfcFundData&cod=FR&client=FCPE&fId=%s&SH=%s&lId=fr' % m.groups()).page elif not self.page.browser.history.is_here(): @@ -336,7 +365,9 @@ class ItemInvestment(ItemElement): self.env['code'] = NotAvailable self.env['code_type'] = NotAvailable return - elif url.startswith('http://sggestion-ede.com/product') or url.startswith('https://www.lyxorfunds.com/part'): + elif (url.startswith('http://sggestion-ede.com/product') or + url.startswith('https://www.lyxorfunds.com/part') or + url.startswith('https://www.societegeneralegestion.fr')): self.env['_link'] = url # Try to fetch ISIN code from URL with re.match @@ -392,10 +423,12 @@ class ItemInvestment(ItemElement): if isinstance(page, CodePage): self.env['code'] = page.get_code() self.env['code_type'] = page.CODE_TYPE + self.env['asset_category'] = page.get_asset_category() else: # The page is not handled and does not have a get_code method. self.env['code'] = NotAvailable self.env['code_type'] = NotAvailable + self.env['asset_category'] = NotAvailable class MultiPage(HTMLPage): @@ -494,8 +527,10 @@ class AccountsPage(LoggedPage, MultiPage): form[select_id] = Attr('//option[contains(text(), "%s")]' % accid, 'value')(self.doc) form[input_id] = "onglet4" if pocket else "onglet2" # Select display : amount or quantity - radio_txt = ("En montant" if valuation else [u"Quantité", "En parts"]) if self.browser.LANG == "fr" else \ - ("In amount" if valuation else ["Quantity", "In units"]) + if self.browser.LANG == "fr": + radio_txt = "En montant" if valuation else ["Quantité", "En parts", "Nombre de parts"] + else: + radio_txt = "In amount" if valuation else ["Quantity", "In units", "Number of units"] if isinstance(radio_txt, list): radio_txt = '" or text()="'.join(radio_txt) input_id = Regexp(Attr('%s//span[text()="%s"]/preceding-sibling::a[1]' \ @@ -701,8 +736,11 @@ class EtoileGestionPage(HTMLPage, CodePage): if characteristics_url is not None: detail_page = self.browser.open(characteristics_url).page + if not isinstance(detail_page, EtoileGestionCharacteristicsPage): + return NotAvailable + # We prefer to return an ISIN code by default - code_isin = detail_page.get_code_isin() + code_isin = detail_page.get_isin_code() if code_isin is not None: self.CODE_TYPE = Investment.CODE_TYPE_ISIN return code_isin @@ -715,9 +753,12 @@ class EtoileGestionPage(HTMLPage, CodePage): return NotAvailable + def get_asset_category(self): + return CleanText('//label[contains(text(), "Classe d\'actifs")]/following-sibling::span')(self.doc) + class EtoileGestionCharacteristicsPage(PartialHTMLPage): - def get_code_isin(self): + def get_isin_code(self): code = CleanText('//td[contains(text(), "Code Isin")]/following-sibling::td', default=None)(self.doc) return code @@ -726,18 +767,29 @@ class EtoileGestionCharacteristicsPage(PartialHTMLPage): return code +class EsaliaDetailsPage(LoggedPage, HTMLPage): + def get_asset_category(self): + return CleanText('//label[text()="Classe d\'actifs:"]/following-sibling::span')(self.doc) + + class ProfilePage(LoggedPage, MultiPage): def get_company_name(self): return CleanText('//div[contains(@class, "operation-bloc")]//span[contains(text(), "Entreprise")]/following-sibling::span[1]')(self.doc) class APIInvestmentDetailsPage(LoggedPage, JsonPage): - def get_investment_performances(self): - # Fetching the performance history (1 year, 3 years & 5 years) - perfs = {} - for item in Dict('sharePerf')(self.doc): - if item['name'] in ('1Y', '3Y', '5Y'): - duration = int(item['name'][0]) - value = item['value'] - perfs[duration] = Eval(lambda x: x / 100, CleanDecimal.US(value))(self.doc) - return perfs + @method + class fill_investment(ItemElement): + obj_srri = Eval(int, Dict('risque')) + obj_asset_category = Dict('classification') + obj_recommended_period = Dict('dureePlacement') + + def obj_performance_history(self): + # Fetching the performance history (1 year, 3 years & 5 years) + perfs = {} + for item in Dict('sharePerf')(self): + if item['name'] in ('1Y', '3Y', '5Y'): + duration = int(item['name'][0]) + value = item['value'] + perfs[duration] = Eval(lambda x: x / 100, CleanDecimal.US(value))(self) + return perfs diff --git a/modules/senscritique/browser.py b/modules/senscritique/browser.py index d3c51eda3..42e337030 100644 --- a/modules/senscritique/browser.py +++ b/modules/senscritique/browser.py @@ -47,7 +47,7 @@ class SenscritiqueBrowser(PagesBrowser): def get_event(self, _id, event=None): if not event: try: - event = self.films_page.go().iter_films(_id=_id).next() + event = next(self.films_page.go().iter_films(_id=_id)) except StopIteration: raise UserError('This event (%s) does not exists' % _id) diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 78143ea05..3fcc37407 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -31,13 +31,14 @@ from weboob.capabilities.base import find_object, NotAvailable from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError from weboob.capabilities.profile import ProfileMissing from weboob.tools.value import Value, ValueBool +from weboob.tools.decorators import retry from .pages.accounts_list import ( AccountsMainPage, AccountDetailsPage, AccountsPage, LoansPage, HistoryPage, CardHistoryPage, PeaLiquidityPage, AdvisorPage, HTMLProfilePage, CreditPage, CreditHistoryPage, OldHistoryPage, MarketPage, LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, - UnavailableServicePage, LoanDetailsPage, + UnavailableServicePage, LoanDetailsPage, TemporaryBrowserUnavailable, ) from .pages.transfer import AddRecipientPage, SignRecipientPage, TransferJson, SignTransferPage from .pages.login import MainPage, LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage @@ -114,7 +115,7 @@ class SocieteGenerale(LoginBrowser, StatesMixin): r'.*/Technical-pages/service-indisponible/service-indisponible.html', UnavailableServicePage) error = URL(r'https://static.societegenerale.fr/pri/erreur.html', ErrorPage) - login = URL(r'/sec/vk', LoginPage) + login = URL(r'https://particuliers.societegenerale.fr//sec/vk/', LoginPage) # yes, it works only with double slash main_page = URL(r'https://particuliers.societegenerale.fr', MainPage) context = None @@ -188,7 +189,8 @@ class SocieteGenerale(LoginBrowser, StatesMixin): else: account_ibans = self.page.get_account_ibans_dict() - self.accounts.go() + go = retry(TemporaryBrowserUnavailable)(self.accounts.go) + go() if not self.page.is_new_website_available(): # return in old pages to get accounts @@ -230,6 +232,14 @@ class SocieteGenerale(LoginBrowser, StatesMixin): yield account + def next_page_retry(self, condition): + next_page = self.page.hist_pagination(condition) + if next_page: + location = retry(TemporaryBrowserUnavailable)(self.location) + location(next_page) + return True + return False + @need_login def iter_history(self, account): if account.type in (account.TYPE_LOAN, account.TYPE_MARKET, account.TYPE_CONSUMER_CREDIT, ): @@ -248,7 +258,9 @@ class SocieteGenerale(LoginBrowser, StatesMixin): account.type == account.TYPE_REVOLVING_CREDIT and account._loan_type != 'PR_CONSO', account.type in (account.TYPE_REVOLVING_CREDIT, account.TYPE_SAVINGS) and not account._is_json_histo )): - self.account_details_page.go(params={'idprest': account._prestation_id}) + go = retry(TemporaryBrowserUnavailable)(self.account_details_page.go) + go(params={'idprest': account._prestation_id}) + history_url = self.page.get_history_url() # history_url return NotAvailable when history page doesn't exist @@ -265,21 +277,31 @@ class SocieteGenerale(LoginBrowser, StatesMixin): return if account.type == account.TYPE_CARD: - self.history.go(params={'b64e200_prestationIdTechnique': account.parent._internal_id}) - for summary_card_tr in self.page.iter_card_transactions(card_number=account.number): - yield summary_card_tr - - for card_tr in summary_card_tr._card_transactions: - card_tr.date = summary_card_tr.date - # We use the Raw pattern to set the rdate automatically, but that make - # the transaction type to "CARD", so we have to correct it in the browser. - card_tr.type = TransactionType.DEFERRED_CARD - yield card_tr + go = retry(TemporaryBrowserUnavailable)(self.history.go) + go(params={'b64e200_prestationIdTechnique': account.parent._internal_id}) + + next_page = True + while next_page: + for summary_card_tr in self.page.iter_card_transactions(card_number=account.number): + yield summary_card_tr + + for card_tr in summary_card_tr._card_transactions: + card_tr.date = summary_card_tr.date + # We use the Raw pattern to set the rdate automatically, but that make + # the transaction type to "CARD", so we have to correct it in the browser. + card_tr.type = TransactionType.DEFERRED_CARD + yield card_tr + next_page = self.next_page_retry('history') return - self.history.go(params={'b64e200_prestationIdTechnique': account._internal_id}) - for transaction in self.page.iter_history(): - yield transaction + go = retry(TemporaryBrowserUnavailable)(self.history.go) + go(params={'b64e200_prestationIdTechnique': account._internal_id}) + + next_page = True + while next_page: + for transaction in self.page.iter_history(): + yield transaction + next_page = self.next_page_retry('history') @need_login def iter_coming(self, account): @@ -298,23 +320,31 @@ class SocieteGenerale(LoginBrowser, StatesMixin): internal_id = account._internal_id if account.type == account.TYPE_CARD: internal_id = account.parent._internal_id - self.history.go(params={'b64e200_prestationIdTechnique': internal_id}) + + go = retry(TemporaryBrowserUnavailable)(self.history.go) + go(params={'b64e200_prestationIdTechnique': internal_id}) if account.type == account.TYPE_CARD: - for transaction in self.page.iter_future_transactions(acc_prestation_id=account._prestation_id): - # coming transactions on this page are not include in coming balance - # use it only to retrive deferred card coming transactions - if transaction._card_coming: - for card_coming in transaction._card_coming: - card_coming.date = transaction.date - # We use the Raw pattern to set the rdate automatically, but that make - # the transaction type to "CARD", so we have to correct it in the browser. - card_coming.type = TransactionType.DEFERRED_CARD - yield card_coming + next_page = True + while next_page: + for transaction in self.page.iter_future_transactions(acc_prestation_id=account._prestation_id): + # coming transactions on this page are not included in coming balance + # use it only to retrive deferred card coming transactions + if transaction._card_coming: + for card_coming in transaction._card_coming: + card_coming.date = transaction.date + # We use the Raw pattern to set the rdate automatically, but that makes + # the transaction type to "CARD", so we have to correct it in the browser. + card_coming.type = TransactionType.DEFERRED_CARD + yield card_coming + next_page = self.next_page_retry('future') return - for intraday_tr in self.page.iter_intraday_comings(): - yield intraday_tr + next_page = True + while next_page: + for intraday_tr in self.page.iter_intraday_comings(): + yield intraday_tr + next_page = self.next_page_retry('intraday') @need_login def iter_investment(self, account): diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 2c4a66c49..2824e7c06 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -43,6 +43,12 @@ from weboob.browser.pages import HTMLPage, XMLPage, JsonPage, LoggedPage, pagina from weboob.exceptions import BrowserUnavailable, ActionNeeded, NoAccountsException +class TemporaryBrowserUnavailable(BrowserUnavailable): + # To handle temporary errors (like 'err_tech') that are usually + # solved by just making a retry + pass + + def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) @@ -63,6 +69,10 @@ class JsonBasePage(LoggedPage, JsonPage): if action and 'BLOCAGE' in action: raise ActionNeeded() + if reason and 'err_tech' in reason: + # This error is temporary and usually do not happens on the next try + raise TemporaryBrowserUnavailable() + if ('le service est momentanement indisponible' in reason and Dict('commun/origine')(self.doc) != 'cbo'): raise BrowserUnavailable() @@ -475,9 +485,6 @@ class HistoryPage(JsonBasePage): # in JsonBasePage and we can't have history for now. return Dict('commun/statut')(self.el).upper() != 'NOK' - def next_page(self): - return self.page.hist_pagination('history') - item_xpath = 'donnees/listeOperations' class item(TransactionItemElement): @@ -487,9 +494,6 @@ class HistoryPage(JsonBasePage): @pagination @method class iter_card_transactions(DictElement): - def next_page(self): - return self.page.hist_pagination('history') - item_xpath = 'donnees/listeOperations' class item(TransactionItemElement): @@ -527,9 +531,6 @@ class HistoryPage(JsonBasePage): # in JsonBasePage and we can't have history for now. return Dict('commun/statut')(self.el).upper() != 'NOK' - def next_page(self): - return self.page.hist_pagination('intraday') - item_xpath = 'donnees/listeOperations' class item(TransactionItemElement): @@ -539,9 +540,6 @@ class HistoryPage(JsonBasePage): @pagination @method class iter_future_transactions(DictElement): - def next_page(self): - return self.page.hist_pagination('future') - item_xpath = 'donnees/listeOperationsFutures' class item(ItemElement): diff --git a/modules/societegenerale/pages/login.py b/modules/societegenerale/pages/login.py index 4a0d1a08b..36893300a 100644 --- a/modules/societegenerale/pages/login.py +++ b/modules/societegenerale/pages/login.py @@ -39,7 +39,10 @@ class PasswordPage(object): strange_map = None def decode_grid(self, infos): - grid = b64decode(infos['grid']).decode('ascii') + grid = infos['grid'] + if isinstance(infos['grid'], list): + grid = infos['grid'][0] + grid = b64decode(grid).decode('ascii') grid = [int(x) for x in re.findall('[0-9]{3}', grid)] n = int(infos['nbrows']) * int(infos['nbcols']) @@ -64,15 +67,28 @@ class PasswordPage(object): class MainPage(BasePage, PasswordPage): - def get_authentication_data(self): - url = self.browser.BASEURL + '//sec/vkm/gen_crypto?estSession=0' + """ + be carefull : those differents methods and PREFIX_URL are used + in another page of an another module which is an abstract of this page + """ + PREFIX_URL = '//sec' + + def get_url(self, path): + return (self.browser.BASEURL + self.PREFIX_URL + path) + + def get_authentication_infos(self): + url = self.get_url('/vkm/gen_crypto?estSession=0') infos_data = self.browser.open(url).text infos_data = re.match('^_vkCallback\((.*)\);$', infos_data).group(1) infos = json.loads(infos_data.replace("'", '"')) + return infos + + def get_authentication_data(self): + infos = self.get_authentication_infos() infos['grid'] = self.decode_grid(infos) - url = self.browser.BASEURL + '//sec/vkm/gen_ui?modeClavier=0&cryptogramme=' + infos["crypto"] + url = self.get_url('/vkm/gen_ui?modeClavier=0&cryptogramme=' + infos['crypto']) img = Captcha(BytesIO(self.browser.open(url).content), infos) try: @@ -104,7 +120,7 @@ class MainPage(BasePage, PasswordPage): 'cryptocvcs': authentication_data['infos']['crypto'].encode('iso-8859-1'), 'vkm_op': 'auth', } - self.browser.location(self.browser.absurl('/sec/vk/authent.json'), data=data) + self.browser.location(self.get_url('/vk/authent.json'), data=data) def handle_error(self): error_msg = CleanText('//span[@class="error_msg"]')(self.doc) diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index 45e272341..44cb38d4a 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -97,16 +97,30 @@ class ChangePassPage(SGPEPage): class LoginPage(SGPEPage): + """ + be carefull : those differents methods and PREFIX_URL are used + in another page of an another module which is an abstract of this page + """ + PREFIX_URL = '/sec' + @property def logged(self): return self.doc.xpath('//a[text()="Déconnexion" and @href="/logout"]') - def get_authentication_data(self): - infos_data = self.browser.open('/sec/vk/gen_crypto?estSession=0').text + def get_url(self, path): + return (self.browser.BASEURL + self.PREFIX_URL + path) + + def get_authentication_infos(self): + url = self.get_url('/vk/gen_crypto?estSession=0') + infos_data = self.browser.open(url).text infos_data = re.match('^_vkCallback\((.*)\);$', infos_data).group(1) infos = json.loads(infos_data.replace("'", '"')) + return infos - url = '/sec/vk/gen_ui?modeClavier=0&cryptogramme=' + infos["crypto"] + def get_authentication_data(self): + infos = self.get_authentication_infos() + + url = self.get_url('/vk/gen_ui?modeClavier=0&cryptogramme=' + infos['crypto']) img = Captcha(BytesIO(self.browser.open(url).content), infos) try: @@ -121,6 +135,9 @@ class LoginPage(SGPEPage): 'img': img, } + def get_authentication_url(self): + return self.browser.absurl('/authent.html') + def login(self, login, password): authentication_data = self.get_authentication_data() @@ -130,7 +147,7 @@ class LoginPage(SGPEPage): 'cryptocvcs': authentication_data['infos']['crypto'], 'vk_op': 'auth', } - self.browser.location(self.browser.absurl('/authent.html'), data=data) + self.browser.location(self.get_authentication_url(), data=data) class CardsPage(LoggedPage, SGPEPage): diff --git a/modules/trainline/pages.py b/modules/trainline/pages.py index 796463c9d..a283d43f7 100644 --- a/modules/trainline/pages.py +++ b/modules/trainline/pages.py @@ -6,8 +6,9 @@ from __future__ import unicode_literals from weboob.browser.pages import LoggedPage, JsonPage from weboob.browser.elements import DictElement, ItemElement, method -from weboob.browser.filters.standard import Date, CleanDecimal, Format, Env, Currency, Eval +from weboob.browser.filters.standard import Date, CleanDecimal, Format, Env, Currency, Field from weboob.browser.filters.json import Dict +from weboob.capabilities import NotAvailable from weboob.capabilities.bill import Subscription, Bill @@ -34,34 +35,45 @@ class UserPage(LoggedPage, JsonPage): class DocumentsPage(LoggedPage, JsonPage): def build_doc(self, text): """ - this json contains several important lists + this json contains several lists - pnrs - proofs - folders - trips + - after_sales_logs_dict + and others - each bill has data inside theses lists - this function rebuild doc to put data within same list we call 'bills' + the most important is proofs, because it contains url with a pdf + => so one proof gives one bill (for purchase only) """ doc = super(DocumentsPage, self).build_doc(text) pnrs_dict = {pnr['id']: pnr for pnr in doc['pnrs']} - proofs_dict = {proof['pnr_id']: proof for proof in doc['proofs']} - folders_dict = {folder['pnr_id']: folder for folder in doc['folders']} - trips_dict = {trip['folder_id']: trip for trip in doc['trips']} + after_sales_logs_dict = {asl['id']: asl for asl in doc['after_sales_logs']} bills = [] - for key, pnr in pnrs_dict.items(): - proof = proofs_dict[key] - folder = folders_dict[key] - trip = trips_dict[folder['id']] - - bills.append({ - 'pnr': pnr, - 'proof': proof, - 'folder': folder, - 'trip': trip, - }) + for proof in doc['proofs']: + pnr = pnrs_dict[proof['pnr_id']] + bill = { + 'id': proof['id'], # hash of 32 char length + 'url': proof['url'], + 'date': proof['created_at'], + 'type': proof['type'], # can be 'purchase' or 'refund' + 'currency': pnr['currency'] or '', # because pnr['currency'] can be None + } + + assert proof['type'] in ('purchase', 'refund'), proof['type'] + if proof['type'] == 'purchase': + # pnr['cents'] is 0 if this purchase has a refund, but there is nowhere to take it + # except make an addition, but we don't do that + bill['price'] = pnr['cents'] + bills.append(bill) + else: # proof['type'] == 'refund' + after_sales_logs = [after_sales_logs_dict[asl_id] for asl_id in pnr['after_sales_log_ids']] + for asl in after_sales_logs: + new_bill = dict(bill) + new_bill['price'] = asl['refunded_cents'] + bills.append(new_bill) return {'bills': bills} @@ -72,10 +84,26 @@ class DocumentsPage(LoggedPage, JsonPage): class item(ItemElement): klass = Bill - obj_id = Format('%s_%s', Env('subid'), Dict('pnr/id')) - obj_url = Dict('proof/url') - obj_date = Date(Dict('proof/created_at')) + obj_id = Format('%s_%s', Env('subid'), Dict('id')) + obj_url = Dict('url') + obj_date = Date(Dict('date')) obj_format = 'pdf' - obj_label = Format('Trajet du %s', Date(Dict('trip/departure_date'))) - obj_price = Eval(lambda x: x / 100, CleanDecimal(Dict('pnr/cents'))) - obj_currency = Currency(Dict('pnr/currency')) + obj_currency = Currency(Dict('currency'), default=NotAvailable) + + def obj_price(self): + price = CleanDecimal(Dict('price'), default=NotAvailable)(self) + if price: + return price / 100 + return NotAvailable + + def obj_income(self): + if Dict('type')(self) == 'purchase': + return False + else: # type is 'refund' + return True + + def obj_label(self): + if Field('income')(self): + return Format('Remboursement du %s', Field('date'))(self) + else: + return Format('Trajet du %s', Field('date'))(self) diff --git a/modules/transilien/browser.py b/modules/transilien/browser.py index 444730783..f9156c74d 100644 --- a/modules/transilien/browser.py +++ b/modules/transilien/browser.py @@ -39,8 +39,8 @@ class Transilien(PagesBrowser): roadmap_page = URL('itineraire/trajet', RoadMapPage) def get_roadmap(self, departure, arrival, filters): - dep = self.get_stations(departure, False).next() - arr = self.get_stations(arrival, False).next() + dep = next(self.get_stations(departure, False)) + arr = next(self.get_stations(arrival, False)) self.roadmap_page.go().request_roadmap(dep, arr, filters.departure_time, filters.arrival_time) if self.page.is_ambiguous(): self.page.fix_ambiguity() diff --git a/modules/vimeo/browser.py b/modules/vimeo/browser.py index 223ad5711..62d6f1217 100644 --- a/modules/vimeo/browser.py +++ b/modules/vimeo/browser.py @@ -18,192 +18,51 @@ # You should have received a copy of the GNU Affero General Public License # along with this weboob module. If not, see . -from base64 import b64encode - from weboob.browser import PagesBrowser, URL -from weboob.browser.exceptions import HTTPNotFound -from weboob.capabilities.base import NotAvailable -from weboob.tools.compat import urljoin, quote_plus - -from .pages import VideoJsonPage, CategoriesPage, ListPage, APIPage, XMLAPIPage +from weboob.capabilities.file import SearchSort -import time -import hmac -from hashlib import sha1 +from .pages import ListPage, APIPage __all__ = ['VimeoBrowser'] -class VimeoBrowser(PagesBrowser): - - BASEURL = 'https://vimeo.com' - APIURL = 'http://vimeo.com/api/rest/v2' - CONSUMER_KEY = 'ae4ac83f9facda375a72fed704a3643a' - CONSUMER_SECRET = 'b6072a4aba1eaaed' - - video_url = URL(r'https://player.vimeo.com/video/(?P<_id>.*)/config', VideoJsonPage) - - list_page = URL(r'categories/(?P.*)/videos/.*?', - ListPage) - categories_page = URL('categories', CategoriesPage) - - api_page = URL('https://api.vimeo.com/search\?filter_mature=191&filter_type=clip&sort=featured&direction=desc&page=(?P\d*)&per_page=20&sizes=590x332&_video_override=true&c=b&query=&filter_category=(?P\w*)&fields=search_web%2Cmature_hidden_count&container_fields=parameters%2Ceffects%2Csearch_id%2Cstream_id%2Cmature_hidden_count', APIPage) - - _api = URL(APIURL, XMLAPIPage) - - def __init__(self, method, quality, *args, **kwargs): - self.method = method - self.quality = quality - PagesBrowser.__init__(self, *args, **kwargs) - - def fill_video_infos(self, _id, video=None): - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - data = {'method': 'vimeo.videos.getInfo', - 'video_id': _id} - self._prepare_request(self.APIURL, method='POST', headers=headers, data=data) - return self._api.go(data=data).fill_video_infos(obj=video) - - def get_video(self, _id, video=None): - video = self.fill_video_infos(_id, video) - if video._is_hd == "0": - video._quality = 2 - else: - video._quality = self.quality - video._method = self.method - return self.fill_video_url(video) - - def fill_video_url(self, video): - self._setup_session(self.PROFILE) - try: - video = self.video_url.open(_id=video.id).fill_url(obj=video) - if self.method == u'hls': - streams = [] - for item in self.read_url(video.url): - item = item.decode('ascii') - if not item.startswith('#') and item.strip(): - streams.append(item) - - if streams: - streams.reverse() - url = streams[self.quality] if self.quality < len(streams) else streams[0] - video.url = urljoin(video.url, url) - else: - video.url = NotAvailable - return video - except HTTPNotFound: - return video +SORT_NAME = { + SearchSort.RELEVANCE: 'relevance', + SearchSort.RATING: 'popularity', + SearchSort.VIEWS: 'popularity', + SearchSort.DATE: 'latest', +} - def read_url(self, url): - r = self.open(url, stream=True) - buf = r.iter_lines() - return buf +NSFW_FLAGS = { + True: 255, + False: 191, +} - def search_videos(self, pattern, sortby): - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - data = {'method': 'vimeo.videos.search', - 'sort': 'relevant', - 'page': '1', - 'full_response': '1', - 'query': quote_plus(pattern.encode('utf-8'))} - self._prepare_request(self.APIURL, method='POST', headers=headers, data=data) - return self._api.go(data=data).iter_videos() - - def get_channels(self): - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - data = {'method': 'vimeo.channels.getAll', - 'page': '1', - 'sort': 'most_subscribed'} - # 'newest', 'oldest', 'alphabetical', 'most_videos', 'most_subscribed', 'most_recently_updated' - self._prepare_request(self.APIURL, method='POST', headers=headers, data=data) - return self._api.go(data=data).iter_channels() - - def get_channel_videos(self, channel): - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - data = {'method': 'vimeo.channels.getVideos', - 'sort': 'newest', # 'oldest', 'most_played', 'most_commented', 'most_liked' - 'page': '1', - 'channel_id': channel, - 'full_response': '1'} - self._prepare_request(self.APIURL, method='POST', headers=headers, data=data) - return self._api.go(data=data).iter_videos() - - def get_categories(self): - self._setup_session(self.PROFILE) - return self.categories_page.go().iter_categories() - - def get_category_videos(self, category): - token = self.list_page.go(category=category).get_token() - self.session.headers.update({"Authorization": "jwt %s" % token, - "Accept": "application/vnd.vimeo.*+json;version=3.3"}) - return self.api_page.go(page=1, category=category).iter_videos() - - def _create_authorization(self, url, method, params=None): - def _percent_encode(s): - result = quote_plus(s).replace('+', '%20').replace('*', '%2A').replace('%7E', '~') - # the implementation of the app has a bug. someone double escaped the '@' so we have to correct this - # on our end. - result = result.replace('%40', '%2540') - return result - - def _compute_signature(s): - key = _percent_encode(self.CONSUMER_SECRET) + '&' + _percent_encode('') - key = key.encode('ascii') - s = s.encode('ascii') - a = hmac.new(key, s, sha1) - sig = b64encode(a.digest()).decode('ascii') - sig = sig.rstrip('\n') - return sig - - def _normalize_parameters(_params): - sorted_keys = sorted(_params.keys()) - list_of_params = [] - for key in sorted_keys: - value = _params[key] - # who wrote the android app should burn in hell! No clue of correct encoding - make up your mind - if url == 'https://secure.vimeo.com/oauth/access_token' and key != 'x_auth_password': - list_of_params.append('%s=%s' % (key, value)) - pass - else: - list_of_params.append('%s=%s' % (key, _percent_encode(value))) - pass - pass - return '&'.join(list_of_params) - - if not params: - params = {} - pass - - all_params = {'oauth_consumer_key': self.CONSUMER_KEY, - 'oauth_signature_method': 'HMAC-SHA1', - 'oauth_timestamp': str(time.time()), - 'oauth_nonce': str(time.time()), - 'oauth_version': '1.0'} - all_params.update(params) - - base_string = _percent_encode(method.upper()) - base_string += '&' - base_string += _percent_encode(url) - base_string += '&' - base_string += _percent_encode(_normalize_parameters(all_params)) - - all_params['oauth_signature'] = _compute_signature(base_string) - - authorization = [] - for key in all_params: - if key.startswith('oauth_'): - authorization.append('%s="%s"' % (key, _percent_encode(all_params[key]))) - pass - pass - return {'Authorization': 'OAuth %s' % (', '.join(authorization))} +class VimeoBrowser(PagesBrowser): + BASEURL = 'https://vimeo.com' - def _prepare_request(self, url, method='GET', headers={}, data={}): - _headers = { - 'User-Agent': 'VimeoAndroid/1.1.42 (Android ver=4.4.2 sdk=19; Model\ - samsung GT-I9505; Linux 3.4.0-3423977 armv7l)', - 'Host': 'vimeo.com', - 'Accept-Encoding': 'gzip, deflate'} - self.session.headers.update(_headers) - self.session.headers.update(headers) - self.session.headers.update(self._create_authorization(url, method, data)) + api_page = URL(r'https://api.vimeo.com/search', APIPage) + html_search = URL(r'https://vimeo.com/search/page:(?P\d+)/sort:(?P\w+)', ListPage) + + def search_videos(self, pattern, sortby, nsfw): + sortby = SORT_NAME[sortby] + nsfw = NSFW_FLAGS[nsfw] + + self.html_search.go(page=1, sort=sortby, params={'q': pattern}) + jwt = self.page.get_token() + + params = { + 'query': pattern, + 'filter_type': 'clip', + 'per_page': 18, + 'page': 1, + 'sort': sortby, + 'fields': 'search_web,mature_hidden_count', + 'container_fields': 'parameters,effects,search_id,stream_id,mature_hidden_count', + 'direction': 'desc', + 'filter_mature': nsfw, + } + self.api_page.go(params=params, headers={'Authorization': 'jwt %s' % jwt}) + return self.page.iter_videos() diff --git a/modules/vimeo/module.py b/modules/vimeo/module.py index 106470a57..ce2428b30 100644 --- a/modules/vimeo/module.py +++ b/modules/vimeo/module.py @@ -18,22 +18,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this weboob module. If not, see . -from collections import OrderedDict - from weboob.capabilities.video import CapVideo, BaseVideo -from weboob.capabilities.collection import CapCollection, CollectionNotFound, Collection -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import Value +from weboob.tools.backend import Module +from weboob.tools.capabilities.video.ytdl import video_info from .browser import VimeoBrowser -import re __all__ = ['VimeoModule'] -class VimeoModule(Module, CapVideo, CapCollection): +class VimeoModule(Module, CapVideo): NAME = 'vimeo' MAINTAINER = u'François Revol' EMAIL = 'revol@free.fr' @@ -42,76 +38,17 @@ class VimeoModule(Module, CapVideo, CapCollection): LICENSE = 'AGPLv3+' BROWSER = VimeoBrowser - SORTBY = ['relevance', 'rating', 'views', 'time'] - - quality_choice = OrderedDict([(k, v) for k, v in sorted( - {u'0': u'high', u'1': u'medium', u'2': u'low'}.items())]) - - method_choice = [u'hls', u'progressive'] - - CONFIG = BackendConfig(Value('method', label='Choose a stream method', choices=method_choice), - Value('quality', label='Choosen a quality', - choices=quality_choice)) - - def create_default_browser(self): - return self.create_browser(method=self.config['method'].get(), - quality=int(self.config['quality'].get())) - def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): - return self.browser.search_videos(pattern, self.SORTBY[sortby]) - - def get_video(self, _id): - _id = self.parse_id(_id) - if _id: - return self.browser.get_video(self.parse_id(_id)) + return self.browser.search_videos(pattern, sortby, nsfw) def fill_video(self, video, fields): if fields != ['thumbnail']: # if we don't want only the thumbnail, we probably want also every fields - video = self.browser.get_video(video.id, video) + video = video_info(self.browser.absurl('/%s' % video.id, base=True)) + if 'thumbnail' in fields and video and video.thumbnail: video.thumbnail.data = self.browser.open(video.thumbnail.url).content return video - def parse_id(self, _id): - m = re.match('https?://vimeo.com/(.*)', _id) - if m: - return m.group(1) - elif not _id.startswith('http'): - return _id - - return None - - def iter_resources(self, objs, split_path): - if BaseVideo in objs: - collection = self.get_collection(objs, split_path) - if collection.path_level == 0: - yield Collection([u'vimeo-categories'], u'Vimeo categories') - yield Collection([u'vimeo-channels'], u'Vimeo channels') - - if collection.path_level == 1: - if collection.split_path == [u'vimeo-categories']: - for category in self.browser.get_categories(): - yield category - if collection.split_path == [u'vimeo-channels']: - for channel in self.browser.get_channels(): - yield channel - - if collection.path_level == 2: - if collection.split_path[0] == u'vimeo-channels': - for video in self.browser.get_channel_videos(collection.split_path[1]): - yield video - if collection.split_path[0] == u'vimeo-categories': - for video in self.browser.get_category_videos(collection.split_path[1]): - yield video - - def validate_collection(self, objs, collection): - if collection.path_level == 0: - return - if BaseVideo in objs and (collection.split_path[0] == u'vimeo-categories' or - collection.split_path[0] == u'vimeo-channels'): - return - raise CollectionNotFound(collection.split_path) - OBJECTS = {BaseVideo: fill_video} diff --git a/modules/vimeo/pages.py b/modules/vimeo/pages.py index e1e395ac2..cd520bdc7 100644 --- a/modules/vimeo/pages.py +++ b/modules/vimeo/pages.py @@ -20,43 +20,16 @@ from weboob.capabilities.video import BaseVideo from weboob.capabilities.image import Thumbnail -from weboob.capabilities.collection import Collection - -from weboob.exceptions import ParseError -from weboob.browser.elements import ItemElement, ListElement, method, DictElement -from weboob.browser.pages import HTMLPage, pagination, JsonPage, XMLPage -from weboob.browser.filters.standard import Regexp, Env, CleanText, DateTime, Duration, Field, BrowserURL -from weboob.browser.filters.html import Attr, Link +from weboob.browser.elements import ItemElement, method, DictElement +from weboob.browser.pages import HTMLPage, pagination, JsonPage +from weboob.browser.filters.standard import Regexp, CleanText from weboob.browser.filters.json import Dict -import re - - -class VimeoDuration(Duration): - _regexp = re.compile(r'(?P\d+)') - kwargs = {'seconds': 'ss'} - class ListPage(HTMLPage): def get_token(self): return Regexp(CleanText('//script'), '"jwt":"(.*)","url"', default=None)(self.doc) - @pagination - @method - class iter_videos(ListElement): - item_xpath = '//div[@id="browse_content"]/ol/li' - next_page = Link(u'//a[text()="Next"]') - - class item(ItemElement): - klass = BaseVideo - obj_id = Regexp(Attr('.', 'id'), 'clip_(.*)') - obj_title = Attr('./a', 'title') - - def obj_thumbnail(self): - thumbnail = Thumbnail(self.xpath('./a/img')[0].attrib['src']) - thumbnail.url = thumbnail.id - return thumbnail - class APIPage(JsonPage): @pagination @@ -64,12 +37,7 @@ class APIPage(JsonPage): class iter_videos(DictElement): item_xpath = 'data' - def parse(self, el): - self.env['next_page'] = Regexp(Dict('paging/next'), 'page=(\d*)', default=None)(el) - - def next_page(self): - if Env('next_page')(self) is not None: - return BrowserURL('api_page', page=int(Env('next_page')(self)), category=Env('category'))(self) + next_page = Dict('paging/next') class item(ItemElement): klass = BaseVideo @@ -81,111 +49,3 @@ class APIPage(JsonPage): thumbnail = Thumbnail(Dict('clip/pictures/sizes/0/link')(self)) thumbnail.url = thumbnail.id return thumbnail - - -class VideoJsonPage(JsonPage): - @method - class fill_url(ItemElement): - klass = BaseVideo - - def obj_url(self): - data = self.el - - if not data['request']['files']: - raise ParseError('Unable to detect any stream method for id: %r (available: %s)' - % (int(Field('id')(self)), - data['request']['files'].keys())) - - # Choosen method is not available, we choose an other one - method = self.obj._method - if method not in data['request']['files']: - method = list(data['request']['files'].keys())[0] - - streams = data['request']['files'][method] - if not streams: - raise ValueError('There is no url available for id: %r' % (int(Field('id')(self)))) - - stream = None - if method == 'hls': - if 'url' in streams: - stream = streams['url'] - else: - stream = streams['cdns'][streams['default_cdn']]['url'] - - # ...but a list for progressive - # we assume the list is sorted by quality with best first - if not stream: - quality = self.obj._quality - stream = streams[quality]['url'] if quality < len(streams) else streams[0]['url'] - - return stream - - -class CategoriesPage(HTMLPage): - @method - class iter_categories(ListElement): - item_xpath = '//div[@class="category_grid"]/div/a' - - class item(ItemElement): - klass = Collection - - obj_id = CleanText('./@href') - obj_title = CleanText('./div/div/p') - - def obj_split_path(self): - split_path = ['vimeo-categories'] - category = CleanText('./@href', replace=[('/categories/', '')])(self) - split_path.append(category) - return split_path - - -class VimeoItem(ItemElement): - klass = BaseVideo - - obj_id = CleanText('./@id') - obj_title = CleanText('./title') - obj_description = CleanText('./description') - obj_author = CleanText('./owner/@display_name') - obj_date = DateTime(CleanText('./upload_date')) - obj__is_hd = CleanText('./@is_hd') - obj_duration = VimeoDuration(CleanText('./duration')) - obj_ext = u'mp4' - - def obj_thumbnail(self): - t = CleanText('./thumbnails/thumbnail[1]', default='')(self) - if t: - thumbnail = Thumbnail(t) - thumbnail.url = thumbnail.id - return thumbnail - - -class XMLAPIPage(XMLPage): - @method - class iter_videos(ListElement): - item_xpath = '//video' - - class item(VimeoItem): - pass - - @method - class fill_video_infos(VimeoItem): - def __init__(self, *args, **kwargs): - super(VimeoItem, self).__init__(*args, **kwargs) - self.el = self.el.xpath('/rsp/video')[0] - - @pagination - @method - class iter_channels(ListElement): - item_xpath = '//channel' - - class item(ItemElement): - klass = Collection - - obj_title = CleanText('./name') - obj_id = CleanText('./@id') - - def obj_split_path(self): - split_path = ['vimeo-channels'] - split_path.append(Regexp(CleanText('./url'), - 'http://vimeo.com/channels/(.*)/?')(self)) - return split_path diff --git a/modules/wellsfargo/browser.py b/modules/wellsfargo/browser.py deleted file mode 100644 index e3286ee4c..000000000 --- a/modules/wellsfargo/browser.py +++ /dev/null @@ -1,300 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Oleg Plakhotniuk -# -# This file is part of a weboob module. -# -# This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This weboob module is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this weboob module. If not, see . - - -import json -import os -import ssl -from subprocess import STDOUT, CalledProcessError, check_output -from tempfile import mkstemp - -from weboob.browser import URL, LoginBrowser, need_login -from weboob.capabilities.bank import AccountNotFound -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable -from weboob.tools.compat import unquote - -from .pages import (ActivityCardPage, ActivityCashPage, CodeRequestPage, CodeSubmitPage, DocumentsPage, LoggedInPage, - LoginProceedPage, LoginRedirectPage, StatementPage, StatementsEmbeddedPage, StatementsPage, - SummaryPage) - -__all__ = ['WellsFargo'] - - -class WellsFargo(LoginBrowser): - BASEURL = 'https://online.wellsfargo.com' - TIMEOUT = 30 - MAX_RETRIES = 10 - login_proceed = URL('/das/cgi-bin/session.cgi\?screenid=SIGNON.*$', - '/login\?ERROR_CODE=.*LOB=CONS&$', - LoginProceedPage) - login_redirect = URL('/das/cgi-bin/session.cgi\?screenid=SIGNON.*$', - '/login\?ERROR_CODE=.*LOB=CONS&$', - LoginRedirectPage) - code_request = URL('https://oam.wellsfargo.com/oam/access' - '/twoFAAARDisplay\?OAM_TKN=.+$', CodeRequestPage) - code_submit = URL('https://oam.wellsfargo.com/oam/access' - '/twoFAAARDisplay\?OAM_TKN=.+$', - 'https://oam.wellsfargo.com/oam/access' - '/twoFAAARSubmitCode\?OAM_TKN=.+$', CodeSubmitPage) - summary = URL('/das/channel/accountSummary$', SummaryPage) - activity_cash = URL('/das/cgi-bin/session.cgi\?sessargs=.+$', - ActivityCashPage) - activity_card = URL('/das/cgi-bin/session.cgi\?sessargs=.+$', - ActivityCardPage) - documents = URL('https://connect.secure.wellsfargo.com' - '/accounts/start\?.+$', DocumentsPage) - statements_embedded = URL('https://connect.secure.wellsfargo.com' - '/accounts/start\?.+$', StatementsEmbeddedPage) - statements = URL('https://connect.secure.wellsfargo.com' - '/accounts/documents/statement/list.+$', - StatementsPage) - statement = URL('https://connect.secure.wellsfargo.com' - '/accounts/documents/retrieve/.+$', - StatementPage) - unknown = URL('/.*$', LoggedInPage) # e.g. random advertisement pages. - - def __init__(self, question1, answer1, question2, answer2, - question3, answer3, phone_last4, code_file, *args, **kwargs): - super(WellsFargo, self).__init__(*args, **kwargs) - self.question1 = question1 - self.answer1 = answer1 - self.question2 = question2 - self.answer2 = answer2 - self.question3 = question3 - self.answer3 = answer3 - self.phone_last4 = phone_last4 - self.code_file = code_file - - def do_login(self): - ''' - There's a bunch of dynamically generated obfuscated JavaScript, - which uses DOM. For now the easiest option seems to be to run it in - PhantomJs. - ''' - for i in range(self.MAX_RETRIES): - scrf, scrn = mkstemp('.js') - cookf, cookn = mkstemp('.json') - os.write(scrf, LOGIN_JS % { - 'scriptTimeout': self.TIMEOUT*2, - 'resourceTimeout': self.TIMEOUT, - 'username': self.username, - 'password': self.password, - 'output': cookn, - 'question1': self.question1, - 'answer1': self.answer1, - 'question2': self.question2, - 'answer2': self.answer2, - 'question3': self.question3, - 'answer3': self.answer3}) - os.close(scrf) - os.close(cookf) - try: - check_output(["phantomjs", scrn], stderr=STDOUT) - with open(cookn) as cookf: - cookies = json.loads(cookf.read()) - except CalledProcessError: - continue - finally: - os.remove(scrn) - os.remove(cookn) - self.session.cookies.clear() - for c in cookies: - for k in ['expiry', 'expires', 'httponly']: - c.pop(k, None) - c['value'] = unquote(c['value']) - self.session.cookies.set(**c) - self.summary.go() - if self.page.logged: - break - else: - raise BrowserIncorrectPassword - - def location(self, *args, **kwargs): - """ - Wells Fargo inserts redirecting pages from time to time, - so we should follow them whenever we see them. - """ - r = super(WellsFargo, self).location(*args, **kwargs) - if self.login_proceed.is_here(): - return self.page.proceed() - elif self.login_redirect.is_here(): - return self.page.redirect() - elif self.code_request.is_here(): - return self.page.request_code() - elif self.code_submit.is_here(): - return self.page.submit_code() - else: - return r - - def prepare_request(self, req): - """ - Wells Fargo uses TLS v1.0. See issue #1647 for details. - """ - preq = super(WellsFargo, self).prepare_request(req) - conn = self.session.adapters['https://'].get_connection(preq.url) - conn.ssl_version = ssl.PROTOCOL_TLSv1 - return preq - - def get_account(self, id_): - self.to_activity() - if id_ not in self.page.accounts_ids(): - raise AccountNotFound() - else: - self.to_activity(id_) - return self.page.get_account() - - def iter_accounts(self): - self.to_activity() - for id_ in self.page.accounts_ids(): - self.to_activity(id_) - yield self.page.get_account() - - @need_login - def to_summary(self): - self.summary.stay_or_go() - assert self.summary.is_here() - - def is_activity(self): - return self.activity_cash.is_here() or self.activity_card.is_here() - - @need_login - def to_activity(self, id_=None): - if not self.is_activity(): - self.to_summary() - self.page.to_activity() - assert self.is_activity() - if id_ and self.page.account_id() != id_: - self.page.to_account(id_) - assert self.is_activity() - assert self.page.account_id() == id_ - - @need_login - def to_statements(self, id_=None, year=None): - if not self.statements.is_here() \ - and not self.statements_embedded.is_here(): - self.to_summary() - self.page.to_documents() - if self.documents.is_here(): - self.page.to_statements() - assert self.statements.is_here() - else: - assert self.statements_embedded.is_here() - if id_ and self.page.parser().account_id() != id_: - self.page.parser().to_account(id_) - assert self.statements.is_here() - assert self.page.parser().account_id() == id_ - if year and self.page.parser().year() != year: - self.page.parser().to_year(year) - assert self.statements.is_here() - assert self.page.parser().year() == year - - @need_login - def to_statement(self, uri): - for i in range(self.MAX_RETRIES): - self.location(uri) - if self.statement.is_here(): - break - else: - raise BrowserUnavailable() - - def iter_history(self, account): - self.to_activity(account.id) - # Skip transactions on web page if we cannot apply - # "since last statement" filter. - # This might be the case, for example, if Wells Fargo - # is processing the current statement: - # "Since your credit card account statement is being processed, - # transactions grouped by statement period will not be available - # for up to seven days." - # (www.wellsfargo.com, 2014-07-20) - if self.page.since_last_statement(): - assert self.page.account_id() == account.id - while True: - for trans in self.page.iter_transactions(): - yield trans - if not self.page.next_(): - break - - self.to_statements(account.id) - for year in self.page.parser().years(): - self.to_statements(account.id, year) - for stmt in self.page.parser().statements(): - self.to_statement(stmt) - for trans in self.page.iter_transactions(): - yield trans - - -LOGIN_JS = u'''\ -var page = require('webpage').create(); - -page.settings.resourceTimeout = %(resourceTimeout)s*1000; -page.open('https://www.wellsfargo.com/'); - -var waitForForm = function() { - var hasForm = page.evaluate(function(){ - return !!document.getElementById('frmSignon') - }); - if (hasForm) { - page.evaluate(function(){ - document.getElementById('userid').value = '%(username)s'; - document.getElementById('password').value = '%(password)s'; - document.getElementById('frmSignon').submit(); - }); - } else { - setTimeout(waitForForm, 1000); - } -} - -var waitForQuestions = function() { - var isQuestion = page.content.indexOf('Confirm Your Identity') != -1; - if (isQuestion) { - var questions = { - "%(question1)s": "%(answer1)s", - "%(question2)s": "%(answer2)s", - "%(question3)s": "%(answer3)s" - }; - for (var question in questions) { - if (page.content.indexOf(question)) { - page.evaluate(function(answer){ - document.getElementById('answer').value = answer; - document.getElementById('command').submit.click(); - }, questions[question]); - } - } - } - setTimeout(waitForQuestions, 2000); -} - -var waitForLogin = function() { - var isSplash = page.content.indexOf('Splash Page') != -1; - var hasSignOff = page.content.indexOf('Sign Off') != -1; - if (isSplash || hasSignOff) { - var cookies = JSON.stringify(phantom.cookies); - require('fs').write('%(output)s', cookies, 'w'); - phantom.exit(); - } else { - setTimeout(waitForLogin, 2000); - } -} - -waitForForm(); -waitForQuestions(); -waitForLogin(); -setTimeout(function(){phantom.exit(-1);}, %(scriptTimeout)s*1000); -''' diff --git a/modules/wellsfargo/favicon.png b/modules/wellsfargo/favicon.png deleted file mode 100644 index cf4fd68778441915b597f5d34db57408ce0eaaf7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 285 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0P3?wHke>@jRu?6^qxSnBP{QH5~J0jpAP~?)Q zi(`n!`Q(HJ$_e?K|Nmal^p}Czc)r1DM&?cywzRrbHU;O##*NXD4;mVIn3*5vJmqKS zc_J+#p>ECWdPCm8AjXa-Y3knL2jSn9tPHScpbZ|@hegFU0=lm1iF8cpH z=YO-Lt(_2iB2P!5jg64I0?$AB`U@@<0$C4uI?609S2U)yo}b_HtIhrs56>AL*0%PW zLJf>ZKOB4@FhkMc@B@j@`{*_i&A*-+xY#I}u%5. - - -from weboob.capabilities.bank import CapBank -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword - -from .browser import WellsFargo - - -__all__ = ['WellsFargoModule'] - - -class WellsFargoModule(Module, CapBank): - NAME = 'wellsfargo' - MAINTAINER = u'Oleg Plakhotniuk' - EMAIL = 'olegus8@gmail.com' - VERSION = '1.6' - LICENSE = 'LGPLv3+' - DESCRIPTION = u'Wells Fargo' - CONFIG = BackendConfig( - ValueBackendPassword('login', label='Username', masked=False), - ValueBackendPassword('password', label='Password'), - ValueBackendPassword('question1', label='Question 1', masked=False), - ValueBackendPassword('answer1', label='Answer 1', masked=False), - ValueBackendPassword('question2', label='Question 2', masked=False), - ValueBackendPassword('answer2', label='Answer 2', masked=False), - ValueBackendPassword('question3', label='Question 3', masked=False), - ValueBackendPassword('answer3', label='Answer 3', masked=False), - ValueBackendPassword('phone_last4', label='Last 4 digits of phone number to request access code to', masked=False), - ValueBackendPassword('code_file', label='File to read access code from', masked=False)) - BROWSER = WellsFargo - - def create_default_browser(self): - return self.create_browser( - username = self.config['login'].get(), - password = self.config['password'].get(), - question1 = self.config['question1'].get(), - answer1 = self.config['answer1'].get(), - question2 = self.config['question2'].get(), - answer2 = self.config['answer2'].get(), - question3 = self.config['question3'].get(), - answer3 = self.config['answer3'].get(), - phone_last4 = self.config['phone_last4'].get(), - code_file = self.config['code_file'].get()) - - def iter_accounts(self): - return self.browser.iter_accounts() - - def get_account(self, id_): - return self.browser.get_account(id_) - - def iter_history(self, account): - return self.browser.iter_history(account) diff --git a/modules/wellsfargo/pages.py b/modules/wellsfargo/pages.py deleted file mode 100644 index f76dd3220..000000000 --- a/modules/wellsfargo/pages.py +++ /dev/null @@ -1,486 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Oleg Plakhotniuk -# -# This file is part of a weboob module. -# -# This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This weboob module is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this weboob module. If not, see . - -import Cookie -import datetime -import itertools -import json -import os -import re -from decimal import Decimal -from time import sleep - -from requests.cookies import morsel_to_cookie - -from weboob.browser.pages import HTMLPage, LoggedPage, RawPage -from weboob.capabilities.bank import Account, Transaction -from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr - -from .parsers import StatementParser, clean_label - - -try: - cmp = cmp -except NameError: - def cmp(x, y): - return (x > y) - (x < y) - - -class LoginProceedPage(LoggedPage, HTMLPage): - is_here = '//script[contains(text(),"setAndCheckCookie")]' - - def proceed(self): - script = self.doc.xpath('//script/text()')[0] - cookieStr = re.match('.*document\.cookie = "([^"]+)".*', - script, re.DOTALL).group(1) - morsel = Cookie.Cookie(cookieStr).values()[0] - self.browser.session.cookies.set_cookie(morsel_to_cookie(morsel)) - form = self.get_form() - return form.submit() - - -class LoginRedirectPage(LoggedPage, HTMLPage): - is_here = 'contains(//meta[@http-equiv="Refresh"]/@content,' \ - '"SIGNON_PORTAL_PAUSE")' - - def redirect(self): - refresh = self.doc.xpath( - '//meta[@http-equiv="Refresh"]/@content')[0] - url = re.match(r'^.*URL=(.*)$', refresh).group(1) - return self.browser.location(url) - - -class LoggedInPage(HTMLPage): - @property - def logged(self): - return bool(self.doc.xpath(u'//a[text()="Sign Off"]')) \ - or bool(self.doc.xpath(u'//title[text()="Splash Page"]')) \ - or bool(self.doc.xpath(u'//title[contains(text(),' - u'"Advanced Access Required")]')) - - -class CodeRequestPage(LoggedInPage): - is_here = u'contains(//label/text(),"Send Code to")' - - def request_code(self): - phone = self.browser.page.doc.xpath( - '//select[@name="telephone"]/option[contains(text(),"%s")]/@value' - % self.browser.phone_last4)[0] - form = self.get_form(name='otp') - form['telephone'] = [phone] - form['deliveryMode'] = ['sms'] - form['sendcode'] = ['Request Code'] - del form['cancelBtn'] - del form['sendcode2'] - return form.submit() - - -class CodeSubmitPage(LoggedInPage): - is_here = u'contains(//title/text(),"Enter and Submit Advanced Access Code")' - - def submit_code(self): - self.browser.logger.warning( - 'The code has been sent to the phone number ending with %s. ' - 'Please write this code into the file %s' - % (self.browser.phone_last4, self.browser.code_file)) - while True: - try: - with open(self.browser.code_file) as f: - code = f.read().strip() - break - except IOError: - sleep(1) - os.remove(self.browser.code_file) - self.browser.logger.info('The code %s has been successfully read' % code) - form = self.get_form(name='otp') - form['passcode'] = [code] - del form['cancelBtn'] - del form['btnSubmit'] - return form.submit() - - -class SummaryPage(LoggedInPage): - is_here = u'//title[contains(text(),"Account Summary")]' - - def to_activity(self): - href = self.doc.xpath(u'//a[text()="Account Activity"]/@href')[0] - self.browser.location(href) - - def to_documents(self): - href = self.doc.xpath('//a[text()="Statements & Documents"]' - '/@href')[0] - self.browser.location(href) - - -class AccountPage(object): - def account_id(self, name=None): - if name: - return name[-4:] # Last 4 digits of "BLAH XXXXXXX1234" - else: - return self.account_id(self.account_name()) - - -class ActivityPage(AccountPage, LoggedInPage): - def is_here(self): - return bool(self.doc.xpath( - u'contains(//title/text(),"Account Activity")')) - - def accounts_names(self): - return self.doc.xpath( - u'//select[@name="selectedAccountUID"]/option/text()') - - def accounts_ids(self): - return [self.account_id(name) for name in self.accounts_names()] - - def account_uid(self, id_=None): - if id_: - return self.doc.xpath( - u'//select[@name="selectedAccountUID"]' - u'/option[contains(text(),"%s")]/@value' % id_)[0] - else: - return self.doc.xpath( - u'//select[@name="selectedAccountUID"]' - u'/option[@selected="selected"]/@value')[0] - - def account_name(self): - for name in self.doc.xpath(u'//select[@name="selectedAccountUID"]' - u'/option[@selected="selected"]/text()'): - return name - return u'' - - def account_type(self, name=None): - raise NotImplementedError() - - def account_balance(self): - raise NotImplementedError() - - def account_paydatemin(self): - return None, None - - def account_cardlimit(self): - return None - - def to_account(self, id_): - form = self.get_form(xpath='//form[@name="AccountActivityForm"]') - form['selectedAccountUID'] = [self.account_uid(id_)] - form.submit() - - def get_account(self): - name = self.account_name() - balance = self.account_balance() - currency = Account.get_currency(balance) - id_ = self.account_id() - type_ = self.account_type() - paydate, paymin = self.account_paydatemin() - cardlimit = self.account_cardlimit() - - account = Account() - account.id = id_ - account.label = name - account.currency = currency - account.balance = AmTr.decimal_amount(balance) - account.type = type_ - if paydate is not None: - account.paydate = paydate - if paymin is not None: - account.paymin = paymin - if cardlimit is not None: - account.cardlimit = AmTr.decimal_amount(cardlimit) - - return account - - def since_last_statement(self): - raise NotImplementedError() - - def iter_transactions(self): - raise NotImplementedError() - - def next_(self): - raise NotImplementedError() - - -class ActivityCashPage(ActivityPage): - def is_here(self): - return super(ActivityCashPage, self).is_here() and \ - (u'CHECKING' in self.account_name() or - u'SAVINGS' in self.account_name()) - - def account_type(self, name=None): - name = name or self.account_name() - if u'CHECKING' in name: - return Account.TYPE_CHECKING - elif u'SAVINGS' in name: - return Account.TYPE_SAVINGS - else: - return Account.TYPE_UNKNOWN - - def account_balance(self): - return self.doc.xpath( - u'//td[@headers="currentPostedBalance"]/span/text()')[0] - - def since_last_statement(self): - if not self.doc.xpath(u'//p[@id="noactivitymessage"]'): - form = self.get_form(xpath='//form[@id="ddaShowForm"]') - form['showTabDDACommand.transactionTypeFilterValue'] = [ - u'All Transactions'] - form['showTabDDACommand.timeFilterValue'] = ['8'] - form.submit() - return True - - def iter_transactions(self): - for row in self.doc.xpath('//tr/th[@headers=' - '"postedHeader dateHeader"]/..'): - date = row.xpath('th[@headers="postedHeader ' - 'dateHeader"]/text()')[0] - desc = row.xpath('td[@headers="postedHeader descriptionHeader"]' - '//span[@class="OneLinkNoTx"]/text()')[0] - deposit = row.xpath('td[@headers="postedHeader ' - 'depositsConsumerHeader"]/span/text()')[0] - withdraw = row.xpath('td[@headers="postedHeader ' - 'withdrawalsConsumerHeader"]/span/text()')[0] - - date = datetime.datetime.strptime(date, '%m/%d/%y') - - desc = clean_label(desc) - - deposit = deposit.strip() - deposit = AmTr.decimal_amount(deposit or '0') - withdraw = withdraw.strip() - withdraw = AmTr.decimal_amount(withdraw or '0') - - amount = deposit - withdraw - - trans = Transaction(u'') - trans.date = date - trans.rdate = date - trans.type = Transaction.TYPE_UNKNOWN - trans.raw = desc - trans.label = desc - trans.amount = amount - yield trans - - def next_(self): - links = self.doc.xpath('//a[@title="Go To Next Page"]/@href') - if links: - self.browser.location(links[0]) - return True - else: - return False - - -class ActivityCardPage(ActivityPage): - def is_here(self): - return super(ActivityCardPage, self).is_here() and \ - u'CARD' in self.account_name() - - def account_type(self, name=None): - return Account.TYPE_CARD - - def account_balance(self): - return self.doc.xpath( - u'//td[@headers="outstandingBalance"]/text()')[0] - - def account_cardlimit(self): - return self.doc.xpath( - u'//td[@headers="totalCreditLimit"]/text()')[0] - - def account_paydatemin(self): - if self.doc.xpath(u'//p[contains(text(),' - '"payment due date is not yet scheduled")]'): - # If payment date is not scheduled yet, set it somewhere in a - # distant future, so that we always have a valid date. - return datetime.datetime.now() + datetime.timedelta(days=999), \ - Decimal(0) - else: - date = self.doc.xpath(u'//span[contains(text(),"Minimum Payment")]' - '/span/text()')[0] - date = re.match(u'.*(../../..).*', date).group(1) - date = datetime.datetime.strptime(date, '%m/%d/%y') - amount = self.doc.xpath(u'//td[@headers="minimumPaymentDue"]' - '//text()')[0].strip() - return date, AmTr.decimal_amount(amount) - - def get_account(self): - account = ActivityPage.get_account(self) - - # Credit card is essentially a liability. - # Negative amount means there's a payment due. - account.balance = -account.balance - - return account - - def since_last_statement(self): - if self.doc.xpath('//select[@name="showTabCommand.' - 'transactionTypeFilterValue"]' - '/option[@value="sincelastStmt"]'): - form = self.get_form(xpath='//form[@id="creditCardShowForm"]') - form['showTabCommand.transactionTypeFilterValue'] = [ - 'sincelastStmt'] - form.submit() - return True - - def iter_transactions(self): - for row in self.doc.xpath('//tr/th[@headers=' - '"postedHeader transactionDateHeader"]/..'): - tdate = row.xpath('th[@headers="postedHeader ' - 'transactionDateHeader"]/text()')[0] - pdate = row.xpath('td[@headers="postedHeader ' - 'postingDateHeader"]/text()')[0] - desc = row.xpath('td[@headers="postedHeader ' - 'descriptionHeader"]/span/text()')[0] - ref = row.xpath('td[@headers="postedHeader ' - 'descriptionHeader"]/text()')[0] - amount = row.xpath('td[@headers="postedHeader ' - 'amountHeader"]/text()')[0] - - tdate = datetime.datetime.strptime(tdate, '%m/%d/%y') - pdate = datetime.datetime.strptime(pdate, '%m/%d/%y') - - desc = clean_label(desc) - - ref = re.match('.*]+)>.*', ref).group(1) - - if amount.startswith('+'): - amount = AmTr.decimal_amount(amount[1:]) - else: - amount = -AmTr.decimal_amount(amount) - - trans = Transaction(ref) - trans.date = tdate - trans.rdate = pdate - trans.type = Transaction.TYPE_UNKNOWN - trans.raw = desc - trans.label = desc - trans.amount = amount - yield trans - - def next_(self): - # As of 2014-07-05, there's only one page for cards history. - return False - - -class DocumentsPage(LoggedInPage): - HEADER_XPATH = u'//h1[contains(text(),"Statements and Documents")]' - LINK_XPATH = u'//a[@data-async-load-template="stmtdisc.html"]' \ - u'/@data-async-load-url' - - def is_here(self): - return self.doc.xpath(self.HEADER_XPATH) \ - and self.doc.xpath(self.LINK_XPATH) - - def to_statements(self): - url = self.doc.xpath(self.LINK_XPATH)[0] - self.browser.location(url) - - -class StatementsEmbeddedPage(LoggedInPage): - HEADER_XPATH = u'//h1[contains(text(),"Statements and Documents")]' - SCRIPT_XPATH = '//script[contains(text(),"stmtdisc.html")]/text()' - - def is_here(self): - return self.doc.xpath(self.HEADER_XPATH) \ - and self.doc.xpath(self.SCRIPT_XPATH) - - def get_embedded_data(self): - scr = self.doc.xpath(self.SCRIPT_XPATH)[0] - data = json.loads('\n'.join(scr.split('\n')[2:-2]).replace( - "'appendTo'", '"appendTo"')) - return json.loads(data['data']) - - def parser(self): - return StatementsPageParser(self.get_embedded_data(), self.browser) - - -class WfJsonPage(LoggedPage, RawPage): - def __init__(self, *args, **kwArgs): - RawPage.__init__(self, *args, **kwArgs) - clean = self.doc.replace('"/*WellFargoProprietary%', '') \ - .replace('%WellFargoProprietary*/"', '').decode('string_escape') - self.doc = json.loads(clean) - - -class StatementsPage(WfJsonPage): - def parser(self): - return StatementsPageParser(self.doc, self.browser) - - -class StatementsPageParser(AccountPage): - def __init__(self, doc, browser): - self.doc = doc - self.browser = browser - - def account_name(self): - for account in self.accounts(): - if account['selected']: - return account['accountDisplayName'] - - def to_account(self, id_): - for account in self.accounts(): - if account['accountDisplayName'].endswith(id_): - self.browser.location(account['url']) - - def year(self): - for period in self.periods(): - if period['selected']: - try: - return int(period['timePeriod']) - except ValueError: - pass - - def years(self): - for period in self.periods(): - try: - yield int(period['timePeriod']) - except ValueError: - pass - - def to_year(self, year): - for period in self.periods(): - if period['timePeriod'] == str(year): - self.browser.location(period['url']) - - def statements(self): - stmts = self.doc['statementsDisclosuresInfo'].get('statements', []) - for stmt in stmts: - yield stmt['url'] - - def accounts(self): - return self.doc['statementsDisclosuresInfo']['accountList'] - - def periods(self): - return self.doc['statementsDisclosuresInfo']['timePeriodList'] - - -class StatementPage(LoggedPage, RawPage): - def __init__(self, *args, **kwArgs): - RawPage.__init__(self, *args, **kwArgs) - self._parser = StatementParser(self.doc) - - def is_here(self): - return self.doc[:4] == '%PDF' - - def iter_transactions(self): - # Maintain a nice consistent newer-to-older order of transactions. - return sorted( - itertools.chain( - self._parser.read_cash_transactions(), - self._parser.read_card_transactions()), - cmp=lambda t1, t2: cmp(t2.date, t1.date) or - cmp(t1.label, t2.label) or - cmp(t1.amount, t2.amount)) diff --git a/modules/wellsfargo/parsers.py b/modules/wellsfargo/parsers.py deleted file mode 100644 index bc5e730a5..000000000 --- a/modules/wellsfargo/parsers.py +++ /dev/null @@ -1,321 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Oleg Plakhotniuk -# -# This file is part of a weboob module. -# -# This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This weboob module is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this weboob module. If not, see . - -from weboob.capabilities.bank import Transaction -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.tools.date import closest_date -from weboob.tools.pdf import decompress_pdf -from weboob.tools.tokenizer import ReTokenizer -from weboob.tools.compat import unicode -import re -import datetime - - -def clean_label(text): - """ - Web view and statements use different label formatting. - User shouldn't be able to see the difference, so we - need to make labels from both sources look the same. - """ - return re.sub(u' +', u' ', text.strip().upper(), re.UNICODE) - - -class StatementParser(object): - """ - Each "read_*" method takes position as its argument, - and returns next token position if read was successful, - or the same position if it was not. - """ - - LEX = [ - ('amount', r'^\[\(([0-9,]+\.\d+)\)\] TJ$'), - ('date', r'^\[\((\d+/\d+)\)\] TJ$'), - ('date_range_1', r'^\[\(([A-z]+ \d+, \d{4})' - r' - ([A-z]+ \d+, \d{4})\)\] TJ$'), - ('date_range_2', r'^\[\((\d{2}/\d{2}/\d{4})' - r' to (\d{2}/\d{2}/\d{4})\)\] TJ$'), - ('layout_tz', r'^(\d+\.\d{2}) Tz$'), - ('layout_tc', r'^(\d+\.\d{2}) Tc$'), - ('layout_tw', r'^(\d+\.\d{2}) Tw$'), - ('layout_tf', r'^/F(\d) (\d+\.\d{2}) Tf$'), - ('layout_tm', r'^' + (r'(\d+\.\d+ )'*6) + r'Tm$'), - ('ref', r'^\[\(([0-9A-Z]{17})\)\] TJ$'), - - ('text', r'^\[\(([^\)]+)\)\] TJ$') - ] - - def __init__(self, pdf): - self._pdf = decompress_pdf(pdf) - self._tok = ReTokenizer(self._pdf, '\n', self.LEX) - - def read_card_transactions(self): - # Early check if this is a card account statement at all. - if '[(Transactions)] TJ' not in self._pdf: - return - - # Read statement dates range. - date_from, date_to = self.read_first_date_range() - - # Read transactions. - pos = 0 - while not self._tok.tok(pos).is_eof(): - pos, trans = self.read_card_transaction(pos, date_from, date_to) - if trans: - yield trans - else: - pos += 1 - - def read_cash_transactions(self): - # Early check if this is a cash account statement at all. - if '[(Transaction history)] TJ' not in self._pdf: - return - - # Read statement dates range. - date_from, date_to = self.read_first_date_range() - - # Read transactions. - pos = 0 - while not self._tok.tok(pos).is_eof(): - pos, trans = self.read_cash_transaction(pos, date_from, date_to) - if trans: - yield trans - else: - pos += 1 - - def read_first_date_range(self): - pos = 0 - while not self._tok.tok(pos).is_eof(): - pos, date_range = self.read_date_range(pos) - if date_range is not None: - return date_range - else: - pos += 1 - - def read_card_transaction(self, pos, date_from, date_to): - INDENT_CHARGES = 520 - - startPos = pos - - pos, tdate = self.read_date(pos) - pos, pdate_layout = self.read_layout_tm(pos) - pos, pdate = self.read_date(pos) - pos, ref_layout = self.read_layout_tm(pos) - pos, ref = self.read_ref(pos) - pos, desc = self.read_multiline_desc(pos) - pos, amount = self.read_indent_amount( - pos, - range_minus = (INDENT_CHARGES, 9999), - range_plus = (0, INDENT_CHARGES)) - - if tdate is None or pdate_layout is None or pdate is None \ - or ref_layout is None or ref is None or desc is None or amount is None: - return startPos, None - else: - tdate = closest_date(tdate, date_from, date_to) - pdate = closest_date(pdate, date_from, date_to) - - trans = Transaction(ref) - trans.date = tdate - trans.rdate = pdate - trans.type = Transaction.TYPE_UNKNOWN - trans.raw = desc - trans.label = desc - trans.amount = amount - return pos, trans - - def read_cash_transaction(self, pos, date_from, date_to): - INDENT_BALANCE = 520 - INDENT_WITHDRAWAL = 470 - - startPos = pos - - pos, date = self.read_date(pos) - pos, _ = self.read_star(pos) - pos, desc = self.read_multiline_desc(pos) - pos, amount = self.read_indent_amount( - pos, - range_plus = (0, INDENT_WITHDRAWAL), - range_minus = (INDENT_WITHDRAWAL, INDENT_BALANCE), - range_skip = (INDENT_BALANCE, 9999)) - - if desc is None or date is None or amount is None: - return startPos, None - else: - date = closest_date(date, date_from, date_to) - - trans = Transaction(u'') - trans.date = date - trans.rdate = date - trans.type = Transaction.TYPE_UNKNOWN - trans.raw = desc - trans.label = desc - trans.amount = amount - return pos, trans - - def read_multiline_desc(self, pos): - startPos = pos - - descs = [] - while True: - prevPos = pos - pos, layout = self.read_layout_tm(pos) - pos, desc = self.read_text(pos) - if layout is None or desc is None: - pos = prevPos - break - else: - descs.append(desc) - - if descs: - return pos, clean_label(' '.join(descs)) - else: - return startPos, None - - def read_indent_amount(self, pos, range_skip=(0,0), range_plus=(0,0), - range_minus=(0,0)): - startPos = pos - - # Read layout-amount pairs. - amounts = [] - while True: - prevPos = pos - pos, layout = self.read_layout_tm(pos) - pos, amount = self.read_amount(pos) - if layout is None or amount is None: - pos = prevPos - break - else: - amounts.append((layout, amount)) - - if not amounts: - return startPos, None - else: - # Infer amount type by its indentation in the layout. - amount_total = AmTr.decimal_amount('0') - for (_, _, _, _, indent, _), amount in amounts: - within = lambda xmin_xmax: xmin_xmax[0] <= indent <= xmin_xmax[1] - if within(range_skip): - continue - elif within(range_plus): - amount_total += amount - elif within(range_minus): - amount_total -= amount - return pos, amount_total - - def read_star(self, pos): - pos1, star1 = self.read_star_1(pos) - pos2, star2 = self.read_star_2(pos) - if star1 is not None: - return pos1, star1 - else: - return pos2, star2 - - def read_star_1(self, pos): - startPos = pos - - vals = list() - pos, v = self.read_layout_tz(pos); vals.append(v) - pos, v = self.read_layout_tc(pos); vals.append(v) - pos, v = self.read_layout_tw(pos); vals.append(v) - pos, v = self.read_layout_tf(pos); vals.append(v) - pos, v = self.read_layout_tm(pos); vals.append(v) - pos, star = self.read_text(pos) - pos, v = self.read_layout_tz(pos); vals.append(v) - pos, v = self.read_layout_tc(pos); vals.append(v) - pos, v = self.read_layout_tw(pos); vals.append(v) - pos, v = self.read_layout_tf(pos); vals.append(v) - - if star == 'S' and None not in vals: - return pos, star - else: - return startPos, None - - def read_star_2(self, pos): - startPos = pos - - vals = list() - pos, v = self.read_layout_tf(pos); vals.append(v) - pos, v = self.read_layout_tm(pos); vals.append(v) - pos, star = self.read_text(pos) - pos, v = self.read_layout_tf(pos); vals.append(v) - - if star == 'S' and None not in vals: - return pos, star - else: - return startPos, None - - def read_date(self, pos): - def parse_date(v): - for year in [1900, 1904]: # try leap and non-leap years - fullstr = '%s/%i' % (v, year) - try: - return datetime.datetime.strptime(fullstr, '%m/%d/%Y') - except ValueError as e: - last_error = e - raise last_error - - return self._tok.simple_read('date', pos, parse_date) - - def read_text(self, pos): - t = self._tok.tok(pos) - # TODO: handle PDF encodings properly. - return (pos+1, unicode(t.value(), errors='ignore')) \ - if t.is_text() else (pos, None) - - def read_amount(self, pos): - t = self._tok.tok(pos) - return (pos+1, AmTr.decimal_amount(t.value())) \ - if t.is_amount() else (pos, None) - - def read_date_range(self, pos): - t = self._tok.tok(pos) - if t.is_date_range_1(): - return (pos+1, [datetime.datetime.strptime(v, '%B %d, %Y') - for v in t.value()]) - elif t.is_date_range_2(): - return (pos+1, [datetime.datetime.strptime(v, '%m/%d/%Y') - for v in t.value()]) - else: - return (pos, None) - - def read_ref(self, pos): - t = self._tok.tok(pos) - return (pos+1, t.value()) if t.is_ref() else (pos, None) - - def read_layout_tz(self, pos): - t = self._tok.tok(pos) - return (pos+1, t.value()) if t.is_layout_tz() else (pos, None) - - def read_layout_tc(self, pos): - t = self._tok.tok(pos) - return (pos+1, t.value()) if t.is_layout_tc() else (pos, None) - - def read_layout_tw(self, pos): - t = self._tok.tok(pos) - return (pos+1, t.value()) if t.is_layout_tw() else (pos, None) - - def read_layout_tf(self, pos): - t = self._tok.tok(pos) - return (pos+1, t.value()) if t.is_layout_tf() else (pos, None) - - def read_layout_tm(self, pos): - t = self._tok.tok(pos) - return (pos+1, [float(v) for v in t.value()]) \ - if t.is_layout_tm() else (pos, None) -- GitLab