From 4bb66891158aa416b851f868b640702b32f94ba6 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Sat, 5 Jan 2019 16:00:08 +0100 Subject: [PATCH] Update of modules --- modules/alloresto/browser.py | 66 ----- modules/alloresto/favicon.png | Bin 2320 -> 0 bytes modules/alloresto/module.py | 61 ----- modules/alloresto/pages.py | 75 ------ modules/alloresto/test.py | 32 --- modules/{voyagessncf => asana}/__init__.py | 8 +- modules/asana/browser.py | 148 +++++++++++ modules/asana/favicon.png | Bin 0 -> 2620 bytes modules/asana/module.py | 233 ++++++++++++++++++ modules/asana/test.py | 148 +++++++++++ modules/caissedepargne/browser.py | 11 +- modules/canaltp/browser.py | 76 ------ modules/canaltp/favicon.png | Bin 2440 -> 0 bytes modules/canaltp/module.py | 49 ---- modules/canaltp/test.py | 31 --- modules/creditmutuel/pages.py | 15 +- modules/ehentai/__init__.py | 22 -- modules/ehentai/browser.py | 104 -------- modules/ehentai/favicon.png | Bin 917 -> 0 bytes modules/ehentai/gallery.py | 32 --- modules/ehentai/module.py | 117 --------- modules/ehentai/pages.py | 93 ------- modules/ehentai/test.py | 43 ---- modules/geolocip/__init__.py | 3 - modules/geolocip/favicon.png | Bin 3850 -> 0 bytes modules/geolocip/module.py | 72 ------ modules/geolocip/test.py | 28 --- modules/lcl/browser.py | 4 +- .../{canaltp => meslieuxparis}/__init__.py | 9 +- modules/meslieuxparis/browser.py | 52 ++++ .../test.py => meslieuxparis/module.py} | 34 ++- modules/meslieuxparis/pages.py | 94 +++++++ modules/meslieuxparis/test.py | 48 ++++ .../{alloresto => pagesjaunes}/__init__.py | 9 +- modules/pagesjaunes/browser.py | 50 ++++ modules/pagesjaunes/module.py | 52 ++++ modules/pagesjaunes/pages.py | 72 ++++++ modules/societegenerale/pages/transfer.py | 2 +- modules/societegenerale/sgpe/browser.py | 6 +- modules/voyagessncf/browser.py | 58 ----- modules/voyagessncf/favicon.png | Bin 5350 -> 0 bytes modules/voyagessncf/module.py | 124 ---------- modules/voyagessncf/pages.py | 118 --------- 43 files changed, 964 insertions(+), 1235 deletions(-) delete mode 100644 modules/alloresto/browser.py delete mode 100644 modules/alloresto/favicon.png delete mode 100644 modules/alloresto/module.py delete mode 100644 modules/alloresto/pages.py delete mode 100644 modules/alloresto/test.py rename modules/{voyagessncf => asana}/__init__.py (83%) create mode 100644 modules/asana/browser.py create mode 100644 modules/asana/favicon.png create mode 100644 modules/asana/module.py create mode 100644 modules/asana/test.py delete mode 100644 modules/canaltp/browser.py delete mode 100644 modules/canaltp/favicon.png delete mode 100644 modules/canaltp/module.py delete mode 100644 modules/canaltp/test.py delete mode 100644 modules/ehentai/__init__.py delete mode 100644 modules/ehentai/browser.py delete mode 100644 modules/ehentai/favicon.png delete mode 100644 modules/ehentai/gallery.py delete mode 100644 modules/ehentai/module.py delete mode 100644 modules/ehentai/pages.py delete mode 100644 modules/ehentai/test.py delete mode 100644 modules/geolocip/__init__.py delete mode 100644 modules/geolocip/favicon.png delete mode 100644 modules/geolocip/module.py delete mode 100644 modules/geolocip/test.py rename modules/{canaltp => meslieuxparis}/__init__.py (82%) create mode 100644 modules/meslieuxparis/browser.py rename modules/{voyagessncf/test.py => meslieuxparis/module.py} (50%) create mode 100644 modules/meslieuxparis/pages.py create mode 100644 modules/meslieuxparis/test.py rename modules/{alloresto => pagesjaunes}/__init__.py (82%) create mode 100644 modules/pagesjaunes/browser.py create mode 100644 modules/pagesjaunes/module.py create mode 100644 modules/pagesjaunes/pages.py delete mode 100644 modules/voyagessncf/browser.py delete mode 100644 modules/voyagessncf/favicon.png delete mode 100644 modules/voyagessncf/module.py delete mode 100644 modules/voyagessncf/pages.py diff --git a/modules/alloresto/browser.py b/modules/alloresto/browser.py deleted file mode 100644 index d790c92a9..000000000 --- a/modules/alloresto/browser.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword - -from .pages import LoginPage, AccountsPage - - -__all__ = ['AlloRestoBrowser'] - - -class AlloRestoBrowser(LoginBrowser): - BASEURL = 'https://www.alloresto.fr' - - login = URL('/identification-requise.*', LoginPage) - accounts = URL('/chez-moi/releve-compte-miams', AccountsPage) - - def do_login(self): - assert isinstance(self.username, basestring) - assert isinstance(self.password, basestring) - - self.accounts.stay_or_go() - self.page.login(self.username, self.password) - - if self.login.is_here(): - raise BrowserIncorrectPassword() - - @need_login - def get_accounts_list(self): - return self.accounts.stay_or_go().iter_accounts() - - @need_login - def get_account(self, id): - assert isinstance(id, basestring) - - for a in self.get_accounts_list(): - if a.id == id: - return a - - return None - - @need_login - def get_history(self, account): - return self.accounts.stay_or_go().get_transactions(type='consommable') - - @need_login - def get_coming(self, account): - return self.accounts.stay_or_go().get_transactions(type='acquisition') diff --git a/modules/alloresto/favicon.png b/modules/alloresto/favicon.png deleted file mode 100644 index b1b365c6dac425feb9450fa5c73a1957bb68d000..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2320 zcmV+r3GeoaP)ZSky02y>e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{00?+VL_t(|+U;3uaFo>nyN98fD) zX)06{sI)>L0!fSlNz9ETyO~@!*~_=zcb=y|zHisfZZ?UM?XT~d*&n;_d(Qiw=e*}V z?|T;T;lqayA3l8e@ZrOU4<9~!`0(*5qQC{_;fFgJ4tMqT_MVe1H{IQJtgQ_@!1srn zXS8~mh`sVkDR=HX%I0R9mL>M|kX2PZ&d!}LG&VM#{(m9BEL*mZ#z>Nqt*t-KmdgWy z=y3ClRxbs+tn3~}B1U<@`udY?ZEa^1bn(_*cm0SxJ!zxPu3anhElhw$L%>LS0FWpz zznbwlC5h|Ti?e2l!Jv5FdE!ksiS6yKKl}IpwV(v>{r5X1iDhMyBvV$lqtFCYR8$1n z($XbKeC8PuXt9C0A>hm;@GSxdc>VQm6r4EGRj|d|FT8NCBysm{!$pf+gZuV17McLN zvT~Uu@uQDqfgrGw~5`V)zm!tH)dO}8XDci;Vtx%AQjNn%q|vd{#WqerVH ziHjGDz+2h!5O5}gL6?9;LWSA=va%&!h$Wmv;!z$w+MDvf@1ch-DkK36hZB;-Y13qY zpJvNjfC-#B)g_>{m1Dnn+rEPB?U8)h*IBvpcLgQDvfShs49Ws#<|Ihq%e?t!mw+8R z5@WIE(*}L%=~+@VdGgr+psFh9@yDUdE`zG80RTmlCVj7+6$}#4E3qfb@qOHn^;B-ySY#1eBH8lEl@k6$ieQlOtBF zD3>HQH_JKaoIi|h_=iG^C5c~j?=c?Yxl^JO53(C z>*(k>D;A4QHO6vwZLzLS4lqBx zW1h1#MkF2YLy|Rvc6O9{26%+G+#<$eLsx*dE#7yZ2y7brtMB;nK0f`l2;>wz2KYH& zd{I1fNCY;g$86jvI!>;mwY4IUbb#wVi(WVI1p-Z+Ia7S@Iq{QE#6&_2g~ZL9#mg@j zf$lgkZR`mE`US`Vev&O~1WqT=B0vNp8Do00kISH=1FZcldH|3FW~Xcoq1W#K*No-5 zGjD-r0Xz-(8-e2lwi#e?3ou~@U=g@eKo5Xdy+!>K00azoANP1V4|`*W&JnHD z_S@pprD7l;0>^FOJ30D?4vFu*CmuK;wzY||m>aI)u>1S{_lv-qtWx$$0{y)9T5-=F zabQ5~?G^X!vy3sfdc>|0Ahx!Cnm79U#b{JaCf)Cmh}h5|e(-^~ZCiDw4KIS41c(=1 zB(7cS>ROiA-7VJDoixV$DyO9$GU=3;i!)}3fb9S`jHEwkY*iH<;0Ng|9yr$kA+B0A zv_4r201j|9fqp*qlTD<~E)J0mseMd(C9{)EJ9CkH8{2xvP60xU+OZrEe?Y~cf082pn*2oNh z?{-<3Cg7fXYCL=Y$c+GRXBY~JbLXZOvm*g~YBV`a#t<^B_-k$Pq>9geZ9uz=?PORpI7rOb! z7?PypIM(_FfRdpePS$@mw;3!DL})v)TCpK`DJbh9X#l;yj%pVtmH9|t5$XK$Pp3<5%?nj z*z@N7ncKI!xgvEOl1Z_%Gn1^f-W>A@7|xj^ZrC8Uw79i-d%L)GE6oEBY#8AP_42?0 z@%7h-J#LwuWKK&R4%0L>{l~WLD~25G0M`&$H*Ck073Kzbl)zyEeFTiRXp2B6f$}79 zX~z8X3H+Bpj6g4eN&-(?z{MlJR=G{oY6n;}l5PZ;L|`R>V+0Ze+6b(-fN%K!_WJPQ q!-o$aK79D_;lqayA3l8e0R9IFRg;Xs2z5aK0000. - - -from weboob.capabilities.bank import CapBank, AccountNotFound -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword - -from .browser import AlloRestoBrowser - - -__all__ = ['AlloRestoModule'] - - -class AlloRestoModule(Module, CapBank): - NAME = 'alloresto' - MAINTAINER = u'Romain Bignon' - EMAIL = 'romain@weboob.org' - VERSION = '1.4' - DESCRIPTION = u'Allo Resto' - LICENSE = 'AGPLv3+' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Mot de passe')) - BROWSER = AlloRestoBrowser - - def create_default_browser(self): - return self.create_browser(self.config['login'].get(), - self.config['password'].get()) - - def iter_accounts(self): - return self.browser.get_accounts_list() - - def get_account(self, _id): - account = self.browser.get_account(_id) - - if account: - return account - else: - raise AccountNotFound() - - def iter_history(self, account): - return self.browser.get_history(account) - - def iter_coming(self, account): - return self.browser.get_coming(account) diff --git a/modules/alloresto/pages.py b/modules/alloresto/pages.py deleted file mode 100644 index 1e248da5b..000000000 --- a/modules/alloresto/pages.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -import datetime -from decimal import Decimal - -from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.elements import ItemElement, method -from weboob.browser.filters.standard import CleanDecimal, CleanText, Filter, TableCell -from weboob.capabilities.bank import Account -from weboob.tools.capabilities.bank.transactions import FrenchTransaction as Transaction - - -class LoginPage(HTMLPage): - def login(self, username, password): - form = self.get_form(xpath='//form[has-class("form_o")]') - form['uname'] = username - form['pass'] = password - form.submit() - - -class AccountsPage(LoggedPage, HTMLPage): - @method - class iter_accounts(ItemElement): - def __call__(self): - return self - - klass = Account - - obj_id = '0' - obj_label = u'Compte miams' - obj_balance = CleanDecimal('//div[@class="compteur"]//strong', replace_dots=True) - obj_currency = u'MIAM' - obj_coming = CleanDecimal('//table[@id="solde_acquisition_lignes"]//th[@class="col_montant"]', default=Decimal('0'), replace_dots=True) - - class MyDate(Filter): - MONTHS = ['janv', u'févr', u'mars', u'avr', u'mai', u'juin', u'juil', u'août', u'sept', u'oct', u'nov', u'déc'] - - def filter(self, txt): - day, month, year = txt.split(' ') - day = int(day) - year = int(year) - month = self.MONTHS.index(month.rstrip('.')) + 1 - return datetime.date(year, month, day) - - def get_transactions(self, type='consommable'): - class get_history(Transaction.TransactionsElement): - head_xpath = '//table[@id="solde_%s_lignes"]//thead//tr/th/text()' % type - item_xpath = '//table[@id="solde_%s_lignes"]//tbody/tr' % type - - col_date = u"Date de valeur" - col_raw = u"Motif" - - class item(Transaction.TransactionElement): - obj_amount = Transaction.Amount('./td[last()]') - obj_date = AccountsPage.MyDate(CleanText(TableCell('date'))) - - return get_history(self)() diff --git a/modules/alloresto/test.py b/modules/alloresto/test.py deleted file mode 100644 index a31ea0d08..000000000 --- a/modules/alloresto/test.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest - - -class AlloRestoTest(BackendTest): - MODULE = 'alloresto' - - def test_alloresto(self): - l = list(self.backend.iter_accounts()) - - a = l[0] - list(self.backend.iter_history(a)) - list(self.backend.iter_coming(a)) diff --git a/modules/voyagessncf/__init__.py b/modules/asana/__init__.py similarity index 83% rename from modules/voyagessncf/__init__.py rename to modules/asana/__init__.py index 55230f6d6..80aafb858 100644 --- a/modules/voyagessncf/__init__.py +++ b/modules/asana/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2013 Romain Bignon +# Copyright(C) 2017 Vincent Ardisson # # This file is part of weboob. # @@ -17,8 +17,10 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from .module import VoyagesSNCFModule +from .module import AsanaModule -__all__ = ['VoyagesSNCFModule'] + +__all__ = ['AsanaModule'] diff --git a/modules/asana/browser.py b/modules/asana/browser.py new file mode 100644 index 000000000..d5c1ea228 --- /dev/null +++ b/modules/asana/browser.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent Ardisson +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import time + +from weboob.browser.browsers import APIBrowser +from weboob.browser.exceptions import ClientError +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bugtracker import User, Project, Issue, Status, Update +from weboob.exceptions import BrowserIncorrectPassword +from dateutil.parser import parse as parse_date + + +class AsanaBrowser(APIBrowser): + BASEURL = 'https://app.asana.com/api/1.0/' + + STATUS_OPEN = Status(0, 'Open', Status.VALUE_NEW) + STATUS_CLOSED = Status(1, 'Closed', Status.VALUE_RESOLVED) + + def __init__(self, token, *args, **kwargs): + super(AsanaBrowser, self).__init__(*args, **kwargs) + self.token = token + self.session.headers['Authorization'] = 'Bearer %s' % token + + def open(self, *args, **kwargs): + try: + return super(AsanaBrowser, self).open(*args, **kwargs) + except ClientError as e: + if e.response.status_code == 401: + raise BrowserIncorrectPassword() + elif e.response.status_code == 429: + self.logger.warning('reached requests quota...') + waiting = int(e.response.headers['Retry-After']) + if waiting <= 60: + self.logger.warning('waiting %s seconds', waiting) + time.sleep(waiting) + return super(AsanaBrowser, self).open(*args, **kwargs) + else: + self.logger.warning('not waiting %s seconds, just fuck it', waiting) + + raise + + def _make_user(self, data): + u = User(data['id'], None) + if 'name' in data: + u.name = data['name'] + return u + + def _make_project(self, data): + p = Project(str(data['id']), data['name']) + p.url = 'https://app.asana.com/0/%s' % p.id + if 'members' in data: + p.members = [self._make_user(u) for u in data['members']] + + p.statuses = [self.STATUS_OPEN, self.STATUS_CLOSED] + p._workspace = data['workspace']['id'] + + # these fields don't exist in asana + p.priorities = [] + p.versions = [] + return p + + def _make_issue(self, data): + if data['name'].endswith(':'): + # section, not task + return None + + i = Issue(str(data['id'])) + i.url = 'https://app.asana.com/0/0/%s/f' % i.id + i.title = data['name'] + if 'notes' in data: + i.body = data['notes'] + if data.get('assignee'): + i.assignee = self._make_user(data['assignee']) + if data.get('created_by'): + # created_by is not documented + i.author = self._make_user(data['created_by']) + if 'created_at' in data: + i.creation = parse_date(data['created_at']) + if 'modified_at' in data: + i.updated = parse_date(data['modified_at']) + if 'due_at' in data: + if data['due_at']: + i.due = parse_date(data['due_at']) + else: + i.due = NotAvailable + if 'due_on' in data: + if data['due_on']: + i.due = parse_date(data['due_on']) + else: + i.due = NotAvailable + if data.get('projects'): + i.project = self._make_project(data['projects'][0]) + if 'completed' in data: + i.status = self.STATUS_CLOSED if data['completed'] else self.STATUS_OPEN + if 'custom_fields' in data: + def get(d): + for k in ('string_value', 'number_value', 'enum_value', 'text_value'): + if k in d: + return d[k] + assert False, 'custom type not handled' + i.fields = {d['name']: get(d) for d in data['custom_fields']} + if 'tags' in data: + i.tags = [d['name'] for d in data['tags']] + if data.get('memberships') and data['memberships'][0]['section']: + i.category = data['memberships'][0]['section']['name'] + + i.version = NotAvailable + i.priority = NotAvailable + return i + + def _make_update(self, data): + u = Update(str(data['id'])) + if 'created_at' in data: + u.date = parse_date(data['created_at']) + u.message = '%s: %s' % (data['type'], data['text']) + if 'created_by' in data: + u.author = self._make_user(data['created_by']) + return u + + def paginate(self, *args, **kwargs): + params = kwargs.setdefault('params', {}) + params['limit'] = 20 + reply = self.request(*args, **kwargs) + for d in reply['data']: + yield d + while reply.get('next_page') and reply['next_page'].get('uri'): + reply = self.request(reply['next_page']['uri']) + for d in reply['data']: + yield d diff --git a/modules/asana/favicon.png b/modules/asana/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..60ff9a2b9169e202a8ff626bd825c1e85ff1569c GIT binary patch literal 2620 zcmV-C3d8k@P)00009a7bBm000XU z000XU0RWnu7ytkO2XskIMF-&t6BRBIB`UDO0000TX;fHrLvL+uWo~o;00000Lvm$d zbY)~9cWHEJAV*0}P-HG;2LJ#GvPnciRA}DqTHA8lxDxCk2LP|K9cTCdzb|V$wkV0h zg`D#+(8XTK(ZqAHl%ai2E5K%DBaHkjE!{dz7aq3~L!j#E5AIfUL^kpmaM1T|pm1REJ zKa1fS0AiBOh?k{2e#f^*`Rgw{FJ&`gGa^RJGXN9?j1eCw1eqoR1Vls`gEgN7Km^l_ z5HOBdt?~U?9v1laFd-#`fX#^2S~eSOMvS9) z4_*F!A)7@+h=8IHYe;uVK~58dKVt_3m}kT&A;Nq3fDjR*q=YnKnsWT?(Gp2Y6eB`} z5Aa?>Bt0iTt4cmb=48e+BPE0=F(M}D)RRtK^1L%I63lb{-;@xOD3X*g&5-%u$42Gd zAO9^}hyWrGK$^>G_YHvnIp(29-$P%D!>@mi9w-WwC5$2G$XPfGYhVlvqbzoib}X6- zlm!gK7+3>m$yss^#z4#2=j=dHfEm^vlDe}LqL>*X(2}L=uf>>+w#n_S1@lkO{=)!> zz$FaBS+p(nJ=Pm7UIYkXcbXMvN!FpQ(KOVx)c0sxI19rcp`{-NP!!~xG!6BgY&IB2 zib+HOA}-(P0XT=Yp{~Wv0yjPN9khylCKoCS7)Dbew1{^!%>qzah+%9-gd;9w4Vs$T z26wmg$5(v0L*JsQU_LQ~auhLDl`Q66)*&&5mIIm*A<8rX$c(M7vFPdU7GJ;6ms>1) zs;bYX(~7|wnkx~IQvasLqLcLq?`f|GoTIiz-&5b=rbpjXRegkqUKK!5pmJ1@6uKb} zZ3E>O#{jHBQ=@IrHdI%z_9M}s2fvy(x>xieh!5}qF(SqT0ApZs`rl>GE9iQi<5?n%GuH>xza{&AH@614VLuLpvOUta`lqn=~y4%YTUIswhXfSA6 z%&dlT27DjP$p*?&1(;G+cKf_cgmNd8e*}XsjqG_B73%X45%Vq{iuW|($uBk1cJy0P zL`<1;a0zB&hG7`~q1xmhdsQJw2-te;>AOnA%-alARb~Virj&U`O7KCp9^;4*vIV9! z+ZtFahEdm4S8(qAp836DV#GGcdX4o4>y3nv2h(+R4%SdrN!OrlsA*u$kE_x=YkHMo zh1FVi@*h6r;ItObqOPcGrEAf5d4bNHdwyq_EO4KfvRYv|U|37=#{j6Vr0?K+3Q=aE zxXHRGXbbhQzky^w0WP0dHjxt zM_CS7t+Ni)GhmH4hrXlR8~Nit^N4qhDh7*^zi zh!7y&?NeB@$y$Ir3gY&|F4<6m-_8ngt=qdX49$JbssN z4|sT#VTJVuTYoTMxD;a%B84Ec5FkWU5LiXh+XKy~owlvVYAr9zJk0Y;c6(H_t3G6g z_mUFDbl`B-z`7G7i6A8j0qc#dR(N>C0jvS3 z4$`R%9ng28JiM;+-X3A#zr|0|)QQcQowoIE_Sq{JXt>Oi1}H$n zOwM6gN#Bw3pLs@kWVR`}h*3hou+9OwIqEjsFa(-0Yd{j1VT}wcx>+D5Nr|S{-JT2~ zNih@2trzdH^=en;awgfL(zGo^!~t6m?`7OlZaLxa1%8?|-%Id_DSLAI%u_A`ki5Ey zFinU-LL{Yz$42E@S3*m=_U^TKC8Q)NW_3Hujsi$F*0W2sH|IP;6dmdKo^PH-TZWij zu#`)C-qK@Il-sA=`M&<>9A0B6&KQQ0o9-D4;W_nk-Z+Vz&o&&LgQaZJptljx#@263{v{FLiELkJQq6}cp=S(E03$?WL zu@;8OI#>&9-Xw}62g$%1sw;60m4o+`Qs(n&Q0t=b&b_M~X-nl@d`TrP3t|?-s4MC_ zSq{oTNy9VI&N0suqPogvN!Oxp$=ZB>`Ub<^gsdv*JM2ngp|s5tt@&BD+)pR4}maElIb*vO?FC}8eL0^9*Z7rgUX>e zb=^`HnY#L(Zf_(6E#UwWYl=MR0W>w5n))tVU|oy42J_#K!Kn~3Om!t)3m?eu2#JPl z47B4yU`FN8Hq>{x|AMdg)OBbZ(v$OMuZd8Wa29PtA<7^3nLZg7<}5aLTnN_YExrFj ze}1K#1-h0h2g4VHP#(#l??^9&l#ti~*2s~dfnjnEeTSO`{rQ#d?{K@IwuQAQ&PX%E z)HHcTls;KY&goT+0w6WFbxXhA)BPRpZmI9iNdi7QQdKDt7b0>NeJ8_;v?6Mo%As;ZkpeW!hoI_Jn*P?5w@1SP?JBmyLudb*pr695f&Pm^qR&hB3LT|nLIb3a|D-Op?+oJz$vF. + +from __future__ import unicode_literals + +from weboob.capabilities.base import empty +from weboob.capabilities.bugtracker import CapBugTracker, Project, Issue, User +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import ValueBackendPassword + +from .browser import AsanaBrowser + + +__all__ = ['AsanaModule'] + + +class AsanaModule(Module, CapBugTracker): + NAME = 'asana' + DESCRIPTION = 'Asana' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + CONFIG = BackendConfig(ValueBackendPassword('token', label='Personal access token')) + + BROWSER = AsanaBrowser + + def create_default_browser(self): + return self.create_browser(self.config['token'].get()) + + ## read-only issues and projects + def iter_issues(self, query): + query = query.copy() + + params = {} + + if query.title: + params['text'] = query.title + + if query.project: + if not isinstance(query.project, Project): + query.project = next(p for p in self.iter_projects() if query.project.lower() in p.name.lower()) + params['project'] = query.project.id + + if query.tags: + params['tags.all'] = ','.join(query.project._tagdict[tag] for tag in query.tags) + + if query.assignee: + if isinstance(query.assignee, User): + params['assignee'] = query.assignee.id + else: + params['assignee'] = query.assignee + + if query.author: + if isinstance(query.author, User): + params['created_by'] = query.author.id + else: + params['created_by'] = query.author + + if query.status: + if query.status.lower() == 'closed': + params['completed'] = 'true' + else: + params['completed'] = 'false' + params['completed_since'] = 'now' # completed=false is not enough... + + if not query.project: + workspaces = list(self._iter_workspaces()) + if len(workspaces) == 1: + params['workspace'] = workspaces[0] + + if query.project and query.assignee: + # asana's wtf api doesn't allow more than 1 filter... + del params['project'] + params['workspace'] = query.project._workspace + + opt = '?opt_fields=%s' % ','.join([ + 'name', 'completed', 'due_at', 'due_on', 'created_at', 'modified_at', + 'notes', 'custom_fields', 'tags.name', 'assignee.name', 'created_by.name', + 'projects.name', 'projects.workspace', + ]) + if params: + data = self.browser.paginate('tasks%s' % opt, params=params) + else: + data = [] + + for issue in data: + issue = self.browser._make_issue(issue) + if issue is None: # section + continue + + # post-filter because many server-side filters don't always work... + if query.title and query.title.lower() not in issue.title.lower(): + self.logger.debug('"title" filter failed on issue %r', issue) + continue + + if query.status and query.status.lower() != issue.status.name.lower(): + self.logger.debug('"completed" filter failed on issue %r', issue) + continue + + if query.tags and not (set(query.tags) <= set(issue.tags)): + self.logger.debug('"tags" filter failed on issue %r', issue) + continue + + if query.author: + if isinstance(query.author, User): + if query.author.id != issue.author.id: + continue + else: + if query.author.lower() != issue.author.name.lower(): + continue + + yield issue + + def _set_stories(self, issue): + ds = self.browser.request('tasks/%s/stories' % issue.id)['data'] + issue.history = [self.browser._make_update(d) for d in ds] + + def get_issue(self, id): + if not id.isdigit(): + return + + data = self.browser.request('tasks/%s' % id)['data'] + return self.browser._make_issue(data) + + def iter_projects(self): + for w in self._iter_workspaces(): + tags = self.browser.request('tags?workspace=%s' % w)['data'] + data = self.browser.paginate('projects?opt_fields=name,members.name,workspace&workspace=%s' % w) + for p in data: + project = self.browser._make_project(p) + self._assign_tags(tags, project) + yield project + + def _assign_tags(self, data, project): + project._tagdict = {d['name']: str(d['id']) for d in data} + project.tags = list(project._tagdict) + + def get_project(self, id): + if not id.isdigit(): + return + + data = self.browser.request('projects/%s' % id)['data'] + return self.browser._make_project(data) + + def _iter_workspaces(self): + return (d['id'] for d in self.browser.paginate('workspaces')) + + ## writing issues + def create_issue(self, project): + issue = Issue(0) + issue._project = project + return issue + + def post_issue(self, issue): + data = {} + if issue.title: + data['name'] = issue.title + if issue.body: + data['notes'] = issue.body + if issue.due: + data['due_at'] = issue.due.strftime('%Y-%m-%d') + if issue.assignee: + data['assignee'] = issue.assignee.id + + if issue.id and issue.id != '0': + data['projects'] = issue._project + self.browser.request('tasks', data=data) + if issue.tags: + self._set_tag_list(issue, True) + else: + self.browser.request('tasks/%s' % issue.id, data=data, method='PUT') + if not empty(issue.tags): + self._set_tag_list(issue) + + def _set_tag_list(self, issue, add=False): + to_remove = set() + to_add = set(issue.tags) + + if not add: + existing = set(self.get_issue(issue.id).tags) + to_add = to_add - existing + to_remove = existing - to_add + + for old in to_remove: + tagid = issue.project._tagdict[old] + self.browser.request('tasks/%s/removeTag', data={'tag': tagid}) + for new in to_add: + tagid = issue.project._tagdict[new] + self.browser.request('tasks/%s/addTag', data={'tag': tagid}) + + def update_issue(self, issue, update): + assert not update.changes, 'changes are not supported yet' + assert update.message + self.browser.request('tasks/%s/stories' % issue.id, data={'text': update.message}) + + def remove_issue(self, issue): + self.browser.request('tasks/%s' % issue.id, method='DELETE') + + ## filling + def fill_project(self, project, fields): + if set(['members']) & set(fields): + return self.get_project(project.id) + + def fill_issue(self, issue, fields): + if set(['body', 'assignee', 'due', 'creation', 'updated', 'project']) & set(fields): + new = self.get_issue(issue.id) + for f in fields: + if getattr(new, f): + setattr(issue, f, getattr(new, f)) + if 'history' in fields: + self._set_stories(issue) + + OBJECTS = { + Project: fill_project, + Issue: fill_issue, + } diff --git a/modules/asana/test.py b/modules/asana/test.py new file mode 100644 index 000000000..f35db7b7e --- /dev/null +++ b/modules/asana/test.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent Ardisson +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.capabilities.base import empty +from weboob.capabilities.bugtracker import Query +from weboob.tools.test import BackendTest + + +class AsanaTest(BackendTest): + MODULE = 'asana' + + def test_iter_projects(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + self.assertTrue(projects[0].statuses) + self.assertTrue(projects[0].members) + + def test_find_any_issues(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + + issues = [issue for issue, _ in zip(self.backend.iter_issues(q), range(30))] + self.assertTrue(issues) + + for issue in issues: + self.assertTrue(issue.project) + self.assertEquals(issue.project.id, projects[0].id) + self.assertTrue(issue.title) + self.assertFalse(empty(issue.body)) + self.assertTrue(issue.creation) + + self.assertTrue(issue.author, issue.title) + + def _test_find_by_criterion(self, attr, first_cb=None, matcher_cb=None): + if matcher_cb is None: + def matcher_cb(expected, actual): + self.assertEquals(expected.id, actual.id, 'different id on: %s != %s' % (expected, actual)) + + if first_cb is None: + def first_cb(obj): + return bool(obj) + + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + criterion_obj = getattr(issue, attr) + if first_cb(criterion_obj): + break + else: + assert False, 'not a single issue has this criterion' + + setattr(q, attr, criterion_obj) + + some = False + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + some = True + fetched_obj = getattr(issue, attr) + matcher_cb(criterion_obj, fetched_obj) + assert some, 'the issue searched for was not found' + + def test_find_by_assignee(self): + self._test_find_by_criterion('assignee') + + def test_find_by_author(self): + self._test_find_by_criterion('author') + + def test_find_by_title(self): + self._test_find_by_criterion( + 'title', + matcher_cb=lambda crit, actual: self.assertIn(crit.lower(), actual.lower()) + ) + + def test_find_by_tags(self): + self._test_find_by_criterion( + 'tags', + first_cb=lambda tags: bool(tags), + matcher_cb=lambda crit, actual: self.assertLessEqual(set(crit), set(actual)) + ) + + def _test_find_by_fields(self): + self._test_find_by_criterion( + 'fields', + first_cb=lambda tags: bool(tags), + matcher_cb=lambda crit, actual: self.assertLessEqual(set(crit), set(actual)) + ) + + def test_find_by_status(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + q.status = 'open' + + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + self.assertEquals(issue.status.name.lower(), 'open', issue.title) + + q.status = 'closed' + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + self.assertEquals(issue.status.name.lower(), 'closed', issue.title) + + def test_read_comments(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + self.backend.fillobj(issue, ['history']) + self.assertNotEmpty(issue.history) + if issue.history: + for update in issue.history: + self.assertTrue(update.author) + self.assertTrue(update.author.id) + self.assertTrue(update.author.name) + self.assertTrue(update.date) + self.assertTrue(update.message) + self.assertNotEmpty(update.changes) + + break + else: + assert 0, 'no issue had history' diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 2b783a036..eae03e34b 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -33,7 +33,9 @@ from weboob.capabilities.bank import Account, AddRecipientStep, Recipient, Trans from weboob.capabilities.base import NotAvailable from weboob.capabilities.profile import Profile from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPError, BrowserPasswordExpired +from weboob.exceptions import ( + BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPError, BrowserPasswordExpired, ActionNeeded +) from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction from weboob.tools.capabilities.bank.investments import create_french_liquidity from weboob.tools.compat import urljoin @@ -269,7 +271,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): if data.get('authMode', '') == 'redirect': # the connection type EU could also be used as a criteria raise SiteSwitch('cenet') - typeAccount = accounts_types[0] + typeAccount = data['account'][0] if self.multi_type: assert typeAccount == self.typeAccount @@ -309,6 +311,11 @@ class CaisseEpargne(LoginBrowser, StatesMixin): # the only possible way to log in w/o nuser is on WE. if we're here no need to go further. if not self.nuser and self.typeAccount == 'WE': raise BrowserIncorrectPassword(response['error']) + + # we tested all, next iteration will throw the assertion + if self.inexttype == len(accounts_types) and 'Temporairement votre abonnement est bloqué' in response['error']: + raise ActionNeeded(response['error']) + if self.multi_type: # try to log in with the next connection type's value self.do_login() diff --git a/modules/canaltp/browser.py b/modules/canaltp/browser.py deleted file mode 100644 index 1b0228913..000000000 --- a/modules/canaltp/browser.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from datetime import datetime, date, time - -from weboob.deprecated.browser import Browser -from weboob.tools.misc import to_unicode -from weboob.deprecated.browser import BrokenPageError - - -__all__ = ['CanalTP'] - - -class CanalTP(Browser): - DOMAIN = 'widget.canaltp.fr' - - def __init__(self, **kwargs): - Browser.__init__(self, '', **kwargs) - - def iter_station_search(self, pattern): - url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/gare.php?txtrech=%s' % unicode(pattern) - result = self.openurl(url.encode('utf-8')).read() - for station in result.split('&'): - try: - _id, name = station.split('=') - except ValueError: - continue - else: - yield _id, to_unicode(name) - - def iter_station_departures(self, station_id, arrival_id=None): - url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/index.php?gare=%s' % unicode(station_id) - result = self.openurl(url.encode('utf-8')).read() - result = result - departure = '' - for line in result.split('&'): - if '=' not in line: - raise BrokenPageError('Unable to parse result: %s' % line) - key, value = line.split('=', 1) - if key == 'nomgare': - departure = value - elif key.startswith('ligne'): - _type, unknown, _time, arrival, served, late, late_reason = value.split(';', 6) - yield {'type': to_unicode(_type), - 'time': datetime.combine(date.today(), time(*[int(x) for x in _time.split(':')])), - 'departure': to_unicode(departure), - 'arrival': to_unicode(arrival).strip(), - 'late': late and time(0, int(late.split()[0])) or time(), - 'late_reason': to_unicode(late_reason).replace('\n', '').strip()} - - def home(self): - pass - - def login(self): - pass - - def is_logged(self): - """ Do not need to be logged """ - return True diff --git a/modules/canaltp/favicon.png b/modules/canaltp/favicon.png deleted file mode 100644 index 9f61523abaa67e94dd9aa0497b6419a8d83e6100..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2440 zcmV;333v91P)Px#24YJ`L;(K){{a7>y{D4^000SaNLh0L01FcU01FcV0GgZ_00007bV*G`2igZ8 z4J97m4{+xI00{_5L_t(|+U=WPY+F|y$3N%Z>w6t1wqv(;XxFlI87k`rtJaNdtkWj` zXx)Hl5__Nz2!seAq*aKg@iZZUkS2jd_5zDE!2=o+n#KsAtpYS&rnOr(+6~Iql(}8D zG*03qzP4ju``*KYuRSNV(|eqy?b`fa?0e7m{_gpG&%fXO{my|c*^+BoM7x{gN1_6V z0x=-%-*$oXKw)&uE^g$d<42-9fdjx-{9k}cU;>yN9kbuxfbK!yKHwYvFTjhyZ-Ey_ z$L!3y1o*~Z2Rsbi3q0f(Wr5!TF9Uz+mIUAUM}Wh?HlWWhDgX<>)4&tmkl-8t81T&w zklz45ZY4qQVEp%ip8%h1DeeXC0&WM+jSO3viDOQ+OXI%->;TM`ViV{G_5sGouytzU zm}7s4#slmJ`bUPX6BEarB>-&=AnLdLkJeGL2l#1A*Hw{0;4?sS9Zb^440fS=Zvfoh zR^4j}i22?AVzNCTZmoO{xH|yuZXc^6U-oy7EmsoQo9-t<}r{EUL`WB`3lx%u;t?tJ%+ zXUS*ZB9^W%2I(8_0buv75A*Kno^)zJi^b#Q^ZB~xmd(~@RLjC~oaQxvZP#54 z@rS3MlFcO`oo<5C=`^WSYSlx@WRk_j#Z}C=mbWTy&Xmz;TivZPG&ID~qemGU8X}QM z0I;~Y$eA-|R_SQyZd|YT-oW1iPd)idc`bhoXKp9!(J}i#+nC_^k?0N}a4c?g z%obh5?M?$X28925bj+a}(kRZz$;8&N@NSOCgQ|b~zyKCB-`IyPO25gwetQ zaT$gA0$L)@mxBP6uvnlF-RPLT1WYOD-F7(%h$~=MHmMG~+Y;cXmo{5;1?>M6uyYDn zyHgJsb&H&|^0YxLA+6X0j7FKG|hrl2Hwg}KciN)MP; zP~u(!0%8i-zEuyV_Xffm|2X>GY0I(*~a`%C+PG^45n)Mw4{?zi@ zl;YyMx7Vr_&B{-TT&et-KJz6-WR2jI^wJ#yXKmkjv$|Zm!mZ zM=Ta2pU=y=xj845N|``C)+IeH7Vl?j>NM$e`wGTlG4jjHAn+Cza*IL;M+i}wot-TX z3=BkPXJ?lO1_lyRO2hNKn3U4UX0y%t>tacg*2K!DBu1jqD7I}AjYd~xgo0M^l7_@K zW%2e6v~Bx}GIpci=2xX`B}Ur1cq?U=^EQsZ3(hn{#A8u?RT;0x`yj|^!4@Px-MoY#L7y!i8pi|-OyJhGeSm)o}Mr(D=SoM zwN=-pl`@4w2~F4N30tcYC51wXLa{_F+Q;DayNL9Jn~h)>iph zR*AS(q*4pz&e`cl+ijM985j z5BL`_K7J(n6u~!4YMuN5m9?y!b?!Jv;ua%69Mz8N)#otiSyzv1?;&_1auWx zP>k~heJ-o0TO}&=Rwn{Z5-is?qd1-$aXtrlJRtmWMfED%b|*U#@G9_100po3*ly(S z0b+`$sh{;xk>fxm;4pl<69F0ES>VJv3QqW#uEz7|m^}%6%{OwUP0uU-iY!#?)v)@PO?e@. - -from weboob.capabilities.travel import CapTravel, Station, Departure -from weboob.tools.backend import Module - -from .browser import CanalTP - - -__all__ = ['CanalTPModule'] - - -class CanalTPModule(Module, CapTravel): - NAME = 'canaltp' - MAINTAINER = u'Romain Bignon' - EMAIL = 'romain@weboob.org' - VERSION = '1.4' - LICENSE = 'AGPLv3+' - DESCRIPTION = "French trains" - BROWSER = CanalTP - - def iter_station_search(self, pattern): - for _id, name in self.browser.iter_station_search(pattern): - yield Station(_id, name) - - def iter_station_departures(self, station_id, arrival_id=None, date=None): - for i, d in enumerate(self.browser.iter_station_departures(station_id, arrival_id)): - departure = Departure(i, d['type'], d['time']) - departure.departure_station = d['departure'] - departure.arrival_station = d['arrival'] - departure.late = d['late'] - departure.information = d['late_reason'] - yield departure diff --git a/modules/canaltp/test.py b/modules/canaltp/test.py deleted file mode 100644 index b5d191c8b..000000000 --- a/modules/canaltp/test.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest - - -class CanalTPTest(BackendTest): - MODULE = 'canaltp' - - def test_canaltp(self): - stations = list(self.backend.iter_station_search('defense')) - self.assertTrue(len(stations) > 0) - - list(self.backend.iter_station_departures(stations[0].id)) diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 6966ba1b2..ed2a5426c 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -1218,13 +1218,21 @@ class PorPage(LoggedPage, HTMLPage): class item(ItemElement): klass = Investment + def condition(self): + return not any(not x.isdigit() for x in Attr('.', 'id')(self)) + obj_label = CleanText(TableCell('label'), default=NotAvailable) - obj_code = CleanText('.//td[1]/a/@title') & Regexp(pattern=r'^([^ ]+)') obj_quantity = CleanDecimal(TableCell('quantity'), default=Decimal(0), replace_dots=True) obj_unitprice = CleanDecimal(TableCell('unitprice'), default=Decimal(0), replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) obj_diff = CleanDecimal(TableCell('diff'), default=Decimal(0), replace_dots=True) + def obj_code(self): + code = Regexp(CleanText('.//td[1]/a/@title'), r'^([^ ]+)')(self) + if 'masquer' in code: + return Regexp(CleanText('./following-sibling::tr[1]//a/@title'), r'^([^ ]+)')(self) + return code + def obj_unitvalue(self): r = CleanText(TableCell('unitvalue'))(self) if r[-1] == '%': @@ -1865,7 +1873,6 @@ class NewCardsListPage(LoggedPage, HTMLPage): obj_type = Account.TYPE_CARD obj__new_space = True obj__is_inv = False - load_details = Field('_link_id') & AsyncLoad def obj__secondpage(self): # Necessary to reach the good history page @@ -1909,8 +1916,8 @@ class NewCardsListPage(LoggedPage, HTMLPage): def parse(self, el): # We have to reach the good page with the information of the type of card - async_page = Async('details').loaded_page(self) - card_type_page = Link('//div/ul/li/a[contains(text(), "Fonctions")]')(async_page.doc) + history_page = self.page.browser.open(Field('_link_id')(self)).page + card_type_page = Link('//div/ul/li/a[contains(text(), "Fonctions")]')(history_page.doc) doc = self.page.browser.open(card_type_page).page.doc card_type_line = doc.xpath('//tbody/tr[th[contains(text(), "Débit des paiements")]]') if card_type_line: diff --git a/modules/ehentai/__init__.py b/modules/ehentai/__init__.py deleted file mode 100644 index 954af271c..000000000 --- a/modules/ehentai/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from .module import EHentaiModule - -__all__ = ['EHentaiModule'] diff --git a/modules/ehentai/browser.py b/modules/ehentai/browser.py deleted file mode 100644 index 79b8f5694..000000000 --- a/modules/ehentai/browser.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from weboob.deprecated.browser import Browser, BrowserIncorrectPassword -from weboob.tools.compat import urlencode - -from .pages import IndexPage, GalleryPage, ImagePage, HomePage, LoginPage -from .gallery import EHentaiImage - - -__all__ = ['EHentaiBrowser'] - - -class EHentaiBrowser(Browser): - ENCODING = None - PAGES = { - r'http://[^/]+/': IndexPage, - r'http://[^/]+/\?.*': IndexPage, - r'http://[^/]+/g/.+': GalleryPage, - r'http://[^/]+/s/.*': ImagePage, - r'http://[^/]+/home\.php': HomePage, - r'http://e-hentai\.org/bounce_login\.php': LoginPage, - } - - def __init__(self, domain, username, password, *args, **kwargs): - self.DOMAIN = domain - self.logged = False - Browser.__init__(self, parser=('lxmlsoup',), *args, **kwargs) - if password: - self.login(username, password) - - def _gallery_url(self, gallery): - return 'http://%s/g/%s/' % (self.DOMAIN, gallery.id) - - def _gallery_page(self, gallery, n): - return gallery.url + ('?p='+str(n)) - - def search_galleries(self, pattern): - self.location(self.buildurl('/', f_search=pattern.encode('utf-8'))) - assert self.is_on_page(IndexPage) - return self.page.iter_galleries() - - def latest_gallery(self): - self.location('/') - assert self.is_on_page(IndexPage) - return self.page.iter_galleries() - - def iter_gallery_images(self, gallery): - self.location(gallery.url) - assert self.is_on_page(GalleryPage) - for n in self.page._page_numbers(): - self.location(self._gallery_page(gallery, n)) - assert self.is_on_page(GalleryPage) - - for img in self.page.image_pages(): - yield EHentaiImage(img) - - def get_image_url(self, image): - self.location(image.id) - assert self.is_on_page(ImagePage) - return self.page.get_url() - - def gallery_exists(self, gallery): - gallery.url = self._gallery_url(gallery) - self.location(gallery.url) - assert self.is_on_page(GalleryPage) - return self.page.gallery_exists(gallery) - - def fill_gallery(self, gallery, fields): - gallery.url = self._gallery_url(gallery) - self.location(gallery.url) - assert self.is_on_page(GalleryPage) - self.page.fill_gallery(gallery) - - def login(self, username, password): - assert isinstance(username, basestring) - assert isinstance(password, basestring) - - data = {'ipb_login_username': username, - 'ipb_login_password': password} - self.location('http://e-hentai.org/bounce_login.php', urlencode(data), no_login=True) - - assert self.is_on_page(LoginPage) - if not self.page.is_logged(): - raise BrowserIncorrectPassword() - - # necessary in order to reach the fjords - self.home() diff --git a/modules/ehentai/favicon.png b/modules/ehentai/favicon.png deleted file mode 100644 index 4022e0080fd3838a587d7fc7c2c409c826a191da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 917 zcmV;G18V$Px#24YJ`L;(K){{a7>y{D4^000SaNLh0L01FcU01FcV0GgZ_00007bV*G`2ipY! z6D%RGNyvZz000?uMObu0Z*6U5Zgc=ca%Ew3Wn>_CX>@2HM@dakSAh-}0008;NkljjlZOl5(&b&ViP1So{)$dK! zt5;P7K@bE%5ClOG1VJ!vOyr&HDLjiS2hiWPqO4}?gn1`hkB_hets1p?JX4g_!J0N6 z#|zk8^Zt2!j$K7r&9|^;K*l#=EAHrHykyw!KJwjtjF;Npfk|lNLF_`-rF0-Y-$)+6 zP!2TIJBqt;8@6Cys(-#q=QKX()5fE?4_DRL^8*eSWpxB5YKGB5lOfxbf74#x$+qGv zD9UQ$zv{m=?m;jd5Do|j41|;T6LZ7nfcAj;*@KIUvif}_9gugjxA1LIR{ML&i?W&- z)+x|l1>>*7Blr#b`{-@LcHEk(j^P9@#^pGRQ#ezU)xugD0hgxdh86U-<4H`V@-z

6*+0zrkPW*@Sh=R*nf=wq{2_QC4U1F!nZlWg-1WXD?8c)q4#B(L*WmnNDf*#w0qA z@5Uhpyoi@lmiI1x#5sJ4cSe0Xyp-iF_4zuESUib7fthh|z!K+DZt=x1#DF*w3kQS) z!U1DVJMUzZ7B3UC9`YyhPBuS?56L^(CAb&2q^iwbvgf)~PviZftWMUnb1m-o&&$zX z_RQkLHr~WtHQ%*wfPcnbyj&w&e!IL={T2CZc%Y#@Pq*+fej0+&IMd`@!QyrOCw=6f rYIdc6JRQyiK@bE%5ClOG1Y^hFii5i0+2Xwq00000NkvXXu0mjfF>0kX diff --git a/modules/ehentai/gallery.py b/modules/ehentai/gallery.py deleted file mode 100644 index 4b839288b..000000000 --- a/modules/ehentai/gallery.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from weboob.capabilities.gallery import BaseGallery, BaseImage - -__all_ = ['EHentaiGallery', 'EHentaiImage'] - - -class EHentaiGallery(BaseGallery): - def __init__(self, *args, **kwargs): - BaseGallery.__init__(self, *args, **kwargs) - - -class EHentaiImage(BaseImage): - def __init__(self, *args, **kwargs): - BaseImage.__init__(self, *args, **kwargs) diff --git a/modules/ehentai/module.py b/modules/ehentai/module.py deleted file mode 100644 index 6b390b064..000000000 --- a/modules/ehentai/module.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -import re -from weboob.capabilities.gallery import CapGallery, BaseGallery -from weboob.capabilities.collection import CapCollection, CollectionNotFound -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.misc import ratelimit -from weboob.tools.value import Value, ValueBackendPassword - -from .browser import EHentaiBrowser -from .gallery import EHentaiGallery, EHentaiImage - - -__all__ = ['EHentaiModule'] - - -class EHentaiModule(Module, CapGallery, CapCollection): - NAME = 'ehentai' - MAINTAINER = u'Roger Philibert' - EMAIL = 'roger.philibert@gmail.com' - VERSION = '1.4' - DESCRIPTION = 'E-Hentai galleries' - LICENSE = 'AGPLv3+' - BROWSER = EHentaiBrowser - CONFIG = BackendConfig( - Value('domain', label='Domain', default='g.e-hentai.org'), - Value('username', label='Username', default=''), - ValueBackendPassword('password', label='Password')) - - def create_default_browser(self): - username = self.config['username'].get() - if username: - password = self.config['password'].get() - else: - password = None - return self.create_browser(self.config['domain'].get(), username, password) - - def search_galleries(self, pattern, sortby=None): - with self.browser: - return self.browser.search_galleries(pattern) - - def iter_gallery_images(self, gallery): - self.fillobj(gallery, ('url',)) - with self.browser: - return self.browser.iter_gallery_images(gallery) - - ID_REGEXP = r'/?\d+/[\dabcdef]+/?' - URL_REGEXP = r'.+/g/(%s)' % ID_REGEXP - - def get_gallery(self, _id): - match = re.match(r'^%s$' % self.URL_REGEXP, _id) - if match: - _id = match.group(1) - else: - match = re.match(r'^%s$' % self.ID_REGEXP, _id) - if match: - _id = match.group(0) - else: - return None - - gallery = EHentaiGallery(_id) - with self.browser: - if self.browser.gallery_exists(gallery): - return gallery - else: - return None - - def fill_gallery(self, gallery, fields): - if not gallery.__iscomplete__(): - with self.browser: - self.browser.fill_gallery(gallery, fields) - - def fill_image(self, image, fields): - with self.browser: - image.url = self.browser.get_image_url(image) - if 'data' in fields: - ratelimit("ehentai_get", 2) - image.data = self.browser.readurl(image.url) - - def iter_resources(self, objs, split_path): - if BaseGallery in objs: - collection = self.get_collection(objs, split_path) - if collection.path_level == 0: - yield self.get_collection(objs, [u'latest_nsfw']) - if collection.split_path == [u'latest_nsfw']: - for gallery in self.browser.latest_gallery(): - yield gallery - - def validate_collection(self, objs, collection): - if collection.path_level == 0: - return - if BaseGallery in objs and collection.split_path == [u'latest_nsfw']: - collection.title = u'Latest E-Hentai galleries (NSFW)' - return - raise CollectionNotFound(collection.split_path) - - OBJECTS = { - EHentaiGallery: fill_gallery, - EHentaiImage: fill_image} diff --git a/modules/ehentai/pages.py b/modules/ehentai/pages.py deleted file mode 100644 index 66584f7b8..000000000 --- a/modules/ehentai/pages.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.deprecated.browser import Page -from weboob.capabilities.image import Thumbnail - -from datetime import datetime -import re - -from .gallery import EHentaiGallery - - -class LoginPage(Page): - def is_logged(self): - success_p = self.document.xpath( - '//p[text() = "Login Successful. You will be returned momentarily."]') - if len(success_p): - return True - else: - return False - - -class HomePage(Page): - pass - - -class IndexPage(Page): - def iter_galleries(self): - lines = self.document.xpath('//table[@class="itg"]//tr[@class="gtr0" or @class="gtr1"]') - for line in lines: - a = line.xpath('.//div[@class="it3"]/a')[-1] - url = a.attrib["href"] - title = a.text.strip() - yield EHentaiGallery(re.search('(?<=/g/)\d+/[\dabcdef]+', url).group(0), title=title) - - -class GalleryPage(Page): - def image_pages(self): - return self.document.xpath('//div[@class="gdtm"]//a/attribute::href') - - def _page_numbers(self): - return [n for n in self.document.xpath("(//table[@class='ptt'])[1]//td/text()") if re.match(r"\d+", n)] - - def gallery_exists(self, gallery): - if self.document.xpath("//h1"): - return True - else: - return False - - def fill_gallery(self, gallery): - gallery.title = self.document.xpath("//h1[@id='gn']/text()")[0] - cardinality_string = self.document.xpath("//div[@id='gdd']//tr[td[@class='gdt1']/text()='Length:']/td[@class='gdt2']/text()")[0] - gallery.cardinality = int(re.match(r"\d+", cardinality_string).group(0)) - date_string = self.document.xpath("//div[@id='gdd']//tr[td[@class='gdt1']/text()='Posted:']/td[@class='gdt2']/text()")[0] - gallery.date = datetime.strptime(date_string, "%Y-%m-%d %H:%M") - rating_string = self.document.xpath("//td[@id='rating_label']/text()")[0] - rating_match = re.search(r"\d+\.\d+", rating_string) - if rating_match is None: - gallery.rating = None - else: - gallery.rating = float(rating_match.group(0)) - - gallery.rating_max = 5 - - try: - thumbnail_url = self.document.xpath("//div[@class='gdtm']/a/img/attribute::src")[0] - except IndexError: - thumbnail_style = self.document.xpath("//div[@class='gdtm']/div/attribute::style")[0] - thumbnail_url = re.search(r"background:[^;]+url\((.+?)\)", thumbnail_style).group(1) - - gallery.thumbnail = Thumbnail(thumbnail_url) - - -class ImagePage(Page): - def get_url(self): - return self.document.xpath('//div[@class="sni"]/a/img/attribute::src')[0] diff --git a/modules/ehentai/test.py b/modules/ehentai/test.py deleted file mode 100644 index f6c307214..000000000 --- a/modules/ehentai/test.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest -from weboob.capabilities.gallery import BaseGallery - - -class EHentaiTest(BackendTest): - MODULE = 'ehentai' - - def test_search(self): - l = list(self.backend.search_galleries('lol')) - self.assertTrue(len(l) > 0) - v = l[0] - self.backend.fillobj(v, ('url',)) - self.assertTrue(v.url and v.url.startswith('http://'), 'URL for gallery "%s" not found: %s' % (v.id, v.url)) - self.backend.browser.openurl(v.url) - - img = self.backend.iter_gallery_images(v).next() - self.backend.fillobj(img, ('url',)) - self.assertTrue(v.url and v.url.startswith('http://'), 'URL for first image in gallery "%s" not found: %s' % (v.id, img.url)) - self.backend.browser.openurl(img.url) - - def test_latest(self): - l = list(self.backend.iter_resources([BaseGallery], [u'latest_nsfw'])) - assert len(l) > 0 diff --git a/modules/geolocip/__init__.py b/modules/geolocip/__init__.py deleted file mode 100644 index c3ae46d45..000000000 --- a/modules/geolocip/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .module import GeolocIpModule - -__all__ = ['GeolocIpModule'] diff --git a/modules/geolocip/favicon.png b/modules/geolocip/favicon.png deleted file mode 100644 index 37a05af29048596edc25dca9cb8cd8c826d799c0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3850 zcmV+l5B2bgP)Px#24YJ`L;(K){{a7>y{D4^000SaNLh0L01FcU01FcV0GgZ_00007bV*G`2igl8 z3pO=eL(b6v01k>tL_t(|+U;9?j2z{8|Gn?LGdsI`yX@T=9Ag_C99uvm8Zg9mC{Ros z2Lh%76bBTbX&@>JNm{jO8#Qf%P}3rX5R$01`WHl1BPtXG4kp+I8E`Nsmy|%@jBQ-Q z!EwGj-|gP+%scPAU;Sfe#&c&MzIA+uPt7BZ_G@-$p5OERvB3XxB6D$(L$AH&`MzH% zxb8Aa=}O0OF0j^~ZLO_ot)Yz>)yBN1wSHA=Js_pr*KD;~D=)n?n3n)F_U&6%^8ICu zu^Ry_F~(d>DJ_B03aoVj0M^171EmzCQczk$DFtK9goxvRZ?#&_aUq_(X#M)&lm=kn zSHJRV)#|cRvAB{_dL4k}aGbRugaYtkEub~5wUfT5wT9LjQYpkzLTe3U45U&JLLlJ$ z?Pja>SDf=F*KgVsoN@pRzVb@Br&e22C>A%tb#DStgw}0_tpx}HjDbz0thI*H8pc}K zMEW|BdMqW3F_}gPDG@~xLe9smwU5_MJMH;ewfYKU?BLRiE~p>F&N3g~FeK(#5dWfiZxTkU}7qvTe^P0Li1tYrzx%(jzqy zn*c#6h39$jJrADefiaf;j#5hXD`V~lu>FE7uN<5c02=%Dt*CZ)KLFQV3u}8)##J06 z5&|NQA*DntWn0?DKw1h72ilUh&C3Wz3TP$T=800<%3;khn6&x6Mp7-QLa#u(`K z5oxWTl2YEc`kHI{W)*l4uD80#KwUrFG5`Y2((*-1z-)1iU3F5L@)}(0qDda zNZbO|91bC13G*p1>2m;?>F_8*5QhKFqv-A%zx?IClM2A${{9s`y}b{BvCpNGEA5CW|* zd@|(xE4SW#cSk?uKPs*MM#Mt48hys2~iXwh&4hK5LT8#^sIz( zd>CU-+#^v3aEq{A3t=h?p^H7RI)*g{R>d%h4=}kfngVUCU9Oc{@JCl)z5B7PTbnbz zB%AJo?pkey>$+cu);(4!@F+TRa1O34MDcWl<;Bo$7mT$4>`4mF1SnL2LKV7LgAq;8 z;eHU&ghNbb25H0q0M~IG-}g6#Vb}*?^KYdA2m1QF-dgQe#@O}NT9&Fjk0L}-1gYl3WdU2LdX(hOgpSenT0wjC4^IgW?ge%yE-1s!rz1KS~&eFjzc}q zyVV%8=7a$7Jg;Jm*`&1YQd%RHGGkmS)qeD9u#|l`1w3_O7Oa3}{xMufJ4!IeVV>)5 z`tG;B<#h}|(f6wWR-_r1OftJTVO~%Vx_tVF*ufyM#ctTHGmc_Drv;?oc{fR=mUSEj z7-OHX)-Kmc{+^nEQVN24P`&2^u0NxD{mi*X@ZGk@O#9g+4J;IK=2^?J_Nwdg`8&Re za#!`3An4u|Agy5#6;1XBX(q{7*`f3T0DC*q0AtLetZ+#Rn^g6c*9}wZomBzHg5%)6 z-{7a4{}Erk;bPRsMvi@Ap)yGevTQLi14^mqdERY*dDmTC9RZ-VUYDksG6#Ux8v3-Q zz%1DQwmk;h_82_%y2Otk-Hmh3{Y1Ox@Xmfb{NUeDYYMt&IV|%t^-rS#Nz!QF_dn)1 zPNgFNq*7I#tJ~AgjW!0hP?=pP?>Jcb@m2Wi`yS5T+x6_0>Az5@!uqwNnW7Yqd&6ndd~RZh*0i5O8TiEV2r`{{aZSU0>+q%);eokW*wXEGiRM~ ztu+qp`8DqU+dH!N)?9h*ar1&>q{M9vU<{N}zas#wwWTzyl`Vin6RyE?zQq7 zkO3e=%mAwPMNN;PK=fVIO@`)z~}P*I0%jh=cUkHZgSK9&+fh_O!cfL%(dJM-nG zwHCx$u=@W(R~McX=XSCjAFaQ06aZNnuQjBU@5fTkTmy_T16pgDPl7UaBhldfNf-Svi8kJytom9bRw^6O@hA#fLGWZp0DS7Y z>jGoULs1mH7IKa_jwg+)i3~wU-h|ci!)1PCFGx5tiFuhYpKId}x5DsXM*t+b^d2t6 z_KXTS4%25Cqu7=SbMDo7kmVm^vvaLy5i zA)2k0ZA?rY7#SPee%vO^bRRsnb!+pQb?e?T)?V#UT6Gu$O_HLvBowG!?M5ke zW{G(Nh$ftoe$erQATpf9JQo6-bF_j0%^*O%(HI^a8v679eBz0nCky}p{CxZN_da*k zRSgibficD$$C;d38U+#upv^&$)(D6eux_zE&N>+qw;F_x!#VUioYB3Y;b9P?CiM@Z zI7S$T8S~@y`r+Zx(FX<(9eVVwx8IIBZi`Q)QmOQm)_Q9cMb{BRnBzD{M%782_hpPYJl=FAPzr}aceE4lxPM4Mh_px(8x$UI5hMRZ@>HQ=BIb; znDHjrjCX{-f6qM^T5JEg==+~8`aUgq9$eQ2V=P-~$peuuP4j8>$*lTP0ES0L^}&IG zNB8XC|EEtp{d7lbbTjr^Pw&_~3c*k@Agq+i6t7VofTJ*^4Z@wvK4FCXmp|4L~wtDqj%^-Lo48u_o$DUGZiByWH zWaU+B4V@pW<@SN|OsA=w{n*rak6@6fh-K?7p3B4t5j>f*z>%h^;cZ+#&f%N z$&>B_jzxFfetQ|fLdMvQrDE~2a;fy$qVKz|>%wI$^9kh0z%myii*Q{x`;9aMjN`aH4`d83W71`8tLwV|^oKXx&^IRlOr>krulKCA6++0eg6FL) z`u=%-p|FfGRwaZiu-1Bnkh2IOvDR8ErT!-pVz3p4kG5K^gU{^TdGNB;s~0U^wCLgG zXP^D)vzII(J>A_X6^m`nN;D|V1Oa5DutejOF{Yn$e&=UzxutK;0XUiTH(&eOi)Srf zymmoP56Y!d20*_1n@XSMwn`%*5ORLp1xk9V1JG3}-J;t6BAK*po?np41CnY6063Ih z4B+p6@}nP}dD5c5T+_3^{AK<64IADAu#q_qOZ~{ye1j0uUd^4-($~g(R0yFrZrITO z_%D9ZoOc=k0H;-}Pf|)ZM?%CRjv>;8t#orG3AvL-d8uhQj>{b9qBxG1%}WyifSd2S zD**6saU8$c3Ic>-cw|#O%`tKy{&& zpyypCcs#oQJKynwFkEGf`Haihb&lh#Qc7XG(RjJuXlxl78{0NIHumz1yLZR)G8LW) z-S_Qpd!u7xmBz$G509b>k0LZCCYsG47?V;qUw-w~7{8<6(J|@&03^iQXG;f9r~m)} M07*qoM6N<$f}J^5c>n+a diff --git a/modules/geolocip/module.py b/modules/geolocip/module.py deleted file mode 100644 index 32ac7caea..000000000 --- a/modules/geolocip/module.py +++ /dev/null @@ -1,72 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Julien Veyssier -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.capabilities.geolocip import CapGeolocIp, IpLocation -from weboob.tools.backend import Module -from weboob.deprecated.browser import Browser, BrowserUnavailable - - -__all__ = ['GeolocIpModule'] - - -class GeolocIpModule(Module, CapGeolocIp): - NAME = 'geolocip' - MAINTAINER = u'Julien Veyssier' - EMAIL = 'julien.veyssier@aiur.fr' - VERSION = '1.4' - LICENSE = 'AGPLv3+' - DESCRIPTION = u"GeolocIP IP addresses geolocation service" - BROWSER = Browser - - def get_location(self, ipaddr): - with self.browser: - - content = self.browser.readurl('http://www.geolocip.com/?s[ip]=%s&commit=locate+IP!' % str(ipaddr)) - - if content is None: - raise BrowserUnavailable() - - tab = {} - last_line = '' - line = '' - for line in content.split('\n'): - if len(line.split('

')) > 1: - key = last_line.split('
')[1].split('
')[0][0:-2] - value = line.split('
')[1].split('
')[0] - tab[key] = value - last_line = line - iploc = IpLocation(ipaddr) - iploc.city = u'%s'%tab['City'] - iploc.region = u'%s'%tab['Region'] - iploc.zipcode = u'%s'%tab['Postal code'] - iploc.country = u'%s'%tab['Country name'] - if tab['Latitude'] != '': - iploc.lt = float(tab['Latitude']) - else: - iploc.lt = 0.0 - if tab['Longitude'] != '': - iploc.lg = float(tab['Longitude']) - else: - iploc.lg = 0.0 - #iploc.host = 'NA' - #iploc.tld = 'NA' - #iploc.isp = 'NA' - - return iploc diff --git a/modules/geolocip/test.py b/modules/geolocip/test.py deleted file mode 100644 index b62dd31e9..000000000 --- a/modules/geolocip/test.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Julien Veyssier -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest - - -class GeolocIPTest(BackendTest): - MODULE = 'geolocip' - - def test_geolocip(self): - self.backend.get_location('88.198.11.130') diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index 5bd5b3807..4281ae109 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -229,7 +229,9 @@ class LCLBrowser(LoginBrowser, StatesMixin): continue self.location('/outil/UWRI/Accueil/') - if self.page.has_iban_choice(): + if self.no_perm.is_here(): + self.logger.warning('RIB is unavailable.') + elif self.page.has_iban_choice(): self.rib.go(data={'compte': '%s/%s/%s' % (a.id[0:5], a.id[5:11], a.id[11:])}) if self.rib.is_here(): iban = self.page.get_iban() diff --git a/modules/canaltp/__init__.py b/modules/meslieuxparis/__init__.py similarity index 82% rename from modules/canaltp/__init__.py rename to modules/meslieuxparis/__init__.py index ebee8eec3..a7d12f5a6 100644 --- a/modules/canaltp/__init__.py +++ b/modules/meslieuxparis/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2011 Romain Bignon +# Copyright(C) 2018 Vincent A # # This file is part of weboob. # @@ -17,7 +17,10 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from .module import CanalTPModule -__all__ = ['CanalTPModule'] +from .module import MeslieuxparisModule + + +__all__ = ['MeslieuxparisModule'] diff --git a/modules/meslieuxparis/browser.py b/modules/meslieuxparis/browser.py new file mode 100644 index 000000000..d37457577 --- /dev/null +++ b/modules/meslieuxparis/browser.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser import PagesBrowser, URL + +from .pages import ListPage + + +class MeslieuxparisBrowser(PagesBrowser): + BASEURL = 'https://meslieux.paris.fr' + + list = URL(r'/proxy/data/get/equipements/get_equipements\?m_tid=(?P\d+)&limit=5000&order=name%20ASC&lat=48.8742&lon=2.38', ListPage) + search = URL(r'/proxy/data/get/equipements/search_equipement\?cid=(?P[\d,]+)&limit=100', ListPage) + + # all categories can be found at https://meslieux.paris.fr/proxy/data/get/equipements/get_categories_equipement?id=all&type_name=search + + PARKS = [7, 14, 65, 91] + POOLS = [27, 29] + MARKETS = [289, 300] + MUSEUMS = [67] + HALLS = [100] + SCHOOLS = [41, 43] + + ALL = [2, 5, 6, 7, 9, 14, 16, 17, 26, 27, 28, 30, 32, 36, 37, 39, 40, 41, 43, + 46, 47, 60, 62, 64, 65, 67, 70, 71, 76, 80, 82, 84, 85, 87, 91, 100, + 175, 177, 181, 235, 253, 267, 280, 287, 289, 290, 293, 300, 303, + ] + + def search_contacts(self, pattern): + ids = ','.join(str(id) for id in self.ALL) + self.search.go(cid=ids, params={'keyword': pattern}) + for res in self.page.iter_contacts(): + yield res + diff --git a/modules/voyagessncf/test.py b/modules/meslieuxparis/module.py similarity index 50% rename from modules/voyagessncf/test.py rename to modules/meslieuxparis/module.py index b061dd781..e71ebecc0 100644 --- a/modules/voyagessncf/test.py +++ b/modules/meslieuxparis/module.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2013 Romain Bignon +# Copyright(C) 2018 Vincent A # # This file is part of weboob. # @@ -17,21 +17,29 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from weboob.tools.test import BackendTest +from weboob.tools.backend import Module +from weboob.capabilities.contact import CapDirectory -class VoyagesSNCFTest(BackendTest): - MODULE = 'voyagessncf' +from .browser import MeslieuxparisBrowser - def test_stations(self): - stations = list(self.backend.iter_station_search('paris')) - self.assertTrue(len(stations) > 0) - self.assertTrue('Paris Massy' in stations[-1].name) - def test_departures(self): - departure = list(self.backend.iter_station_search('paris'))[0] - arrival = list(self.backend.iter_station_search('lyon'))[0] +__all__ = ['MeslieuxparisModule'] - prices = list(self.backend.iter_station_departures(departure.id, arrival.id)) - self.assertTrue(len(prices) > 0) + +class MeslieuxparisModule(Module, CapDirectory): + NAME = 'meslieuxparis' + DESCRIPTION = 'MesLieux public Paris places' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + + BROWSER = MeslieuxparisBrowser + + def search_contacts(self, query, sortby): + if query.city and query.city.lower() != 'paris': + return [] + return self.browser.search_contacts(query.name.lower()) diff --git a/modules/meslieuxparis/pages.py b/modules/meslieuxparis/pages.py new file mode 100644 index 000000000..0f2bb445b --- /dev/null +++ b/modules/meslieuxparis/pages.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from datetime import time, date + +from dateutil import rrule +from weboob.browser.elements import method, ItemElement, DictElement +from weboob.browser.filters.standard import CleanText, Regexp +from weboob.browser.filters.json import Dict +from weboob.browser.pages import JsonPage +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.contact import Place, OpeningRule, OpeningHours + + +def parsetime(s): + return time(*map(int, s.split(':'))) + +def parsedate(s): + return date(*map(int, s.split('-'))) + + +class ListPage(JsonPage): + def build_doc(self, content): + content = content.strip() + return super(ListPage, self).build_doc(content) + + @method + class iter_contacts(DictElement): + def find_elements(self): + return self.el + + def condition(self): + return 'ERR' not in self.el + + class item(ItemElement): + klass = Place + + obj_id = Dict('idequipement') + obj_name = Dict('name') + obj_address = Dict('details/address') + obj_postcode = Dict('details/zip_code') + obj_city = Dict('details/city') + obj_country = 'FR' + obj_phone = Regexp(CleanText(Dict('details/phone'), replace=[(' ', '')]), r'^0(.*)$', r'+33\1', default=None) + + def obj_opening(self): + if self.el['calendars'] == []: + # yes, sometimes it's a list + return NotAvailable + + if self.el['calendars'].get('everyday'): + rule = OpeningRule() + rule.dates = rrule.rrule(rrule.DAILY) + rule.times = [(time(0, 0), time(23, 59, 59))] + rule.is_open = True + + res = OpeningHours() + res.rules = [rule] + return res + + rules = [] + for day, hours in self.el['calendars'].items(): + rule = OpeningRule() + rule.is_open = True + + day = parsedate(day) + rule.dates = rrule.rrule(rrule.DAILY, count=1, dtstart=day) + rule.times = [(parsetime(t[0]), parsetime(t[1])) for t in hours if t[0] != 'closed'] + rule.is_open = True + + if rule.times: + rules.append(rule) + + res = OpeningHours() + res.rules = rules + return res diff --git a/modules/meslieuxparis/test.py b/modules/meslieuxparis/test.py new file mode 100644 index 000000000..527b02481 --- /dev/null +++ b/modules/meslieuxparis/test.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.capabilities.contact import SearchQuery +from weboob.tools.test import BackendTest + + +class MeslieuxparisTest(BackendTest): + MODULE = 'meslieuxparis' + + def test_search(self): + q = SearchQuery() + q.name = 'champ-de-mars' # site has no result for "champ de mars"... + + res = list(self.backend.search_contacts(q, None)) + self.assertEqual(len(res), 1) + self.assertEqual(res[0].name, 'Parc du Champ-de-Mars') + self.assertEqual(res[0].city, 'Paris') + self.assertEqual(res[0].postcode, '75007') + self.assertEqual(res[0].country, 'FR') + self.assertEqual(res[0].address, '2 allée Adrienne-Lecouvreur') + self.assertTrue(res[0].opening.is_open_now) + + def test_not(self): + q = SearchQuery() + q.name = 'champ de mars' + q.city = 'marseille' + + res = list(self.backend.search_contacts(q, None)) + self.assertFalse(res) diff --git a/modules/alloresto/__init__.py b/modules/pagesjaunes/__init__.py similarity index 82% rename from modules/alloresto/__init__.py rename to modules/pagesjaunes/__init__.py index 712ff08d2..cc0002423 100644 --- a/modules/alloresto/__init__.py +++ b/modules/pagesjaunes/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2014 Romain Bignon +# Copyright(C) 2018 Vincent A # # This file is part of weboob. # @@ -17,7 +17,10 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from .module import AlloRestoModule -__all__ = ['AlloRestoModule'] +from .module import PagesjaunesModule + + +__all__ = ['PagesjaunesModule'] diff --git a/modules/pagesjaunes/browser.py b/modules/pagesjaunes/browser.py new file mode 100644 index 000000000..30b8f0113 --- /dev/null +++ b/modules/pagesjaunes/browser.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.browser import PagesBrowser, URL +from weboob.capabilities.contact import OpeningHours + +from .pages import ResultsPage, PlacePage + + +class PagesjaunesBrowser(PagesBrowser): + BASEURL = 'https://www.pagesjaunes.fr' + + search = URL('/recherche/(?P[a-z0-9-]+)/(?P[a-z0-9-]+)', ResultsPage) + company = URL('/pros/\d+', PlacePage) + + def simplify(self, name): + return re.sub(r'[^a-z0-9-]+', '-', name.lower()) + + def search_contacts(self, query): + assert query.name + assert query.city + + self.search.go(city=self.simplify(query.city), pattern=self.simplify(query.name)) + return self.page.iter_contacts() + + def fill_hours(self, contact): + self.location(contact.url) + contact.opening = OpeningHours() + contact.opening.rules = list(self.page.iter_hours()) + diff --git a/modules/pagesjaunes/module.py b/modules/pagesjaunes/module.py new file mode 100644 index 000000000..d733d0b39 --- /dev/null +++ b/modules/pagesjaunes/module.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.backend import Module +from weboob.capabilities.contact import CapDirectory, Place + +from .browser import PagesjaunesBrowser + + +__all__ = ['PagesjaunesModule'] + + +class PagesjaunesModule(Module, CapDirectory): + NAME = 'pagesjaunes' + DESCRIPTION = 'Pages Jaunes' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + + BROWSER = PagesjaunesBrowser + + def search_contacts(self, query, sortby): + return self.browser.search_contacts(query) + + def fill_contact(self, obj, fields): + if 'opening' in fields: + self.browser.fill_hours(obj) + + OBJECTS = { + Place: fill_contact, + } + diff --git a/modules/pagesjaunes/pages.py b/modules/pagesjaunes/pages.py new file mode 100644 index 000000000..ab85b037e --- /dev/null +++ b/modules/pagesjaunes/pages.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from datetime import time +import re + +from dateutil import rrule +from weboob.browser.elements import method, ListElement, ItemElement +from weboob.browser.filters.standard import CleanText, Regexp +from weboob.browser.filters.html import AbsoluteLink, HasElement +from weboob.browser.pages import HTMLPage +from weboob.capabilities.base import NotLoaded, NotAvailable +from weboob.capabilities.contact import Place, OpeningRule + + +class ResultsPage(HTMLPage): + @method + class iter_contacts(ListElement): + item_xpath = '//section[@id="listResults"]/article' + + class item(ItemElement): + klass = Place + + obj_name = CleanText('.//a[has-class("denomination-links")]') + obj_address = CleanText('.//a[has-class("adresse")]') + obj_phone = Regexp(CleanText('.//strong[@class="num"]', replace=[(' ', '')]), r'^0(\d{9})$', r'+33\1') + obj_url = AbsoluteLink('.//a[has-class("denomination-links")]') + obj_opening = HasElement('.//span[text()="Horaires"]', NotLoaded, NotAvailable) + + +class PlacePage(HTMLPage): + @method + class iter_hours(ListElement): + item_xpath = '//ul[@class="liste-horaires-principaux"]/li[@class="horaire-ouvert"]' + + class item(ItemElement): + klass = OpeningRule + + def obj_dates(self): + wday = CleanText('./span')(self) + wday = ['lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi', 'dimanche'].index(wday) + assert wday >= 0 + return rrule.rrule(rrule.DAILY, byweekday=wday) + + def obj_times(self): + times = [] + for sub in self.el.xpath('.//li[@itemprop]'): + t = CleanText('./@content')(sub) + m = re.match(r'\w{2} (\d{2}):(\d{2})-(\d{2}):(\d{2})$', t) + m = [int(x) for x in m.groups()] + times.append((time(m[0], m[1]), time(m[2], m[3]))) + return times + + obj_is_open = True diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index e2d3059af..e48c2ace3 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -114,7 +114,7 @@ class TransferJson(LoggedPage, JsonPage): transfer.label = json_response['motif'] transfer.amount = CleanDecimal.French((CleanText(Dict('montantToDisplay'))))(json_response) transfer.currency = json_response['devise'] - transfer.exec_date = Date(Dict('dateExecution'))(json_response) + transfer.exec_date = Date(Dict('dateExecution'), dayfirst=True)(json_response) transfer.account_id = Format('%s%s', Dict('codeGuichet'), Dict('numeroCompte'))(json_response['compteEmetteur']) transfer.account_iban = json_response['compteEmetteur']['iban'] diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index fe6746c9c..def1102ce 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -61,7 +61,7 @@ class SGPEBrowser(LoginBrowser): '/gae/afficherInscriptionUtilisateur.html', '/gae/afficherChangementCodeSecretExpire.html', ChangePassPage) - inscription_page = URL('/icd-web/gax/gax-inscription.html', InscriptionPage) + inscription_page = URL('/icd-web/gax/gax-inscription-utilisateur.html', InscriptionPage) def check_logged_status(self): if not self.page or self.login.is_here(): @@ -108,6 +108,10 @@ class SGPEBrowser(LoginBrowser): @need_login def get_cb_operations(self, account): self.location('/Pgn/NavigationServlet?PageID=Cartes&MenuID=%sOPF&Classeur=1&NumeroPage=1&Rib=%s&Devise=%s' % (self.MENUID, account.id, account.currency)) + + if self.inscription_page.is_here(): + raise ActionNeeded(self.page.get_error()) + for coming in self.page.get_coming_list(): if coming['date'] == 'Non definie': # this is a very recent transaction and we don't know his date yet diff --git a/modules/voyagessncf/browser.py b/modules/voyagessncf/browser.py deleted file mode 100644 index 8eb115f99..000000000 --- a/modules/voyagessncf/browser.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2013 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from random import randint - -from weboob.deprecated.browser import Browser - -from .pages import CitiesPage, SearchPage, SearchErrorPage, \ - SearchInProgressPage, ResultsPage, ForeignPage - - -__all__ = ['VoyagesSNCFBrowser'] - - -class VoyagesSNCFBrowser(Browser): - PROTOCOL = 'http' - DOMAIN = 'www.voyages-sncf.com' - ENCODING = 'utf-8' - - PAGES = { - 'http://www.voyages-sncf.com/completion/VSC/FR/fr/cityList.js': (CitiesPage, 'raw'), - 'http://www.voyages-sncf.com/billet-train': SearchPage, - 'http://www.voyages-sncf.com/billet-train\?.+': SearchErrorPage, - 'http://www.voyages-sncf.com/billet-train/recherche-en-cours.*': SearchInProgressPage, - 'http://www.voyages-sncf.com/billet-train/resultat.*': ResultsPage, - 'http://(?P\w{2})\.voyages-sncf.com/\w{2}/.*': ForeignPage, - } - - def __init__(self, *args, **kwargs): - Browser.__init__(self, *args, **kwargs) - self.addheaders += (('X-Forwarded-For', '82.228.147.%s' % randint(1,254)),) - - - def get_stations(self): - self.location('/completion/VSC/FR/fr/cityList.js') - return self.page.get_stations() - - def iter_departures(self, departure, arrival, date, age, card, comfort_class): - self.location('/billet-train') - self.page.search(departure, arrival, date, age, card, comfort_class) - - return self.page.iter_results() diff --git a/modules/voyagessncf/favicon.png b/modules/voyagessncf/favicon.png deleted file mode 100644 index 49674464f22c0ff22724a9fa77f2f31d0c4ba0f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5350 zcmVe zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{02EG1L_t(|+U=Wr>}A(=*S~A+ z$2sSodw=hFc^h7~XcdfP8V;|th@?-h2{QoR;y({qOLuW4Qz~85j0ML&Zts=?=m5mMYkngTbR8>iZ4qk#^TfKuM&?wg0)U)2s_;aGCMvuMUoO z#{$5k51nx+`h}uQN7&_?ROko+`$f1SC^U0`(t3{OjAT9|$y@|*2|)Y(5a>Eo-tgjd ze=&%+OQ!b_aGOMX3_^>Vxg=7BP+-;#0IxymyD_+vxwohGAT(FHgT3AYUb8q|C``JO zvfhtG?`7rV$h03qr6#sYPyk6)w~2Hg8>wv>Es^%Ab3K55FE(6%0G{tLIKj;wP@tO^5Vp63&uT~aSLk1)S0c65P&f6lLNx(%(rU8(z8vws?=YQgm z&M#D@e=i$*9~*O+t^i{$1K{f- zTf1dkmG$%|1l@Qe6QS^rE+`T=mdK|=DeRqB(%dx7x@R(Hc#W`$6#L)5$SZVP`Wk&^q=|%9Dn19C7AkdOZ*Y4><850h`KyL&Xl`|>9|ByXNUtVD=aD4 zWk@u;fshnrGnBK!_AzH!V~g|T_yUAy*K8&jZ}^d1XKieZ_+q_EyV;;uD_5MVO0q?9CWOU7$9(Mga!yL50IuUzw*0-(p2 z=ddxklf?XKc>iHoy;`Kj7^Q~t**JK>-H?kqgJk{15M13xr~aO=-9b>lANSSOj%4^B-^Xl}Q zvLw%!7t8Tyi-+Fu+3f2mKY%qW7*&Tbtd8lZI!INyLS;38D~uW&L1ssLzJM6dYM8w0 zkMtL2Kl|&``nP^Qy;GFP2@`~^dh`@ytDpWYTU)2`GOEACZq?D;el4mwh%K}e!Yn9a zFX;LfDV?B{ZU^BObp3bKmEY-){mEBCymza7`3rAIUi@L6-gd`97vuX8a1SC5BU1qd zQ`14WW`Or#@8Aat3;?47mof$ukGbHGERek_Y0d^|$BMqB%#Wqces!3Nljq!Rvo~Xj z_KKmx0HxcHx;(DK@+YVs-l22zN3B(>4Yc*)vh0aXbvhXAU*)l4XW+{ z1`AJUted~$s{9yC@!xk6e;4b`d`kJ4B;488)3!CnQDoZYXN2OnG2LzV_RH>))LFtDpZnJpk-4 z&s7z-H!19W3c3d=9Yszh2@%)VZiI5`~5yXacD4;L?GGpaMmd0MT=r^;_`v zQR(HGr7!=xtsizLvU^F61ycoQ=HcBuTrsbv@S4Cn4#0Q@%%(uhyUM#7oimtu#@J=d zoKj%1N;;pRHm9Z##imc#C?a-TLTeI1k^!_fS&tbdqr|q!fy(LN@mCi9V0x1* zpJ=_j&nJ637rIQOmLw8ofszBPg%X1d?_MPWy?|l{I0M{6i3Ex%WjzI^$6@TZsZSg3 z+?VyO08kJK2m(R|v1%5jlz}b+u?N^f+yHG+V5?M|t0LB2z(x_!X+VzzqD+~HnP@=7 zM-2@3YGB&nr*`$#u`ll$8of)&9@N4YR4{=BkOWEw07I>Zk^$K4Cil9R!KwgR6`BJy zuR`%8#6AJ=d4Kkc`A5w}?OuN@LUjhv4ir03Z6K17q9SL45|k|xGo~)%l25o;wAeB; z>`(_t0LLDg6n$YR8aCP6Dmi~76RCmm6CS@d!{l%&gByj4k@x{?;X$PVH61jy*ezxS zFdoWguW?7a0THUC5`nx&3FF_f?~Tc@Sb zONgDPA{hb$7I%@iE1F}KkSS{Nlr)ydi&!Ox)a5)2WP{(&6(m`U>Bw|cX#K?%Av9&Tnzpw~b zEn`uaaBRAOL$e|F^#um4pxR8pW~QL62JfjiGjvkHq!O(x0}jyrCHi6k#qt_>G6DJs ziG<{=1XqW?Dpq0ala)`8bC=JS-~MuZyBae8S2?`b=?UTEzi=L!8jxz?Dw@Biv&!rR%7d)%-Lb-#@W#H0ML|X?GVFDXH2n=FHZ+j$e)P`~N-E7Swyi{^~ z`r?ft;3~+QZi^3p|L?b_?|bM}zf$GIlC!m{)|)}H`v=MRG8*DYh^;|8N#N}X?D#xf zw*^lbqp887&uGekXfrnU_1LLq&|w9W9iRjkGZa0B<5qpwc7s`e!ALw^gs}O~f9s|6 zul^_j0KoBYJluZeV}JJ*O5?BCJZGwiA}4lrPb(&p$sJQ{6y2mlwR0A(ISbqGz_<#- z)&koWgorXE6d__`sX?e5gm!0ong#PaNf$R`9?s&{+*Do}Hu^GW-SMlBD*mVe0KiXu z=~u%SKK7SS_YPlCR}PJu+9WC@V=ne;x<(Mg1Vvl}#Z6G00>;2bk72XBT-iqyAz^Vl zqA3}|1Q4O@fI$n)?`S`4;`-b~PnA8NDMPuEPq+%rrMkeP&U2eVVlrlA-GzC%ssR4`! z$bhLt5gmLokf2DM_YQU`OiaqL7mi`u|34=4=?PW`D*j@jy6w*7>}#T2f%%iIcaN& zI?gClf=vLV2jvcGE0FO3@$P{RNzfdSHEJ;0cPv|-jcavm0{5$WH37WkbAosO8tHVV zJ7Fr{iG6&EB%f2meOGu)D~IjMp*4b>dmsfMPC;P<)La71)J_39u?i+4cYHUR&bo|V7ZmupV* zvj*|*%=Tbrx4cmgOkhG}VBBTQg$O4KC=k3wX9!z$rL(fkY|0%f{P{}t;$SxJt}6hK z{l~wqKlky6n}MlcwxUn<>W(rE%V^VZbHqF;Ef!}j>Mp`%0RhO&n3RQzb>gXLN-ED3 zT%J5IO{>3L+SC4er`2!_}0|QF;Fq6;NK;bt69a|dP@q0%?bu!6iUQ0 ziqt~gd57Zp`8J$f-rm{Rck*lR2mteT|KP7zhf=5(`!YtdK*>O51)d=`_yU;;VvGt~ zG33%hPtCj*XXmAQ?qHC|Z`heVz3<{{s|Z{9A8EMi0IteALwe#9#dC24u0``@;(7!mGj!#xwtEVtIw?nd(@eavV zduvR2kuZO>K)JBg)=P&o^Wol_l|(usRV7KWit&=^n@iMeo++B0b#&v&JE{Ct1#krj zgL+>HHyFYX2OzF?rw}2bpUu!u8+79tfcQ?C{gv{+0bNd@6GsMw0ssI207*qoM6N<$ Eg7d`!_W%F@ diff --git a/modules/voyagessncf/module.py b/modules/voyagessncf/module.py deleted file mode 100644 index 4aaaaa7cc..000000000 --- a/modules/voyagessncf/module.py +++ /dev/null @@ -1,124 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2013 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from collections import OrderedDict - -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import Value -from weboob.capabilities.travel import CapTravel, Station, Departure -from weboob.capabilities import UserError - -from .browser import VoyagesSNCFBrowser - - -__all__ = ['VoyagesSNCFModule'] - - -class VoyagesSNCFModule(Module, CapTravel): - NAME = 'voyagessncf' - DESCRIPTION = u'Voyages SNCF' - MAINTAINER = u'Romain Bignon' - EMAIL = 'romain@weboob.org' - LICENSE = 'AGPLv3+' - VERSION = '1.4' - CONFIG = BackendConfig(Value('age', label='Passenger age', default='ADULT', - choices=OrderedDict((('ADULT', '26-59 ans'), - ('SENIOR', '60 et +'), - ('YOUNG', '12-25 ans'), - ('CHILD_UNDER_FOUR', '0-3 ans'), - ('CHILDREN', '4-11 ans')))), - Value('card', label='Passenger card', default='default', - choices=OrderedDict((('default', u'Pas de carte'), - ('YOUNGS', u'Carte Jeune'), - ('ESCA', u'Carte Escapades'), - ('WEEKE', u'Carte Week-end'), - ('FQ2ND', u'Abo Fréquence 2e'), - ('FQ1ST', u'Abo Fréquence 1e'), - ('FF2ND', u'Abo Forfait 2e'), - ('FF1ST', u'Abo Forfait 1e'), - ('ACCWE', u'Accompagnant Carte Week-end'), - ('ACCCHD', u'Accompagnant Carte Enfant+'), - ('ENFAM', u'Carte Enfant Famille'), - ('FAM30', u'Carte Familles Nombreuses 30%'), - ('FAM40', u'Carte Familles Nombreuses 40%'), - ('FAM50', u'Carte Familles Nombreuses 50%'), - ('FAM75', u'Carte Familles Nombreuses 75%'), - ('MI2ND', u'Carte Militaire 2e'), - ('MI1ST', u'Carte Militaire 1e'), - ('MIFAM', u'Carte Famille Militaire'), - ('THBIZ', u'Thalys ThePass Business'), - ('THPREM', u'Thalys ThePass Premium'), - ('THWE', u'Thalys ThePass Weekend')))), - Value('class', label='Comfort class', default='2', - choices=OrderedDict((('1', u'1e classe'), - ('2', u'2e classe'))))) - - BROWSER = VoyagesSNCFBrowser - STATIONS = [] - - def _populate_stations(self): - if len(self.STATIONS) == 0: - with self.browser: - self.STATIONS = self.browser.get_stations() - - def iter_station_search(self, pattern): - self._populate_stations() - - pattern = pattern.lower() - already = set() - - # First stations whose name starts with pattern... - for _id, name in enumerate(self.STATIONS): - if name.lower().startswith(pattern): - already.add(_id) - yield Station(_id, unicode(name)) - # ...then ones whose name contains pattern. - for _id, name in enumerate(self.STATIONS): - if pattern in name.lower() and _id not in already: - yield Station(_id, unicode(name)) - - def iter_station_departures(self, station_id, arrival_id=None, date=None): - self._populate_stations() - - if arrival_id is None: - raise UserError('The arrival station is required') - - try: - station = self.STATIONS[int(station_id)] - arrival = self.STATIONS[int(arrival_id)] - except (IndexError, ValueError): - try: - station = list(self.iter_station_search(station_id))[0].name - arrival = list(self.iter_station_search(arrival_id))[0].name - except IndexError: - raise UserError('Unknown station') - - with self.browser: - for i, d in enumerate(self.browser.iter_departures(station, arrival, date, - self.config['age'].get(), - self.config['card'].get(), - self.config['class'].get())): - departure = Departure(i, d['type'], d['time']) - departure.departure_station = d['departure'] - departure.arrival_station = d['arrival'] - departure.arrival_time = d['arrival_time'] - departure.price = d['price'] - departure.currency = d['currency'] - departure.information = d['price_info'] - yield departure diff --git a/modules/voyagessncf/pages.py b/modules/voyagessncf/pages.py deleted file mode 100644 index c04075afc..000000000 --- a/modules/voyagessncf/pages.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2013 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -import re -from datetime import datetime, time, timedelta -from decimal import Decimal - -from mechanize import TextControl - -from weboob.capabilities.base import Currency, UserError -from weboob.deprecated.browser import Page -from weboob.tools.json import json - - -class ForeignPage(Page): - def on_loaded(self): - raise UserError('Your IP address is localized in a country not supported by this module (%s). Currently only the French website is supported.' % self.group_dict['country']) - - -class CitiesPage(Page): - def get_stations(self): - result = json.loads(self.document[self.document.find('{'):-2]) - return result['CITIES'] - - -class SearchPage(Page): - def search(self, departure, arrival, date, age, card, comfort_class): - self.browser.select_form(name='saisie') - self.browser['ORIGIN_CITY'] = departure.encode(self.browser.ENCODING) - self.browser['DESTINATION_CITY'] = arrival.encode(self.browser.ENCODING) - - if date is None: - date = datetime.now() + timedelta(hours=1) - elif date < datetime.now(): - raise UserError("You cannot look for older departures") - - self.browser['OUTWARD_DATE'] = date.strftime('%d/%m/%y') - self.browser['OUTWARD_TIME'] = [str(date.hour)] - self.browser['PASSENGER_1'] = [age] - self.browser['PASSENGER_1_CARD'] = [card] - self.browser['COMFORT_CLASS'] = [str(comfort_class)] - self.browser.controls.append(TextControl('text', 'nbAnimalsForTravel', {'value': ''})) - self.browser['nbAnimalsForTravel'] = '0' - self.browser.submit() - - -class SearchErrorPage(Page): - def on_loaded(self): - p = self.document.getroot().cssselect('div.messagesError p') - if len(p) > 0: - message = p[0].text.strip() - raise UserError(message) - - -class SearchInProgressPage(Page): - def on_loaded(self): - link = self.document.xpath('//a[@id="url_redirect_proposals"]')[0] - self.browser.location(link.attrib['href']) - - -class ResultsPage(Page): - def get_value(self, div, name, last=False): - i = -1 if last else 0 - p = div.cssselect(name)[i] - sub = p.find('p') - if sub is not None: - txt = sub.tail.strip() - if txt == '': - p.remove(sub) - else: - return unicode(txt) - - return unicode(self.parser.tocleanstring(p)) - - def parse_hour(self, div, name, last=False): - txt = self.get_value(div, name, last) - hour, minute = map(int, txt.split('h')) - return time(hour, minute) - - def iter_results(self): - for div in self.document.getroot().cssselect('div.train_info'): - info = None - price = None - currency = None - for td in div.cssselect('td.price'): - txt = self.parser.tocleanstring(td) - p = Decimal(re.sub('([^\d\.]+)', '', txt)) - if price is None or p < price: - info = list(div.cssselect('strong.price_label')[0].itertext())[-1].strip().strip(':') - price = p - currency = Currency.get_currency(txt) - - yield {'type': self.get_value(div, 'div.transporteur-txt'), - 'time': self.parse_hour(div, 'div.departure div.hour'), - 'departure': self.get_value(div, 'div.departure div.station'), - 'arrival': self.get_value(div, 'div.arrival div.station', last=True), - 'arrival_time': self.parse_hour(div, 'div.arrival div.hour', last=True), - 'price': price, - 'currency': currency, - 'price_info': info, - } -- GitLab