diff --git a/modules/alloresto/browser.py b/modules/alloresto/browser.py deleted file mode 100644 index d790c92a96607fd17141ecc8a9b406fc0f42869f..0000000000000000000000000000000000000000 --- a/modules/alloresto/browser.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword - -from .pages import LoginPage, AccountsPage - - -__all__ = ['AlloRestoBrowser'] - - -class AlloRestoBrowser(LoginBrowser): - BASEURL = 'https://www.alloresto.fr' - - login = URL('/identification-requise.*', LoginPage) - accounts = URL('/chez-moi/releve-compte-miams', AccountsPage) - - def do_login(self): - assert isinstance(self.username, basestring) - assert isinstance(self.password, basestring) - - self.accounts.stay_or_go() - self.page.login(self.username, self.password) - - if self.login.is_here(): - raise BrowserIncorrectPassword() - - @need_login - def get_accounts_list(self): - return self.accounts.stay_or_go().iter_accounts() - - @need_login - def get_account(self, id): - assert isinstance(id, basestring) - - for a in self.get_accounts_list(): - if a.id == id: - return a - - return None - - @need_login - def get_history(self, account): - return self.accounts.stay_or_go().get_transactions(type='consommable') - - @need_login - def get_coming(self, account): - return self.accounts.stay_or_go().get_transactions(type='acquisition') diff --git a/modules/alloresto/favicon.png b/modules/alloresto/favicon.png deleted file mode 100644 index b1b365c6dac425feb9450fa5c73a1957bb68d000..0000000000000000000000000000000000000000 Binary files a/modules/alloresto/favicon.png and /dev/null differ diff --git a/modules/alloresto/module.py b/modules/alloresto/module.py deleted file mode 100644 index cf5bdbeffa256df4aa646ee3a2029deacc495661..0000000000000000000000000000000000000000 --- a/modules/alloresto/module.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.capabilities.bank import CapBank, AccountNotFound -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword - -from .browser import AlloRestoBrowser - - -__all__ = ['AlloRestoModule'] - - -class AlloRestoModule(Module, CapBank): - NAME = 'alloresto' - MAINTAINER = u'Romain Bignon' - EMAIL = 'romain@weboob.org' - VERSION = '1.4' - DESCRIPTION = u'Allo Resto' - LICENSE = 'AGPLv3+' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Mot de passe')) - BROWSER = AlloRestoBrowser - - def create_default_browser(self): - return self.create_browser(self.config['login'].get(), - self.config['password'].get()) - - def iter_accounts(self): - return self.browser.get_accounts_list() - - def get_account(self, _id): - account = self.browser.get_account(_id) - - if account: - return account - else: - raise AccountNotFound() - - def iter_history(self, account): - return self.browser.get_history(account) - - def iter_coming(self, account): - return self.browser.get_coming(account) diff --git a/modules/alloresto/pages.py b/modules/alloresto/pages.py deleted file mode 100644 index 1e248da5b8bb65228c8877df94993504975e508b..0000000000000000000000000000000000000000 --- a/modules/alloresto/pages.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -import datetime -from decimal import Decimal - -from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.elements import ItemElement, method -from weboob.browser.filters.standard import CleanDecimal, CleanText, Filter, TableCell -from weboob.capabilities.bank import Account -from weboob.tools.capabilities.bank.transactions import FrenchTransaction as Transaction - - -class LoginPage(HTMLPage): - def login(self, username, password): - form = self.get_form(xpath='//form[has-class("form_o")]') - form['uname'] = username - form['pass'] = password - form.submit() - - -class AccountsPage(LoggedPage, HTMLPage): - @method - class iter_accounts(ItemElement): - def __call__(self): - return self - - klass = Account - - obj_id = '0' - obj_label = u'Compte miams' - obj_balance = CleanDecimal('//div[@class="compteur"]//strong', replace_dots=True) - obj_currency = u'MIAM' - obj_coming = CleanDecimal('//table[@id="solde_acquisition_lignes"]//th[@class="col_montant"]', default=Decimal('0'), replace_dots=True) - - class MyDate(Filter): - MONTHS = ['janv', u'févr', u'mars', u'avr', u'mai', u'juin', u'juil', u'août', u'sept', u'oct', u'nov', u'déc'] - - def filter(self, txt): - day, month, year = txt.split(' ') - day = int(day) - year = int(year) - month = self.MONTHS.index(month.rstrip('.')) + 1 - return datetime.date(year, month, day) - - def get_transactions(self, type='consommable'): - class get_history(Transaction.TransactionsElement): - head_xpath = '//table[@id="solde_%s_lignes"]//thead//tr/th/text()' % type - item_xpath = '//table[@id="solde_%s_lignes"]//tbody/tr' % type - - col_date = u"Date de valeur" - col_raw = u"Motif" - - class item(Transaction.TransactionElement): - obj_amount = Transaction.Amount('./td[last()]') - obj_date = AccountsPage.MyDate(CleanText(TableCell('date'))) - - return get_history(self)() diff --git a/modules/alloresto/test.py b/modules/alloresto/test.py deleted file mode 100644 index a31ea0d08b5037069c5d01b3415a3ca209699b1d..0000000000000000000000000000000000000000 --- a/modules/alloresto/test.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2014 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest - - -class AlloRestoTest(BackendTest): - MODULE = 'alloresto' - - def test_alloresto(self): - l = list(self.backend.iter_accounts()) - - a = l[0] - list(self.backend.iter_history(a)) - list(self.backend.iter_coming(a)) diff --git a/modules/voyagessncf/__init__.py b/modules/asana/__init__.py similarity index 83% rename from modules/voyagessncf/__init__.py rename to modules/asana/__init__.py index 55230f6d6108f9ec26eb176f6d9a2269721685e6..80aafb85833f15d450aa645e4fbbe6474223835e 100644 --- a/modules/voyagessncf/__init__.py +++ b/modules/asana/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2013 Romain Bignon +# Copyright(C) 2017 Vincent Ardisson # # This file is part of weboob. # @@ -17,8 +17,10 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from .module import VoyagesSNCFModule +from .module import AsanaModule -__all__ = ['VoyagesSNCFModule'] + +__all__ = ['AsanaModule'] diff --git a/modules/asana/browser.py b/modules/asana/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..d5c1ea2287525e19891307d17f169f017f0431b5 --- /dev/null +++ b/modules/asana/browser.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent Ardisson +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import time + +from weboob.browser.browsers import APIBrowser +from weboob.browser.exceptions import ClientError +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bugtracker import User, Project, Issue, Status, Update +from weboob.exceptions import BrowserIncorrectPassword +from dateutil.parser import parse as parse_date + + +class AsanaBrowser(APIBrowser): + BASEURL = 'https://app.asana.com/api/1.0/' + + STATUS_OPEN = Status(0, 'Open', Status.VALUE_NEW) + STATUS_CLOSED = Status(1, 'Closed', Status.VALUE_RESOLVED) + + def __init__(self, token, *args, **kwargs): + super(AsanaBrowser, self).__init__(*args, **kwargs) + self.token = token + self.session.headers['Authorization'] = 'Bearer %s' % token + + def open(self, *args, **kwargs): + try: + return super(AsanaBrowser, self).open(*args, **kwargs) + except ClientError as e: + if e.response.status_code == 401: + raise BrowserIncorrectPassword() + elif e.response.status_code == 429: + self.logger.warning('reached requests quota...') + waiting = int(e.response.headers['Retry-After']) + if waiting <= 60: + self.logger.warning('waiting %s seconds', waiting) + time.sleep(waiting) + return super(AsanaBrowser, self).open(*args, **kwargs) + else: + self.logger.warning('not waiting %s seconds, just fuck it', waiting) + + raise + + def _make_user(self, data): + u = User(data['id'], None) + if 'name' in data: + u.name = data['name'] + return u + + def _make_project(self, data): + p = Project(str(data['id']), data['name']) + p.url = 'https://app.asana.com/0/%s' % p.id + if 'members' in data: + p.members = [self._make_user(u) for u in data['members']] + + p.statuses = [self.STATUS_OPEN, self.STATUS_CLOSED] + p._workspace = data['workspace']['id'] + + # these fields don't exist in asana + p.priorities = [] + p.versions = [] + return p + + def _make_issue(self, data): + if data['name'].endswith(':'): + # section, not task + return None + + i = Issue(str(data['id'])) + i.url = 'https://app.asana.com/0/0/%s/f' % i.id + i.title = data['name'] + if 'notes' in data: + i.body = data['notes'] + if data.get('assignee'): + i.assignee = self._make_user(data['assignee']) + if data.get('created_by'): + # created_by is not documented + i.author = self._make_user(data['created_by']) + if 'created_at' in data: + i.creation = parse_date(data['created_at']) + if 'modified_at' in data: + i.updated = parse_date(data['modified_at']) + if 'due_at' in data: + if data['due_at']: + i.due = parse_date(data['due_at']) + else: + i.due = NotAvailable + if 'due_on' in data: + if data['due_on']: + i.due = parse_date(data['due_on']) + else: + i.due = NotAvailable + if data.get('projects'): + i.project = self._make_project(data['projects'][0]) + if 'completed' in data: + i.status = self.STATUS_CLOSED if data['completed'] else self.STATUS_OPEN + if 'custom_fields' in data: + def get(d): + for k in ('string_value', 'number_value', 'enum_value', 'text_value'): + if k in d: + return d[k] + assert False, 'custom type not handled' + i.fields = {d['name']: get(d) for d in data['custom_fields']} + if 'tags' in data: + i.tags = [d['name'] for d in data['tags']] + if data.get('memberships') and data['memberships'][0]['section']: + i.category = data['memberships'][0]['section']['name'] + + i.version = NotAvailable + i.priority = NotAvailable + return i + + def _make_update(self, data): + u = Update(str(data['id'])) + if 'created_at' in data: + u.date = parse_date(data['created_at']) + u.message = '%s: %s' % (data['type'], data['text']) + if 'created_by' in data: + u.author = self._make_user(data['created_by']) + return u + + def paginate(self, *args, **kwargs): + params = kwargs.setdefault('params', {}) + params['limit'] = 20 + reply = self.request(*args, **kwargs) + for d in reply['data']: + yield d + while reply.get('next_page') and reply['next_page'].get('uri'): + reply = self.request(reply['next_page']['uri']) + for d in reply['data']: + yield d diff --git a/modules/asana/favicon.png b/modules/asana/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..60ff9a2b9169e202a8ff626bd825c1e85ff1569c Binary files /dev/null and b/modules/asana/favicon.png differ diff --git a/modules/asana/module.py b/modules/asana/module.py new file mode 100644 index 0000000000000000000000000000000000000000..f454665eb466a1d9f39dd4299af9b799f3d63551 --- /dev/null +++ b/modules/asana/module.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent Ardisson +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.capabilities.base import empty +from weboob.capabilities.bugtracker import CapBugTracker, Project, Issue, User +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import ValueBackendPassword + +from .browser import AsanaBrowser + + +__all__ = ['AsanaModule'] + + +class AsanaModule(Module, CapBugTracker): + NAME = 'asana' + DESCRIPTION = 'Asana' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + CONFIG = BackendConfig(ValueBackendPassword('token', label='Personal access token')) + + BROWSER = AsanaBrowser + + def create_default_browser(self): + return self.create_browser(self.config['token'].get()) + + ## read-only issues and projects + def iter_issues(self, query): + query = query.copy() + + params = {} + + if query.title: + params['text'] = query.title + + if query.project: + if not isinstance(query.project, Project): + query.project = next(p for p in self.iter_projects() if query.project.lower() in p.name.lower()) + params['project'] = query.project.id + + if query.tags: + params['tags.all'] = ','.join(query.project._tagdict[tag] for tag in query.tags) + + if query.assignee: + if isinstance(query.assignee, User): + params['assignee'] = query.assignee.id + else: + params['assignee'] = query.assignee + + if query.author: + if isinstance(query.author, User): + params['created_by'] = query.author.id + else: + params['created_by'] = query.author + + if query.status: + if query.status.lower() == 'closed': + params['completed'] = 'true' + else: + params['completed'] = 'false' + params['completed_since'] = 'now' # completed=false is not enough... + + if not query.project: + workspaces = list(self._iter_workspaces()) + if len(workspaces) == 1: + params['workspace'] = workspaces[0] + + if query.project and query.assignee: + # asana's wtf api doesn't allow more than 1 filter... + del params['project'] + params['workspace'] = query.project._workspace + + opt = '?opt_fields=%s' % ','.join([ + 'name', 'completed', 'due_at', 'due_on', 'created_at', 'modified_at', + 'notes', 'custom_fields', 'tags.name', 'assignee.name', 'created_by.name', + 'projects.name', 'projects.workspace', + ]) + if params: + data = self.browser.paginate('tasks%s' % opt, params=params) + else: + data = [] + + for issue in data: + issue = self.browser._make_issue(issue) + if issue is None: # section + continue + + # post-filter because many server-side filters don't always work... + if query.title and query.title.lower() not in issue.title.lower(): + self.logger.debug('"title" filter failed on issue %r', issue) + continue + + if query.status and query.status.lower() != issue.status.name.lower(): + self.logger.debug('"completed" filter failed on issue %r', issue) + continue + + if query.tags and not (set(query.tags) <= set(issue.tags)): + self.logger.debug('"tags" filter failed on issue %r', issue) + continue + + if query.author: + if isinstance(query.author, User): + if query.author.id != issue.author.id: + continue + else: + if query.author.lower() != issue.author.name.lower(): + continue + + yield issue + + def _set_stories(self, issue): + ds = self.browser.request('tasks/%s/stories' % issue.id)['data'] + issue.history = [self.browser._make_update(d) for d in ds] + + def get_issue(self, id): + if not id.isdigit(): + return + + data = self.browser.request('tasks/%s' % id)['data'] + return self.browser._make_issue(data) + + def iter_projects(self): + for w in self._iter_workspaces(): + tags = self.browser.request('tags?workspace=%s' % w)['data'] + data = self.browser.paginate('projects?opt_fields=name,members.name,workspace&workspace=%s' % w) + for p in data: + project = self.browser._make_project(p) + self._assign_tags(tags, project) + yield project + + def _assign_tags(self, data, project): + project._tagdict = {d['name']: str(d['id']) for d in data} + project.tags = list(project._tagdict) + + def get_project(self, id): + if not id.isdigit(): + return + + data = self.browser.request('projects/%s' % id)['data'] + return self.browser._make_project(data) + + def _iter_workspaces(self): + return (d['id'] for d in self.browser.paginate('workspaces')) + + ## writing issues + def create_issue(self, project): + issue = Issue(0) + issue._project = project + return issue + + def post_issue(self, issue): + data = {} + if issue.title: + data['name'] = issue.title + if issue.body: + data['notes'] = issue.body + if issue.due: + data['due_at'] = issue.due.strftime('%Y-%m-%d') + if issue.assignee: + data['assignee'] = issue.assignee.id + + if issue.id and issue.id != '0': + data['projects'] = issue._project + self.browser.request('tasks', data=data) + if issue.tags: + self._set_tag_list(issue, True) + else: + self.browser.request('tasks/%s' % issue.id, data=data, method='PUT') + if not empty(issue.tags): + self._set_tag_list(issue) + + def _set_tag_list(self, issue, add=False): + to_remove = set() + to_add = set(issue.tags) + + if not add: + existing = set(self.get_issue(issue.id).tags) + to_add = to_add - existing + to_remove = existing - to_add + + for old in to_remove: + tagid = issue.project._tagdict[old] + self.browser.request('tasks/%s/removeTag', data={'tag': tagid}) + for new in to_add: + tagid = issue.project._tagdict[new] + self.browser.request('tasks/%s/addTag', data={'tag': tagid}) + + def update_issue(self, issue, update): + assert not update.changes, 'changes are not supported yet' + assert update.message + self.browser.request('tasks/%s/stories' % issue.id, data={'text': update.message}) + + def remove_issue(self, issue): + self.browser.request('tasks/%s' % issue.id, method='DELETE') + + ## filling + def fill_project(self, project, fields): + if set(['members']) & set(fields): + return self.get_project(project.id) + + def fill_issue(self, issue, fields): + if set(['body', 'assignee', 'due', 'creation', 'updated', 'project']) & set(fields): + new = self.get_issue(issue.id) + for f in fields: + if getattr(new, f): + setattr(issue, f, getattr(new, f)) + if 'history' in fields: + self._set_stories(issue) + + OBJECTS = { + Project: fill_project, + Issue: fill_issue, + } diff --git a/modules/asana/test.py b/modules/asana/test.py new file mode 100644 index 0000000000000000000000000000000000000000..f35db7b7eba7f9a09e4da3279d272b4281089801 --- /dev/null +++ b/modules/asana/test.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent Ardisson +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.capabilities.base import empty +from weboob.capabilities.bugtracker import Query +from weboob.tools.test import BackendTest + + +class AsanaTest(BackendTest): + MODULE = 'asana' + + def test_iter_projects(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + self.assertTrue(projects[0].statuses) + self.assertTrue(projects[0].members) + + def test_find_any_issues(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + + issues = [issue for issue, _ in zip(self.backend.iter_issues(q), range(30))] + self.assertTrue(issues) + + for issue in issues: + self.assertTrue(issue.project) + self.assertEquals(issue.project.id, projects[0].id) + self.assertTrue(issue.title) + self.assertFalse(empty(issue.body)) + self.assertTrue(issue.creation) + + self.assertTrue(issue.author, issue.title) + + def _test_find_by_criterion(self, attr, first_cb=None, matcher_cb=None): + if matcher_cb is None: + def matcher_cb(expected, actual): + self.assertEquals(expected.id, actual.id, 'different id on: %s != %s' % (expected, actual)) + + if first_cb is None: + def first_cb(obj): + return bool(obj) + + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + criterion_obj = getattr(issue, attr) + if first_cb(criterion_obj): + break + else: + assert False, 'not a single issue has this criterion' + + setattr(q, attr, criterion_obj) + + some = False + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + some = True + fetched_obj = getattr(issue, attr) + matcher_cb(criterion_obj, fetched_obj) + assert some, 'the issue searched for was not found' + + def test_find_by_assignee(self): + self._test_find_by_criterion('assignee') + + def test_find_by_author(self): + self._test_find_by_criterion('author') + + def test_find_by_title(self): + self._test_find_by_criterion( + 'title', + matcher_cb=lambda crit, actual: self.assertIn(crit.lower(), actual.lower()) + ) + + def test_find_by_tags(self): + self._test_find_by_criterion( + 'tags', + first_cb=lambda tags: bool(tags), + matcher_cb=lambda crit, actual: self.assertLessEqual(set(crit), set(actual)) + ) + + def _test_find_by_fields(self): + self._test_find_by_criterion( + 'fields', + first_cb=lambda tags: bool(tags), + matcher_cb=lambda crit, actual: self.assertLessEqual(set(crit), set(actual)) + ) + + def test_find_by_status(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + q.status = 'open' + + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + self.assertEquals(issue.status.name.lower(), 'open', issue.title) + + q.status = 'closed' + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + self.assertEquals(issue.status.name.lower(), 'closed', issue.title) + + def test_read_comments(self): + projects = list(self.backend.iter_projects()) + self.assertTrue(projects) + + q = Query() + q.project = projects[0] + + for issue, _ in zip(self.backend.iter_issues(q), range(30)): + self.backend.fillobj(issue, ['history']) + self.assertNotEmpty(issue.history) + if issue.history: + for update in issue.history: + self.assertTrue(update.author) + self.assertTrue(update.author.id) + self.assertTrue(update.author.name) + self.assertTrue(update.date) + self.assertTrue(update.message) + self.assertNotEmpty(update.changes) + + break + else: + assert 0, 'no issue had history' diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 2b783a036c3e28bf5985d147b7f461d68f859406..eae03e34b19e80f337343c3dd4ab0ae2ff93299b 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -33,7 +33,9 @@ from weboob.capabilities.bank import Account, AddRecipientStep, Recipient, Trans from weboob.capabilities.base import NotAvailable from weboob.capabilities.profile import Profile from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPError, BrowserPasswordExpired +from weboob.exceptions import ( + BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPError, BrowserPasswordExpired, ActionNeeded +) from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction from weboob.tools.capabilities.bank.investments import create_french_liquidity from weboob.tools.compat import urljoin @@ -269,7 +271,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): if data.get('authMode', '') == 'redirect': # the connection type EU could also be used as a criteria raise SiteSwitch('cenet') - typeAccount = accounts_types[0] + typeAccount = data['account'][0] if self.multi_type: assert typeAccount == self.typeAccount @@ -309,6 +311,11 @@ class CaisseEpargne(LoginBrowser, StatesMixin): # the only possible way to log in w/o nuser is on WE. if we're here no need to go further. if not self.nuser and self.typeAccount == 'WE': raise BrowserIncorrectPassword(response['error']) + + # we tested all, next iteration will throw the assertion + if self.inexttype == len(accounts_types) and 'Temporairement votre abonnement est bloqué' in response['error']: + raise ActionNeeded(response['error']) + if self.multi_type: # try to log in with the next connection type's value self.do_login() diff --git a/modules/canaltp/browser.py b/modules/canaltp/browser.py deleted file mode 100644 index 1b0228913148f38d9c7709d42baac68d7d7e7a77..0000000000000000000000000000000000000000 --- a/modules/canaltp/browser.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from datetime import datetime, date, time - -from weboob.deprecated.browser import Browser -from weboob.tools.misc import to_unicode -from weboob.deprecated.browser import BrokenPageError - - -__all__ = ['CanalTP'] - - -class CanalTP(Browser): - DOMAIN = 'widget.canaltp.fr' - - def __init__(self, **kwargs): - Browser.__init__(self, '', **kwargs) - - def iter_station_search(self, pattern): - url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/gare.php?txtrech=%s' % unicode(pattern) - result = self.openurl(url.encode('utf-8')).read() - for station in result.split('&'): - try: - _id, name = station.split('=') - except ValueError: - continue - else: - yield _id, to_unicode(name) - - def iter_station_departures(self, station_id, arrival_id=None): - url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/index.php?gare=%s' % unicode(station_id) - result = self.openurl(url.encode('utf-8')).read() - result = result - departure = '' - for line in result.split('&'): - if '=' not in line: - raise BrokenPageError('Unable to parse result: %s' % line) - key, value = line.split('=', 1) - if key == 'nomgare': - departure = value - elif key.startswith('ligne'): - _type, unknown, _time, arrival, served, late, late_reason = value.split(';', 6) - yield {'type': to_unicode(_type), - 'time': datetime.combine(date.today(), time(*[int(x) for x in _time.split(':')])), - 'departure': to_unicode(departure), - 'arrival': to_unicode(arrival).strip(), - 'late': late and time(0, int(late.split()[0])) or time(), - 'late_reason': to_unicode(late_reason).replace('\n', '').strip()} - - def home(self): - pass - - def login(self): - pass - - def is_logged(self): - """ Do not need to be logged """ - return True diff --git a/modules/canaltp/favicon.png b/modules/canaltp/favicon.png deleted file mode 100644 index 9f61523abaa67e94dd9aa0497b6419a8d83e6100..0000000000000000000000000000000000000000 Binary files a/modules/canaltp/favicon.png and /dev/null differ diff --git a/modules/canaltp/module.py b/modules/canaltp/module.py deleted file mode 100644 index 488f75a546a5e054de71e7753e4d5b0889f4c16e..0000000000000000000000000000000000000000 --- a/modules/canaltp/module.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from weboob.capabilities.travel import CapTravel, Station, Departure -from weboob.tools.backend import Module - -from .browser import CanalTP - - -__all__ = ['CanalTPModule'] - - -class CanalTPModule(Module, CapTravel): - NAME = 'canaltp' - MAINTAINER = u'Romain Bignon' - EMAIL = 'romain@weboob.org' - VERSION = '1.4' - LICENSE = 'AGPLv3+' - DESCRIPTION = "French trains" - BROWSER = CanalTP - - def iter_station_search(self, pattern): - for _id, name in self.browser.iter_station_search(pattern): - yield Station(_id, name) - - def iter_station_departures(self, station_id, arrival_id=None, date=None): - for i, d in enumerate(self.browser.iter_station_departures(station_id, arrival_id)): - departure = Departure(i, d['type'], d['time']) - departure.departure_station = d['departure'] - departure.arrival_station = d['arrival'] - departure.late = d['late'] - departure.information = d['late_reason'] - yield departure diff --git a/modules/canaltp/test.py b/modules/canaltp/test.py deleted file mode 100644 index b5d191c8b94b397287a1f6dc726346e42cc09794..0000000000000000000000000000000000000000 --- a/modules/canaltp/test.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest - - -class CanalTPTest(BackendTest): - MODULE = 'canaltp' - - def test_canaltp(self): - stations = list(self.backend.iter_station_search('defense')) - self.assertTrue(len(stations) > 0) - - list(self.backend.iter_station_departures(stations[0].id)) diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 6966ba1b22b088e551fe2f21b7338ac8faa78322..ed2a5426c84f5b9b0b82e40971e1e2e5f64fa532 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -1218,13 +1218,21 @@ class PorPage(LoggedPage, HTMLPage): class item(ItemElement): klass = Investment + def condition(self): + return not any(not x.isdigit() for x in Attr('.', 'id')(self)) + obj_label = CleanText(TableCell('label'), default=NotAvailable) - obj_code = CleanText('.//td[1]/a/@title') & Regexp(pattern=r'^([^ ]+)') obj_quantity = CleanDecimal(TableCell('quantity'), default=Decimal(0), replace_dots=True) obj_unitprice = CleanDecimal(TableCell('unitprice'), default=Decimal(0), replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) obj_diff = CleanDecimal(TableCell('diff'), default=Decimal(0), replace_dots=True) + def obj_code(self): + code = Regexp(CleanText('.//td[1]/a/@title'), r'^([^ ]+)')(self) + if 'masquer' in code: + return Regexp(CleanText('./following-sibling::tr[1]//a/@title'), r'^([^ ]+)')(self) + return code + def obj_unitvalue(self): r = CleanText(TableCell('unitvalue'))(self) if r[-1] == '%': @@ -1865,7 +1873,6 @@ class NewCardsListPage(LoggedPage, HTMLPage): obj_type = Account.TYPE_CARD obj__new_space = True obj__is_inv = False - load_details = Field('_link_id') & AsyncLoad def obj__secondpage(self): # Necessary to reach the good history page @@ -1909,8 +1916,8 @@ class NewCardsListPage(LoggedPage, HTMLPage): def parse(self, el): # We have to reach the good page with the information of the type of card - async_page = Async('details').loaded_page(self) - card_type_page = Link('//div/ul/li/a[contains(text(), "Fonctions")]')(async_page.doc) + history_page = self.page.browser.open(Field('_link_id')(self)).page + card_type_page = Link('//div/ul/li/a[contains(text(), "Fonctions")]')(history_page.doc) doc = self.page.browser.open(card_type_page).page.doc card_type_line = doc.xpath('//tbody/tr[th[contains(text(), "Débit des paiements")]]') if card_type_line: diff --git a/modules/ehentai/__init__.py b/modules/ehentai/__init__.py deleted file mode 100644 index 954af271cdcaa67bdbc7814c81f95cc24ad6976b..0000000000000000000000000000000000000000 --- a/modules/ehentai/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from .module import EHentaiModule - -__all__ = ['EHentaiModule'] diff --git a/modules/ehentai/browser.py b/modules/ehentai/browser.py deleted file mode 100644 index 79b8f569439bf1d752c2da4e1226f4a2417182c1..0000000000000000000000000000000000000000 --- a/modules/ehentai/browser.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from weboob.deprecated.browser import Browser, BrowserIncorrectPassword -from weboob.tools.compat import urlencode - -from .pages import IndexPage, GalleryPage, ImagePage, HomePage, LoginPage -from .gallery import EHentaiImage - - -__all__ = ['EHentaiBrowser'] - - -class EHentaiBrowser(Browser): - ENCODING = None - PAGES = { - r'http://[^/]+/': IndexPage, - r'http://[^/]+/\?.*': IndexPage, - r'http://[^/]+/g/.+': GalleryPage, - r'http://[^/]+/s/.*': ImagePage, - r'http://[^/]+/home\.php': HomePage, - r'http://e-hentai\.org/bounce_login\.php': LoginPage, - } - - def __init__(self, domain, username, password, *args, **kwargs): - self.DOMAIN = domain - self.logged = False - Browser.__init__(self, parser=('lxmlsoup',), *args, **kwargs) - if password: - self.login(username, password) - - def _gallery_url(self, gallery): - return 'http://%s/g/%s/' % (self.DOMAIN, gallery.id) - - def _gallery_page(self, gallery, n): - return gallery.url + ('?p='+str(n)) - - def search_galleries(self, pattern): - self.location(self.buildurl('/', f_search=pattern.encode('utf-8'))) - assert self.is_on_page(IndexPage) - return self.page.iter_galleries() - - def latest_gallery(self): - self.location('/') - assert self.is_on_page(IndexPage) - return self.page.iter_galleries() - - def iter_gallery_images(self, gallery): - self.location(gallery.url) - assert self.is_on_page(GalleryPage) - for n in self.page._page_numbers(): - self.location(self._gallery_page(gallery, n)) - assert self.is_on_page(GalleryPage) - - for img in self.page.image_pages(): - yield EHentaiImage(img) - - def get_image_url(self, image): - self.location(image.id) - assert self.is_on_page(ImagePage) - return self.page.get_url() - - def gallery_exists(self, gallery): - gallery.url = self._gallery_url(gallery) - self.location(gallery.url) - assert self.is_on_page(GalleryPage) - return self.page.gallery_exists(gallery) - - def fill_gallery(self, gallery, fields): - gallery.url = self._gallery_url(gallery) - self.location(gallery.url) - assert self.is_on_page(GalleryPage) - self.page.fill_gallery(gallery) - - def login(self, username, password): - assert isinstance(username, basestring) - assert isinstance(password, basestring) - - data = {'ipb_login_username': username, - 'ipb_login_password': password} - self.location('http://e-hentai.org/bounce_login.php', urlencode(data), no_login=True) - - assert self.is_on_page(LoginPage) - if not self.page.is_logged(): - raise BrowserIncorrectPassword() - - # necessary in order to reach the fjords - self.home() diff --git a/modules/ehentai/favicon.png b/modules/ehentai/favicon.png deleted file mode 100644 index 4022e0080fd3838a587d7fc7c2c409c826a191da..0000000000000000000000000000000000000000 Binary files a/modules/ehentai/favicon.png and /dev/null differ diff --git a/modules/ehentai/gallery.py b/modules/ehentai/gallery.py deleted file mode 100644 index 4b839288b57a7889cefa1161e527f39a93765f87..0000000000000000000000000000000000000000 --- a/modules/ehentai/gallery.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from weboob.capabilities.gallery import BaseGallery, BaseImage - -__all_ = ['EHentaiGallery', 'EHentaiImage'] - - -class EHentaiGallery(BaseGallery): - def __init__(self, *args, **kwargs): - BaseGallery.__init__(self, *args, **kwargs) - - -class EHentaiImage(BaseImage): - def __init__(self, *args, **kwargs): - BaseImage.__init__(self, *args, **kwargs) diff --git a/modules/ehentai/module.py b/modules/ehentai/module.py deleted file mode 100644 index 6b390b0648392d3f8b4de6500a07739bbcde1b66..0000000000000000000000000000000000000000 --- a/modules/ehentai/module.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -import re -from weboob.capabilities.gallery import CapGallery, BaseGallery -from weboob.capabilities.collection import CapCollection, CollectionNotFound -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.misc import ratelimit -from weboob.tools.value import Value, ValueBackendPassword - -from .browser import EHentaiBrowser -from .gallery import EHentaiGallery, EHentaiImage - - -__all__ = ['EHentaiModule'] - - -class EHentaiModule(Module, CapGallery, CapCollection): - NAME = 'ehentai' - MAINTAINER = u'Roger Philibert' - EMAIL = 'roger.philibert@gmail.com' - VERSION = '1.4' - DESCRIPTION = 'E-Hentai galleries' - LICENSE = 'AGPLv3+' - BROWSER = EHentaiBrowser - CONFIG = BackendConfig( - Value('domain', label='Domain', default='g.e-hentai.org'), - Value('username', label='Username', default=''), - ValueBackendPassword('password', label='Password')) - - def create_default_browser(self): - username = self.config['username'].get() - if username: - password = self.config['password'].get() - else: - password = None - return self.create_browser(self.config['domain'].get(), username, password) - - def search_galleries(self, pattern, sortby=None): - with self.browser: - return self.browser.search_galleries(pattern) - - def iter_gallery_images(self, gallery): - self.fillobj(gallery, ('url',)) - with self.browser: - return self.browser.iter_gallery_images(gallery) - - ID_REGEXP = r'/?\d+/[\dabcdef]+/?' - URL_REGEXP = r'.+/g/(%s)' % ID_REGEXP - - def get_gallery(self, _id): - match = re.match(r'^%s$' % self.URL_REGEXP, _id) - if match: - _id = match.group(1) - else: - match = re.match(r'^%s$' % self.ID_REGEXP, _id) - if match: - _id = match.group(0) - else: - return None - - gallery = EHentaiGallery(_id) - with self.browser: - if self.browser.gallery_exists(gallery): - return gallery - else: - return None - - def fill_gallery(self, gallery, fields): - if not gallery.__iscomplete__(): - with self.browser: - self.browser.fill_gallery(gallery, fields) - - def fill_image(self, image, fields): - with self.browser: - image.url = self.browser.get_image_url(image) - if 'data' in fields: - ratelimit("ehentai_get", 2) - image.data = self.browser.readurl(image.url) - - def iter_resources(self, objs, split_path): - if BaseGallery in objs: - collection = self.get_collection(objs, split_path) - if collection.path_level == 0: - yield self.get_collection(objs, [u'latest_nsfw']) - if collection.split_path == [u'latest_nsfw']: - for gallery in self.browser.latest_gallery(): - yield gallery - - def validate_collection(self, objs, collection): - if collection.path_level == 0: - return - if BaseGallery in objs and collection.split_path == [u'latest_nsfw']: - collection.title = u'Latest E-Hentai galleries (NSFW)' - return - raise CollectionNotFound(collection.split_path) - - OBJECTS = { - EHentaiGallery: fill_gallery, - EHentaiImage: fill_image} diff --git a/modules/ehentai/pages.py b/modules/ehentai/pages.py deleted file mode 100644 index 66584f7b82dc6420cf18836a3c4d2a292e45613a..0000000000000000000000000000000000000000 --- a/modules/ehentai/pages.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.deprecated.browser import Page -from weboob.capabilities.image import Thumbnail - -from datetime import datetime -import re - -from .gallery import EHentaiGallery - - -class LoginPage(Page): - def is_logged(self): - success_p = self.document.xpath( - '//p[text() = "Login Successful. You will be returned momentarily."]') - if len(success_p): - return True - else: - return False - - -class HomePage(Page): - pass - - -class IndexPage(Page): - def iter_galleries(self): - lines = self.document.xpath('//table[@class="itg"]//tr[@class="gtr0" or @class="gtr1"]') - for line in lines: - a = line.xpath('.//div[@class="it3"]/a')[-1] - url = a.attrib["href"] - title = a.text.strip() - yield EHentaiGallery(re.search('(?<=/g/)\d+/[\dabcdef]+', url).group(0), title=title) - - -class GalleryPage(Page): - def image_pages(self): - return self.document.xpath('//div[@class="gdtm"]//a/attribute::href') - - def _page_numbers(self): - return [n for n in self.document.xpath("(//table[@class='ptt'])[1]//td/text()") if re.match(r"\d+", n)] - - def gallery_exists(self, gallery): - if self.document.xpath("//h1"): - return True - else: - return False - - def fill_gallery(self, gallery): - gallery.title = self.document.xpath("//h1[@id='gn']/text()")[0] - cardinality_string = self.document.xpath("//div[@id='gdd']//tr[td[@class='gdt1']/text()='Length:']/td[@class='gdt2']/text()")[0] - gallery.cardinality = int(re.match(r"\d+", cardinality_string).group(0)) - date_string = self.document.xpath("//div[@id='gdd']//tr[td[@class='gdt1']/text()='Posted:']/td[@class='gdt2']/text()")[0] - gallery.date = datetime.strptime(date_string, "%Y-%m-%d %H:%M") - rating_string = self.document.xpath("//td[@id='rating_label']/text()")[0] - rating_match = re.search(r"\d+\.\d+", rating_string) - if rating_match is None: - gallery.rating = None - else: - gallery.rating = float(rating_match.group(0)) - - gallery.rating_max = 5 - - try: - thumbnail_url = self.document.xpath("//div[@class='gdtm']/a/img/attribute::src")[0] - except IndexError: - thumbnail_style = self.document.xpath("//div[@class='gdtm']/div/attribute::style")[0] - thumbnail_url = re.search(r"background:[^;]+url\((.+?)\)", thumbnail_style).group(1) - - gallery.thumbnail = Thumbnail(thumbnail_url) - - -class ImagePage(Page): - def get_url(self): - return self.document.xpath('//div[@class="sni"]/a/img/attribute::src')[0] diff --git a/modules/ehentai/test.py b/modules/ehentai/test.py deleted file mode 100644 index f6c30721481969a989c191d1a9e67e430067cb6d..0000000000000000000000000000000000000000 --- a/modules/ehentai/test.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Roger Philibert -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest -from weboob.capabilities.gallery import BaseGallery - - -class EHentaiTest(BackendTest): - MODULE = 'ehentai' - - def test_search(self): - l = list(self.backend.search_galleries('lol')) - self.assertTrue(len(l) > 0) - v = l[0] - self.backend.fillobj(v, ('url',)) - self.assertTrue(v.url and v.url.startswith('http://'), 'URL for gallery "%s" not found: %s' % (v.id, v.url)) - self.backend.browser.openurl(v.url) - - img = self.backend.iter_gallery_images(v).next() - self.backend.fillobj(img, ('url',)) - self.assertTrue(v.url and v.url.startswith('http://'), 'URL for first image in gallery "%s" not found: %s' % (v.id, img.url)) - self.backend.browser.openurl(img.url) - - def test_latest(self): - l = list(self.backend.iter_resources([BaseGallery], [u'latest_nsfw'])) - assert len(l) > 0 diff --git a/modules/geolocip/__init__.py b/modules/geolocip/__init__.py deleted file mode 100644 index c3ae46d455661a4f184ffbda36bc9b886ab5f670..0000000000000000000000000000000000000000 --- a/modules/geolocip/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .module import GeolocIpModule - -__all__ = ['GeolocIpModule'] diff --git a/modules/geolocip/favicon.png b/modules/geolocip/favicon.png deleted file mode 100644 index 37a05af29048596edc25dca9cb8cd8c826d799c0..0000000000000000000000000000000000000000 Binary files a/modules/geolocip/favicon.png and /dev/null differ diff --git a/modules/geolocip/module.py b/modules/geolocip/module.py deleted file mode 100644 index 32ac7caeadbf4e3b49bb81bc2e78fb29528992b2..0000000000000000000000000000000000000000 --- a/modules/geolocip/module.py +++ /dev/null @@ -1,72 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Julien Veyssier -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.capabilities.geolocip import CapGeolocIp, IpLocation -from weboob.tools.backend import Module -from weboob.deprecated.browser import Browser, BrowserUnavailable - - -__all__ = ['GeolocIpModule'] - - -class GeolocIpModule(Module, CapGeolocIp): - NAME = 'geolocip' - MAINTAINER = u'Julien Veyssier' - EMAIL = 'julien.veyssier@aiur.fr' - VERSION = '1.4' - LICENSE = 'AGPLv3+' - DESCRIPTION = u"GeolocIP IP addresses geolocation service" - BROWSER = Browser - - def get_location(self, ipaddr): - with self.browser: - - content = self.browser.readurl('http://www.geolocip.com/?s[ip]=%s&commit=locate+IP!' % str(ipaddr)) - - if content is None: - raise BrowserUnavailable() - - tab = {} - last_line = '' - line = '' - for line in content.split('\n'): - if len(line.split('
')) > 1: - key = last_line.split('
')[1].split('
')[0][0:-2] - value = line.split('
')[1].split('
')[0] - tab[key] = value - last_line = line - iploc = IpLocation(ipaddr) - iploc.city = u'%s'%tab['City'] - iploc.region = u'%s'%tab['Region'] - iploc.zipcode = u'%s'%tab['Postal code'] - iploc.country = u'%s'%tab['Country name'] - if tab['Latitude'] != '': - iploc.lt = float(tab['Latitude']) - else: - iploc.lt = 0.0 - if tab['Longitude'] != '': - iploc.lg = float(tab['Longitude']) - else: - iploc.lg = 0.0 - #iploc.host = 'NA' - #iploc.tld = 'NA' - #iploc.isp = 'NA' - - return iploc diff --git a/modules/geolocip/test.py b/modules/geolocip/test.py deleted file mode 100644 index b62dd31e924edad0c028055ccdea2c0c6ca0b695..0000000000000000000000000000000000000000 --- a/modules/geolocip/test.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2010-2011 Julien Veyssier -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -from weboob.tools.test import BackendTest - - -class GeolocIPTest(BackendTest): - MODULE = 'geolocip' - - def test_geolocip(self): - self.backend.get_location('88.198.11.130') diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index 5bd5b3807c3665b86223948cf77372236085c800..4281ae109024ec4326a2fe187b14d992c2d01fe2 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -229,7 +229,9 @@ class LCLBrowser(LoginBrowser, StatesMixin): continue self.location('/outil/UWRI/Accueil/') - if self.page.has_iban_choice(): + if self.no_perm.is_here(): + self.logger.warning('RIB is unavailable.') + elif self.page.has_iban_choice(): self.rib.go(data={'compte': '%s/%s/%s' % (a.id[0:5], a.id[5:11], a.id[11:])}) if self.rib.is_here(): iban = self.page.get_iban() diff --git a/modules/canaltp/__init__.py b/modules/meslieuxparis/__init__.py similarity index 82% rename from modules/canaltp/__init__.py rename to modules/meslieuxparis/__init__.py index ebee8eec31df3928771fcb94015ecb7b9c142430..a7d12f5a665d90d39c8bbeffa4ecc7268485295f 100644 --- a/modules/canaltp/__init__.py +++ b/modules/meslieuxparis/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2011 Romain Bignon +# Copyright(C) 2018 Vincent A # # This file is part of weboob. # @@ -17,7 +17,10 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from .module import CanalTPModule -__all__ = ['CanalTPModule'] +from .module import MeslieuxparisModule + + +__all__ = ['MeslieuxparisModule'] diff --git a/modules/meslieuxparis/browser.py b/modules/meslieuxparis/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..d374575773c058610f85bb0b21a557158f6f0e5e --- /dev/null +++ b/modules/meslieuxparis/browser.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser import PagesBrowser, URL + +from .pages import ListPage + + +class MeslieuxparisBrowser(PagesBrowser): + BASEURL = 'https://meslieux.paris.fr' + + list = URL(r'/proxy/data/get/equipements/get_equipements\?m_tid=(?P\d+)&limit=5000&order=name%20ASC&lat=48.8742&lon=2.38', ListPage) + search = URL(r'/proxy/data/get/equipements/search_equipement\?cid=(?P[\d,]+)&limit=100', ListPage) + + # all categories can be found at https://meslieux.paris.fr/proxy/data/get/equipements/get_categories_equipement?id=all&type_name=search + + PARKS = [7, 14, 65, 91] + POOLS = [27, 29] + MARKETS = [289, 300] + MUSEUMS = [67] + HALLS = [100] + SCHOOLS = [41, 43] + + ALL = [2, 5, 6, 7, 9, 14, 16, 17, 26, 27, 28, 30, 32, 36, 37, 39, 40, 41, 43, + 46, 47, 60, 62, 64, 65, 67, 70, 71, 76, 80, 82, 84, 85, 87, 91, 100, + 175, 177, 181, 235, 253, 267, 280, 287, 289, 290, 293, 300, 303, + ] + + def search_contacts(self, pattern): + ids = ','.join(str(id) for id in self.ALL) + self.search.go(cid=ids, params={'keyword': pattern}) + for res in self.page.iter_contacts(): + yield res + diff --git a/modules/voyagessncf/test.py b/modules/meslieuxparis/module.py similarity index 50% rename from modules/voyagessncf/test.py rename to modules/meslieuxparis/module.py index b061dd781c416e6bf96d03cd41593f4d0572bdc7..e71ebecc0cd8a24fa0b5f3dcaab162c6b3857f4c 100644 --- a/modules/voyagessncf/test.py +++ b/modules/meslieuxparis/module.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2013 Romain Bignon +# Copyright(C) 2018 Vincent A # # This file is part of weboob. # @@ -17,21 +17,29 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from weboob.tools.test import BackendTest +from weboob.tools.backend import Module +from weboob.capabilities.contact import CapDirectory -class VoyagesSNCFTest(BackendTest): - MODULE = 'voyagessncf' +from .browser import MeslieuxparisBrowser - def test_stations(self): - stations = list(self.backend.iter_station_search('paris')) - self.assertTrue(len(stations) > 0) - self.assertTrue('Paris Massy' in stations[-1].name) - def test_departures(self): - departure = list(self.backend.iter_station_search('paris'))[0] - arrival = list(self.backend.iter_station_search('lyon'))[0] +__all__ = ['MeslieuxparisModule'] - prices = list(self.backend.iter_station_departures(departure.id, arrival.id)) - self.assertTrue(len(prices) > 0) + +class MeslieuxparisModule(Module, CapDirectory): + NAME = 'meslieuxparis' + DESCRIPTION = 'MesLieux public Paris places' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + + BROWSER = MeslieuxparisBrowser + + def search_contacts(self, query, sortby): + if query.city and query.city.lower() != 'paris': + return [] + return self.browser.search_contacts(query.name.lower()) diff --git a/modules/meslieuxparis/pages.py b/modules/meslieuxparis/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..0f2bb445bbcce07142aa243ffc9eed00d5d6a8b8 --- /dev/null +++ b/modules/meslieuxparis/pages.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from datetime import time, date + +from dateutil import rrule +from weboob.browser.elements import method, ItemElement, DictElement +from weboob.browser.filters.standard import CleanText, Regexp +from weboob.browser.filters.json import Dict +from weboob.browser.pages import JsonPage +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.contact import Place, OpeningRule, OpeningHours + + +def parsetime(s): + return time(*map(int, s.split(':'))) + +def parsedate(s): + return date(*map(int, s.split('-'))) + + +class ListPage(JsonPage): + def build_doc(self, content): + content = content.strip() + return super(ListPage, self).build_doc(content) + + @method + class iter_contacts(DictElement): + def find_elements(self): + return self.el + + def condition(self): + return 'ERR' not in self.el + + class item(ItemElement): + klass = Place + + obj_id = Dict('idequipement') + obj_name = Dict('name') + obj_address = Dict('details/address') + obj_postcode = Dict('details/zip_code') + obj_city = Dict('details/city') + obj_country = 'FR' + obj_phone = Regexp(CleanText(Dict('details/phone'), replace=[(' ', '')]), r'^0(.*)$', r'+33\1', default=None) + + def obj_opening(self): + if self.el['calendars'] == []: + # yes, sometimes it's a list + return NotAvailable + + if self.el['calendars'].get('everyday'): + rule = OpeningRule() + rule.dates = rrule.rrule(rrule.DAILY) + rule.times = [(time(0, 0), time(23, 59, 59))] + rule.is_open = True + + res = OpeningHours() + res.rules = [rule] + return res + + rules = [] + for day, hours in self.el['calendars'].items(): + rule = OpeningRule() + rule.is_open = True + + day = parsedate(day) + rule.dates = rrule.rrule(rrule.DAILY, count=1, dtstart=day) + rule.times = [(parsetime(t[0]), parsetime(t[1])) for t in hours if t[0] != 'closed'] + rule.is_open = True + + if rule.times: + rules.append(rule) + + res = OpeningHours() + res.rules = rules + return res diff --git a/modules/meslieuxparis/test.py b/modules/meslieuxparis/test.py new file mode 100644 index 0000000000000000000000000000000000000000..527b02481b73b8238d03c4ea965722a014c47af7 --- /dev/null +++ b/modules/meslieuxparis/test.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.capabilities.contact import SearchQuery +from weboob.tools.test import BackendTest + + +class MeslieuxparisTest(BackendTest): + MODULE = 'meslieuxparis' + + def test_search(self): + q = SearchQuery() + q.name = 'champ-de-mars' # site has no result for "champ de mars"... + + res = list(self.backend.search_contacts(q, None)) + self.assertEqual(len(res), 1) + self.assertEqual(res[0].name, 'Parc du Champ-de-Mars') + self.assertEqual(res[0].city, 'Paris') + self.assertEqual(res[0].postcode, '75007') + self.assertEqual(res[0].country, 'FR') + self.assertEqual(res[0].address, '2 allée Adrienne-Lecouvreur') + self.assertTrue(res[0].opening.is_open_now) + + def test_not(self): + q = SearchQuery() + q.name = 'champ de mars' + q.city = 'marseille' + + res = list(self.backend.search_contacts(q, None)) + self.assertFalse(res) diff --git a/modules/alloresto/__init__.py b/modules/pagesjaunes/__init__.py similarity index 82% rename from modules/alloresto/__init__.py rename to modules/pagesjaunes/__init__.py index 712ff08d2434c607ed4917075560711e85e30413..cc0002423e0100a0781e543494e97190af3fe686 100644 --- a/modules/alloresto/__init__.py +++ b/modules/pagesjaunes/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2014 Romain Bignon +# Copyright(C) 2018 Vincent A # # This file is part of weboob. # @@ -17,7 +17,10 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from .module import AlloRestoModule -__all__ = ['AlloRestoModule'] +from .module import PagesjaunesModule + + +__all__ = ['PagesjaunesModule'] diff --git a/modules/pagesjaunes/browser.py b/modules/pagesjaunes/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..30b8f011342942dfb7cfb70a99aff756e76f4a64 --- /dev/null +++ b/modules/pagesjaunes/browser.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.browser import PagesBrowser, URL +from weboob.capabilities.contact import OpeningHours + +from .pages import ResultsPage, PlacePage + + +class PagesjaunesBrowser(PagesBrowser): + BASEURL = 'https://www.pagesjaunes.fr' + + search = URL('/recherche/(?P[a-z0-9-]+)/(?P[a-z0-9-]+)', ResultsPage) + company = URL('/pros/\d+', PlacePage) + + def simplify(self, name): + return re.sub(r'[^a-z0-9-]+', '-', name.lower()) + + def search_contacts(self, query): + assert query.name + assert query.city + + self.search.go(city=self.simplify(query.city), pattern=self.simplify(query.name)) + return self.page.iter_contacts() + + def fill_hours(self, contact): + self.location(contact.url) + contact.opening = OpeningHours() + contact.opening.rules = list(self.page.iter_hours()) + diff --git a/modules/pagesjaunes/module.py b/modules/pagesjaunes/module.py new file mode 100644 index 0000000000000000000000000000000000000000..d733d0b3927725862b003c926ec072f1e578bd27 --- /dev/null +++ b/modules/pagesjaunes/module.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.backend import Module +from weboob.capabilities.contact import CapDirectory, Place + +from .browser import PagesjaunesBrowser + + +__all__ = ['PagesjaunesModule'] + + +class PagesjaunesModule(Module, CapDirectory): + NAME = 'pagesjaunes' + DESCRIPTION = 'Pages Jaunes' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + + BROWSER = PagesjaunesBrowser + + def search_contacts(self, query, sortby): + return self.browser.search_contacts(query) + + def fill_contact(self, obj, fields): + if 'opening' in fields: + self.browser.fill_hours(obj) + + OBJECTS = { + Place: fill_contact, + } + diff --git a/modules/pagesjaunes/pages.py b/modules/pagesjaunes/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..ab85b037ee5799388ff19f3441c47b7114343e84 --- /dev/null +++ b/modules/pagesjaunes/pages.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from datetime import time +import re + +from dateutil import rrule +from weboob.browser.elements import method, ListElement, ItemElement +from weboob.browser.filters.standard import CleanText, Regexp +from weboob.browser.filters.html import AbsoluteLink, HasElement +from weboob.browser.pages import HTMLPage +from weboob.capabilities.base import NotLoaded, NotAvailable +from weboob.capabilities.contact import Place, OpeningRule + + +class ResultsPage(HTMLPage): + @method + class iter_contacts(ListElement): + item_xpath = '//section[@id="listResults"]/article' + + class item(ItemElement): + klass = Place + + obj_name = CleanText('.//a[has-class("denomination-links")]') + obj_address = CleanText('.//a[has-class("adresse")]') + obj_phone = Regexp(CleanText('.//strong[@class="num"]', replace=[(' ', '')]), r'^0(\d{9})$', r'+33\1') + obj_url = AbsoluteLink('.//a[has-class("denomination-links")]') + obj_opening = HasElement('.//span[text()="Horaires"]', NotLoaded, NotAvailable) + + +class PlacePage(HTMLPage): + @method + class iter_hours(ListElement): + item_xpath = '//ul[@class="liste-horaires-principaux"]/li[@class="horaire-ouvert"]' + + class item(ItemElement): + klass = OpeningRule + + def obj_dates(self): + wday = CleanText('./span')(self) + wday = ['lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi', 'dimanche'].index(wday) + assert wday >= 0 + return rrule.rrule(rrule.DAILY, byweekday=wday) + + def obj_times(self): + times = [] + for sub in self.el.xpath('.//li[@itemprop]'): + t = CleanText('./@content')(sub) + m = re.match(r'\w{2} (\d{2}):(\d{2})-(\d{2}):(\d{2})$', t) + m = [int(x) for x in m.groups()] + times.append((time(m[0], m[1]), time(m[2], m[3]))) + return times + + obj_is_open = True diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index e2d3059af116cdbfd05cd643262df0195f333a5f..e48c2ace3b954a73983b60faa68a5fdbccea34da 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -114,7 +114,7 @@ class TransferJson(LoggedPage, JsonPage): transfer.label = json_response['motif'] transfer.amount = CleanDecimal.French((CleanText(Dict('montantToDisplay'))))(json_response) transfer.currency = json_response['devise'] - transfer.exec_date = Date(Dict('dateExecution'))(json_response) + transfer.exec_date = Date(Dict('dateExecution'), dayfirst=True)(json_response) transfer.account_id = Format('%s%s', Dict('codeGuichet'), Dict('numeroCompte'))(json_response['compteEmetteur']) transfer.account_iban = json_response['compteEmetteur']['iban'] diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index fe6746c9c4d486676501bb6bcf8cf6f4ec6bdf94..def1102ceaa6b88affae7dcc48c7774712a98519 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -61,7 +61,7 @@ class SGPEBrowser(LoginBrowser): '/gae/afficherInscriptionUtilisateur.html', '/gae/afficherChangementCodeSecretExpire.html', ChangePassPage) - inscription_page = URL('/icd-web/gax/gax-inscription.html', InscriptionPage) + inscription_page = URL('/icd-web/gax/gax-inscription-utilisateur.html', InscriptionPage) def check_logged_status(self): if not self.page or self.login.is_here(): @@ -108,6 +108,10 @@ class SGPEBrowser(LoginBrowser): @need_login def get_cb_operations(self, account): self.location('/Pgn/NavigationServlet?PageID=Cartes&MenuID=%sOPF&Classeur=1&NumeroPage=1&Rib=%s&Devise=%s' % (self.MENUID, account.id, account.currency)) + + if self.inscription_page.is_here(): + raise ActionNeeded(self.page.get_error()) + for coming in self.page.get_coming_list(): if coming['date'] == 'Non definie': # this is a very recent transaction and we don't know his date yet diff --git a/modules/voyagessncf/browser.py b/modules/voyagessncf/browser.py deleted file mode 100644 index 8eb115f9940c26d651d50b59db4ae32bb55442ae..0000000000000000000000000000000000000000 --- a/modules/voyagessncf/browser.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2013 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from random import randint - -from weboob.deprecated.browser import Browser - -from .pages import CitiesPage, SearchPage, SearchErrorPage, \ - SearchInProgressPage, ResultsPage, ForeignPage - - -__all__ = ['VoyagesSNCFBrowser'] - - -class VoyagesSNCFBrowser(Browser): - PROTOCOL = 'http' - DOMAIN = 'www.voyages-sncf.com' - ENCODING = 'utf-8' - - PAGES = { - 'http://www.voyages-sncf.com/completion/VSC/FR/fr/cityList.js': (CitiesPage, 'raw'), - 'http://www.voyages-sncf.com/billet-train': SearchPage, - 'http://www.voyages-sncf.com/billet-train\?.+': SearchErrorPage, - 'http://www.voyages-sncf.com/billet-train/recherche-en-cours.*': SearchInProgressPage, - 'http://www.voyages-sncf.com/billet-train/resultat.*': ResultsPage, - 'http://(?P\w{2})\.voyages-sncf.com/\w{2}/.*': ForeignPage, - } - - def __init__(self, *args, **kwargs): - Browser.__init__(self, *args, **kwargs) - self.addheaders += (('X-Forwarded-For', '82.228.147.%s' % randint(1,254)),) - - - def get_stations(self): - self.location('/completion/VSC/FR/fr/cityList.js') - return self.page.get_stations() - - def iter_departures(self, departure, arrival, date, age, card, comfort_class): - self.location('/billet-train') - self.page.search(departure, arrival, date, age, card, comfort_class) - - return self.page.iter_results() diff --git a/modules/voyagessncf/favicon.png b/modules/voyagessncf/favicon.png deleted file mode 100644 index 49674464f22c0ff22724a9fa77f2f31d0c4ba0f7..0000000000000000000000000000000000000000 Binary files a/modules/voyagessncf/favicon.png and /dev/null differ diff --git a/modules/voyagessncf/module.py b/modules/voyagessncf/module.py deleted file mode 100644 index 4aaaaa7cc672d1c97660506367e96746f7772d1a..0000000000000000000000000000000000000000 --- a/modules/voyagessncf/module.py +++ /dev/null @@ -1,124 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2013 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from collections import OrderedDict - -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import Value -from weboob.capabilities.travel import CapTravel, Station, Departure -from weboob.capabilities import UserError - -from .browser import VoyagesSNCFBrowser - - -__all__ = ['VoyagesSNCFModule'] - - -class VoyagesSNCFModule(Module, CapTravel): - NAME = 'voyagessncf' - DESCRIPTION = u'Voyages SNCF' - MAINTAINER = u'Romain Bignon' - EMAIL = 'romain@weboob.org' - LICENSE = 'AGPLv3+' - VERSION = '1.4' - CONFIG = BackendConfig(Value('age', label='Passenger age', default='ADULT', - choices=OrderedDict((('ADULT', '26-59 ans'), - ('SENIOR', '60 et +'), - ('YOUNG', '12-25 ans'), - ('CHILD_UNDER_FOUR', '0-3 ans'), - ('CHILDREN', '4-11 ans')))), - Value('card', label='Passenger card', default='default', - choices=OrderedDict((('default', u'Pas de carte'), - ('YOUNGS', u'Carte Jeune'), - ('ESCA', u'Carte Escapades'), - ('WEEKE', u'Carte Week-end'), - ('FQ2ND', u'Abo Fréquence 2e'), - ('FQ1ST', u'Abo Fréquence 1e'), - ('FF2ND', u'Abo Forfait 2e'), - ('FF1ST', u'Abo Forfait 1e'), - ('ACCWE', u'Accompagnant Carte Week-end'), - ('ACCCHD', u'Accompagnant Carte Enfant+'), - ('ENFAM', u'Carte Enfant Famille'), - ('FAM30', u'Carte Familles Nombreuses 30%'), - ('FAM40', u'Carte Familles Nombreuses 40%'), - ('FAM50', u'Carte Familles Nombreuses 50%'), - ('FAM75', u'Carte Familles Nombreuses 75%'), - ('MI2ND', u'Carte Militaire 2e'), - ('MI1ST', u'Carte Militaire 1e'), - ('MIFAM', u'Carte Famille Militaire'), - ('THBIZ', u'Thalys ThePass Business'), - ('THPREM', u'Thalys ThePass Premium'), - ('THWE', u'Thalys ThePass Weekend')))), - Value('class', label='Comfort class', default='2', - choices=OrderedDict((('1', u'1e classe'), - ('2', u'2e classe'))))) - - BROWSER = VoyagesSNCFBrowser - STATIONS = [] - - def _populate_stations(self): - if len(self.STATIONS) == 0: - with self.browser: - self.STATIONS = self.browser.get_stations() - - def iter_station_search(self, pattern): - self._populate_stations() - - pattern = pattern.lower() - already = set() - - # First stations whose name starts with pattern... - for _id, name in enumerate(self.STATIONS): - if name.lower().startswith(pattern): - already.add(_id) - yield Station(_id, unicode(name)) - # ...then ones whose name contains pattern. - for _id, name in enumerate(self.STATIONS): - if pattern in name.lower() and _id not in already: - yield Station(_id, unicode(name)) - - def iter_station_departures(self, station_id, arrival_id=None, date=None): - self._populate_stations() - - if arrival_id is None: - raise UserError('The arrival station is required') - - try: - station = self.STATIONS[int(station_id)] - arrival = self.STATIONS[int(arrival_id)] - except (IndexError, ValueError): - try: - station = list(self.iter_station_search(station_id))[0].name - arrival = list(self.iter_station_search(arrival_id))[0].name - except IndexError: - raise UserError('Unknown station') - - with self.browser: - for i, d in enumerate(self.browser.iter_departures(station, arrival, date, - self.config['age'].get(), - self.config['card'].get(), - self.config['class'].get())): - departure = Departure(i, d['type'], d['time']) - departure.departure_station = d['departure'] - departure.arrival_station = d['arrival'] - departure.arrival_time = d['arrival_time'] - departure.price = d['price'] - departure.currency = d['currency'] - departure.information = d['price_info'] - yield departure diff --git a/modules/voyagessncf/pages.py b/modules/voyagessncf/pages.py deleted file mode 100644 index c04075afce57a1ab7a84e7380ce3e55ec2767eca..0000000000000000000000000000000000000000 --- a/modules/voyagessncf/pages.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2013 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - - -import re -from datetime import datetime, time, timedelta -from decimal import Decimal - -from mechanize import TextControl - -from weboob.capabilities.base import Currency, UserError -from weboob.deprecated.browser import Page -from weboob.tools.json import json - - -class ForeignPage(Page): - def on_loaded(self): - raise UserError('Your IP address is localized in a country not supported by this module (%s). Currently only the French website is supported.' % self.group_dict['country']) - - -class CitiesPage(Page): - def get_stations(self): - result = json.loads(self.document[self.document.find('{'):-2]) - return result['CITIES'] - - -class SearchPage(Page): - def search(self, departure, arrival, date, age, card, comfort_class): - self.browser.select_form(name='saisie') - self.browser['ORIGIN_CITY'] = departure.encode(self.browser.ENCODING) - self.browser['DESTINATION_CITY'] = arrival.encode(self.browser.ENCODING) - - if date is None: - date = datetime.now() + timedelta(hours=1) - elif date < datetime.now(): - raise UserError("You cannot look for older departures") - - self.browser['OUTWARD_DATE'] = date.strftime('%d/%m/%y') - self.browser['OUTWARD_TIME'] = [str(date.hour)] - self.browser['PASSENGER_1'] = [age] - self.browser['PASSENGER_1_CARD'] = [card] - self.browser['COMFORT_CLASS'] = [str(comfort_class)] - self.browser.controls.append(TextControl('text', 'nbAnimalsForTravel', {'value': ''})) - self.browser['nbAnimalsForTravel'] = '0' - self.browser.submit() - - -class SearchErrorPage(Page): - def on_loaded(self): - p = self.document.getroot().cssselect('div.messagesError p') - if len(p) > 0: - message = p[0].text.strip() - raise UserError(message) - - -class SearchInProgressPage(Page): - def on_loaded(self): - link = self.document.xpath('//a[@id="url_redirect_proposals"]')[0] - self.browser.location(link.attrib['href']) - - -class ResultsPage(Page): - def get_value(self, div, name, last=False): - i = -1 if last else 0 - p = div.cssselect(name)[i] - sub = p.find('p') - if sub is not None: - txt = sub.tail.strip() - if txt == '': - p.remove(sub) - else: - return unicode(txt) - - return unicode(self.parser.tocleanstring(p)) - - def parse_hour(self, div, name, last=False): - txt = self.get_value(div, name, last) - hour, minute = map(int, txt.split('h')) - return time(hour, minute) - - def iter_results(self): - for div in self.document.getroot().cssselect('div.train_info'): - info = None - price = None - currency = None - for td in div.cssselect('td.price'): - txt = self.parser.tocleanstring(td) - p = Decimal(re.sub('([^\d\.]+)', '', txt)) - if price is None or p < price: - info = list(div.cssselect('strong.price_label')[0].itertext())[-1].strip().strip(':') - price = p - currency = Currency.get_currency(txt) - - yield {'type': self.get_value(div, 'div.transporteur-txt'), - 'time': self.parse_hour(div, 'div.departure div.hour'), - 'departure': self.get_value(div, 'div.departure div.station'), - 'arrival': self.get_value(div, 'div.arrival div.station', last=True), - 'arrival_time': self.parse_hour(div, 'div.arrival div.hour', last=True), - 'price': price, - 'currency': currency, - 'price_info': info, - }