diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 02e08f9d1026ec7cfd3631524717bb59582003e6..6d0bbf02a39c12a5531f345e68352c9d8b1e6f5f 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -24,7 +24,7 @@ from dateutil.relativedelta import relativedelta from weboob.tools.browser2.page import HTMLPage, method, ListElement, ItemElement, SkipItem, FormNotFound, TableElement -from weboob.tools.browser2.filters import Filter, Env, CleanText, CleanDecimal, Link, TableCell +from weboob.tools.browser2.filters import Filter, Env, CleanText, CleanDecimal, Link, TableCell, Attr from weboob.tools.browser import BrowserIncorrectPassword from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account @@ -78,11 +78,11 @@ class iter_accounts(ListElement): class item(ItemElement): klass = Account - def __filter__(self, el): - if len(el.xpath('./td')) < 2: + def condition(self): + if len(self.el.xpath('./td')) < 2: return False - first_td = el.xpath('./td')[0] + first_td = self.el.xpath('./td')[0] return ((first_td.attrib.get('class', '') == 'i g' or first_td.attrib.get('class', '') == 'p g') and first_td.find('a') is not None) @@ -90,6 +90,12 @@ class Label(Filter): def filter(self, text): return text.lstrip(' 0123456789').title() + class Type(Filter): + def filter(self, label): + for pattern, actype in AccountsPage.TYPES.iteritems(): + if label.startswith(pattern): + return actype + obj_id = Env('id') obj_label = Label(CleanText('./td[1]/a')) obj_balance = CleanDecimal('./td[2] | ./td[3]') @@ -98,11 +104,7 @@ def filter(self, text): obj_currency = FrenchTransaction.Currency('./td[2] | ./td[3]') obj__link_id = Link('./td[1]/a') obj__card_links = [] - - def obj_type(self): - for pattern, actype in AccountsPage.TYPES.iteritems(): - if self.obj.label.startswith(pattern): - return actype + obj_type = Type(Attr('label')) def parse(self, el): link = el.xpath('./td[1]/a')[0].get('href', '') @@ -196,7 +198,7 @@ class get_history(Pagination, TableElement): class item(ItemElement): klass = Transaction - __filter__ = lambda el: len(el.xpath('./td')) >= 4 and len(el.xpath('./td[@class="i g" or @class="p g" or contains(@class, "_c1 c _c1")]')) > 0 + condition = lambda self: len(self.el.xpath('./td')) >= 4 and len(self.el.xpath('./td[@class="i g" or @class="p g" or contains(@class, "_c1 c _c1")]')) > 0 class OwnRaw(Filter): def __call__(self, item): @@ -209,9 +211,9 @@ def __call__(self, item): return u' '.join(parts) - obj_raw = Transaction.Raw(OwnRaw()) obj_date = Transaction.Date(TableCell('date')) obj_vdate = Transaction.Date(TableCell('vdate', 'date')) + obj_raw = Transaction.Raw(OwnRaw()) obj_amount = Transaction.Amount(TableCell('credit'), TableCell('debit')) def find_amount(self, title): @@ -238,7 +240,7 @@ class get_history(Pagination, ListElement): class item(ItemElement): klass = Transaction - __filter__ = lambda el: len(el.xpath('./td')) >= 3 + condition = lambda self: len(self.el.xpath('./td')) >= 3 obj_date = Transaction.Date('./td[1]') obj_raw = Transaction.Raw('./td[2]') @@ -255,8 +257,8 @@ class list_cards(ListElement): class item(ItemElement): def __iter__(self): card_link = self.el.get('href') - history_url = '%s/%s/fr/banque/%s' % (self.browser.BASEURL, self.browser.currentSubBank, card_link) - page = self.browser.location(history_url) + history_url = '%s/%s/fr/banque/%s' % (self.page.browser.BASEURL, self.page.browser.currentSubBank, card_link) + page = self.page.browser.location(history_url) for op in page.get_history(): yield op @@ -272,7 +274,7 @@ def parse(self, el): class item(ItemElement): klass = Transaction - __filter__ = lambda el: len(el.xpath('./td')) >= 4 + condition = lambda self: len(self.el.xpath('./td')) >= 4 obj_raw = Transaction.Raw('./td[last()-2] | ./td[last()-1]') obj_type = Transaction.TYPE_CARD diff --git a/weboob/tools/browser2/filters.py b/weboob/tools/browser2/filters.py index 9cad7579924ebfcfac354543e7ae1093c176b207..7e0aa408272459abdea23b095d6430a6b2d882b0 100644 --- a/weboob/tools/browser2/filters.py +++ b/weboob/tools/browser2/filters.py @@ -23,7 +23,15 @@ import re -class Filter(object): +class _Filter(object): + _creation_counter = 0 + + def __init__(self): + self._creation_counter = _Filter._creation_counter + _Filter._creation_counter += 1 + + +class Filter(_Filter): """ Class used to filter on a HTML element given as call parameter to return matching elements. @@ -38,6 +46,7 @@ class Filter(object): """ def __init__(self, selector=None): + super(Filter, self).__init__() self.selector = selector def __call__(self, item): @@ -56,7 +65,8 @@ def filter(self, value): """ return value -class Env(Filter): + +class Env(_Filter): """ Filter to get environment value of the item. @@ -64,12 +74,13 @@ class Env(Filter): method on ItemElement. """ def __init__(self, name): + super(Env, self).__init__() self.name = name def __call__(self, item): return item.env[self.name] -class TableCell(Filter): +class TableCell(_Filter): """ Used with TableElement, it get the cell value from its name. @@ -89,6 +100,7 @@ class item(ItemElement): """ def __init__(self, *names): + super(TableCell, self).__init__() self.names = names def __call__(self, item): @@ -136,3 +148,15 @@ class Link(Filter): """ def filter(self, el): return el[0].attrib.get('href', '') + + +class Attr(_Filter): + """ + Get the attribute of object. + """ + def __init__(self, name): + super(Attr, self).__init__() + self.name = name + + def __call__(self, item): + return item.use_selector(getattr(item, 'obj_%s' % self.name)) diff --git a/weboob/tools/browser2/page.py b/weboob/tools/browser2/page.py index fe27ae415395957de9fcd9e3094dfc7ebad87a17..2efb3cc6c9d48ad951fbd5b94e8d9153ba9ac6ee 100644 --- a/weboob/tools/browser2/page.py +++ b/weboob/tools/browser2/page.py @@ -21,6 +21,7 @@ import requests import re +import sys from copy import deepcopy from cStringIO import StringIO @@ -30,7 +31,7 @@ from weboob.tools.log import getLogger from .browser import DomainBrowser -from .filters import Filter, CleanText +from .filters import _Filter, CleanText class URL(object): @@ -114,7 +115,7 @@ def __new__(cls, name, bases, attrs): new_class = super(_PagesBrowserMeta, cls).__new__(cls, name, bases, attrs) if new_class._urls is None: - new_class._urls = {} + new_class._urls = OrderedDict() else: new_class._urls = deepcopy(new_class._urls) new_class._urls.update(urls) @@ -387,15 +388,18 @@ def __init__(self, page, parent=None, el=None): self.env = deepcopy(page.params) def use_selector(self, func): - if isinstance(func, Filter): + if isinstance(func, _Filter): value = func(self) elif callable(func): value = func() else: - value = func + value = deepcopy(func) return value + def parse(self, obj): + pass + def xpath(self, *args, **kwargs): return self.el.xpath(*args, **kwargs) @@ -412,9 +416,6 @@ def __init__(self, *args, **kwargs): def __call__(self): return self.__iter__() - def parse(self, el): - pass - def __iter__(self): self.parse(self.el) @@ -464,12 +465,31 @@ def handle_element(self, el): for obj in attr(self.page, self, el): yield self.store(obj) + class SkipItem(Exception): pass + +class _ItemElementMeta(type): + """ + Private meta-class used to keep order of obj_* attributes in ItemElement. + """ + def __new__(cls, name, bases, attrs): + filters = [(re.sub('^obj_', '', attr_name), attrs[attr_name]) for attr_name, obj in attrs.items() if attr_name.startswith('obj_')] + # constants first, then filters, then methods + filters.sort(key=lambda x: x[1]._creation_counter if hasattr(x[1], '_creation_counter') else (sys.maxint if callable(x[1]) else 0)) + + new_class = super(_ItemElementMeta, cls).__new__(cls, name, bases, attrs) + new_class._attrs = [f[0] for f in filters] + return new_class + + class ItemElement(AbstractElement): + __metaclass__ = _ItemElementMeta + + _attrs = None klass = None - __filter__ = None + condition = None class Index(object): pass @@ -478,9 +498,6 @@ def __init__(self, *args, **kwargs): super(ItemElement, self).__init__(*args, **kwargs) self.obj = None - def parse(self, obj): - pass - def build_object(self): return self.klass() @@ -492,22 +509,15 @@ def __call__(self, obj=None): return obj def __iter__(self): - if self.__filter__ is not None: - try: - skip = not self.__filter__(self.el) - except TypeError: - skip = not self.__filter__.im_func(self.el) - if skip: - return + if self.condition is not None and not self.condition(): + return try: if self.obj is None: self.obj = self.build_object() self.parse(self.el) - for attr in dir(self): - m = re.match('obj_(.*)', attr) - if m: - self.handle_attr(m.group(1), getattr(self, attr)) + for attr in self._attrs: + self.handle_attr(attr, getattr(self, 'obj_%s' % attr)) except SkipItem: return diff --git a/weboob/tools/capabilities/bank/transactions.py b/weboob/tools/capabilities/bank/transactions.py index dc04ce2fb92a81ec0d9eabe7b8b0ab87f863469c..a1b9e653f11d80479a4e41313ce82cdb74c2d087 100644 --- a/weboob/tools/capabilities/bank/transactions.py +++ b/weboob/tools/capabilities/bank/transactions.py @@ -23,10 +23,11 @@ import re from weboob.capabilities.bank import Transaction, Account -from weboob.capabilities import NotAvailable +from weboob.capabilities import NotAvailable, NotLoaded from weboob.tools.misc import to_unicode from weboob.tools.log import getLogger +from weboob.tools.browser2.page import TableElement from weboob.tools.browser2.filters import Filter, CleanText, CleanDecimal @@ -166,10 +167,17 @@ def inargs(key): return + class TransactionsElement(TableElement): + columns = {'date': [u'Date'], + 'vdate': [u'Valeur'], + 'raw': [u'Opération', u'Libellé'], + 'credit': [u'Crédit', 'Montant'], + 'debit': [u'Débit'], + } + class Date(CleanText): def __call__(self, item): date = super(FrenchTransaction.Date, self).__call__(item) - item.obj.rdate = date return date def filter(self, date): @@ -195,6 +203,8 @@ def Raw(klass, *args, **kwargs): class Filter(CleanText): def __call__(self, item): raw = super(Filter, self).__call__(item) + if item.obj.rdate is NotLoaded: + item.obj.rdate = item.obj.date item.obj.category = NotAvailable if ' ' in raw: item.obj.category, useless, item.obj.label = [part.strip() for part in raw.partition(' ')]