# -*- coding: utf-8 -*- # Copyright(C) 2014 Oleg Plakhotniuk # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this weboob module. If not, see . from __future__ import unicode_literals import datetime import re from weboob.capabilities.bank import Transaction from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr from weboob.tools.compat import unicode from weboob.tools.date import closest_date from weboob.tools.pdf import decompress_pdf from weboob.tools.tokenizer import ReTokenizer def clean_label(text): """ Web view and statements use different label formatting. User shouldn't be able to see the difference, so we need to make labels from both sources look the same. """ for pattern in [r' \d+\.\d+ +POUND STERLING', 'Subject to Foreign Fee', 'Description']: text = re.sub(pattern, '', text, re.UNICODE) return re.sub(r' +', ' ', text.strip().upper(), re.UNICODE) def formatted(read_func): """ Reads boilerplate PDF formatting around the data of interest. """ def wrapped(self, pos): startPos = pos pos, ws = self.read_whitespace(pos) pos, bt = self.read_layout_bt(pos) pos, tf = self.read_layout_tf(pos) pos, tm = self.read_layout_tm(pos) pos, data = read_func(self, pos) pos, et = self.read_layout_et(pos) if ws is None or bt is None or tf is None \ or tm is None or data is None or et is None: return startPos, None else: return pos, data return wrapped class StatementParser(object): """ Each "read_*" method takes position as its argument, and returns next token position if read was successful, or the same position if it was not. """ LEX = [ ('date_range', r'^\((\d{2}/\d{2}/\d{2})-(\d{2}/\d{2}/\d{2})\) Tj$'), ('amount', r'^\((-?\$\d+(,\d{3})*\.\d{2})\) Tj$'), ('date', r'^\((\d{2}/\d{2})\) Tj$'), ('text', r'^\((.*)\) Tj$'), ('layout_tf', r'^.* Tf$'), ('layout_tm', r'^' + (6*r'([^ ]+) ') + r'Tm$'), ('layout_bt', r'^BT$'), ('layout_et', r'^ET$'), ('whitespace', r'^$') ] def __init__(self, pdf): self._pdf = decompress_pdf(pdf) self._tok = ReTokenizer(self._pdf, '\n', self.LEX) def read_transactions(self): # Read statement dates range. date_from, date_to = self.read_first_date_range() # Read transactions. pos = 0 while not self._tok.tok(pos).is_eof(): pos, trans = self.read_transaction(pos, date_from, date_to) if trans: yield trans else: pos += 1 def read_first_date_range(self): pos = 0 while not self._tok.tok(pos).is_eof(): pos, date_range = self.read_date_range(pos) if date_range is not None: return date_range else: pos += 1 def read_date_range(self, pos): t = self._tok.tok(pos) if t.is_date_range(): return (pos+1, [datetime.datetime.strptime(v, '%m/%d/%y') for v in t.value()]) else: return (pos, None) def read_transaction(self, pos, date_from, date_to): startPos = pos pos, tdate = self.read_date(pos) pos, pdate = self.read_date(pos) # Early check to call read_multiline_desc() only when needed. if tdate is None: return startPos, None pos, desc = self.read_multiline_desc(pos) pos, amount = self.read_amount(pos) if desc is None or amount is None: return startPos, None else: # Sometimes one date is missing. pdate = pdate or tdate tdate = closest_date(tdate, date_from, date_to) pdate = closest_date(pdate, date_from, date_to) trans = Transaction() trans.date = tdate trans.rdate = pdate trans.type = Transaction.TYPE_UNKNOWN trans.raw = desc trans.label = desc trans.amount = -amount return pos, trans def read_multiline_desc(self, pos): """ Read transaction description which can span over multiple lines. Amount must always follow the multiline description. But multiline description might be split by page break. After reading first line of the description, we skip everything which is not an amount and which has different horizontal offset than the first read line. """ startPos = pos descs = [] xofs = None while not self._tok.tok(pos).is_eof(): pos, desc_tm = self.read_text(pos) if desc_tm is None: if not descs: break prev_pos = pos pos, amount = self.read_amount(pos) if amount is not None: pos = prev_pos break pos += 1 else: desc, tm = desc_tm if xofs is None: _, _, _, _, xofs, _ = tm _, _, _, _, xofs_new, _ = tm if xofs == xofs_new: descs.append(desc) else: pos += 1 if descs: return pos, clean_label(' '.join(descs)) else: return startPos, None def __getattr__(self, name): if name.startswith('read_'): return lambda pos: self._tok.simple_read(name[5:], pos) raise AttributeError() @formatted def read_date(self, pos): def parse_date(v): for year in [1900, 1904]: # try leap and non-leap years fullstr = '%s/%i' % (v, year) try: return datetime.datetime.strptime(fullstr, '%m/%d/%Y') except ValueError as e: last_error = e raise last_error return self._tok.simple_read('date', pos, parse_date) @formatted def read_amount(self, pos): return self._tok.simple_read('amount', pos, lambda xs: AmTr.decimal_amount(xs[0])) def read_text(self, pos): startPos = pos pos, ws = self.read_whitespace(pos) pos, bt = self.read_layout_bt(pos) pos, tf = self.read_layout_tf(pos) pos, tm = self.read_layout_tm(pos) pos, text = self._tok.simple_read('text', pos, lambda v: unicode(v, errors='ignore')) pos, et = self.read_layout_et(pos) if ws is None or bt is None or tf is None \ or tm is None or text is None or et is None: return startPos, None else: return pos, (text, tm)