From 51348f7a38f5ddc39b3797ea95547bf71d0c0967 Mon Sep 17 00:00:00 2001 From: Bruno Chabrier Date: Sat, 23 Jan 2021 11:41:42 +0100 Subject: [PATCH] New version, faster (multi thread) and with new options (--list) --- weboob/applications/money/money.py | 902 ++++++++++++++++++++--------- 1 file changed, 619 insertions(+), 283 deletions(-) diff --git a/weboob/applications/money/money.py b/weboob/applications/money/money.py index 4f7f06d1af..1f6bdfa351 100644 --- a/weboob/applications/money/money.py +++ b/weboob/applications/money/money.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2009-2011 Romain Bignon, Christophe Benz +# Copyright(C) 2018-2021 Bruno Chabrier # # This file is part of woob. # @@ -20,13 +20,13 @@ from __future__ import print_function # start with: -# set PYTHONPATH=D:\Dropbox\Projets\money -# python d:\Dropbox\Projets\money\scripts\money -N +# set PYTHONPATH=D:\Dropbox\Projets\boomoney +# D:\Dropbox\Projets\boomoney\scripts\bin\woob.exe money -N +import signal +import sys from threading import Thread, Lock -import copy -import sys from io import StringIO import os import re @@ -34,31 +34,56 @@ import datetime from optparse import OptionGroup +import asyncio +from asyncio.subprocess import PIPE +from asyncio import create_subprocess_exec + import shutil from colorama import init, Fore, Style from weboob.tools.compat import unicode -from weboob.exceptions import BrowserUnavailable -from weboob.capabilities.bank import AccountNotFound, AccountType -from weboob.applications.bank.bank import Appbank, OfxFormatter +from weboob.capabilities.bank import AccountType +from weboob.applications.bank import Appbank +from weboob.applications.bank.bank import OfxFormatter from weboob.tools.application.formatters.simple import SimpleFormatter __all__ = ['AppMoney'] + +def handler(signum, frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + + print('Interrupted', file=sys.__stderr__) + for t in HistoryThread.allthreads: + t.terminate() + sys.exit() + + +signal.signal(signal.SIGINT, handler) + printMutex = Lock() numMutex = Lock() backupMutex = Lock() class MoneyOfxFormatter(OfxFormatter): + + MANDATORY_FIELDS = tuple(set(OfxFormatter.MANDATORY_FIELDS).union({"type"}) - {"id"}) + def start_format(self, **kwargs): self.seen = set() + # MSMoney only supports CHECKING accounts - t = kwargs['account'].type + self.original_type = kwargs['account'].type + + # we collect this formatter output because we will need to do some processing to restore the account type + self.original_outfile = self.outfile + self.outfile = StringIO() + kwargs['account'].type = AccountType.CHECKING super(MoneyOfxFormatter, self).start_format(**kwargs) - kwargs['account'].type = t + kwargs['account'].type = self.original_type def format_obj(self, obj, alias): cat = obj.category @@ -73,6 +98,16 @@ def output(self, formatted): else: super(MoneyOfxFormatter, self).output(formatted) + def flush(self): + super(MoneyOfxFormatter, self).flush() + + # we process the output by restoring the account type + collected_output = self.outfile.getvalue() + if hasattr(self, 'original_outfile'): + # do the test because when killed, flush might be called before original_outfile is called + self.outfile = self.original_outfile + self.outfile.write(re.sub('[^\r]*\r', '' + str(self.original_type) + '\r', collected_output)) + class ListFormatter(SimpleFormatter): def output(self, formatted): @@ -82,125 +117,412 @@ def output(self, formatted): super(ListFormatter, self).output(formatted) -class AppbankNoBackend(Appbank): - EXTRA_FORMATTERS = {'ops_list': MoneyOfxFormatter} - COMMANDS_FORMATTERS = {'history': 'ops_list'} +class HistoryThread(Thread): + allthreads = [] - def load_default_backends(self): + def __init__(self, money, accounts): + Thread.__init__(self) + self.daemon = True + HistoryThread.allthreads.append(self) + self.money = money + self.accounts = accounts + self.last_dates = {} + for a in self.accounts: + self.last_dates[a] = self.money.config.get(a, 'last_date', default='') + + def terminate(self): pass - def bcall_error_handler(self, backend, error, backtrace): - handled = False - if isinstance(error, BrowserUnavailable): - handled = True - self.error = True - if isinstance(error, AccountNotFound): - handled = True - self.error = True - if isinstance(error, NotImplementedError): - handled = True - self.error = False - if not handled: - self.error = True - self.woob.logger.error("Unsupported error %s in AppbankNoBackend" % type(error)) - return super(Appbank, self).bcall_error_handler(backend, error, backtrace) + def retrieve_history(self, account): + return self.money.retrieve_history(account) + def run(self): + for account in self.accounts: + last_date = self.money.get_history_from_thread(account, self) + if not self.money.options.no_import: + self.last_dates[account] = last_date -class HistoryThread(Thread): - def __init__(self, money, account): - Thread.__init__(self) - self.money = money - self.account = account - self.disabled = money.config.get(account, 'disabled', default=False) - self.date_min = money.config.get(account, 'date_min', default='') - self.last_date = money.config.get(account, 'last_date', default='') - - @property - def label(self): - if self.account in self.money.labels: - return self.money.labels[self.account] - else: - return self.money.config.get(self.account, 'label', default='') - def dumpTransaction(self, output, fields, field): - output.write("\n") - if "DTUSER" in field and "DTPOSTED" in field and not field["DTUSER"] == field["DTPOSTED"]: - # the payment date is a deferred payment - # MSMoney takes DTPOSTED, which is the payment date - # I prefer to have the date of the operation, so I set DTPOSTED - # as DTUSER - field["DTPOSTED"] = field["DTUSER"] - for f in fields.strip().split(" "): - value = field[f] - if f == "NAME": - if value == "": - # MSMoney does not support empty NAME field - value = "" - else: - # MSMoney does not support NAME field longer than 64 - value = value[:64] - output.write("<%s>%s\n" % (f, value)) - output.write("\n") +class HistoryThreadAsAProcess(HistoryThread): + + def __init__(self, money, accounts): + super(HistoryThreadAsAProcess, self).__init__(money, accounts) + self.ofxcontent = "" + self.stderr = "" + self.loop = asyncio.new_event_loop() + + def terminate(self): + self.process.terminate() + return super(HistoryThreadAsAProcess, self).terminate() + + def retrieve_history(self, account): + self.ofxcontent[account] = re.sub(r'\r\r\n', r'\n', self.ofxcontent[account]) + return self.ofxcontent[account], self.stderrcontent[account] + + async def _read_stream(self, stream, callback): + while True: + line = await stream.readline() + if line: + callback(line) + else: + break + + async def run_process(self): + id, backend = self.accounts[0].split("@") + + propagated_options = [] + for o in vars(self.money.options): + switcher = { + 'backends': False, + 'exclude_backends': False, + 'insecure': True, + 'nss': True, + 'debug': True, + 'quiet': False, + 'verbose': True, + 'logging_file': False, + 'save_responses': False, + 'export_session': False, + 'shell_completion': False, + 'auto_update': False, + 'condition': False, + 'count': True, + 'select': False, + 'formatter': False, + 'no_header': False, + 'no_keys': False, + 'outfile': False, + 'list': False, + 'force': True, + 'accounts': False, + 'until_date': True, + 'no_import': False, + 'display': False + } + propagate = switcher.get(o, None) + if propagate is None: + self.money.logger.warning("Unhandled option %s." % o) + propagate = False + if propagate: + value = getattr(self.money.options, o) + o = o.replace("_", "-") + if value is not None: + if value == 0: + pass + elif value == 1: + propagated_options += ["--" + o] + else: + propagated_options += ["--" + o + "=" + str(value)] + + self.cmd = [ + sys.executable, # D:\...\woob.exe + sys.path[0], + 'money', + '--no-import', + '--display', + '--backends=' + backend, + '--accounts=' + ",".join(self.accounts) + ] + propagated_options + self.money.logger.info(" ".join(self.cmd)) + + self.process = await create_subprocess_exec(*self.cmd, stdout=PIPE, stderr=PIPE) + + def handle_stdout_line(line): + line = line.decode("CP1252") + if not re.search(r"^\([0-9]+/[0-9]+\) ", line) is None: + account = re.split(r"\s", line)[1] + self.ofxcontent[account] = self.stdout + self.stderrcontent[account] = self.stderr + last_date = self.money.get_history_from_thread(account, self) + + if not self.money.options.no_import: + self.last_dates[account] = last_date + + self.stdout = "" + self.stderr = "" + self.stdouterr = "" + self.intransaction = False + return + + if line.startswith("Hint: There are more results available"): + self.stderr += line + self.stdouterr += line + return + + if line.startswith("OFXHEADER:"): + self.intransaction = True + + if self.intransaction: + self.stdout += line + else: + self.stderr += line + self.stdouterr += line + + # print("STDOUT: {}".format(line)) + + def handle_stderr_line(line): + line = line.decode("CP1252") + self.stderr += line + self.stdouterr += line + # print("STDERR: {}".format(line)) + + self.stdout = "" + self.stderr = "" + self.stdouterr = "" + self.ofxcontent = dict() + self.stderrcontent = dict() + self.intransaction = False + + await asyncio.wait( + [ + self._read_stream(self.process.stdout, handle_stdout_line), + self._read_stream(self.process.stderr, handle_stderr_line) + ] + ) + + await self.process.wait() def run(self): + self.loop.run_until_complete(self.run_process()) + if self.stdouterr != "": + self.money.print(" ".join(self.cmd) + "\n" + self.stdouterr) + +class AppMoney(Appbank): + APPNAME = 'money' + OLD_APPNAME = 'boomoney' + VERSION = '3.0' + COPYRIGHT = 'Copyright(C) 2018-YEAR Bruno Chabrier' + DESCRIPTION = "Console application that imports bank accounts into Microsoft Money" + SHORT_DESCRIPTION = "import bank accounts into Microsoft Money" + + EXTRA_FORMATTERS = {'list': ListFormatter, 'ops_list': MoneyOfxFormatter} + COMMANDS_FORMATTERS = {'list': 'list', 'history': 'ops_list', 'coming': 'ops_list'} + + def __init__(self): + super(AppMoney, self).__init__() + self.importIndex = 0 + application_options = OptionGroup(self._parser, 'Money Options') + application_options.add_option('-l', '--list', action='store_true', help='list the accounts and balance, without generating any import to MSMoney') + application_options.add_option('-F', '--force', action='store_true', help='forces the retrieval of transactions (10 maximum), otherwise retrieves only the transactions newer than the previous retrieval date') + application_options.add_option('-U', '--until-date', help='retrieves until date YYYY-MM-DD max') + application_options.add_option('-A', '--accounts', help='retrieves only the specified accounts. By default, all accounts are retrieved') + application_options.add_option('-N', '--no-import', action='store_true', help='no import. Generates the files, but they are not imported in MSMoney. Last import dates are not modified') + application_options.add_option('-D', '--display', action='store_true', help='displays the generated OFX file') + self._parser.add_option_group(application_options) + self.labels = dict() + self.commands_formatters["select"] = "simple" + self._backupDone = False + + def str2bool(self, str): + if str is True: + return True + if str is False: + return False + if str.upper() == "True".upper(): + return True + if str.upper() == "False".upper(): + return False + self.logger.error("Cannot convert '%s' to boolean." % str) + raise ValueError + + def print(self, *args, **kwargs): + with printMutex: + print(*args, **kwargs) + + def write(self, *args): + with printMutex: + sys.stdout.write(*args) + + def get_downloads_path(self): + if not hasattr(self, '_downloadsPath'): + s = subprocess.check_output('reg query "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\User Shell Folders" /v "{374DE290-123F-4565-9164-39C4925E467B}"', encoding='CP850') + t = re.sub(r'^(.|\r|\n)+REG_EXPAND_SZ\s+([^\n\r]+)(.|\r|\n)*$', r'\2', s) + self._downloadsPath = os.path.expandvars(t) + return self._downloadsPath + + def get_money_path(self): + if not hasattr(self, '_moneyPath'): + s = subprocess.check_output('reg query HKEY_CLASSES_ROOT\\money\\Shell\\Open\\Command /ve', encoding='CP850') + t = re.sub(r'^(.|\r|\n)+REG_SZ\s+([^\n\r]+)(.|\r|\n)*$', r'\2', s) + self._moneyPath = os.path.expandvars(os.path.dirname(t)) + return self._moneyPath + + def get_money_file(self): + if not hasattr(self, '_moneyFile'): + s = subprocess.check_output('reg query HKEY_CURRENT_USER\\Software\\Microsoft\\Money\\14.0 /v CurrentFile', encoding='CP850') + t = re.sub(r'^(.|\r|\n)+REG_SZ\s+([^\n\r]+)(.|\r|\n)*$', r'\2', s) + self._moneyFile = os.path.expandvars(t) + return self._moneyFile + + def backup_if_needed(self): + if not self._backupDone: + with backupMutex: + # redo the test in mutual exclusion + if not (hasattr(self, '_backupDone') and self._backupDone): + file = self.get_money_file() + filename = os.path.splitext(os.path.basename(file))[0] + dir = os.path.dirname(file) + self.logger.info(Fore.YELLOW + Style.BRIGHT + "Creating backup of %s..." % file + Style.RESET_ALL) + target = os.path.join(dir, filename + datetime.datetime.now().strftime("_%Y_%m_%d_%H%M%S.mny")) + shutil.copy2(file, target) + self._backupDone = True + + def save_config(self): + for t in self.threads: + for a in t.accounts: + self.config.set(a, 'label', self.config.get(a, 'label', default='')) + self.config.set(a, 'disabled', self.str2bool(self.config.get(a, 'disabled', default=False))) + self.config.set(a, 'date_min', self.config.get(a, 'date_min', default='')) + self.config.set(a, 'last_date', t.last_dates[a]) + + self.config.save() + + def get_list(self): + self.onecmd("select id label number balance") + self.options.outfile = StringIO() + self.onecmd("list") + listContent = self.options.outfile.getvalue() + self.options.outfile.close() + self.options.outfile = None + + # find max columns width + id_maxlength = 0 + label_maxlength = 0 + number_maxlength = 0 + balance_maxlength = 0 + self.ids = [] + labels = [] + balances = [] + self.numbers = [] + for line in listContent.split(os.linesep): + if not line == "": + idspec, labelspec, numberspec, balancespec = line.split("\t") + notusedid, id = idspec.split("=") + self.ids.append(id) + id_maxlength = max(id_maxlength, len(id)) + notusedlabel, label = labelspec.split("=") + bmnlabel = self.config.get(id, 'label', default='') + if bmnlabel != '' and bmnlabel != label: + label = bmnlabel + ' (' + label + ')' + labels.append(label) + label_maxlength = max(label_maxlength, len(label)) + notusednumber, number = numberspec.split("=") + self.numbers.append(number) + number_maxlength = max(number_maxlength, len(number)) + notusedbalance, balance = balancespec.split("=") + balances.append(balance) + balance_maxlength = max(balance_maxlength, len(balance)) + + # use the label if not already set + if self.config.get(id, 'label', default='') == '': + self.config.set(id, 'label', label) + + # print columns + self.print(Style.BRIGHT + "%d accounts:" % len(self.ids)) + sepline = "-".ljust(id_maxlength, "-") + "-" + "-".ljust(number_maxlength, "-") + "-" + "-".ljust(balance_maxlength, "-") + "-" + "-".ljust(label_maxlength, "-") + "-" + "--------" + self.print(sepline) + for i in range(len(self.ids)): + disabled = self.str2bool(self.config.get(self.ids[i], "disabled", default=False)) + self.print( + self.ids[i].ljust(id_maxlength), + self.numbers[i].ljust(number_maxlength), + balances[i].ljust(balance_maxlength), + labels[i].ljust(label_maxlength), + "Disabled" if disabled else "") + self.print(sepline + Style.RESET_ALL) + self.print() + + def get_history_from_thread(self, account, thread): now = datetime.datetime.now().strftime("%Y-%m-%d") + last_date = self.config.get(account, 'last_date', default='') + label = self.config.get(account, 'label', default='') + + ofxcontent, stderrcontent = thread.retrieve_history(account) - if self.money.options.force: - from_date = self.date_min + if ofxcontent == '': + self.write(stderrcontent) + self.importIndex += 1 + self.print(Style.BRIGHT + Fore.RED + "(%i/%i) %s (%s): Got error!" % ( + self.importIndex, self.nb_accounts, account, label) + Style.RESET_ALL) + return last_date else: - from_date = self.last_date + self.handle_ofx_content(account, ofxcontent, stderrcontent) + return now - if from_date >= now: - self.money.print(Style.BRIGHT + "%s (%s): Last import date is %s, no need to import again..." % ( - self.account, self.label, self.last_date) + Style.RESET_ALL) - return + def retrieve_history(self, account): - appbank = self.money.createAppbank(self.account) - if appbank is None: - with numMutex: - self.money.importIndex = self.money.importIndex + 1 - return + date_min = self.config.get(account, 'date_min', default='') + last_date = self.config.get(account, 'last_date', default='') + label = self.config.get(account, 'label', default='') - appbank.stderr = StringIO() - appbank.stdout = appbank.stderr - id, backend = self.account.split("@") - module_name, foo = appbank.woob.backends_config.get_backend(backend) - moduleHandler = "%s.bat" % os.path.join(os.path.dirname(self.money.getMoneyFile()), module_name) - self.money.logger.info("Starting history of %s (%s)..." % (self.account, self.label)) + if self.options.force: + from_date = date_min + else: + if last_date != '': + from_date = datetime.date.fromisoformat(last_date).isoformat() + else: + from_date = '' + + if self.options.count: + from_date = '' + if self.options.until_date: + from_date = self.options.until_date + + self.stderr = StringIO() + self.stdout = self.stderr + id, backend = account.split("@") + module_name, foo = self.woob.backends_config.get_backend(backend) + moduleHandler = "%s.bat" % os.path.join(os.path.dirname(self.get_money_file()), module_name) + self.logger.info("Starting history of %s (%s)..." % (account, label)) MAX_RETRIES = 3 count = 0 found = False content = '' - appbank.error = False - while count <= MAX_RETRIES and not (found and not appbank.error): - appbank.options.outfile = StringIO() - appbank.error = False + self.error = False + while count <= MAX_RETRIES and not (found and not self.error): + self.options.outfile = StringIO() + self.error = False # executing history command - appbank.onecmd("history " + self.account + " " + from_date) + self.logger.info("select " + " ".join(MoneyOfxFormatter.MANDATORY_FIELDS)) + self.onecmd("select " + " ".join(MoneyOfxFormatter.MANDATORY_FIELDS)) + self.logger.info("history " + account + " " + from_date) + self.onecmd("history " + account + " " + from_date) + expected_outputs = 1 + + # For CARD accounts, let's also get coming transactions + # We check number and string until MR!300 is merged. Can take a loooong time... + regexp = re.compile('\r\n(' + str(AccountType.CARD) + '|' + list(AccountType._keys)[int(AccountType.CARD)] + ')\r\n') + if regexp.search(self.options.outfile.getvalue()): + self.logger.info("coming " + account + " " + from_date) + self.onecmd("coming " + account + " " + from_date) + expected_outputs += 1 + + historyContent = self.options.outfile.getvalue() + self.options.outfile.close() + self.options.outfile = None if count > 0: - self.money.logger.info("Retrying %s (%s)... %i/%i" % (self.account, self.label, count, MAX_RETRIES)) - found = re.match(r'^OFXHEADER:100', appbank.options.outfile.getvalue()) - if found and not appbank.error: - content = appbank.options.outfile.getvalue() - appbank.options.outfile.close() + self.logger.info("Retrying %s (%s)... %i/%i" % (account, label, count, MAX_RETRIES)) + found = re.findall(r'OFXHEADER:100', historyContent) + nb_output = len(found) + if found and nb_output == expected_outputs and not self.error: + content = historyContent count = count + 1 + if content == '': # error occurred with numMutex: - self.money.importIndex = self.money.importIndex + 1 - index = self.money.importIndex - self.money.logger.error("(%i/%i) %s (%s): %saborting after %i retries.%s" % ( - index, len(self.money.threads), - self.account, - self.label, + self.importIndex = self.importIndex + 1 + self.logger.error("%s (%s): %saborting after %i retries.%s" % ( + account, + label, Fore.RED + Style.BRIGHT, MAX_RETRIES, Style.RESET_ALL)) - return + return '', self.stderr.getvalue() # postprocessing of the ofx content to match MSMoney expectations content = re.sub(r'Not loaded', r'', content) @@ -208,14 +530,94 @@ def run(self): output = StringIO() field = {} fields = ' ' + transaction = '' + output_id = 1 # used because we have 2 commands, history and coming for line in input: + if output_id != nb_output: + # skip trailer of first commands + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith('20200526'): + continue + if line.startswith(''): + output_id += 1 + continue + if output_id != 1: + # skip header of last commands + if line.startswith('OFXHEADER:100'): + continue + if line.startswith('DATA:OFXSGML'): + continue + if line.startswith('VERSION:'): + continue + if line.startswith('SECURITY:NONE'): + continue + if line.startswith('ENCODING:'): + continue + if line.startswith('CHARSET:'): + continue + if line.startswith('COMPRESSION:'): + continue + if line.startswith('OLDFILEUID:'): + continue + if line.startswith('NEWFILEUID:'): + continue + if line.startswith('\r'): + continue + if line.startswith('0INFO'): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith('0INFOnull'): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if line.startswith(''): + continue + if re.match(r'^OFXHEADER:100', line): inTransaction = False if re.match(r'^', line): inTransaction = True + # MSMoney only supports CHECKING accounts + if re.match(r'^', line): + line = 'CHECKING\r\n' + if not inTransaction: output.write(line) + else: + transaction = transaction + line if re.match(r'^', line): + # debug: display transaction: + # print(transaction, file=sys.stderr) + # if output_id == 2: print(transaction, file=sys.stderr) + # MSMoney expects CHECKNUM instead of NAME for CHECK transactions if "TRNTYPE" in field and field["TRNTYPE"] == "CHECK": if "NAME" in field and unicode(field["NAME"]).isnumeric(): @@ -229,7 +631,6 @@ def run(self): origfields = fields origfield = field.copy() if os.path.exists(moduleHandler): - self.money.logger.info("Calling backend handler %s..." % moduleHandler) # apply the transformations, in the form # field_NAME=... # field_MEMO=... @@ -239,9 +640,24 @@ def run(self): for f in field: value = field[f] cmd = cmd + 'set field_%s=%s& ' % (f, value) - cmd = cmd + '"' + moduleHandler + '"' - result = subprocess.check_output(cmd.encode(sys.stdout.encoding)) + + self.logger.info(cmd) + result = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = result.communicate() + if result.returncode != 0: + print(stderr.decode('CP850'), file=sys.stderr) + raise subprocess.CalledProcessError(result.returncode, cmd) + if stderr != b'' or self.options.verbose is True: + if stderr != b'': + self.logger.warning(cmd) + for f in field: + self.print('field_%s=%s' % (f, field[f])) + self.print('Output:') + self.print(stdout.decode('CP850'), end='') + self.print(stderr.decode('CP850'), end='') + + result = stdout.decode('CP850') for line in re.split(r'[\r\n]+', result): if not line == "": @@ -266,9 +682,19 @@ def run(self): else: fields = fields + f + " " - if not IGNORE: - # dump transaction - self.dumpTransaction(output, fields, field) + if "DTUSER" in field and "DTPOSTED" in field and not field["DTUSER"] == field["DTPOSTED"]: + # the payment date is a deferred payment + # MSMoney takes DTPOSTED, which is the payment date + # I prefer to have the date of the operation, so I set DTPOSTED + # as DTUSER + field["DTPOSTED"] = field["DTUSER"] + + if not IGNORE and (from_date == '' or field["DTPOSTED"] >= from_date[0:4] + from_date[5:7] + from_date[8:10]): + # dump transaction if not ignored and date matches + # (still needed because 'coming "date"' still returns older transactions - bug? + # and anyway, because for CARDs all transactions are listed by 'history "date"' as registered at the payment date) + # Also, we keep the transactions listed by 'coming' (output_id == 2) + self.dump_transaction(output, fields, field) if NEW is not None: for n in NEW.strip().split(" "): @@ -285,13 +711,14 @@ def run(self): if f not in fields.strip().split(" "): fields = fields + f + " " # dump secondary transaction - self.dumpTransaction(output, fields, field) + self.dump_transaction(output, fields, field) inTransaction = False if inTransaction: if re.match(r'^', line): field = {} fields = ' ' + transaction = '' else: t = line.split(">", 1) v = re.split(r'[\r\n]', t[1]) @@ -299,227 +726,136 @@ def run(self): fields = fields + t[0][1:] + ' ' ofxcontent = output.getvalue() - stderrcontent = appbank.stderr.getvalue() + stderrcontent = self.stderr.getvalue() input.close() output.close() - appbank.stderr.close() + self.stderr.close() + + return ofxcontent, stderrcontent + + def dump_transaction(self, output, fields, field): + output.write("\n") + for f in fields.strip().split(" "): + value = field[f] + if f == "NAME": + if value == "": + # MSMoney does not support empty NAME field + value = "" + else: + # MSMoney does not support NAME field longer than 64 + value = value[:64] + output.write("<%s>%s\n" % (f, value)) + output.write("\n") - if self.money.options.display: - self.money.print(Style.BRIGHT + ofxcontent + Style.RESET_ALL) + def handle_ofx_content(self, account, ofxcontent, stderrcontent): + + label = self.config.get(account, 'label', default='') + + if self.options.display: + self.print(Style.BRIGHT + ofxcontent + Style.RESET_ALL) nbTransactions = ofxcontent.count('') # create ofx file - fname = re.sub(r'[^\w@\. ]', '_', self.account + " " + self.label) - ofxfile = os.path.join(self.money.getDownloadsPath(), fname + ".ofx") + fname = re.sub(r'[^\w@\. ]', '_', account + " " + label) + ofxfile = os.path.join(self.get_downloads_path(), fname + datetime.datetime.now().strftime("_%Y_%m_%d_%H%M%S") + ".ofx") with open(ofxfile, "w") as ofx_file: - ofx_file.write(re.sub(r'\r\n', r'\n', ofxcontent.encode(sys.stdout.encoding))) + ofx_file.write(re.sub(r'\r\n', r'\n', ofxcontent)) with numMutex: - self.money.write(stderrcontent) - self.money.importIndex = self.money.importIndex + 1 - index = self.money.importIndex - if not (self.money.options.noimport or nbTransactions == 0): - self.money.backupIfNeeded() + self.importIndex = self.importIndex + 1 + index = self.importIndex + + self.write(stderrcontent) + + if not (self.options.no_import or nbTransactions == 0): + self.backup_if_needed() with printMutex: - if self.money.options.noimport or nbTransactions == 0: + if self.options.no_import or nbTransactions == 0: if nbTransactions == 0: print(Style.BRIGHT + '(%i/%i) %s (%s) (no transaction).' % ( - index, len(self.money.threads), - self.account, - self.label + index, self.nb_accounts, + account, + label ) + Style.RESET_ALL) else: print(Fore.GREEN + Style.BRIGHT + '(%i/%i) %s (%s) (%i transaction(s)).' % ( - index, len(self.money.threads), - self.account, - self.label, + index, self.nb_accounts, + account, + label, nbTransactions ) + Style.RESET_ALL) else: # import into money print(Fore.GREEN + Style.BRIGHT + '(%i/%i) Importing "%s" into MSMoney (%i transaction(s))...' % ( - index, len(self.money.threads), + index, self.nb_accounts, ofxfile, nbTransactions ) + Style.RESET_ALL) - if not self.money.options.noimport: + if not self.options.no_import: if nbTransactions > 0: subprocess.check_call('"%s" %s' % ( - os.path.join(self.money.getMoneyPath(), "mnyimprt.exe"), + os.path.join(self.get_money_path(), "mnyimprt.exe"), ofxfile)) - self.last_date = now - - -class AppMoney(Appbank): - APPNAME = 'money' - VERSION = '2.1' - COPYRIGHT = 'Copyright(C) 2018-YEAR Bruno Chabrier' - DESCRIPTION = "Console application that imports bank accounts into Microsoft Money" - SHORT_DESCRIPTION = "import bank accounts into Microsoft Money" - - EXTRA_FORMATTERS = {'list': ListFormatter} - COMMANDS_FORMATTERS = {'list': 'list'} - - def __init__(self): - super(Appbank, self).__init__() - self.importIndex = 0 - application_options = OptionGroup(self._parser, 'AppMoney Options') - application_options.add_option('-F', '--force', action='store_true', help='forces the retrieval of transactions (10 maximum), otherwise retrieves only the transactions newer than the previous retrieval date') - application_options.add_option('-A', '--account', help='retrieves only the specified account. By default, all accounts are retrieved') - application_options.add_option('-N', '--noimport', action='store_true', help='no import. Generates the files, but they are not imported in MSMoney. Last import dates are not modified') - application_options.add_option('-D', '--display', action='store_true', help='displays the generated OFX file') - application_options.add_option('-P', '--parallel', action='store_true', help='retrieves all accounts in parallel instead of one by one (experimental)') - self._parser.add_option_group(application_options) - self.labels = dict() - - def print(self, *args): - with printMutex: - print(*args) - - def write(self, *args): - with printMutex: - sys.stdout.write(*args) - - def createAppbank(self, account): - accountId, backendName = account.split("@") - if not self.woob.backends_config.backend_exists(backendName): - self.logger.warning("Unknown backend '%s' of account '%s' (not found in backends)" % (backendName, account)) - return None - - # create a Appbank instance - appbank = AppbankNoBackend() - appbank.options = copy.copy(self.options) - - moduleName = self.woob.backends_config._read_config().get(backendName, "_module") - module = self.woob.modules_loader.loaded[moduleName] - backend = self.woob.backend_instances[backendName] - - params = {} - for param in backend.config: - params[param] = backend.config[param].get() - dedicatedBackendInstanceName = "backend instance for " + account - appbank.APP_NAME = "bank app for " + account - instance = module.create_instance(self.woob, dedicatedBackendInstanceName, params, storage=appbank.create_storage()) - - appbank.enabled_backends = set() - appbank.enabled_backends.add(instance) - appbank.woob.backend_instances[dedicatedBackendInstanceName] = instance - - appbank.selected_fields = ["$full"] - appbank.formatter = self.formatter - - appbank._interactive = False - return appbank - - def getHistory(self, account): - t = HistoryThread(self, account) - return t - - def getDownloadsPath(self): - if not hasattr(self, '_downloadsPath'): - s = subprocess.check_output( - 'reg query "HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\User Shell Folders" /v "{374DE290-123F-4565-9164-39C4925E467B}"') - t = re.sub(r'^(.|\r|\n)+REG_EXPAND_SZ\s+([^\n\r]+)(.|\r|\n)*$', r'\2', s) - self._downloadsPath = os.path.expandvars(t).decode('CP850') - return self._downloadsPath - - def getMoneyPath(self): - if not hasattr(self, '_moneyPath'): - s = subprocess.check_output('reg query HKEY_CLASSES_ROOT\\money\\Shell\\Open\\Command /ve') - t = re.sub(r'^(.|\r|\n)+REG_SZ\s+([^\n\r]+)(.|\r|\n)*$', r'\2', s) - self._moneyPath = os.path.expandvars(os.path.dirname(t)).decode('CP850') - return self._moneyPath + def main(self, argv): - def getMoneyFile(self): - if not hasattr(self, '_moneyFile'): - s = subprocess.check_output('reg query HKEY_CURRENT_USER\\Software\\Microsoft\\Money\\14.0 /v CurrentFile') - t = re.sub(r'^(.|\r|\n)+REG_SZ\s+([^\n\r]+)(.|\r|\n)*$', r'\2', s) - self._moneyFile = os.path.expandvars(t).decode('CP850') - return self._moneyFile + init() - def backupIfNeeded(self): - if not (hasattr(self, '_backupDone') and self._backupDone): - with backupMutex: - # redo the test in mutual exclusion - if not (hasattr(self, '_backupDone') and self._backupDone): - file = self.getMoneyFile() - filename = os.path.splitext(os.path.basename(file))[0] - dir = os.path.dirname(file) - self.print(Fore.YELLOW + Style.BRIGHT + "Creating backup of %s..." % file + Style.RESET_ALL) - target = os.path.join(dir, filename + datetime.datetime.now().strftime("_%Y_%m_%d_%H%M%S.mny")) - shutil.copy2(file, target) - self._backupDone = True + self.load_config() - def save_config(self): - for t in self.threads: - self.config.set(t.account, 'label', t.label) - self.config.set(t.account, 'disabled', t.disabled) - self.config.set(t.account, 'date_min', t.date_min) - self.config.set(t.account, 'last_date', t.last_date) + self._interactive = False - self.config.save() + if self.options.list: + self.get_list() + return - def getList(self): - self.onecmd("select id label") - self.options.outfile = StringIO() - self.onecmd("list") - listContent = self.options.outfile.getvalue() - self.options.outfile.close() - self.print(Style.BRIGHT + "Accounts:%s----------%s%s----------" % ( - os.linesep, - os.linesep, - listContent) + Style.RESET_ALL) - for line in listContent.split(os.linesep): - if not line == "": - idspec, labelspec = line.split("\t") - notusedid, id = idspec.split("=") - notusedlabel, label = labelspec.split("=") - self.labels[id] = label + self.threads = set() - def checkNew(self): - new = set() - for account in self.labels: - if account not in self.config.config.sections(): - new.add(HistoryThread(self, account)) - return new + accounts = [] + if not self.options.accounts: + self.get_list() + accounts = self.ids + else: + accounts = self.options.accounts.split(",") - def main(self, argv): + # take only enabled accounts + accounts = list(filter( + lambda account: self.str2bool(self.config.get(account, "disabled", default=False)) is False, + accounts)) + self.nb_accounts = len(accounts) - init() + # accounts.sort(key = lambda x: x.split("@")[1]) - self.load_config() + # make a unique list + backends = list(set(map(lambda x: x.split("@")[1], accounts))) - self._interactive = False + # order backends by number of accounts + backends.sort(key=lambda b: len(list(filter( + lambda x: x.split("@")[1] == b, + accounts))), reverse=True) - self.threads = set() + self.main_thread = None + for backend in backends: + backend_accounts = list(filter( + lambda x: x.split("@")[1] == backend, + accounts)) - self.logger.info(self.config.config.sections()) - for account in self.config.config.sections(): - if self.options.account == None or account == self.options.account: - if self.config.config.getboolean(account, "disabled") == False: - # time.sleep(3) - self.threads.add(self.getHistory(account)) + if backend == backends[0]: + self.main_thread = HistoryThread(self, backend_accounts) + self.threads.add(self.main_thread) + else: + self.threads.add(HistoryThreadAsAProcess(self, backend_accounts)) - if self.options.parallel: - self.print(Fore.MAGENTA + Style.BRIGHT + "Starting %i history threads..." % len(self.threads) + Style.RESET_ALL) + if self.main_thread is not None: for t in self.threads: - t.start() - self.getList() - for t in self.checkNew(): - t.start() - self.threads.add(t) - self.print(Fore.MAGENTA + Style.BRIGHT + "Waiting for %i threads to complete..." % len(self.threads) + Style.RESET_ALL) - for t in self.threads: - t.join() - else: - self.getList() - for t in self.checkNew(): - self.threads.add(t) + if t != self.main_thread: + t.start() + self.main_thread.run() + for t in self.threads: - t.start() - t.join() + if t != self.main_thread: + t.join() self.save_config() return -- GitLab