From e2a8f085814e9220c33131ede5b686609b478de6 Mon Sep 17 00:00:00 2001 From: Pavol Rusnak Date: Mon, 13 Apr 2015 15:39:23 +0200 Subject: [PATCH] add local version of filecache (used just for unittests), cache globally (to tmpdir) --- trezorlib/filecache.py | 184 +++++++++++++++++++++++++++++++++++++++++ trezorlib/tx_api.py | 11 +-- 2 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 trezorlib/filecache.py diff --git a/trezorlib/filecache.py b/trezorlib/filecache.py new file mode 100644 index 0000000..b1132e7 --- /dev/null +++ b/trezorlib/filecache.py @@ -0,0 +1,184 @@ +''' +filecache + +filecache is a decorator which saves the return value of functions even +after the interpreter dies. For example this is useful on functions that download +and parse webpages. All you need to do is specify how long +the return values should be cached (use seconds, like time.sleep). + +USAGE: + + from filecache import filecache + + @filecache(24 * 60 * 60) + def time_consuming_function(args): + # etc + + @filecache(filecache.YEAR) + def another_function(args): + # etc + + +NOTE: All arguments of the decorated function and the return value need to be + picklable for this to work. + +NOTE: The cache isn't automatically cleaned, it is only overwritten. If your + function can receive many different arguments that rarely repeat, your + cache may forever grow. One day I might add a feature that once in every + 100 calls scans the db for outdated stuff and erases. + +NOTE: This is less useful on methods of a class because the instance (self) + is cached, and if the instance isn't the same, the cache isn't used. This + makes sense because class methods are affected by changes in whatever + is attached to self. + +Tested on python 2.7 and 3.1 + +License: BSD, do what you wish with this. Could be awesome to hear if you found +it useful and/or you have suggestions. ubershmekel at gmail + + +A trick to invalidate a single value: + + @filecache.filecache + def somefunc(x, y, z): + return x * y * z + + del somefunc._db[filecache._args_key(somefunc, (1,2,3), {})] + # or just iterate of somefunc._db (it's a shelve, like a dict) to find the right key. + + +''' + + +import collections as _collections +import datetime as _datetime +import functools as _functools +import inspect as _inspect +import os as _os +import pickle as _pickle +import shelve as _shelve +import sys as _sys +import time as _time +import traceback as _traceback +import types + +_retval = _collections.namedtuple('_retval', 'timesig data') +_SRC_DIR = _os.path.dirname(_os.path.abspath(__file__)) + +SECOND = 1 +MINUTE = 60 * SECOND +HOUR = 60 * MINUTE +DAY = 24 * HOUR +WEEK = 7 * DAY +MONTH = 30 * DAY +YEAR = 365 * DAY +FOREVER = None + +OPEN_DBS = dict() + +def _get_cache_name(function): + """ + returns a name for the module's cache db. + """ + module_name = _inspect.getfile(function) + module_name = _os.path.abspath(module_name) + cache_name = module_name + + # fix for '' or '' in exec or interpreter usage. + cache_name = cache_name.replace('<', '_lt_') + cache_name = cache_name.replace('>', '_gt_') + + tmpdir = _os.getenv('TMPDIR') or _os.getenv('TEMP') or _os.getenv('TMP') + if tmpdir: + cache_name = tmpdir + '/filecache_' + cache_name.replace(_os.sep, '@') + + cache_name += '.cache' + return cache_name + + +def _log_error(error_str): + try: + error_log_fname = _os.path.join(_SRC_DIR, 'filecache.err.log') + if _os.path.isfile(error_log_fname): + fhand = open(error_log_fname, 'a') + else: + fhand = open(error_log_fname, 'w') + fhand.write('[%s] %s\r\n' % (_datetime.datetime.now().isoformat(), error_str)) + fhand.close() + except Exception: + pass + +def _args_key(function, args, kwargs): + arguments = (args, kwargs) + # Check if you have a valid, cached answer, and return it. + # Sadly this is python version dependant + if _sys.version_info[0] == 2: + arguments_pickle = _pickle.dumps(arguments) + else: + # NOTE: protocol=0 so it's ascii, this is crucial for py3k + # because shelve only works with proper strings. + # Otherwise, we'd get an exception because + # function.__name__ is str but dumps returns bytes. + arguments_pickle = _pickle.dumps(arguments, protocol=0).decode('ascii') + + key = function.__name__ + arguments_pickle + return key + +def filecache(seconds_of_validity=None, fail_silently=False): + ''' + filecache is called and the decorator should be returned. + ''' + def filecache_decorator(function): + @_functools.wraps(function) + def function_with_cache(*args, **kwargs): + try: + key = _args_key(function, args, kwargs) + + if key in function._db: + rv = function._db[key] + if seconds_of_validity is None or _time.time() - rv.timesig < seconds_of_validity: + return rv.data + except Exception: + # in any case of failure, don't let filecache break the program + error_str = _traceback.format_exc() + _log_error(error_str) + if not fail_silently: + raise + + retval = function(*args, **kwargs) + + # store in cache + # NOTE: no need to _db.sync() because there was no mutation + # NOTE: it's importatnt to do _db.sync() because otherwise the cache doesn't survive Ctrl-Break! + try: + function._db[key] = _retval(_time.time(), retval) + function._db.sync() + except Exception: + # in any case of failure, don't let filecache break the program + error_str = _traceback.format_exc() + _log_error(error_str) + if not fail_silently: + raise + + return retval + + # make sure cache is loaded + if not hasattr(function, '_db'): + cache_name = _get_cache_name(function) + if cache_name in OPEN_DBS: + function._db = OPEN_DBS[cache_name] + else: + function._db = _shelve.open(cache_name) + OPEN_DBS[cache_name] = function._db + + function_with_cache._db = function._db + + return function_with_cache + + if type(seconds_of_validity) == types.FunctionType: + # support for when people use '@filecache.filecache' instead of '@filecache.filecache()' + func = seconds_of_validity + return filecache_decorator(func) + + return filecache_decorator diff --git a/trezorlib/tx_api.py b/trezorlib/tx_api.py index 94214f0..55d39c1 100644 --- a/trezorlib/tx_api.py +++ b/trezorlib/tx_api.py @@ -2,16 +2,7 @@ import binascii import urllib2 import json from decimal import Decimal -try: - raise Exception() # remove this line to enable caching - from filecache import filecache, DAY -except: - def filecache(x): - def _inner(y): - return y - return _inner - DAY = None - +from filecache import filecache, DAY import types_pb2 as proto_types def op_push_data(data):