import datetime import hashlib import logging import os import pickle from pathlib import Path import pandas as pd from tadas.platform import config logger = logging.getLogger(__name__) THIS_DIR = Path(__file__).parent PROJECT_ROOT = THIS_DIR.parent.parent.parent CACHE_DIR = PROJECT_ROOT / '.cache' CONFIG_USE_CACHE = config.get('USE_CACHE') CACHE_TTL = datetime.timedelta(minutes=config.get('CACHE_TTL_MINUTES')) def delete_outdated_files(cache_dir: Path, ttl: datetime.timedelta = CACHE_TTL): for filename in os.listdir(cache_dir): file_path = os.path.join(cache_dir, filename) if os.path.isfile(file_path): file_age = os.path.getmtime(file_path) file_timestamp = datetime.datetime.fromtimestamp(file_age) if file_timestamp < datetime.datetime.now() - ttl: logger.info( f"File created at {file_timestamp}, TTL is {ttl}. It is expired. file: {file_path}") logger.info(f"Removing old cache file: {file_path}") os.remove(file_path) def _to_pickle_df(cache_path, df): logger.info(f'Saving {len(df)} rows to cache file: {cache_path}') df.to_pickle(cache_path) def _from_pickle_df(cache_path): df = pd.read_pickle(cache_path) logger.info(f'Loaded {len(df) if df is not None else None} rows from file: {cache_path}') return df def cached_df(**kwargs): return cached( to_picke=_to_pickle_df, from_picle=_from_pickle_df, **kwargs, ) def cached( cache_dir=CACHE_DIR, cache_ttl=CACHE_TTL, hashing_kwargs=None, to_picke=lambda cache_path, values: cache_path.write_bytes(pickle.dumps(values)), from_picle=lambda cache_path: pickle.loads(cache_path.read_bytes()), ): """ Cache function results to a file system. WARNING: hashes only kwargs. Positional args are ignored! Features: * limit of cached time * stores hash key in the cache folder alongside the result * cache can be disabled by passing `use_cache=False` to the decorated function """ def actual_decorator(func): def wrapper(*args, use_cache=True, **kwargs): use_cache = use_cache and CONFIG_USE_CACHE if not use_cache: return func(*args, **kwargs) if args: logger.warning(f"Args non empty (={len(args)}). *args are ignored when caching") os.makedirs(cache_dir, exist_ok=True) delete_outdated_files(cache_dir, ttl=cache_ttl) hashing_kwargs_params = {k: v for k, v in kwargs.items() if not hashing_kwargs or k in hashing_kwargs} hash_key = str(hashing_kwargs_params) query_hash = hashlib.sha256(hash_key.encode("utf-8")).hexdigest() cache_path = cache_dir / f"{query_hash}.pkl" hash_info_file = cache_dir / f"{query_hash}.txt" logger.info(f'Lookup cache file at: {cache_path}') if not os.path.exists(cache_path): logger.info("Executing wrapped query...") result = func(*args, **kwargs) hash_info_file.write_text(hash_key) logger.info(f'Saved Hash Info to {hash_info_file}') to_picke(cache_path, result) logger.info("Loading from cache...") cached_result = from_picle(cache_path) return cached_result return wrapper return actual_decorator