import simplejson as json from copy import deepcopy from flask import g from inspect import getfullargspec from typing import Any, Callable, Dict, Generator, Iterable, List, Set, Tuple from src.cache import redis_client from src.cache.constants import CacheMode from src.config.base import REDIS_SCAN_CHUNK_SIZE from src.legacy.redis_db import keys from src.legacy.redis_db.keys import get_key from src.utils.json import date_hook, str_to_date def cache_requests( cache_key_template: str, cache_ttl: int, ids_index: int or None = 1, ids_kwargs_name: str or None = None, cache_key_parts: Tuple[int] or None = (2,), cache_key_kwargs_names: Tuple[str] or None = None, item_key_name: str = "id", ) -> Callable: """Get multiple items from cache and get missing. Args: cache_key_template (str): Cache key template. cache_ttl (int): Cache TTL. ids_index (int or None): IDs list argument index (or None if ids are in kwargs). ids_kwargs_name (str or None): IDs list kwargs key (or None if ids are in args) cache_key_parts (Tuple[int] or None): Cache key parts indexes in args (or None). cache_key_kwargs_names (Tuple[str] or None): Cache key parts names in kwargs (or None). item_key_name (str): Item key field. Returns: Callable: Decorated function. """ def decorator(f: Callable) -> Callable: if not ((ids_index is None) ^ (ids_kwargs_name is None)): raise ValueError("One and only one of 'ids_index', 'ids_kwargs_name' should be passed.") def wrapped(*args, **kwargs) -> List[dict]: result = [] mode = g._cache_mode if mode == CacheMode.IGNORE or not cache_ttl or cache_ttl <= 0: return f(*args, **kwargs) cache_args = tuple(args[i] for i in cache_key_parts or []) cache_kwargs = {k: kwargs[k] for k in cache_key_kwargs_names or []} ids_list = args[ids_index] if ids_index is not None else kwargs[ids_kwargs_name] cache_keys = [get_key(cache_key_template, *cache_args, _id, **cache_kwargs) for _id in ids_list] if mode != CacheMode.UPDATE: result = redis_client.mget(cache_keys) result = [decode_str(item) for item in result if item] items_to_check = [item for item in ids_list if item not in set([x[item_key_name] for x in result])] if items_to_check: if ids_index is not None: args_list = list(args) args_list[ids_index] = items_to_check args = tuple(args_list) if ids_kwargs_name is not None: kwargs[ids_kwargs_name] = items_to_check api_result = f(*args, **kwargs) result.extend(api_result) redis_pipeline = redis_client.pipeline() for item in api_result: key = get_key(cache_key_template, *cache_args, item[item_key_name], **cache_kwargs) redis_pipeline.setex(key, cache_ttl, json.dumps(item, default=str)) redis_pipeline.execute() return result return wrapped return decorator def decode_str(json_data: str): """Decode json string, correctly handle lists and dates. Args: json_data (str): JSON string. Returns: list or dict or flat value: Decoded data. """ data = json.loads(json_data, strict=False, object_hook=date_hook) return str_to_date(data) def get_key_by_template( cache_key_template: str, defined_args: List[str], args: Tuple, kwargs: Dict, key_args_indexes: Tuple[int] or None = None, key_args: Tuple[Any] or None = None, key_kwargs: Dict[str, Any] or None = None, ): key_parts_args = list(args) + [kwargs[a] for a in defined_args if a in kwargs] if key_args_indexes: key_parts_args = [key_parts_args[i] for i in key_args_indexes] if key_args: key_parts_args = list(key_args) + key_parts_args key_parts_kwargs = {} if kwargs: key_parts_kwargs.update(kwargs) if args: key_parts_kwargs.update({k: v for k, v in zip(defined_args, args)}) if key_kwargs: key_parts_kwargs.update(key_kwargs) return get_key(cache_key_template, *key_parts_args, **key_parts_kwargs) def cache_chart_last_dates( cache_key_template: str, cache_ttl: int, chart_date_index: int, key_args_indexes: Tuple[int] or None = None, key_args: Tuple[Any] or None = None, key_kwargs: Dict[str, Any] or None = None, ) -> Callable: """Cache chart last dates, generate key using args. Custom cache decorator ensures that requests for dates with chart_date=None (meaning that the most recent date is required) and with a chart_date equal to the date that actually matches the most recent one will return the same results. This behavior is necessary for the consistency of the returned data. This is achieved by caching the result for the most recent date by two keys with None and with this date in the key. Args: cache_key_template (str): Cache key template. cache_ttl (int): Cache TTL. chart_date_index (int): index of chart_date in the arguments. key_args_indexes: (Tuple[int] or None): Cache key parts indexes in args or None if use all args. key_args (Tuple[Any] or None): Additional key parts. key_kwargs (Dict[str, Any] or None): Additional key parts. Returns: Callable: Decorated function. """ def decorator(f: Callable) -> Callable: argspec = getfullargspec(f) def wrapped(*args, **kwargs) -> List[dict]: result = None mode = g._cache_mode if mode == CacheMode.IGNORE or not cache_ttl or cache_ttl <= 0: return f(*args, **kwargs) defined_args = argspec.args args_chart_date = args[chart_date_index] cache_key = get_key_by_template( cache_key_template, defined_args, args, kwargs, key_args_indexes, key_args, key_kwargs ) if mode != CacheMode.UPDATE: result = redis_client.get(cache_key) if result: return decode_str(result) result = f(*args, **kwargs) # if the result was got for the most recent date (according to the request with chart_date=None) # it also should be saved by key for case chart_date=most recent date for the consistency between the cases. if args_chart_date is None and result and result[0]: most_recent_date = result[0] args_with_date = list(args) args_with_date[chart_date_index] = most_recent_date pair_cache_key = get_key_by_template( cache_key_template, defined_args, args_with_date, kwargs, key_args_indexes, key_args, key_kwargs ) dumped_result = json.dumps(result, default=str) with redis_client.pipeline() as pipe: pipe.setex(cache_key, cache_ttl, dumped_result) pipe.setex(pair_cache_key, cache_ttl, dumped_result) pipe.execute() else: redis_client.setex(cache_key, cache_ttl, json.dumps(result, default=str)) return result return wrapped return decorator def get_redis_value( cache_key_template: str, defined_args: List[str] or None = None, func: Callable or None = None, key_args_indexes: Tuple[int] or None = None, args: Tuple[Any] or None = None, kwargs: Dict[str, Any] or None = None, key_args: Tuple[Any] or None = None, key_kwargs: Dict[str, Any] or None = None, ) -> Tuple[Any, str]: """Generate key and get data from redis. Args: cache_key_template: Cache key template. defined_args: Function full args spec. func: Function which results are cached. key_args_indexes: Cache key parts indexes in args or None if use all args. args: Func call args. kwargs: Func call kwargs. key_args: Additional key args. key_kwargs: Additional key kwargs. Returns: Redis data and generated key. """ result = None mode = g._cache_mode if defined_args is None: defined_args = getfullargspec(func).args cache_key = get_key_by_template( cache_key_template, defined_args, args, kwargs, key_args_indexes, key_args, key_kwargs ) if mode == CacheMode.REGULAR: result = redis_client.get(cache_key) if result: return decode_str(result), cache_key return result, cache_key def set_redis_value(cache_key: str or None, cache_ttl: int, data: Any): """Put data to redis. Args: cache_key: Key. cache_ttl: TTL. data: Value. """ if cache_key: serialized_data = json.dumps(data, default=str) redis_client.setex(cache_key, cache_ttl, serialized_data) def cache_value( cache_key_template: str, cache_ttl: int, key_args_indexes: Tuple[int] or None = None, key_args: Tuple[Any] or None = None, key_kwargs: Dict[str, Any] or None = None, ) -> Callable: """Cache one value, generate key using args. Args: cache_key_template (str): Cache key template. cache_ttl (int): Cache TTL. key_args_indexes: (Tuple[int] or None): Cache key parts indexes in args or None if use all args. key_args (Tuple[Any] or None): Additional key parts. key_kwargs (Dict[str, Any] or None): Additional key parts. Returns: Callable: Decorated function. """ def decorator(f: Callable) -> Callable: argspec = getfullargspec(f) def wrapped(*args, **kwargs) -> List[dict]: mode = g._cache_mode if mode == CacheMode.IGNORE or not cache_ttl or cache_ttl <= 0: return f(*args, **kwargs) result, cache_key = get_redis_value( cache_key_template, defined_args=argspec.args, key_args_indexes=key_args_indexes, args=args, kwargs=kwargs, key_args=key_args, key_kwargs=key_kwargs, ) if not result: result = f(*args, **kwargs) set_redis_value(cache_key, cache_ttl, result) return result return wrapped return decorator def get_cache_builder_query_key_part(cache_key_template: str, builder, query_dict: dict, local_params: object) -> str: """Return cache key (query part) for caching ChartsResponseBuilder._process_all. :param cache_key_template: string template for caching key. :param builder: response builder object. :param query_dict: dict got from query_params builder object. :param local_params: object: local_parameters builder object. """ params = { "builder_name": builder.__class__.__name__, "start_position": None, "end_position": None, "change": None, "compact": False, "chart_type": None, "is_sony": None, } params.update(query_dict) params["chart_date"] = local_params.last_date return get_key(cache_key_template, **params) def filter_existing_cache_keys_by_fields( existing_cache_keys: Iterable[str], key_fields_set: Set[str], fields_delimiter: str, inner_delimiter: str ) -> Generator[str, None, None]: """ Filters the passed keys, leaving only those that contain all the requested fields. Returns filtered keys in descending order of the number of fields included in the key. This is necessary so that, if possible, data is returned from the most complete cached instance of the appropriate ones, which contributes to the consistency of the returned information for requests containing different lists of requested fields. :param existing_cache_keys: existing cache keys to filter by requested fields. :param key_fields_set: requested fields set. :param fields_delimiter: cache key delimiter for separating query key part from fields key part. :param inner_delimiter: cache key delimiter for separating fields from each other in the fields key part. :return: generator of ordered filtered keys. """ key_and_fields_number = [] for existing_key in existing_cache_keys: key_parts = existing_key.split(fields_delimiter) existing_key_fields_set = set(key_parts[1].split(inner_delimiter)) if len(key_parts) > 1 else {} if key_fields_set.issubset(existing_key_fields_set): key_and_fields_number.append((existing_key, len(existing_key_fields_set))) return (kr[0] for kr in sorted(key_and_fields_number, key=lambda kr: kr[1], reverse=True)) def cache_builder( cache_query_template: str = keys.CHARTS_BUILDER_BASE_QUERY_PATTERN, fields_delimiter: str = keys.CHARTS_BUILDER_FIELDS_DELIMITER, inner_delimiter: str = "/", ignored_fields: tuple = ("is_starred",), cache_ttl: int = keys.CHARTS_RESPONSE_TTL, scan_count: int = REDIS_SCAN_CHUNK_SIZE, ) -> Callable: """Decorator for caching ResponseBuilder._process_all by request parameters value. :param cache_query_template: string base template for caching key (only query part without fields part). :param cache_ttl: expiration time. :param fields_delimiter: cache key delimiter for separating query key part from fields key part. :param inner_delimiter: cache key delimiter for separating fields from each other in the fields key part. :param ignored_fields: fields to be excluded from cache key. :param scan_count: chunk size for redis iter scan """ def decorator(f: Callable) -> Callable: def wrapped( builder, vendor, query_params: object, fields_params: object, local_params: object ) -> Dict[str, Any]: """Function to wrap ResponseBuilder._process_all, has same signature.""" existing_cache_keys = [] mode = g._cache_mode if mode == CacheMode.IGNORE or not cache_ttl or cache_ttl <= 0: return f(builder, vendor, query_params, fields_params, local_params) query_dict = deepcopy(query_params.__dict__) fields_set = {k for k, v in fields_params.__dict__.items() if v and k not in ignored_fields} key_query_part = get_cache_builder_query_key_part(cache_query_template, builder, query_dict, local_params) if mode != CacheMode.UPDATE: key_prefix_pattern = f"{key_query_part}{fields_delimiter}*" existing_cache_keys = [ k.decode() for k in redis_client.scan_iter(match=key_prefix_pattern, count=scan_count) ] for existing_key in filter_existing_cache_keys_by_fields( existing_cache_keys, fields_set, fields_delimiter, inner_delimiter ): redis_response = redis_client.get(existing_key) if redis_response: result = decode_str(redis_response) break else: new_cache_key = f"{key_query_part}{fields_delimiter}{inner_delimiter.join(sorted(fields_set))}" result = f(builder, vendor, query_params, fields_params, local_params) redis_client.setex(new_cache_key, cache_ttl, json.dumps(result, default=str)) return result return wrapped return decorator