from datetime import datetime, timedelta from typing import Any, Dict, List, Optional, Tuple from redis import asyncio as aioredis from sentry_sdk import capture_exception import config import context from server.core.cache.utils import is_image_caching_collection from server.core.constants import StorageField from server.core.utils import compress, count_time, decompress CACHE_KEY_PREFIX = "cache" redis_client: Optional[aioredis.Redis] = None def init(): global redis_client redis_client = aioredis.from_url(config.REDIS_URL) return redis_client async def check_health() -> Dict[str, Any]: """Invokes healthcheck.""" return await redis_client.info() async def close() -> None: """Closes cache client session.""" return await redis_client.close() @count_time(context.CACHE_TIME) async def get_documents( collection_name: str, keys: List[str], timeout: int or None, by_record_id: bool ) -> Tuple[Dict[str, dict], datetime or None, datetime or None]: """ Get cached data from Redis. Storage logic is implemented as a compound key from generic prefix, compression type, collection name, specific record key and a compressed with configured lib value. Args: collection_name: Collection name used for keys namespacing. keys: items ID list. timeout: Cached data considered as actual if it was saved to mongo less than this interval (in seconds) ago, 0 or None - do not use cache, -1 - ignore timeout. by_record_id: Return results mapping by record ID or by object ID. Returns: ID to data mapping and datetime stats. """ if not timeout or not keys: return {}, None, None try: raw_data = zip(keys, await redis_client.mget(*[get_full_key(collection_name, key) for key in keys])) except Exception as e: capture_exception(e) return {}, None, None data = {} dates = [] for key, raw_value in raw_data: if not raw_value: continue try: value = decompress(config.REDIS_COMPRESSION_LIB, raw_value) date = datetime.fromisoformat(value.get(StorageField.CREATED_AT)) except Exception as e: capture_exception(e) continue if timeout > 0: if date <= datetime.utcnow() - timedelta(seconds=timeout): continue if ( is_image_caching_collection(collection_name) and timeout > config.TEMP_IMAGE_MAX_TTL and ( (value.get(StorageField.IMAGES_SAVED) and date <= datetime.utcnow() - timedelta(seconds=timeout)) or (date <= datetime.utcnow() - timedelta(seconds=config.TEMP_IMAGE_MAX_TTL)) ) ): continue key_field = StorageField.RECORD_ID if by_record_id else StorageField.OBJECT_ID data[value.get(key_field)] = value.get(StorageField.DATA) dates.append(date) return data, min(dates) if dates else None, max(dates) if dates else None async def save_documents( collection_name: str, records: Dict[str, Tuple[str, dict] or Tuple[int, dict]], created_at: datetime ): """ Save data to Redis. Storage logic is implemented as a compound key from generic prefix, compression type, collection name, specific record key and a compressed with configured lib value. Args: collection_name: Collection name for keys namespacing. records: Record ID to object ID and data mapping. created_at: Records created_at datetime. """ cache_mapping = {} for key, value in records.items(): try: cache_mapping[get_full_key(collection_name, key)] = compress( config.REDIS_COMPRESSION_LIB, { StorageField.RECORD_ID: key, StorageField.OBJECT_ID: value[0], StorageField.DATA: value[1], StorageField.CREATED_AT: created_at.isoformat(timespec="milliseconds"), }, ) except Exception as e: capture_exception(e) continue try: await redis_client.mset(cache_mapping) except Exception as e: capture_exception(e) async def save_document(collection_name: str, record_id: str, data: Dict, created_at: datetime): """ Save single record to Redis. Storage logic is implemented as a compound key from generic prefix, compression type, collection name, specific record key and a compressed with configured lib value. Args: collection_name: Collection name for keys namespacing. record_id: Record ID. data: Object data. created_at: Record created_at datetime. """ try: await redis_client.set( get_full_key(collection_name, record_id), compress( config.REDIS_COMPRESSION_LIB, { StorageField.RECORD_ID: record_id, StorageField.DATA: data, StorageField.CREATED_AT: created_at.isoformat(timespec="milliseconds"), }, ), ) except Exception as e: capture_exception(e) def get_full_key(collection_name, key): """ Creates the full redis key from generic prefix, configured compression type and passed specific params Args: collection_name: Collection name for keys namespacing. key: key of the specific record. """ return f"{CACHE_KEY_PREFIX}:{config.REDIS_COMPRESSION_LIB}:{collection_name}:{key}"