import asyncio import inspect import math from typing import Any, Callable, List from server import config from server.constants.delphi.group_by import GROUP_BY_DATE from server.utils.args.getter_setter import get_arg, set_arg def _get_page( offset: int, limit: int, offset_field: str, limit_field: str, args_spec: inspect.FullArgSpec, f: Callable, args: tuple, kwargs: dict, ) -> Any: """Get single page. Args: offset: Page offset. limit: Page limit. offset_field: Offset arg name. limit_field: Limit arg name. args_spec: Function args spec. f: Function. args: Function args. kwargs: Function kwargs. """ args, kwargs = set_arg(args, kwargs, args_spec, offset_field, offset) args, kwargs = set_arg(args, kwargs, args_spec, limit_field, limit) return f(*args, **kwargs) def delphi_pagination( page_size: int = config.DEFAULT_DELPHI_PAGE_SIZE, offset_field: str = "offset", limit_field: str = "limit", response_items_node: str = "items", response_count_node: str = "count", in_parallel: bool = False, ): """Get full response data from pagination. Args: limit_field: Limit arg name. offset_field: Offset arg name. response_items_node: Response items node. page_size: Page size. response_count_node: Name of response key with original number of returned items. in_parallel: Make several requests in parallel for group by date case only (some requests per month). """ def inner(f: Callable): args_spec = inspect.getfullargspec(f) async def wrapped(*args, **kwargs) -> List: if get_arg(args, kwargs, args_spec, offset_field) or get_arg(args, kwargs, args_spec, limit_field): response = await f(*args, **kwargs) return response[response_items_node] if response_items_node else response items = [] offset = 0 requests_count = 1 if ( in_parallel and config.DELPHI_MAX_REQUESTS_COUNT > 1 and GROUP_BY_DATE in (get_arg(args, kwargs, args_spec, "group_by") or []) ): start_date = get_arg(args, kwargs, args_spec, "start_date") end_date = get_arg(args, kwargs, args_spec, "end_date") if start_date and end_date and end_date > start_date: requests_count = min( config.DELPHI_MAX_REQUESTS_COUNT, math.ceil((end_date - start_date).days / config.DELPHI_REQUESTS_INTERVAL_SIZE) * config.DELPHI_REQUESTS_PER_INTERVAL, ) while True: tasks = [ _get_page(offset + i * page_size, page_size, offset_field, limit_field, args_spec, f, args, kwargs) for i in range(requests_count) ] responses = await asyncio.gather(*tasks) for response in responses: response_items = response[response_items_node] if response_items_node else response items = items + response_items response_len = response[response_count_node] if response_count_node else len(response_items) if response_len < page_size: return items offset += page_size * requests_count wrapped.__signature__ = inspect.signature(f) return wrapped return inner