import os import re from typing import Any, Dict, List, Optional from urllib import parse import boto3 from botocore.paginate import PageIterator from pynamodb.pagination import ResultIterator from slz_dynamodb.const import ContentStatus as ContentStatusEnum from structlog import getLogger from slz_api_service.const import ARCHIVE_BUCKET, DEFAULT_PAGE_SIZE, USE_S3 from slz_api_service.errors import Codes, S3QueryError from slz_api_service.v1.data_models.dynamo_db import DyContentStatus from slz_api_service.v1.utils import QueryUtils from slz_api_service.v1.view_models.query_params import QueryParams LOG = getLogger(__name__) class ContentStatusViewModel: def __init__(self, params: QueryParams, use_s3=USE_S3): self.params = params self.use_s3 = use_s3 self.client = boto3.client('dynamodb') self.s3 = boto3.client('s3') self._s3_delimiter = ',' @property def uow_id(self) -> str: """Constructs a UnitOfWork ID from the parameters in the request Returns: str: constructed ``UnitOfWorkCode`` """ whitelist = {'dsp', 'report_date', 'report_licensor', 'report_type', 'version'} data = {k: v for k, v in self.params.to_dict().items() if k in whitelist} return QueryUtils.get_uow_id(**data) def get_s3_uri(self, filename: str) -> str: """Creates an S3 URI in the format ``s3://bucket/path/to/object`` Args: filename: last part of a filepath """ protocol = 's3' bucket_name = ARCHIVE_BUCKET dsp = self.params.dsp report_type: str = self.params.report_type if dsp == 'apple': # remove ``am`` prefix report_type = re.sub(r'^(am)', '', report_type, flags=re.IGNORECASE) path = os.path.join( bucket_name, dsp, report_type.lower(), self.params.version, f"report_date={self.params.report_date}", f"report_licensor={self.params.report_licensor}", filename, ) return f'{protocol}://{path}' def get_s3_prefix(self) -> str: dsp = self.params.dsp report_type: str = self.params.report_type path = os.path.join( dsp, report_type, self.params.version, f"report_date={self.params.report_date}", f"report_licensor={self.params.report_licensor}", ) return f'{path}/' def get_context(self, context: str): """Sanitizes a passed ``context`` value Args: context: value from the context field in the data """ if self.params.dsp == 'apple': # only return the YYY from XXX::YYY format context = context.split('::')[-1] # Nullify context if it is the same as the report_type if self.params.report_type == context: context = None return context def get_files_by_report_date(self) -> Dict[str, Any]: """Masks the ``NextToken`` DynamoDB pagination flow to use a standard ``offset`` field. Returns: dictionary response body with data from the database """ limit: int = self.params.limit offset: int = self.params.offset limit_offset = limit + offset if self.use_s3: results = self.query_s3() mapped_items = self.map_data_s3(results.get('Contents', [])) # If client requested less items than we found, including offset, don't continue continue_querying = limit_offset < len(mapped_items) # If we have a paginated result from s3, get the other pages of results while results.get('NextToken', False) and continue_querying: pagination_config = self._get_pagination_config() result = self.query_s3(pagination_config) mapped_items.extend(self.map_data_s3(result.get('Items', []))) continue_querying = limit_offset < len(mapped_items) else: result = self.query_dynamodb() mapped_items = self.map_data_dynamodb(result) returned_items = mapped_items[offset:limit_offset] response_body = { 'count': len(returned_items), 'items': returned_items, } return response_body @staticmethod def _get_pagination_config(start_token: Optional[str] = None) -> dict: """S3 Pagination Specific ``StartingToken`` is a token to specify where to start paginating. This is the ``NextToken`` from a previous response. :return: S3 config for ``PaginationConfig`` """ config = { 'MaxItems': DEFAULT_PAGE_SIZE, 'PageSize': DEFAULT_PAGE_SIZE, } if start_token: config.update({'StartingToken': start_token}) return config def query_s3(self, pagination_config: Optional[dict] = None) -> Dict[str, Any]: """DynamoDB Specific Args: pagination_config: optional S3 PaginationConfig object parameter """ if pagination_config is None: pagination_config = self._get_pagination_config() try: paginator = self.s3.get_paginator('list_objects_v2') iterator: PageIterator = paginator.paginate( Bucket=ARCHIVE_BUCKET, Delimiter=self._s3_delimiter, EncodingType='url', Prefix=self.get_s3_prefix(), FetchOwner=False, PaginationConfig=pagination_config, ) return iterator.build_full_result() except Exception as err: LOG.exception(str(err)) raise S3QueryError({'code': Codes.s3_query.value, 'description': str(err)}) def map_data_s3(self, data: iter) -> List[Dict[str, Any]]: """Sanitizes the data objects for the API response from S3 data source Args: data: iterable of query results """ out = [] for row in data: key = row.get('Key', '') uri = f's3://{ARCHIVE_BUCKET}/{parse.unquote(key)}' out.append({ 'file_size_bytes': row.get('Size'), 'uri': uri, }) return out def query_dynamodb(self) -> ResultIterator: model = DyContentStatus filter_condition = model.content_status.__eq__(ContentStatusEnum.COMPLETE.value) results = model.query(self.uow_id, filter_condition=filter_condition, consistent_read=True) return results def map_data_dynamodb(self, data) -> List[Dict[str, Any]]: """Sanitizes the data objects for the API response from DynamoDB data source Args: data: iterable of query results """ out = [] row: DyContentStatus for row in data: content_name = ( row.original_content_name if row.original_content_name else row.content_name) out.append({ 'file_size_bytes': row.file_size, 'uri': self.get_s3_uri(content_name), }) return out