"""Interface for fetching assets from s3.""" import os import boto3 from botocore.exceptions import ResponseStreamingError from src.common.exceptions import exceptions s3_client = boto3.client('s3') ENV_CONFIG = { 'dev': { 'bucket': 'dev-orcd-raw-assets', 'prefix': 'fingerprinting/acrcloud/' }, 'qa': { 'bucket': 'qa-orcd-mezzanine-assets', 'prefix': '' }, 'prod': { 'bucket': 'prod-orcd-mezzanine-assets', 'prefix': '' } } # 2 GiB MAX_FILESIZE_BYTES = 1.074e+9 * 2 class FileTooLarge(Exception): """File too large to load into memory.""" def download_asset(filename, extension): """Fetch raw bytes of asset from s3. Args: filename (str): file identifier extension (str): file suffix Returns: bytes """ (bucket, prefix) = _config() obj_key = f'{prefix}{filename}.{extension}' try: # verify file is not too large to be fingerprinted head_s3_response = s3_client.head_object( Bucket=bucket, Key=obj_key ) size_bytes = head_s3_response['ContentLength'] if size_bytes > MAX_FILESIZE_BYTES: raise FileTooLarge() # fetch actual bytes of the file s3_response = s3_client.get_object( Bucket=bucket, Key=obj_key ) return s3_response['Body'].read() except ResponseStreamingError as e: raise exceptions.RetryableException(f'Error retrieving file: {e}') def _config(): """Get config based on value of Environment var.""" env = os.environ['ENVIRONMENT'] config = ENV_CONFIG[env] return ( config['bucket'], config['prefix'] )