"""S3 common functions.""" import boto3 import botocore from config import logger client = boto3.client('s3') resource = boto3.resource('s3') def put_object(bucket, key, data, **kwargs): """Put object to S3. Args: bucket (str): target S3 bucket. key (str): target S3 key. data (bytes): file data. kwargs (dict): extra arguments. """ return client.put_object(Bucket=bucket, Key=key, Body=data, **kwargs) def get_object(bucket, key): """Get S3 object. Args: bucket (str): target S3 bucket. key (str): target S3 key. """ return client.get_object(Bucket=bucket, Key=key) def object_exists(bucket, key): """Check if S3 object exists. Args: bucket (str): target S3 bucket. key (str): target S3 key. """ try: client.head_object(Bucket=bucket, Key=key) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == '404': return False else: raise else: return True def head_object(bucket, key): """Retrieve object metadata without object downloading. Args: bucket (str): target S3 bucket. key (str): target object key. Returns: dict: Response from S3. """ return client.head_object(Bucket=bucket, Key=key) def download_file_object(bucket, key, file_object, **kwargs): """Download s3 object to file-like buffer. Args: bucket (str): target S3 bucket. key (str): target S3 key. file_object (io.BytesIO): file data. kwargs (dict): extra arguments. """ return client.download_fileobj( Bucket=bucket, Key=key, Fileobj=file_object, **kwargs) def upload_file_objects(file_descriptors, bucket, content_type='binary/octet'): """Upload file objects to s3. Args: file_descriptors (list): List of file descriptors. bucket (str): target S3 bucket. content_type (str): Content Type """ try: for descriptor in file_descriptors: descriptor['output_image_buffer'].seek(0) put_object( bucket=bucket, key=descriptor['key'], data=descriptor['output_image_buffer'], ContentType=content_type ) except Exception as e: logger.exception(str(e)) raise return True def create_presigned_url( bucket, key, client_method='get_object', expires_in=86400): """Generate a presigned url. Args: bucket (str): Bucket name. key (str): Existing S3 object key. client_method (str): Client method for operation by result URL. expires_in (int): The number of seconds the presigned url is valid for. By default it expires in 24 hours (86400 seconds). Returns: str: Presigned url. """ return client.generate_presigned_url( ClientMethod=client_method, ExpiresIn=expires_in, Params={ 'Bucket': bucket, 'Key': key } ) def copy_object(source_bucket, source_key, destination_bucket, destination_key, destination_metadata): """Copy S3 object from one bucket to another. Args: source_bucket (str): Bucket name for source S3 object. source_key (str): Key name of the source S3 object. destination_bucket (str):Bucket name for destination S3 object. destination_key (str): Key name of the destination S3 object. destination_metadata (dict): Metadata for destination object. Returns: dict: Response from S3. """ return client.copy_object( Bucket=destination_bucket, Key=destination_key, CopySource={'Bucket': source_bucket, 'Key': source_key}, Metadata=destination_metadata)