"""A class with methods to work with sme buckets.""" __all__ = [ 'SMES3Adapter', ] import os import re import tempfile from typing import Dict, List, Optional, Union import boto3 from boto3.s3.transfer import TransferConfig from botocore.exceptions import ClientError from garcon import task from garcon.activity import ActivityWorker from garcon_contrib.aws.utils import garcon_s3 class SMES3Adapter: """The class allows to work with sme buckets.""" _PAGINATOR_PAGE_SIZE = 1000 _SME_AWS_ACCESS_KEY_ID_SECRET_NAME = 'SME_AWS_ACCESS_KEY_ID' _SME_AWS_SECRET_ACCESS_KEY_SECRET_NAME = 'SME_AWS_SECRET_ACCESS_KEY' _TASKS_TIMEOUT = 1000 def __init__( self, sme_access_key_id_name: Optional[str] = None, sme_secret_access_key_name: Optional[str] = None, ) -> None: """Construct the adapter object. Args: sme_access_key_id_name: secret name with SME access_key_id. If it's not provided then default one will be used. sme_secret_access_key_name: secret name with SME secret_access_key. If it's not provided then default one will be used. """ # Init The orchard S3 client self.s3_orcd_client = boto3.client('s3') # Init SME S3 client self._sme_access_key_id_name = ( sme_access_key_id_name or self._SME_AWS_ACCESS_KEY_ID_SECRET_NAME ) self._sme_access_key_id = os.environ.get(self._sme_access_key_id_name) self._sme_secret_access_key_name = ( sme_secret_access_key_name or self._SME_AWS_SECRET_ACCESS_KEY_SECRET_NAME ) self._sme_secret_access_key = os.environ.get( self._sme_secret_access_key_name) self.s3_sme_client = boto3.client( 's3', aws_access_key_id=self._sme_access_key_id, aws_secret_access_key=self._sme_secret_access_key, ) @task.decorate(timeout=_TASKS_TIMEOUT) def copy_files( self, activity: ActivityWorker, s3_archive_path: str, s3_download_path: str, source_files_dict: Dict[str, List[Dict[str, str]]], ) -> Dict[str, Dict[str, List[Dict[str, Union[bool, str]]]]]: """Copy files from SME bucket to The Orchard bucket. Args: activity: an activity object. s3_archive_path: The Orchard path with bucket and prefix to copy blobs to. s3_download_path: SME path to copy blobs from. source_files_dict: a structure contains blob names to copy. """ files = [] for blob in source_files_dict['files']: success_flag = True blob_name = blob['file_name'] try: self.copy_blob_from_sme_to_the_orchard( blob_name=blob_name, s3_archive_path=s3_archive_path, s3_source_path=s3_download_path, ) except ClientError as e: success_flag = False activity.logger.error( # noqa f'blob {blob_name} has not been copied because of {e}' ) files.append( { 'file_name': blob_name, 'file_size': blob['file_size'], 'file_path': f'{s3_download_path}/{blob_name}', 'found': success_flag }, ) return {'source_files_dict': {'files': files}} def copy_blob_from_sme_to_the_orchard( self, blob_name: str, s3_archive_path: str, s3_source_path: str, ) -> None: """Copy the blob from SME bucket to The Orchard one. Args: blob_name: name of the blob to be copied. s3_archive_path: full prefix of blob (s3://...) to be copied in The Orchard bucket. s3_source_path: full prefix of blob (s3://...) to be copied in SME bucket. """ ( source_bucket_name, source_blob_prefix, ) = garcon_s3.extract_bucket_path(s3_source_path) source_key = f'{source_blob_prefix}/{blob_name}' ( archive_bucket_name, archive_blob_prefix, ) = garcon_s3.extract_bucket_path(s3_archive_path) archive_key = f'{archive_blob_prefix}/{blob_name}' with tempfile.NamedTemporaryFile('wb') as file: # 1. Download the blob from SME bucket self.s3_sme_client.download_fileobj( Bucket=source_bucket_name, Key=source_key, Fileobj=file, Config=TransferConfig(), ) file.flush() # 2. Upload the blob to The Orchard bucket self.s3_orcd_client.upload_file( Filename=file.name, Bucket=archive_bucket_name, Key=archive_key, Config=TransferConfig(), ) def get_blobs_list_by_prefix(self, prefix: str, wildcard: str) -> List[ Dict[str, str], ]: """List all blob with given prefix. Args: prefix: prefix for blobs to search. wildcard: a blob name pattern. Returns: list of blob names and their sizes. """ bucket, bucket_path = garcon_s3.extract_bucket_path(prefix) paginator = self.s3_sme_client.get_paginator('list_objects') blobs_parameters = { 'Bucket': bucket, 'Prefix': bucket_path, 'PaginationConfig': { 'PageSize': self._PAGINATOR_PAGE_SIZE, } } pattern = re.compile(wildcard) return [ { 'file_name': key['Key'].split('/')[-1], 'file_size': key['Size'], } for page in paginator.paginate(**blobs_parameters) for key in page.get('Contents', []) if pattern.match(key['Key']) ] @task.decorate(timeout=_TASKS_TIMEOUT) def source_files( self, activity: ActivityWorker, s3_bucket: str, s3_path: str, file_pattern: str, ) -> Dict[str, Dict[str, List[Dict[str, str]]]]: """List all blobs in the bucket with given prefix and name pattern. Args: activity: an activity object. s3_bucket: source bucket name. s3_path: target blobs prefix. file_pattern: wildcard for target blobs names. Returns: a structure contains info about found blob names and their sizes. """ s3_prefix = 's3://{}/{}'.format(s3_bucket, s3_path) files = self.get_blobs_list_by_prefix( prefix=s3_prefix, wildcard=file_pattern, ) activity.logger.info( # noqa f'The next files were found in {s3_prefix}: {files}' ) return {'source_files_dict': {'files': files}}