"""S3 utils.""" from datetime import date, datetime from typing import List import boto3 from botocore import exceptions import config from constants import common as consts import utils class S3Client: """S3 logic container. """ def __init__(self): """Init client. """ self.boto_session = boto3.Session() self.s3_resource = self.boto_session.resource('s3') @staticmethod def get_relative_path(vendor: str, playlist_id: str, folder_type: str, postfix: str = config.CURRENT_RUN_ID) -> str: """Get S3 path without bucket name. Args: vendor (str): Vendor name. playlist_id (str): Playlist ID. folder_type (str): In or out folder. postfix (str): Folder name postfix. Returns: str: S3 path. """ return f'{config.ENVIRONMENT}/{postfix}/{vendor}_{folder_type}/{utils.trim_playlist_id(playlist_id)}/' @staticmethod def get_full_by_relative(relative_path: str) -> str: """Get full path by relative path. Args: relative_path (str): Relative S3 path. Returns: str: Full S3 path. """ return f's3://{config.S3_BUCKET}/{relative_path}' @staticmethod def get_relative_by_full(full_path: str) -> str: """Get relative path by full path. Args: full_path (str): Full S3 path. Returns: str: Relative S3 path. """ return full_path.replace(f's3://{config.S3_BUCKET}/', '') @staticmethod def get_full_path(vendor: str, playlist_id: str, folder_type: str, postfix: str = config.CURRENT_RUN_ID) -> str: """Get full S3 path. Args: vendor (str): Vendor name. playlist_id (str): Playlist ID. folder_type (str): In or out folder. postfix (str): Folder name postfix. Returns: str: S3 path. """ return S3Client.get_full_by_relative(S3Client.get_relative_path(vendor, playlist_id, folder_type, postfix)) @staticmethod def get_temp_path(postfix: str = config.CURRENT_RUN_ID): """Get temp S3 path. Args: postfix (str): Folder name postfix. Returns: str: S3 path. """ return f's3://{config.S3_BUCKET}/{config.ENVIRONMENT}/{postfix}/temp/' def delete_folder_files(self, folder_path: str): """Delete all files in folder. Args: folder_path (str): Path prefix. """ bucket = self.s3_resource.Bucket(config.S3_BUCKET) bucket.objects.filter(Prefix=folder_path).delete() def get_s3_folders(self, base_folder: str) -> List[str]: """Get all s3 folders. Args: base_folder (str): Base s3 folder. Returns: List[str]: S3 folders. """ bucket = self.s3_resource.Bucket(name=config.S3_BUCKET) entries = [] for obj in bucket.objects.filter(Prefix=base_folder): folder = obj.key.replace(base_folder, '').split('/')[1] if folder: entries.append(folder) return entries def delete_s3_files_except_csv(self, s3_path: str): """Delete all extra files from S3 to prevent load to MySQL errors. Args: s3_path (str): S3 folder path. """ bucket = self.s3_resource.Bucket(config.S3_BUCKET) for obj in bucket.objects.filter(Prefix=s3_path): if not obj.key.endswith('.csv'): obj.delete() def delete_temp(self): """Delete temp folder. """ self.delete_folder_files(self.get_relative_by_full(self.get_temp_path())) @staticmethod def _get_key(vendor: str, key: str) -> str: """Compose a key. Args: vendor (str): Vendor name. key (str): Key. Returns: str: Complete key constructed by concatenated prefix + key. """ return f'{config.ENVIRONMENT}/{consts.APP_NAME}/{vendor}/{key}' def get_last_date(self, vendor: str, default_value: date or None = None) -> date or None: """Get value from S3. Args: vendor (str): Vendor name. default_value (date or None): Return if key is not found. Returns: date or None: Value or None if not found. """ try: obj = self.s3_resource.Object(config.S3_BUCKET, self._get_key(vendor, consts.KEY_LAST_UPDATE_DATE)) value = obj.get()['Body'].read().decode('utf-8') return datetime.strptime(value, consts.DEFAULT_DATE_FORMAT).date() except exceptions.ClientError as e: error_code = e.response['Error']['Code'] if error_code == 'NoSuchKey': return default_value else: raise def set_last_date(self, vendor: str, value: date): """Set value to S3. Args: vendor (str): Vendor name. value (str): Value to save. """ obj = self.s3_resource.Object(config.S3_BUCKET, self._get_key(vendor, consts.KEY_LAST_UPDATE_DATE)) obj.put(Body=value.strftime(consts.DEFAULT_DATE_FORMAT))