"""S3 utils.""" import boto3 import config s3_resource = boto3.resource("s3", region_name=config.AWS_DEFAULT_REGION) def get_relative_path(table_type: str, folder_type: str, postfix: str = config.CURRENT_RUN_ID) -> str: """Get S3 path without bucket name. Args: table_type: Table type. folder_type: In or out folder. postfix: Folder name postfix. Returns: str: S3 path. """ return f"{config.ENVIRONMENT}/hh/{postfix}/{table_type}/{folder_type}/" def get_full_by_relative(relative_path: str) -> str: """Get full path by relative path. Args: relative_path: Relative S3 path. Returns: str: Full S3 path. """ return f"s3://{config.S3_BUCKET}/{relative_path}" def get_relative_by_full(full_path: str) -> str: """Get relative path by full path. Args: full_path: Full S3 path. Returns: str: Relative S3 path. """ return full_path.replace(f"s3://{config.S3_BUCKET}/", "") def get_full_path(table_type: str, folder_type: str, postfix: str = config.CURRENT_RUN_ID) -> str: """Get full S3 path. Args: table_type: Table type. folder_type: In or out folder. postfix: Folder name postfix. Returns: str: S3 path. """ return get_full_by_relative(get_relative_path(table_type, folder_type, postfix)) def get_temp_path(postfix: str = config.CURRENT_RUN_ID): """Get temp S3 path. Args: postfix: Folder name postfix. Returns: str: S3 path. """ return f"s3://{config.S3_BUCKET}/{config.ENVIRONMENT}/hh/{postfix}/temp/" def delete_folder_files(folder_path: str): """Delete all files in folder. Args: folder_path: Path prefix. """ bucket = s3_resource.Bucket(config.S3_BUCKET) bucket.objects.filter(Prefix=folder_path).delete() def delete_s3_files_except_csv(s3_path: str): """Delete all extra files from S3 to prevent load to MySQL errors. Args: s3_path: S3 folder path. """ bucket = s3_resource.Bucket(config.S3_BUCKET) for obj in bucket.objects.filter(Prefix=s3_path): if not obj.key.endswith(".csv"): obj.delete() def delete_temp(): """Delete temp folder. """ delete_folder_files(get_relative_by_full(get_temp_path()))