"""Boto3 s3 file operations.""" from collections import defaultdict from typing import Any from botocore.exceptions import ClientError from assets.connectors import s3 def delete_s3_files(files: list[dict[str, str]]) -> dict[str, dict[str, Any]]: """Delete s3 files. Args: files (list): List of dictionaries with file info. Returns: dict: Keys of response are bucket names. For each bucket we have 2 lists of objects - `succeeded` and `failed` and optional field `error_code`. Each object has field `Key` e. g. filename. For objects in `failed` list we may have custom errors specified for in each object in fields `Message` and `Code`. Or we may have an error common for all failed files specified in `error_code` field. Example: { 'bucket_1': { 'succeeded': [ {'Key': 'bucket_1_file_1'}, {'Key': 'bucket_1_file_2'}], 'failed': [] }, 'bucket_2': { 'succeeded': [{'Key': 'bucket_2_file_1'}], 'failed': [{ 'Key': 'bucket_2_file_2', 'Message': 'Access Denied', 'Code': 'AccessDenied'}] }, 'bucket_3': { 'failed': [{'Key': 'bucket_2_file_1'}], 'succeeded': [], 'error_code': 'some_critical_error' } } """ s3_client = s3.get_s3_client() bucket_mapping = defaultdict(list) delete_status: dict[str, Any] = {} for file_info in files: bucket = file_info["bucket"] key = file_info["file"] bucket_mapping[bucket].append(key) for bucket, bucket_files in bucket_mapping.items(): try: delete_response = s3_client.delete_objects( Bucket=bucket, Delete={"Objects": [{"Key": file} for file in bucket_files]}, ) delete_status[bucket] = { "succeeded": delete_response.get("Deleted", []), "failed": delete_response.get("Errors", []), } except ClientError as e: delete_status[bucket] = { "succeeded": [], "failed": [{"Key": file} for file in bucket_files], "error_code": e.response["Error"]["Code"], } except Exception as e: delete_status[bucket] = { "succeeded": [], "failed": [{"Key": file} for file in bucket_files], "error_code": str(e), } return delete_status def check_s3_file_exists(bucket_name: str, file_key: str) -> bool: """Check if the file exists in the s3 bucket. Args: bucket_name (str): name of bucket where file is stored. file_key (str): path to file for which url should be generated. Returns: bool """ try: s3_client = s3.get_s3_client() file_response = s3_client.head_object(Bucket=bucket_name, Key=file_key) status = file_response["ResponseMetadata"]["HTTPStatusCode"] return status == 200 except ClientError: return False def create_presigned_url(bucket: str, key: str, expiration: int = 3600) -> str: """Generate a presigned URL to share an S3 object. Args: bucket (str): The bucket where the S3 object is located in. key (str): The key where the S3 object is located at. expiration (int, optional): The expiration time in seconds. Defaults to 3600. Returns: str: The presigned URL. """ return s3.get_s3_client().generate_presigned_url( "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=expiration )