"""Common S3 DDEX helper functions.""" import json from typing import Dict, List import boto3 from ddex_ingester_common.release_correction.release_correction_diffs import ( ReleaseCorrectionDiffs) def load_rc_json(event: Dict) -> Dict: """Get Release Correction JSON data from S3 file.""" bucket = event.get('bucket') key = event.get('key') s3_client = boto3.client('s3') split_key = f"{key.rsplit('/', 1)[0]}/" json_file_path = f'{split_key}release_corrections.json' release_correction_json = s3_client.get_object( Bucket=bucket, Key=json_file_path )['Body'].read().decode('utf-8') return json.loads(release_correction_json) def load_lambda_rc_json_files(event: Dict) -> Dict: """Get all Release Correction JSON files from S3 file.""" bucket = event.get('bucket') key = event.get('key') split_key = f"{key.rsplit('/', 1)[0]}/" s3_client = boto3.client('s3') s3_res = boto3.resource('s3', 'us-east-1') my_bucket = s3_res.Bucket(bucket) release_correction_json = [] for object_summary in my_bucket.objects.filter(Prefix=split_key): if 'release_correction' in object_summary.key: release_correction_json.append(json.loads(s3_client.get_object( Bucket=bucket, Key=object_summary.key )['Body'].read().decode('utf-8'))) return release_correction_json def write_rc_json(event: object, data: Dict, filename=None): """Write Release Correction to S3.""" bucket = event.get('bucket') key = event.get('key') s3_client = boto3.client('s3') split_key = f"{key.rsplit('/', 1)[0]}/" if filename: json_file_path = f'{split_key}{filename}' else: json_file_path = f'{split_key}release_corrections.json' s3_client.put_object( Bucket=bucket, Key=json_file_path, Body=json.dumps(data).encode(encoding='UTF-8') ) def create_blank_rc_json(event: Dict, filename=None): """Write a blank Release Correction to S3.""" bucket = event.get('bucket') key = event.get('key') s3_client = boto3.client('s3') blank_rc = ReleaseCorrectionDiffs({'changes': []}) split_key = f"{key.rsplit('/', 1)[0]}/" if filename: json_file_path = f'{split_key}{filename}' else: json_file_path = f'{split_key}release_corrections.json' s3_client.put_object( Bucket=bucket, Key=json_file_path, Body=json.dumps(blank_rc.changes).encode(encoding='UTF-8') ) def delete_all_rc_json_files(event: Dict): """Delete a list of all RC JSON files in S3 bucket.""" bucket = event.get('bucket') key = event.get('key') s3_res = boto3.resource('s3', 'us-east-1') my_bucket = s3_res.Bucket(bucket) split_key = f"{key.rsplit('/', 1)[0]}/" keys = [] for object_summary in my_bucket.objects.filter(Prefix=split_key): if 'release_correction' in object_summary.key: keys.append( { 'Key': object_summary.key } ) if keys: delete_objects(bucket, keys) def delete_objects(bucket: str, keys: List[Dict]) -> object: """Delete S3 object list by filename.""" s3_client = boto3.client('s3') response = s3_client.delete_objects( Bucket=bucket, Delete={ 'Objects': keys } ) return response