"""Main Handler.""" import json from typing import Dict from bulk_metadata_ingester_common.constants.file import ( CSV_DELIMITER, CSV_QUOTECHAR) from bulk_metadata_ingester_common.utils.jitter import jitter from bulk_metadata_ingester_common.utils.logging import get_current_logger from bulk_metadata_ingester_common.utils.release_utils import ( get_release_track_list) import config from config import graphql_gateway from lambdacommon.aws import s3 import pandas as pd def handler(event: Dict, context: object) -> Dict: """Lambda Entrypoint.""" key = event.get('key') bucket = event.get('bucket') correlation_id = event.get('correlation_id') logger = get_current_logger( config.ENVIRONMENT, config.LAMBDA_NAME, logging_level=config.LOGGING_LEVEL, correlation_id=correlation_id) # Jitter calls - Sleep for randomness jitter(logger) graphql_gateway.set_headers( { 'Orchard-User-Id': config.OA_USER, 'Correlation-Id': correlation_id } ) # Log lambda begins message logger.info(f'write_to_csv received: {event}') json_data = get_release_track_list(bucket, key) write_to_csv(bucket, key, json_data, logger) return event def write_to_csv( bucket: str, key: str, json_data: Dict, logger: object): """Write JSON to CSV.""" # Remove the index values from the data to be converted data = list(json_data.values()) flattened_data = [] # Turn a list of lists of dicts into a list of dicts # Each list represents a product and each dict represents a track for product in data: for track in product: flattened_data.append(track) # Convert model data to JSON string json_str = json.dumps(flattened_data) # Convert JSON string to CSV binary data df = pd.read_json(json_str) csv_data = df.to_csv( encoding='utf-8', sep=CSV_DELIMITER, escapechar=CSV_QUOTECHAR, index=None ).encode('utf-8') # Make CSV key with the same name as the JSON key split_key = key.split('/') file_name = split_key[-1].split('.')[0] csv_key = 'generated_csv/' + file_name + '.csv' logger.info(f'Writing CSV file to bucket: {bucket} and key: {csv_key}') # Write JSON in same dir as source key as list s3.resource.Object(bucket, csv_key).put(Body=csv_data)