"""Entrypoint for lambda function.""" from urllib.parse import unquote_plus import re import csv import boto3 import src.query as query from logger import get_current_logger from owsrequest import request as requests from lambdacommon import util import config class InvocationContext: """This is for passing around the invoke context to logger.""" correlation_id = None def handler(event, context): """Lambda entry point. Args: event (optional): AWS Lambda event dependent structure with metadata. context (LambdaContext): AWS Lambda context. """ InvocationContext.correlation_id = context.aws_request_id log = get_current_logger(InvocationContext.correlation_id) log.info('Running "{}"'.format(config.SCRIPT_NAME)) log.info('Event "{}"'.format(event)) event_type = event['Records'][0]['eventName'] if event_type in ['ObjectCreated:Put']: file_name = event['Records'][0]['s3']['object']['key'] file_bucket = event['Records'][0]['s3']['bucket']['name'] file_name = unquote_plus(file_name) if not file_name.endswith('.csv'): return log.error( 'Uploaded file "{}" is not in .CSV format'.format(file_name)) process_csv(file_bucket, file_name) log.info('Finished "{}"'.format(config.SCRIPT_NAME)) def process_csv(bucket, filename): """Loop over csv and build data for sql. Args: bucket (str): The name of the bucket to pull from filename (str): The filename we want to loop over """ # To read from S3 log = get_current_logger() s3_client = boto3.client('s3') file_obj = s3_client.get_object(Bucket=bucket, Key=filename) csv_data = file_obj['Body'].read().decode().splitlines() log.info('Raw data from CSV "{}"'.format(csv_data)) data = [] reader = csv.DictReader(csv_data) fieldnames = reader.fieldnames required_fields = [ 'album_upc', 'album_ean', 'album_live_from', 'date_inserted', 'countries', 'album_uri'] result = all(elem in fieldnames for elem in required_fields) if not result: return log.error( 'The csv file is either not comma delimited or' ' it does not contain required fields: album_upc, album_ean,' ' album_live_from, date_inserted, countries, album_uri') index = 1 for row in csv.DictReader(csv_data): if row['album_upc'].lower() == 'none': row['album_upc'] = row['album_ean'] # Null-like CSV inputs become SQL NULL. Under the cluster's empty # sql_mode, anything else that isn't a real date gets silently # stored as '0000-00-00 00:00:00'. live_from = (row['album_live_from'] or '').strip() if live_from.lower() in ( '', 'none', 'null', '0000-00-00', '0000-00-00 00:00:00', ): row['album_live_from'] = None if not (row['album_upc'].strip() or row['album_upc'] .lower() == 'none'): log.error( 'UPC value is empty at row {} in file {}'.format( index + 1, filename)) index += 1 continue data.append({ 'upc': row['album_upc'], 'ean': row['album_ean'], 'date_inserted': row['date_inserted'], 'countries': (row['countries']).replace(' ', ','), 'live_from': row['album_live_from'], 'album_uri': re.sub('spotify:album:', '', row['album_uri']), }) index += 1 if len(data) < 1: return log.error('Spreadsheet {} is empty'.format(filename)) backfill_data_in_tables(data) def backfill_data_in_tables(data): """Insert or update rows in table. Args: data (list(dict)): The data we are insert or updating """ log = get_current_logger() log.info('Begin records backfill to {}'.format( config.OWS_STORE_AVAILABILITY_DB_CREDENTIALS['database'])) with util.mysql_connection( config.OWS_STORE_AVAILABILITY_DB_CREDENTIALS['host'], config.OWS_STORE_AVAILABILITY_DB_CREDENTIALS['user'], config.OWS_STORE_AVAILABILITY_DB_CREDENTIALS['password'], config.OWS_STORE_AVAILABILITY_DB_CREDENTIALS['database'] ) as rds_db_connection: for single_record in data: product_data = get_product_data(single_record) log.info('ows-product response "{}"'.format(product_data)) if product_data: # merge the product data list with csv row list data single_record.update(product_data) log.info('Record row to be added "{}"'.format(single_record)) insert_record_in_product_and_product_in_store( single_record, rds_db_connection) def get_product_data(data): """Fetch records for given upc and return to the calling function. Args: data (list(dict)): The data we are insert or updating Returns: product data dict """ log = get_current_logger() product_response = requests.process( config.SCRIPT_NAME, config.ENVIRONMENT, 'GET', config.SERVICE_NAME_OWS_PRODUCT, '/product/upc/{}'.format(data['upc']), json={'blocked': True}) if not product_response: return log.error( 'Received empty response from ows-product for UPC {}'. format(data['upc'])) return product_response.json() def insert_record_in_product_and_product_in_store(data, rds_db_connection): """Fetch records for given upc and return to the calling function. Args: data (list(dict)): The data we are insert or updating rds_db_connection: pymysql.connections.Connection """ log = get_current_logger() log.info('Begin transaction to insert record in tables') with rds_db_connection.cursor() as cursor: cursor.execute(query.INSERT_INTO_PRODUCT, data) cursor.execute(query.INSERT_INTO_PRODUCT_IN_STORE, data) rds_db_connection.commit() log.info('Transaction completed') if __name__ == '__main__': handler(None, None)