"""Lambda split_sales_sheet function module.""" import csv import os.path from urllib.parse import unquote_plus import boto3 from common_config import logger from constants import general from smart_open import smart_open from warmer_util import catch_warmer_event @catch_warmer_event() def handler(event, context): """Lambda entry point.""" try: for record in event['Records']: bucket = record['s3']['bucket']['name'] key = unquote_plus(record['s3']['object']['key']) logger.info('Bucket: ' + bucket + ' Key: ' + key) s3 = boto3.resource('s3') obj = s3.Object(bucket, unquote_plus(key)) lines = obj.get()['Body'].read().decode().splitlines(True) reader = csv.reader(lines) logger.info('file read complete') count = 0 file_count = 1 inner_list = [] header = [] written_row_count = 0 logger.info('writing to output file : ') chunk_size = get_chunk_size() for line in reader: if count == 0: header = line else: inner_list.append(line) # if count % 40000 == 0: if count % chunk_size == 0: written_row_count = count write_to_s3( bucket, key, file_count, header, inner_list) file_count += 1 inner_list.clear() count += 1 if count > written_row_count: write_to_s3(bucket, key, file_count, header, inner_list) except Exception as e: logger.exception(str(e)) raise def write_to_s3(bucket, key, file_count, header, data): """Write updated CSV with filename and filesize, into new file. Args: bucket (string): S3 bucket name key (string): S3 key name file_count (int): file count to identify split file header (list): header of csv file data (list): data of csv file without header """ source_url = 's3://{bucket}/{path}/{file_count}_{filename}'.format( bucket=bucket, path=general.SPLIT_SOUNDEXCHANGE_STMT_OUTPUT_PATH, file_count=file_count, filename=os.path.basename(key)) with smart_open(source_url, 'w') as fileobj: writer = csv.writer(fileobj, quoting=csv.QUOTE_ALL) logger.info('writing to file : ' + source_url) writer.writerow(header) writer.writerows(data) def get_chunk_size(): """Get chunk size. Returns: int: chunk size """ return general.CHUNK_SIZE