"""Lambda function module.""" import logging import boto3 import smart_open import config # noqa import s3 # noqa import util # noqa logger = logging.getLogger() logger.setLevel(logging.INFO) def split( source_bucket, source_key, new_bucket, new_key, chunk_max_line_count=config.CHUNK_MAX_LINE_COUNT): """Split source file and put chunks to the destination path. Args: source_bucket (str): source S3 bucket. source_key (str): source S3 key. new_bucket (str): new S3 bucket. new_key (str): new S3 key. chunk_max_line_count (int): max amount of lines in chunk. """ source_url = 's3://{bucket}/{path}'.format( bucket=source_bucket, path=source_key) chunk = [] chunk_number = 0 for line in smart_open.smart_open(source_url): chunk.append(line) if len(chunk) == chunk_max_line_count: s3.upload_key( new_bucket, new_key + '_chunk_{}'.format(chunk_number), b''.join(chunk)) chunk_number += 1 chunk = [] if chunk: s3.upload_key( new_bucket, new_key + '_chunk_{}'.format(chunk_number), b''.join(chunk)) def _need_split(bucket, key): """Check if file size is too big. Args: bucket (str): S3 bucket. key (str): S3 key. Returns: Boolean: True if file size is more than max allowable. """ s3_client = boto3.client('s3') response = s3_client.head_object(Bucket=bucket, Key=key) return response['ContentLength'] > config.MAX_FILE_SIZE def _get_file_type(key_name): """Detect file type by file name (should be one of SoS data type). Args: key_name (str): file name. Returns: str: file type (on of config.FILE_TYPES). Raises: ValueError: if incorrect file was passed. """ for file_type in config.FILE_TYPES: if file_type in key_name: return file_type raise ValueError('Incorrect file name or type.') def _identify_retailer(key): """Extract retailer name from S3 key. Args: key(str): S3 key. Returns: str: Retailer name (for further use in destination S3 path of chunk). Raises: ValueError: If incorrect key was passed. """ if 'apple-music-sos' in key: return 'apple-music-sos' elif 'spotify-sos' in key: return 'spotify-sos' else: raise ValueError('Can not identify retailer by S3 key {}'.format(key)) def process_key(bucket, key): """Process passed key. Split source key if it is to big and put chunks to destination directory. If key size acceptable move it to destination directory. Args: bucket (str): S3 bucket. key (str): S3 key. """ file_type = _get_file_type(key) file_name = key.split('/')[-1] retailer = _identify_retailer(key) new_key_path = ''.join( [config.TARGET_S3_PATH['suffix'].format( retailer=retailer), file_type, '/', file_name]) if _need_split(bucket, key): split(bucket, key, bucket, new_key_path) else: s3.copy_key(bucket, key, bucket, new_key_path) def handler(event, context): """Lambda entry point. Args: event (dict): AWS Lambda event. context (object): AWS Lambda context. """ bucket, key = util.extract_triggered_key(event) logger.info('event with bucket: {} and key {}'.format(bucket, key)) if util.object_is_file(key): process_key(bucket, key) logger.info( 'was successfully processed bucket: {} and key {}'.format( bucket, key)) s3.delete_key(bucket, key) logger.info( 'was successfully deleted bucket: {} and key {}'.format( bucket, key))