"""Lambda support_copy_assets function module.""" import ast import math import boto3 import paramiko from lambdacommon import util import config from logger import get_current_logger from sql import queries from constants import general class InvocationContext: """This is for passing around the invoke context to logger.""" correlation_id = None @util.handle_with_sentry def handler(event, context): """Lambda entry point. Args: event (optional): AWS Lambda event dependent structure with metadata. context (LambdaContext): AWS Lambda context. """ log = get_current_logger(InvocationContext.correlation_id) log.info('Running {}'.format(config.SCRIPT_NAME)) log.info('Event {}'.format(event)) log.info('Environment {}'.format(config.ENVIRONMENT)) if not config.SENTRY_DSN and \ config.ENVIRONMENT in [ config.PROD_ENVIRONMENT, config.QA_ENVIRONMENT]: log.info('Sentry is not configured in {}'.format(config.ENVIRONMENT)) upc_list = ast.literal_eval(event['upcs']) is_downloadable = event.get('is_downloadable', True) asset_type = int(event['asset_type']) physical_location_id = event.get('physical_location_id') if not physical_location_id: physical_location_id = [config.DEFAULT_PHYSICAL_LOCATION_ID] else: physical_location_id = ast.literal_eval(physical_location_id) offset = event.get('offset', general.DEFAULT_OFFSET) limit = event.get('limit', general.DEFAULT_LIMIT) log.info('Target S3 bucket @ {}'.format(config.RAW_BUCKET)) log.info( 'Received upc list as {}, asset type as {} ' 'and physical location id as {} ' . format(upc_list, asset_type, physical_location_id)) log.info('Querying DD db for UPCs: {} and asset_type: {}'.format( upc_list, asset_type)) with util.dd_connection(config.DD_MYSQL_CONN_INFO) as conn: with conn.cursor() as cursor: cursor.execute(queries.DD_SELECT_ASSET_PATH.format( ','.join(map(str, upc_list)), ','.join(map(str, physical_location_id)), asset_type, offset, limit)) direct_delivery_result = cursor.fetchall() if not direct_delivery_result: log.info( 'For UPCs: {} DD query did not fetch any records'.format(upc_list)) return process_direct_delivery_records( direct_delivery_result, physical_location_id, is_downloadable) log.info('Finished {}'.format(config.SCRIPT_NAME)) def transfer_chunk_from_sftp_to_s3( ftp_file, s3_connection, multipart_upload, bucket_name, ftp_file_path, s3_file_path, part_number, chunk_size): """Transfers chunks of a file from SFTP to S3. Args: ftp_file: File from SFTP server s3_connection: s3 connection object multipart_upload: Part of the file to upload bucket_name: Name of s3 bucket ftp_file_path: SFTP Path to file s3_file_path: s3 path part_number: Number of the file part chunk_size: Chunk size for multi-part Returns: part_output object """ chunk = ftp_file.read(int(chunk_size)) part = s3_connection.upload_part( Bucket=bucket_name, Key=s3_file_path, PartNumber=part_number, UploadId=multipart_upload['UploadId'], Body=chunk) part_output = { 'PartNumber': part_number, 'ETag': part['ETag']} return part_output def transfer_file_from_sftp_to_s3( bucket_name, file_name, ftp_file_path, s3_file_path, server_ip, ftp_username, ftp_password, chunk_size, s3_client, is_downloadable=True): """SFTP to S3 File transfer. Args: bucket_name: Name of s3 bucket file_name: File from SFTP server ftp_file_path: SFTP Path to file s3_file_path: s3 file path server_ip: IP of SFTP server ftp_username: Username of SFTP server ftp_password: Password of SFTP server chunk_size: Chunk size for multi-part s3_client: s3 connection object is_downloadable: bool """ log = get_current_logger() ftp_connection = open_sftp_connection( server_ip, config.SFTP_PORT, ftp_username, ftp_password) ftp_file = ftp_connection.file(ftp_file_path, 'r') ftp_file_size = ftp_file._get_size() if not ftp_file_size: log.error('File: {} at SFTP path {} does not exist ' 'or has file size 0'. format( file_name, ftp_file_path)) return if not is_downloadable and ftp_file_size > 0: log.info('File: {} exist at SFTP path {} '. format( file_name, ftp_file_path)) return log.info('File {} transferring for the first time'.format(file_name)) if ftp_file_size <= int(chunk_size): log.info('Transferring complete file: {} from SFTP to S3' ' to path: {}'.format(file_name, s3_file_path)) ftp_file_data = ftp_file.read() s3 = boto3.resource('s3') s3.Object(config.RAW_BUCKET, s3_file_path).put(Body=ftp_file_data) log.info('Completed transferring file: {}'.format(file_name)) ftp_file.close() return log.info( 'Transferring file {} from SFTP to S3 in chunks'.format(file_name) ) chunk_count = int(math.ceil(ftp_file_size / float(chunk_size))) multipart_upload = s3_client.create_multipart_upload( Bucket=bucket_name, Key=s3_file_path) parts = [] for i in range(chunk_count): log.info('Transferring chunk.. {}'.format(i + 1)) part = transfer_chunk_from_sftp_to_s3( ftp_file, s3_client, multipart_upload, bucket_name, ftp_file_path, s3_file_path, i + 1, chunk_size) parts.append(part) log.info('Chunk {} for file {} transferred Successfully!'.format( i + 1, file_name)) part_info = {'Parts': parts} s3_client.complete_multipart_upload( Bucket=bucket_name, Key=s3_file_path, UploadId=multipart_upload['UploadId'], MultipartUpload=part_info) log.info('All chunks for file {} Transferred to bucket'.format(file_name)) ftp_file.close() def open_sftp_connection(ftp_host, ftp_port, ftp_username, ftp_password): """SFTP Connection. Args: ftp_host: SFTP host ftp_port: SFTP port number ftp_username: SFTP username ftp_password: SFTP password Returns: SFTP connection with the host """ log = get_current_logger() log.info('Opening SFTP connection with host : {}'.format(ftp_host)) client = paramiko.SSHClient() client.load_system_host_keys() log.info('Instantiating host with port : {}'.format(ftp_port)) transport = paramiko.Transport(ftp_host, ftp_port) log.info('Completed instantiating host with port') transport.connect(username=ftp_username, password=ftp_password) ftp_connection = paramiko.SFTPClient.from_transport(transport) log.info('FTP Connection for host {} acquired.'.format(ftp_host)) return ftp_connection def process_direct_delivery_records(direct_delivery_result, physical_location_id, is_downloadable=True): """SFTP Connection. Args: direct_delivery_result: Result of DD query physical_location_id: storage drive id is_downloadable: bool """ log = get_current_logger() client = boto3.client( config.S3, region_name=config.REGION_NAME) for records in direct_delivery_result: server_ip = records['ip'] sftp_info = config.AVL_SFTP_INFO if config.REMOTE_STORAGE_IP: server_ip = sftp_info['server_ip'] if records['physical_location_id'] == config.PHYSICAL_LOCATION_ID_NYC: sftp_info = config.NYC_SFTP_INFO if config.REMOTE_STORAGE_IP_FOR_NYC: server_ip = sftp_info['server_ip'] file_path = records['file_path'] file_name = file_path.split('/')[-1] s3_file_path = '{}/{}/{}'.format( sftp_info['folder_name'], records['upc'], file_path) attr_log = 'file attributes. file_path: {}. ' \ 'file_name: {} server_ip: {}'. \ format(file_path, file_name, server_ip) log.info(attr_log) transfer_file_from_sftp_to_s3( config.RAW_BUCKET, file_name, file_path, s3_file_path, server_ip, sftp_info['sftp_username'], sftp_info['sftp_password'], config.CHUNK_SIZE, client, is_downloadable) if __name__ == '__main__': handler(None, None)