"""Lambda function which processes a file uploaded on S3 by Looker. Then it uploads processed file on the GfK's FTP server. Basically it removes leading zeros from the first column, and wraps it with double quotes. """ import csv import datetime import os import tempfile from urllib.parse import unquote_plus import boto3 import config import pysftp import sentry_sdk from config import FTP_CREDENTIALS from config import get_ from sentry_sdk import capture_exception from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration FILENAME_TO_UPLOAD = 'Releases_Deliverable_to_GfK.txt' def _send_sns(): """Email data-squad@theorchard.com.""" try: client = boto3.client('sns', region_name='us-east-1') client.publish( TopicArn='arn:aws:sns:us-east-1:437795906767' ':prod-swf-feed-ingestion', Message='Releases were successfuly delivered to Gfk ' 'via https://theorchard.looker.com/looks/13413 and ' 'https://github.com/theorchard/lambda-looker', Subject='{} GfK delivery'.format( datetime.date.today().strftime('%Y-%m-%d'))) except Exception as e: print(str(e)) # TODO: remove after successful message delivery def _upload_to_ftp(tmpfile): """Upload a file to IN directory on the GfK's FTP server.""" cnopts = pysftp.CnOpts() cnopts.hostkeys = None with pysftp.Connection(FTP_CREDENTIALS['host'], username=get_('GFK_FTP_USERNAME'), password=get_('GFK_FTP_PASSWORD'), cnopts=cnopts) as sftp: sftp.put(tmpfile.name, '{}/{}'.format( FTP_CREDENTIALS['remote_ftp_dir'], FILENAME_TO_UPLOAD)) sftp.close() _send_sns() def handler(event, context): """Get a file from S3, process it, and upload to FTP.""" if get_('SENTRY_DSN'): sentry_sdk.init( dsn=get_('SENTRY_DSN'), environment=config.ENVIRONMENT, integrations=[AwsLambdaIntegration(timeout_warning=True)], ) # save tests from failing on init step s3 = boto3.resource('s3') bucket = s3.Bucket(event['detail']['bucket']['name']) # get full filename of a file, uploaded by Looker to S3 filename = unquote_plus(event['detail']['object']['key']) # download file local_path = '/tmp/{}'.format(os.path.basename(filename)) bucket.download_file(filename, local_path) with open(local_path) as csvfile: reader = csv.reader(csvfile) with tempfile.NamedTemporaryFile(mode='w+t') as tmpfile: for index, row in enumerate(reader): if index == 0: # write header tmpfile.write('\t'.join([row[0], row[1]]) + '\n') else: # write rows tmpfile.write( '\t'.join( ['"' + row[0].lstrip('0') + '"', row[1]]) + '\n') tmpfile.flush() try: _upload_to_ftp(tmpfile) except Exception as e: capture_exception(e) raise e