"""Theatrical cuts ETL Tasks.""" from garcon import task from flows import datastore from flows import s3 from flows.theatrical_cuts import log from flows.theatrical_cuts import queries from flows.theatrical_cuts import status from flows.theatrical_cuts import util @task.decorate(timeout=900) def bootstrap(activity, correlation_id, workflow_run_id, s3_bucket, s3_key): """Prepare parameters for flow execution. UPC value and file date are taken from s3_key. Initial log record is created since all required data is ready for it. Generate required data for SWF context. Args: activity (Activity): SWF Activity. correlation_id (str): correlation id. workflow_run_id (str): workflow run id generated by AWS SWF service. s3_bucket (str): S3 bucket name of ingested CSV file. s3_key (str): S3 key path of ingested CSV file. Returns: dict: Dict with update to SWF context. """ drop_url = 's3://{bucket}/{key}'.format(bucket=s3_bucket, key=s3_key) upc = util.get_cuts_upc(drop_url) log.create(correlation_id, workflow_run_id, upc) return {'upc': upc, 'drop_url': drop_url} @task.decorate(timeout=900) def determine_anchor_date(activity, correlation_id, upc): """Determine the anchor date for a upc. Args: activity (ActivityWorker): activity worker. correlation_id (str): etl correlation ID. upc (str): release UPC. Returns: dict: YYYY-MM-DD format anchor date string. """ date = util.get_upc_theatrical_cuts_anchor_date(upc) if date: log.update_status(correlation_id, status.ANCHOR_DATE_DETERMINED) return {'anchor_date': date.isoformat()} log.update_status(correlation_id, status.ANCHOR_DATE_UNDETERMINED) return {'anchor_date': None, 'stop': True} @task.decorate(timeout=900) def download_cuts_to_db(activity, anchor_date, correlation_id, drop_url, upc): """Download cuts data from S3 bucket to db. Args: activity (ActivityWorker): activity worker. anchor_date (str): iso formatted date 'YYYY-MM-DD' correlation_id (str): etl correlation ID. drop_url (str): S3 url of source file. upc (str): release UPC. """ with datastore.context() as (cursor, connection): cursor.execute('START TRANSACTION;') csv_row = s3.download_csv(drop_url)[0] cursor.execute(queries.DELETE_EXIST_UPC_CUTS, {'upc': upc}) cursor.executemany( queries.INSERT_UPC_CUTS, util.transpose_percentage_data( csv_row, anchor_date, upc)) log.update_status( correlation_id, status.THEATRICAL_CUTS_RAW_TABLE_UPDATED) @task.decorate(timeout=900) def update_db(activity, correlation_id, upc): """Download data from S3 bucket to db. Args: activity (ActivityWorker): activity worker. correlation_id (str): etl correlation ID. upc (str): release UPC. """ datastore.execute(queries.UPDATE_REVENUE, {'upc': upc}) log.update_status(correlation_id, status.REVENUE_TABLE_UPDATED) @task.decorate(timeout=900) def archive_dropped_csv( activity, archive_path, bucket, correlation_id, drop_url): """Grab dropped CSV file into the archive location. Args: activity (ActivityWorker): activity worker. archive_path (str): location to store csv file to on S3. bucket (str): S3 bucket of source and destination files. correlation_id (str): etl correlation ID. drop_url (str): dropped csv source location on S3. Returns: dict: full archive s3 location. """ drop_obj = s3.get_object(drop_url) filename = drop_url.split('/')[-1] archive_key = archive_path.format(filename=filename) archive_url = 's3://{bucket}/{key}'.format(bucket=bucket, key=archive_key) archive_obj = s3.get_object(archive_url) archive_obj.copy({'Bucket': drop_obj.bucket_name, 'Key': drop_obj.key}) drop_obj.delete() log.update_status(correlation_id, status.DROP_FILE_ARCHIVED) return {'archive_url': archive_url} @task.decorate(timeout=100) def set_final_status(activity, correlation_id): """Set status INGESTED to the db log. Args: activity (Activity): SWF Activity. correlation_id (str): correlation id. """ log.update_status(correlation_id, status.INGESTED)