"""Helper functions for iTunes data ingestion workflow.""" import datetime import os import boto3 from garcon_contrib.aws.utils import garcon_s3 from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.util import itunes_reporter STATUS_ORDERING = [ False, None, garcon_feed_status.STATUS_NOT_AVAILABLE, garcon_feed_status.STATUS_DOWNLOADED, garcon_feed_status.STATUS_POPULATED_RAW_TABLE, garcon_feed_status.STATUS_INGESTED] def check_status(date, feed_name, checkpoint_status): """Check if current overall status is at or before the checkpoint status. Checkpoint status is the status after which the task will be skipped. Args: date (str): Date in YYYYMMDD format (without '-', e.g. 20150920). feed_name (str): Name of the feed. checkpoint_status (str): Status after which task will be skipped. Returns: bool: True or False of whether task should be skipped. """ overall_status = garcon_feed_status.get_overall_status(feed_name, date) overall_status_index = STATUS_ORDERING.index(overall_status) checkpoint_status_index = STATUS_ORDERING.index(checkpoint_status) return overall_status_index < checkpoint_status_index def download_raw_file(date, vendor_name): """Download raw file using iTunes's tool. Args: date (str): date without '-' eg. 20150920 vendor_name (str): Vendor we want to use with the Reporter App ('ORCHARD' or 'IODA'). Returns: str: standard output of check_output. """ reporter = itunes_reporter.get_reporter(vendor_name, date) return reporter.download_itunes_file() def create_full_s3_path(date, account_details): """Create a full S3 path to a downloaded file. Args: date (str): Date on which files are being ingested. account_details (dict): Dictionary contains descriptions of each file which is being downloaded. """ date_obj = datetime.datetime.strptime(date, '%Y-%m-%d') file_name = account_details['file_info']['file_name'].format( vendor_id=account_details['vendor_id'], date=date_obj, version=itunes_reporter.REPORT_VERSIONS.itunes) s3_path = account_details['destination_s3_path'].format( file_name=file_name, date=date_obj) return s3_path def upload_raw_file_to_s3( date, account_details, expected_bucket_owner='437795906767'): """Upload raw file to S3. Args: date (str): Date on which files are being ingested. account_details (dict): Dictionary contains descriptions of each file which is being downloaded. expected_bucket_owner (str): Expected bucket owner. """ date_obj = datetime.datetime.strptime(date, '%Y-%m-%d') file_name = account_details['file_info']['file_name'].format( vendor_id=account_details['vendor_id'], date=date_obj, version=itunes_reporter.REPORT_VERSIONS.itunes) file_path = './{}'.format(file_name) assert os.path.exists(file_path), ( 'File {} does not exist.').format(file_path) s3_client = boto3.client('s3') s3_path = create_full_s3_path(date, account_details) bucket, bucket_path = garcon_s3.extract_bucket_path(s3_path) s3_client.upload_file( file_path, bucket, bucket_path, ExtraArgs={'ExpectedBucketOwner': expected_bucket_owner} ) def remove_raw_file_from_local(file_name): """Remove file which has already been uploaded to S3. Args: file_name (str): Downloaded raw file to be removed. """ os.remove('./{}'.format(file_name))