"""Tasks for Apple's Reporter Tool.""" import os import subprocess from zipfile import ZipFile import boto3 from botocore.exceptions import ClientError from garcon import task from garcon_contrib.aws.utils import garcon_s3 from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.util import itunes_reporter from feed_ingestion.util.itunes_reporter import ReporterException from feed_ingestion.util.sentry_util import send_error_or_warning @task.decorate(timeout=14400) def extract_reporter_file_to_s3( activity, reporter_account, report_type, vendor_id, date, secrets_path, destination_s3_path, feed_name=None, date_type=None, report_role='sales', etl_name=None ): """Extract a report from Apple's Reporter Tool and upload it to S3. Args: activity (ActivityWorker): The activity worker. reporter_account (str): Apple Reporter Account ID. report_type (str): report_type to download (see util.itunes_reporter module) vendor_id (int): vendor id to download the report file. date (str): Reporting date (YYYY-MM-DD). secrets_path (str): aws secrets manager path. destination_s3_path (str): Destination S3 path to the archive location in s3://foo/bar/ format. feed_name (str): If passed, updates feed's status table entry for this date with whether or not the file was Downloaded or Not Available. report_role (str): Report role (sales or finance). Returns: dict: Adds an entry to the context with a key of the file name and a value of whether the file was Downloaded or Not Available. """ reporter = itunes_reporter.get_apple_podcast_reporter( reporter_account, vendor_id, date, secrets_path) file_name = _get_report_file_name( report_role, report_type, reporter, date_type, etl_name) context = { 'file_name': file_name, 'status': garcon_feed_status.STATUS_NOT_AVAILABLE } # abort if already downloaded if feed_name is not None: file_status = garcon_feed_status.get_status(feed_name, date, file_name) if file_status == garcon_feed_status.STATUS_DOWNLOADED: activity.logger.info( 'Already Downloaded File: {}'.format(file_name)) context['status'] = garcon_feed_status.STATUS_DOWNLOADED return context # download report file via Reporter try: activity.logger.info('Downloading File: {}'.format(file_name)) _download_report(report_role, report_type, reporter, date_type) except subprocess.TimeoutExpired: activity.logger.info( 'TimeoutExpired error. {} file not yet available'.format( file_name)) if feed_name is not None: garcon_feed_status.set_status( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) return context except ReporterException as exception: if exception.error_code == ReporterException.FILE_UNAVAILABLE_ERR_CODE: activity.logger.info( 'File unavailable error. {} file not yet available'.format( file_name)) if feed_name is not None: garcon_feed_status.set_status( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) return context elif exception.error_code == ReporterException.NO_REPORT_IS_AVAILABLE: activity.logger.info('No report is available for {}'.format(date)) if feed_name is not None: garcon_feed_status.set_status( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) return context elif exception.error_code == ReporterException.NO_SALES_FOR_DATE: activity.logger.info( 'There were no sales for the date specified. File: {}'.format( file_name)) return context else: if exception.error_code == ReporterException.INVALID_TOKEN: # Let's see when Token is invalid in dev environment activity.logger.info( 'Apple Podcasts Reporter Access Token is invalid') if os.environ.get('SENTRY_DSN'): send_error_or_warning(exception) return context activity.logger.info('Download complete for {}'.format(file_name)) context['status'] = garcon_feed_status.STATUS_DOWNLOADED if os.path.exists(file_name): activity.logger.info('In single file upload') # upload report file to s3 copy_on_s3_result = _upload_to_s3( file_name, destination_s3_path, activity) elif os.path.exists(file_name.replace('.txt.gz', '.zip')): activity.logger.info('In multi-part file upload') file_name = file_name.replace('.txt.gz', '.zip') activity.logger.info('Uploading zipped file {}'.format(file_name)) with ZipFile(file_name, 'r') as zf: zf.extractall() file_list = zf.namelist() activity.logger.info('Start uploading files list - {}'.format( file_list)) copy_on_s3_result = len(file_list) > 0 for name in file_list: copy_on_s3_result = copy_on_s3_result and _upload_to_s3( name, destination_s3_path, activity) os.remove(name) else: if feed_name is not None: garcon_feed_status.set_status( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) error = Exception( 'Wrong name given for downloaded file: {}'.format(file_name) ) if os.environ.get('SENTRY_DSN'): send_error_or_warning(error) return context # remove local report file os.remove(file_name) if feed_name is not None: if not copy_on_s3_result: garcon_feed_status.set_status( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) return {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE} garcon_feed_status.set_status( feed_name, date, file_name, status=garcon_feed_status.STATUS_DOWNLOADED) return context def _get_report_file_name( report_role, report_type, reporter, date_type, etl_name): """Get report filename. Args: report_role (str): Report role (sales of finance). report_type (str): report_type to download (see util.itunes_reporter module). reporter (ItunesReporter): Reporter instance. Returns: str: Report file name string. """ if report_role == 'sales': if date_type in ['Weekly', 'Monthly'] and \ etl_name == 'apple_podcasts_sales_summary_monthly': report_type = report_type + date_type file_name = reporter.get_sales_report_file_name(report_type) else: raise Exception('Unknown Report Role: {}.'.format(report_role)) return file_name def _download_report(report_role, report_type, reporter, date_type): """Download report file. Args: report_role (str): Report role (sales of finance). report_type (str): report_type to download (see util.itunes_reporter module). reporter (ItunesReporter): Reporter instance. """ if report_role == 'sales': if date_type in ['Weekly', 'Monthly']: report_type = report_type + date_type reporter.download_sales_report(report_type) def _upload_to_s3(file_name, destination_s3_path, activity): """Upload report file to s3. Args: file_name (str): The name of the file on the local system destination_s3_path (str): The s3 destination location activity (ActivityWorker): The activity worker. """ s3 = boto3.client('s3') bucket_name, key_path = garcon_s3.extract_bucket_path(destination_s3_path) local_file_path = os.path.join(os.path.curdir, file_name) file_key_path = f'{key_path}{file_name}' activity.logger.info( 'Uploading file: {} to {} has started'.format( file_name, destination_s3_path ) ) try: s3.upload_file(local_file_path, bucket_name, file_key_path) s3.head_object(Bucket=bucket_name, Key=file_key_path) activity.logger.info( 'Uploaded file: {} to {}'.format( file_name, destination_s3_path)) return True except ClientError as e: activity.logger.info( 'Uploading file: {} to {} failed'.format( file_name, destination_s3_path)) raise e