"""Apple Music generators module.""" import re from feed_ingestion.flows.apple_music_streams import config from feed_ingestion.flows.apple_music_streams import tasks from feed_ingestion.flows.apple_music_streams import vendor_accounts from feed_ingestion.util import itunes_reporter def itunes_reports_generator(context): """Generate reporter_account, report_type params that we want to process. Used by reporter_to_s3 activity. Args: context (dict): The current context. Must have a 'bootstrap.reports_status_names'. Yields: (dict): Dictionary for each reporter account / report we want to download and process. """ reports_status_names = context['bootstrap.reports_status_names'] date_str = context['bootstrap.date'] for report_name, description in reports_status_names.items(): vendors = vendor_accounts.get_vendors( context['bootstrap.date'], context['licensor'], report_name) vendors_config = tasks.get_vendors_config( context['licensor'], vendors) for reporter_account in description['vendors']: reporter = itunes_reporter.get_reporter( reporter_account, date_str, vendors=vendors_config) filename = reporter.get_sales_report_file_name(report_name) yield dict( reporter_account=reporter_account, feed_name=description['feed_name'], report_type=report_name, vendors_config={ reporter_account: vendors_config[reporter_account]}, filename=filename) def update_feed_file_status_generator(context): """Generate params needed to update_feed_file_status. Args: context (dict): The current context. Must have a 'bootstrap.reports_status_names'. Yields: (dict): Dictionary for each reporter account / report we want to download and process. """ reports_status_names = context['bootstrap.reports_status_names'] date_str = context['bootstrap.date'] for report_name, description in reports_status_names.items(): vendors = vendor_accounts.get_vendors( context['bootstrap.date'], context['licensor'], report_name) vendors_config = tasks.get_vendors_config( context['licensor'], vendors) expected_download_files = [] for reporter_account in description['vendors']: reporter = itunes_reporter.get_reporter( reporter_account, date_str, vendors=vendors_config) filename = reporter.get_sales_report_file_name(report_name) expected_download_files.append(filename) yield dict( feed_name=reports_status_names[report_name]['feed_name'], report_type=report_name, expected_download_files=expected_download_files) def temp_staging_raw_generator(context): """Generate temp_tables params needs to create & drop temp tables. Used by populate_temp_stage_tables & drop_temp_stage_tables activities. Args: context (dict): the current context. Must have a 'bootstrap.temp_tables' and 'bootstrap.reports_status_names'. Yields: (dict): Dictionary for each temp table. Dictionary has a key for 'temp_table_name' & 'schema'. """ reports_status_names = context[ 'check_available_reports.available_reports'] key_dir = context['bootstrap.s3_archive_bucket'] date = context['bootstrap.date'] for report_name, description in reports_status_names.items(): vendors = vendor_accounts.get_vendors( context['bootstrap.date'], context['licensor'], report_name) vendors_config = tasks.get_vendors_config( context['licensor'], vendors) for reporter_account in description['vendors']: reporter = itunes_reporter.get_reporter( reporter_account, date, vendors=vendors_config) filename = reporter.get_sales_report_file_name(report_name) temp_table_name = tasks.get_temp_table_name( date, report_name, reporter_account, context['licensor']) yield { 'temp_table_name': temp_table_name, 'feed_name': description['feed_name'], 'key_dir': key_dir, 'kwargs': { 'date': context['bootstrap.date'], 'file_pattern': '.*{}.*'.format(re.escape( filename.replace('.txt.gz', ''))), 'report_name': report_name, 'licensor': context['licensor'], } } def populate_staging_raw_generator(context): """Generate params needed to populate staging_raw. Args: context (dict): the current context. Must have a 'check_available_reports.available_reports' key. Yields: (dict): Dictionary for each report / vendor with corresponding tables info. """ reports_status_names = context[ 'check_available_reports.available_reports'] date_str = context['bootstrap.date'] for report_name, description in reports_status_names.items(): staging_raw_table = config.reports[report_name].get( 'staging_raw_table', None) if staging_raw_table: vendors_config = tasks.get_vendors_config( context['licensor'], description['vendors']) vendors = {} for vendor in description['vendors']: reporter = itunes_reporter.get_reporter( vendor, date_str, vendors=vendors_config) filename = reporter.get_sales_report_file_name(report_name) vendors[vendor] = filename yield { 'staging_raw_table': staging_raw_table, 'vendors': vendors, 'report_name': report_name, 'feed_name': description['feed_name']} def update_staging_raw_library_reports_generator(context): """Generate params needed to update_staging_raw_library_reports. Args: context (dict): The current context. Yields: dict: Dictionary of feed names. """ reports_status_names = context[ 'check_available_reports.available_reports'] for report_name, description in reports_status_names.items(): if report_name in config.reports_to_update: yield dict( feed_name=description['feed_name'], report_name=report_name, licensor=context['licensor'])