"""Apple Music generators module.""" import re from feed_ingestion.flows.apple_financial import config from feed_ingestion.flows.apple_financial import tasks def itunes_reports_generator(context): """Generate reporter_account, report_type params that we want to process. Used by reporter_to_s3 activity. Args: context (dict): The current context. Must have a 'bootstrap.apple_reports' key that has a list of all (reporter_account, report_type) tuples to download and process. Yields: (dict): Dictionary for each reporter account / report we want to download and process. """ for (reporter_account, report_type) in context['bootstrap.apple_reports']: yield { 'generator.reporter_account': reporter_account, 'generator.report_type': report_type, 'generator.report_country': 'ZZ' } def process_drop_files_generator(context): """Generate params needed to process_drop_files. Args: context (dict): The current context. Yields: dict: Dictionary of the reports to be processed. """ files = [ {'file_name': file['file_name']} for file in context['check_files_on_s3.source_files_dict']['files'] ] yield dict( files=files, ) def temp_staging_raw_generator(context): """Generate temp_tables params needs to create & drop temp tables. Used by create_temp_staging_raw_tables, load_temp_staging_raw_tables and drop_temp_stage_tables activities. Args: context (dict): the current context. Must have a 'process_drop_files.reports_to_ingest', 'bootstrap.s3_archive_path', 'bootstrap.date' and bootstrap.feed_name. Yields: (dict): Dictionary for each temp table. """ reports_to_ingest = context[ 'process_drop_files.reports_to_ingest'] key_dir = context['bootstrap.s3_archive_path'] date = context['bootstrap.date'] for items in reports_to_ingest: vendor_id = items['filename'].split('_')[1] temp_table_name = tasks.get_temp_table_name( date, items['report'], vendor_id ) yield { 'temp_table_name': temp_table_name, 'feed_name': context['bootstrap.feed_name'], 'key_dir': key_dir, 'kwargs': { 'date': context['bootstrap.date'], 'file_pattern': '.*{}.*'.format(re.escape( items['filename'].replace('.txt.gz', ''))), 'report_name': items['report'], 'licensor': 'theorchard', 'error_limit': config.snowflake_error_limit, 'vendor_id': vendor_id, } } def populate_staging_raw_generator(context): """Generate params needed to populate staging_raw. Args: context (dict): the current context. Must have a 'process_drop_files.reports_to_ingest' key. Yields: (dict): Dictionary for each report / vendor with corresponding tables info. """ reports_to_ingest = context[ 'process_drop_files.reports_to_ingest'] for items in reports_to_ingest: vendor_id = items['filename'].split('_')[1] staging_raw_table = config.expected_reports[items['report']].get( 'staging_raw_table', None) yield { 'staging_raw_table': staging_raw_table, 'vendor_id': vendor_id, 'report_name': items['report'], 'feed_name': context['bootstrap.feed_name'], 'filename': items['filename'] }