""" Update apple_id_mapping table and dim_zip table. Tasks for updating apple_id_mapping and dim_zip table. Transformation of fact data depends on these 2 tables to find the correct asset. """ from garcon import task from snowflake_connector.etl_connector import SnowflakeSQLExecutor from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common import apple_id_mapping from feed_ingestion.conf.config import merge_configs from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.flows.itunes import config from feed_ingestion.tasks import check_status sql_loader = SQLLoader(__file__) @task.decorate(timeout=7200) @check_status() def insert_new_zipcode( activity, sfdb_params, date, feed_name, licensor, secrets_path=None): """Insert into dim_zip table with new zip code. Args: activity (ActivityWorker): The activity worker. sfdb_params (dict): Dictionary stores Snowflake params. date (str): YYYY-MM-DD date of rows to delete from staging_raw table. feed_name (str): Name of the feed. licensor (str): Name of the licensor to ingest. secrets_path (str): Secrets manager path of the flow. """ activity.logger.info('Inserting new entries into into dim_zip') sf_config = merge_configs(get_sf_config(secrets_path), sfdb_params) SnowflakeSQLExecutor(sf_config).execute_query( sql_loader, 'insert_new_zipcodes', { 'db': sf_config['db'], 'schema': sf_config['schema'], 'staging_raw_table': config.staging_raw_table, 'date': date, 'vendor_ids': config.licensors[licensor] } ) @task.decorate(timeout=10800) def update_apple_id_mapping( activity, sfdb_params, date, licensor, skip_mapping, secrets_path=None): """Update apple_id_mapping table. Args: activity (ActivityWorker): The activity worker. sfdb_params (dict): Dictionary stores optional Snowflake db and schema. date (str): YYYY-MM-DD date of rows to delete from staging_raw table. licensor (str): Name of the licensor to ingest. skip_mapping (str): if 'True' skip apple_id_mapping process. secrets_path (str): Secrets manager path of the flow. """ if skip_mapping == 'True': return {'skip_mapping': True} sf_config = merge_configs(get_sf_config(secrets_path), sfdb_params) sql_loader = SQLLoader(apple_id_mapping.query_path, folder='/queries/') if licensor == 'theorchard': query_names = [ '00_itunes_insert_new_entries', '01_update_vendor_offer_code', '02_update_vendor_identifier', '03_clean_blank_apple_release_id', '04_clean_blank_orchard_release_id', '05_clean_zero_apple_track_id', '06_clean_zero_orchard_track_id', '06_itunes_update_apple_release_track_id_from_upc_isrc', ('07_update_apple_track_id', {'term': 1}), ('07_update_apple_track_id', {'term': 2}), ('07_update_apple_track_id', {'term': 3}), ('07_update_apple_track_id', {'term': 4}), ('08_update_apple_release_id', {'term': 1}), ('08_update_apple_release_id', {'term': 2}), ('08_update_apple_release_id', {'term': 3}), '09_update_apple_release_id_one_track', '10_update_apple_release_id_from_upc', '11_update_apple_track_id_one_track', '12_update_orchard_track_id', '13_update_orchard_release_id', '13_itunes_update_album_level_apple_release_id', '13_itunes_update_album_level_orchard_release_id', '14_update_season_pass_orchard_release_track_id', '15_update_with_vendor_identifier_mapping', '16_update_ioda_video_mapping_case', '17_update_ioda_tv_mapping_case', '18_itunes_update_orchard_release_track_id_from_isrc', '19_update_orchard_release_id_from_manufacturer_upc_in_vid', '20_update_orchard_release_id_from_manufacturer_upc_in_upc' ] elif licensor == 'awal': query_names = ['25_itunes_populate_awal_apple_id_mapping'] else: query_names = [ '21_itunes_populate_sony_apple_id_mapping'] common_params = { 'db': sf_config['db'], 'schema': sf_config['schema'], 'date': date, 'vendor_ids': config.licensors[licensor], 'staging_raw_table': config.staging_raw_table } with SnowflakeSQLExecutor(sf_config) as executor: for query in query_names: if isinstance(query, tuple): query_name, add_params = query else: query_name, add_params = query, {} activity.logger.info('Mapping: ' + query_name) params = common_params.copy() params.update(add_params) executor.execute_query(sql_loader, query_name, params)