""" Load transformed iTunes data into fact_analytics table. Tasks for loading transformed iTunes data into fact_analytics table. """ from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SnowflakeSQLExecutor from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.conf.config import merge_configs from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.flows.itunes import config from feed_ingestion.flows.itunes.util import itunes_helpers from feed_ingestion.util import task_status sql_loader = SQLLoader(__file__) @task.decorate(timeout=7200) def load_staging_fact( activity, date, sfdb_params, licensor, secrets_path=None): """Load temp staging fact table. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date of the data file. sfdb_params (dict): Dictionary stores Snowflake params. licensor (str): The licensor to ingest. secrets_path (str): Secrets manager path of the flow. """ date_obj = datetime.strptime(date, '%Y-%m-%d') activity.logger.info('Loading into staging fact') sf_config = merge_configs(get_sf_config(secrets_path), sfdb_params) with SnowflakeSQLExecutor(sf_config) as executor: executor.execute_query( sql_loader, 'load_staging_fact', { 'db': sf_config['db'], 'schema': sf_config['schema'], 'reportdate': date, 'staging_fact_analytics_table': config.staging_fact_table_template.format( date=date_obj, licensor=licensor), 'staging_raw_table': config.staging_raw_table, 'feedid': config.feedid, 'storeid': 1, 'vendor_ids': config.licensors[licensor], 'licensor': licensor } ) @task.decorate(timeout=2000) def drop_staging_fact( activity, date, sfdb_params, licensor, secrets_path=None): """Drop temp staging fact table. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date of the data file. sfdb_params (dict): Dictionary stores Snowflake params. licensor (str): The licensor to ingest. secrets_path (str): Secrets manager path of the flow. """ date_obj = datetime.strptime(date, '%Y-%m-%d') staging_fact_analytics_table = ( config.staging_fact_table_template.format( date=date_obj, licensor=licensor)) activity.logger.info('Dropping: %s', staging_fact_analytics_table) sf_config = merge_configs(get_sf_config(secrets_path), sfdb_params) with SnowflakeSQLExecutor(sf_config) as executor: executor.execute_query( sql_loader, 'drop_temp_table', { 'db': sf_config['db'], 'schema': sf_config['schema'], 'table_name': staging_fact_analytics_table } ) @task.decorate(timeout=7200) def delete_fact_data( activity, date, feed_name, sfdb_params, licensor, secrets_path=None): """Delete data from fact_analytics and fact_analytics_error table. Args: date (str): Date on which data is deleted. feed_name (str): Name of the feed. sfdb_params (dict): Dictionary stores Snowflake params. licensor (str): The licensor to ingest. secrets_path (str): Secrets manager path of the flow. """ completed = task_status.is_completed_report(feed_name, date) if (itunes_helpers.check_status( date, feed_name, garcon_feed_status.STATUS_INGESTED) or not completed): activity.logger.info('Deleting from fact tables') sf_config = merge_configs(get_sf_config(secrets_path), sfdb_params) with SnowflakeSQLExecutor(sf_config) as executor: executor.execute_query( sql_loader, 'delete_from_fact_table', { 'db': sf_config['db'], 'schema': sf_config['schema'], 'fact_table': 'fact_analytics', 'reportdate': date, 'feedid': config.feedid, 'storeid': 1, 'licensor': licensor } ) executor.execute_query( sql_loader, 'delete_from_fact_table', { 'db': sf_config['db'], 'schema': sf_config['schema'], 'fact_table': 'fact_analytics_error', 'reportdate': date, 'feedid': config.feedid, 'storeid': 1, 'licensor': licensor } ) else: activity.logger.info('Status is INGESTED. Task skipped.') @task.decorate(timeout=7200) def load_fact_tables( activity, date, feed_name, sfdb_params, licensor, secrets_path=None): """Load data into fact analytics tables. (Into fact_analytics, fact_analytics_error and fact_analytics_geo). Args: activity (ActivityWorker): The activity worker. date (str): Reporting date of the data file. feed_name (str): Name of the feed. sfdb_params (dict): Dictionary stores optional Snowflake db and schema. licensor (str): The licensor to ingest. secrets_path (str): Secrets manager path of the flow. """ date_obj = datetime.strptime(date, '%Y-%m-%d') staging_fact_analytics_table = ( config.staging_fact_table_template.format( date=date_obj, licensor=licensor)) completed = task_status.is_completed_report(feed_name, date) if (itunes_helpers.check_status( date, feed_name, garcon_feed_status.STATUS_INGESTED) or not completed): sf_config = merge_configs(get_sf_config(secrets_path), sfdb_params) with SnowflakeSQLExecutor(sf_config) as executor: activity.logger.info('Loading fact_analytics table') executor.execute_query( sql_loader, 'load_fact_analytics', { 'db': sf_config['db'], 'schema': sf_config['schema'], 'reportdate': date, 'staging_fact_analytics_table': staging_fact_analytics_table, 'fact_table': 'fact_analytics', 'feedid': config.feedid, 'storeid': 1, 'staging_raw_table': config.staging_raw_table } ) activity.logger.info('Loading fact_analytics_error table') executor.execute_query( sql_loader, 'load_fact_analytics_error', { 'db': sf_config['db'], 'schema': sf_config['schema'], 'reportdate': date, 'fact_table': 'fact_analytics_error', 'staging_fact_analytics_table': staging_fact_analytics_table, 'feedid': config.feedid, 'storeid': 1, 'staging_raw_table': config.staging_raw_table, 'vendor_ids': config.licensors[licensor], 'licensor': licensor } ) @task.decorate(timeout=2000) def set_overall_status_to_ingested(activity, feed_name, date): """Set overall status to INGESTED in DynamoDB. Args: activity (ActivityWorker): The activity worker. feed_name (str): Name of the feed. date (str): Feed delivery date. """ garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_INGESTED) processed_contexts = task_status.mark_report_processed_contexts( feed_name, date) activity.logger.info(f'Change context status ' f'to processed for: {processed_contexts}') overall_feed_name = feed_name is_completed = task_status.is_completed_report(feed_name, date) task_status.mark_completed_overall_job( overall_feed_name, date, is_completed) # task_status.set_values(overall_feed_name, date, # 'not_completed_reports', not_completed_reports)