"""Component for loading source files directly to staging_raw via stage.""" from datetime import datetime from garcon import task from garcon.param import StaticParam from snowflake_connector.etl_connector import SnowflakeSQLExecutor from feed_ingestion.common.staging_raw_sf.snowflake_stage_loader import \ LicensorSL from feed_ingestion.conf.config import merge_configs from feed_ingestion.flows import base from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.tasks import check_status class YouTubeClaimSL(LicensorSL): """StageLoader class.""" def load_temp_staging_raw_table( self, temp_staging_raw_table, date, stage_name, file_size, skip_corrupted_rows=False, **kwargs): """Load the temp_staging_raw data to the feed's staging_raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). stage_name (str): Name of Snowflake stage containing source files. file_size (int): File size in bytes. skip_corrupted_rows (bool): If True, add ON_ERROR=CONTINUE. """ params = dict( db=self.executor.sf_config['db'], schema=self.executor.sf_config['schema'], stage=stage_name, temp_staging_raw_table=temp_staging_raw_table, file_size=file_size, **kwargs) self.resolve_sql_loader_and_execute( 'load_temp_staging_raw', params=params) @classmethod def load_activity( cls, feed_name, requirements, sql_loader=None, executor_class=SnowflakeSQLExecutor, secrets_path=None): """Load from stage activity to temp_staging_raw_table. Args: feed_name (str): name of the feed. requirements (dict): activity requirements dict. sql_loader (SQLLoader): sql loader instance. executor_class: Snowflake executor class. secrets_path (str): Secrets manager path of the flow. """ @task.decorate(timeout=7200) @check_status(task_id='load_temp_staging_raw_table') def load_task( activity, feed_name, date, s3_dir_path, temp_staging_raw_table, sfdb_params, aws, file_size, skip_corrupted_rows=False, licensor=None): """Copy data from Snowflake stage to staging_raw table. Args: activity (ActivityWorker): The activity worker. feed_name (str): name of the feed. date (str): Reporting date (YYYY-MM-DD). s3_dir_path (str): Name of the feed to get executor class. temp_staging_raw_table (str): name of staging raw table. sfdb_params (dict): Dict with params to optionally override default ones (Snowflake db and schema name). aws (dict): aws credentials. file_size (int): File size in bytes. skip_corrupted_rows (bool): If True, add ON_ERROR=CONTINUE. licensor (str): Optional licensor name. """ activity.logger.info('Loading temp staging raw table: %s', date) sf_config = get_sf_config(secrets_path) sf_config_custom = merge_configs(sf_config, sfdb_params) with executor_class(sf_config_custom) as executor: stage_loader = cls(executor, sql_loader) kwargs = { 'skip_corrupted_rows': skip_corrupted_rows, 'licensor': licensor } stage_name = '{feed_name}_stage_{date:%Y%m%d}'.format( feed_name=feed_name, date=datetime.strptime(date, '%Y-%m-%d')) stage_loader.create_stage( stage_name, s3_dir_path, aws, **kwargs) args = [ temp_staging_raw_table, date, stage_name, file_size] stage_loader.load_temp_staging_raw_table(*args, **kwargs) stage_loader.drop_stage(stage_name) return property( lambda flow_self: flow_self.create( name='load_temp_staging_raw_table_from_stage', tasks=base.SyncRunner( load_task.fill( aws=StaticParam(flow_self.conf_aws), **requirements) ) ) )