"""Component for loading source files directly to staging_raw via stage.""" from datetime import datetime from feed_ingestion.common.staging_raw_sf.snowflake_stage_loader import \ LicensorSL class YouTubeAssetSL(LicensorSL): """StageLoader class.""" def create_stage(self, stage_name, s3_dir_path, aws, **kwargs): """Create Snowflake stage. Args: stage_name (str): Name of Snowflake stage containing source files. s3_dir_path (str): s3 path to directory containing source files. aws (dict): aws credentials. """ aws_params = self.get_aws_params() licensor = kwargs['licensor'] if licensor in ['theorchard']: query = 'create_snowflake_stage' else: query = 'create_snowflake_stage_{licensor}'.format( licensor=licensor) self.resolve_sql_loader_and_execute( query, params=dict( db=self.executor.sf_config['db'], schema=self.executor.sf_config['schema'], stage=stage_name, s3_dir_path=s3_dir_path, **aws_params ) ) def clean_staging_raw_table(self, staging_raw_table, date, **kwargs): """Delete rows from previous unsuccessful workflow run. Args: staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ self.resolve_sql_loader_and_execute( 'delete_from_staging_raw', params=dict( db=self.executor.sf_config['db'], schema=self.executor.sf_config['schema'], staging_raw_table=staging_raw_table, date=date, **kwargs)) def load_staging_raw_table(self, staging_raw_table, source_files_dict, date, stage_name, **kwargs): """Load the temp_staging_raw data to the feed's staging_raw table. Args: staging_raw_table (str): A table name in Snowflake. source_files_dict (dict): A dict with source files metadata. date (str): Date of the data being process (YYYY-MM-DD). stage_name (str): Name of Snowflake stage containing source files. """ ingestion_time = datetime.now() licensor = kwargs['licensor'] for file_dict in source_files_dict['files']: if licensor in ['theorchard']: file_name = file_dict['file_name'] if licensor == 'theorchard' and 'dmgi' in file_name: sql_template = 'load_staging_raw_theorchard_dmgi' elif licensor == 'theorchard' and '.ent.' in file_name.lower(): sql_template = 'load_staging_raw_theorchard_ent' else: sql_template = 'load_staging_raw' else: assert licensor == 'sme' sql_template = 'load_staging_raw_sme' self.resolve_sql_loader_and_execute( sql_template, params=dict( db=self.executor.sf_config['db'], schema=self.executor.sf_config['schema'], stage=stage_name, staging_raw_table=staging_raw_table, file_name=file_dict['file_name'], file_size=file_dict['file_size'], download_date=date, ingestion_time=ingestion_time, **kwargs))