"""Component for loading source files directly to staging_raw via stage.""" from datetime import datetime from feed_ingestion.common.staging_raw_sf.snowflake_stage_loader \ import StageLoader from feed_ingestion.util import dates_util class GFKSL(StageLoader): """StageLoader class for the GFK ETL.""" def get_release_year_and_week(self, date, stage_name): """Get release week and year. We have to attribute dayid to all the rows of the file, but not all of them has week_number value. If there is no value, it means this row contains only cumulative data. But we still have to add dayid to it, so we take the week_number from a row which has that value. Args: date (str): Date of the data being process (YYYY-MM-DD). stage_name (str): Name of Snowflake stage containing source files. Returns: tuple: First and last day of a release week in YYYY-MM-DD format. """ result = self.executor.fetchone_query( self.sql_loader, 'get_release_year_and_week', params=dict( db=self.executor.sf_config['db'], schema=self.executor.sf_config['schema'], stage_name=stage_name, date=date)) year_and_week_number = result[0] # e.g., split '322014' string into 32 and 2014 ints week_number, year = int( year_and_week_number[:2]), int(year_and_week_number[2:]) release_week = dates_util.calculate_first_and_last_day_of_release_week( year, week_number) return release_week.get('first_day'), release_week.get('last_day') def load_staging_raw_table( self, staging_raw_table, source_files_dict, date, stage_name): """Load the temp_staging_raw data to the feed's staging_raw table. Args: staging_raw_table (str): A table name in Snowflake. source_files_dict (dict): A dict with source files metadata. date (str): Date of the data being process (YYYY-MM-DD). stage_name (str): Name of Snowflake stage containing source files. """ ingestion_time = datetime.now() first_day, last_day = self.get_release_year_and_week( date, stage_name) for file_dict in filter( lambda f: f['found'], source_files_dict['files']): self.resolve_sql_loader_and_execute( 'load_staging_raw', params=dict( db=self.executor.sf_config['db'], schema=self.executor.sf_config['schema'], stage=stage_name, staging_raw_table=staging_raw_table, file_name=file_dict['file_name'].replace('.zip', '.gz'), file_size=file_dict['file_size'], download_date=date, ingestion_time=ingestion_time, first_day_of_release_week=first_day, last_day_of_release_week=last_day))