from dataclasses import dataclass from chartmetric_snowflake.base_args import BaseArgs @dataclass class SetupProcessDictionary(BaseArgs): def setup_process_dictionary(self, *args, **kwargs): """ Setup the workflow required for creating the Process Dictionary """ super(SetupProcessDictionary, self).setup_base_args(*args, **kwargs) if self.args.artists: self.snowflake_utils.log( message= f'Data being fetched for GRAS_IDs: {self.args.artists} for sources: {self.args.sources}', _level="info") else: self.snowflake_utils.log( message= f'Data being fetched from {self.since} till {self.until} for sources {self.source_list}', _level="info") self._check_if_source_data_updated() self._prepare_data_dictionary() return self def _check_if_source_data_updated(self): """ Check for V_RUN_DATES to confirm if the data is updated for any of the sources """ try: result = self.snow_flake_instance.run_query( query=self.queries.get('data_update_confirmation_query')) return result.fetchall() except Exception as e: self.snowflake_utils.log( message=f"Error fetching data confirmation: {e}", _level='error') def _prepare_data_dictionary(self): """ Process dictionary creates a an object with differentiated scheduled and manual run data :return: """ _is_data_updated = self._check_if_source_data_updated() for items in _is_data_updated: if items['SOURCE'] not in self.process_dictionary['scheduled']: self.process_dictionary['scheduled'][items['SOURCE']] = {} self.process_dictionary['scheduled'][items['SOURCE']].update( items) self.process_dictionary['manual'][items['SOURCE']] = {} self.process_dictionary['manual'][items['SOURCE']].update( items) else: self.process_dictionary['scheduled'][items['SOURCE']].update( items) self.process_dictionary['manual'][items['SOURCE']].update( items) del _is_data_updated if self.since or self.until or self.source_list or self.args.artists: self.process_to_bigtable = True if self.args.artists: self.is_manual_run = True for _source, _data in self.process_dictionary['manual'].items(): if self.since: # Replace the since date with the provided date self.process_dictionary['manual'][_source][ 'REPORT_DATE'] = str(self.since).split(',')[0] self.is_manual_run = True if self.until: # Replace the until date with the provided date self.process_dictionary['manual'][_source][ 'LATEST_DATE'] = str(self.until).split(',')[0] self.is_manual_run = True if self.source_list: self.is_manual_run = True if _source not in self.source_list: self.process_dictionary['manual'][_source][ 'REQUIRED'] = False else: self.process_dictionary['manual'][_source][ 'REQUIRED'] = True else: self.process_dictionary['manual'][_source][ 'REQUIRED'] = True