import datetime import sys from concurrent.futures.thread import ThreadPoolExecutor from dataclasses import dataclass from chartmetric_snowflake.setup_chartmetric_workflow import SetupWorkflow @dataclass class BaseArgs(SetupWorkflow): since = None until = None is_manual_run = False # Manual run flag to be set if its a manual run process_to_bigtable = False # Process only if we have updated data (Not for Manual run) source_list = None data = [] process_dictionary = {'scheduled': {}, 'manual': {}} threadpool = ThreadPoolExecutor(40) def _setup_since_until_dates(self): try: # Edge cases to check if valid dates are provided if self.args.since: self.since = f"{datetime.datetime.strptime(self.args.since, '%Y-%m-%d')}".split( ' ')[0] if self.args.until: self.until = f"{datetime.datetime.strptime(self.args.until, '%Y-%m-%d')}".split( ' ')[0] except Exception as e: self.snowflake_utils.log( message=f"Invalid date provided: {self.args.since}", _level="error") self.snowflake_utils.notify_slack() exit(1) def _setup_chartmetric_required_sources_list(self): try: # Check if the list of sources is in a proper format self.source_list = eval(self.args.sources) self.snowflake_source_table_names = [ f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_BANDSINTOWN_STAT', f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_DEEZER_STAT', f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_FACEBOOK_STAT', f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_INSTAGRAM_STAT', f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_SPOTIFY_STAT', f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_TWITTER_STAT', f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_WIKIPEDIA_STAT', f'{self.chartmetric_constants.get("SNOWFLAKE_DATABASE")}.{self.chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.SME_YOUTUBE_STAT' ] return True except Exception as e: self.snowflake_utils.log( message=f"Invalid source type provided: {sys.argv[3]}", _level="error") self.snowflake_utils.notify_slack() exit(1) def setup_base_args(self, *args, **kwargs): super(BaseArgs, self).setup_instances(*args, **kwargs) self._setup_chartmetric_required_sources_list() self._setup_since_until_dates()