import argparse import datetime import json import os from dataclasses import dataclass from sme_btutils import GcpUtils from sme_snowflake import SnowflakeUtils from aws_utils import AwsUtils from const import ENV from snowflake_utils import ChartmetricSnowflakeUtils @dataclass class ArgParser: _now = datetime.datetime.now() parser = argparse.ArgumentParser() parser.add_argument( '--since', help="Since date for data fetch", type=str, default=(_now - datetime.timedelta(days=7)).strftime("%Y-%m-%d")) parser.add_argument('--until', help="Until date for data fetch", type=str, default=_now.strftime("%Y-%m-%d")) parser.add_argument( '--sources', help="Sources for data fetch", type=str, default=json.loads(AwsUtils().get_secret( required_secret=f"delphi/{ENV}/chartmetric_constants")).get( "VALID_SOURCES")) parser.add_argument('--artists', help="Artists date for data fetch", type=str) args = parser.parse_args() @dataclass class SetupChartmetricConstants(ArgParser): start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") chartmetric_constants = json.loads(AwsUtils().get_secret( required_secret=f"delphi/{ENV}/chartmetric_constants")) queries = { "data_update_confirmation_query": f'select * from {chartmetric_constants.get("SNOWFLAKE_DATABASE")}.' f'"{chartmetric_constants.get("SNOWFLAKE_SCHEMA")}".v_run_dates; ', "schedule_value_query": f'select NVL(max(schedule),0) AS schedule from {chartmetric_constants.get("SNOWFLAKE_DATABASE")}.' f'{chartmetric_constants.get("SNOWFLAKE_SCHEMA")}.CHARTMETRIC_STATUS WHERE ' f"RUN_DATE=current_date(); ", } snowflake_source_table_names = None @dataclass class SetupWorkflow(SetupChartmetricConstants): """ Setup all the instances required for end to end process: - fetch snowflake credentials - sme_snowflake -sme_btutils """ snow_flake_instance = None gcp_instance = None snowflake_utils = None def setup_instances(self, *args, **kwargs): self.snow_flake_instance = SnowflakeUtils( user=self.chartmetric_constants.get( "CHARTMETRIC_SNOWFLAKE_USERNAME"), password=self.chartmetric_constants.get( "CHARTMETRIC_SNOWFLAKE_PASSWORD"), account=self.chartmetric_constants.get("SNOWFLAKE_ACCOUNT"), warehouse=self.chartmetric_constants.get("SNOWFLAKE_WAREHOUSE"), database=self.chartmetric_constants.get("SNOWFLAKE_DATABASE"), schema=self.chartmetric_constants.get("SNOWFLAKE_SCHEMA"), slack_channel=self.chartmetric_constants.get( "CHARTMETRIC_SLACK_CHANNEL"), slack_token=self.chartmetric_constants.get( "CHARTMETRIC_SLACK_TOKEN"), filename=str(os.getcwd()).split("/")[-1], private_key=self.chartmetric_constants.get( "CHARTMETRIC_SNOWFLAKE_PRIVATE_KEY") ) self.gcp_instance = GcpUtils( column_family_meta="meta", column_family_metrics="metrics", gcp_credentials=AwsUtils().get_secret( required_secret=self.chartmetric_constants.get( "GOOGLE_SECRET_NAME")), project_id=self.chartmetric_constants.get("GCP_PROJECT_ID"), instance=self.chartmetric_constants.get("GCP_INSTANCE_ID"), table=self.chartmetric_constants.get("GCP_TABLE_NAME"), slack_channel=self.chartmetric_constants.get( "CHARTMETRIC_SLACK_CHANNEL"), slack_token=self.chartmetric_constants.get( "CHARTMETRIC_SLACK_TOKEN"), filename=str(os.getcwd()).split("/")[-1], ) self.snowflake_utils = ChartmetricSnowflakeUtils()