import os from datetime import date, datetime today = datetime.utcnow().date() s = str(date.today()).split("-") year, month, day = s[0], s[1], s[2] ENV = os.getenv('ENVIRONMENT') ENV = 'dev' if ENV is None else ENV.lower() chartmetric = { "CHARTMETRIC_VERSION": "v1", "REPORT_LICENSOR": "sme", "RATE_LIMIT": 1000, "TOKEN_REFRESH_DELTA": 3500, "REFRESH_SLEEP_TIME": 5, "TIME_DELTA": 15, "urls": { "CM_REFRESH_TOKEN_URL": "https://api.chartmetric.com/api/token", "CM_FAN_METRICS_URL": "https://api.chartmetric.com/api/artist/{0}/stat/{1}?since={2}&until={3}&interpolated=true", "CM_CHARTMETRIC_ID_URL_SPOTIFY_SEARCH": "https://api.chartmetric.com/api/artist/spotify/{0}/get-ids", "CM_CHARTMETRIC_ID_URL_NAME_SEARCH": "https://api.chartmetric.com/api/search?q={0}&limit=1", }, "DATA_SOURCES": ("spotify", "deezer", "facebook", "twitter", "youtube_channel", "youtube_artist", "instagram", "wikipedia", "bandsintown", "soundcloud", "youtube"), "DATA_SOURCES_METRICS": { "spotify": ["followers", "popularity", "listeners"], "deezer": ["fans"], "facebook": ["likes"], "twitter": ["followers"], "youtube_channel": ["subscribers", "views"], "youtube_artist": ["views", "daily_views"], "instagram": ["followers"], "wikipedia": ["views"], "bandsintown": ["followers"], "soundcloud": ["followers"], "youtube": ["views"] } } aws = { "AWS_DEFAULT_REGION": "us-east-1", "AWS_DEV_PROFILE": f"gdb-delphi-{ENV}", "AWS_FIREHOSE_CHARTMETRIC_DATA_STREAM": f"{ENV}-delphi-chartmetric_stream", "AWS_FIREHOSE_CHARTMETRIC_HISTORICAL_DATA_STREAM": f"{ENV}-delphi-chartmetric_historical_stream", "RDS": { "POSTGRESQL_PORT": "5432", "DB_HOST": f"{ENV}-delphi-slz-master.cic9zqsxd2y6.us-east-1.rds.amazonaws.com", 'USERNAME': 'chartmetric_user', "DB_NAME": "profile", "queries": { "GET_LAST_UPDATED": "select cast(update_date as VARCHAR) from public.chartmetric_status LIMIT 1;", # for 20 sample artist "FETCH_ARTIST_CHARTMETRIC_ID": "SELECT * FROM public.artist WHERE chartmetric_id IS NULL;", "GET_ARTIST_CHARTMETRIC_IDS": "SELECT * FROM public.artist WHERE chartmetric_id IS NOT NULL;", "GET_ARTIST_CHARTMETRIC_IDS_REPROC": "select id, re_vw.chartmetric_id as artist_id, artist_name, " "array_agg(src) as src " "from public.chartmetric_status_vw re_vw, " "public.artist a where cast(re_vw.chartmetric_id as VARCHAR)= " "a.chartmetric_id group by re_vw.chartmetric_id, id, artist_name;", # Insert artist "INSERT_ARTIST_QUERY_USING_SPOTIFY": "UPDATE public.artist_temp SET chartmetric_id = {} where SPOTIFY_ID in ('{}');", "INSERT_ARTIST_QUERY_USING_NAME": "UPDATE public.artist_temp set chartmetric_id = {} where artist_name in ('{}');", # Insert artist logs "INSERT_ARTIST_LOGS_QUERY": "INSERT INTO public.chartmetric_status {} values {} ON CONFLICT(gras_id) DO UPDATE SET {};", "COUNT_MANAGE_QUERY": "UPDATE public.chartmetric_status SET status = 'ACTIVE', count = 0 WHERE coalesce(update_date,CURRENT_DATE) <> CURRENT_DATE AND gras_id IN {0}; \n " "UPDATE public.chartmetric_status SET count = count + 1 WHERE coalesce(update_date,CURRENT_DATE) = CURRENT_DATE AND gras_id " "IN {0} AND status = 'ACTIVE'; \n " "UPDATE public.chartmetric_status SET status = 'DNP' WHERE count > 3 AND gras_id IN {0} ;\n", "LAST_DATE_QUERY": "SELECT max(coalesce(update_date,CURRENT_DATE)) FROM public.chartmetric_status;\n", "INSERT_REPROC_EXEMPT": "INSERT INTO public.chartmetric_status_exempt (gras_id, chartmetric_id, status, update_date, count, facebook_req, facebook_res, facebook_val, spotify_req, spotify_res, spotify_val, deezer_req, deezer_res, deezer_val, twitter_req" ", twitter_res, twitter_val, youtube_channel_req, youtube_channel_res, youtube_channel_val, youtube_artist_req, youtube_artist_res, youtube_artist_val, instagram_req, instagram_res, instagram_val, wikipedia_req, wikipedia_res, wikipedia_val" ", bandsintown_req, bandsintown_res, bandsintown_val, soundcloud_req, soundcloud_res, soundcloud_val) " "SELECT * FROM public.chartmetric_status " "WHERE status = 'DNP';", "TRUNCATE_STATUS_TABLE": "TRUNCATE public.chartmetric_status;", "UPDATE_FILE_PROCESSING": "UPDATE public.chartmetric_file SET data_write_big_table= TRUE, update_date=now(), end_time='{}' WHERE filename IN ('{}');", "GET_FILE_PROCESSING_INFO": "select filename, report_date from public.chartmetric_file where data_write_big_table = FALSE AND data_quality = TRUE AND data_movement = TRUE;", 'GET_LATEST_RUN_COUNT': "SELECT max(schedule) from chartmetric_performance where run_date = '{0}' and process_type = '{1}';", 'INSERT_EXECUTION_TIME_METRICS': "INSERT INTO chartmetric_performance (process_type,schedule,start_time,end_time,total_time,run_date,parent_id) " "VALUES('{0}',{1},'{2}','{3}',age('{4}','{5}'),'{6}',{7});", 'INSERT_EXECUTION_TIME_FAN_METRICS': "INSERT INTO chartmetric_performance (process_type,schedule," "start_time,run_date,parent_id) values ('{}',{},'{}','{}',{});", 'UPDATE_EXECUTION_TIME_FAN_METRICS': "update chartmetric_performance set end_time = '{0}', total_time = age('{1}','{2}'), stats='{3}' where process_type='{4}' and schedule = {5} and start_time ='{6}';", }, }, "s3": { "BUCKET_QUARANTINE": f"{ENV}-delphi-quarantine-sme-data-decompressed", "BUCKET_LOGS": f"{ENV}-delphi-chartmetric-logs", "BUCKET_DECOMPRESSED": f"{ENV}-sme-data-decompressed", "BUCKET_HISTORICAL_PRE_QUARANTINE": f"{ENV}-delphi-chartmetric-hist-data-decompressed", "BUCKET_PATH": f"chartmetric/{chartmetric.get('CHARTMETRIC_VERSION', None)}/" f"report_date={today}/" f"report_licensor={chartmetric.get('REPORT_LICENSOR', None)}", "HISTORICAL_BUCKET_PATH": "chartmetric/{0}/" "report_date={1}/" "report_licensor={2}" }, "secrets_manager": { "SECRETS_CHARTMETRIC_ACCESS_KEY": f"delphi/{ENV}/chartmetric_api", "SECRETS_CHARTMETRIC_SNOWFLAKE_CREDS": f"delphi/{ENV}/chartmetric_snowflake", "SLACK_SECRET": f'delphi/{ENV}/chartmetricslack' }, } if ENV == 'stage': aws['RDS'][ 'DB_HOST'] = 'stage-delphi-slz.cluster-cic9zqsxd2y6.us-east-1.rds.amazonaws.com' elif ENV == 'prod': aws['RDS'][ 'DB_HOST'] = 'prod-delphi-slz.cluster-cjzbf0a3lwld.us-east-1.rds.amazonaws.com' aws['s3']['BUCKET_DECOMPRESSED'] = 'sme-data-decompressed' gcp = { "PROJECT_ID": f"delphi-{ENV}", "INSTANCE_ID": f"{ENV}-delphi-consumer-analytics-hdd", "TABLE": "chartmetric", "GOOGLE_SECRET_NAME": f"delphi/{ENV}/gcp/external-bigtable-user", "GOOGLE_CREDENTIALS": "google_credentials.json", "BIGTABLE_INT_BYTE_FORMAT": ">q" } APP_NAME: str = "Chartmetric" DEFAULT_VALID_CODES = '200' # LOG FILES CHARTMETRIC_API_LOGS_FILE = os.path.abspath( "delphi-chartmetric/error_logs.log")