"""Config for Store Spicific Spotify Charts Workflow.""" import os environment = os.environ.get('Environment', 'dev') feed_name = 'spotify_charts' secrets_path = 'spotify_charts' feed_version = '1.0' expected_bucket_owner = os.environ.get('EXPECTED_BUCKET_OWNER', '437795906767') # bucket for archived data data_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') if environment == 'production': data_bucket = 'cucumbers' # S3 locations for dropped and archived files s3 = { 'archive': 'SpotifyCharts/archives/{chart}/{date:%Y-%m-%d}/' } expected_countries = [ 'AD', 'AE', 'AR', 'AT', 'AU', 'BE', 'BG', 'BO', 'BR', 'BY', 'CA', 'CH', 'CL', 'CO', 'CR', 'CY', 'CZ', 'DE', 'DK', 'DO', 'EC', 'EE', 'EG', 'ES', 'FI', 'FR', 'GB', 'GLOBAL', 'GR', 'GT', 'HK', 'HN', 'HU', 'ID', 'IE', 'IL', 'IN', 'IS', 'IT', 'JP', 'KR', 'LI', 'LT', 'LU', 'LV', 'MA', 'MC', 'MT', 'MX', 'MY', 'NG', 'NI', 'NL', 'NO', 'NZ', 'PA', 'PE', 'PH', 'PK', 'PL', 'PT', 'PY', 'RO', 'RU', 'SA', 'SE', 'SG', 'SK', 'SV', 'TH', 'TR', 'TW', 'UA', 'US', 'UY', 'VE', 'VN', 'ZA'] charts = { 'daily_top_songs': { 'chart_type': 'regional', 'frequency': 'daily', 'number_of_countries': 68 }, 'weekly_top_songs': { 'chart_type': 'regional', 'frequency': 'weekly', 'number_of_countries': 69 }, } file_pattern = '{chart_type}_{{country}}_{frequency}_{date:%Y-%m-%d}.json' # snowflake table name template staging_raw_table = 'staging_raw_spotify_charts' # https://whimsical.com/spotify-charts-denormalisation-D43dK53mVCWJ8UcL2tPv8C chart_table = 'spotify_chart' spotify_metadata_tables = [ 'spotify_chart', 'spotify_track', 'spotify_label', 'spotify_track_artist', 'spotify_artist'] temp_staging_raw_table = ( 'temp_staging_raw_spotify_charts_{chart}_{{country}}_{date:%Y%m%d}') snowflake_error_limit = 1 jenkins_username = os.environ.get('JENKINS_USER', 'jenkinsjobrunner') jenkins_secrets_path = 'swf-shared-jenkins' jenkins_url = 'https://scheduler.theorchard.io' jenkins_job = 'swf-chartmetric-charts-exec' jenkins_job_params = { 'Environment': environment, 'PLATFORM_NAMES': 'spotify', 'KAFKA': True, 'WAIT_UNTIL_COMPLETE': True, 'DAYS_BACK': 3 }