"""Config for Ingestion Workflow.""" import os feed_name = 'spotify_marquee' secrets_path = 'spotify_marquee' feed_version = '1.0' s3_feed_name = 'SpotifyMarquee' environment = os.environ.get('Environment', 'dev') # bucket for archived data data_bucket = os.environ.get('FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') # bucket for dropped data drop_bucket = os.environ.get( 'FEED_INGESTION_DROP_BUCKET', 'dev-orcdbucket') s3 = { 'archive': ( 's3://{data_bucket}/{feed_name}/archives/{{date:%Y-%m-%d}}/'.format( data_bucket=data_bucket, feed_name=s3_feed_name)), 'drop': ( 's3://{data_bucket}/feed-drop/SpotifyMarquee/'.format( data_bucket=drop_bucket)), } temp_table_name = '{report_name}_{date:%Y%m%d}' snowflake_error_limit = 10 create_temp_table_query_name = 'create_temp_table_{report_name}' load_query_name = 'load_staging_raw_{report_name}' reports = { 'segment_level': { 'file_pattern': 'Marketplace Campaign Report - Segment Level - {date:%m%d%Y}.csv', # NOQA: E501 'staging_raw_table': 'staging_raw_spotify_marquee_segment_level_v2', }, 'campaign_level': { 'file_pattern': 'Marketplace Campaign Report - Campaign Level - {date:%m%d%Y}.csv', # NOQA: E501 'staging_raw_table': 'staging_raw_spotify_marquee_campaign_level_v2', }, }