"""Config for Meta Daily Ingestion Workflow.""" import os environment = os.environ.get('Environment', 'dev') feed_name = 'meta_daily' secrets_path = 'meta_daily' sme_secrets_path = 'swf-sme-ca-prod-partners-s3-access' data_bucket = os.environ.get('FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') sme_drop_bucket = os.environ.get( 'SME_FEED_INGESTION_DROP_BUCKET', 'sme-ca-prod-partners' ) LICENSOR_SME = 'sme' LICENSOR_ORCHARD = 'theorchard' # Both tables share the same source schema (tab-delimited): # isrc | country_code | product | upc | events # Note: delivery spec says pipe delimiter; actual files are tab-delimited. reports = { 'consumption': { 'staging_raw_table': 'staging_raw_meta_consumption', 'events_column': 'VIEWS', 'sources': [ { 'licensor': LICENSOR_SME, 'drop_path': 'facebook/in/sme/daily/v1/{date}', 'file_name': 'sony_consumption_export_{date}.txt', }, { 'licensor': LICENSOR_ORCHARD, 'drop_path': 'facebook/in/orchard/daily/v1/{date}', 'file_name': 'orchard_consumption_export_{date}.txt', }, ], }, 'production': { 'staging_raw_table': 'staging_raw_meta_production', 'events_column': 'CREATIONS', 'sources': [ { 'licensor': LICENSOR_SME, 'drop_path': 'facebook/in/sme/daily/v1/{date}', 'file_name': 'sony_production_export_{date}.txt', }, { 'licensor': LICENSOR_ORCHARD, 'drop_path': 'facebook/in/orchard/daily/v1/{date}', 'file_name': 'orchard_production_export_{date}.txt', }, ], }, } s3 = { 'drop': { 'bucket': sme_drop_bucket, }, 'archive': { 'bucket': data_bucket, 'path': 'MetaDaily/archives/{date}', }, } # Snowflake database/schema for temp tables and Snowflake stages # (separate from the final staging_raw tables visible to end users) stage_sf = { 'db': os.environ.get('SNOWFLAKE_STAGE_DATABASE', 'FACTS'), 'schema': os.environ.get('SNOWFLAKE_STAGE_SCHEMA', 'PROD'), }