"""Config for Pandora Ingestion Workflow.""" import os # feed information feed_name = 'pandora' secrets_path = 'pandora' feedid = 3 # SELECT feedid from dim_feed where feedname = 'Pandora' feed_version = '2.0' # environment to run in environment = os.environ.get('ENVIRONMENT', 'dev') # bucket for dropped data drop_bucket = os.environ.get('FEED_INGESTION_DROP_BUCKET', 'dev-feed-drop') sme_drop_bucket = os.environ.get('SME_FEED_INGESTION_DROP_BUCKET', 'sme-ca-prod-partners') # bucket for archived data data_bucket = os.environ.get('FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') # name of streams file pandora_streams_file = '{licensor}_{{country}}_{YYYY}-{MM}-{DD}.txt.bz2' # name of metadata file pandora_metadata_file = '{licensor}_metadata_{YYYY}-{MM}-{DD}.txt.bz2' # list of countries countries = ['US'] # mapping from countries as they appear in the Pandora data # and countryname in the dim_country table country_mappings = { 'US': 'USA'} # mapping from countries to currencyid currencyid_mappings = { 'US': 170} # mapping from countries to currencycode and royaltycurrency currency_mappings = { 'US': 'USD'} # mapping licensor and licensor name in filename licensors = { 'theorchard': 'orchard', 'sme': 'sme', } # S3 locations for dropped and archived files s3 = { 'drop': { 'theorchard': ( 's3://{drop_bucket}/data/{{YYYY}}/{{MM}}/{{DD}}/'.format( drop_bucket=drop_bucket)), 'sme': ( 's3://{drop_bucket}/pandora/in/pandora-streams/' '{{YYYY}}/{{MM}}/{{DD}}/'.format(drop_bucket=sme_drop_bucket)) }, 'archive': { 'theorchard': ( 's3://{data_bucket}/{feed_name}/archives/{{date}}/'.format( data_bucket=data_bucket, feed_name=feed_name.capitalize())), 'sme': ( 's3://{data_bucket}/{feed_name}/archives_sme/{{date}}/'.format( data_bucket=data_bucket, feed_name=feed_name.capitalize())), } } snowflake_table_names = { 'staging_raw': 'staging_raw_pandora', 'streams_temp_raw_template': 'pandora_streams_{licensor}_{country}_{date}', 'metadata_temp_raw_template': 'pandora_metadata_{licensor}_{date}' } # config for the update_dim_tables task dimension_tables = { 'tables_to_update': [ 'dim_sourcetype', 'dim_subscriptiontype', 'dim_subscriptionpaytier', 'dim_playlist', 'dim_user' ], 'include_to_report': [ 'dim_sourcetype', 'dim_subscriptiontype', 'dim_subscriptionpaytier' ], 'sns_topic': os.environ.get( 'FEED_INGESTION_SNS_TOPIC', 'arn:aws:sns:us-east-1:437795906767:dev-swf-feed-ingestion:' 'e17576f8-b905-45a9-b863-0eaceca54f4d') } default_snowflake_error_limit = 10 jenkins_config = { 'feeds_required_for_jenkins_build': ['pandora_licensorplaceholder'], # plus licensor in runtime, this is for get_overall_status + date # noqa 'jenkins_username': os.environ.get('JENKINS_USER', 'jenkinsjobrunner'), 'jenkins_secrets_path': 'swf_feed_ingestion', 'jenkins_url': 'https://scheduler.theorchard.io', 'jenkins_job': 'dbt-scheduler-analytics-pipeline', 'jenkins_job_params': {'TRIGGERED_BY': feed_name} }