"""Config for Soundcloud Ingestion Workflow.""" import os environment = os.environ.get('Environment', 'dev') feed_name = 'soundcloud' secrets_path = 'swf-soundcloud' feedid = 43 storeid = 1505 # licensors licensors = { 'theorchard': 'orchard', 'sme': 'sme', } # filenames for theorchard and sme # Generally same for both - keeping this for flexiblity) filename = { 'theorchard': 'PLT2_D_{date:%Y%m%d}_{date:%Y%m%d}_{type}.tsv', 'sme': 'PLT2_D_{date:%Y%m%d}_{date:%Y%m%d}_{type}.tsv', } # orchard bucket for dropped data drop_bucket = os.environ.get( 'FEED_INGESTION_DROP_BUCKET', 'dev-feed-drop') # sme bucket for dropped data sme_drop_bucket = os.environ.get( 'SME_FEED_INGESTION_DROP_BUCKET', 'sme-ca-prod-partners') # bucket for archived data data_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') # drop and data buckets for different environments if environment in ['qa']: drop_bucket = 'qa-orcdbucket' data_bucket = 'qa-cucumbers' if environment in ['production', 'prod']: drop_bucket = 'prod-orcdbucket' data_bucket = 'cucumbers' def build_path(lst_dirs): """Build paths from a list of directories. params: - lst_dirs (List[str]): Receive a list of strings. """ return os.path.join(*lst_dirs) # S3 locations for dropped and archived files s3 = { 'drop': { 'theorchard': build_path([ 'feed-drop', 'SoundCloud', '{date:%Y}/{date:%m}/{date:%d}/' ]), 'sme': build_path([ 'soundcloud/in', '{date:%Y}/{date:%m}/{date:%d}/' ]), }, 'archive': { 'theorchard': build_path([ 'SoundCloud', 'archives', '{date:%Y}/{date:%m}/{date:%d}/' ]), 'sme': build_path([ 'SoundCloud', 'sme', 'archives', '{date:%Y}/{date:%m}/{date:%d}/' ]), } } # List of Reports reports = [ 'Trackinformation', 'Streamlevelreport', 'Playlistreporting', 'Customeranalytics', 'Customerdata' ] # snowflake table names # ORDER - licensor, report and date snowflake_table_names = { 'temp_staging_raw_table_template': 'temp_staging_raw_soundcloud_{}_{}_{}', 'staging_raw_table_template': 'staging_raw_soundcloud_{}', 'staging_raw_table': 'staging_raw_soundcloud_streamlevelreport' } # config for the update_dim_tables task dimension_tables = { 'tables_to_update': [ 'dim_playlist' ], 'include_to_report': [ 'dim_playlist' ], 'sns_topic': os.environ.get( 'FEED_INGESTION_SNS_TOPIC', 'arn:aws:sns:us-east-1:437795906767:dev-swf-feed-ingestion:' 'e17576f8-b905-45a9-b863-0eaceca54f4d' ) } snowflake_error_limit = 120