"""Config file for the iTunes data ingestion workflow.""" import os feed_name = 'itunes' s3_feed_name = 'iTunes' secrets_path = 'iTunes' feedid = 13 # SELECT feedid from dim_feed where feedname = 'iTunes' expected_bucket_owner = os.environ.get('EXPECTED_BUCKET_OWNER', '437795906767') storeid = 1 staging_raw_table = 'staging_raw_itunes' staging_fact_table_template = ( 'staging_fact_analytics_itunes_{licensor}_{date:%Y%m%d}') environment = os.environ.get('Environment', 'dev') s3_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') if environment == 'production': s3_bucket = 'cucumbers' # The maximum allowed number of errors in the file snowflake_error_limit = int(os.environ.get('SNOWFLAKE_ERROR_LIMIT', 1)) # DS-2156 Refactoring settings max_concurrent_executions = { 'theorchard': 1, 'sme': 5, 'awal': 5, 'altafonte': 5, } awal_version = '1' # for itunes/version=1/ awal_file_pattern = 'D_D_{vendor}_{date:%Y%m%d}{version}{format}' # todo reimplement to use reporter get_report_versions() awal_s3_versions = { 's3_folder': 'itunes', 'versions': [ {'since': '2016-07-01', 'version': '', 'format': '.txt.gz'}, {'since': '2019-05-13', 'version': '_V1_2', 'format': '.txt.gz'}, {'since': '2022-09-20', 'version': '_V1_3', 'format': '.txt.gz'}, ], } awal_drop_bucket = os.environ.get( 'FEED_INGESTION_AWAL_DROP_BUCKET', 'prod-orcd-awal-drop') awal_s3_path = ( 's3://{drop_bucket}/AWAL_TO_ORCH/ANALYTICS_BACKFILL/{s3_folder}/' 'version={version}/year={date:%Y}/month={date:%Y%m}/' 'day={date:%Y%m%d}/') # '85356999': 5516 was removed from theorchard due to DS-3306 licensors = { 'theorchard': ['80029727', '85420853', '80028967', '80031813', '80032046', '80032543', '80034465', '80031727', '80032084', '80035150', '80028547', '85320019'], 'sme': [ '80026921', '80030469', '80030905', '80030906', '80031738', '80031739', '80031740', '80031741', '80031742', '80031743', '80031744', '80031763', '80031941', '80033089', '80033090', '80033091', '80033092', '80033230', '85004044', '85406437', '85406441', '85406443', '85455523', '85455524', '85455525', '85455526', '85455527', '85520934', '85625470', '85625472', '85625473', '85625474', '85625475', '85625477', '85625478', '85625694', '85741870', '85741871', '85741872', '85741873', '85757805', '85774842'], 'awal': ['80031998', ], } s3 = { 'archive_bucket': ( f's3://{s3_bucket}/{s3_feed_name}/archives/{{date}}/') } report_name = 'itunes' temp_table_name = 'staging_raw_itunes_{licensor}_{reporter_account}_{date}' vendor_account_mapping = { '80028967': 1526, '80029727': 1682, '80032046': 2243, '80032543': 2341, '80034465': 2969, '85420853': 2170, '80031813': 2170, '80031727': 2151, '80032084': 2246, '80035150': 2913, '80028547': 1522, 'sme': 4632, 'awal': 2234, '85320019': 2818, } # TODO set correct values # Required/optional contexts. contexts_config = { # licensor, e.g. default, theorchard, sme 'default': { # report 'default': { 'optional': ['80035150'] } } }