"""Config for AWA Ingestion Workflow.""" import os environment = os.environ.get('Environment', 'dev') feed_name = 'qq' secrets_path = 'qq' feedid = 51 storeid = 1603 # bucket for dropped data sme_drop_bucket = os.environ.get( 'SME_FEED_INGESTION_DROP_BUCKET', 'dev-feed-drop') # bucket for archived data data_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') if environment == 'production': data_bucket = 'cucumbers' licensors = ('theorchard', 'sme') # File name/format versions keyed by the reporting date they take effect, # mirroring the queries// sub folders. Pick one with format_for(date). # 2022-10-01 is the earliest QQ reporting date; 2026-07-01 is the new P0CY # layout (4 new columns, CP_NAME removed, TheOrchard abbreviated to TO). formats = { '2022-10-01': { 'filename': 'P926_{date:%Y%m%d}_{date:%Y%m%d}_{licensor_filename}.txt', 'licensors': {'theorchard': 'TheOrchard', 'sme': 'SME,SMEJ'}, 'source_file_pattern': { 'theorchard': r'^P926_\d{8}_\d{8}_TheOrchard\.txt$', 'sme': r'^P926_\d{8}_\d{8}_SME(J)*\.txt$', }, }, '2026-07-01': { 'filename': 'P0CY_D_{date:%Y%m%d}_QQ_{licensor_filename}.txt', 'licensors': {'theorchard': 'TO', 'sme': 'SME,SMEJ'}, 'source_file_pattern': { 'theorchard': r'^P0CY_D_\d{8}_QQ_TO\.txt$', 'sme': r'^P0CY_D_\d{8}_QQ_SME(J)*\.txt$', }, }, } def format_for(date): """Return the file-format spec effective for a reporting date. Selects the latest version whose effective date is <= date, matching the SQLLoader queries// resolution. Args: date (str): Reporting date (YYYY-MM-DD). Returns: dict: The effective file-format spec (filename, licensors, source_file_pattern). """ effective = [ spec for since, spec in sorted(formats.items()) if since <= date] return effective[-1] # S3 locations for dropped and archived files s3 = { 'drop': { 'theorchard': ( 's3://{drop_bucket}/qq/in/'.format(drop_bucket=sme_drop_bucket)), 'sme': ( 's3://{drop_bucket}/qq/in/'.format(drop_bucket=sme_drop_bucket)) }, 'archive': { 'theorchard': ( '{feed_name}/archives/{{date}}/'.format(feed_name=feed_name)), 'sme': ( '{feed_name}/archives_sme/{{date}}/'.format(feed_name=feed_name)), } } snowflake_table_names = { 'staging_raw': 'staging_raw_qq', } snowflake_error_limit = 100