"""Config for Line Ingestion Workflow.""" import os environment = os.environ.get('Environment', 'dev') feed_name = 'line' secrets_path = 'swf-line' feedid = 42 storeid = 1503 licensors = { 'theorchard': dict( zipped_filename='LMA_bl00000000029c7cd8_{date:%Y%m%d}.zip', filename='LMA_bl00000000029c7cd8_{date:%Y%m%d}_{type}.csv', drop='feed-drop/LINE/line-music-analytics/{date:%Y%m%d}/', zipped_archive='LINE/zipped_archives/{date:%Y-%m-%d}/', archive='LINE/archives/{date:%Y-%m-%d}/', line_api_credentials={ 'client_id': os.environ.get('LINE_API_CLIENT_ID'), 'client_secret': os.environ.get('LINE_API_CLIENT_SECRET') } ), 'smej': dict( zipped_filename='LMA_bl0000000002904827_{date:%Y%m%d}.zip', filename='LMA_bl0000000002904827_{date:%Y%m%d}_{type}.csv', drop='feed-drop/LINE/smej/line-music-analytics/{date:%Y%m%d}/', zipped_archive='LINE/smej/zipped_archives/{date:%Y-%m-%d}/', archive='LINE/smej/archives/{date:%Y-%m-%d}/', line_api_credentials={ 'client_id': os.environ.get('LINE_SMEJ_API_CLIENT_ID'), 'client_secret': os.environ.get('LINE_SMEJ_API_CLIENT_SECRET') } ) } # bucket for archived data data_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') if environment == 'production': data_bucket = 'cucumbers' # types of reports reports = [ 'place', 'type', 'pref', 'sex_age_type', 'sex_age_region', 'sex_age', 'sex_age_hour'] # snowflake table name template staging_raw_table = 'staging_raw_line_{}' staging_fact_table = 'staging_fact_line_{date}' fa_staging_raw_table = 'staging_raw_line_place' # actual table names: # staging_raw_line_place, staging_raw_line_type, staging_raw_line_pref # staging_raw_line_sex_age_type, staging_raw_line_sex_age_region, # staging_raw_line_sex_age, staging_raw_line_sex_age_hour