"""Config for Amazon DataPulse Ingestion Workflow.""" import os from feed_ingestion.conf.config import ENV, SF_PARAMS feed_name = 'amazon_datapulse' feed_version = '1.0' secrets_path = 'amazon_datapulse' DATE_FORMAT = '%Y-%m-%d' archive_bucket = os.environ.get('FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') archive_s3_full_path_template = 'AmazonDataPulse/{report_name}/full/{date}/' archive_s3_partitioned_path_template = ( 'AmazonDataPulse/{report_name}/partitioned/{partition}/' ) def get_staging_raw_database_schema(report): """Get staging raw database, schema, and table name for a report.""" report_config = reports[report] if ENV in {'prod', 'qa'}: database = report_config['snowflake_database'] schema = report_config['snowflake_schema'] if ENV == 'qa': database = f'qa_{database}' else: # for dev and others fallback to defaults specified in ENV database = SF_PARAMS['db'] schema = SF_PARAMS['schema'] return database, schema, report_config['staging_raw_table'] reports = { 'fraud_report': { 'athena_source_table': 'fraud_report', 'snowflake_database': 'fraud_reporting', 'snowflake_schema': 'amazon', 'staging_raw_table': 'staging_raw_amazon_fraud_report', 'partitions': { 'cadence': str, 'region': str, 'report_date': str, 'entity_name': str, 'service': str, 'music_territory': str, }, }, 'daily_play_events': { 'athena_source_table': 'daily_play_events', 'snowflake_database': 'consumer_reporting', 'snowflake_schema': 'amazon', 'staging_raw_table': 'staging_raw_amazon_daily_play_events', 'partitions': { 'service': str, 'region': str, 'marketplace': str, 'territory': str, 'year': str, 'month': str, 'day': str, }, }, 'daily_customer_metadata': { 'athena_source_table': 'daily_customer_metadata', 'snowflake_database': 'consumer_reporting', 'snowflake_schema': 'amazon', 'staging_raw_table': 'staging_raw_amazon_daily_customer_metadata', 'partitions': { 'region': str, 'marketplace': str, 'territory': str, 'year': str, 'month': str, 'day': str, }, }, 'daily_playlist_metadata': { 'athena_source_table': 'daily_playlist_metadata', 'snowflake_database': 'consumer_reporting', 'snowflake_schema': 'amazon', 'staging_raw_table': 'staging_raw_amazon_daily_playlist_metadata', 'partitions': { 'region': str, 'marketplace': str, 'territory': str, 'year': str, 'month': str, 'day': str, }, }, } athena_source_database = os.environ.get( 'AMAZON_FRAUD_REPORT_ATHENA_SOURCE_DATABASE', 'amazon_music_reporting_datalake_link', ) athena_workgroup = os.environ.get( 'AMAZON_FRAUD_REPORT_ATHENA_WORKGROUP', 'primary' ) def get_contextified_feed_name(report_name, partition=None): """Get feed name with report name context.""" assert report_name in reports parts = [feed_name, report_name] if partition: # lowercase all partition keys to match report config partition = {k.lower(): str(v).lower() for k, v in partition.items()} report_config = reports[report_name] parts.extend( partition[key] for key in report_config['partitions'].keys() ) contextified_feed_name = '_'.join(parts) contextified_feed_name = ''.join( c if c.isalnum() else '_' for c in contextified_feed_name ) return contextified_feed_name def get_partition_date(partition): """Return YYYY-MM-DD date string derived from partition keys. Supports two partition shapes: * `report_date` — used as-is. * `year` / `month` / `day` — combined into YYYY-MM-DD with zero-padding. """ if 'report_date' in partition: return partition['report_date'] if all(k in partition for k in ('year', 'month', 'day')): return '{}-{}-{}'.format( partition['year'], partition['month'].zfill(2), partition['day'].zfill(2), ) return None