"""Config for the Chartmetric Charts Ingestion Workflow.""" import os environment = os.getenv('Environment', 'dev') feed_name = 'chartmetric_charts' secrets_path = 'chartmetric_charts' sf_batch_size = 5000 all_platform_names = [ 'amazon', 'appleMusic', 'deezer', 'linemusic', 'shazam', 'spotify', 'soundcloud', 'recochoku', 'tiktok', 'youtube', 'itunes'] platforms_to_check_if_chart_is_changed = [ 'appleMusic', 'deezer', 'soundcloud', 'shazam' ] log_table_name = 'chartmetric_charts_log' STAGING_TABLE_NAME = 'STAGING_FACT_CHARTMETRIC_CHARTS' KAFKA_CLUSTER_NAME = '{}-managed-kafka-cdc-destination'.format(environment) KAFKA_CONSUMER_GROUP = os.getenv('KAFKA_CONSUMER_GROUP', 'connect-neo4j_sink') KAFKA_SECURITY_PROTOCOL = 'SSL' KAFKA_TOPIC_PREFIX = 'etl.chartmetric.charts.' KAFKA_DLQ_TOPIC = 'dlq.chartmetric.charts.neo4jsink.status' ALBUM_ID_COLUMN_MAPPING = { 'amazon': 'AMAZON_ID', 'appleMusic': 'APPLE_MUSIC_ID', 'deezer': 'DEEZER_ID', 'itunes': 'APPLE_MUSIC_ID', 'shazam': 'APPLE_MUSIC_ID', 'spotify': 'SPOTIFY_ID' } OWS_CHARTS_BATCH_SIZE = 100