"""Config for TikTok and Douyin Ingestion Workflow.""" import os # Environment environment = os.environ.get('Environment', 'dev') # Feed configuration store_id = 1202 feed_id = 45 feed_name = 'tiktok' secrets_path = 'tiktok' sme_secrets_path = 'swf-sme-ca-prod-partners-s3-access' # Supported reports reports = ['TikTok', 'Douyin'] DBT_REQUIRED_LICENSORS = ['theorchard', 'sme'] # orchard bucket for dropped data drop_bucket = os.environ.get( 'FEED_INGESTION_DROP_BUCKET', 'dev-feed-drop') # sme bucket for dropped data sme_drop_bucket = os.environ.get( 'SME_FEED_INGESTION_DROP_BUCKET', 'sme-ca-prod-partners') # altafonte bucket for dropped data altafonte_drop_bucket = os.environ.get( 'ALTAFONTE_FEED_INGESTION_DROP_BUCKET', 'prod-orcd-ftp') # bucket for archived data data_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') # drop and data buckets for different environments if environment in ['dev']: drop_bucket = 'prod-orcd-ftp' data_bucket = 'dev-cucumbers' if environment in ['qa']: drop_bucket = 'prod-orcd-ftp' data_bucket = 'qa-cucumbers' if environment in ['production', 'prod']: drop_bucket = 'prod-orcd-ftp' data_bucket = 'cucumbers' licensors = { 'theorchard': { 'reports': reports, 's3': { 'drop': 'ftp/data_feed/tiktok/', 'archive': 'TikTok/archives/{date:%Y-%m-%d}/', 'processed': 'TikTok/processed/{date:%Y-%m-%d}/', 'filename': '{report}_Daily_Usage_Report_{date:%Y%m%d}.tsv.zip', 'drop_bucket': drop_bucket }, 's3_sme': { 'drop': 'tiktok/in/orchard/daily_trends/', 'archive': 'TikTok/archives/{date:%Y-%m-%d}/', 'processed': 'TikTok/processed/{date:%Y-%m-%d}/', 'filename': '{report}_Daily_Usage_Report_{date:%Y%m%d}.tsv.zip', 'drop_bucket': sme_drop_bucket }, }, 'sme': { 'reports': reports, 's3': { 'drop': 'tiktok/in/sme/daily_trends/', 'archive': 'TikTok/archives_sme/{date:%Y-%m-%d}/', 'processed': 'TikTok/archives_sme/{date:%Y-%m-%d}/', 'filename': 'PLY3_D_{sme_report}_{date:%Y%m%d}.txt', 'drop_bucket': sme_drop_bucket }, }, 'smejp': { 'reports': ['TikTok'], 's3': { 'drop': 'tiktok/in/smej/japan/', 'archive': 'TikTok/archives_smejp/{date:%Y-%m-%d}/', 'processed': 'TikTok/archives_smejp/{date:%Y-%m-%d}/', 'filename': 'PLY3_D_{sme_report}_{date:%Y%m%d}.txt', 'drop_bucket': sme_drop_bucket }, }, 'smejpintl': { 'reports': ['TikTok'], 's3': { 'drop': 'tiktok/in/smej/ww/', 'archive': 'TikTok/archives_smejpintl/{date:%Y-%m-%d}/', 'processed': 'TikTok/archives_smejpintl/{date:%Y-%m-%d}/', 'filename': 'PLY3_D_{sme_report}_{date:%Y%m%d}.txt', 'drop_bucket': sme_drop_bucket }, }, 'altafonte': { 'reports': reports, 's3': { 'drop': 'ftp/altafonte/merlin/tiktok/{date:%Y-%m-%d}/', 'archive': 'TikTok/archives_altafonte/{date:%Y-%m-%d}/', 'processed': 'TikTok/archives_altafonte/{date:%Y-%m-%d}/', 'filename': ('altafonte-network-s-l_{report}_' '{date:%Y%m%d}_Daily-Usage.tsv.zip'), 'drop_bucket': altafonte_drop_bucket }, 'reports_new': ['TikTok'], 's3_new': { 'drop': 'tiktok/in/altafonte/daily_trends/', 'archive': 'TikTok/archives_altafonte/{date:%Y-%m-%d}/', 'processed': 'TikTok/archives_altafonte/{date:%Y-%m-%d}/', 'filename': '{report}_Daily_Usage_Report_{date:%Y%m%d}.tsv.zip', 'drop_bucket': sme_drop_bucket }, } } # uses in filename template sme_report_mapping = { 'TikTok': 'ROW', 'Douyin': 'CN' } altafonte_source_bucket_switch_date = '2024-10-30' # Snowflake destination for the staging_raw and temp tables # (separate from the default facts.prod used for dim tables) consumer_sf = { 'db': os.environ.get('CONSUMER_DB', 'consumer'), 'schema': os.environ.get('CONSUMER_SCHEMA', 'tiktok'), } # Temp Staging Raw Table temp_staging_raw_table = 'temp_staging_raw_{licensor}_{report}_{date:%Y%m%d}' # Staging Raw Table staging_raw_table = 'staging_raw_tiktok' # The maximum allowed number of errors in the file snowflake_error_limit = 10 # trigger DBT: dbt-scheduler-analytics-build-views # MODEL=tag:tiktok_v2 # FORCE_MODEL_UPGRADE=True jenkins_config = { 'feeds_required_for_jenkins_build': [ f'tiktok_{licensor}' for licensor in DBT_REQUIRED_LICENSORS ], 'jenkins_username': os.environ.get('JENKINS_USER', 'jenkinsjobrunner'), 'jenkins_secrets_path': 'swf-shared-jenkins', 'jenkins_url': 'https://scheduler.theorchard.io', 'jenkins_job': 'dbt-scheduler-analytics-parametrized-pipeline', 'jenkins_job_params': { 'MODELS': 'tag:tiktok_v2', 'CLEAR_CACHE': True, 'ENVIRONMENT': environment, 'TRIGGER_TADAS': True, } }