"""Config for YouTube claim Snowflake-only Ingestion Workflow.""" import os environment = os.environ.get('Environment', 'dev') feed_name = 'youtube_claim' secrets_path = 'youtube_claim' # SELECT feedid from dim_feed where feedname = 'YouTube Claim' feedid = 14 storeid = 453 youtube_report_full_name = 'content_owner_active_claims_a3' expected_bucket_owner = os.environ.get('EXPECTED_BUCKET_OWNER', '437795906767') licensors = ['theorchard', 'sme'] # root AWS bucket s3_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') if environment == 'production': s3_bucket = 'cucumbers' s3_dir_path = 'Youtube_claims/archives/{date:%Y-%m-%d}/{licensor}/' s3_archive_path_template = f's3://{s3_bucket}/{s3_dir_path}' s3_split_dir_path = ('temp-feed-ingestion/' 'Youtube_claims/{date:%Y-%m-%d}/{licensor}/') s3_split_path_template = f's3://{s3_bucket}/{s3_split_dir_path}' snowflake_table_names = { 'staging_raw_theorchard': 'staging_raw_youtube_claim', 'staging_raw_sme': 'staging_raw_youtube_claim_sme', 'staging_fact_analytics': 'staging_fact_analytics_{feed_name}_{date:%Y%m%d}' } temp_table_name_template = '{staging_raw_table}_{date:%Y%m%d}_{account}' source_file_pattern = fr'^{youtube_report_full_name}\.\w+\.csv\.gz$' source_filename_template = f'{youtube_report_full_name}.{{account}}.csv.gz' youtube_reporting_api_service_name = 'youtubereporting' youtube_reporting_api_version = 'v1' jobs_meta_path = os.path.join( os.path.dirname(__file__), 'jobs_no_playlists.json') credentials_paths = { 'theorchard': '/'.join( os.path.dirname(__file__).split('/')[:-2] + ['conf', 'youtube_access_token.json']) } # Content owner IDs generated by YouTube. orch = 'J8vAyKuNSYBIN_9RIdxggQ' ent = 'oVAQ96fyMa57I6WozL2gUw' cms_dict = { orch: 'ORCH', ent: 'ENT' } sme_athena_database = os.environ.get('SME_ATHENA_DATABASE') sme_athena_temp_database = os.environ.get('SME_ATHENA_TEMP_DATABASE') athena_workgroup = os.environ.get('ATHENA_WORKGROUP', 'primary') athena_aws_region = os.environ.get('ATHENA_AWS_REGION', 'us-east-1')