"""Config for Deezer Ingestion Workflow.""" from datetime import datetime import os environment = os.environ.get('Environment', 'dev') feed_name = 'deezer_daily' secrets_path = 'deezer_daily' if environment == 'prod' else 'swf-deezer' feedid = 9 # SELECT feedid from dim_feed where feedname = 'Deezer' licensors = ['theorchard', 'sme', 'altafonte'] # file name as ingested from the service drop_file_name = { 'theorchard_v1': 'TheOrchard_{datestamp_YYYYMMDD}_{datestamp_YYYYMMDD}.zip', 'theorchard_v3': 'deezer_daily_report_theorchard_activity_metrics_{datestamp_YYYYMMDD}.zip', # noqa 'sme_v2': 'sony_{datestamp_YYYYMMDD}_{datestamp_YYYYMMDD}.zip', 'sme_v3': 'deezer_daily_report_sony_activity_metrics_{datestamp_YYYYMMDD}.zip', 'altafonte_v4': '{datestamp_YYYYMMDD}-AltafonteMERLIN.zip' } local_downloads_dir = './deezer_downloads' # root AWS bucket s3_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') sme_drop_bucket = os.environ.get('SME_FEED_INGESTION_DROP_BUCKET', 'sme-ca-prod-partners') sme_drop_path = os.environ.get('SME_DROP_PATH', 'deezer/in/PG45/') altafonte_drop_bucket = os.environ.get('ALTAFONTE_FEED_INGESTION_DROP_BUCKET', 'prod-orcd-ftp') altafonte_drop_path = os.environ.get('ALTAFONTE_DROP_PATH', 'ftp/altafonte/merlin/dzr-deezer/{date:%Y-%m-%d}/') # noqa altafonte_reports_prefix = 'AltafonteMERLIN_{date:%Y%m%d}_{date:%Y%m%d}' if environment == 'production': s3_bucket = 'cucumbers' # TODO: define exact dates for v3 switch spec_version = { 'theorchard': [ {'since': '2000-01-01', 'version': 1}, {'since': '2024-02-01', 'version': 3} ], 'sme': [ {'since': '2000-01-01', 'version': 2}, {'since': '2024-01-01', 'version': 3} ], 'altafonte': [ {'since': '2024-05-01', 'version': 4}, ], } zephir = { 'username': os.environ.get('ZEPHIR_DEEZER_USERNAME', 'fake_deezer_user'), 'password': os.environ.get('ZEPHIR_DEEZER_PASSWORD', 'FakePassword123'), 'host': os.environ.get('ZEPHIR_DEEZER_HOST', 'https://zephir.deezer.com'), 'path': os.environ.get('ZEPHIR_DEEZER_PATH', '/external/download?file=daily/') } # TODO: rename _bucket to _full_path s3 = { 'archive_bucket': 's3://{s3_bucket}/DeezerV{spec_version}' '/archives/{datestamp}/{licensor}/', 'temp_staging_raw_bucket': 's3://{s3_bucket}/DeezerV{spec_version}' '/temp/{datestamp}/{licensor}/', } # intermediate entity to avoid v3 names duplication names_alias = { 'theorchard_v1': 'v1', 'sme_v2': 'v2', 'theorchard_v3': 'v3', 'sme_v3': 'v3', 'altafonte_v4': 'v4', } snowflake_table_names = { 'staging_raw': { 'v1': 'staging_raw_deezer_v1', 'v2': 'staging_raw_deezer_v2', 'v3': [ 'staging_raw_deezer_v3_activity_metrics', 'staging_raw_deezer_v3_detailed_content_identifiers', 'staging_raw_deezer_v3_playlist_metrics', 'staging_raw_deezer_v3_stream_source', 'staging_raw_deezer_v3_user_activity_metrics', 'staging_raw_deezer_v3_user_details', 'staging_raw_deezer_v3_vendor_product_details', 'staging_raw_deezer_v3_fraud_report' ], 'v4': 'staging_raw_deezer_v4', }, } # source files format specs file_suffixes = { 'v1': ['.txt', '_TB.txt'], 'v2': ['_FR.txt', '_ROW.txt', '_TB.txt'], 'v3': [ 'sony_activity_metrics_daily.csv', 'sony_detailed_content_identifiers_daily.csv', 'sony_playlist_metrics.csv', 'sony_stream_source.csv', 'sony_user_activity_metrics.csv', 'sony_user_details.csv', 'sony_vendor_product_details_daily.csv', 'fraud_report_*.txt' ], 'v4': ['.txt', '_TB.txt'], } v3_files_fraud = ['fraud_report_*.txt'] # Snowflake destination for the fraud report table # (separate from the default facts.prod) fraud_report_sf = { 'db': os.environ.get('FRAUD_REPORTING_DB', 'fraud_reporting'), 'schema': os.environ.get('FRAUD_REPORTING_SCHEMA', 'deezer'), } # Mapping from licensor name to the identifier Deezer uses # in fraud report filenames. # e.g. sme -> fraud_report_sony-YYYYMMDD.txt, # theorchard -> fraud_report_theorchard-YYYYMMDD.txt fraud_report_licensor_names = { 'sme': 'sony', 'theorchard': 'theorchard', } # config for the update_dim_tables task dimension_tables = { 'tables_to_update': [ 'dim_playlist', 'dim_user', ], 'include_to_report': [ 'dim_playlist', 'dim_user', ], 'sns_topic': os.environ.get( 'FEED_INGESTION_SNS_TOPIC', 'arn:aws:sns:us-east-1:437795906767:dev-swf-feed-ingestion:' 'e17576f8-b905-45a9-b863-0eaceca54f4d') } jenkins_config = { 'feeds_needed_for_jenkins_build': ['deezer_daily_licensorplaceholder'], # plus licensor in runtime, this is for get_overall_status + date # noqa 'jenkins_username': os.environ.get('JENKINS_USER', 'jenkinsjobrunner'), 'jenkins_secrets_path': 'swf-shared-jenkins', 'jenkins_url': 'https://scheduler.theorchard.io', 'jenkins_job': 'dbt-scheduler-analytics-pipeline', 'jenkins_job_params': {'TRIGGERED_BY': feed_name} } def get_version(rules, lookup_date): """Get version of the report by date.""" version = '' for rule in rules: period_start_str = rule['since'] period_start = datetime.strptime( period_start_str, '%Y-%m-%d').date() if period_start > lookup_date: break version = rule['version'] return version