"""Config for Apple Music Streams Ingestion Workflow.""" import os from feed_ingestion.flows.apple_music_streams.vendor_accounts import licensors environment = os.environ.get('Environment', 'dev') feed_name = 'apple_music' secrets_path = 'apple_music' feed_version = '3.0' cucumber_feed_name = 'AppleMusicStreams' store_id = 1 feedid = 4 # SELECT feedid from dim_feed where feedname = 'Apple Music' # root AWS bucket s3_bucket = os.environ.get('FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') if environment == 'production': s3_bucket = 'cucumbers' drop_bucket = os.environ.get('FEED_INGESTION_DROP_BUCKET', 'dev-feed-drop') s3 = { 'archive_bucket': ( 's3://{s3_bucket}/{feed_name}/archives/{{date:%Y-%m-%d}}/').format( s3_bucket=s3_bucket, feed_name=cucumber_feed_name), 'drop_bucket': ( 's3://{s3_drop_bucket}/feed-drop/{feed_name}/{{date:%Y-%m-%d}}/' ).format( s3_drop_bucket=drop_bucket, feed_name=cucumber_feed_name) } # naming queries and temporary table names convention create_temp_table_query_name = 'create_temp_table_{report}' temp_table_name = '{report}_{licensor}_{reporter_account}_{date:%Y%m%d}' temp_table_s3_full_path = '{s3_archive_bucket}{filename}' load_query_name = 'load_staging_raw_{report}' vendor_names_mapping = { '80028967': 'ORCHARD', '80029727': 'IODA', '80032046': 'FINETUNES', '80032543': 'PHONOFILE', '80034465': 'DKUA', '80031998': 'AWAL', '85320019': 'ALTAFONTE', } vendor_account_mapping = { '80028967': 1526, '80029727': 1682, '80032046': 2243, '80032543': 2341, '80034465': 2969, '80031727': 2151, '80031813': 2170, '85420853': 2170, '80032084': 2246, '80035150': 2913, '80028547': 1522, 'sme': 4632, 'awal': 2234, '85320019': 2818, } reports = { 'amStreams': { 'staging_raw_table': 'staging_raw_apple_music_streams', 'licensors': licensors.keys(), 'start_date': '2016-02-28', 'end_date': '2021-04-14', }, 'amStreamsSummary': { 'staging_raw_table': 'staging_raw_apple_music_summary_streams', 'licensors': licensors.keys(), 'start_date': '2021-03-01', 'is_consumer_reporting': True, }, 'amContent': { 'report_type': 'amContent', 'licensors': licensors.keys(), 'start_date': '2016-02-28', }, 'amContainer': { 'licensors': licensors.keys(), 'staging_raw_table': 'staging_raw_apple_music_container', # real start date 2021-04-14, # but data was backfilled from amStreams till 2021-04-15 'start_date': '2021-04-01', 'is_consumer_reporting': True, }, 'amContentDemographics': { 'staging_raw_table': 'staging_raw_apple_music_demographics', 'licensors': licensors.keys(), 'start_date': '2017-08-02', }, 'amShazam': { 'staging_raw_table': 'apple_music_shazam_daily_events', 'licensors': licensors.keys(), 'start_date': '2019-11-01', }, 'amNonRoyaltyStreams': { 'staging_raw_table': 'staging_raw_apple_music_nonroyalty_streams', # available for the all theorchard accounts since 2020-05-15 'licensors': licensors.keys() - {'altafonte'}, 'start_date': '2018-02-23', 'end_date': '2021-04-14', }, 'amNonRoyaltySummaryStreams': { 'staging_raw_table': 'staging_raw_apple_music_nonroyalty_summary_streams', 'licensors': licensors.keys(), 'start_date': '2021-04-08', }, 'amLibraryEvents': { 'staging_raw_table': 'staging_raw_apple_music_libraryevents', 'licensors': licensors.keys(), 'start_date': '2020-03-08', }, 'amTotalLibraryAdds': { 'staging_raw_table': 'staging_raw_apple_music_totallibraryadds', 'licensors': licensors.keys(), 'start_date': '2020-03-08', }, 'amSubscriptionReference': { 'staging_raw_table': 'staging_raw_apple_music_sub_reference', 'licensors': licensors.keys(), 'start_date': '2020-11-09', 'end_date': '2021-04-12', }, # Apple Music uniques reports 'amArtists': { 'staging_raw_table': 'staging_raw_apple_music_artists', 'licensors': licensors.keys(), 'start_date': '2019-05-13', }, 'amPlaylists': { 'staging_raw_table': 'staging_raw_apple_music_playlists', 'licensors': licensors.keys(), 'start_date': '2019-05-13', }, 'amSongs': { 'staging_raw_table': 'staging_raw_apple_music_songs', 'licensors': licensors.keys(), 'start_date': '2019-05-13', }, } # Consumer reporting destination for staging_raw tables of reports flagged # with 'is_consumer_reporting'. All other tables stay on the flow's db/schema. consumer_sf = { 'db': os.environ.get('CONSUMER_DB', 'CONSUMER_REPORTING'), 'schema': os.environ.get('CONSUMER_SCHEMA', 'APPLE'), } common_reports = ['amContent'] # update upc & isrc using apple_id_mapping table for these reports reports_to_update = ['amLibraryEvents', 'amTotalLibraryAdds'] # licensors to update dim_tables, load to fa active_licensors = ['theorchard', 'sme', 'awal'] apple_id_mapping_tables = { 'theorchard': 'apple_id_mapping', 'sme': 'sony_apple_id_mapping', 'awal': 'awal_apple_id_mapping', 'altafonte': 'apple_id_mapping', } # config for the update_dim_tables task dimension_tables = { 'tables_to_update': [ 'dim_sourcetype', 'dim_subscriptiontype', 'dim_subscriptionpaytier', 'dim_licensor', 'dim_playlist', 'dim_playlist_container' ], 'include_to_report': [ 'dim_sourcetype', 'dim_subscriptiontype', 'dim_subscriptionpaytier', 'dim_licensor' ], 'sns_topic': os.environ.get( 'FEED_INGESTION_SNS_TOPIC', 'arn:aws:sns:us-east-1:437795906767:dev-swf-feed-ingestion:' 'e17576f8-b905-45a9-b863-0eaceca54f4d') } # The maximum allowed number of errors in the file snowflake_error_limit = int(os.environ.get('SNOWFLAKE_ERROR_LIMIT', 10)) max_concurrent_executions = { 'theorchard': 1, 'sme': 10, 'awal': 5, 'altafonte': 10, } switch_over_date_on_summary_reports = '2021-04-14' switch_over_date_on_summary_reports_awal = '2021-03-31' # uploading awal data from s3 awal_drop_bucket = os.environ.get( 'FEED_INGESTION_AWAL_DROP_BUCKET', 'prod-orcd-awal-drop') awal_s3_path = ( 's3://{drop_bucket}/AWAL_TO_ORCH/ANALYTICS_BACKFILL/{s3_folder}/' 'version={version}/year={date:%Y}/month={date:%Y%m}/' 'day={date:%Y%m%d}/') awal_s3_versions = { 'amStreams': { 's3_folder': 'applemusic_streams', 'versions': [ {'since': '2016-04-06', 'version': '1'}, {'since': '2017-03-23', 'version': '2'}, {'since': '2017-12-18', 'version': '3'}, {'since': '2019-05-13', 'version': '4'}, ]}, 'amStreamsSummary': { 's3_folder': 'applemusic_summary_streams', 'versions': [ {'since': '2021-03-01', 'version': '1'}, {'since': '2021-06-07', 'version': '2'}, ]}, 'amContent': { 's3_folder': 'applemusic_content', 'versions': [ {'since': '2015-06-30', 'version': '1'}, {'since': '2015-07-23', 'version': '2'}, {'since': '2016-04-06', 'version': '3'}, {'since': '2017-08-02', 'version': '4'}, {'since': '2017-12-18', 'version': '5'}, {'since': '2021-07-19', 'version': '6'}, ]}, 'amContainer': { 's3_folder': 'applemusic_container_summary', 'versions': [ {'since': '2021-04-01', 'version': '1'}, ]}, 'amContentDemographics': { 's3_folder': 'applemusic_content_demographics', 'versions': [ {'since': '2017-12-18', 'version': '1'}, {'since': '2023-04-24', 'version': '2'}, ]}, 'amShazam': { 's3_folder': 'applemusic_shazam', 'versions': [ {'since': '2019-08-07', 'version': '1'}, {'since': '2019-12-10', 'version': '2'}, ]}, 'amNonRoyaltyStreams': {}, 'amNonRoyaltySummaryStreams': {}, 'amLibraryEvents': { 's3_folder': 'applemusic_library_events', 'versions': [ {'since': '2021-03-02', 'version': '1'}, ]}, 'amTotalLibraryAdds': {}, 'amSubscriptionReference': {}, 'amArtists': {}, 'amPlaylists': {}, 'amSongs': {}, } DBT_REQUIRED_REPORTS = [ 'amContentDemographics', 'amContainer', 'amStreamsSummary' ] jenkins_config = { 'feeds_required_for_jenkins_build': [ f'apple_music_{licensor}_{report}' for licensor in ['theorchard', 'sme'] for report in DBT_REQUIRED_REPORTS ], 'jenkins_username': os.environ.get('JENKINS_USER', 'jenkinsjobrunner'), 'jenkins_secrets_path': 'swf-shared-jenkins', 'jenkins_url': 'https://scheduler.theorchard.io', 'jenkins_job': 'dbt-scheduler-analytics-pipeline', 'jenkins_job_params': {'TRIGGERED_BY': feed_name} }