"""Unit tests for Apple Music Streams Ingestion Workflow.""" from unittest import mock from unittest.mock import MagicMock from feed_ingestion.flows.apple_music_streams import flow def test_flow_decider_with_theorchard_licensor(): """Test normal decider execution...""" activity_statuses = MagicMock() activity_statuses.result = { 'check_available_reports.load_fact_analytics': True, 'update_dim_tables.sns_report_subject': 'Test subj', 'check_available_reports.update_library_reports': True, } schedule = MagicMock(return_value=activity_statuses) am_flow = flow.Flow() am_flow.decider(schedule, {'licensor': 'theorchard'}) # collect task names from schedule calls tasks = [call[0][0] for call in schedule.call_args_list] assert tasks == [ 'check_concurrent_status', 'bootstrap', 'reporter_to_s3', 'update_feed_file_status', 'check_available_reports', 'create_temp_staging_raw_tables', 'load_temp_staging_raw_tables', 'populate_staging_raw', 'drop_temp_stage_tables', 'set_status_to_populated_raw_table', 'set_status_ingested', 'update_dim_tables', 'update_apple_id_mapping', 'load_fact_analytics', 'set_status_ingested_to_fact_analytics_report', 'update_staging_raw_library_reports', 'load_aggregated_skips_and_saves', 'set_overall_status_ingested', 'build_jenkins_dbt', ] def test_flow_decider_with_sme_licensor(): """Test normal decider execution...""" activity_statuses = MagicMock() activity_statuses.result = { 'check_available_reports.load_fact_analytics': True, 'update_dim_tables.sns_report_subject': 'Test subj', 'check_available_reports.update_library_reports': True, } schedule = MagicMock(return_value=activity_statuses) am_flow = flow.Flow() am_flow.decider(schedule, {'licensor': 'sme'}) schedule.assert_has_calls([ mock.call('check_concurrent_status', mock.ANY), mock.call('bootstrap', mock.ANY, requires=[mock.ANY]), mock.call('reporter_to_s3', mock.ANY, requires=[mock.ANY]), mock.call('update_feed_file_status', mock.ANY, requires=[mock.ANY]), mock.call('check_available_reports', mock.ANY, requires=[mock.ANY]), mock.call( 'create_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('load_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('populate_staging_raw', mock.ANY, requires=mock.ANY), mock.call('drop_temp_stage_tables', mock.ANY, requires=mock.ANY), mock.call( 'set_status_to_populated_raw_table', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('update_dim_tables', mock.ANY, requires=mock.ANY), mock.call('update_apple_id_mapping', mock.ANY, requires=mock.ANY), mock.call('load_fact_analytics', mock.ANY, requires=mock.ANY), mock.call( 'set_status_ingested_to_fact_analytics_report', mock.ANY, requires=mock.ANY), mock.call( 'update_staging_raw_library_reports', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_overall_status_ingested', mock.ANY, requires=mock.ANY), ]) def test_flow_decider_if_fact_analytics_report_is_not_available_for_orchard(): """Test normal decider execution...""" activity_statuses = MagicMock() activity_statuses.result = { 'check_available_reports.load_fact_analytics': True, 'update_dim_tables.sns_report_subject': 'Test subj' } schedule = MagicMock(return_value=activity_statuses) am_flow = flow.Flow() am_flow.decider(schedule, {}) schedule.assert_has_calls([ mock.call('check_concurrent_status', mock.ANY), mock.call('bootstrap', mock.ANY, requires=[mock.ANY]), mock.call('reporter_to_s3', mock.ANY, requires=[mock.ANY]), mock.call('update_feed_file_status', mock.ANY, requires=[mock.ANY]), mock.call('check_available_reports', mock.ANY, requires=[mock.ANY]), mock.call( 'create_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('load_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('populate_staging_raw', mock.ANY, requires=mock.ANY), mock.call('drop_temp_stage_tables', mock.ANY, requires=mock.ANY), mock.call( 'set_status_to_populated_raw_table', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY) ]) def test_flow_decider_if_fact_analytics_report_is_not_available(): """Test decider execution if fact analytics report is not available.""" check_status = MagicMock() check_status.result = { 'check_available_reports.load_fact_analytics': False} schedule = MagicMock(return_value=check_status) am_flow = flow.Flow() am_flow.decider(schedule, {}) schedule.assert_has_calls([ mock.call('check_concurrent_status', mock.ANY), mock.call('bootstrap', mock.ANY, requires=[mock.ANY]), mock.call('reporter_to_s3', mock.ANY, requires=[mock.ANY]), mock.call('update_feed_file_status', mock.ANY, requires=[mock.ANY]), mock.call('check_available_reports', mock.ANY, requires=[mock.ANY]), mock.call( 'create_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('load_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('populate_staging_raw', mock.ANY, requires=mock.ANY), mock.call('drop_temp_stage_tables', mock.ANY, requires=mock.ANY), mock.call( 'set_status_to_populated_raw_table', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), ]) def test_flow_decider_wirh_use_s3_param_sme(): """Test decider execution if use_s3 param is 'True'.""" activity_statuses = MagicMock() activity_statuses.result = { 'bootstrap.use_s3': 'True', 'check_available_reports.load_fact_analytics': True, 'update_dim_tables.sns_report_subject': 'Test subj', 'check_available_reports.update_library_reports': True, } schedule = MagicMock(return_value=activity_statuses) am_flow = flow.Flow() am_flow.decider(schedule, {'licensor': 'sme'}) schedule.assert_has_calls([ mock.call('check_concurrent_status', mock.ANY), mock.call('bootstrap', mock.ANY, requires=[mock.ANY]), mock.call('grab_drop_files', mock.ANY, requires=[mock.ANY]), mock.call('update_feed_file_status', mock.ANY, requires=[mock.ANY]), mock.call('check_available_reports', mock.ANY, requires=[mock.ANY]), mock.call( 'create_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('load_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('populate_staging_raw', mock.ANY, requires=mock.ANY), mock.call('drop_temp_stage_tables', mock.ANY, requires=mock.ANY), mock.call( 'set_status_to_populated_raw_table', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('update_dim_tables', mock.ANY, requires=mock.ANY), mock.call('update_apple_id_mapping', mock.ANY, requires=mock.ANY), mock.call('load_fact_analytics', mock.ANY, requires=mock.ANY), mock.call( 'set_status_ingested_to_fact_analytics_report', mock.ANY, requires=mock.ANY), mock.call( 'update_staging_raw_library_reports', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_overall_status_ingested', mock.ANY, requires=mock.ANY), ]) def test_flow_decider_wirh_use_s3_param_awal(): """Test decider execution if use_s3 param is 'True'.""" activity_statuses = MagicMock() activity_statuses.result = { 'bootstrap.use_s3': 'True', 'check_available_reports.load_fact_analytics': True, 'update_dim_tables.sns_report_subject': 'Test subj', 'check_available_reports.update_library_reports': True, } schedule = MagicMock(return_value=activity_statuses) am_flow = flow.Flow() am_flow.decider(schedule, {'licensor': 'awal'}) schedule.assert_has_calls([ mock.call('check_concurrent_status', mock.ANY), mock.call('bootstrap', mock.ANY, requires=[mock.ANY]), mock.call( 'grab_drop_files_from_s3_awal', mock.ANY, requires=[mock.ANY]), mock.call('update_feed_file_status', mock.ANY, requires=[mock.ANY]), mock.call('check_available_reports', mock.ANY, requires=[mock.ANY]), mock.call( 'create_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('load_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('populate_staging_raw', mock.ANY, requires=mock.ANY), mock.call('drop_temp_stage_tables', mock.ANY, requires=mock.ANY), mock.call( 'set_status_to_populated_raw_table', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('update_dim_tables', mock.ANY, requires=mock.ANY), mock.call('update_apple_id_mapping', mock.ANY, requires=mock.ANY), mock.call('load_fact_analytics', mock.ANY, requires=mock.ANY), mock.call( 'set_status_ingested_to_fact_analytics_report', mock.ANY, requires=mock.ANY), mock.call( 'update_staging_raw_library_reports', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_overall_status_ingested', mock.ANY, requires=mock.ANY), ]) def test_flow_decider_with_altafonte_licensor(): """Test decider execution if use_s3 param is 'True'.""" activity_statuses = MagicMock() activity_statuses.result = { 'bootstrap.stop_after_staging_raw': True, } schedule = MagicMock(return_value=activity_statuses) am_flow = flow.Flow() am_flow.decider(schedule, {'licensor': 'altafonte'}) schedule.assert_has_calls([ mock.call('check_concurrent_status', mock.ANY), mock.call('bootstrap', mock.ANY, requires=[mock.ANY]), mock.call('reporter_to_s3', mock.ANY, requires=[mock.ANY]), mock.call('update_feed_file_status', mock.ANY, requires=[mock.ANY]), mock.call('check_available_reports', mock.ANY, requires=[mock.ANY]), mock.call( 'create_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('load_temp_staging_raw_tables', mock.ANY, requires=mock.ANY), mock.call('populate_staging_raw', mock.ANY, requires=mock.ANY), mock.call('drop_temp_stage_tables', mock.ANY, requires=mock.ANY), mock.call( 'set_status_to_populated_raw_table', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), ]) def test_flow_decider_soft_reload(): """Test normal decider execution...""" activity_statuses = MagicMock() activity_statuses.result = { 'check_available_reports.load_fact_analytics': True, 'check_available_reports.update_library_reports': True, 'bootstrap.soft_reload': True, } schedule = MagicMock(return_value=activity_statuses) am_flow = flow.Flow() am_flow.decider(schedule, {'licensor': 'theorchard'}) # collect task names from schedule calls tasks = [call[0][0] for call in schedule.call_args_list] assert tasks == [ 'check_concurrent_status', 'bootstrap', # ... skipped with soft_reload tasks are commented # 'reporter_to_s3', # 'update_feed_file_status', 'check_available_reports', # 'create_temp_staging_raw_tables', # 'load_temp_staging_raw_tables', # 'populate_staging_raw', # 'drop_temp_stage_tables', # 'set_status_to_populated_raw_table', # 'set_status_ingested', # 'update_dim_tables', 'update_apple_id_mapping', 'load_fact_analytics', 'set_status_ingested_to_fact_analytics_report', 'update_staging_raw_library_reports', 'load_aggregated_skips_and_saves', 'set_overall_status_ingested', 'build_jenkins_dbt', ]