"""Unit tests for Spotify Workflow.""" from unittest import mock from unittest.mock import MagicMock from feed_ingestion.flows.spotify.flow import Flow def test_decider_reports_unavailable(): """Test decider execution when reports are unavailable.""" check_status = MagicMock() check_status.result = { 'check_available_reports.stop': True} schedule = MagicMock(return_value=check_status) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('check_date', mock.ANY), mock.call('check_feed_status', mock.ANY, requires=mock.ANY), mock.call('bootstrap', mock.ANY, requires=mock.ANY), mock.call('grab_drop_files', mock.ANY, requires=mock.ANY), mock.call('check_available_reports', mock.ANY, requires=mock.ANY)]) def test_decider_reports_grab_drop_files_partitioned_stop(): """Test decider execution when reports are unavailable.""" check_status = MagicMock() check_status.result = { 'bootstrap.use_partitioned': 'True', 'grab_drop_files.stop': True } schedule = MagicMock(return_value=check_status) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('check_date', mock.ANY), mock.call('check_feed_status', mock.ANY, requires=mock.ANY), mock.call('bootstrap', mock.ANY, requires=mock.ANY), mock.call('grab_drop_files_partitioned', mock.ANY, requires=mock.ANY), ]) def test_decider_common_reports_are_available(): """Test decider execution when anly common reports are unavailable.""" check_status = MagicMock() check_status.result = { 'check_available_reports.necessary_activities': { 'load_common_table': True }} schedule = MagicMock(return_value=check_status) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('check_date', mock.ANY), mock.call('check_feed_status', mock.ANY, requires=mock.ANY), mock.call('bootstrap', mock.ANY, requires=mock.ANY), mock.call('grab_drop_files', mock.ANY, requires=mock.ANY), mock.call('check_available_reports', mock.ANY, requires=mock.ANY), mock.call( 'create_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'load_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'create_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call( 'load_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call('drop_temp_staging_table', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('build_jenkins_dbt', mock.ANY, requires=mock.ANY), mock.call( 'set_overall_status_ingested', mock.ANY, requires=mock.ANY) ]) def test_decider_should_stop_when_data_already_ingested(): """Decider quits if data is already ingested.""" check_feed_status = MagicMock() check_feed_status.result = {'check_feed_status.stop': True} schedule = MagicMock(return_value=check_feed_status) flow = Flow() result = flow.decider(schedule) assert result is None schedule.assert_has_calls( [mock.call('check_feed_status', mock.ANY, requires=mock.ANY)]) def test_decider_should_load_data_into_fact_analytics(): """Test decider if load_fact_analytics is True.""" activity_statuses = MagicMock() necessary_activities = { 'load_staging_raw': True, 'load_fact_analytics': True, 'load_common_table': True } activity_statuses.result = { 'check_available_reports.necessary_activities': necessary_activities, 'update_dim_tables.sns_report_subject': 'Test subj' } schedule = MagicMock(return_value=activity_statuses) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('check_date', mock.ANY), mock.call('check_feed_status', mock.ANY, requires=mock.ANY), mock.call('bootstrap', mock.ANY, requires=mock.ANY), mock.call('grab_drop_files', mock.ANY, requires=mock.ANY), mock.call('check_available_reports', mock.ANY, requires=mock.ANY), mock.call( 'create_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'load_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'create_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call( 'load_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call('load_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call('drop_temp_staging_table', mock.ANY, requires=mock.ANY), mock.call('update_dim_tables', mock.ANY, requires=mock.ANY), mock.call('load_staging_fact_table', mock.ANY, requires=mock.ANY), mock.call('load_fact_tables', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('build_jenkins_dbt', mock.ANY, requires=mock.ANY), mock.call( 'set_overall_status_ingested', mock.ANY, requires=mock.ANY) ]) def test_decider_should_load_data_into_staging_raw(): """Test decider if load_staging_raw is True.""" check_status = MagicMock() necessary_activities = { 'load_staging_raw': True, } check_status.result = { 'check_available_reports.necessary_activities': necessary_activities} schedule = MagicMock(return_value=check_status) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('check_date', mock.ANY), mock.call('check_feed_status', mock.ANY, requires=mock.ANY), mock.call('bootstrap', mock.ANY, requires=mock.ANY), mock.call('grab_drop_files', mock.ANY, requires=mock.ANY), mock.call('check_available_reports', mock.ANY, requires=mock.ANY), mock.call( 'create_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'load_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call('load_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call('drop_temp_staging_table', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('build_jenkins_dbt', mock.ANY, requires=mock.ANY), mock.call( 'set_overall_status_ingested', mock.ANY, requires=mock.ANY) ]) def test_decider_use_s3_sme(): """Test decider if use_s3 is True for sme licensor.""" activity_statuses = MagicMock() necessary_activities = { 'load_staging_raw': True, 'load_fact_analytics': True, 'load_common_table': True } activity_statuses.result = { 'check_available_reports.necessary_activities': necessary_activities, 'update_dim_tables.sns_report_subject': 'Test subj', 'bootstrap.use_s3': 'True', 'bootstrap.licensor': 'sme' } schedule = MagicMock(return_value=activity_statuses) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('check_date', mock.ANY), mock.call('check_feed_status', mock.ANY, requires=mock.ANY), mock.call('bootstrap', mock.ANY, requires=mock.ANY), mock.call('grab_drop_files_from_s3', mock.ANY, requires=mock.ANY), mock.call('check_available_reports', mock.ANY, requires=mock.ANY), mock.call( 'create_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'load_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'create_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call( 'load_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call('load_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call('drop_temp_staging_table', mock.ANY, requires=mock.ANY), mock.call('update_dim_tables', mock.ANY, requires=mock.ANY), mock.call('load_staging_fact_table', mock.ANY, requires=mock.ANY), mock.call('load_fact_tables', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('build_jenkins_dbt', mock.ANY, requires=mock.ANY), mock.call( 'set_overall_status_ingested', mock.ANY, requires=mock.ANY) ]) def test_decider_use_partitioned(): """Test decider if use_partitioned is True.""" activity_statuses = MagicMock() necessary_activities = { } activity_statuses.result = { 'check_available_reports.necessary_activities': necessary_activities, 'bootstrap.use_partitioned': 'True', } schedule = MagicMock(return_value=activity_statuses) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('grab_drop_files_partitioned', mock.ANY, requires=mock.ANY), ]) def test_decider_trigger_grps_ingester(): """Test decider if create_grps_products is True.""" activity_statuses = MagicMock() necessary_activities = { 'load_staging_raw': True, 'load_fact_analytics': True, 'load_common_table': True } activity_statuses.result = { 'check_available_reports.necessary_activities': necessary_activities, 'bootstrap.create_grps_products': 'True' } schedule = MagicMock(return_value=activity_statuses) flow = Flow() flow.decider(schedule) schedule.assert_has_calls([ mock.call('check_date', mock.ANY), mock.call('check_feed_status', mock.ANY, requires=mock.ANY), mock.call('bootstrap', mock.ANY, requires=mock.ANY), mock.call('grab_drop_files', mock.ANY, requires=mock.ANY), mock.call('check_available_reports', mock.ANY, requires=mock.ANY), mock.call( 'create_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'load_temp_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call( 'create_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call( 'load_transitional_common_tables', mock.ANY, requires=mock.ANY), mock.call('load_staging_raw_table', mock.ANY, requires=mock.ANY), mock.call('drop_temp_staging_table', mock.ANY, requires=mock.ANY), mock.call('update_dim_tables', mock.ANY, requires=mock.ANY), mock.call('load_staging_fact_table', mock.ANY, requires=mock.ANY), mock.call('load_fact_tables', mock.ANY, requires=mock.ANY), mock.call( 'load_aggregated_skips_and_saves', mock.ANY, requires=mock.ANY), mock.call('set_status_ingested', mock.ANY, requires=mock.ANY), mock.call('build_jenkins_dbt', mock.ANY, requires=mock.ANY), mock.call( 'set_overall_status_ingested', mock.ANY, requires=mock.ANY) ])