import argparse import datetime import json from unittest.mock import patch, call import pytest from feed_ingestion.bin import flow_timings @patch.object(flow_timings.swf, 'list_closed_swf_executions') def test_list_executions(mock_load_executions): executions = flow_timings.list_executions( workflow_name='spotify_feed_ingestion', dates=[ datetime.datetime.strptime('2024-01-14', '%Y-%m-%d'), datetime.datetime.strptime('2024-01-15', '%Y-%m-%d'), ], ) assert executions == mock_load_executions.return_value assert mock_load_executions.call_args == ( { 'from_': datetime.datetime.strptime('2024-01-14', '%Y-%m-%d'), 'to': datetime.datetime.strptime('2024-01-15', '%Y-%m-%d'), 'swf_domain': 'prod_swf_feed_ingestion', 'execution_type': 'spotify_feed_ingestion' }, ) @patch.object(flow_timings.swf, 'load_all_execution_events') def test_list_event(mock_load_all_execution_events): execution = { 'execution': { 'workflowId': 'workflow_id', 'runId': 'run_id', } } events = flow_timings.list_events(execution) assert events == mock_load_all_execution_events.return_value assert mock_load_all_execution_events.call_args == ( { 'swf_domain': 'prod_swf_feed_ingestion', 'workflow_id': 'workflow_id', 'run_id': 'run_id', }, ) def test_datetime_to_utc_and_truncate(): dt = datetime.datetime(2024, 1, 14, 16, 10, tzinfo=datetime.timezone.utc) assert (flow_timings.datetime_to_utc_and_truncate(dt) == datetime.datetime(2024, 1, 14, 16, 10, tzinfo=datetime.timezone.utc)) def test_first_not_empty(): assert flow_timings.first_not_empty([None, 1, 2]) == 1 assert flow_timings.first_not_empty([None, None, 2]) == 2 assert flow_timings.first_not_empty([None, None, None]) is None def test_get_feed_name_and_date(): tasks = [ { 'name': 'spotify_bootstrap', 'closed': { 'activityTaskCompletedEventAttributes': { 'result': json.dumps({ 'bootstrap_feed.date': '2024-01-14', 'bootstrap_feed.feed_name': 'spotify', }) } } } ] result = flow_timings.get_feed_name_and_date(tasks) assert result == ('spotify', '2024-01-14') def test_find_tasks_by_name(): tasks = [ {'name': 'spotify_bootstrap'}, {'name': 'spotify_ingest'}, {'name': 'amazon_music_bootstrap'}, {'name': 'amazon_music_ingest'}, ] result = flow_timings.find_tasks_by_name('bootstrap', tasks) assert result == [ {'name': 'spotify_bootstrap'}, {'name': 'amazon_music_bootstrap'}, ] def test_get_task_result(): task = { 'closed': { 'activityTaskCompletedEventAttributes': { 'result': json.dumps({'a': 1}), } } } result = flow_timings.get_task_result(task) assert result == {'a': 1} def test_workflow_name_from_feed_name(): assert (flow_timings.workflow_name_from_feed_name('spotify') == 'spotify_feed_ingestion') @pytest.mark.parametrize( 'tasks, expected', [ ([], False), ([{'name': 'spotify_bootstrap'}], False), ([{'name': 'spotify_load_fact_tables'}], True), ([ {'name': 'spotify_bootstrap'}, {'name': 'spotify_load_fact_tables'} ], True), ] ) def test_find_completion_tasks(tasks, expected): assert flow_timings.find_completion_tasks(tasks) is expected @patch.object(flow_timings.swf, 'list_closed_swf_executions') @patch.object(flow_timings.swf, 'load_all_execution_events') @patch.object(flow_timings.swf, 'tasks_from_events') def test_main( mock_tasks_from_events, mock_load_all_execution_events, mock_load_executions): mock_load_executions.return_value = [ { 'execution': { 'workflowId': 'workflow_id', 'runId': 'run_id', }, 'startTimestamp': datetime.datetime(2024, 1, 14, 16, 0), 'closeTimestamp': datetime.datetime(2024, 1, 14, 16, 10), } ] mock_load_all_execution_events.return_value = [] mock_tasks_from_events.return_value = [] swf_domain = 'swf_domain' date = datetime.date(2024, 1, 14) flows = ['spotify', 'amazon_music'] flow_timings.main(swf_domain, date, flows) assert mock_load_executions.call_args_list == [ call( from_=datetime.datetime(2024, 1, 14, 0, 0), to=datetime.datetime(2024, 1, 15, 0, 0), swf_domain='swf_domain', execution_type='spotify_feed_ingestion'), call( from_=datetime.datetime(2024, 1, 14, 0, 0), to=datetime.datetime(2024, 1, 15, 0, 0), swf_domain='swf_domain', execution_type='amazon_music_feed_ingestion')] assert mock_load_all_execution_events.call_args_list == [ call(swf_domain='swf_domain', workflow_id='workflow_id', run_id='run_id'), call(swf_domain='swf_domain', workflow_id='workflow_id', run_id='run_id') ] assert mock_tasks_from_events.call_args_list == [ call(mock_load_all_execution_events.return_value), call(mock_load_all_execution_events.return_value) ] def test_process_dbt_triggerer(): date = datetime.date(2024, 1, 14) execution = { 'execution': { 'workflowId': 'workflow_id', 'runId': 'run_id', } } tasks = [ { 'name': 'spotify_build_jenkins_dbt', 'closed': { 'activityTaskCompletedEventAttributes': { 'result': json.dumps({ 'build_jenkins_dbt.job_url': 'job_url', }) } } } ] assert flow_timings.process_dbt_triggerer(date, execution, tasks) == tasks @pytest.mark.parametrize( 'argv, expected', [ ( [ '--swf-domain', 'swf_domain', '--date', '2024-01-14', '--flows', 'spotify,amazon_music', '--cache', ], { 'swf_domain': 'swf_domain', 'date': datetime.date(2024, 1, 14), 'flows': ['spotify', 'amazon_music'], 'cache': True, } ), ( [ '--date', '2024-01-14', ], { 'swf_domain': flow_timings.SWF_DOMAIN_PROD, 'date': datetime.date(2024, 1, 14), 'flows': flow_timings.FLOWS, 'cache': False, } ), ( [ '--date', '2024-01-14', '--no-cache', ], { 'swf_domain': flow_timings.SWF_DOMAIN_PROD, 'date': datetime.date(2024, 1, 14), 'flows': flow_timings.FLOWS, 'cache': False, } ), ] ) def test_parseargs(argv, expected): args = flow_timings.parseargs(argv) assert args == argparse.Namespace(**expected)