"""Unit tests for TikTok Weekly tasks workflow.""" from datetime import datetime from datetime import timedelta from unittest.mock import call from unittest.mock import MagicMock from unittest.mock import patch import pytest from feed_ingestion.flows.tiktok_weekly import tasks # Sunday _date = '2021-01-03' @pytest.fixture def expected_bootstrap_response(): """Response for bootstrap task.""" return { 'feed_name': 'tiktok_weekly', 'date': _date, 'drop_path': 'tiktok/in/sme/weekly_toplists/', 'archive_path': 'TikTok_weekly/archives/2021-01-03/', 'filename_template': '{platform}_Trends_{report}_20210103.txt'} @pytest.fixture def get_downloaded_files(): """Return downloaded_files.""" return { 'TikTok': { 'TopHashtag': 'TikTok_Trends_TopHashtag_20210103.txt', 'TopSong': 'TikTok_Trends_TopSong_20210103.txt'}, 'Douyin': { 'TopHashtag': 'Douyin_Trends_TopHashtag_20210103.txt', 'TopSong': 'Douyin_Trends_TopSong_20210103.txt'}} @pytest.fixture def mock_get_overall_status(): """Yield get overall status.""" overall_status_path = ( 'feed_ingestion.flows.tiktok_weekly.tasks.garcon_feed_status.' 'get_overall_status') with patch(overall_status_path) as overall_status: yield overall_status def test_bootstrap(expected_bootstrap_response, mock_get_overall_status): """Test bootstrap task with last day of a week.""" result = tasks.bootstrap(MagicMock(), _date, reload=False) assert result == expected_bootstrap_response def test_bootstrap_ingested_status(mock_get_overall_status): """Test bootstrap task if overall status is already INGESTED.""" mock_get_overall_status.return_value = 'INGESTED' result = tasks.bootstrap(MagicMock(), _date, reload=False) assert result == {'stop': True} def test_bootstrap_context_date(mock_get_overall_status): """Test bootstrap task.""" # test with Monday context date date_obj = datetime.strptime(_date, '%Y-%m-%d') monday_date = (date_obj + timedelta(days=1)).strftime('%Y-%m-%d') result = tasks.bootstrap(MagicMock(), monday_date, reload=False) assert result['date'] == _date # test with Wednesday context date date_obj = datetime.strptime(_date, '%Y-%m-%d') print(date_obj) wednesday_date = (date_obj + timedelta(days=3)).strftime('%Y-%m-%d') result = tasks.bootstrap(MagicMock(), wednesday_date, reload=False) assert result['date'] == _date @pytest.fixture def mock_executor_context(): """Yield executor context.""" sf_executor_class_path = ( 'feed_ingestion.flows.tiktok_weekly.tasks.TikTokWeekly') with patch(sf_executor_class_path) as sf_executor: mock_executor_context = sf_executor.return_value.__enter__.return_value yield mock_executor_context @pytest.fixture def mock_task_status(): """Yield task status.""" task_status_path = 'feed_ingestion.flows.tiktok_weekly.tasks.task_status' with patch(task_status_path) as task_status: task_status.is_completed_task.return_value = False task_status.mark_completed_task = MagicMock() task_status.get_values = MagicMock() yield task_status @pytest.fixture def mock_set_missing_files(): """Yield overall status.""" set_missing_files_path = ( 'feed_ingestion.flows.tiktok_weekly.tasks.garcon_feed_status.' 'set_missing_files') with patch(set_missing_files_path) as set_missing_files_status: yield set_missing_files_status @pytest.fixture def mock_set_overall_status(): """Yield overall status.""" overall_status_path = ( 'feed_ingestion.flows.tiktok_weekly.tasks.garcon_feed_status.' 'set_overall_status') with patch(overall_status_path) as overall_status: yield overall_status @pytest.fixture def mock_s3_tasks(): """Yield overall status.""" path = 'feed_ingestion.flows.tiktok_weekly.tasks.s3_tasks' with patch(path) as mock_s3: yield mock_s3 def test_grab_drop_files_files_are_not_available( mock_task_status, mock_set_overall_status, mock_s3_tasks, mock_set_missing_files): """Test test_grab_drop_files when all files are not available.""" mock_task_status.get_values.return_value = [] mock_s3_tasks.copy_file_from_sme_s3_to_theocrhard.return_value = {} result = tasks.grab_drop_files( MagicMock(), 'feed_name', _date, 'tiktok/in/2021/01/03/', 'TikTok_weekly/archives/2021-01-03/', 'TikTok, Douyin', '{platform}_Trends_{report}_20210103.txt') assert result == { 'stop': True, 'missing_files': [ 'TikTok_Trends_TopHashtag_20210103.txt', 'TikTok_Trends_TopSong_20210103.txt', 'Douyin_Trends_TopHashtag_20210103.txt', 'Douyin_Trends_TopSong_20210103.txt' ], } mock_set_overall_status.assert_called_with( 'feed_name', _date, 'NOT_AVAILABLE') mock_set_missing_files.assert_called_with( 'feed_name', _date, [ 'TikTok_Trends_TopHashtag_20210103.txt', 'TikTok_Trends_TopSong_20210103.txt', 'Douyin_Trends_TopHashtag_20210103.txt', 'Douyin_Trends_TopSong_20210103.txt']) def test_grab_drop_files_some_files_are_not_available( mock_task_status, mock_set_overall_status, mock_s3_tasks, mock_set_missing_files): """Test test_grab_drop_files when not all files are available.""" mock_task_status.get_values.return_value = [] mock_s3_tasks.copy_file_from_sme_s3_to_theocrhard.return_value = { 'TikTok_Trends_TopHashtag_20210103.txt': True, 'TikTok_Trends_TopSong_20210103.txt': True} result = tasks.grab_drop_files( MagicMock(), 'feed_name', _date, 'tiktok/in/2021/01/03/', 'TikTok_weekly/archives/2021-01-03/', 'TikTok, Douyin', '{platform}_Trends_{report}_20210103.txt') assert result == { 'stop': True, 'missing_files': [ 'Douyin_Trends_TopHashtag_20210103.txt', 'Douyin_Trends_TopSong_20210103.txt' ], } mock_set_overall_status.assert_called_with( 'feed_name', _date, 'NOT_AVAILABLE') mock_set_missing_files.assert_called_with( 'feed_name', _date, [ 'Douyin_Trends_TopHashtag_20210103.txt', 'Douyin_Trends_TopSong_20210103.txt']) def test_grab_drop_files_all_files_are_available( mock_task_status, mock_set_overall_status, mock_s3_tasks, mock_set_missing_files, get_downloaded_files): """Test test_grab_drop_files when all files are available.""" mock_task_status.get_values.return_value = [] mock_s3_tasks.copy_file_from_sme_s3_to_theocrhard.return_value = { 'TikTok_Trends_TopHashtag_20210103.txt': True, 'TikTok_Trends_TopSong_20210103.txt': True, 'Douyin_Trends_TopHashtag_20210103.txt': True, 'Douyin_Trends_TopSong_20210103.txt': True} result = tasks.grab_drop_files( MagicMock(), 'feed_name', _date, 'tiktok/in/2021/01/03/', 'TikTok_weekly/archives/2021-01-03/', 'TikTok, Douyin', '{platform}_Trends_{report}_20210103.txt') assert result == { 'downloaded_files': get_downloaded_files, 'platforms': ['TikTok', 'Douyin']} mock_set_overall_status.assert_not_called() mock_set_missing_files.assert_called_with('feed_name', '2021-01-03', []) def test_grab_drop_files_all_files_are_available_with_one_platform( mock_task_status, mock_set_overall_status, mock_s3_tasks, mock_set_missing_files, get_downloaded_files): """Test test_grab_drop_files when all files are available.""" mock_task_status.get_values.return_value = [] mock_s3_tasks.copy_file_from_sme_s3_to_theocrhard.return_value = { 'TikTok_Trends_TopHashtag_20210103.txt': True, 'TikTok_Trends_TopSong_20210103.txt': True} result = tasks.grab_drop_files( MagicMock(), 'feed_name', _date, 'tiktok/in/2021/01/03/', 'TikTok_weekly/archives/2021-01-03/', 'TikTok', '{platform}_Trends_{report}_20210103.txt') assert result == {'downloaded_files': { 'TikTok': get_downloaded_files['TikTok']}, 'platforms': ['TikTok']} mock_set_overall_status.assert_not_called() mock_set_missing_files.assert_called_with('feed_name', '2021-01-03', []) def test_grab_drop_files_all_files_are_available_with_incorrect_platform( mock_task_status, mock_set_overall_status, mock_s3_tasks, mock_set_missing_files, get_downloaded_files): """Test test_grab_drop_files when all files are available.""" mock_task_status.get_values.return_value = [] mock_s3_tasks.copy_file_from_sme_s3_to_theocrhard.return_value = { 'TikTok_Trends_TopHashtag_20210103.txt': True, 'TikTok_Trends_TopSong_20210103.txt': True} with pytest.raises(ValueError): tasks.grab_drop_files( MagicMock(), 'feed_name', _date, 'tiktok/in/2021/01/03/', 'TikTok_weekly/archives/2021-01-03/', 'Test', '{platform}_Trends_{report}_20210103.txt') mock_s3_tasks.copy_file_from_sme_s3_to_theocrhard.assert_not_called() mock_set_overall_status.assert_not_called() mock_set_missing_files.assert_not_called() def test_grab_drop_files_some_files_were_already_ingested( mock_task_status, mock_set_overall_status, mock_s3_tasks, mock_set_missing_files): """Test test_grab_drop_files when some files were ingested.""" mock_task_status.get_values.return_value = [ 'TikTok_Trends_TopHashtag_20210103.txt', 'TikTok_Trends_TopSong_20210103.txt'] mock_s3_tasks.copy_file_from_sme_s3_to_theocrhard.return_value = { 'TikTok_Trends_TopHashtag_20210103.txt': True, 'TikTok_Trends_TopSong_20210103.txt': True, 'Douyin_Trends_TopHashtag_20210103.txt': True, 'Douyin_Trends_TopSong_20210103.txt': True} downloaded_files = { 'Douyin': { 'TopHashtag': 'Douyin_Trends_TopHashtag_20210103.txt', 'TopSong': 'Douyin_Trends_TopSong_20210103.txt'}} result = tasks.grab_drop_files( MagicMock(), 'feed_name', _date, 'tiktok/in/2021/01/03/', 'TikTok_weekly/archives/2021-01-03/', 'TikTok, Douyin', '{platform}_Trends_{report}_20210103.txt') assert result == { 'downloaded_files': downloaded_files, 'platforms': ['TikTok', 'Douyin']} mock_set_overall_status.assert_not_called() mock_set_missing_files.assert_called_with('feed_name', '2021-01-03', []) def test_load_staging_raw_table(mock_executor_context, get_downloaded_files): """Test load_staging_raw_table.""" sfdb_params = {'db': 'db', 'schema': 'schema'} tasks.load_staging_raw_table( MagicMock(), _date, sfdb_params, get_downloaded_files) assert mock_executor_context.clean_staging_raw_table.call_args_list == [ call('staging_raw_TikTok_trends_tophashtag', _date), call('staging_raw_TikTok_trends_topsong', _date), call('staging_raw_Douyin_trends_tophashtag', _date), call('staging_raw_Douyin_trends_topsong', _date) ] assert mock_executor_context.drop_table.call_args_list == [ call('temp_staging_raw_TikTok_weekly_TopHashtag_20210103'), call('temp_staging_raw_TikTok_weekly_TopSong_20210103'), call('temp_staging_raw_Douyin_weekly_TopHashtag_20210103'), call('temp_staging_raw_Douyin_weekly_TopSong_20210103') ] assert mock_executor_context.drop_table.call_args_list == [ call('temp_staging_raw_TikTok_weekly_TopHashtag_20210103'), call('temp_staging_raw_TikTok_weekly_TopSong_20210103'), call('temp_staging_raw_Douyin_weekly_TopHashtag_20210103'), call('temp_staging_raw_Douyin_weekly_TopSong_20210103')] def test_set_status_to_ingested( mock_task_status, mock_set_overall_status, get_downloaded_files): """Test set_status_to_ingested.""" tasks.set_status_to_ingested( MagicMock(), 'feed_name', _date, get_downloaded_files, ['TikTok', 'Douyin']) mock_set_overall_status.assert_called_with( 'feed_name', _date, 'INGESTED') def test_set_status_to_ingested_with_one_platform( mock_task_status, mock_set_overall_status, get_downloaded_files): """Test set_status_to_ingested.""" downloaded_files = {'TikTok': get_downloaded_files['TikTok']} tasks.set_status_to_ingested( MagicMock(), 'feed_name', _date, downloaded_files, ['TikTok']) mock_set_overall_status.assert_called_with( 'feed_name', _date, 'INGESTED') def test_set_status_to_ingested_some_files_are_not_available( mock_task_status, mock_set_overall_status): """Test set_status_to_ingested.""" mock_task_status.get_values.return_value = [ 'TikTok_Trends_TopHashtag_20210103.txt', 'TikTok_Trends_TopSong_20210103.txt'] downloaded_files = { 'Douyin': { 'TopHashtag': 'Douyin_Trends_TopHashtag_20210103.txt', 'TopSong': 'Douyin_Trends_TopSong_20210103.txt'}} tasks.set_status_to_ingested( MagicMock(), 'feed_name', _date, downloaded_files, ['TikTok', 'Douyin']) mock_set_overall_status.assert_called_with( 'feed_name', _date, 'INGESTED')