"""Unit tests for tasks of YouTube Channel Names Workflow.""" from unittest.mock import MagicMock from unittest.mock import patch import pytest from feed_ingestion.flows.youtube_channel_names import config from feed_ingestion.flows.youtube_channel_names import tasks @pytest.fixture(params=[ 'theorchard', 'sme' ]) def context_and_params(request): """Yield context, licensor.""" licensor = request.param context = { 'activity': MagicMock(), 'date': '2020-07-16', 'licensor': licensor, 'reload': False} yield context, licensor @pytest.fixture def mock_task_status(): """Yield task status.""" task_status_path = 'feed_ingestion.tasks.task_status' with patch(task_status_path) as task_status: task_status.is_completed_task.return_value = False task_status.mark_completed_task = MagicMock() yield task_status @pytest.fixture def mock_executor_context(): """Yield executor context.""" reg_executors_class_path = ( 'feed_ingestion.flows.youtube_channel_names.' 'tasks.registered_executors') with patch(reg_executors_class_path) as reg_executors: mock_executor = MagicMock() reg_executors.get = mock_executor yield mock_executor.return_value.return_value.__enter__.return_value @pytest.fixture def mock_s3_utils(): """Mock s3utils.""" path = 'feed_ingestion.flows.youtube_channel_names.tasks.s3utils' with patch(path) as s3utils: yield s3utils @pytest.fixture def mock_save_youtube_access_token_from_secrets(): """Yield overall status.""" overall_status_path = ( 'feed_ingestion.flows.youtube_channel_names.tasks.' 'save_youtube_access_token_from_secrets') with patch(overall_status_path) as overall_status: yield overall_status @pytest.fixture def mock_youtube_util(): """Mock youtube_util.""" path = 'feed_ingestion.flows.youtube_channel_names.tasks.youtube_util' with patch(path) as youtube_util: yield youtube_util @pytest.fixture def expected_bootstrap_response(context_and_params): """Response for bootstrap task.""" context, licensor = context_and_params feed_name = '_'.join([config.feed_name, licensor]) s3_path = ( f's3://dev-cucumbers/YouTubeChannelNames/preprocessed/{licensor}/') temp_staging_raw_table = f'dim_youtube_channel_names_2020_07_16_{licensor}' credentials_path = config.credentials_paths[licensor] return { 'date': context['date'], 'feed_name': feed_name, 'licensor': licensor, 's3_preprocessed_path': s3_path, 'processed_filename': 'yt_channel_names_2020-07-16.tsv.gz', 'temp_staging_raw_table': temp_staging_raw_table, 'temp_staging_table_kwargs': {'file_pattern': 'yt_channel_names_2020-07-16.tsv.gz'}, 'credentials_path': credentials_path } @patch('feed_ingestion.flows.youtube_channel_names.' 'tasks.garcon_feed_status.get_overall_status') def test_bootstrap(mock_get_overall_status, expected_bootstrap_response): """Check that bootstrap returns expected results.""" licensor = expected_bootstrap_response['licensor'] context = { 'activity': MagicMock(), 'date': '2020-07-16', 'reload': True, 'licensor': licensor } result = tasks.bootstrap(**context) feed_name = expected_bootstrap_response['feed_name'] mock_get_overall_status.assert_called_with( feed_name, '2020-07-16') assert result == expected_bootstrap_response def test_get_missing_channels( mock_task_status, mock_executor_context, expected_bootstrap_response, mock_s3_utils, mock_youtube_util, mock_save_youtube_access_token_from_secrets): """Test get_missing_channels task.""" tasks.get_missing_channels( MagicMock(), expected_bootstrap_response['date'], expected_bootstrap_response['feed_name'], expected_bootstrap_response['processed_filename'], expected_bootstrap_response['s3_preprocessed_path'], expected_bootstrap_response['credentials_path']) assert mock_task_status.is_completed_task.called assert mock_task_status.mark_completed_task.called assert mock_executor_context.get_missing_channels.called assert mock_youtube_util.get_authenticated_services.called assert mock_s3_utils.upload_processed_to_s3.called def test_update_dim_table( mock_task_status, mock_executor_context, expected_bootstrap_response): """Test update_dim_table task.""" tasks.update_dim_table( MagicMock(), expected_bootstrap_response['date'], expected_bootstrap_response['feed_name'], expected_bootstrap_response['processed_filename'], expected_bootstrap_response['s3_preprocessed_path'], expected_bootstrap_response['temp_staging_raw_table'] ) assert mock_task_status.is_completed_task.called assert mock_task_status.mark_completed_task.called assert mock_executor_context.update_dim_table.called