"""Unit tests for tasks of YouTube Neo4j Workflow.""" from unittest.mock import call, MagicMock from unittest.mock import patch from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from feed_ingestion.flows.youtube_neo4j import config from feed_ingestion.flows.youtube_neo4j import tasks @pytest.fixture def mock_activity(): """Return mock activity.""" return MagicMock() @pytest.fixture def context(): """Context for YouTube Bulk Reports tasks.""" return { 'activity': MagicMock(), 'date': '2020-08-01', 'reload': False} @pytest.fixture def expected_bootstrap_response(context): """Response for bootstrap task.""" return { 'date': '2020-08-01', 'feed_name': config.feed_name} @patch('feed_ingestion.flows.youtube_neo4j.tasks.garcon_feed_status.' 'get_overall_status', return_value=garcon_feed_status.STATUS_NOT_INGESTED) def test_bootstrap( mock_get_overall_status, context, expected_bootstrap_response): """Test bootstrap task.""" result = tasks.bootstrap(**context) assert result == expected_bootstrap_response @patch('feed_ingestion.flows.youtube_neo4j.tasks.garcon_feed_status.' 'get_overall_status', return_value=garcon_feed_status.STATUS_INGESTED) def test_check_staging_status(mock_get_overall_status, mock_activity): """Test check_staging_status_task.""" result = tasks.check_staging_status(mock_activity, '2020-08-01') assert result == {} @patch('feed_ingestion.flows.youtube_neo4j.tasks.garcon_feed_status.' 'get_overall_status', return_value=garcon_feed_status.STATUS_NOT_INGESTED) def test_check_staging_status_not_available( mock_get_overall_status, mock_activity): """Should return STOP_RESPONSE if reports are not available.""" result = tasks.check_staging_status(mock_activity, '2020-08-01') assert result == {'stop': True} @patch('feed_ingestion.flows.youtube_neo4j.tasks.get_sf_config') @patch('feed_ingestion.flows.youtube_neo4j.tasks.YouTubeNeo4J') @patch('feed_ingestion.flows.youtube_neo4j.tasks.get_neo4j_config') @patch('feed_ingestion.flows.youtube_neo4j.tasks.Neo4jExecutor') @pytest.mark.parametrize( 'query_type, query_name, butch_size', [ ('asset', 'ingest_youtube_asset', 1000), ('channel', 'ingest_youtube_channel', 100), ('video', 'ingest_youtube_video', 1000), ] ) def test_ingest_youtube_data( mock_neo4j_executor_class, mock_get_neo4j_config, mock_sf_executor_class, mock_get_sf_config, query_type, query_name, butch_size): """Test aggregation ingest queries.""" mock_neo4j_config = {} mock_get_neo4j_config.return_value = mock_neo4j_config mock_neo4j_executor = MagicMock() mock_neo4j_executor_class.return_value.__enter__.return_value = \ mock_neo4j_executor mock_get_sf_config.return_value = { 'db': 'DB', 'schema': 'SCHEMA', 'table_name': 'temp_table', } mock_sf_executor = MagicMock() mock_sf_executor_class.return_value.__enter__.return_value = \ mock_sf_executor first_batch = [{'key': 'value'}] second_batch = [] mock_sf_executor.fetchall_dict_query.side_effect = \ (first_batch, second_batch) date = '2020-08-01' tasks.ingest_youtube_data( MagicMock(), config.feed_name, date, query_type, 'temp_table') mock_get_neo4j_config.assert_called_once_with('youtube_neo4j') mock_neo4j_executor_class.assert_called_once_with( 'youtube_neo4j', mock_neo4j_config) mock_neo4j_executor.execute_write_query.assert_called_once_with( query_name, {'rows': first_batch}) params_expected = { 'limit': butch_size, 'offset': 0, 'table_name': 'temp_table', } params_expected2 = { 'limit': butch_size, 'offset': butch_size, 'table_name': 'temp_table', } assert mock_sf_executor.fetchall_dict_query.call_args_list == [ call(query_name, **params_expected), call(query_name, **params_expected2), ] @patch('feed_ingestion.flows.youtube_neo4j.tasks.get_sf_config') @patch('feed_ingestion.flows.youtube_neo4j.tasks.YouTubeNeo4J') def test_create_table(mock_executor_class, mock_get_sf_config): """Test create table.""" mock_conifg = {} mock_get_sf_config.return_value = mock_conifg mock_executor = MagicMock() mock_executor_class.return_value.__enter__.return_value = mock_executor feed_name = 'youtube_neo4j' date = '2020-01-01' query_name = 'asset' temp_table = 'temp_table' full_owner_changed_refresh = False tasks.create_table( MagicMock(), feed_name, date, query_name, temp_table, full_owner_changed_refresh) mock_get_sf_config.assert_called_once_with(feed_name) mock_executor_class.assert_called_once_with(mock_conifg) mock_executor.execute_query.assert_has_calls([ call( 'create_temp_asset_table', ar_db='orchard_app_reporting_v2', ar_schema='art_relations_prod_art_relations', temp_table_name=temp_table, log_table_name='youtube_neo4j_log', ingest_date=date, full_owner_changed_refresh=False) ]) @patch('feed_ingestion.flows.youtube_neo4j.tasks.get_sf_config') @patch('feed_ingestion.flows.youtube_neo4j.tasks.YouTubeNeo4J') def test_clear_log_table(mock_executor_class, mock_get_sf_config): """Test clear log table.""" mock_config = {} mock_get_sf_config.return_value = mock_config mock_executor = MagicMock() mock_executor_class.return_value.__enter__.return_value = mock_executor feed_name = 'youtube_neo4j' date = '2020-01-01' force_refresh = 'True' tasks.clear_log_table(MagicMock(), feed_name, date, force_refresh) mock_get_sf_config.assert_called_once_with(feed_name) mock_executor_class.assert_called_once_with(mock_config) mock_executor.execute_query.assert_called_once_with( 'clear_log_table', log_table_name=config.snowflake['log_table_name'], ingest_date=date) mock_executor.execute_query.reset_mock() force_refresh = 'False' tasks.clear_log_table(MagicMock(), feed_name, date, force_refresh) assert mock_executor.execute_query.call_count == 0 @patch('feed_ingestion.flows.youtube_neo4j.tasks.get_sf_config') @patch('feed_ingestion.flows.youtube_neo4j.tasks.YouTubeNeo4J') def test_insert_into_log_table(mock_executor_class, mock_get_sf_config): """Test insert_into_log_table.""" mock_config = {} mock_get_sf_config.return_value = mock_config mock_executor = MagicMock() mock_executor_class.return_value.__enter__.return_value = mock_executor feed_name = 'youtube_neo4j' date = '2020-01-01' tasks.insert_into_log_table( MagicMock(), feed_name, date, 'asset', 'temp_table') mock_get_sf_config.assert_called_once_with(feed_name) mock_executor_class.assert_called_once_with(mock_config) mock_executor.execute_query.assert_called_once_with( 'insert_into_log_table', ingest_date='2020-01-01', table_name='temp_table', log_table_name='youtube_neo4j_log' )