"""Unit tests for tasks for loading iTunes data into fact_analytics table.""" from unittest import mock from unittest.mock import MagicMock from unittest.mock import patch import boto3 from garcon import activity from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from feed_ingestion.conf.config import SF_CONFIG from feed_ingestion.flows.itunes import load_fact_table_tasks from feed_ingestion.flows.itunes.util import itunes_helpers from feed_ingestion.util import task_status MOCK_SF_CONFIG = { 'db': 'db', 'schema': 'schema', 'role': 'role', 'warehouse': 'warehouse', } MOCK_AWS_CONFIG = { 'access_key': 'access_key', 'access_secret': 'access_secret' } @pytest.fixture def mock_sql_loader(): """Mock sql_loader instance.""" with patch('feed_ingestion.flows.itunes.' 'load_fact_table_tasks.sql_loader') as loader: yield loader @pytest.fixture def mock_config(): """Mock config.""" with patch('feed_ingestion.flows.itunes.config') as mconfig: yield mconfig @pytest.fixture def mock_sf_config(): """Mock Snowflake config.""" with patch.dict(SF_CONFIG, MOCK_SF_CONFIG, clean=True) as mconfig: yield mconfig @pytest.fixture def mock_executor_context(): """Yield executor context.""" sf_executor_class_path = ( 'feed_ingestion.flows.itunes.' 'load_fact_table_tasks.SnowflakeSQLExecutor') with patch(sf_executor_class_path) as sf_executor: mock_executor_context = \ sf_executor.return_value.__enter__.return_value yield mock_executor_context def test_drop_staging_fact( mock_config, mock_sf_config, mock_executor_context, mock_sql_loader): """Test drop_staging_fact task.""" mock_config.staging_fact_table_template = 'staging_fact' load_fact_table_tasks.drop_staging_fact( activity.Activity(boto3.client('swf', 'us-east-1')), '2015-10-09', {}, 'licensor') mock_executor_context.execute_query.assert_any_call( mock_sql_loader, 'drop_temp_table', mock.ANY) def test_load_staging_fact( mock_config, mock_sf_config, mock_executor_context, mock_sql_loader): """Test load_staging_fact task.""" mock_config.staging_fact_table_template = 'staging_fact' load_fact_table_tasks.load_staging_fact( MagicMock(), '2015-10-09', {}, 'theorchard') mock_executor_context.execute_query.assert_any_call( mock_sql_loader, 'load_staging_fact', mock.ANY) def test_delete_fact_data( mock_config, mock_sf_config, mock_executor_context, mock_sql_loader, monkeypatch): """Test delete_fact_data.""" monkeypatch.setattr( itunes_helpers, 'check_status', MagicMock(return_value=True)) monkeypatch.setattr( task_status, 'is_completed_report', MagicMock(return_value=False)) load_fact_table_tasks.delete_fact_data( MagicMock(), '2015-10-09', 'theorchard', {}, 'theorchard') mock_executor_context.execute_query.assert_any_call( mock_sql_loader, 'delete_from_fact_table', mock.ANY) mock_executor_context.execute_query.call_count = 2 def test_delete_fact_data_already_ingested( mock_config, mock_sf_config, mock_executor_context, mock_sql_loader, monkeypatch): """Test delete_fact_data.""" monkeypatch.setattr( itunes_helpers, 'check_status', MagicMock(return_value=False)) monkeypatch.setattr( task_status, 'is_completed_report', MagicMock(return_value=True)) load_fact_table_tasks.delete_fact_data( activity.Activity(boto3.client('swf', 'us-east-1')), '2015-10-09', 'feed_name', {}, 'theorchard') assert mock_executor_context.execute_query.call_count == 0 def test_load_fact_tables( mock_config, mock_sf_config, mock_executor_context, mock_sql_loader, monkeypatch): """Test load_fact_tables.""" monkeypatch.setattr( garcon_feed_status, 'set_overall_status', MagicMock(return_value=None)) # Check what is called when status is not INGESTED monkeypatch.setattr( itunes_helpers, 'check_status', MagicMock(return_value=True)) monkeypatch.setattr( task_status, 'is_completed_report', MagicMock(return_value=False)) load_fact_table_tasks.load_fact_tables( MagicMock(), '2015-10-09', 'feed_name', {}, 'theorchard') mock_executor_context.execute_query.assert_any_call( mock_sql_loader, 'load_fact_analytics', mock.ANY) mock_executor_context.execute_query.assert_any_call( mock_sql_loader, 'load_fact_analytics_error', mock.ANY) def test_load_fact_tables_already_ingested( mock_config, mock_sf_config, mock_executor_context, mock_sql_loader, monkeypatch): """Test load_fact_tables when date is already ingested.""" # Check what is called when status is INGESTED monkeypatch.setattr( itunes_helpers, 'check_status', MagicMock(return_value=False)) monkeypatch.setattr( task_status, 'is_completed_report', MagicMock(return_value=True)) load_fact_table_tasks.load_fact_tables( activity.Activity(boto3.client('swf', 'us-east-1')), '2015-10-09', 'feed_name', {}, 'theorchard') assert mock_executor_context.execute_query.call_count == 0 def test_set_overall_status_to_ingested(monkeypatch): """Test set overall status to INGESTED.""" monkeypatch.setattr( garcon_feed_status, 'set_overall_status', MagicMock(return_value=None)) monkeypatch.setattr( task_status, 'mark_report_processed_contexts', MagicMock(return_value=None)) monkeypatch.setattr( task_status, 'is_completed_report', MagicMock(return_value=True)) monkeypatch.setattr( task_status, 'mark_completed_overall_job', MagicMock(return_value=None)) load_fact_table_tasks.set_overall_status_to_ingested( activity.Activity(boto3.client('swf', 'us-east-1')), 'iTunes_theorchard', '2015-08-10') garcon_feed_status.set_overall_status.assert_any_call( 'iTunes_theorchard', '2015-08-10', 'INGESTED')