"""Unit tests for tasks of Apple Music Demographics Preaggregation Workflow.""" import datetime from unittest.mock import MagicMock from unittest.mock import Mock from unittest.mock import call from unittest.mock import patch from garcon.contrib.dynamo_feed_status import \ feed_status_ingestion as feed_status from analytics_aggregation import base_config from analytics_aggregation.flows.apple_music_demographics import tasks class _DateWithTodayMock(datetime.date): """Mock datetime.date.today to return constant date.""" @classmethod def today(cls): return cls(2017, 1, 1) @patch('analytics_aggregation.flows.apple_music_demographics.tasks.datetime') @patch('analytics_aggregation.util.common') def test_bootstrap_all_processed(common, date_mock, monkeypatch): """Test bootstrap task.""" date_mock.date = _DateWithTodayMock activity_mock = MagicMock() range_result_mock = { 'found_days': False, 'week_range': '2016-12-25_2017-01-01'} range_to_process_mock = MagicMock(return_value=range_result_mock) monkeypatch.setattr(tasks, '_find_range_to_process', range_to_process_mock) tasks.bootstrap(activity_mock, 'feed_name', None, None) assert common.exit_message.called_with( 'There is no data to aggregate for 2016-12-25_2017-01-01 range') @patch('analytics_aggregation.flows.apple_music_demographics.tasks.datetime') def test_bootstrap_not_processed(date_mock, monkeypatch): """Test bootstrap task.""" date_mock.date = _DateWithTodayMock activity_mock = MagicMock() week_ago = '2016-12-25' today = '2017-01-01' range_result_mock = { 'found_days': True, 'start_date': week_ago, 'end_date': today } range_to_process_mock = MagicMock(return_value=range_result_mock) monkeypatch.setattr(tasks, '_find_range_to_process', range_to_process_mock) result = tasks.bootstrap(activity_mock, 'feed_name', None, None) assert not result['reload'] assert result['date_range']['start_date'] == week_ago assert result['date_range']['end_date'] == today @patch('analytics_aggregation.flows.apple_music_demographics.tasks.datetime') def test_bootstrap_not_processed_with_date_range(date_mock, monkeypatch): """Test bootstrap task.""" date_mock.date = _DateWithTodayMock activity_mock = MagicMock() last_day_apple_music_ingested_mock = MagicMock(return_value='2017-01-01') monkeypatch.setattr( tasks, '_last_day_apple_music_ingested', last_day_apple_music_ingested_mock) start_date = '2016-10-21' end_date = '2017-01-01' result = tasks.bootstrap( activity_mock, 'feed_name', '{}_{}'.format(start_date, end_date), True) assert result['reload'] assert result['date_range']['start_date'] == '2016-10-21' assert result['date_range']['end_date'] == '2017-01-01' @patch('analytics_aggregation.flows.apple_music_demographics.tasks.datetime') @patch('analytics_aggregation.util.common') def test_bootstrap_raw_data_is_not_available(common, date_mock, monkeypatch): """Test bootstrap task.""" date_mock.date = _DateWithTodayMock activity_mock = MagicMock() last_day_apple_music_ingested_mock = MagicMock(return_value='2016-01-01') monkeypatch.setattr( tasks, '_last_day_apple_music_ingested', last_day_apple_music_ingested_mock) tasks.bootstrap( activity_mock, 'feed_name', '2016-12-25_2017-01-01', True) assert common.exit_message.called_with( 'Raw Apple Music available only till 2016-01-01') @patch('analytics_aggregation.flows.apple_music_demographics.tasks.datetime') def test_find_range_to_process_all_processed(date_mock, monkeypatch): """Test _find_range_to_process function.""" date_mock.date = _DateWithTodayMock apple_music_ingested_mock = MagicMock() apple_music_ingested_mock.__contains__.return_value = True last_day_apple_music_ingested_mock = MagicMock( return_value=apple_music_ingested_mock) monkeypatch.setattr( tasks, '_last_dates_apple_music_ingested', last_day_apple_music_ingested_mock) get_overall_feed_status_mock = Mock( return_value=feed_status.STATUS_INGESTED) monkeypatch.setattr( feed_status, 'get_overall_status', get_overall_feed_status_mock) assert tasks._find_range_to_process('feed_name') == { 'found_days': False, 'week_range': '2016-12-18_2017-01-01'} @patch( 'analytics_aggregation.flows.apple_music_demographics.tasks.datetime') def test_find_range_to_process_not_processed(date_mock, monkeypatch): """Test _find_range_to_process function.""" date_mock.date = _DateWithTodayMock apple_music_ingested_mock = MagicMock() apple_music_ingested_mock.__contains__.return_value = True last_day_apple_music_ingested_mock = MagicMock( return_value=apple_music_ingested_mock) monkeypatch.setattr( tasks, '_last_dates_apple_music_ingested', last_day_apple_music_ingested_mock) get_overall_feed_status_mock = MagicMock(return_value='other_status') monkeypatch.setattr( feed_status, 'get_overall_status', get_overall_feed_status_mock) assert tasks._find_range_to_process('feed_name') == { 'found_days': True, 'week_range': '2016-12-18_2017-01-01', 'start_date': '2016-12-18', 'end_date': '2017-01-01'} @patch( 'analytics_aggregation.flows.apple_music_demographics.tasks.' '_cleanup_fact_demographics') @patch( 'analytics_aggregation.flows.apple_music_demographics.tasks.' '_populate_fact_demographics') def test_populate_fact_demographics( _cleanup_fact_demographics, _populate_fact_demographics, mock_snowflake_db): """Test for populate_fact_demographics.""" activity_mock = MagicMock() date_range = { 'start_date': '2017-01-01', 'end_date': '2017-01-01'} # test function call tasks.populate_fact_demographics(activity_mock, date_range) # checks mock_snowflake_db.SnowflakeSQLExecutor.assert_called_once_with( base_config.SNOWFLAKE_CONFIG) _cleanup_fact_demographics.assert_called_once_with( mock_snowflake_db.snowflake_sql_executor, date_range) _populate_fact_demographics.assert_called_once_with( mock_snowflake_db.snowflake_sql_executor, date_range) activity_mock.logger.info.assert_has_calls([ call('fact_demographics is cleared up.'), call('fact_demographics is populated.')]) @patch( 'analytics_aggregation.flows.apple_music_demographics.tasks._sql_loader') def test__cleanup_fact_demographics(_sql_loader_mock): """Test _cleanup_fact_demographics.""" executor_mock = MagicMock() date_range = {'test': 'date'} formated_sql = Mock() non_identifier_params = Mock() executor_mock.validator.format_identifiers.return_value = ( formated_sql, non_identifier_params) tasks._cleanup_fact_demographics(executor_mock, date_range) _sql_loader_mock.load_query.assert_called_once_with( 'cleanup_fact_demographics') executor_mock.execute.assert_called_once_with( formated_sql, non_identifier_params) @patch( 'analytics_aggregation.flows.apple_music_demographics.tasks._sql_loader') def test__populate_fact_demographics(_sql_loader_mock): """Test _populate_fact_demographics.""" executor_mock = MagicMock() date_range = {'test': 'date'} formated_sql = Mock() non_identifier_params = Mock() executor_mock.validator.format_identifiers.return_value = ( formated_sql, non_identifier_params) tasks._populate_fact_demographics(executor_mock, date_range) _sql_loader_mock.load_query.assert_called_once_with( 'populate_fact_demographics') executor_mock.execute.assert_called_once_with( formated_sql, non_identifier_params) @patch( 'analytics_aggregation.flows.apple_music_demographics.tasks._sql_loader') def test_last_day_apple_music_ingested_with_date( sql_loader_mock, mock_snowflake_db): """Test _last_day_apple_music_ingested function with date result.""" sql = Mock() non_identifier_params = Mock() mock_snowflake_db.snowflake_sql_executor.validator.format_identifiers = ( Mock(return_value=(sql, non_identifier_params))) last_date = datetime.datetime(2017, 1, 1) mock_snowflake_db.snowflake_sql_executor.fetchone.return_value = ( last_date,) result = tasks._last_day_apple_music_ingested(7) sql_loader_mock.load_query.assert_called_once_with( 'get_max_apple_music_raw_data_date') mock_snowflake_db.snowflake_sql_executor.fetchone.assert_called_once_with( sql, non_identifier_params) assert result == last_date.strftime('%Y-%m-%d') @patch( 'analytics_aggregation.flows.apple_music_demographics.tasks._sql_loader') def test_last_day_apple_music_ingested_without_date( sql_loader_mock, mock_snowflake_db): """Test _last_day_apple_music_ingested function without date result.""" sql = Mock() non_identifier_params = Mock() mock_snowflake_db.snowflake_sql_executor.validator.format_identifiers = ( Mock(return_value=(sql, non_identifier_params))) last_date = None mock_snowflake_db.snowflake_sql_executor.fetchone.return_value = ( last_date,) result = tasks._last_day_apple_music_ingested(7) sql_loader_mock.load_query.assert_called_once_with( 'get_max_apple_music_raw_data_date') mock_snowflake_db.snowflake_sql_executor.fetchone.assert_called_once_with( sql, non_identifier_params) assert result == last_date def test_last_dates_apple_music_ingested_with_result(mock_snowflake_db): """Test last_dates_apple_music_ingested function with result.""" # Mocking days_back = 10 sql = Mock() non_identifier_params = Mock() mock_snowflake_db.snowflake_sql_executor.validator.format_identifiers = ( Mock(return_value=(sql, non_identifier_params))) db_dates = [datetime.datetime(2018, 1, 1), datetime.datetime(2018, 1, 2)] mock_snowflake_db.snowflake_sql_executor.fetchall.return_value = ( [(date,) for date in db_dates]) expected_result = {date.strftime('%Y-%m-%d') for date in db_dates} # Tested function call result = tasks._last_dates_apple_music_ingested(days_back) # Checks assert expected_result == result def test_last_dates_apple_music_ingested_with_empty_result(mock_snowflake_db): """Test last_dates_apple_music_ingested function with empty result.""" # Mocking days_back = 10 sql = Mock() non_identifier_params = Mock() mock_snowflake_db.snowflake_sql_executor.validator.format_identifiers = ( Mock(return_value=(sql, non_identifier_params))) expected_result = set() mock_snowflake_db.snowflake_sql_executor.fetchall.return_value = ( [(datetime.datetime.strptime(date, '%Y-%m-%d'),) for date in expected_result]) # Tested function call result = tasks._last_dates_apple_music_ingested(days_back) # Checks assert expected_result == result