"""Unit tests for the tasks of datadog_caching_monitor.""" from datetime import datetime from unittest.mock import call, MagicMock, patch from snowflake.connector.errors import ProgrammingError from snowflake_connector.metadata_connector import SnowflakeMetadataConnector from swf_monitoring.flows.datadog_caching_monitor import config from swf_monitoring.flows.datadog_caching_monitor import tasks def test__create_metric(): """Test _create_metric.""" event = { 'test_metric': 10, 'start_time': datetime.fromtimestamp(1575376877.1), 'correlation_id': 'test_correlation_id', 'query_id': 'test_query_id', 'day': '20191201', 'hour': '05', 'user_name': 'TEST_SF_USER', 'warehouse_name': 'TEST_WAREHOUSE' } expected_result = { 'metric': 'snowflake_caching_monitor.test_metric', 'points': [(1575376877.1, 10)], 'tags': [ 'snowflake_caching_monitor', 'correlation_id:test_correlation_id', 'query_id:test_query_id', 'day:20191201', 'hour:05', 'user_name:TEST_SF_USER', 'warehouse_name:TEST_WAREHOUSE' ]} assert expected_result == tasks._create_metric('test_metric', event) def test__get_cache_stats(monkeypatch): """Test _get_cache_stats.""" activity_mock = MagicMock() get_query_stats_by_sfqid_mock = MagicMock( return_value=[{ 'remote_bytes': 0, 'local_bytes': 10}]) monkeypatch.setattr( SnowflakeMetadataConnector, 'authenticate', MagicMock()) monkeypatch.setattr( SnowflakeMetadataConnector, 'get_query_stats_by_sfqid', get_query_stats_by_sfqid_mock) expected_result = { 'q1': [{'remote_bytes': 0, 'local_bytes': 10}], 'q2': [{'remote_bytes': 0, 'local_bytes': 10}]} assert expected_result == tasks._get_cache_stats( activity_mock, ['q1', 'q2']) @patch( 'swf_monitoring.flows.datadog_caching_monitor.tasks.executor_class') def test_populate_cache_stats(executor_mock, monkeypatch): """Test populate_cache_stats.""" activity_mock = MagicMock() get_snowflake_warehouses_to_monitor_mock = MagicMock( return_value=['WH1', 'WH2']) monkeypatch.setattr( config, 'get_snowflake_warehouses_to_monitor', get_snowflake_warehouses_to_monitor_mock) tasks.populate_cache_stats(activity_mock) activity_mock.logger.info.assert_has_calls([ call('Populating stats table...'), call('The table cleared from queries not send to Datadog...'), call('Populating table for WH1...'), call('Populating table for WH2...'), call('Stats table populated!') ]) @patch( 'swf_monitoring.flows.datadog_caching_monitor.tasks.executor_class') def test_populate_cache_stats_timeout_on_metadada_query( executor_mock, monkeypatch): """Test populate_cache_stats when timeout on metadata query happens.""" activity_mock = MagicMock() executor_mock.return_value.__enter__.return_value.fetchone_query.\ side_effect = [ None, None, ProgrammingError( 'Statement reached its statement or warehouse timeout')] result = tasks.populate_cache_stats(activity_mock) assert result == {'stop': True, 'result': 'Statement reached its timeout'} @patch('swf_monitoring.flows.datadog_caching_monitor.tasks.executor_class') @patch( 'swf_monitoring.flows.datadog_caching_monitor.tasks.' 'SnowflakeMetadataConnector') def test_update_metadata_in_cache_stats_table( metadata_connector_mock, executor_mock, monkeypatch): """Test update_metadata_in_cache_stats_table.""" activity_mock = MagicMock() executor_mock.return_value.__enter__.return_value.validator.\ format_identifiers.return_value = ('SQL', {}) tasks.update_metadata_in_cache_stats_table(activity_mock) activity_mock.logger.info.assert_has_calls([ call('Getting metadata from Snowflake API...'), call('Metadata updated!') ]) @patch( 'swf_monitoring.flows.datadog_caching_monitor.tasks.executor_class') @patch('swf_monitoring.flows.datadog_caching_monitor.tasks.datadog') @patch('swf_monitoring.flows.datadog_caching_monitor.tasks.config') def test_send_query_level_metrics_to_datadog( config_mock, datadog_mock, executor_mock): """Test send_query_level_metrics_to_datadog.""" activity_mock = MagicMock() fetchall_mock = MagicMock() executor_mock.return_value.__enter__.return_value.\ fetchall_query = fetchall_mock config_mock.DATADOG_API_KEY = 'test_api_key' tasks.send_query_level_metrics_to_datadog(activity_mock) datadog_mock.initialize.assert_called_once_with(api_key='test_api_key') fetchall_mock.call_count = 2 activity_mock.logger.info.assert_has_calls([ call('Sending metrics to Datadog...'), call('Metrics sent!') ]) @patch( 'swf_monitoring.flows.datadog_caching_monitor.snowflake_executor.' 'DatadogCachingMonitorExecutor') @patch('swf_monitoring.flows.datadog_caching_monitor.tasks.datadog') def send_send_hits_to_kw_cache_percentage_to_datadog( datadog_mock, executor_mock): """Test send_hits_to_kw_cache_percentage_to_datadog.""" activity_mock = MagicMock() fetchone_mock = MagicMock() executor_mock.return_value.__enter__.return_value. \ fetchone_query = fetchone_mock tasks.send_hits_to_kw_cache_percentage_to_datadog(activity_mock) datadog_mock.initialize.assert_called_once_with(api_key='test_api_key') fetchone_mock.call_count = 1 activity_mock.logger.info.assert_has_calls([ call('Sending daily aggregated metrics to Datadog...'), call('Updated kv hits percentage for a day sent!') ])