"""Unit tests for youtube_asset_conflict specific Snowflake SQL executor.""" from unittest.mock import MagicMock from tests.flows.fixtures import conflict_ids_with_es_ids as es_ids_records from yt_conflict_elasticsearch.flows.elasticsearch_export \ import snowflake_executor def _find_execute_query_in_mock_calls(mock_calls): """Return query ran in the execute call.""" for name, args, _ in mock_calls: if 'execute' in name: return args def test_store_csv_to_s3_conflicts(mocker, sf_config_mock): """Test store_csv_to_s3_conflicts function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) snowflake_executor.YouTubeConflictSFExecutor( sf_config_mock).store_csv_to_s3_conflicts( 's3://bucket/folder/prefix', 'new') execute_query, _ = _find_execute_query_in_mock_calls( mock_connect.mock_calls) if execute_query: assert 'fc.resolved_datetime is NULL and a.conflict_id is NULL' \ in execute_query else: assert False def test_mark_indexed_conflicts(mocker, sf_config_mock): """Test mark_indexed_conflicts function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) snowflake_executor.YouTubeConflictSFExecutor( sf_config_mock).mark_indexed_conflicts() execute_query, _ = _find_execute_query_in_mock_calls( mock_connect.mock_calls) expected_query = ( 'UPDATE facts.test_schema.fact_conflict fc\n' + 'SET fc.es_indexed = true\n' + 'WHERE fc.es_indexed = false;\n') assert execute_query == expected_query def test_get_responded_conflicts(mocker, sf_config_mock): """Test get_responded_conflicts function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) snowflake_executor.YouTubeConflictSFExecutor( sf_config_mock).get_responded_conflicts() execute_query, _ = _find_execute_query_in_mock_calls( mock_connect.mock_calls) expected_query = ( 'SELECT conflict.es_id\n' 'FROM facts.test_schema.fact_conflict conflict\n' 'JOIN ows_conflict_manager.test_schema.action action\n' 'ON conflict.conflict_id = action.conflict_id\n' 'WHERE conflict.es_id IS NOT NULL\n' 'GROUP BY conflict.es_id;\n') assert execute_query == expected_query def test_get_resolved_conflicts(mocker, sf_config_mock): """Test get_resolved_conflicts function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) snowflake_executor.YouTubeConflictSFExecutor( sf_config_mock).get_resolved_conflicts() execute_query, _ = _find_execute_query_in_mock_calls( mock_connect.mock_calls) expected_query = ( 'SELECT a.es_id\n' 'FROM\n' '(\n' ' SELECT es_id, COUNT(conflict_id) as' ' t1 FROM facts.test_schema.fact_conflict\n' ' WHERE es_id IS NOT NULL\n' ' GROUP BY es_id\n' ') a\n' 'JOIN\n' '(\n' ' SELECT es_id, COUNT(conflict_id) as' ' t2 FROM facts.test_schema.fact_conflict\n' ' WHERE resolved_datetime IS NOT NULL\n' ' AND es_id IS NOT NULL\n' ' GROUP BY es_id\n' ') b\n' 'ON a.es_id = b.es_id\n' 'WHERE t1 = t2;\n') assert execute_query == expected_query def test_create_temp_conflict_to_es_id(mocker, sf_config_mock): """Test create_temp_conflict_to_es_id function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) snowflake_executor.YouTubeConflictSFExecutor( sf_config_mock).create_temp_conflict_to_es_id() execute_query, _ = _find_execute_query_in_mock_calls( mock_connect.mock_calls) expected_query = ( 'CREATE OR REPLACE TABLE {db}.{schema}.{table}\n' '(conflict_id NUMBER, es_id VARCHAR);\n'.format( db=sf_config_mock['db'], schema=sf_config_mock['schema'], table=sf_config_mock['temp_conflict_es_id_table'])) assert execute_query == expected_query def test_fill_temp_table_with_es_ids(mocker, sf_config_mock): """Test fill_temp_table_with_es_ids function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) snowflake_executor.YouTubeConflictSFExecutor(sf_config_mock).\ fill_temp_table_with_es_ids( sf_config_mock['temp_conflict_es_id_table'], es_ids_records.ES_ID_RECORDS) execute_query, query_params = _find_execute_query_in_mock_calls( mock_connect.mock_calls) expected_table_name = '{db}.{schema}.{table}'.format( db=sf_config_mock['db'], schema=sf_config_mock['schema'], table=sf_config_mock['temp_conflict_es_id_table']) assert execute_query assert expected_table_name in execute_query assert query_params == es_ids_records.ES_ID_RECORDS def test_populate_es_ids_to_conflicts(mocker, sf_config_mock): """Test populate_es_ids_to_conflicts function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) snowflake_executor.YouTubeConflictSFExecutor( sf_config_mock).populate_es_ids_to_conflicts() execute_query, _ = _find_execute_query_in_mock_calls( mock_connect.mock_calls) expected_query = ( 'UPDATE {db}.{schema}.fact_conflict fc\n' 'SET fc.es_id = teid.es_id\n' 'FROM {db}.{schema}.{table} teid\n' 'WHERE fc.conflict_id = teid.conflict_id;\n'.format( db=sf_config_mock['db'], schema=sf_config_mock['schema'], table=sf_config_mock['temp_conflict_es_id_table'])) assert execute_query == expected_query def test_remove_selected_es_ids(mocker, sf_config_mock): """Test remove_selected_es_ids function in snowflake_executor.""" mock_connect = MagicMock() mocker.patch('snowflake.connector.connect', return_value=mock_connect) expected_params = {'es_ids': ['hash1', 'hash2']} sample_es_ids = [('hash1',), ('hash2',)] snowflake_executor.YouTubeConflictSFExecutor( sf_config_mock).remove_selected_es_ids(sample_es_ids) expected_table_name = 'facts.{schema}.fact_conflict'.format( schema=sf_config_mock['schema']) execute_query, query_params = _find_execute_query_in_mock_calls( mock_connect.mock_calls) assert execute_query assert expected_table_name in execute_query assert query_params == expected_params