"""Tests for the sync_sql_tables_snapshot script.""" import argparse import subprocess import time from unittest.mock import call from unittest.mock import MagicMock import boto3 import pytest from bin_scripts import sync_sql_tables_snapshot from snowflake_etl.flows.sql2sf.config import SourcesConf sources = SourcesConf() @pytest.fixture() def argparse_mock(): """Fixture returning monkeypatched argparse.ArgumentParser class.""" args_mock = MagicMock() args_mock.tables = None args_mock.destination_sf_db_name = None args_mock.destination_sf_schema_name = None parser_mock = MagicMock() parse_args_mock = MagicMock(return_value=args_mock) ArgumentParser_mock = MagicMock(return_value=parser_mock) parser_mock.parse_args = parse_args_mock return ArgumentParser_mock @pytest.fixture() def argparse_mock_tables(): """Fixture returning monkeypatched argparse.ArgumentParser class.""" args_mock = MagicMock() args_mock.tables = ['supply_chain_vendors', 'passthru_processed'] args_mock.destination_sf_db_name = None args_mock.destination_sf_schema_name = None parser_mock = MagicMock() parse_args_mock = MagicMock(return_value=args_mock) ArgumentParser_mock = MagicMock(return_value=parser_mock) parser_mock.parse_args = parse_args_mock return ArgumentParser_mock @pytest.fixture() def argparse_mock_custom_dest_db_and_schema(): """Fixture returning monkeypatched argparse.ArgumentParser class.""" args_mock = MagicMock() args_mock.tables = None args_mock.destination_sf_db_name = 'testdb' args_mock.destination_sf_schema_name = 'testschema' parser_mock = MagicMock() parse_args_mock = MagicMock(return_value=args_mock) ArgumentParser_mock = MagicMock(return_value=parser_mock) parser_mock.parse_args = parse_args_mock return ArgumentParser_mock @pytest.fixture() def argparse_mock_custom_dest_db(): """Fixture returning monkeypatched argparse.ArgumentParser class.""" args_mock = MagicMock() args_mock.tables = None args_mock.destination_sf_db_name = 'testdb' args_mock.destination_sf_schema_name = None parser_mock = MagicMock() parse_args_mock = MagicMock(return_value=args_mock) ArgumentParser_mock = MagicMock(return_value=parser_mock) parser_mock.parse_args = parse_args_mock return ArgumentParser_mock def test_create_workflow_context(monkeypatch, argparse_mock): """Test create_workflow_context function.""" monkeypatch.setattr(argparse, 'ArgumentParser', argparse_mock) context = sync_sql_tables_snapshot.create_workflow_context( ('reportsar02', 'accountingflat', 'dig_sales_detail', None), sfdb_params_override=None) assert context == dict( db_type='mysql', source_db_host='reportsar02', source_schema='accountingflat', source_table='dig_sales_detail', load_strategy='snapshot', query_type='snapshot', file_format=None, sfdb_params_override=None) def test_create_workflow_context_override_db_and_schema( monkeypatch, argparse_mock_custom_dest_db_and_schema): """Test create_workflow_context function.""" monkeypatch.setattr( argparse, 'ArgumentParser', argparse_mock_custom_dest_db_and_schema) context = sync_sql_tables_snapshot.create_workflow_context( ('reportsar02', 'accountingflat', 'dig_sales_detail', None), sfdb_params_override={ 'db': 'snowflake_database', 'schema': 'production'}) assert context == dict( db_type='mysql', source_db_host='reportsar02', source_schema='accountingflat', source_table='dig_sales_detail', load_strategy='snapshot', query_type='snapshot', file_format=None, sfdb_params_override={ 'db': 'snowflake_database', 'schema': 'production'}) def test_delayed_execution(monkeypatch, argparse_mock_tables): """Test if workflow execution postponed. (If 3 another workflows are already working on same db host and schema). """ monkeypatch.setattr(argparse, 'ArgumentParser', argparse_mock_tables) subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) sleep_mock = MagicMock() monkeypatch.setattr(time, 'sleep', sleep_mock) layer_mock1 = MagicMock() layer_mock2 = MagicMock() layer_mock3 = MagicMock() layer_mock4 = MagicMock() executioninfos_list = [{ 'execution': { 'workflowId': 'mysql-accountingflat.passthru_processed-snapshot-into'}, 'workflowType': {'name': 'sql2sf_snowflake_etl'}}, { 'execution': { 'workflowId': 'mysql-accountingflat.passthru_processed-snapshot-into'}, 'workflowType': {'name': 'sql2sf_snowflake_etl'}}, { 'execution': { 'workflowId': 'mysql-accountingflat.passthru_processed-snapshot-into'}, 'workflowType': {'name': 'sql2sf_snowflake_etl'}}] executioninfos_list_3 = {'executionInfos': executioninfos_list} executioninfos_list_2 = {'executionInfos': executioninfos_list[:2]} executioninfos_list_1 = {'executionInfos': [executioninfos_list[0]]} list_method_mock1 = MagicMock(side_effect=[executioninfos_list_3]) list_method_mock2 = MagicMock(side_effect=[executioninfos_list_2]) list_method_mock3 = MagicMock(side_effect=[executioninfos_list_1]) list_method_mock4 = MagicMock(side_effect=[{'executionInfos': []}]) layer_mock1.list_open_workflow_executions = list_method_mock1 layer_mock2.list_open_workflow_executions = list_method_mock2 layer_mock3.list_open_workflow_executions = list_method_mock3 layer_mock4.list_open_workflow_executions = list_method_mock4 swf_mock = MagicMock( side_effect=[layer_mock1, layer_mock2, layer_mock3, layer_mock4]) monkeypatch.setattr(boto3, 'client', swf_mock) sync_sql_tables_snapshot.main() assert sleep_mock.call_count == 1 assert subprocess_call_mock.call_count == 2 def test_all_workflows_executed(monkeypatch, argparse_mock): """Test if all workflows from the tables_to_sync list were started.""" monkeypatch.setattr(argparse, 'ArgumentParser', argparse_mock) subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) layer_mock = MagicMock() list_method_mock = MagicMock(return_value={}) layer_mock.list_open_workflow_executions = list_method_mock tables_to_sync = sources.get_tables_to_sync() swf_mock = MagicMock(side_effect=[layer_mock] * len(tables_to_sync)) monkeypatch.setattr(boto3, 'client', swf_mock) sync_sql_tables_snapshot.main() assert subprocess_call_mock.call_count == len(tables_to_sync) def test_all_workflows_executed_list_tables(monkeypatch, argparse_mock_tables): """Test if all workflows from the passed list were started.""" monkeypatch.setattr(argparse, 'ArgumentParser', argparse_mock_tables) subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) layer_mock = MagicMock() list_method_mock = MagicMock(return_value={}) layer_mock.list_open_workflow_executions = list_method_mock swf_mock = MagicMock(side_effect=[layer_mock] * 2) monkeypatch.setattr(boto3, 'client', swf_mock) sync_sql_tables_snapshot.main() assert subprocess_call_mock.call_count == 2 def test_exec_cmd(monkeypatch): """Test exec_cmd helper.""" subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) sync_sql_tables_snapshot.exec_cmd({'some': 'context'}) subprocess_call_mock.assert_called_once_with([ 'garcon', 'exec', 'sql2sf', '-c', '{"some": "context"}']) def test_no_sf_params_override(monkeypatch, argparse_mock): """Test exec_cmd helper when no custom db and schema were specified.""" monkeypatch.setattr(argparse, 'ArgumentParser', argparse_mock) subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) layer_mock = MagicMock() list_method_mock = MagicMock(return_value={}) layer_mock.list_open_workflow_executions = list_method_mock tables_to_sync = sources.get_tables_to_sync() swf_mock = MagicMock(side_effect=[layer_mock] * len(tables_to_sync)) monkeypatch.setattr(boto3, 'client', swf_mock) exec_cmd_mock = MagicMock() monkeypatch.setattr(sync_sql_tables_snapshot, 'exec_cmd', exec_cmd_mock) sync_sql_tables_snapshot.main() exec_cmd_mock.assert_has_calls([ call({'query_type': 'snapshot', 'source_schema': 'stmt-db', 'load_strategy': 'snapshot', 'source_table': 'dig_sales_period_id', 'file_format': None, 'sfdb_params_override': {'schema': None, 'db': 'stmt_db'}, # db was overrided to default to this table 'source_db_host': 'statements', 'db_type': 'mysql'})], any_order=True) def test_hyphen_in_db_cleared(monkeypatch, argparse_mock): """Test exec_cmd helper when hyphen was in schema name.""" monkeypatch.setattr(argparse, 'ArgumentParser', argparse_mock) subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) layer_mock = MagicMock() list_method_mock = MagicMock(return_value={}) layer_mock.list_open_workflow_executions = list_method_mock tables_to_sync = sources.get_tables_to_sync() swf_mock = MagicMock(side_effect=[layer_mock] * len(tables_to_sync)) monkeypatch.setattr(boto3, 'client', swf_mock) exec_cmd_mock = MagicMock() monkeypatch.setattr(sync_sql_tables_snapshot, 'exec_cmd', exec_cmd_mock) sync_sql_tables_snapshot.main() exec_cmd_mock.assert_has_calls([ call({'query_type': 'snapshot', 'source_schema': 'stmt-db', 'load_strategy': 'snapshot', 'source_table': 'fx_rates', 'file_format': None, 'sfdb_params_override': {'schema': None, 'db': 'stmt_db'}, # db was overrided to default to this table 'source_db_host': 'statements', 'db_type': 'mysql'})], any_order=True) def test_sf_params_db_override(monkeypatch, argparse_mock_custom_dest_db): """Test exec_cmd helper when custom db was specified.""" monkeypatch.setattr( argparse, 'ArgumentParser', argparse_mock_custom_dest_db) subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) layer_mock = MagicMock() list_method_mock = MagicMock(return_value={}) layer_mock.list_open_workflow_executions = list_method_mock tables_to_sync = sources.get_tables_to_sync() swf_mock = MagicMock(side_effect=[layer_mock] * len(tables_to_sync)) monkeypatch.setattr(boto3, 'client', swf_mock) exec_cmd_mock = MagicMock() monkeypatch.setattr(sync_sql_tables_snapshot, 'exec_cmd', exec_cmd_mock) sync_sql_tables_snapshot.main() exec_cmd_mock.assert_has_calls([ call({'query_type': 'snapshot', 'source_schema': 'stmt-db', 'load_strategy': 'snapshot', 'source_table': 'fx_rates', 'file_format': None, 'sfdb_params_override': {'schema': None, 'db': 'testdb'}, # db was overrided 'source_db_host': 'statements', 'db_type': 'mysql'})], any_order=True) def test_sf_params_db_and_schema_override( monkeypatch, argparse_mock_custom_dest_db_and_schema): """Test exec_cmd helper when custom db and schema were specified.""" monkeypatch.setattr( argparse, 'ArgumentParser', argparse_mock_custom_dest_db_and_schema) subprocess_call_mock = MagicMock() monkeypatch.setattr(subprocess, 'call', subprocess_call_mock) layer_mock = MagicMock() list_method_mock = MagicMock(return_value={}) layer_mock.list_open_workflow_executions = list_method_mock tables_to_sync = sources.get_tables_to_sync() swf_mock = MagicMock(side_effect=[layer_mock] * len(tables_to_sync)) monkeypatch.setattr(boto3, 'client', swf_mock) exec_cmd_mock = MagicMock() monkeypatch.setattr(sync_sql_tables_snapshot, 'exec_cmd', exec_cmd_mock) sync_sql_tables_snapshot.main() exec_cmd_mock.assert_has_calls([ call({'query_type': 'snapshot', 'source_schema': 'stmt-db', 'load_strategy': 'snapshot', 'source_table': 'fx_rates', 'file_format': None, 'sfdb_params_override': {'schema': 'testschema', 'db': 'testdb'}, # both were overrided 'source_db_host': 'statements', 'db_type': 'mysql'})], any_order=True)