"""Unit tests for MySQL related tasks.""" import subprocess import tempfile from unittest.mock import MagicMock, Mock, patch, PropertyMock import boto3 from moto import mock_aws import pymysql import pytest from garcon_contrib.mysql import garcon_mysql # noqa from garcon_contrib.mysql import garcon_mysql_utils # noqa @patch('subprocess.Popen') def test_pipe_mysql_from_stdin_to_stdout(popen_mock, monkeypatch): tmpfile = MagicMock() tempfile_mock = MagicMock(side_effect=[tmpfile, tmpfile]) monkeypatch.setattr(tempfile, 'TemporaryFile', tempfile_mock) with pytest.raises(Exception): garcon_mysql.pipe_mysql_from_stdin_to_stdout( None, 'host', 'port', 'user', 'password') mock_pipe = MagicMock(stdout=subprocess.PIPE) resp = garcon_mysql.pipe_mysql_from_stdin_to_stdout( MagicMock(), mock_pipe, 'host', 'port', 'user', 'password') assert isinstance(resp, dict) assert subprocess.Popen.called assert resp.get('pipe') assert resp.get('mysql_stderr') subprocess.Popen.assert_called_with([ 'mysql', '-q', '-N', '--host=host', '--port=port', '-u', 'user', '--password=password', '--protocol=TCP'], stdout=-1, stderr=tmpfile, close_fds=True, stdin=-1) @patch('tempfile.NamedTemporaryFile') @mock_aws def test_ingest_csv_from_s3(mock_named_tempfile, monkeypatch): """Tests db call of execution upload command """ activity_mock = Mock() # mock aws bucket test_file_name = 'test_file' test_bucket_name = 'test-bucket' file_to_ingest = 's3://{bucket}/{file}'.format( bucket=test_bucket_name, file=test_file_name) s3 = boto3.client('s3') # just a note, if we use boto3.resource('s3'), not used in the method # then the test raises botocore.errorfactory.NoSuchBucket s3.create_bucket(Bucket=test_bucket_name) s3.put_object(Body=b'test key', Bucket=test_bucket_name, Key=test_file_name) # mock file we will be ingesting mock_tempfile = MagicMock() # name property is a reserved word on Mock so need to attach it # as a Property... p = PropertyMock(return_value='temp_ingest_file') type(mock_tempfile).name = p mock_tempfile.seek.return_value = True mock_named_tempfile.return_value.__enter__.return_value = mock_tempfile # mock work with MySQL db_connection = MagicMock() monkeypatch.setattr( pymysql, 'connect', value=MagicMock(return_value=db_connection)) mysql_config = MagicMock() table_name = 'test_table' columns_names = ['column1', 'column2'] replace = True ignore_lines = 0 line_terminator = '\\n' fields_terminator = ',' # mock sql command sql_command_text = 'select 1' sql_command_mock = MagicMock(return_value=sql_command_text) monkeypatch.setattr( garcon_mysql_utils, 'create_sql_for_load_csv', value=sql_command_mock) garcon_mysql.ingest_csv_from_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, replace, ignore_lines=ignore_lines, line_terminator=line_terminator, fields_terminator=fields_terminator) # checking regular execution with db_connection.cursor() as cursor: cursor.execute.assert_called_with(sql_command_text) assert db_connection.commit.called assert db_connection.close.called # checking pymysql.Error exception raising with db_connection.cursor() as cursor: side_effect = MagicMock(side_effect=pymysql.Error()) monkeypatch.setattr(cursor, 'execute', value=side_effect) with pytest.raises(pymysql.Error): garcon_mysql.ingest_csv_from_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, replace, ignore_lines=ignore_lines, line_terminator=line_terminator, fields_terminator=fields_terminator) @patch('tempfile.NamedTemporaryFile') @mock_aws def test_bulk_insert_from_csv_file_on_s3( mock_named_tempfile, monkeypatch): """Tests db call of execution executemany command """ activity_mock = Mock() # mock aws bucket test_file_name = 'test_file' test_bucket_name = 'test-bucket' file_to_ingest = 's3://{bucket}/{file}'.format( bucket=test_bucket_name, file=test_file_name) s3 = boto3.client('s3') s3.create_bucket(Bucket=test_bucket_name) s3.put_object(Body=b'"test, key1",test key2', Bucket=test_bucket_name, Key=test_file_name) # mock file we will be ingesting mock_tempfile = MagicMock() # name property is a reserved word on Mock so need to attach it # as a Property... p = PropertyMock(return_value='temp_ingest_file') type(mock_tempfile).name = p mock_tempfile.seek.return_value = True mock_named_tempfile.return_value.__enter__.return_value = mock_tempfile # mock work with MySQL db_connection = MagicMock() monkeypatch.setattr( pymysql, 'connect', value=MagicMock(return_value=db_connection)) mysql_config = MagicMock() table_name = 'test_table' columns_names = ['column1', 'column2'] ignore_lines = 0 # mock sql command sql_command_text = 'select 1' sql_command_mock = MagicMock(return_value=sql_command_text) monkeypatch.setattr( garcon_mysql_utils, 'create_sql_for_load_csv', value=sql_command_mock) garcon_mysql.bulk_insert_from_csv_file_on_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, ignore_lines=ignore_lines) with db_connection.cursor() as cursor: sql = garcon_mysql_utils.create_sql_for_insert_or_replace_from_csv( table_name, columns_names) cursor.executemany.assert_called_with(sql, []) assert db_connection.commit.called assert db_connection.close.called # checking pymysql.Error exception raising with db_connection.cursor() as cursor: side_effect = MagicMock(side_effect=pymysql.Error()) monkeypatch.setattr(cursor, 'executemany', value=side_effect) with pytest.raises(pymysql.Error): garcon_mysql.bulk_insert_from_csv_file_on_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, ignore_lines=ignore_lines)