"""Unit tests for MySQL related tasks.""" import subprocess import tempfile from unittest.mock import MagicMock, Mock, patch, PropertyMock import boto from boto.s3 import key from moto import mock_s3 import pymysql import pytest from garcon.contrib import mysql # noqa from garcon.contrib import mysql_utils # noqa @patch('subprocess.Popen') def test_pipe_mysql_from_stdin_to_stdout(popen_mock, monkeypatch): tmpfile = MagicMock() tempfile_mock = MagicMock(side_effect=[tmpfile, tmpfile]) monkeypatch.setattr(tempfile, 'TemporaryFile', tempfile_mock) with pytest.raises(Exception): mysql.pipe_mysql_from_stdin_to_stdout( None, 'host', 'port', 'user', 'password') mock_pipe = MagicMock(stdout=subprocess.PIPE) resp = mysql.pipe_mysql_from_stdin_to_stdout( MagicMock(), mock_pipe, 'host', 'port', 'user', 'password') assert isinstance(resp, dict) assert subprocess.Popen.called assert resp.get('pipe') assert resp.get('mysql_stderr') subprocess.Popen.assert_called_with([ 'mysql', '-q', '-N', '--host=host', '--port=port', '-u', 'user', '--password=password', '--protocol=TCP'], stdout=-1, stderr=tmpfile, close_fds=True, stdin=-1) @patch('boto.s3.key.Key') @patch('tempfile.NamedTemporaryFile') @mock_s3 def test_ingest_csv_from_s3(mock_named_tempfile, mock_boto_key, monkeypatch): """Tests db call of execution upload command """ activity_mock = Mock() # mock aws bucket test_file_name = 'test_file' test_bucket_name = 'test-bucket' file_to_ingest = 's3://{bucket}/{file}'.format( bucket=test_bucket_name, file=test_file_name) s3_connection = boto.connect_s3() bucket = s3_connection.create_bucket(test_bucket_name) k = key.Key(bucket) k.key = test_file_name k.set_contents_from_string('test key') # mock S3 bucket read into file mock_boto_key.return_value.get_contents_to_filename.return_value = True # mock file we will be ingesting mock_tempfile = MagicMock() # name property is a reserved word on Mock so need to attach it # as a Property... p = PropertyMock(return_value='temp_ingest_file') type(mock_tempfile).name = p mock_tempfile.seek.return_value = True mock_named_tempfile.return_value.__enter__.return_value = mock_tempfile # mock work with MySQL db_connection = MagicMock() monkeypatch.setattr( pymysql, 'connect', value=MagicMock(return_value=db_connection)) mysql_config = MagicMock() table_name = 'test_table' columns_names = ['column1', 'column2'] replace = True ignore_lines = 0 line_terminator = '\\n' fields_terminator = ',' # mock sql command sql_command_text = 'select 1' sql_command_mock = MagicMock(return_value=sql_command_text) monkeypatch.setattr( mysql_utils, 'create_sql_for_load_csv', value=sql_command_mock) mysql.ingest_csv_from_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, replace, ignore_lines=ignore_lines, line_terminator=line_terminator, fields_terminator=fields_terminator) # checking regular execution with db_connection.cursor() as cursor: cursor.execute.assert_called_with(sql_command_text) assert db_connection.commit.called assert db_connection.close.called # checking pymysql.Error exception raising with db_connection.cursor() as cursor: side_effect = MagicMock(side_effect=pymysql.Error()) monkeypatch.setattr(cursor, 'execute', value=side_effect) with pytest.raises(pymysql.Error): mysql.ingest_csv_from_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, replace, ignore_lines=ignore_lines, line_terminator=line_terminator, fields_terminator=fields_terminator) @patch('boto.s3.key.Key') @patch('tempfile.NamedTemporaryFile') @mock_s3 def test_bulk_insert_from_csv_file_on_s3( mock_named_tempfile, mock_boto_key, monkeypatch): """Tests db call of execution executemany command """ activity_mock = Mock() # mock aws bucket test_file_name = 'test_file' test_bucket_name = 'test-bucket' file_to_ingest = 's3://{bucket}/{file}'.format( bucket=test_bucket_name, file=test_file_name) s3_connection = boto.connect_s3() bucket = s3_connection.create_bucket(test_bucket_name) k = key.Key(bucket) k.key = test_file_name k.set_contents_from_string('"test, key1",test key2') # mock S3 bucket read into file mock_boto_key.return_value.get_contents_to_filename.return_value = True # mock file we will be ingesting mock_tempfile = MagicMock() # name property is a reserved word on Mock so need to attach it # as a Property... p = PropertyMock(return_value='temp_ingest_file') type(mock_tempfile).name = p mock_tempfile.seek.return_value = True mock_named_tempfile.return_value.__enter__.return_value = mock_tempfile # mock work with MySQL db_connection = MagicMock() monkeypatch.setattr( pymysql, 'connect', value=MagicMock(return_value=db_connection)) mysql_config = MagicMock() table_name = 'test_table' columns_names = ['column1', 'column2'] ignore_lines = 0 # mock sql command sql_command_text = 'select 1' sql_command_mock = MagicMock(return_value=sql_command_text) monkeypatch.setattr( mysql_utils, 'create_sql_for_load_csv', value=sql_command_mock) mysql.bulk_insert_from_csv_file_on_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, ignore_lines=ignore_lines) with db_connection.cursor() as cursor: sql = mysql_utils.create_sql_for_insert_or_replace_from_csv( table_name, columns_names) cursor.executemany.assert_called_with(sql, []) assert db_connection.commit.called assert db_connection.close.called # checking pymysql.Error exception raising with db_connection.cursor() as cursor: side_effect = MagicMock(side_effect=pymysql.Error()) monkeypatch.setattr(cursor, 'executemany', value=side_effect) with pytest.raises(pymysql.Error): mysql.bulk_insert_from_csv_file_on_s3( activity_mock, file_to_ingest, mysql_config, table_name, columns_names, ignore_lines=ignore_lines)