from unittest.mock import MagicMock from unittest.mock import patch import boto3 from garcon_contrib.aws.utils import garcon_s3 import pytest from processing_accounting.util import s3 @patch('processing_accounting.util.s3.S3Connection') @patch('boto.s3.key.Key') def test_download(mock_s3_key, mock_s3_connection, monkeypatch): monkeypatch.setattr( garcon_s3, 'extract_bucket_path', MagicMock( return_value=('cucumbers', 'destination_s3_path'))) mock_s3_connection_obj = MagicMock() mock_s3_bucket_object = MagicMock() mock_s3_key_object = MagicMock() monkeypatch.setattr( mock_s3_connection_obj, 'get_bucket', MagicMock( return_value=mock_s3_bucket_object)) mock_s3_connection.return_value = mock_s3_connection_obj monkeypatch.setattr( mock_s3_bucket_object, 'list', MagicMock( return_value=[mock_s3_key_object])) monkeypatch.setattr( mock_s3_key_object, 'get_contents_to_filename', MagicMock( return_value=None)) mock_s3_key.return_value = mock_s3_key_object s3.download('s3://cucumbers/destination_s3_path', 'local_path') mock_s3_key_object.get_contents_to_filename.assert_any_call('local_path') @patch('processing_accounting.util.s3.S3Connection') @patch('boto.s3.key.Key') def test_upload_raw_file_to_s3(mock_s3_key, mock_s3_connection, monkeypatch): monkeypatch.setattr( garcon_s3, 'extract_bucket_path', MagicMock( return_value=('cucumbers', 'destination_s3_path'))) mock_s3_connection_obj = MagicMock() mock_s3_bucket_object = MagicMock() mock_s3_key_object = MagicMock() monkeypatch.setattr( mock_s3_connection_obj, 'get_bucket', MagicMock( return_value=mock_s3_bucket_object)) mock_s3_connection.return_value = mock_s3_connection_obj monkeypatch.setattr( mock_s3_bucket_object, 'new_key', MagicMock( return_value=mock_s3_key_object)) monkeypatch.setattr( mock_s3_key_object, 'get_contents_to_filename', MagicMock( return_value=None)) mock_s3_key.return_value = mock_s3_key_object s3.upload_raw_file_to_s3('source_path', 'destination_s3_path') mock_s3_bucket_object.new_key.assert_any_call('destination_s3_path') mock_s3_key_object.set_contents_from_filename.assert_any_call( 'source_path', None, True) @patch('processing_accounting.util.s3.S3Transfer') def test_upload_boto3(mockS3Transfer, monkeypatch): """test upload_boto3 method """ mock_client = MagicMock() mock_transfer_object = MagicMock() monkeypatch.setattr( mock_transfer_object, 'upload_file', MagicMock(return_value=None)) mockS3Transfer.return_value = mock_transfer_object monkeypatch.setattr( boto3, 'client', MagicMock(return_value=mock_client)) monkeypatch.setattr( garcon_s3, 'extract_bucket_path', MagicMock(return_value=('bucket', 'sample_key'))) s3.upload_boto3('local_path', 's3://bucket/sample_key') mock_transfer_object.upload_file.assert_any_call( 'local_path', 'bucket', 'sample_key') @patch('processing_accounting.util.s3.S3Transfer') def test_download_boto3(mockS3Transfer, monkeypatch): """test download_boto3 method """ mock_client_object = MagicMock() mock_transfer_object = MagicMock() mock_transfer_object.download_file = MagicMock(return_value=None) mock_responses = { 'Contents': [{'Key': 'fileA'}, {'Key': 'fileB'}] } mockS3Transfer.return_value = mock_transfer_object mock_client_object.list_objects.return_value = mock_responses monkeypatch.setattr( boto3, 'client', MagicMock(return_value=mock_client_object)) s3.download_boto3('s3://bucket/sample_key', 'local_path') mock_transfer_object.download_file.assert_any_call( 'bucket', 'fileA', 'local_path_fileA') mock_transfer_object.download_file.assert_any_call( 'bucket', 'fileB', 'local_path_fileB') @patch('processing_accounting.util.s3.S3Transfer') def test_download_boto3_file_doesnt_exist(mockS3Transfer, monkeypatch): """Test download_boto3 method file not found. """ mock_client_object = MagicMock() mock_responses = { 'hmm..weird response' } mock_client_object.list_objects.return_value = mock_responses monkeypatch.setattr( boto3, 'client', MagicMock(return_value=mock_client_object)) with pytest.raises(Exception): s3.download_boto3('s3://bucket/sample_key', 'local_path') @patch('processing_accounting.util.s3.boto3.client') def test_delete_boto3(mock_boto): """Test deleting an S3 object.""" mock_client = MagicMock() mock_boto.return_value = mock_client s3.delete_boto3('s3://bucket/sample_key') mock_boto.assert_called_with('s3') mock_client.delete_object.assert_called_with( Bucket='bucket', Key='sample_key')