import os import random import subprocess from unittest.mock import MagicMock from unittest.mock import patch import uuid import boto from boto.s3 import multidelete from boto.s3.connection import S3Connection from boto.s3.key import Key from garcon import activity from garcon.contrib.aws import s3 from garcon.contrib.aws.utils import s3 as s3_utils from moto import mock_s3 import pytest import smart_open def create_bucket(): """Create an S3 bucket. Note: This cannot be a fixture – otherwise the mock doesn't retain the state of the bucket (including files that may have been saved). Return: boto.s3.bucket.Bucket: newly created S3 bucket """ bucket_name = str(uuid.uuid4()).lower()[:5] connection = boto.connect_s3() connection.create_bucket(bucket_name) return connection.get_bucket(bucket_name) def create_random_file(bucket): """Create a random file in an S3 bucket. Return: boto.s3.key.Key: newly created S3 bucket key containing a random string """ current_file = Key(bucket) current_file.key = str(uuid.uuid4()).lower()[:5] current_file.set_contents_from_string('random value') return current_file def get_bucket_url(bucket): """Get the bucket url for a given bucket. Return: str: a well formed S3 url to the given bucket """ return 's3://{bucket_name}/'.format(bucket_name=bucket.name) @pytest.fixture(params=['error', 'deleted']) def multidelete_result(request): """Create a multidelete result object. Return: boto.s3.multidelete.MultiDeleteResult: result from a multidelete call """ resp = multidelete.MultiDeleteResult() if request.param == 'error': resp.errors = [True] resp.deleted = False else: resp.errors = False resp.deleted = [True] return resp @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') @mock_s3 def test_remove_files_from_path(monkeypatch): """Test the test_remove_files_from_path for best case scenario. File(s) exist and are deleted properly. """ bucket = create_bucket() bucket_url = get_bucket_url(bucket) file_count = int(random.random() * 10) + 1 for i in range(file_count): create_random_file(bucket).key task_response_return_files = s3.remove_files_from_path( activity.Activity(), bucket_url, True) deleted_files = task_response_return_files.get('s3.files_removed') assert len(deleted_files) == file_count task_response_not_return_files = s3.remove_files_from_path( activity.Activity(), bucket_url, False) assert task_response_not_return_files == {} @patch('garcon.contrib.aws.s3.S3Connection') @patch('garcon.contrib.aws.s3.Bucket') def test_remove_file_with_errors(mock_bucket, mock_connection, monkeypatch): """Test the test_remove_files_from_path in case of an error from S3. """ mock_result_object = MagicMock() mock_result_object.deleted = [] mock_result_object.errors = ['errors'] mock_file_bucket = MagicMock() mock_key = MagicMock() mock_key.name = 'test_key' mock_file_bucket.get_key = lambda s3_path: mock_key mock_file_bucket.delete_keys = lambda keys: mock_result_object mock_file_bucket.list = lambda prefix: [mock_key] mock_bucket.return_value = mock_file_bucket monkeypatch.setattr( s3_utils, 'extract_bucket_path', MagicMock( return_value=('cucumbers', 'destination_s3_path'))) with pytest.raises(Exception): s3.remove_files_from_path( activity.Activity(), 's3://cucumbers/test/test', True) @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') @mock_s3 def test_create_object(monkeypatch): """Test file being created on s3 """ monkeypatch.setattr( s3_utils, 'extract_bucket_path', MagicMock(return_value=('somekey', 'somevalue'))) monkeypatch.setattr( boto.s3.connection.Key, 'set_contents_from_string', MagicMock(return_value=None)) with pytest.raises(AssertionError): s3.create_object( activity.Activity(), 's3://somebucket/somefolder/', 'some content') @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') @mock_s3 def test_join_s3_objects(monkeypatch): """Test command in the right syntax """ bucket_name = 'test_bucket' connection = boto.connect_s3() connection.create_bucket(bucket_name) bucket = connection.get_bucket(bucket_name) file_count = 3 for i in range(file_count): current_file = Key(bucket) current_file.key = 'd/file{}'.format(i) current_file.set_contents_from_string('random value') expected_command = 'cat <(aws s3 cp s3://test_bucket/d/file0 -)' expected_command += ' <(aws s3 cp s3://test_bucket/d/file1 -)' expected_command += ' <(aws s3 cp s3://test_bucket/d/file2 -)' expected_command += ' | aws s3 cp - s3://test_bucket/dest_object' expected_command = expected_command.format(bucket=bucket_name) with patch.object( bucket, 'list', return_value=[ {'name': 'd/file0'}, {'name': 'd/file1'}, {'name': 'd/file2'}]): with patch('subprocess.call', return_value=None): s3.join_s3_objects( activity.Activity(), 's3://test_bucket/d', 's3://test_bucket/dest_object') assert subprocess.call.called subprocess.call.assert_called_with( ['bash', '-c', expected_command]) def test_stream_join_s3_objects(monkeypatch): """Test stream join s3 object """ mock_key = MagicMock() mock_key.name = 'obj1' mock_bucket = MagicMock() mock_bucket.list = lambda **arg: [mock_key] smart_open_obj = MagicMock() smart_open_obj.write = lambda line: None monkeypatch.setattr( s3_utils, 'extract_bucket_path', MagicMock(return_value=('sample_bucket', 'sample_key'))) monkeypatch.setattr( smart_open, 'smart_open', MagicMock(return_value=smart_open_obj)) monkeypatch.setattr( S3Connection, 'get_bucket', MagicMock(return_value=mock_bucket)) s3.stream_join_s3_objects( activity.Activity(), 'source_s3_path', 'destination_s3_path') assert smart_open_obj.__enter__.called assert smart_open_obj.__exit__.called