from boto.s3 import multidelete from boto.s3.key import Key from garcon import activity from moto import mock_s3 from unittest.mock import MagicMock import boto import os import pytest import random import uuid import subprocess from octopus.tasks import s3 from octopus.utils.aws import s3 as s3_utils def create_bucket(): """Create an S3 bucket. Note: This cannot be a fixture – otherwise the mock doesn't retain the state of the bucket (including files that may have been saved). Return: boto.s3.bucket.Bucket: newly created S3 bucket """ bucket_name = str(uuid.uuid4()).lower()[:5] connection = boto.connect_s3() connection.create_bucket(bucket_name) return connection.get_bucket(bucket_name) def create_random_file(bucket): """Create a random file in an S3 bucket. Return: boto.s3.key.Key: newly created S3 bucket key containing a random string """ current_file = Key(bucket) current_file.key = str(uuid.uuid4()).lower()[:5] current_file.set_contents_from_string('random value') return current_file def get_bucket_url(bucket): """Get the bucket url for a given bucket. Return: str: a well formed S3 url to the given bucket """ return 's3://{bucket_name}/'.format(bucket_name=bucket.name) @pytest.fixture(params=['error', 'deleted']) def multidelete_result(request): """Create a multidelete result object. Return: boto.s3.multidelete.MultiDeleteResult: result from a multidelete call """ resp = multidelete.MultiDeleteResult() if request.param == 'error': resp.errors = [True] resp.deleted = False else: resp.errors = False resp.deleted = [True] return resp @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') def test_remove_files_from_path(): """Test the test_remove_files_from_path for best case scenario. File(s) exist and are deleted properly. """ with mock_s3(): bucket = create_bucket() bucket_url = get_bucket_url(bucket) file_count = int(random.random() * 10) + 1 for i in range(file_count): create_random_file(bucket).key task_response = s3.remove_files_from_path( activity.Activity(), bucket_url) deleted_files = task_response.get('s3.files_removed') assert len(deleted_files) == file_count @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') def test_remove_file_with_errors(monkeypatch, multidelete_result): """Test the test_remove_files_from_path in case of an error from S3. """ with mock_s3(): bucket = create_bucket() bucket_url = get_bucket_url(bucket) monkeypatch.setattr( boto.s3.connection.Bucket, 'delete_keys', MagicMock( return_value=multidelete_result)) if multidelete_result.errors: with pytest.raises(Exception): s3.remove_files_from_path(activity.Activity(), bucket_url) else: task_response = s3.remove_files_from_path( activity.Activity(), bucket_url) assert len(task_response.get('s3.files_removed')) == 0 assert bucket.delete_keys.called @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') def test_create_object(monkeypatch): """Test file being created on s3 """ with mock_s3(): monkeypatch.setattr( s3_utils, 'extract_bucket_path', MagicMock(return_value=('somekey', 'somevalue'))) monkeypatch.setattr( boto.s3.connection.Key, 'set_contents_from_string', MagicMock(return_value=None)) with pytest.raises(AssertionError): s3.create_object( activity.Activity(), 's3://somebucket/somefolder/', 'some content') @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') def test_join_s3_objects(monkeypatch): """Test command in the right syntax """ with mock_s3(): bucket_name = 'test_join_s3_object_bucket' connection = boto.connect_s3() connection.create_bucket(bucket_name) bucket = connection.get_bucket(bucket_name) file_count = 3 for i in range(file_count): current_file = Key(bucket) current_file.key = 'file{}'.format(i) current_file.set_contents_from_string('random value') expected_command = 'cat <(aws s3 cp s3://{bucket}/file0 -)' expected_command += ' <(aws s3 cp s3://{bucket}/file1 -)' expected_command += ' <(aws s3 cp s3://{bucket}/file2 -)' expected_command += ' | aws s3 cp - s3://{bucket}/destination_object' expected_command = expected_command.format(bucket=bucket_name) monkeypatch.setattr(subprocess, 'call', MagicMock(return_value=None)) actual_command = s3.join_s3_objects( activity.Activity(), bucket_name, 'file', 'destination_object') assert actual_command == expected_command assert subprocess.call.called