import boto3 import pytest from unittest.mock import MagicMock from garcon_contrib.aws import garcon_s3 BUCKET = 'dev-cucumbers' @pytest.fixture() def s3_bucket_and_path(): """Fixture to create some S3 files.""" files = [ 'pathA/fileA1', 'pathA/fileA2', 'pathB/fileB1', ] s3 = boto3.resource('s3') bucket = s3.Bucket(BUCKET) common_path = 'test_garcon_contrib' _clean(bucket, common_path) for file in files: bucket.put_object(Key=f'{common_path}/{file}', Body=file) yield bucket, common_path _clean(bucket, common_path) def _clean(bucket, path): bucket.objects.filter(Prefix=f'{path}/').delete() def _list(bucket, path): return [obj.key for obj in bucket.objects.filter(Prefix=f'{path}/')] def test_remove_files_from_path(s3_bucket_and_path): bucket, common_path = s3_bucket_and_path assert _list(bucket, common_path) == [ 'test_garcon_contrib/pathA/fileA1', 'test_garcon_contrib/pathA/fileA2', 'test_garcon_contrib/pathB/fileB1' ] result = garcon_s3.remove_files_from_path( activity=MagicMock(), path=f's3://{bucket.name}/{common_path}/pathA/', return_deleted_files=True ) assert result == {'s3.files_removed': [ 'test_garcon_contrib/pathA/fileA1', 'test_garcon_contrib/pathA/fileA2', ]} assert _list(bucket, common_path) == [ 'test_garcon_contrib/pathB/fileB1' ] def test_create_object(s3_bucket_and_path): bucket, common_path = s3_bucket_and_path assert _list(bucket, common_path) == [ 'test_garcon_contrib/pathA/fileA1', 'test_garcon_contrib/pathA/fileA2', 'test_garcon_contrib/pathB/fileB1', ] garcon_s3.create_object( activity=MagicMock(), path=f's3://{bucket.name}/{common_path}/pathC/fileC1', content='content', ) assert _list(bucket, common_path) == [ 'test_garcon_contrib/pathA/fileA1', 'test_garcon_contrib/pathA/fileA2', 'test_garcon_contrib/pathB/fileB1', 'test_garcon_contrib/pathC/fileC1' ] assert (bucket.Object(f'{common_path}/pathC/fileC1') .get()['Body'].read().decode()) == 'content' def test_join_s3_objects(s3_bucket_and_path): bucket, common_path = s3_bucket_and_path garcon_s3.join_s3_objects( activity=MagicMock(), source_s3_path=f's3://{bucket.name}/{common_path}/pathA/', destination_s3_path=f's3://{bucket.name}/{common_path}/joined_pathA', ) assert _list(bucket, common_path) == [ 'test_garcon_contrib/joined_pathA', 'test_garcon_contrib/pathA/fileA1', 'test_garcon_contrib/pathA/fileA2', 'test_garcon_contrib/pathB/fileB1', ] assert (bucket.Object(f'{common_path}/joined_pathA') .get()['Body'].read().decode()) == 'pathA/fileA1pathA/fileA2' def test_stream_join_s3_objects(s3_bucket_and_path): bucket, common_path = s3_bucket_and_path garcon_s3.stream_join_s3_objects( activity=MagicMock(), source_s3_path=f's3://{bucket.name}/{common_path}/pathA/', destination_s3_path=f's3://{bucket.name}/{common_path}/stream_joined_pathA', ) assert _list(bucket, common_path) == [ 'test_garcon_contrib/pathA/fileA1', 'test_garcon_contrib/pathA/fileA2', 'test_garcon_contrib/pathB/fileB1', 'test_garcon_contrib/stream_joined_pathA', ] assert (bucket.Object(f'{common_path}/stream_joined_pathA') .get()['Body'].read().decode()) == 'pathA/fileA1pathA/fileA2'