import os from unittest.mock import Mock from tests.conftest import DECOMPRESSED_BUCKET_TEST import pytest from slz_appreciationengine_clean_up.services.s3 import S3Service @pytest.mark.parametrize( 'key_prefix, initial_files, expected_deleted_files, suffix', [ ( 'appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme', [ 'BE_20210826_010000_040000_20210830150841.json', 'BE_20210826_040000_060000_20210830150841.json', 'BE_20210826_20210830150841.json', 'BE_segment_id.json', 'test.json.gz', ], [ 'BE_20210826_010000_040000_20210830150841.json', 'BE_20210826_040000_060000_20210830150841.json', 'BE_20210826_20210830150841.json', 'BE_segment_id.json', 'test.json.gz', ], None, ), ( 'appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme', [], [], None, ), ( 'appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme', [ 'BE_20210826_20210830150841.json', 'BE_20210826_010000_040000_20210830150841.json', 'BE_20210826_040000_060000_20210830150841.json', 'BE_segment_id.json', 'test.json.gz', ], [ 'BE_20210826_040000_060000_20210830150841.json', ], '_040000_060000', ), ] ) def test_s3_service(key_prefix, initial_files, expected_deleted_files, suffix, s3_client): # test file in different folder other_file_path = os.path.join(key_prefix.rsplit('/', maxsplit=1)[0], 'other', 'test.txt') s3_client.put_object( Body=b'data', Bucket=DECOMPRESSED_BUCKET_TEST, Key=other_file_path, ) # files that should be deleted for key in initial_files: s3_client.put_object( Body=b'data', Bucket=DECOMPRESSED_BUCKET_TEST, Key=f'{key_prefix}/{key}' ) key_count = s3_client.list_objects_v2( Bucket=DECOMPRESSED_BUCKET_TEST, Prefix=key_prefix, )['KeyCount'] assert len(initial_files) == key_count service = S3Service( logger=Mock(), s3_client=s3_client, buckets_to_clean=[DECOMPRESSED_BUCKET_TEST], keys_prefix=key_prefix, ) actual_deleted_files = service.clean_uow_s3_bucket_files(filename_suffix=suffix) assert [{'Key': f'{key_prefix}/{k}'} for k in expected_deleted_files] == actual_deleted_files # check that desired files are deleted key_count = s3_client.list_objects_v2( Bucket=DECOMPRESSED_BUCKET_TEST, Prefix=key_prefix, )['KeyCount'] assert key_count == len(initial_files) - len(expected_deleted_files) # check that test file in different folder was not deleted key_count = s3_client.list_objects_v2( Bucket=DECOMPRESSED_BUCKET_TEST, Prefix=other_file_path, )['KeyCount'] assert key_count == 1