from unittest.mock import Mock from tests.conftest import DECOMPRESSED_BUCKET_TEST import pytest from slz_clean_up.services.s3 import S3Service @pytest.mark.parametrize( 'key_prefix, expected_deleted_files', [ ( 'apple/artistdemographics/v1_0/report_date=2019-05-01/report_licensor=sme', [ 'apple/artistdemographics/v1_0/report_date=2019-05-01/report_licensor=sme/test_1.txt', 'apple/artistdemographics/v1_0/report_date=2019-05-01/report_licensor=sme/test_2.txt', ] ), ( 'appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme', [ 'appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme/BE_20210826_20210830150841.json', 'appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme/BE_segment_id.json', 'appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme/test.json.gz', ] ), ('appreciationengine/membersvisittotals/v1/report_date=2021-08-26/report_licensor=sme', []) ] ) def test_s3_service(key_prefix, expected_deleted_files, s3_client): # test file in different folder other_file_path = key_prefix.rsplit('/', maxsplit=1)[0] s3_client.put_object( Body=b'data', Bucket=DECOMPRESSED_BUCKET_TEST, Key=other_file_path + 'test.txt' ) # files that should be deleted for key in expected_deleted_files: s3_client.put_object(Body=b'data', Bucket=DECOMPRESSED_BUCKET_TEST, Key=key) key_count = s3_client.list_objects_v2( Bucket=DECOMPRESSED_BUCKET_TEST, Prefix=key_prefix, )['KeyCount'] assert len(expected_deleted_files) == key_count service = S3Service( logger=Mock(), s3_client=s3_client, buckets_to_clean=[DECOMPRESSED_BUCKET_TEST], keys_prefix=key_prefix, ) actual_deleted_files = service.clean_uow_s3_bucket_files() assert [{'Key': k} for k in expected_deleted_files] == actual_deleted_files # check that desired files are deleted key_count = s3_client.list_objects_v2( Bucket=DECOMPRESSED_BUCKET_TEST, Prefix=key_prefix, )['KeyCount'] assert key_count == 0 # check that test file in different folder was not deleted key_count = s3_client.list_objects_v2( Bucket=DECOMPRESSED_BUCKET_TEST, Prefix=other_file_path, )['KeyCount'] assert key_count == 1