from contextlib import nullcontext as does_not_raise from typing import Any, Dict, List, Union import mock.mock import pytest from _pytest.python_api import RaisesContext from mypy_boto3_s3 import S3Client from smelog.factory import SmeBoundLogger from slz_spotify_charts_clean_up.const import Environment from slz_spotify_charts_clean_up.entities import Config from slz_spotify_charts_clean_up.manager import Manager from slz_spotify_charts_clean_up.services.db import DBService from slz_spotify_charts_clean_up.services.file_storage import FileStorageService from slz_spotify_charts_clean_up.services.job import JobService from tests.conftest import CommonValues @pytest.mark.parametrize( 'given, expected, expected_exception', [ ({}, '', pytest.raises(KeyError)), ( dict( dsp='dsp', report_type='report_type', report_date='report_date', licensor='report_licensor', ), '', pytest.raises(KeyError), ), ( dict( dsp='dsp', report_type='report_type', version='version', report_date='report_date', licensor='report_licensor', ), 'dsp/report_type/version/report_date=report_date/report_licensor=report_licensor', does_not_raise(), ), ( dict( dsp='spotify', report_type='charts_daily_regional', version='v1', report_date='2021-12-10', licensor='sme', ), 'spotify/charts_daily_regional/v1/report_date=2021-12-10/report_licensor=sme', does_not_raise(), ), ] ) def test__get_reports_prefix( given: Dict[str, Any], expected: str, expected_exception: Union[RaisesContext, does_not_raise], ): manager = Manager( config=mock.MagicMock(spec=Config), logger=mock.MagicMock(spec=SmeBoundLogger), db_service=mock.MagicMock(spec=DBService), job_service=mock.MagicMock(spec=JobService), file_storage=mock.MagicMock(spec=FileStorageService), ) with expected_exception: result = manager.get_reports_prefix(given) assert result == expected @pytest.mark.parametrize( 'initial_files, expected_result', [ ( { CommonValues.decompressed_bucket: ['test.txt', 'check.json', 'report.json'], CommonValues.corrupted_bucket: ['test.txt', 'check.json', 'report.json'], }, { CommonValues.decompressed_bucket: [ f'{CommonValues.file_prefix}/check.json', f'{CommonValues.file_prefix}/report.json', f'{CommonValues.file_prefix}/test.txt', ], CommonValues.corrupted_bucket: [ f'{CommonValues.file_prefix}/check.json', f'{CommonValues.file_prefix}/report.json', f'{CommonValues.file_prefix}/test.txt', ], }, ), ( { CommonValues.decompressed_bucket: ['check.json', 'report.json'], CommonValues.corrupted_bucket: ['test.txt', 'check.json', 'report.json'], }, { CommonValues.decompressed_bucket: [ f'{CommonValues.file_prefix}/check.json', f'{CommonValues.file_prefix}/report.json', ], CommonValues.corrupted_bucket: [ f'{CommonValues.file_prefix}/check.json', f'{CommonValues.file_prefix}/report.json', f'{CommonValues.file_prefix}/test.txt', ], }, ), ( { CommonValues.decompressed_bucket: ['test.txt', 'check.json', 'report.json'], }, { CommonValues.decompressed_bucket: [ f'{CommonValues.file_prefix}/check.json', f'{CommonValues.file_prefix}/report.json', f'{CommonValues.file_prefix}/test.txt', ], CommonValues.corrupted_bucket: [], }, ), ], ) def test_process( common_values: CommonValues, s3_client: S3Client, initial_files: Dict[str, List[str]], expected_result: Dict[str, List[str]] ): # yapf: disable buckets_to_clean = [common_values.corrupted_bucket, common_values.decompressed_bucket] config = Config( environment=Environment.DEV, sentry_secret_key='', buckets_to_clean=buckets_to_clean, step_function_arn=common_values.step_function_arn, rds_secret_key='rds/key', ) # Put few files in each bucket, including control bucket. for bucket in buckets_to_clean: for file_name in initial_files.get(bucket, []): s3_client.put_object( Body=b'dummy', Bucket=bucket, Key=f'{common_values.file_prefix}/{file_name}', ) # Assert that files actually uploaded as expected for bucket in buckets_to_clean: initial_files_count = len(initial_files.get(bucket, [])) assert s3_client.list_objects_v2(Bucket=bucket)['KeyCount'] == initial_files_count s3_client.put_object( Body=b'dummy', Bucket=common_values.control_sample_bucket, Key=f'{common_values.file_prefix}/test_file.bat', ) assert s3_client.list_objects_v2(Bucket=common_values.control_sample_bucket)['KeyCount'] == 1 manager = Manager( config=config, logger=mock.MagicMock(spec=SmeBoundLogger), db_service=mock.MagicMock(spec=DBService), job_service=mock.MagicMock(spec=JobService), file_storage=FileStorageService( logger=mock.MagicMock(spec=SmeBoundLogger), client=s3_client, ), # yapf: disable ) result = manager.process(mock.Mock(), common_values.file_prefix) assert result['deleted_files'].keys() == expected_result.keys() for key, item in result['deleted_files'].items(): assert sorted(item) == sorted(expected_result[key]) # 4. Check that files were deleted. for bucket in buckets_to_clean: assert s3_client.list_objects_v2(Bucket=bucket)['KeyCount'] == 0 assert s3_client.list_objects_v2(Bucket=common_values.control_sample_bucket)['KeyCount'] == 1