import os from datetime import datetime, timezone from unittest.mock import Mock import pytest import smart_open from slz_config.entities import ( CompleteCriteriaConfig, DSPConfig, DSPConfigUoWArgs, DSPImplementation, DSPSettings, ) from slz_config.pipeline_config.entities import PipelineConfig, S3Location from slz_config.pipeline_config.exceptions import CorruptedConfigsError from slz_config.pipeline_config.service import PipelineConfigService from . import BUCKET_TEST, FIXTURES_PATH def test_merged_config_ok(s3_client): for config in [ 'dsp-complete-criteria.json', 'dsp-specific-settings.json', 'dsp_config.json', 'slz-flow.json', ]: with smart_open.open(os.path.join(FIXTURES_PATH, 'pipeline_config', config)) as _file: s3_client.put_object( Body=_file.read(), Bucket=BUCKET_TEST, Key=config, ) config_service = PipelineConfigService( logger=Mock(), dsp_config_path=S3Location(bucket=BUCKET_TEST, key='dsp_config.json'), dsp_complete_criteria_config_path=S3Location( bucket=BUCKET_TEST, key='dsp-complete-criteria.json' ), dsp_specific_settings_path=S3Location(bucket=BUCKET_TEST, key='dsp-specific-settings.json'), slz_flow_config_path=S3Location(bucket=BUCKET_TEST, key='slz-flow.json') ) expected = { 'amazonadsupported-{yyyymmdd}-sme-activity-v1': PipelineConfig( dsp_config=DSPConfig( unit_of_work='amazonadsupported-{yyyymmdd}-sme-activity-v1', uow_active_hours_threshold=72, schedule='*/10 * * * *', time_zone='UTC', args=DSPConfigUoWArgs( dsp='amazonadsupported', extension='txt', type='activity', version='v1', licensor='sme', valid_from=datetime(2019, 4, 1, 0, 0, tzinfo=timezone.utc), subtype=None, valid_until=None ), frequency=None, uow_active_days_limit=None, validation_thresholds=None, uow_complete_after_hours=None ), dsp_complete_criteria=[ CompleteCriteriaConfig(context='US', required=True, job_id=None, query=None), CompleteCriteriaConfig(context='AT', required=False, job_id=None, query=None) ], dsp_specific_settings=DSPSettings( uow_active_days_limit=180, default_priority_hrs=24, implementations=[ DSPImplementation( rule='theorchard/*', label='amazonadsupported-theorchard', flow='standard' ), DSPImplementation(rule='*/*', label='amazonadsupported', flow='standard') ], validation_thresholds={ 'low': 30, 'high': 90 }, days_offset=1 ), sf_arn='slzMainFlow', slz_main_batch_config_location=None ), 'appreciationengine-{yyyymmdd}-sme-request_to_forget-v1': PipelineConfig( dsp_config=DSPConfig( unit_of_work='appreciationengine-{yyyymmdd}-sme-request_to_forget-v1', uow_active_hours_threshold=24, schedule='*/30 6-23 * * *', time_zone='UTC', args=DSPConfigUoWArgs( dsp='appreciationengine', extension='json', type='request_to_forget', version='v1', licensor='sme', valid_from=datetime(2021, 4, 1, 0, 0, tzinfo=timezone.utc), subtype=None, valid_until=None ), frequency='0 0 * * 5', uow_active_days_limit=None, validation_thresholds=None, uow_complete_after_hours=24 ), dsp_complete_criteria=[], dsp_specific_settings=DSPSettings( uow_active_days_limit=2, default_priority_hrs=36, concurrency_limit=5, implementations=[ DSPImplementation( rule='sme/request_to_forget', label='appreciationengine_email', flow='gdpr_appreciationengine_email' ), DSPImplementation( rule='*/*', label='appreciationengine-paginate-partial', flow='standard' ) ], validation_thresholds={ 'low': 0, 'high': 1000 }, days_offset=1 ), sf_arn='slzGdprAeEmailFlow', slz_main_batch_config_location=None ), 'gras-{yyyymmdd}-sme-dim_country-v1': PipelineConfig( dsp_config=DSPConfig( unit_of_work='gras-{yyyymmdd}-sme-dim_country-v1', uow_active_hours_threshold=36, schedule='*/10 * * * *', time_zone='UTC', args=DSPConfigUoWArgs( dsp='gras', extension='csv', type='dim_country', version='v1', licensor='sme', valid_from=datetime(2019, 1, 12, 0, 0, tzinfo=timezone.utc), subtype=None, valid_until=None ), frequency=None, uow_active_days_limit=None, validation_thresholds=None, uow_complete_after_hours=None ), dsp_complete_criteria=[ CompleteCriteriaConfig( context='dim_country', required=True, job_id=None, query='gras-queries/dim_country.sql' ) ], dsp_specific_settings=DSPSettings( uow_active_days_limit=2, default_priority_hrs=24, implementations=[DSPImplementation(rule='*/*', label='gras', flow='standard')], validation_thresholds={ 'low': 10, 'high': 20 }, days_offset=1 ), sf_arn='slzMainFlow', slz_main_batch_config_location=None ), 'youtubereporting-{yyyymmdd}-sme-active_claims-a1': PipelineConfig( dsp_config=DSPConfig( unit_of_work='youtubereporting-{yyyymmdd}-sme-active_claims-a1', uow_active_hours_threshold=36, schedule='*/10 * * * *', time_zone='US/Eastern', args=DSPConfigUoWArgs( dsp='youtubereporting', extension='csv', type='active_claims', version='a1', licensor='sme', valid_from=datetime(2020, 1, 13, 0, 0, tzinfo=timezone.utc), subtype=None, valid_until=None ), frequency=None, uow_active_days_limit=None, validation_thresholds={ 'low': 50, 'high': 70 }, uow_complete_after_hours=None ), dsp_complete_criteria=[ CompleteCriteriaConfig( context='content_owner_active_claims_a1', required=True, job_id='d956202e-6f16-46f1', query=None ) ], dsp_specific_settings=DSPSettings( uow_active_days_limit=30, default_priority_hrs=24, implementations=[ DSPImplementation( rule='*/active_claims', label='youtubereporting-activeclaims', flow='standard' ), DSPImplementation(rule='*/*', label='youtubereporting', flow='standard') ], validation_thresholds={ 'low': 50, 'high': 70 }, days_offset=2 ), sf_arn='slzMainFlow', slz_main_batch_config_location=None ) } assert config_service.merged_config == expected def test_merged_config_raise_err(s3_client): for config in ['dsp-complete-criteria.json', 'dsp_config_error.json']: with smart_open.open(os.path.join(FIXTURES_PATH, 'pipeline_config', config)) as _file: s3_client.put_object( Body=_file.read(), Bucket=BUCKET_TEST, Key=config, ) with pytest.raises(CorruptedConfigsError): PipelineConfigService( logger=Mock(), dsp_config_path=S3Location(bucket=BUCKET_TEST, key='dsp_config_error.json'), dsp_complete_criteria_config_path=S3Location( bucket=BUCKET_TEST, key='dsp-complete-criteria.json' ) ) @pytest.mark.parametrize( 'uow_id, found_config', [ ('amazonadsupported-20210202-sme-activity-v1', True), ('amazonadsupported-20210202-sme-activity-v1-pr2021020202020', True), ('appreciationengine-20190909-sme-request_to_forget-v1', True), ('appreciationengine-20190909-sme-request_to_forget-v2', False), ] ) def test_merged_config_get_uow_config(uow_id, found_config, s3_client): for config in [ 'dsp-complete-criteria.json', 'dsp-specific-settings.json', 'dsp_config.json', 'slz-flow.json', ]: with smart_open.open(os.path.join(FIXTURES_PATH, 'pipeline_config', config)) as _file: s3_client.put_object( Body=_file.read(), Bucket=BUCKET_TEST, Key=config, ) config_service = PipelineConfigService( logger=Mock(), dsp_config_path=S3Location(bucket=BUCKET_TEST, key='dsp_config.json'), dsp_complete_criteria_config_path=S3Location( bucket=BUCKET_TEST, key='dsp-complete-criteria.json' ), dsp_specific_settings_path=S3Location(bucket=BUCKET_TEST, key='dsp-specific-settings.json'), slz_flow_config_path=S3Location(bucket=BUCKET_TEST, key='slz-flow.json') ) expected = config_service.get(uow_id) assert bool(expected) == found_config