import io import json from os import path from typing import Any, Dict from unittest.mock import Mock import boto3 import pytest from botocore.stub import Stubber from slz_appreciationengine_prepare_configuration import handler from slz_appreciationengine_prepare_configuration.entities import Config from slz_appreciationengine_prepare_configuration.exceptions import PrepareConfigurationError from . import FIXTURES_PATH logger = Mock() def read_json_fixture(filepath: str) -> Dict[str, Any]: with open(path.join(FIXTURES_PATH, filepath), 'r') as f: return json.load(f) def prepare_response(filepath: str) -> Dict[str, Any]: body = open(path.join(FIXTURES_PATH, filepath), 'rb').read() resp = read_json_fixture('generic_s3_response.json') resp['Body'] = io.BytesIO(body) return resp @pytest.fixture def s3_client(): return boto3.client( 's3', region_name='us-east-1', aws_access_key_id='***', aws_secret_access_key='***' ) @pytest.fixture def stubber(s3_client): stubber = Stubber(s3_client) yield stubber stubber.deactivate() @pytest.fixture def config_mock(): return Config( environment='dev', dsp_settings_path='my-dsp-settings-path', dsp_complete_criterias_path='complete-criterias.json', sqs_arns_meta='', sqs_arns_split='', rds_secrets_key='some/key', sentry_secret_key='some/key', config_key='cfg.json', https_proxy_host='', https_proxy_dsps=[], map_max_executions=5, bucket_troubleshooting_logs='', notification_filter_regex='', optional_config={}, ) def test_success_with_disabled_validation(s3_client, stubber, config_mock): event = { 'dsp': 'appreciationengine', 'report_type': 'memberslogin', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } stubber.add_response( 'get_object', prepare_response(config_mock.config_key), { 'Bucket': event['config_bucket'], 'Key': config_mock.config_key } ) stubber.activate() resp = handler(event, s3_client, logger, config_mock) assert '--url=http://sdm.lbl.gov/fastbit/data/star2000.csv.gz' == resp['Command'][0][0] assert '--enable-historical-validation' not in resp['Command'][0] assert 'arn:aws:ecs:us-east-1:123543223:cluster/slz' == resp['Cluster'] stubber.assert_no_pending_responses() @pytest.mark.parametrize( 'contexts, cmd_len', [ (['AD'], 1), (['AD', 'AE', 'AF', 'GB', 'DE', 'US', 'FR', 'QW', 'QQ', 'EE', 'TT'], 5), ] ) def test_success_command(contexts, cmd_len, stubber, s3_client, config_mock): event = { 'dsp': 'spotify', 'report_type': 'partneremails', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': contexts } stubber.add_response( 'get_object', prepare_response(config_mock.config_key), { 'Bucket': event['config_bucket'], 'Key': config_mock.config_key } ) stubber.activate() resp = handler(event, s3_client, logger, config_mock) assert len(resp['Command']) == cmd_len @pytest.mark.parametrize( 'dsp, proxy_setting_in_cmd', [ ('spotify', False), ('appreciationengine', True), ] ) def test_success_with_https_proxy(dsp, proxy_setting_in_cmd, stubber, s3_client, config_mock): config_mock.https_proxy_host = 'dev-kong.delphiplatform.io' config_mock.https_proxy_dsps = ['appreciationengine', ] event = { 'dsp': dsp, 'report_type': 'memberslogin', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } stubber.add_response( 'get_object', prepare_response(config_mock.config_key), { 'Bucket': event['config_bucket'], 'Key': config_mock.config_key } ) stubber.activate() resp = handler(event, s3_client, logger, config_mock) assert ('--https-proxy-host=dev-kong.delphiplatform.io' in resp['Command'][0]) == \ proxy_setting_in_cmd @pytest.mark.parametrize( 'dsp, report_date, report_type, should_contain_meta_arn', [ ('appreciationengine', '2021-07-09', 'activityfeed', True), ('appreciationengine', '2021-07-08', 'activityfeed', True), ('appreciationengine', '2021-07-08', 'membersextended', True), ('appreciationengine', '2019-07-08', 'membersextended', False), ] ) def test_ae_contain_meta_arn( dsp, report_date, report_type, should_contain_meta_arn, stubber, s3_client, config_mock ): config_mock.sqs_arns_meta = '123' config_mock.notification_filter_regex = 'appreciationengine-(?!2019).+' event = { 'uow_id': f'{dsp}-{report_date}-sme-{report_type}-v1', 'dsp': dsp, 'report_date': report_date, 'report_type': report_type, 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } stubber.add_response( 'get_object', prepare_response(config_mock.config_key), { 'Bucket': event['config_bucket'], 'Key': config_mock.config_key } ) stubber.activate() resp = handler(event, s3_client, logger, config_mock) assert ('--sqs-arns-meta=123' in resp['Command'][0]) == should_contain_meta_arn def test_not_found(stubber, s3_client, config_mock): config_mock.config_key = '123.json' event = { 'dsp': 'appreciationengine', 'report_type': 'memberslogin', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } not_found_resp = read_json_fixture('not_found_response.json') stubber.add_client_error( 'get_object', 'NoSuchKey', 'The specified key does not exist.', 404, not_found_resp, { 'Bucket': event['config_bucket'], 'Key': config_mock.config_key } ) stubber.activate() with pytest.raises(PrepareConfigurationError): handler(event, s3_client, logger, config_mock) stubber.assert_no_pending_responses() def test_generic_error(stubber, s3_client, config_mock): event = { 'dsp': 'appreciationengine', 'report_type': 'memberslogin', 'config_bucket_2': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } stubber.activate() with pytest.raises(PrepareConfigurationError): handler(event, s3_client, logger, config_mock) stubber.assert_no_pending_responses() def test_job_queue_change(stubber, s3_client, config_mock): event = { 'dsp': 'appreciationengine', 'report_type': 'memberslogin', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 8, 'contexts': ['AD', ] } stubber.add_response( 'get_object', prepare_response(config_mock.config_key), { 'Bucket': event['config_bucket'], 'Key': config_mock.config_key } ) stubber.activate() resp = handler(event, s3_client, logger, config_mock) assert 'dev-delphi-spotqueue' == resp['JobQueue']