import io import json from os import path from typing import Any, Dict from unittest.mock import Mock import boto3 import pytest from botocore.stub import Stubber from slz_prepare_configuration import handler from slz_prepare_configuration.exceptions import PrepareConfigurationError from . import FIXTURES_PATH logger = Mock() def read_json_fixture(filepath: str) -> Dict[str, Any]: with open(path.join(FIXTURES_PATH, filepath), 'r') as f: return json.load(f) def prepare_response(filepath: str) -> Dict[str, Any]: body = open(path.join(FIXTURES_PATH, filepath), 'rb').read() resp = read_json_fixture('generic_s3_response.json') resp['Body'] = io.BytesIO(body) return resp @pytest.fixture def s3_client(): return boto3.client( 's3', region_name='us-east-1', aws_access_key_id='***', aws_secret_access_key='***' ) @pytest.fixture def stubber(s3_client): stubber = Stubber(s3_client) yield stubber stubber.deactivate() def test_success_with_disabled_validation(s3_client, stubber): event = { 'dsp': 'apple_music', 'report_type': 'Summary', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } config = { 'dsp_settings_path': 'my-dsp-settings-path', 'dsp_complete_criterias_path': 'complete-criterias.json', 'sqs_arns_meta': '', 'sqs_arns_split': '', 'config_key': 'cfg.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': "0", } stubber.add_response( 'get_object', prepare_response(config['config_key']), { 'Bucket': event['config_bucket'], 'Key': config['config_key'] } ) stubber.activate() resp = handler(event, s3_client, logger, config) assert '--url=http://sdm.lbl.gov/fastbit/data/star2000.csv.gz' == resp['Command'][0][0] assert '--enable-historical-validation' not in resp['Command'][0] assert 'arn:aws:ecs:us-east-1:123543223:cluster/slz' == resp['Cluster'] stubber.assert_no_pending_responses() @pytest.mark.parametrize( 'contexts, cmd_len', [ (['AD'], 1), (['AD', 'AE', 'AF', 'GB', 'DE', 'US', 'FR', 'QW', 'QQ', 'EE', 'TT'], 5), ] ) def test_success_command(contexts, cmd_len, stubber, s3_client): event = { 'dsp': 'spotify', 'report_type': 'partneremails', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': contexts } config = { 'dsp_settings_path': 'my-dsp-settings-path', 'dsp_complete_criterias_path': 'complete-criterias.json', 'sqs_arns_meta': '', 'sqs_arns_split': '', 'config_key': 'cfg.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': '1', 'validation_threshold_file_size': '1024', } stubber.add_response( 'get_object', prepare_response(config['config_key']), { 'Bucket': event['config_bucket'], 'Key': config['config_key'] } ) stubber.activate() resp = handler(event, s3_client, logger, config) assert len(resp['Command']) == cmd_len def test_success_with_enabled_validation(stubber, s3_client): event = { 'dsp': 'apple_music', 'report_type': 'Summary', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } config = { 'dsp_settings_path': 'my-dsp-settings-path', 'dsp_complete_criterias_path': 'complete-criterias.json', 'sqs_arns_meta': '', 'sqs_arns_split': '', 'config_key': 'cfg.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': "1", 'validation_threshold_file_size': "1024", } stubber.add_response( 'get_object', prepare_response(config['config_key']), { 'Bucket': event['config_bucket'], 'Key': config['config_key'] } ) stubber.activate() resp = handler(event, s3_client, logger, config) assert '--enable-historical-validation' in resp['Command'][0] @pytest.mark.parametrize( 'dsp, proxy_setting_in_cmd', [ ('spotify', False), ('appreciationengine', True), ] ) def test_success_with_https_proxy(dsp, proxy_setting_in_cmd, stubber, s3_client): event = { 'dsp': dsp, 'report_type': 'Summary', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } config = { 'dsp_settings_path': 'my-dsp-settings-path', 'dsp_complete_criterias_path': 'complete-criterias.json', 'sqs_arns_meta': '', 'sqs_arns_split': '', 'config_key': 'cfg.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': "0", 'enhanced_storage_job_queue_arn': 'enhanced-storage-arn', 'https_proxy_host': 'dev-kong.delphiplatform.io', 'https_proxy_dsps': ['appreciationengine', ] } stubber.add_response( 'get_object', prepare_response(config['config_key']), { 'Bucket': event['config_bucket'], 'Key': config['config_key'] } ) stubber.activate() resp = handler(event, s3_client, logger, config) assert ('--https-proxy-host=dev-kong.delphiplatform.io' in resp['Command'][0]) == \ proxy_setting_in_cmd @pytest.mark.parametrize( 'dsp, report_date, report_type, should_contain_meta_arn', [ ('appreciationengine', '2021-07-09', 'activityfeed', True), ('appreciationengine', '2021-07-08', 'activityfeed', False), ('appreciationengine', '2021-07-08', 'membersextended', True), ] ) def test_backfill_doesnot_contain_meta_arn( dsp, report_date, report_type, should_contain_meta_arn, stubber, s3_client ): event = { 'dsp': dsp, 'report_date': report_date, 'report_type': report_type, 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } config = { 'dsp_settings_path': 'my-dsp-settings-path', 'dsp_complete_criterias_path': 'complete-criterias.json', 'sqs_arns_meta': '123', 'sqs_arns_split': '', 'config_key': 'cfg.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': "0", 'enhanced_storage_job_queue_arn': 'enhanced-storage-arn', 'https_proxy_host': 'dev-kong.delphiplatform.io', 'https_proxy_dsps': ['appreciationengine', ] } stubber.add_response( 'get_object', prepare_response(config['config_key']), { 'Bucket': event['config_bucket'], 'Key': config['config_key'] } ) stubber.activate() resp = handler(event, s3_client, logger, config) assert ('--sqs-arns-meta=123' in resp['Command'][0]) == should_contain_meta_arn def test_not_found(stubber, s3_client): config = { 'dsp_settings_path': 'my-dsp-settings-path', 'sqs_arns_meta': '', 'sqs_arns_split': '', 'config_key': 'cfg2.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': '0', } event = { 'dsp': 'apple_music', 'report_type': 'Summary', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } not_found_resp = read_json_fixture('not_found_response.json') stubber.add_client_error( 'get_object', 'NoSuchKey', 'The specified key does not exist.', 404, not_found_resp, { 'Bucket': event['config_bucket'], 'Key': config['config_key'] } ) stubber.activate() with pytest.raises(PrepareConfigurationError): handler(event, s3_client, logger, config) stubber.assert_no_pending_responses() def test_generic_error(stubber, s3_client): config = { 'dsp_settings_path': 'my-dsp-settings-path', 'dsp_complete_criterias_path': 'complete-criterias.json', 'sqs_arns_meta': 'ARN', 'sqs_arns_split': 'ARN', 'config_key': 'cfg.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': '0', } event = { 'dsp': 'apple_music', 'report_type': 'Summary', 'config_bucket_2': 'dev-delphi-slz-run-config', 'priority': 5, 'contexts': ['AD', ] } stubber.activate() with pytest.raises(PrepareConfigurationError): handler(event, s3_client, logger, config) stubber.assert_no_pending_responses() @pytest.mark.parametrize( 'event, expected', [ ( { 'dsp': 'apple_music', 'report_type': 'Summary', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 8, 'contexts': ['AD', ] }, 'dev-delphi-spotqueue', ), ( { 'dsp': 'amazonadsupported', 'report_type': 'activity', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 8, 'contexts': [ 'US', 'GB', ], }, 'dev-enhanced_storage_job_queue_arn', ), ( { 'dsp': 'amazonmusicunlimited', 'report_type': 'user', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 8, 'contexts': [ 'US', 'GB', ], }, 'dev-enhanced_storage_job_queue_arn', ), ( { 'dsp': 'amazonprime', 'report_type': 'playlist', 'config_bucket': 'dev-delphi-slz-run-config', 'priority': 8, 'contexts': [ 'US', 'GB', ], }, 'dev-enhanced_storage_job_queue_arn', ), ] ) def test_job_queue_change(stubber, s3_client, event, expected): config = { 'dsp_settings_path': 'my-dsp-settings-path', 'dsp_complete_criterias_path': 'complete-criterias.json', 'sqs_arns_meta': '', 'sqs_arns_split': '', 'config_key': 'cfg.json', 'rds_secrets_key': 'some/key', 'enable_historical_validation': '0', 'enhanced_storage_job_queue_arn': 'dev-enhanced_storage_job_queue_arn', } stubber.add_response( 'get_object', prepare_response(config['config_key']), { 'Bucket': event['config_bucket'], 'Key': config['config_key'] } ) stubber.activate() resp = handler(event, s3_client, logger, config) assert expected == resp['JobQueue']