import json from unittest import mock import boto3 as boto3 import pytest from boto3_type_annotations.sqs import Client as SQSClient from moto import mock_sqs from slz_notification.entities import Message, Metadata from slz_notification.sqs_service import SQSNotificationService from slz_appreciationengine_scrapper.dsp.entities import DecompressedFile, ReportMeta, S3Path from slz_appreciationengine_scrapper.entities import Job from slz_appreciationengine_scrapper.sqs_service import SQSService TEST_QUEUES = { 'arn-1': 'https://queue.amazonaws.com/123456789012/arn-1', 'arn-2': 'https://queue.amazonaws.com/123456789012/arn-2' } @pytest.fixture def sqs_client(aws_credentials) -> SQSClient: with mock_sqs(): sqs_client_: SQSClient = boto3.client('sqs') for queue in TEST_QUEUES.keys(): sqs_client_.create_queue(QueueName=queue) yield sqs_client_ for queue in TEST_QUEUES.keys(): sqs_client_.delete_queue(QueueUrl=queue) @pytest.fixture def report_meta_mock(): return ReportMeta( actual_size=100, expected_size=100, destination_path=S3Path( bucket='sme-archive', name='report.txt.gz', path='sme/v1/', ), destination_path_quarantine=S3Path( bucket='sme-archive-quarantine', name='report.txt.gz', path='sme/v1/', ), destination_decompressed_path_quarantine=None, destination_corrupted_path=None, files=[ DecompressedFile( name='report.txt', path=S3Path(bucket='sme-decompressed', name='report.txt', path='sme/v1/'), path_quarantine=S3Path( bucket='sme-decompressed-quarantine', name='report.txt', path='sme/v1/', ) ) ] ) @pytest.fixture def job_sqs_mock(): return Job( uow_id='uow-id', unit_of_work_id=0, context='US', dsp='apple', report_type='users', subtype='', version='v1', report_date='2020-12-02', licensor='smejp', extension='tsv', config_bucket='sme/bucket', context_params={}, job_id='', ) @pytest.fixture def metadata_sqs_mock(report_meta_mock): return Metadata( version='1.0.0', queues=[ TEST_QUEUES['arn-1'], ], message=Message( uow_id='uow-id', unit_of_work_id=0, compressed_path=report_meta_mock.destination_path.url, decompressed_paths=','.join([file_.path.url for file_ in report_meta_mock.files]), content_name='content-name', context='US', optional_config={}, ) ) @pytest.mark.parametrize( 'dest_path', [ S3Path(bucket='sme-archive', name='report.txt.gz', path='sme/v1/'), None # some AE reports ] ) def test_prepare_metadata(dest_path, sqs_client, job_sqs_mock, report_meta_mock, metadata_sqs_mock): logger = mock.Mock() # mock dest path report_meta_mock.destination_path = dest_path sqs_notification_service = SQSNotificationService(logger, sqs_client) client = SQSService( logger, sqs_notification_service, queues=TEST_QUEUES['arn-1'], optional_config={} ) result = client.prepare_metadata( job=job_sqs_mock, content_name='content-name', meta=report_meta_mock ) expected = Metadata( version='2.0.0', queues=[ TEST_QUEUES['arn-1'], ], message=Message( uow_id='uow-id', unit_of_work_id=0, compressed_path=report_meta_mock.destination_path.url if dest_path else None, decompressed_paths=','.join([file_.path.url for file_ in report_meta_mock.files]), content_name='content-name', context='US', optional_config={}, ) ) assert result == expected @pytest.mark.parametrize( 'dest_path, dest_path_expected', [ ( S3Path(bucket='sme-archive', name='report.txt.gz', path='sme/v1/'), 's3://sme-archive/sme/v1/report.txt.gz' ), (None, None) # some AE reports ] ) def test_push( dest_path, dest_path_expected, sqs_client, report_meta_mock, job_sqs_mock, ): logger = mock.Mock() sqs_notification_service = SQSNotificationService(logger, sqs_client) client = SQSService( logger, sqs_notification_service, queues=TEST_QUEUES['arn-1'], optional_config={} ) report_meta_mock.destination_path = dest_path metadata = client.prepare_metadata( job=job_sqs_mock, content_name='content-name', meta=report_meta_mock ) result = client.push(metadata) assert result is True messages = sqs_client.receive_message(QueueUrl=TEST_QUEUES['arn-1'])['Messages'] assert len(messages) == 1 assert messages[0]['Body'] == json.dumps( { 'UoWID': 'uow-id', 'unit_of_work_id': 0, 'Context': 'US', 'ContentName': 'content-name', 'CompressedPath': dest_path_expected, 'DecompressedPaths': 's3://sme-decompressed/sme/v1/report.txt', 'OptionalConfig': {}, } ) def test_push_multiple_queues(sqs_client, job_sqs_mock, report_meta_mock, metadata_sqs_mock): logger = mock.Mock() sqs_notification_service = SQSNotificationService(logger, sqs_client) queues = ','.join(TEST_QUEUES.values()) client = SQSService(logger, sqs_notification_service, queues=queues, optional_config={}) metadata = client.prepare_metadata( job=job_sqs_mock, content_name='content-name', meta=report_meta_mock ) result = client.push(metadata) assert result is True assert len(sqs_client.receive_message(QueueUrl=TEST_QUEUES['arn-1'])['Messages']) == 1 assert len(sqs_client.receive_message(QueueUrl=TEST_QUEUES['arn-2'])['Messages']) == 1 def test_push_no_queues(sqs_client, metadata_sqs_mock): logger = mock.Mock() sqs_notification_service = SQSNotificationService(logger, sqs_client) client = SQSService(logger, sqs_notification_service, queues=None, optional_config={}) metadata_sqs_mock.queues = [] result = client.push(metadata_sqs_mock) assert result is False assert 'Messages' not in sqs_client.receive_message(QueueUrl=TEST_QUEUES['arn-1']) assert 'Messages' not in sqs_client.receive_message(QueueUrl=TEST_QUEUES['arn-2'])