import types from unittest.mock import MagicMock, patch, call, ANY import freezegun import pytest from feed_ingestion.flows.spotify import smart_downloader @pytest.fixture def download_task1(): """Download_task1.""" yield smart_downloader.DownloadTask( source_url='https://theorchard.com', destination_url='s3://dev-cucumbers/1.html', ) @pytest.fixture def download_task2(): yield smart_downloader.DownloadTask( source_url='https://theorchard.com', destination_url='s3://dev-cucumbers/2.html', ) @pytest.fixture def download_request(download_task1, download_task2): """Download_request.""" yield smart_downloader.DownloadRequest( tasksMessageId='123', tasks=[ download_task1, download_task2, ] ) class TestDownloadRequest: """Test DownloadRequest class.""" def test_to_dict(self, download_request): """Test to_dict.""" result = download_request.to_dict() assert result == { 'tasks': [ { 'destination_url': 's3://dev-cucumbers/1.html', 'source_url': 'https://theorchard.com' }, { 'destination_url': 's3://dev-cucumbers/2.html', 'source_url': 'https://theorchard.com' } ], 'tasksMessageId': '123' } def test_to_json(self, download_request): """Test to_json.""" result = download_request.to_json() assert result == ( '{"tasks": [{"source_url": "https://theorchard.com", ' '"destination_url": ' '"s3://dev-cucumbers/1.html"}, {"source_url": ' '"https://theorchard.com", ' '"destination_url": "s3://dev-cucumbers/2.html"}], ' '"tasksMessageId": "123"}') def test_to_json_indent(self, download_request): """Test to_json_indent.""" result = download_request.to_json(indent=2) expected_result = ( '{\n' ' "tasks": [\n' ' {\n' ' "source_url": "https://theorchard.com",\n' ' "destination_url": "s3://dev-cucumbers/1.html"\n' ' },\n' ' {\n' ' "source_url": "https://theorchard.com",\n' ' "destination_url": "s3://dev-cucumbers/2.html"\n' ' }\n' ' ],\n' ' "tasksMessageId": "123"\n' '}') assert result == expected_result @patch.object(smart_downloader, 'query_dynamodb_paginated_generator') def test_query_for_job_id(query_dynamodb_paginated_generator_mock): """Test query_for_job_id.""" dynamodb_table_mock = MagicMock() query_dynamodb_paginated_generator_mock.return_value = range(3) result = smart_downloader.query_for_job_id( job_id='job-id', dynamodb_table=dynamodb_table_mock ) assert result == [0, 1, 2] assert query_dynamodb_paginated_generator_mock.call_args_list == [ call( dynamodb_table=dynamodb_table_mock, query_args={ 'IndexName': 'jobId-index', 'KeyConditionExpression': ANY, 'ReturnConsumedCapacity': 'TOTAL', 'ProjectionExpression': '#v1', 'ExpressionAttributeNames': {'#v1': 'status'} } ) ] @freezegun.freeze_time('2023-11-11') @patch.object(smart_downloader, 'convert_download_request_to_dynamodb_item') def test_send_download_requests_dynamodb( convert_download_request_to_dynamodb_item_mock, download_request, ): """Test send_download_requests_dynamodb.""" aws_session_mock = MagicMock() table_mock = aws_session_mock.resource.return_value.Table.return_value batch_writer_mock = (table_mock.batch_writer. return_value.__enter__.return_value) result = smart_downloader.send_download_requests( download_requests=(download_request for _ in range(3)), job_id='job-id', table_name='table-name', ttl_timeout_seconds=234, aws_session=aws_session_mock ) assert result == 3 assert batch_writer_mock.put_item.call_args_list == [ call(Item=convert_download_request_to_dynamodb_item_mock.return_value), call(Item=convert_download_request_to_dynamodb_item_mock.return_value), call(Item=convert_download_request_to_dynamodb_item_mock.return_value), ] assert convert_download_request_to_dynamodb_item_mock.call_args_list == [ call(download_request=download_request, job_id='job-id', task_ttl_seconds=1699661034), call(download_request=download_request, job_id='job-id', task_ttl_seconds=1699661034), call(download_request=download_request, job_id='job-id', task_ttl_seconds=1699661034), ] def test_convert_download_request_to_dynamodb_item(download_request): """Test convert_download_request_to_dynamodb_item.""" result = smart_downloader.convert_download_request_to_dynamodb_item( download_request=download_request, job_id='jo-id', task_ttl_seconds=23, ) assert result == { 'jobId': 'jo-id', 'partition': 'jo-id-123', 'request': '{\n' ' "tasks": [\n' ' {\n' ' "source_url": "https://theorchard.com",\n' ' "destination_url": "s3://dev-cucumbers/1.html"\n' ' },\n' ' {\n' ' "source_url": "https://theorchard.com",\n' ' "destination_url": "s3://dev-cucumbers/2.html"\n' ' }\n' ' ],\n' ' "tasksMessageId": "123"\n' '}', 'status': 'NEW', 'taskId': '123', 'ttl': 23} @patch.object(smart_downloader, 'query_for_job_id') @patch.object(smart_downloader.time, 'sleep') def test_await_downloads_completion( sleep_mock, query_for_job_id_mock, ): """Test await_downloads_completion.""" total_sleep_amount = 0 def count_sleep_amount_side_effect(value): nonlocal total_sleep_amount total_sleep_amount += value aws_session_mock = MagicMock() sleep_mock.side_effect = count_sleep_amount_side_effect table_mock = aws_session_mock.resource.return_value.Table.return_value query_for_job_id_mock.side_effect = [ [ *({'status': 'DONE', 'other': 1} for _ in range(3)), *({'status': 'PROCESSING', 'other': 1} for _ in range(3)), ], [ *({'status': 'DONE'} for _ in range(4)), *({'status': 'PROCESSING'} for _ in range(2)), ], [ *({'status': 'DONE', 'other': 1} for _ in range(6)), ], ] result = smart_downloader.await_downloads_completion( job_id='job-id', expected_number_of_items=6, table_name='table-name', aws_session=aws_session_mock, cycle_wait_seconds=11, timeout_seconds=35, ) assert result == {'DONE': 6} assert query_for_job_id_mock.call_args_list == [ call(job_id='job-id', dynamodb_table=table_mock), call(job_id='job-id', dynamodb_table=table_mock), call(job_id='job-id', dynamodb_table=table_mock), ] assert sleep_mock.call_args_list == [ call(11), call(11), ] assert total_sleep_amount == 22 @patch.object(smart_downloader, 'query_for_job_id') @patch.object(smart_downloader, 'time') def test_await_downloads_completion_timed_out( time_mock, query_for_job_id_mock, ): """Test await_downloads_completion.""" aws_session_mock = MagicMock() table_mock = aws_session_mock.resource.return_value.Table.return_value query_for_job_id_mock.side_effect = [ [ *({'status': 'DONE', 'other': 1} for _ in range(3)), *({'status': 'PROCESSING', 'other': 1} for _ in range(3)), ], [ *({'status': 'DONE'} for _ in range(4)), *({'status': 'PROCESSING'} for _ in range(2)), ], [ *({'status': 'DONE'} for _ in range(4)), *({'status': 'PROCESSING'} for _ in range(2)), ], [ *({'status': 'DONE'} for _ in range(5)), *({'status': 'PROCESSING'} for _ in range(1)), ], ] current_time = 1702896587.0 total_sleep_seconds = 0 def count_sleep_amount_side_effect(seconds): nonlocal total_sleep_seconds total_sleep_seconds += seconds time_mock.sleep.side_effect = count_sleep_amount_side_effect time_mock.time.side_effect = lambda: current_time + total_sleep_seconds with pytest.raises(TimeoutError) as excinfo: smart_downloader.await_downloads_completion( job_id='job-id', expected_number_of_items=6, table_name='table-name', aws_session=aws_session_mock, cycle_wait_seconds=11, timeout_seconds=25 ) assert ( 'Waited for 33 seconds. ' "Current states: {'DONE': 5, 'PROCESSING': 1} " 'of total 6 expected responses') == str(excinfo.value) assert query_for_job_id_mock.call_args_list == [ call(job_id='job-id', dynamodb_table=table_mock), call(job_id='job-id', dynamodb_table=table_mock), call(job_id='job-id', dynamodb_table=table_mock), call(job_id='job-id', dynamodb_table=table_mock), ] assert time_mock.sleep.call_args_list == [ call(11), call(11), call(11), ] assert total_sleep_seconds == 33 @patch.object(smart_downloader, 'uuid') def test_download_request_batched_generator( uuid_mock, download_task1, download_task2): """Test download_request_batched_generator.""" uuid_mock.uuid4.side_effect = [ 'uuid-for-task-1', 'uuid-for-task-2', ] result = smart_downloader.download_request_batched_generator( tasks=[download_task1, download_task2, download_task2, download_task2], batch_size=3, ) assert isinstance(result, types.GeneratorType) result = list(result) assert result == [ smart_downloader.DownloadRequest( tasks=[download_task1, download_task2, download_task2], tasksMessageId='uuid-for-task-1' ), smart_downloader.DownloadRequest( tasks=[download_task2], tasksMessageId='uuid-for-task-2' ) ]