import asyncio from dataclasses import dataclass from unittest.mock import AsyncMock, Mock import aiohttp import pytest import pytest_asyncio from src.backend.connectors.youtube_apis.client import BaseClient, YtCid, exceptions def new_mock_yt_exception(status_code: int): """Create a new mock YT exception with a status code.""" class MockException(Exception): def __init__(self): super().__init__("mock_msg") self.res = Mock(status_code=status_code) return MockException() class BaseClientTestImplementation(BaseClient): """Implementation of BaseClient for testing purposes.""" _api = "test" _api_name = "test" _api_version = "v1" _scopes = ("test",) _init_lock = asyncio.Lock() _semaphore = asyncio.Semaphore(BaseClient._api_max_concurrent_requests) @pytest_asyncio.fixture async def instance(): return BaseClientTestImplementation() class TestBaseClient: @pytest.fixture def mocker_aiogoogle(self, mocker, instance): def _(**kwargs): return mocker.patch.object(instance, "_aiogoogle", **kwargs) return _ @pytest.fixture def mocker_dispatch(self, mocker): return mocker.patch.object(BaseClient, "_dispatch") @pytest.fixture def mocker_dispatch_paginated(self, mocker): return mocker.patch.object(BaseClient, "_dispatch_paginated") class TestCredentials: def test_credentials_not_set_returns_none(self, instance): assert instance._aiogoogle is None assert instance.credentials is None def test_credentials_private_key_obfuscated(self, instance, mocker_aiogoogle): mocker_aiogoogle(service_account_creds={"private_key": "1234567890" * 10}) private_key = instance.credentials["private_key"] private_key_is_obfuscated = private_key.endswith("***") assert private_key_is_obfuscated class TestGetListObjects: @pytest.mark.asyncio @pytest.mark.parametrize( "limit,expected_count", [ (None, BaseClient._max_items_per_page), (5, 5), ], ) async def test_get_list_objects_max_results_limit( self, instance, mocker_dispatch_paginated, limit, expected_count ): """Ensure that the max_results limit is respected.""" sample_dicts = [ {"mock_key": str(i)} for i in range(BaseClient._max_items_per_page + 100) ] mocker_dispatch_paginated.return_value = sample_dicts @dataclass class TestObject: mock_key: str result = await instance._get_list_objects(TestObject, Mock(), limit=limit) passed_limit = mocker_dispatch_paginated.call_args.kwargs["limit"] assert passed_limit == limit, ( "The limit passed to the dispatch function is not the same as " "the one passed to the get_list_objects function." if limit is not None else "The limit passed to the dispatch function should have been " "the `_max_items_per_page` value." ) assert ( isinstance(item, TestObject) for item in result ), "The result is not a list of the expected objects." class TestDispatch: def new_aenter_mock(self): async_mock = AsyncMock() async_mock.service_account_creds = {"mock_creds": "value"} async_mock.__aenter__.return_value = async_mock return async_mock @pytest.fixture(autouse=True) def no_sleep(self, mocker): """Disable sleep calls in tests so that they're not slow.""" mocker.patch("asyncio.sleep") @pytest.mark.asyncio async def test_dispatch_increase_request_counter( self, mocker_aiogoogle, instance ): """Ensure that the request counter is increased when dispatching a request. """ assert instance.request_count == 0 async_mock = self.new_aenter_mock() async_mock.as_service_account.return_value = async_mock mocker_aiogoogle(new=async_mock) await instance._dispatch(Mock) assert instance.request_count == 1 @pytest.mark.asyncio @pytest.mark.parametrize( "status_code,expected_exception", [ (404, exceptions.NotFound), (400, exceptions.BadRequest), (500, exceptions.InternalServerError), (429, exceptions.APIQuotaExceeded), (422, Exception), # Broad exception to catch all errors. ], ) async def test_dispatch_exception( self, mocker_aiogoogle, instance, status_code, expected_exception ): async_mock = self.new_aenter_mock() async_mock.as_service_account.side_effect = new_mock_yt_exception( status_code ) mocker_aiogoogle(new=async_mock) with pytest.raises(expected_exception): await instance._dispatch(Mock) # noqa assert instance.request_count >= 1 async def assert_retries(self, mocker_aiogoogle, instance, retries): async_mock = self.new_aenter_mock() async_mock.as_service_account.side_effect = [ new_mock_yt_exception(500), {"mock": "response"}, ] mocker_aiogoogle(new=async_mock) await instance._dispatch(Mock, retries=retries) assert instance.request_count_successful == 1 assert instance.request_count == retries + 1 @pytest.mark.asyncio async def test_dispatch_http_500_retry(self, mocker_aiogoogle, instance): """Ensure that the request is retried a certain number of times when an HTTP 500 error occurs. """ retries = 3 async_mock = self.new_aenter_mock() async_mock.as_service_account.side_effect = [ *[new_mock_yt_exception(500) for _ in range(retries)], {"mock": "response"}, ] mocker_aiogoogle(new=async_mock) await instance._dispatch(Mock, retries=retries) expected_call_count = retries + 1 assert instance.request_count_successful == 1 assert instance.request_count == expected_call_count assert async_mock.as_service_account.call_count == expected_call_count, ( "The request was not retried the expected number of times, " "which are equal to the number of retries plus the initial request." ) @pytest.mark.asyncio @pytest.mark.parametrize( "exception_type", [ aiohttp.client_exceptions.ServerDisconnectedError, aiohttp.client_exceptions.ClientOSError, ], ) async def test_dispatch_aiottp_client_exceptions_retry( self, mocker_aiogoogle, instance, exception_type ): """Ensure that the request is retried a certain number of times when an aiohttp client exception occurs (aiohttp is the underlying library used by aiogoogle). """ retries = 3 mock_exception = exception_type() async_mock = self.new_aenter_mock() async_mock.as_service_account.side_effect = [ *([mock_exception] * retries), {"mock": "response"}, ] mocker_aiogoogle(new=async_mock) await instance._dispatch(Mock, retries=retries) expected_call_count = retries + 1 assert instance.request_count_successful == 1 assert instance.request_count == expected_call_count assert async_mock.as_service_account.call_count == expected_call_count, ( "The request was not retried the expected number of times, " "which are equal to the number of retries plus the initial request." ) @pytest.mark.asyncio async def test_dispatch_exception_not_yt(self, mocker_aiogoogle, instance): """Ensure that an exception is raised if the error is not related to YouTube Content ID. """ async_mock = self.new_aenter_mock() async_mock.as_service_account.side_effect = RuntimeError() mocker_aiogoogle(new=async_mock) with pytest.raises(RuntimeError): await instance._dispatch(Mock) # noqa assert instance.request_count == 1 assert instance.request_count_successful == 0 class TestDispatchPaginated: @pytest_asyncio.fixture async def call(self, instance): async def _call(func=None): return await instance._dispatch_paginated( func or Mock(), "args", {"kwargs": "kwargs"} ) return _call @pytest.mark.asyncio async def test_dispatch_paginated(self, mocker_dispatch, call): _mocker_dispatch_side_effects = ( side_effect for side_effect in [ {YtCid.ITEMS: [1, 2, 3], YtCid.NEXT_PAGE_TOKEN: "next1"}, {YtCid.ITEMS: [4, 5, 6], YtCid.NEXT_PAGE_TOKEN: "next2"}, {YtCid.ITEMS: [7, 8, 9], YtCid.NEXT_PAGE_TOKEN: None}, ] ) def _mocker_dispatch(*args): func = args[0] func() return next(_mocker_dispatch_side_effects) mocker_dispatch.side_effect = _mocker_dispatch mock_func = Mock() result = await call(mock_func) assert result == [1, 2, 3, 4, 5, 6, 7, 8, 9] assert mocker_dispatch.call_count == 3 lambda_page_tokens = [ item.kwargs[YtCid.PAGE_TOKEN] for item in mock_func.call_args_list ] assert lambda_page_tokens == [ None, "next1", "next2", ], "pageToken not passed correctly to the lambda function" @pytest.mark.asyncio async def test_dispatch_paginated_no_items(self, mocker_dispatch, call): mocker_dispatch.side_effect = [ {YtCid.ITEMS: []}, ] result = await call() assert result == [] assert mocker_dispatch.call_count == 1, "Dispatch not called once" @pytest.mark.asyncio async def test_dispatch_paginated_limit(self, mocker_dispatch, instance): mocker_dispatch.side_effect = [ {YtCid.ITEMS: [1, 2, 3], YtCid.NEXT_PAGE_TOKEN: "next1"}, {YtCid.ITEMS: [4, 5, 6], YtCid.NEXT_PAGE_TOKEN: "next2"}, {YtCid.ITEMS: [7, 8, 9], YtCid.NEXT_PAGE_TOKEN: None}, ] result = await instance._dispatch_paginated(Mock(), limit=5) assert result == [1, 2, 3, 4, 5], "Limit not respected" assert mocker_dispatch.call_count == 2, "Limit not respected"