"""Tests for search model layer: Elasticsearch queries.""" import json from typing import Any, Callable from unittest.mock import MagicMock, Mock, patch import pytest from botocore import exceptions as botocore_exceptions from flask.ctx import AppContext from opensearchpy.helpers import query from syrupy.assertion import SnapshotAssertion from werkzeug.exceptions import BadRequest, NotFound from vectororder import config from vectororder.connectors import elasticsearch from vectororder.constants import ( error as error_const, search as search_const, ) from vectororder.models import search as search_models @patch("vectororder.models.search.g", spec=["ows"]) def test_search_vector_order_details_empty_args( mock_g: MagicMock, mocker: Mock, app_context: AppContext ) -> None: """Test search_vector_order_details with empty args.""" execute_search_query_mock = mocker.patch( "vectororder.models.search._execute_search_query" ) build_query_expr_mock = mocker.patch( "vectororder.models.search.build_query_expression" ) build_query_expr_mock.return_value = None with pytest.raises(BadRequest) as exc: search_models.search_vector_order_details() # Check result. assert str(exc.value) == ( "400 Bad Request: " + error_const.ERROR_MESSAGE_SEARCH_REQUEST_EMPTY ) # Check calls. assert build_query_expr_mock.call_count == 1 assert build_query_expr_mock.call_args[1] == { "single": None, "multiple": None, "fulltext": None, "date_range": None, "inner_disjunction": None, } assert execute_search_query_mock.call_count == 0 assert mock_g.ows.log.debug.call_count == 1 assert mock_g.ows.log.info.call_count == 1 @pytest.mark.parametrize( "es_result, total_hits", [ ([{"foo": 1}, {"bar": 2}], 2), ([{"a": 10}, {"b": 20}, {"c": 30}], 3), ([], 0), ], ) @patch("vectororder.models.search.g", spec=["ows"]) def test_search_vector_order_details( mock_g: MagicMock, mocker: Mock, app_context: AppContext, es_result: list[dict[str, Any]], total_hits: int, ) -> None: """Test search_vector_order_details query construction and result.""" single = {"meta_update": True} multiple = {"user_id": [1, 2], "order_id": [3, 4]} fulltext = {"error_log": "Something"} date_range = {"encoding_started": ("lower_limit", "upper_limit")} inner_disjunction = {"_statuses": [{"one": 1}, {"two": 2}]} limit = 10 offset = 5 order_by = ["foo"] query_expr = query.Q({"term": {"f1": "v1"}}) hit_mock = mocker.Mock() hit_mock.to_dict = mocker.Mock(side_effect=es_result) query_result_mock = mocker.Mock() query_result_mock.hits.__iter__ = mocker.Mock() query_result_mock.hits.__iter__.return_value = iter([hit_mock] * len(es_result)) query_result_mock.hits.total = total_hits execute_search_query_mock = mocker.patch( "vectororder.models.search._execute_search_query" ) execute_search_query_mock.return_value = query_result_mock build_query_expr_mock = mocker.patch( "vectororder.models.search.build_query_expression" ) build_query_expr_mock.return_value = query_expr result, hits = search_models.search_vector_order_details( single=single, multiple=multiple, fulltext=fulltext, # Passing any dict with the number of items matching # date_range_expressions just to get into the for loop: date_range=date_range, inner_disjunction=inner_disjunction, offset=offset, limit=limit, order_by=order_by, ) # Check logging calls. assert mock_g.ows.log.debug.call_count == 1 assert mock_g.ows.log.info.call_count == 0 # Check build_query_expression() call. assert build_query_expr_mock.call_count == 1 assert build_query_expr_mock.call_args[1] == { "multiple": multiple, "single": single, "date_range": date_range, "fulltext": fulltext, "inner_disjunction": inner_disjunction, } # Check _execute_search_query() call. assert execute_search_query_mock.call_count == 1 assert execute_search_query_mock.call_args[1] == { "index": search_const.OS_VO_DETAIL_INDEX_NAME, "offset": offset, "limit": limit, "order_by": order_by, "query_expr": query_expr, } # Check result. assert hits == total_hits assert result == es_result @pytest.mark.parametrize( ( "single", "multiple", "fulltext", "date_range_expressions", "inner_disjunction", ), ( pytest.param({"s1": 1}, {}, {}, (), {}, id="single_param"), pytest.param({"s1": 1, "s2": 2}, {}, {}, (), {}, id="single_multi_param"), pytest.param({}, {"m1": [1, 2]}, {}, (), {}, id="multiple_param"), pytest.param( {}, {"m1": [1, 2], "m2": [3]}, {}, (), {}, id="multiple_multi_param" ), pytest.param({}, {}, {"f1": "text1"}, (), {}, id="fulltext_param"), pytest.param( {}, {}, {"f1": "text1", "f2": "text2"}, (), {}, id="fulltext_multi_param" ), pytest.param( {}, {}, {}, ({"d1": {"gte": "v1", "lte": "v2"}},), {}, id="date_range_param" ), pytest.param( {}, {}, {}, ({"d1": {"gte": "v1"}}, {"d2": {"lte": "v2"}}), {}, id="date_range_multi_param", ), pytest.param( {}, {}, {}, (), {"_fake_category_1": [{"multiple": {"status": ["encoded", "encoding"]}}]}, id="inner_disjunction_single", ), pytest.param( {}, {}, {}, (), { "_fake_category_1": [ {"multiple": {"status": ["encoding_stuck"]}}, {"multiple": {"order_id": ["1", "2"]}}, ], "_fake_category_2": [ {"single": {"meta_update": "N"}}, ], }, id="inner_disjunction_multiple", ), pytest.param( {"s1": "s1"}, {"m1": ["m1"]}, {"f1": "f1"}, ({"d1": {"gte": "v1"}},), {"_statuses": [{"multiple": {"status": ["encoded", "encoding"]}}]}, id="all_types", ), ), ) def test_build_query_expression( single: dict[Any, Any], multiple: dict[Any, Any], fulltext: dict[Any, Any], date_range_expressions: dict[Any, Any], inner_disjunction: dict[Any, Any], mocker: Mock, app_context: AppContext, snapshot: SnapshotAssertion, ) -> None: """Test search query construction with fields from different categories.""" num_date_range_expressions = len(date_range_expressions) # Passing any dict with the number of items matching # date_range_expressions just to get into the for loop: date_range = {i: "v" + str(i) for i in range(num_date_range_expressions)} build_range_expr_mock = mocker.patch( "vectororder.models.search._build_range_expression" ) build_range_expr_mock.side_effect = date_range_expressions query_expr = search_models.build_query_expression( single=single, multiple=multiple, fulltext=fulltext, date_range=date_range, inner_disjunction=inner_disjunction, ) if query_expr: query_dict = query_expr.to_dict() assert snapshot == query_dict assert build_range_expr_mock.call_count == num_date_range_expressions assert build_range_expr_mock.call_args_list == [ ({"field_name": k, "date_range": v},) for k, v in date_range.items() ] @pytest.mark.parametrize( "single, multiple, fulltext, date_range", [ # All possible empty parameters combinations. ({}, {}, {}, {}), ({}, {}, {}, None), ({}, {}, None, {}), ({}, {}, None, None), ({}, None, {}, {}), ({}, None, {}, None), ({}, None, None, {}), ({}, None, None, None), (None, {}, {}, {}), (None, {}, {}, None), (None, {}, None, {}), (None, {}, None, None), (None, None, {}, {}), (None, None, {}, None), (None, None, None, {}), (None, None, None, None), ], ) def test_build_query_expression_empty( single: dict[Any, Any], multiple: dict[Any, Any], fulltext: dict[Any, Any], date_range: dict[Any, Any], ) -> None: """Test search query construction returns None for empty arguments.""" assert ( search_models.build_query_expression( single=single, multiple=multiple, fulltext=fulltext, date_range=date_range ) is None ) @pytest.mark.parametrize("iterable_callable", (list, tuple)) @pytest.mark.parametrize( "order_by, expected_order_by", ( ((), [search_const.DOCUMENT_SORT_FIELD]), (("f1",), ["f1", search_const.DOCUMENT_SORT_FIELD]), (("f1", "-f2"), ["f1", "-f2", search_const.DOCUMENT_SORT_FIELD]), ), ) def test_get_ordering_fields( order_by: list[Any], expected_order_by: list[Any], iterable_callable: Callable[..., Any], ) -> None: """Test building a list of fields to order search results by.""" order_by = iterable_callable(order_by) order_by_copy = list(order_by)[:] assert search_models.get_ordering_fields(order_by) == expected_order_by # Make sure the original input is unchanged. assert list(order_by) == order_by_copy @pytest.mark.parametrize( "order_by, expected_order_by", ( (None, [search_const.DOCUMENT_SORT_FIELD]), ("f1", ["f1", search_const.DOCUMENT_SORT_FIELD]), ), ) def test_get_ordering_fields_not_iterable( order_by: list[Any], expected_order_by: list[Any] ) -> None: """Test building a list of fields to order with non-iterable input.""" assert search_models.get_ordering_fields(order_by) == expected_order_by @pytest.mark.parametrize( "field_name, date_range, expected_result", ( ("f1", (None, None), None), ("f1", ("v1", None), {"f1": {"gte": "v1"}}), ("f1", (None, "v1"), {"f1": {"lte": "v1"}}), ("f1", ("v1", "v2"), {"f1": {"gte": "v1", "lte": "v2"}}), ), ) def test_build_range_query( field_name: str, date_range: tuple[Any, Any], expected_result: dict[Any, Any] ) -> None: """Test building ES range query.""" result = search_models._build_range_expression( field_name=field_name, date_range=date_range ) assert result == expected_result @pytest.mark.parametrize( "offset, limit, order_by, expected_order_by, expected_query_params", ( # Additional implicit sort field is defined in search constants. (None, None, None, ["encoding_queue_detail_id"], {"from": 0}), (1, None, None, ["encoding_queue_detail_id"], {"from": 1}), (None, 1, None, ["encoding_queue_detail_id"], {"from": 0, "size": 1}), (0, 0, None, ["encoding_queue_detail_id"], {"from": 0}), (1, 0, None, ["encoding_queue_detail_id"], {"from": 1}), (0, 1, None, ["encoding_queue_detail_id"], {"from": 0, "size": 1}), (1, 2, None, ["encoding_queue_detail_id"], {"from": 1, "size": 2}), (None, None, "f1", ["f1", "encoding_queue_detail_id"], {"from": 0}), (None, None, ["f1"], ["f1", "encoding_queue_detail_id"], {"from": 0}), ( None, None, ["f1", "-f2"], ["f1", "-f2", "encoding_queue_detail_id"], {"from": 0}, ), ), ) @patch("vectororder.models.search.g", spec=["ows"]) def test_execute_search_query( mock_g: MagicMock, mocker: Mock, offset: int | None, limit: int | None, order_by: list[str] | None, expected_order_by: list[str], expected_query_params: dict[str, Any], app_context: AppContext, ) -> None: """Test low level ES query execution.""" expected_result = (1, 2) expected_result_iterable = iter(expected_result) query_expr = query.Q("term", foo="bar") index_name = "someindex" search_request_query_mock = mocker.Mock() search_request_query_mock.execute.return_value = expected_result_iterable search_request_query_mock.to_dict.return_value = {"any": "value"} search_request_mock_sliced = mocker.Mock() search_request_mock_sliced.filter.return_value = search_request_query_mock search_request_mock = mocker.Mock() search_request_mock.sort.return_value = search_request_mock search_request_mock.update_from_dict.return_value = search_request_mock_sliced search_obj_mock = mocker.Mock(return_value=search_request_mock) es_dsl_search_mock = mocker.patch("vectororder.models.search.search") es_dsl_search_mock.Search = search_obj_mock result = search_models._execute_search_query( query_expr=query_expr, index=index_name, offset=offset, limit=limit, order_by=order_by, ) search_obj_mock.assert_called_with( using=elasticsearch.get_client(), index=index_name ) search_request_mock.sort.assert_called_with(*expected_order_by) search_request_mock.update_from_dict.assert_called_with(expected_query_params) search_request_mock_sliced.filter.assert_called_with(query_expr) assert search_request_mock_sliced.filter.call_count == 1 assert search_request_query_mock.execute.call_count == 1 search_request_query_mock.execute.assert_called_once_with(ignore_cache=False) assert mock_g.ows.log.debug.call_count == 1 assert tuple(result) == tuple(expected_result) @patch("vectororder.models.search.g", spec=["ows"]) @patch("vectororder.models.search.config.OPENSEARCH_IGNORE_CACHE", True) def test_execute_search_query_no_cache( mock_g: MagicMock, mocker: Mock, app_context: AppContext, ) -> None: """Test low level ES query execution.""" expected_result = (1, 2) expected_result_iterable = iter(expected_result) query_expr = query.Q("term", foo="bar") index_name = "someindex" search_request_query_mock = mocker.Mock() search_request_query_mock.execute.return_value = expected_result_iterable search_request_query_mock.to_dict.return_value = {"any": "value"} search_request_mock_sliced = mocker.Mock() search_request_mock_sliced.filter.return_value = search_request_query_mock search_request_mock = mocker.Mock() search_request_mock.sort.return_value = search_request_mock search_request_mock.update_from_dict.return_value = search_request_mock_sliced search_obj_mock = mocker.Mock(return_value=search_request_mock) es_dsl_search_mock = mocker.patch("vectororder.models.search.search") es_dsl_search_mock.Search = search_obj_mock result = search_models._execute_search_query( query_expr=query_expr, index=index_name, offset=0, limit=100, order_by=["-created_at"], ) search_obj_mock.assert_called_with( using=elasticsearch.get_client(), index=index_name ) search_request_mock.sort.assert_called_with( *["-created_at", "encoding_queue_detail_id"] ) search_request_mock.update_from_dict.assert_called_with(*[{"from": 0, "size": 100}]) search_request_mock_sliced.filter.assert_called_with(query_expr) assert search_request_mock_sliced.filter.call_count == 1 assert search_request_query_mock.execute.call_count == 1 search_request_query_mock.execute.assert_called_once_with(ignore_cache=True) assert mock_g.ows.log.debug.call_count == 1 assert tuple(result) == tuple(expected_result) @pytest.mark.parametrize("raises", (True, False)) def test_save_search_request(raises: bool, mocker: Mock) -> None: """Test search request is saved to DynamoDB.""" request_data = {"foo": "bar"} expected_request_id = "123abc" mock_uuid4 = mocker.Mock() mock_uuid4.hex = expected_request_id mock_uuid = mocker.patch("vectororder.models.search.uuid") mock_uuid.uuid4.return_value = mock_uuid4 mock_table = mocker.Mock() if raises: mock_table.put_item = mocker.Mock( side_effect=botocore_exceptions.ClientError( error_response={"Error": {"Code": "fancy code"}}, operation_name="bar" ) ) mock_dynamodb_resource = mocker.Mock() mock_dynamodb_resource.Table.return_value = mock_table mock_dynamodb_connector = mocker.patch("vectororder.models.search.dynamodb") mock_dynamodb_connector.get_dynamodb_resource.return_value = mock_dynamodb_resource if raises: with pytest.raises(botocore_exceptions.ClientError): search_models.save_search_request(request_data) else: result = search_models.save_search_request(request_data) assert result == expected_request_id assert mock_dynamodb_connector.get_dynamodb_resource.call_count == 1 mock_dynamodb_connector.get_dynamodb_resource.assert_called_with() assert mock_dynamodb_resource.Table.call_count == 1 mock_dynamodb_resource.Table.assert_called_with(config.DDB_SEARCH_REQUESTS_TABLE) assert mock_table.put_item.call_count == 1 mock_table.put_item.assert_called_with( Item={ "id": expected_request_id, "payload_json": json.dumps(request_data), } ) @pytest.mark.parametrize( "get_item_side_effect", [ [{"Item": {"payload_json": '{"foo": 1}'}}], NotFound(error_const.ERROR_MESSAGE_SEARCH_REQUEST_NOT_FOUND), ], ) def test_get_search_request(get_item_side_effect: Any, mocker: Mock) -> None: """Test search request is retrieved from DynamoDB.""" request_id = "123abc" mock_table = mocker.Mock() mock_table.get_item.side_effect = get_item_side_effect mock_dynamodb_resource = mocker.Mock() mock_dynamodb_resource.Table.return_value = mock_table mock_dynamodb_connector = mocker.patch("vectororder.models.search.dynamodb") mock_dynamodb_connector.get_dynamodb_resource.return_value = mock_dynamodb_resource if isinstance(get_item_side_effect, NotFound): with pytest.raises(NotFound) as exc: search_models.get_search_request(request_id) assert str(exc.value) == ( "404 Not Found: " + error_const.ERROR_MESSAGE_SEARCH_REQUEST_NOT_FOUND ) else: expected_payload = json.loads(get_item_side_effect[0]["Item"]["payload_json"]) result = search_models.get_search_request(request_id) assert result == expected_payload assert mock_dynamodb_connector.get_dynamodb_resource.call_count == 1 mock_dynamodb_connector.get_dynamodb_resource.assert_called_with() assert mock_dynamodb_resource.Table.call_count == 1 mock_dynamodb_resource.Table.assert_called_with(config.DDB_SEARCH_REQUESTS_TABLE) assert mock_table.get_item.call_count == 1 mock_table.get_item.assert_called_with(Key={"id": request_id}) @pytest.mark.parametrize( "value, expected_result", ( ("foo bar", ("match", "foo bar")), ('"foo bar"', ("match_phrase", "foo bar")), ("", ("match", "")), ), ) def test_get_match_type_with_value( value: str, expected_result: tuple[str, str] ) -> None: """Test getting ES match type and value.""" assert search_models._get_match_type_with_value(value) == expected_result