"""Update Opensearch indexes.""" import json from typing import Any, List, Optional, Tuple import config from lambdacommon.common_config import logger from opensearchpy import helpers from opensearchpy.exceptions import OpenSearchException from src.connectors.opensearch import os_client from src.logic.types import OpensearchBulkDocument def upload_bulk_documents( documents: List[OpensearchBulkDocument], ) -> Tuple[int, Optional[list[Any]]]: """Upload documents to Opensearch index. Args: documents: (array) array of documents for this index in format [{ "_op_type": "index", # or "create" "_index": "your_index_name", "_id": "doc_id_1", "_source": {"field1": "value1", "field2": "value2"} }, { "_op_type": "delete", "_index": "your_index_name", "_id": "doc_id_to_delete_1" },...] """ if len(documents) < 1: return 0, [] logger.info(f"Starting to upload {len(documents)} documents") successful_count, os_errors = helpers.bulk( client=os_client, actions=documents, raise_on_error=False, # we will explicitly check for actual errors. max_retries=config.OPENSEARCH_MAX_BULK_RETRIES, ) actual_error = [] ignore_errors = [] for each in os_errors: for operation, error in each.items(): if operation == "delete" and error["result"] == "not_found": ignore_errors.append(each) else: actual_error.append(each) if actual_error: logger.error( f"Finished upload_bulk_documents. success: {successful_count} " f"and errors: {len(os_errors)}", extra={ "actual_error": json.dumps(actual_error), "ignore_errors": json.dumps(ignore_errors), }, ) raise OpenSearchException( "Unexpected error for upload_bulk_documents: {}".format(json.dumps(actual_error)) ) logger.info(f"Finished upload_bulk_documents. success: {successful_count}") return successful_count, os_errors