"""Utilities for Opensearch documents.""" import logging import re from typing import Any, Optional import sentry_sdk from dateutil.parser import parse from src.constants.fields import ( CLEANUP_PATTERN, INVALID_DATE_ERROR, MAX_LIST_LENGTH, MINIMAL_YEAR, VALID_DATE_FORMAT, ZERO_DATE, ) from src.logic.types import OpensearchBulkDocument def cleanup_unicode(value: str) -> str: """Cleanup characters that are not supported by Cloudsearch. Args: value (str): string to clean up Returns: str: cleaned string """ if value: value = re.sub(CLEANUP_PATTERN, "", value) return value def is_expected_datetime_format(value: str) -> bool: """Check date format that is supported by Opensearch. Args: value (str): UTC date Returns: bool """ try: return True if re.match(VALID_DATE_FORMAT, value) else False except Exception: return False def parse_date(value: str) -> str: """Try to parse the value as a date.""" parsed_date = parse(value) formatted_date = parsed_date.strftime("%Y-%m-%dT%H:%M:%SZ") if int(value[:4]) < MINIMAL_YEAR or not is_expected_datetime_format(formatted_date): raise ValueError(INVALID_DATE_ERROR) return formatted_date def prepare_for_upload( schema: dict[str, Any], document: OpensearchBulkDocument ) -> Optional[OpensearchBulkDocument]: """Clean ows response document before upload to Opensearch. Args: schema (dict): JSON schema for the document. document (dict): a dict representing an Opensearch document Returns: dict: cleaned document """ if document.get("_op_type") == "delete": document.pop("doc") return document properties = schema.get("properties") fields = document.get("doc") if not fields: return None if not properties: return None for prop, value in fields.copy().items(): if prop in properties: if properties[prop]["type"] == "integer" and value is None: document["doc"].pop(prop) elif properties[prop]["type"] == "array" and value is None: document["doc"][prop] = [] elif properties[prop]["type"] == "string" and value is None: document["doc"].pop(prop) elif properties[prop]["type"] == "string" and value: document["doc"][prop] = cleanup_unicode(value) elif properties[prop]["type"] == "date_utc" and value is None or value == ZERO_DATE: document["doc"].pop(prop) elif properties[prop]["type"] == "date_utc" and value: try: document["doc"][prop] = parse_date(value) except ValueError: document["doc"].pop(prop) msg = "SKIP: invalid date: {} for release: {}".format( prop, document["doc"].get("release_id") ) logging.warning(msg) elif ( properties[prop]["type"] == "array" and len(document["doc"][prop]) > MAX_LIST_LENGTH ): msg = "SKIP: isrc_list is too long for release: {}".format( document["doc"].get("release_id") ) sentry_sdk.capture_message(msg) logging.warning(msg) return None elif properties[prop]["type"] == "array" and value: document["doc"][prop] = [cleanup_unicode(item) for item in value] else: document["doc"].pop(prop) if not document["doc"]: return None return document