"""Utilities for CloudSearch documents.""" from dateutil.parser import parse import sentry_sdk import config # noqa import const import sentry # noqa from utils import date as date_utils from utils import string as string_utils ZERO_DATE = '0000-00-00' MINIMAL_YEAR = 1000 MAX_LIST_LENGTH = 1000 def prepare_for_upload(schema, document): """Clean a CloudSearch document before upload. Args: schema (dict): JSON schema for the document document (dict): a dict representing a CloudSearch document Returns: dict: cleaned document """ if document.get('type') == 'delete': document.pop('fields') else: properties = schema.get('properties') fields = document.get('fields') for prop, value in fields.copy().items(): if prop in properties: if properties[prop]['type'] == 'integer' and value is None: document['fields'].pop(prop) if properties[prop]['type'] == 'array' and value is None: document['fields'][prop] = [] if properties[prop]['type'] == 'string' and value is None: document['fields'].pop(prop) elif properties[prop]['type'] == 'string' and value: document['fields'][prop] = string_utils.cleanup_unicode( value) if (properties[prop]['type'] == 'date_utc' and value is None # noqa or value == ZERO_DATE): # noqa document['fields'].pop(prop) elif properties[prop]['type'] == 'date_utc' and value: try: parsed_date = parse(value) formatted_date = parsed_date.strftime('%Y-%m-%dT%H:%M:%SZ') if (int(value[:4]) < MINIMAL_YEAR or not date_utils.is_expected_datetime_format(formatted_date)): # noqa raise ValueError(const.INVALID_DATE_ERROR) document['fields'][prop] = formatted_date except ValueError: document['fields'].pop(prop) if (properties[prop]['type'] == 'array' and len(document['fields'][prop]) > MAX_LIST_LENGTH): # noqa sentry_sdk.capture_message( 'isrc_list is too long for release: {}'.format( document['fields'].get('release_id'))) return False elif properties[prop]['type'] == 'array' and value: document['fields'][prop] = [ string_utils.cleanup_unicode(item) for item in value] else: document['fields'].pop(prop) if not document['fields']: return False return document