"""Neo4J CDC event parser.""" from typing import List from neoparse import constants from neoparse import errors class BaseParser(object): """Base class for Neo4J event parser.""" def __init__(self, database_name: str): """Initialize a new instance of BaseParser. Args: database_name (str): database name. """ self.database_name = database_name def _parse_fields(self, input: dict, include_meta: bool = True) -> dict: """Parse the input event fields. Args: input (dict): input event data. include_meta (bool): include the meta info (tx_id, username, etc). Raises: ParserError: if something went wrong during the parsing process. Returns: dict: parsed fields from the event. """ try: payload = input['payload'] payload_type = payload['type'] fields = self._parse_node_relationship_fields( payload, input['meta']['operation']) fields = self._sanitize_fields(fields) if payload_type == constants.RELATIONSHIP: fields.update(self._parse_relationship_metadata(payload)) fields['node_id'] = payload['id'] if include_meta: fields['meta_tx_id'] = input['meta']['txId'] fields['meta_tx_event_id'] = input['meta']['txEventId'] fields['meta_tx_events_count'] = input['meta']['txEventsCount'] fields['meta_username'] = input['meta']['username'] fields['meta_hostname'] = input['meta']['source']['hostname'] return fields except errors.ParserError as parser_error: raise parser_error except KeyError as e: raise errors.ParserError(errors.PARSE_META_ERROR, e) def _sanitize_fields(self, fields: dict) -> dict: """Sanitize the parsed fields format. Concatenate lists to comma-separated strings. E.g. "a,b,c" Concatenate dicts to strings; E.g. "name:a,surname:b" Args: fields (dict): parsed fields. Returns: dict: sanitized fields. """ sanitized = {} for key, value in fields.items(): try: if isinstance(value, list): value = ','.join(value) if isinstance(value, dict): value = ','.join([f'{k}:{v}' for k, v in value.items()]) except: # noqa value = str(value) sanitized[key] = value return sanitized def _parse_node_relationship_fields( self, payload: dict, operation: str) -> dict: """Parse database fields for a node/relationship change event. Args: payload (dict): payload for CDC event. operation (str): operation. Raises: ParserError: if something went wrong during the parsing process. Returns: dict: parsed fields from the event. """ try: if operation in (constants.NEO_INSERT, constants.NEO_UPDATE): fields = payload['after']['properties'] elif operation == constants.NEO_DELETE: fields = payload['before']['properties'] else: fields = {} return fields except KeyError as e: # noqa raise errors.ParserError( errors.PARSE_NODE_RELATIONSHIP_FIELDS_ERROR, e) def _parse_relationship_metadata(self, payload: dict) -> dict: """Parse relationship metadata for a node change event. For node changed event we need to manually construct start node and end node labels and ids and add them to the parsed fields dict. Args: payload (dict): payload for CDC event. Raises: ParserError: if something went wrong during the parsing process. Returns: dict: parsed fields from the event. """ try: relationship_meta = { 'start_node_label': self._make_table_name( payload['start']['labels']), 'start_node_id': payload['start']['id'], 'end_node_label': self._make_table_name( payload['end']['labels']), 'end_node_id': payload['end']['id'], } return relationship_meta except (KeyError, IndexError) as e: raise errors.ParserError( errors.PARSE_RELATIONSHIP_METADATA_ERROR, e) def _parse_operation(self, input_json: dict) -> str: """Parse DB operation (insert/update/delete). Args: input_json (dict): input event data. Raises: ParserError: if something went wrong during the parsing process. Returns: str: operation (can be insert, update, delete or unknown) """ try: neo_operation = input_json['meta']['operation'] return constants.NEO_MAXWELLS_OPERATION_MAP.get( neo_operation, constants.UNKNOWN) except KeyError as e: raise errors.ParserError(errors.PARSE_OPERATION_ERROR, e) def _parse_timestamp(self, input_json: dict) -> int: """Parse timestamp field. Args: input_json (dict): input event data. Raises: ParserError: if something went wrong during the parsing process. Returns: int: timestamp """ try: return input_json['meta']['timestamp'] except KeyError as e: raise errors.ParserError(errors.PARSE_TIMESTAMP_ERROR, e) def _make_table_name(self, labels: List[str]) -> str: """Generate table name from a list of labels. Args: labels (list): list of node labels. Returns: str: table name. """ if len(labels) > 1: return '_'.join(sorted(labels)) else: return labels[0] def _parse_table(self, input_json: dict) -> str: """Parse table name from the event data. If a node has only one label - it will be the table name. For more than one node - we sort them alphabetically and concatenate with an underscore e.g. "ArtistProfile_Orchard" Args: input_json (dict): input event data. Raises: ParserError: if something went wrong during the parsing process. Returns: str: table name. """ try: payload = input_json['payload'] payload_type = payload['type'] if payload_type == constants.NODE: if payload.get('after'): labels = payload['after']['labels'] else: labels = payload['before']['labels'] elif payload_type == constants.RELATIONSHIP: labels = [payload['label']] return self._make_table_name(labels) except (KeyError, IndexError) as e: raise errors.ParserError(errors.PARSE_TABLE_ERROR, e) class NeoMaxwells(BaseParser): """Neo4J CDC event parser and converted to Maxwell's Daemon format.""" # Output event schema. SCHEMA = { 'properties': { 'database': {'type': 'string'}, 'table': {'type': 'string'}, 'ts': {'type': 'number'}, 'type': { 'type': 'string', 'enum': ['insert', 'update', 'delete'] }, 'fields': { 'type': 'object', 'properties': { 'meta_tx_id': {'type': 'number'}, 'meta_tx_events_count': {'type': 'number'}, 'meta_tx_event_id': {'type': 'number'}, 'meta_username': {'type': 'string'}, 'meta_hostname': {'type': 'string'}, 'node_id': {'type': 'number'} }, 'additionalProperties': True } } } def __init__(self, database_name: str, include_schema: bool = False): """Intialize a new instance of NeoMaxwells parser. Args: database_name (str): database name to use in the output json. include_schema (bool): whether to include the schema or not. """ self.include_schema = include_schema super().__init__(database_name) def parse(self, input: dict, include_meta: bool = True) -> dict: """Parse Neo4J CDC event and convert it to Maxwells daemon format. Args: input (dict): input event data received from Neo4J. include_meta (bool): include the meta information or not. meta information includes Neo4J transaction id, username, etc. Returns: dict: output event converted to Maxwells' daemon format. """ operation = self._parse_operation(input) fields = self._parse_fields(input, include_meta=include_meta) timestamp = self._parse_timestamp(input) table = self._parse_table(input) database = self.database_name try: payload = { 'database': database, 'table': table, 'type': operation, 'ts': timestamp, 'xid': input['meta']['txId'], 'data': fields } except KeyError as e: raise errors.ParserError(errors.PARSE_META_ERROR, e) if self.include_schema: result = { 'payload': payload, 'schema': self.SCHEMA } else: result = payload return result