import datetime import logging import connection, utils import consts import queries PR_NAME = 'KDH-308_nr_data_export' logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def main(): ts = datetime.datetime.now() generate_db_pr() print(f"Process took {(datetime.datetime.now() - ts).total_seconds()}") def generate_db_pr(): dump_queries = generate_dump_queries() changesets = [] for i, query in enumerate(dump_queries, start=1): logger.info(f"Generating changeset {i}") changesets.append(consts.DB_CHANGESET_TEMPLATE.format(id=PR_NAME, i=i, query=query)) db_pr_content = consts.DB_PR_TEMPLATE.format(changesets=''.join(changesets)) with open(f'{PR_NAME}.xml', 'w') as f: f.write(db_pr_content) logger.info(f"PR file {PR_NAME}.xml generated") def generate_dump_queries(): result = [] for labels in consts.LABELS: logger.info(f"Generating dump query for {labels}") result.append(_compose_label_dump_query(labels)) for start_label, relationship, end_label in consts.RELATIONS: logger.info(f"Generating dump query for {start_label} {relationship} {end_label}") result.append(_compose_relationship_dump_query(start_label, relationship, end_label)) return result def dump_node_data(labels): ts = datetime.datetime.now() query = _compose_label_dump_query(labels) with connection.driver.session() as s_source: s_source.run(query) te = datetime.datetime.now() print(f"Dumping {labels} data took {(te - ts).total_seconds()}") def dump_relationship_data(start_label, relationship, end_label): query = _compose_relationship_dump_query(start_label, relationship, end_label) with connection.driver.session() as s_source: s_source.run(query) def _compose_relationship_dump_query(start_label, relationship, end_label): filename = utils.generate_relationship_json_filename(start_label, relationship, end_label) slabel = f':{start_label}' relation = f':{relationship}' elabel = f':{end_label}' properties = _get_relation_properties(slabel, relation, elabel) data_query = _compose_relation_data_query(slabel, relation, elabel, properties) return queries.DUMP_JSON.format(query=data_query, filename=filename) def _get_relation_properties(start_label, relationship, end_label): with connection.driver.session() as s_source: cypher = queries.RELATION_UNIQUE_KEYS.format( start_label=start_label, relationship=relationship, end_label=end_label) properties_result = s_source.run(cypher) properties = properties_result.value()[0] return properties def _compose_relation_data_query(start_label, relationship, end_label, properties): """Compose a query to dump the properties of a node.""" properties_str = 'n.id as startId, k.id as endId, ' + ', '.join([f'r.{p} as {p}' for p in properties]) cypher = queries.RELATION_DATA.format( start_label=start_label, relationship=relationship, end_label=end_label, properties=properties_str) return cypher ### Label helpers # def _get_node_properties(node_labels, exclude_labels): # """Find all property names for the given node labels.""" # with connection.driver.session() as s_source: # cypher = queries.LABEL_UNIQUE_KEYS.format(labels=node_labels, exclude_labels=exclude_labels) # properties_result = s_source.run(cypher) # properties = properties_result.value()[0] # return properties def _compose_node_data_query(node_labels, exclude_node_labels, properties): """Compose a query to dump the properties of a node.""" properties_str = ', '.join([f'n.{p} as {p}' for p in properties]) cypher = queries.NODE_DATA.format(labels=node_labels, exclude_labels=exclude_node_labels, properties=properties_str) return cypher def _compose_label_dump_query(labels): query_labels = labels['labels'] exclude_labels = labels.get('exclude', []) filename = utils.generate_labels_json_filename(query_labels) node_labels = ':' + ':'.join(query_labels) if exclude_labels: exclude_node_labels = '\n WHERE ' + ' and '.join([f'NOT n:{label}' for label in exclude_labels]) else: exclude_node_labels = '' properties = utils.get_node_properties(node_labels, exclude_node_labels) node_data_query = _compose_node_data_query(node_labels, exclude_node_labels, properties) return queries.DUMP_JSON.format(query=node_data_query, filename=filename) if __name__ == '__main__': main()