import logging import subprocess import consts, utils import queries logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) PR_NAME = 'KDH-308_nr_data_import' def main(): label_cyphers = _generate_label_cypher() with open('label_cyphers.cypher', 'w') as f: for e in label_cyphers: f.write(e + '\n') relation_cypher = _generate_relation_cypher() with open('relation_cyphers.cypher', 'w') as f: for e in relation_cypher: f.write(e + '\n') def _generate_label_cypher(): result = [] for labels in consts.LABELS: logger.info(f"Generating import query for {labels}") node_labels = labels['labels'] s3_url = _presign_s3_node_files(node_labels) result.append(queries.LABEL_UPLOAD_TEMPLATE.format(s3_url=s3_url, labels=':' + ':'.join(node_labels))) return result def _generate_relation_cypher(): result = [] for start_label, relationship, end_label in consts.RELATIONS: logger.info(f"Generating import query for {start_label} {relationship} {end_label}") s3_url = _presign_s3_relation_files(start_label, relationship, end_label) result.append(queries.RELATION_UPLOAD_TEMPLATE.format( s3_url=s3_url, start_label=f':{start_label}', relation=f':{relationship}', end_label=f':{end_label}')) return result def _presign_s3_node_files(labels): filename = utils.generate_labels_json_filename(labels) command = f'/usr/local/bin/aws s3 presign {consts.S3_LOCATION_PREFIX}/{filename} --expires-in {consts.PRESIGNED_LINK_EXPIRES_IN}' result = subprocess.run(command, shell=True, capture_output=True, text=True) return result.stdout.strip() def _presign_s3_relation_files(start_label, relationship, end_label): filename = utils.generate_relationship_json_filename(start_label, relationship, end_label) command = f'/usr/local/bin/aws s3 presign {consts.S3_LOCATION_PREFIX}/{filename} --expires-in {consts.PRESIGNED_LINK_EXPIRES_IN}' result = subprocess.run(command, shell=True, capture_output=True, text=True) return result.stdout.strip() if __name__ == '__main__': main()