import click import consts import utils import sql_queries def main(schema): generate_node_checks(schema) generate_relation_checks(schema) EXCLUDED_PROPERTIES = ['createdAt', 'createdBy', 'lastModifiedAt', 'lastModifiedBy', 'touchedByBackfill'] def generate_relation_checks(schema): with click.progressbar(consts.RELATIONS_TO_CHECK, label='Processing items') as bar: for start_label, relation, end_label, table in bar: start_table = utils.camel_case_to_upper_snake_case(start_label) end_table = utils.camel_case_to_upper_snake_case(end_label) properties = utils.get_relation_properties(start_label, relation, end_label) properties = list(filter(lambda x: x not in EXCLUDED_PROPERTIES, properties)) relation_properties = list(map(utils.camel_case_to_upper_snake_case, properties)) mg_table = table or f'{start_table}_{relation}_{end_table}' for check_name, check_template in sql_queries.relation_count_checks.items(): _generate_relation_count_check(check_name, check_template, schema, mg_table, start_table, end_table) if relation_properties: key_properties = [f'{start_table}_ID'.lower(), f'{end_table}_ID'.lower()] for check_name, check_template in sql_queries.relation_metadata_checks.items(): _generate_relations_metadata_check( check_name, check_template, schema, mg_table, key_properties, relation_properties) def _generate_relation_count_check(template_name, template, schema, mg_table, start_table, end_table): sql = template.format( schema=schema, mg_table=mg_table, ng_table=f'NRGRAPH_{mg_table}', start_column=f'{start_table}_ID'.lower(), end_column=f'{end_table}_ID'.lower()) with open(f'sql_scripts/relations/{mg_table}_{template_name}', 'w') as f: f.write(sql) def _generate_relations_metadata_check(check_name, check_template, schema, mg_table, key_properties, table_properties): join = ' AND '.join([f'mg.{prop} = ng.{prop} \n' for prop in key_properties]) where = ' OR '.join([f'NOT EQUAL_NULL(mg.{prop}, ng.{prop}) \n' for prop in table_properties]) sql = check_template.format(schema=schema, mg_table=mg_table, ng_table=f'NRGRAPH_{mg_table}', join=join, where=where) with open(f'sql_scripts/relations/{mg_table}_{check_name}', 'w') as f: f.write(sql) def generate_node_checks(schema): with click.progressbar(consts.NODES_TO_CHECK, label='Processing items') as bar: for item in bar: properties = utils.get_node_properties(node_labels=item['labels']) properties = list(filter(lambda x: x not in EXCLUDED_PROPERTIES, properties)) table_name = item['table'] table_properties = list(map(utils.camel_case_to_upper_snake_case, properties)) for check_name, check_template in sql_queries.node_count_checks.items(): _generate_node_count_check('./sql_scripts/nodes', check_name, check_template, schema, table_name) for check_name, check_template in sql_queries.node_metadata_checks.items(): _generate_node_metadata_check('./sql_scripts/nodes', check_name, check_template, schema, table_name, table_properties) def _generate_node_count_check(target_path, template_name, template, schema, mg_table): sql = template.format(schema=schema, mg_table=mg_table, ng_table=f'NRGRAPH_{mg_table}') with open(f'{target_path}/{mg_table}_{template_name}', 'w') as f: f.write(sql) def _generate_node_metadata_check(target_path, check_name, check_template, schema, mg_table, table_properties): where = ' OR '.join([f'NOT EQUAL_NULL(mg.{prop}, ng.{prop}) \n' for prop in table_properties]) sql = check_template.format(schema=schema, mg_table=mg_table, ng_table=f'NRGRAPH_{mg_table}', where=where) with open(f'{target_path}/{mg_table}_{check_name}', 'w') as f: f.write(sql) if __name__ == '__main__': schema = '[ENV]' # schema = 'QA' main(schema=schema)