import math import os import re import sys import warnings from sqlalchemy import create_engine from sqlalchemy.sql import text import colorama import pymysql # Format: mysql+pymysql://:@/ ENGINE_DSN = os.environ.get('ENGINE_DSN') QUERY_INDEX_TABLE_INFORMATION = ( 'SELECT table_name, column_name, column_type ' 'FROM INFORMATION_SCHEMA.COLUMNS ' 'WHERE extra like "%auto_increment%"') QUERY_COUNT_INDEX_ENTRIES = 'SELECT count({field}) from `{table}`' MAP_COLUMN_TYPES_SIZE = { 'bigint': 64, 'int': 32, 'mediumint': 24, 'smallint': 16, 'tinyint': 8, } engine = create_engine(ENGINE_DSN, echo=False) connection = engine.connect() # Ignore warnings from pymysql.err.Warning. This is because some of our comment # fields have invalid utf8 values. warnings.filterwarnings('ignore', category=pymysql.err.Warning) def analyse_database(): """Analyse a database. The systems runs a query that lists all indexes for each table that have an autoincrement field, and analyse the index. Yields: dict: represents the analysis of an index (see analyse_table_index) for more details. """ tables = connection.execute(text(QUERY_INDEX_TABLE_INFORMATION)) for table in tables: table_name, column_name, column_format = table yield analyse_table_index(table_name, column_name, column_format) def analyse_table_index(table_name, column_name, column_format): """Analyse a table index. Args: table_name (str): Name of the table. column_name (str): Name of the column. column_format (str): Format of the column (usually "int(12) unsigned"). Returns: dict: the dictionary contains the different information ( percentage used, current number of items, the maximum number of items the table acn store, the name of the column and its format.) """ query = QUERY_COUNT_INDEX_ENTRIES.format( table=table_name, field=column_name) query = connection.execute(text(query)) total, = query.first() column_information = column_format.split(' ') sign = -1 if len(column_information) == 2 and column_information[1] == 'unsigned': sign = 0 column_information = re.match(r"(.+)\((.+)\)", column_information[0]) column_type = column_information.group(1) max_total = int(math.pow(2, MAP_COLUMN_TYPES_SIZE[column_type] - sign)) percentage = math.ceil(100 * total / max_total) return { 'percentage': percentage, 'total': total, 'max_total': max_total, 'table_name': table_name, 'column_name': column_name, 'column_format': column_format } def csv_printer(analysis): """CSV Printer. Args: analysis (dict): the analysis. """ print( '{table_name},{column_name},{percentage},{total},{max_total}'.format( **analysis)) def pretty_printer(analysis): """Pretty print an analysis. Args: analysis (dict): the analysis. """ table_color = colorama.Fore.BLUE if analysis.get('percentage') > 80: table_color = colorama.Fore.RED print( table_color + analysis.get('table_name') + colorama.Style.RESET_ALL + ': ' + colorama.Fore.YELLOW + analysis.get('column_name'), colorama.Style.RESET_ALL) print(' Percentage:', analysis.get('percentage')) print(' Total:', analysis.get('total')) print(' Max entries:', analysis.get('max_total')) print() if __name__ == '__main__': colorama.init() is_csv = '--csv' in sys.argv if is_csv: print('table name,column name,percentage,total,max total') for analysis in analyse_database(): if is_csv: csv_printer(analysis) continue pretty_printer(analysis) connection.close()