#!/usr/bin/env python3 """Build dashboard-data.json from existing per-service CSV/JSON analysis output. Reads data/{service}/ directories that contain: - all_endpoints.csv (or ows-{service}-endpoints.csv) - dead_endpoints.csv (or ows-{service}-dead-endpoints.csv) - summary.json (or ows-{service}-summary.json) Produces data/dashboard-data.json for the frontend. """ import csv import json from datetime import datetime, timezone from pathlib import Path DATA_DIR = Path(__file__).resolve().parent.parent / 'data' SERVICE_META = { 'ows-users': {'framework': 'flask', 'repo_url': 'https://github.com/theorchard/ows-users', 'team': 'permissions-platform'}, 'ows-permissions': {'framework': 'flask', 'repo_url': 'https://github.com/theorchard/ows-permissions', 'team': 'permissions-platform'}, 'ows-track': {'framework': 'flask', 'repo_url': 'https://github.com/theorchard/ows-track', 'team': 'content-platform'}, } def find_file(service_dir: Path, candidates: list[str]) -> Path | None: for name in candidates: p = service_dir / name if p.exists(): return p return None def load_all_endpoints(service_dir: Path, service_name: str) -> list[dict]: short = service_name.removeprefix('ows-') path = find_file(service_dir, ['all_endpoints.csv', f'ows-{short}-endpoints.csv', f'{service_name}-endpoints.csv']) if not path: return [] rows = [] with open(path) as f: reader = csv.DictReader(f) for row in reader: rows.append(row) return rows def load_dead_endpoints(service_dir: Path, service_name: str) -> list[dict]: short = service_name.removeprefix('ows-') path = find_file(service_dir, ['dead_endpoints.csv', f'ows-{short}-dead-endpoints.csv', f'{service_name}-dead-endpoints.csv']) if not path: return [] rows = [] with open(path) as f: reader = csv.DictReader(f) for row in reader: rows.append(row) return rows def load_summary(service_dir: Path, service_name: str) -> dict | None: short = service_name.removeprefix('ows-') path = find_file(service_dir, ['summary.json', f'ows-{short}-summary.json', f'{service_name}-summary.json']) if not path: return None with open(path) as f: return json.load(f) def build_details(all_eps: list[dict], dead_eps: list[dict]) -> list[dict]: """Build the details array from all_endpoints + dead_endpoints CSVs.""" # Index dead endpoints for dead_functions lookup: (method, path) -> dead_functions dead_map: dict[tuple[str, str], dict] = {} for row in dead_eps: key = (row.get('method', ''), row.get('path', '')) dead_map[key] = row details = [] for row in all_eps: method = row.get('method', '') path = row.get('path', '') in_datadog = row.get('in_datadog', 'no') == 'yes' # Determine status dead_row = dead_map.get((method, path)) if dead_row: status = 'unused' elif in_datadog: status = 'used' else: status = 'unused' # Parse dead functions dead_funcs_str = dead_row.get('dead_functions', '') if dead_row else '' dead_functions = [f.strip() for f in dead_funcs_str.split(';') if f.strip()] if dead_funcs_str else [] details.append( { 'method': method, 'path': path, 'handler': row.get('handler', ''), 'file': row.get('file', ''), 'line': int(row.get('line', 0) or 0), 'status': status, 'dead_functions': dead_functions, } ) return details def build_summary_from_details(details: list[dict], original_summary: dict | None) -> dict: """Build the endpoints/functions summary from details.""" total = len(details) used = sum(1 for d in details if d['status'] == 'used') unused = sum(1 for d in details if d['status'] == 'unused') def pct(count: int) -> float: return round(count / total * 100, 1) if total > 0 else 0.0 # Use original summary for function counts if available if original_summary: functions = original_summary.get('functions', {'total': 0, 'dead': 0, 'dead_percent': 0.0}) dead_functions_list = original_summary.get('dead_functions', []) else: all_dead_fns = set() for d in details: if d['status'] == 'unused': all_dead_fns.update(d['dead_functions']) functions = {'total': 0, 'dead': len(all_dead_fns), 'dead_percent': 0.0} dead_functions_list = sorted(all_dead_fns) return { 'endpoints': { 'total': total, 'used': {'count': used, 'percent': pct(used)}, 'unused': {'count': unused, 'percent': pct(unused)}, }, 'functions': functions, 'dead_functions': dead_functions_list, } def main(): services = [] for service_dir in sorted(DATA_DIR.iterdir()): if not service_dir.is_dir() or not service_dir.name.startswith('ows-'): continue service_name = service_dir.name print(f'Processing {service_name}...') all_eps = load_all_endpoints(service_dir, service_name) dead_eps = load_dead_endpoints(service_dir, service_name) summary = load_summary(service_dir, service_name) if not all_eps: print(f' SKIP: no all_endpoints CSV found') continue details = build_details(all_eps, dead_eps) built_summary = build_summary_from_details(details, summary) meta = SERVICE_META.get(service_name, {'framework': 'flask', 'repo_url': '', 'team': ''}) services.append( { 'name': service_name, 'framework': meta['framework'], 'repo_url': meta['repo_url'], 'team': meta['team'], 'endpoints': built_summary['endpoints'], 'functions': built_summary['functions'], 'details': details, 'dead_functions': built_summary['dead_functions'], } ) ep = built_summary['endpoints'] print(f' {ep["total"]} endpoints: {ep["used"]["count"]} used, {ep["unused"]["count"]} unused') dashboard = { 'generated_at': datetime.now(timezone.utc).isoformat(), 'env': 'prod', 'services': services, } output = DATA_DIR / 'dashboard-data.json' with open(output, 'w') as f: json.dump(dashboard, f, indent=2) print(f'\nWrote {output} ({len(services)} services)') if __name__ == '__main__': main()