#!/usr/bin/env python3 """ Verify cost differences across three data sources. Baseline: Cost Report API (workspace-level, direct costs) Compares against: Usage API and Analytics API Highlights Default workspace vs Non-Default workspace differences. """ import json import sys from pathlib import Path from typing import Any # Add parent directory to path for imports sys.path.insert(0, str(Path(__file__).parent.parent)) from src.usage.cost_analyzer import PRICING def load_json(file_path: Path) -> dict[str, Any]: """Load a JSON file.""" if not file_path.exists(): return {} with open(file_path) as f: return json.load(f) def calculate_cost_from_usage(tokens: dict[str, int], model: str) -> float: """Calculate cost in dollars from token counts.""" if model not in PRICING: print(f'Warning: Unknown model {model}, using fallback pricing') return 0.0 pricing = PRICING[model] input_cost = tokens.get('uncached_input_tokens', 0) * pricing['input'] / 1_000_000 output_cost = tokens.get('output_tokens', 0) * pricing['output'] / 1_000_000 # Cache costs cache_creation = tokens.get('cache_creation', {}) cache_write_5m = cache_creation.get('ephemeral_5m_input_tokens', 0) cache_write_1h = cache_creation.get('ephemeral_1h_input_tokens', 0) cache_creation_cost = cache_write_5m * pricing.get('cache_write_5m', pricing['input']) / 1_000_000 + cache_write_1h * pricing.get('cache_write_1h', pricing['input']) / 1_000_000 cache_read_cost = tokens.get('cache_read_input_tokens', 0) * pricing.get('cache_read', pricing['input'] * 0.1) / 1_000_000 total = input_cost + output_cost + cache_creation_cost + cache_read_cost return total def analyze_usage_for_date(date: str, output_dir: Path, api_keys_lookup: dict) -> dict[str, Any]: """ Analyze usage data for a specific date by workspace. Returns: dict with workspace breakdown """ usage_file = output_dir / 'usage' / 'raw' / f'messages_{date}.json' if not usage_file.exists(): return {'error': f'Usage file not found: {usage_file}'} usage_data = load_json(usage_file) # Track by workspace workspace_costs = {} api_key_details = [] for bucket in usage_data.get('data', []): for result in bucket.get('results', []): api_key_id = result.get('api_key_id') workspace_id = result.get('workspace_id') model = result.get('model') cost = calculate_cost_from_usage(result, model) # Determine workspace workspace = 'Default' if workspace_id is None else workspace_id if workspace not in workspace_costs: workspace_costs[workspace] = 0.0 workspace_costs[workspace] += cost api_key_info = api_keys_lookup.get(api_key_id, {'name': 'Unknown', 'created_by': None}) api_key_name = api_key_info['name'] api_key_details.append( { 'api_key_id': api_key_id, 'api_key_name': api_key_name, 'workspace': workspace, 'model': model, 'cost': round(cost, 2), } ) total_cost = sum(workspace_costs.values()) return { 'total_cost': round(total_cost, 2), 'workspace_costs': {ws: round(cost, 2) for ws, cost in workspace_costs.items()}, 'default_workspace_cost': round(workspace_costs.get('Default', 0.0), 2), 'non_default_workspace_cost': round(sum(c for ws, c in workspace_costs.items() if ws != 'Default'), 2), 'api_key_details': api_key_details, } def analyze_analytics_for_date(date: str, output_dir: Path) -> dict[str, Any]: """ Analyze analytics data for a specific date. Returns: dict with actor breakdown """ analytics_file = output_dir / 'analytics' / 'raw' / f'{date}.json' if not analytics_file.exists(): return {'error': f'Analytics file not found: {analytics_file}'} analytics_data = load_json(analytics_file) total_cost = 0.0 actors = [] for record in analytics_data.get('data', []): actor_info = record.get('actor', {}) actor_name = actor_info.get('email_address') or actor_info.get('api_key_name', 'Unknown') actor_cost = 0.0 for model_data in record.get('model_breakdown', []): cost_cents = model_data.get('estimated_cost', {}).get('amount', 0) actor_cost += cost_cents / 100.0 total_cost += actor_cost actors.append( { 'actor': actor_name, 'terminal_type': record.get('terminal_type'), 'cost': round(actor_cost, 2), } ) return { 'total_cost': round(total_cost, 2), 'actors': actors, } def analyze_cost_report_for_date(date: str, output_dir: Path) -> dict[str, Any]: """ Analyze cost report data for a specific date by workspace. Returns: dict with workspace breakdown """ cost_report_file = output_dir / 'cost_report' / 'cost_report.json' if not cost_report_file.exists(): return {'error': f'Cost report file not found: {cost_report_file}'} cost_report_data = load_json(cost_report_file) workspace_costs = {} for workspace_id, workspace_data in cost_report_data.get('workspaces', {}).items(): workspace_name = workspace_data.get('workspace_name', workspace_id) for period in workspace_data.get('time_periods', []): starting_at = period.get('starting_at', '') period_date = starting_at.split('T')[0] if starting_at else '' if period_date == date: cost_cents = period.get('cost', 0) cost_dollars = cost_cents / 100.0 workspace_costs[workspace_name] = round(cost_dollars, 2) total_cost = sum(workspace_costs.values()) return { 'total_cost': round(total_cost, 2), 'workspace_costs': workspace_costs, 'default_workspace_cost': round(workspace_costs.get('default', 0.0), 2), 'non_default_workspace_cost': round(sum(c for ws, c in workspace_costs.items() if ws != 'default'), 2), } def main() -> None: """Main entry point.""" project_root = Path(__file__).parent.parent.parent differences_path = project_root / 'output' / 'debug' / 'differences.json' output_dir = project_root / 'output' if not differences_path.exists(): print(f'Error: {differences_path} not found!') print('Run: make compare-costs') sys.exit(1) # Load API keys lookup api_keys_data = load_json(output_dir / 'usage' / 'raw' / 'api_keys.json') api_keys_lookup = {} for key in api_keys_data.get('data', []): api_keys_lookup[key['id']] = { 'name': key.get('name', 'Unknown'), 'created_by': key.get('created_by', {}).get('name') if key.get('created_by') else None, } differences = load_json(differences_path) # Filter for significant differences significant = [d for d in differences['differences'] if d['has_significant_difference']] print(f'Found {len(significant)} days with significant differences (>${differences["summary"]["threshold_dollars"]})') print() results = [] for diff in significant: date = diff['date'] analytics_cost = diff['analytics_cost'] usage_cost = diff['usage_cost'] cost_report_cost = diff['cost_report_cost'] print('=' * 100) print(f'DATE: {date}') print('=' * 100) print() # Three-way comparison print('COST COMPARISON (Baseline: Cost Report)') print(f' Cost Report (baseline): ${cost_report_cost:>8.2f}') print(f' Usage API: ${usage_cost:>8.2f} (Δ {usage_cost - cost_report_cost:+.2f})') print(f' Analytics API: ${analytics_cost:>8.2f} (Δ {analytics_cost - cost_report_cost:+.2f})') print() # Analyze cost report breakdown cost_report_analysis = analyze_cost_report_for_date(date, output_dir) if 'error' not in cost_report_analysis: print('COST REPORT BREAKDOWN (by workspace):') print(f' Total: ${cost_report_analysis["total_cost"]:.2f}') for workspace, cost in sorted(cost_report_analysis['workspace_costs'].items()): marker = '⚪ Default' if workspace == 'default' else '🔵 Non-Default' print(f' {marker:15s} {workspace:40s} ${cost:.2f}') print() # Analyze usage breakdown usage_analysis = analyze_usage_for_date(date, output_dir, api_keys_lookup) if 'error' not in usage_analysis: print('USAGE API BREAKDOWN (by workspace):') print(f' Total: ${usage_analysis["total_cost"]:.2f}') for workspace, cost in sorted(usage_analysis['workspace_costs'].items()): marker = '⚪ Default' if workspace == 'Default' else '🔵 Non-Default' print(f' {marker:15s} {workspace:40s} ${cost:.2f}') print() print('USAGE API BREAKDOWN (by API key):') for item in usage_analysis['api_key_details']: ws_marker = '⚪' if item['workspace'] == 'Default' else '🔵' print(f' {ws_marker} {item["api_key_name"]:45s} | {item["workspace"]:15s} | {item["model"]:35s} | ${item["cost"]:.2f}') print() # Analyze analytics breakdown analytics_analysis = analyze_analytics_for_date(date, output_dir) if 'error' not in analytics_analysis: print('ANALYTICS API BREAKDOWN (Claude Code only):') print(f' Total: ${analytics_analysis["total_cost"]:.2f}') for actor in analytics_analysis['actors']: terminal = actor['terminal_type'] or 'unknown' print(f' {actor["actor"]:50s} ${actor["cost"]:>6.2f} [{terminal}]') print() # Analysis print('ANALYSIS:') # Check if usage matches cost report usage_matches_cost_report = abs(usage_cost - cost_report_cost) < 1.0 print(f' Usage ≈ Cost Report: {"✅ YES" if usage_matches_cost_report else "❌ NO"}') # Check if analytics is missing default workspace if 'error' not in usage_analysis and 'error' not in analytics_analysis: default_ws_cost = usage_analysis['default_workspace_cost'] expected_analytics = usage_cost - default_ws_cost analytics_matches_expected = abs(analytics_cost - expected_analytics) < 1.0 print(f' Default workspace cost: ${default_ws_cost:.2f}') print(f' Expected Analytics: ${expected_analytics:.2f} (Usage - Default)') print(f' Actual Analytics: ${analytics_cost:.2f}') print(f' Analytics matches: {"✅ YES" if analytics_matches_expected else "❌ NO"}') print() results.append( { 'date': date, 'cost_report': cost_report_cost, 'usage': usage_cost, 'analytics': analytics_cost, 'default_workspace_cost': usage_analysis.get('default_workspace_cost', 0.0) if 'error' not in usage_analysis else 0.0, } ) # Summary print('=' * 100) print('SUMMARY') print('=' * 100) print(f'Days analyzed: {len(results)}') print() # Calculate how many days have analytics missing default workspace missing_default = sum(1 for r in results if abs(r['analytics'] - (r['usage'] - r['default_workspace_cost'])) < 1.0) print(f'Days where Analytics ≈ (Usage - Default workspace): {missing_default}/{len(results)}') print() print('This confirms that Analytics API excludes Default workspace usage (non-Claude Code).') if __name__ == '__main__': main()