"""Combine Auth0 users, org members, and Neo4j identities into combined user data.""" import argparse import gzip import json import logging import sys from dataclasses import asdict from datetime import datetime from pathlib import Path from src.readers import combine_user_data # Configure logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) def format_file_size(size_bytes: int) -> str: """Format bytes as human-readable string (KB or MB).""" if size_bytes < 1024 * 1024: return f'{size_bytes / 1024:.1f} KB' return f'{size_bytes / (1024 * 1024):.1f} MB' def list_files(pattern: str, directory: str = 'data/input') -> list[tuple[Path, datetime, int]]: """Find files matching pattern with modification date and size.""" input_dir = Path(directory) if not input_dir.exists(): return [] files: list[tuple[Path, datetime, int]] = [] for f in input_dir.glob(pattern): stat = f.stat() files.append((f, datetime.fromtimestamp(stat.st_mtime), stat.st_size)) return sorted(files, key=lambda x: x[1], reverse=True) def prompt_file_selection(files: list[tuple[Path, datetime, int]], label: str) -> Path: """Display numbered menu of files and return selected path.""" print(f'\nAvailable {label} files:') for i, (path, mtime, size) in enumerate(files, 1): date_str = mtime.strftime('%Y-%m-%d %H:%M:%S') print(f' {i}) {path.name} ({date_str}, {format_file_size(size)})') while True: choice = input(f'\nSelect file (1-{len(files)}): ').strip() if choice.isdigit() and 1 <= int(choice) <= len(files): return files[int(choice) - 1][0] print(f'Invalid choice. Enter a number between 1 and {len(files)}.') def main() -> None: """Main entry point.""" parser = argparse.ArgumentParser(description='Combine Auth0 users, org members, and Neo4j identities') parser.add_argument( '--auth0-users', help='Path to Auth0 users JSON Lines file (auto-selects latest job_*.json if omitted)', ) parser.add_argument( '--org-members', help='Path to org members JSON file (auto-selects latest org_members_*.json if omitted)', ) parser.add_argument( '--neo4j-identities', help='Path to Neo4j identities JSON file (auto-selects latest neo4j_identities_*.json if omitted)', ) parser.add_argument( '--vend-contacts', help='Path to vend_contact report JSON file (auto-selects latest vend_contact_report_*.json if omitted)', ) parser.add_argument( '--output', help='Path to output JSON file (default: data/output/combined_users_TIMESTAMP.json.gz)', ) args = parser.parse_args() # Resolve input files - prompt for each if not provided if not args.auth0_users: files = list_files('job_*.json') if not files: logger.error('No job_*.json files found in data/input/') sys.exit(1) args.auth0_users = str(prompt_file_selection(files, 'Auth0 users')) if not args.org_members: files = list_files('org_members_*.json') if not files: logger.error('No org_members_*.json files found in data/input/') sys.exit(1) args.org_members = str(prompt_file_selection(files, 'org members')) if not args.neo4j_identities: files = list_files('neo4j_identities_*.json') if not files: logger.error('No neo4j_identities_*.json files found in data/input/') sys.exit(1) args.neo4j_identities = str(prompt_file_selection(files, 'Neo4j identities')) if not args.vend_contacts: files = list_files('vend_contact_report_*.json') if files: args.vend_contacts = str(prompt_file_selection(files, 'vend_contact report')) else: logger.warning('No vend_contact_report_*.json files found in data/input/ — skipping MySQL data') # Set default output path if not args.output: timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') args.output = f'data/output/combined_users_{timestamp}.json.gz' # Log input files logger.info('Input files:') logger.info(f' Auth0 users: {args.auth0_users}') logger.info(f' Org members: {args.org_members}') logger.info(f' Neo4j identities: {args.neo4j_identities}') logger.info(f' Vend contact rpt: {args.vend_contacts or "(none)"}') # Combine data logger.info('Combining user data...') combined_users = combine_user_data( auth0_users_file=args.auth0_users, org_members_file=args.org_members, neo4j_identities_file=args.neo4j_identities, vend_contact_report_file=args.vend_contacts, ) # Write output (gzip compressed) logger.info(f'Writing {len(combined_users)} combined users to {args.output}') Path(args.output).parent.mkdir(parents=True, exist_ok=True) with gzip.open(args.output, 'wt', encoding='utf-8') as f: json.dump([asdict(user) for user in combined_users], f, indent=2) # Summary users_with_orgs = sum(1 for u in combined_users if u.auth0_data.organizations) users_with_neo4j = sum(1 for u in combined_users if u.neo4j_data is not None) users_with_mysql = sum(1 for u in combined_users if u.mysql_data is not None) logger.info('=' * 60) logger.info('Complete!') logger.info(f'Total users: {len(combined_users)}') logger.info(f'With org memberships: {users_with_orgs}') logger.info(f'With Neo4j identity: {users_with_neo4j}') logger.info(f'With vend_contact rpt:{users_with_mysql}') logger.info(f'Output: {args.output}') logger.info('=' * 60) if __name__ == '__main__': main()