"""Datadog API client for service discovery and endpoint detection. Uses the Software Catalog API to discover ows-* services and the Scalar Query API to fetch resource names that have received traffic. Auth: DD_API_KEY + DD_APP_KEY environment variables (loaded from .env). """ import os import time from dataclasses import dataclass from pathlib import Path import httpx from dotenv import load_dotenv load_dotenv(Path(__file__).resolve().parents[1] / '.env') DD_BASE_URL = 'https://api.datadoghq.com' CATALOG_URL = f'{DD_BASE_URL}/api/v2/services/definitions' SCALAR_URL = f'{DD_BASE_URL}/api/v2/query/scalar' @dataclass class ServiceInfo: name: str repo_url: str team: str def _get_headers() -> dict[str, str]: api_key = os.environ.get('DD_API_KEY', '') app_key = os.environ.get('DD_APP_KEY', '') if not api_key or not app_key: raise RuntimeError('DD_API_KEY and DD_APP_KEY environment variables are required') return { 'DD-API-KEY': api_key, 'DD-APPLICATION-KEY': app_key, 'Content-Type': 'application/json', } def _discover_ows_service_names(env: str = 'prod') -> list[str]: """Discover ows-* service names via a Datadog metrics query (fast, no pagination).""" headers = _get_headers() now_ms = int(time.time() * 1000) from_ms = now_ms - (30 * 86400 * 1000) # 30-day window for discovery payload = { 'data': { 'type': 'scalar_request', 'attributes': { 'formulas': [{'formula': 'hits', 'alias': 'Requests'}], 'queries': [ { 'data_source': 'metrics', 'name': 'hits', 'query': f'sum:trace.flask.request.hits{{env:{env},service:ows-*}} by {{service}}.as_count()', 'aggregator': 'sum', }, ], 'from': from_ms, 'to': now_ms, }, }, } with httpx.Client(timeout=30) as client: resp = client.post(SCALAR_URL, headers=headers, json=payload) resp.raise_for_status() data = resp.json() names: set[str] = set() for col in data.get('data', {}).get('attributes', {}).get('columns', []): if col.get('type') == 'group': for v in col.get('values', []): if v and v[0]: names.add(v[0]) return sorted(names) def _parse_catalog_entry(data: dict) -> tuple[str, str]: """Extract (repo_url, team) from a catalog API response item.""" attrs = data.get('attributes', {}) schema = attrs.get('schema', {}) repo_url = '' for link in schema.get('links', []): if link.get('type') == 'repo': repo_url = link.get('url', '') break if not repo_url: repo_url = schema.get('extensions', {}).get('repo', '') team = ( schema.get('owner', '') or schema.get('team', '') or schema.get('info', {}).get('team', '') ) return repo_url, team def list_ows_services(env: str = 'prod') -> list[ServiceInfo]: """Fetch all ows-* services by discovering names via metrics, then enriching from the catalog.""" names = _discover_ows_service_names(env) services: list[ServiceInfo] = [] headers = _get_headers() with httpx.Client(timeout=15) as client: for name in names: repo_url, team = '', '' try: resp = client.get(f'{CATALOG_URL}/{name}', headers=headers) if resp.status_code == 200: repo_url, team = _parse_catalog_entry(resp.json().get('data', {})) except httpx.HTTPError: pass services.append(ServiceInfo(name=name, repo_url=repo_url, team=team)) return services def get_service_info(service_name: str) -> ServiceInfo: """Look up a single service from the Datadog Software Catalog. Falls back to empty repo_url/team if the catalog call fails. """ headers = _get_headers() try: with httpx.Client(timeout=15) as client: resp = client.get(f'{CATALOG_URL}/{service_name}', headers=headers) if resp.status_code == 200: repo_url, team = _parse_catalog_entry(resp.json().get('data', {})) return ServiceInfo(name=service_name, repo_url=repo_url, team=team) except httpx.HTTPError: pass return ServiceInfo(name=service_name, repo_url='', team='') def get_resource_stats( service: str, env: str = 'prod', framework: str = 'flask', lookback_days: int = 365, ) -> tuple[dict, set[str]]: """Fetch resource stats from Datadog. Returns (raw_response_dict, normalized_resource_names_set). The raw response is saved for offline debugging. """ headers = _get_headers() now_ms = int(time.time() * 1000) from_ms = now_ms - (lookback_days * 86400 * 1000) metric_prefix = 'trace.fastapi.request' if framework == 'fastapi' else 'trace.flask.request' payload = { 'data': { 'type': 'scalar_request', 'attributes': { 'formulas': [ {'formula': 'hits', 'alias': 'Requests'}, ], 'queries': [ { 'data_source': 'metrics', 'name': 'hits', 'query': f'sum:{metric_prefix}.hits{{service:{service},env:{env}}} by {{resource_name}}.as_count()', 'aggregator': 'sum', }, ], 'from': from_ms, 'to': now_ms, }, } } with httpx.Client(timeout=60) as client: resp = client.post(SCALAR_URL, headers=headers, json=payload) resp.raise_for_status() raw_data = resp.json() names = _parse_resource_names(raw_data) return raw_data, names def get_resource_names( service: str, env: str = 'prod', lookback_days: int = 365, framework: str = 'flask', ) -> set[str]: """Fetch resource names that have received traffic from Datadog. Returns a set of normalized (lowercased) resource name strings. """ _, names = get_resource_stats(service, env, framework=framework, lookback_days=lookback_days) return names def _parse_resource_names(data: dict) -> set[str]: """Extract normalized resource names from the Datadog scalar query response. The group column values are lists like ["get_/users/identity/_id"]. """ names: set[str] = set() columns = data.get('data', {}).get('attributes', {}).get('columns', []) for col in columns: if col.get('type') == 'group': for v in col.get('values', []): name = v[0] if v else '' if name: names.add(name.lower().strip()) return names