"""Main application.""" from typing import Any from urllib.parse import urlparse import sentry_sdk from datadog_api_client import ApiClient, Configuration from datadog_api_client.exceptions import ApiException from datadog_api_client.v1.api.metrics_api import MetricsApi from datadog_api_client.v2.model.state import State from lambdacommon.common_config import logger from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration import backoff import config from common.datadog_utils import ( list_frontends_with_details, list_services_with_details, update_scorecard_non_service_outcomes, update_scorecard_service_outcomes_batch, ) if config.SENTRY_DSN: sentry_sdk.init( dsn=config.SENTRY_DSN, environment=config.ENVIRONMENT, integrations=[AwsLambdaIntegration(timeout_warning=True)], ) def handler(event: dict[str, Any], context: Any) -> None: """Lambda handler.""" if not config.DD_API_KEY or not config.DD_APP_KEY: logger.error("Missing DD_API_KEY or DD_APP_KEY") raise ValueError("Missing DD_API_KEY or DD_APP_KEY") logger.info("Fetching services from catalog...") services = list_services_with_details(config.DD_API_KEY, config.DD_APP_KEY) logger.info(f"Found {len(services)} services.") logger.info("Fetching frontends from catalog...") frontends = list_frontends_with_details(config.DD_API_KEY, config.DD_APP_KEY) logger.info(f"Found {len(frontends)} frontends.") # Build mapping of (account, repo) -> list of entity info repo_to_entities: dict[tuple[str, str], list[dict[str, Any]]] = {} entities_without_repo: list[dict[str, Any]] = [] for entity in services + frontends: entity_name = entity["name"] entity_kind = entity["kind"] repo_url = _extract_repository_url(entity) if not repo_url: entities_without_repo.append( {"name": entity_name, "kind": entity_kind} ) continue parsed = _parse_github_url(repo_url) if not parsed: entities_without_repo.append( {"name": entity_name, "kind": entity_kind} ) continue account, repo = parsed key = (account.lower(), repo.lower()) if key not in repo_to_entities: repo_to_entities[key] = [] repo_to_entities[key].append({"name": entity_name, "kind": entity_kind}) logger.info( f"Found {len(repo_to_entities)} unique repositories across " f"{len(services) + len(frontends) - len(entities_without_repo)} " f"entities." ) # Query github.secret_scan_alert metric for repos with open alerts repos_with_alerts = _query_secret_scan_alerts( config.DD_API_KEY, config.DD_APP_KEY ) logger.info( f"Found {len(repos_with_alerts)} repositories with open " f"secret scan alerts." ) # Evaluate outcomes service_outcomes: list[dict[str, Any]] = [] non_service_outcomes: list[dict[str, Any]] = [] # Entities without a repository URL get a SKIP for entity in entities_without_repo: _record_outcome( service_outcomes, non_service_outcomes, entity["kind"] == "service", entity["name"], entity["kind"], config.SCORECARD_RULE_ID, State.SKIP, "No repositoryURL defined in codeLocations", ) # Entities with a repository: check against alerts for (account, repo), entities in repo_to_entities.items(): key = (account.lower(), repo.lower()) has_alerts = key in repos_with_alerts for entity in entities: if has_alerts: alert_count = repos_with_alerts[key] _record_outcome( service_outcomes, non_service_outcomes, entity["kind"] == "service", entity["name"], entity["kind"], config.SCORECARD_RULE_ID, State.FAIL, f"Repository {account}/{repo} has {alert_count} " f"open secret scan alert(s)", ) else: _record_outcome( service_outcomes, non_service_outcomes, entity["kind"] == "service", entity["name"], entity["kind"], config.SCORECARD_RULE_ID, State.PASS, f"No open secret scan alerts for {account}/{repo}", ) logger.info( f"Updating scorecard outcomes: {len(service_outcomes)} services, " f"{len(non_service_outcomes)} non-service entities..." ) update_scorecard_service_outcomes_batch( config.DD_API_KEY, config.DD_APP_KEY, service_outcomes ) update_scorecard_non_service_outcomes( config.DD_API_KEY, config.DD_APP_KEY, non_service_outcomes ) logger.info( f"Scorecard outcomes updated for {len(service_outcomes)} services " f"and {len(non_service_outcomes)} non-service entities." ) def _extract_repository_url(entity: dict[str, Any]) -> str | None: """Extract repositoryURL from entity's codeLocations (schema v3). In v3 schema, codeLocations is at datadog.codeLocations (top-level). """ raw_schema = entity.get("raw_schema", {}) datadog_section = raw_schema.get("datadog", {}) code_locations = datadog_section.get("code_locations") or datadog_section.get( "codeLocations", [] ) if code_locations and isinstance(code_locations, list): for location in code_locations: repo_url = location.get("repository_url") or location.get( "repositoryURL" ) if repo_url: return repo_url return None def _parse_github_url(url: str | None) -> tuple[str, str] | None: """Parse a GitHub URL into (account, repo) tuple. Expects format: https://github.com/${account}/${repo}.git """ if url is None: return None try: parsed = urlparse(url) if parsed.hostname != "github.com": return None parts = parsed.path.strip("/").split("/") if len(parts) < 2: return None account = parts[0] repo = parts[1] if repo.endswith(".git"): repo = repo[:-4] return (account, repo) except Exception: return None @backoff.on_exception( backoff.expo, ApiException, max_tries=5, giveup=lambda e: isinstance(e, ApiException) and e.status != 429, ) def _query_secret_scan_alerts( api_key: str, app_key: str ) -> dict[tuple[str, str], int]: """Query github.secret_scan_alert metric with state:open. Returns a dict of (account, repo) -> alert count. """ import time configuration = Configuration() configuration.api_key["apiKeyAuth"] = api_key configuration.api_key["appKeyAuth"] = app_key results: dict[tuple[str, str], int] = {} with ApiClient(configuration) as api_client: api_instance = MetricsApi(api_client) now = int(time.time()) from_ts = now - config.METRIC_QUERY_PERIOD_SECONDS response = api_instance.query_metrics( _from=from_ts, to=now, query="sum:github.secret_scan_alert{state:open} by {repo,account}", ) if response.series: for series in response.series: scope = series.get("scope", "") tag_values = {} # Parse scope like "account:foo,repo:bar" for part in scope.split(","): if ":" in part: k, v = part.split(":", 1) tag_values[k.strip()] = v.strip() account = tag_values.get("account", "") repo = tag_values.get("repo", "") if account and repo: # Get the last point value as the count pointlist = series.get("pointlist", []) if pointlist: value = int(pointlist[-1].value[1]) if value > 0: results[(account.lower(), repo.lower())] = value return results def _record_outcome( service_outcomes: list[dict[str, Any]], non_service_outcomes: list[dict[str, Any]], is_service: bool, entity_name: str, entity_kind: str, rule_id: str, state: State, remarks: str, ) -> None: """Append an outcome to the appropriate list based on entity kind.""" outcome = {"rule_id": rule_id, "state": state, "remarks": remarks} if is_service: service_outcomes.append({**outcome, "service_name": entity_name}) else: entity_ref = f"{entity_kind}:{entity_name}" non_service_outcomes.append({**outcome, "entity_reference": entity_ref})