{
  "skill_name": "python-service-scan-authz-baseline",
  "evals": [
    {
      "id": "enforced-access-rules",
      "name": "Fully enforced access_rules middleware (Flask)",
      "service": "ows-catalog",
      "prompt": "Scan the Python service at evals/files/ows-catalog and generate a PP migration plan.",
      "expected_output": "A phased migration plan that identifies access_rules.yml enforcement is fully active (verify_access=True, access_log_only=False), lists all endpoints with their rule coverage, flags /catalog/internal/sync as not explicitly covered by a named rule (only catch-all), identifies verify_grass_access calls on GET /catalog/<int> and DELETE /catalog/<int>/items/<int>, includes Phase 1 baseline integration test scaffolding, a Phase 2 shadow step (MigrationAuthorizationBackend, always-allow + would-deny metrics), and a Phase 3 enforce step using PdpAuthorizationBackend.",
      "files": [
        "evals/files/ows-catalog/pyproject.toml",
        "evals/files/ows-catalog/catalog/*"
      ],
      "assertions": [
        "Output states that verify_access=True and access_log_only=False, meaning enforcement is fully active",
        "Output identifies /hello/ as an excluded path that bypasses access_rules enforcement",
        "Output does NOT include a Detailed Findings section or Phase 1/2/3 migration steps for /hello/",
        "Output lists verify_grass_access calls on GET /catalog/<int> and DELETE /catalog/<int>/items/<int>",
        "Output notes that /catalog/internal/sync is not explicitly named in access_rules.yml (only covered by catch-all)",
        "Output assigns a migration priority (e.g. High/Medium/Low) to each endpoint or endpoint group",
        "Phase 1 section includes at least one example integration test or test scaffold",
        "Phase 2 (shadow) or Phase 3 (enforce) section references PdpAuthorizationBackend, python-pdp-sdk, or is_authorized",
        "Output classifies GET /catalog/<int:catalog_id> as having two postures: Access rules (enforced) from the middleware and Grass only (required) from the verify_grass_access call checking vendor_id and subaccount_id",
        "Output classifies DELETE /catalog/<int:catalog_id>/items/<int:item_id> as having two postures: Access rules (enforced) from the middleware and Grass only (required) from the verify_grass_access call",
        "Output classifies POST /catalog/internal/sync as Access rules (enforced) via catch-all rule only — no explicit rule for that path and no handler-level grass check",
        "The Endpoint Auth Posture table columns appear in the order: Posture, Endpoint, Handler, File, Evidence, Confidence — Posture is the first column"
      ]
    },
    {
      "id": "logging-only-enforcement-disabled",
      "name": "Logging-only mode with enforcement disabled (Flask)",
      "service": "ows-releases",
      "prompt": "Scan the Python service at evals/files/ows-releases and generate a PP migration plan.",
      "expected_output": "A phased migration plan that prominently flags that verify_access=False means the access_rules middleware is NOT enforcing rules — requests are never rejected by it. The plan should note that access_rules.yml exists and is registered via set_rules_validator, but enforcement is disabled. It should treat all endpoints as effectively unprotected at the middleware level (except the one inline verify_grass_access call on GET /track/<int>), and scaffold no-fallback PP checks (Template C) for the unprotected endpoints since there is no legacy enforcement to preserve.",
      "files": [
        "evals/files/ows-releases/pyproject.toml",
        "evals/files/ows-releases/releases/*"
      ],
      "assertions": [
        "Output explicitly states that verify_access=False means middleware enforcement is disabled",
        "Output identifies /hello/ as an excluded path that bypasses access_rules enforcement",
        "Output does NOT include a Detailed Findings section or Phase 1/2/3 migration steps for /hello/",
        "Output notes that access_rules.yml exists via set_rules_validator but is not enforced",
        "Output notes that access_log_only is controlled by a config variable (ONLY_LOG_ACCESS_ERRORS)",
        "Output identifies the inline verify_grass_access call in GET /track/<int:track_id> as the only active enforcement",
        "Output does NOT incorrectly claim that access_rules.yml is enforcing access",
        "Phase 1 section emphasizes capturing current (permissive) behavior before adding enforcement",
        "Phase 3 (enforce) section scaffolds a no-fallback PP check (Template C or equivalent) for the unprotected endpoints, given that there is no active enforcement layer to fall back to",
        "Output classifies GET /release/<int:release_id>/tracks as Access rules (disabled) — rules defined in access_rules.yml but verify_access=False means no runtime enforcement and no handler-level auth check",
        "Output classifies GET /track/<int:track_id> as Grass only (required) — the only endpoint with an active inline verify_grass_access call; access rules are not enforced at the middleware level",
        "The Endpoint Auth Posture table columns appear in the order: Posture, Endpoint, Handler, File, Evidence, Confidence — Posture is the first column"
      ]
    },
    {
      "id": "no-legacy-auth-pp-already-wired",
      "name": "No legacy auth — PP backend already wired (Flask)",
      "service": "ows-media",
      "prompt": "Scan the Python service at evals/files/ows-media and generate a PP migration plan.",
      "expected_output": "A migration plan that recognizes ows-media has no access_rules.yml, no verify_grass_access calls, and no legacy owsrequest access enforcement. It should identify that PdpAuthorizationBackend is already set up (factory defined in api.py, instance created in handlers.py). The plan should recommend focusing on adding endpoint-level is_authorized or is_authorized_many calls to the unprotected endpoints rather than setting up middleware from scratch. It should flag /internal/v2/asset/<filename>/owner as a sensitive endpoint with no auth check.",
      "files": [
        "evals/files/ows-media/pyproject.toml",
        "evals/files/ows-media/media/*"
      ],
      "assertions": [
        "Output identifies that there is no access_rules.yml and no verify_grass_access in the service",
        "Output correctly conveys there are no excluded paths (either by stating 0 excluded paths or by noting no access_rules.yml / verify_access is present)",
        "Output recognizes that PdpAuthorizationBackend is already set up in the service (defined in api.py, instantiated in handlers.py)",
        "Output does not recommend adding access_rules.yml or flask_request verify_access setup",
        "Output flags /internal/v2/asset/<filename>/owner as an unprotected internal endpoint",
        "Output identifies /v2/asset (POST) as having a partial JWT identity check that should be migrated to PP",
        "Phase 2 section focuses on adding is_authorized calls at the endpoint level",
        "Output does not claim legacy grass-based auth is present",
        "Output classifies POST /v2/assets/upload as No auth — no PP check, no grass check, relies on upstream gateway for protection",
        "Output classifies POST /v2/asset as Needs human review — has a partial JWT identity check via is_jwt_identity_authorized but no full PP authorization check",
        "Output classifies GET /internal/v2/asset/<filename>/owner as No auth — internal endpoint with no authentication or authorization check",
        "The Endpoint Auth Posture table columns appear in the order: Posture, Endpoint, Handler, File, Evidence, Confidence — Posture is the first column",
        "Phase 2 migration plan instructs creating a NEW migration_auth_backend singleton (e.g. migration_auth_backend = MigrationAuthorizationBackend(...)) rather than overriding or replacing the existing authorization_backend singleton"
      ]
    },
    {
      "id": "conditional-enforcement-mixed-patterns",
      "name": "Conditional enforcement with mixed auth patterns (Flask)",
      "service": "ows-content",
      "prompt": "Scan the Python service at evals/files/ows-content and generate a PP migration plan.",
      "expected_output": "A migration plan that flags that access_log_only is controlled by config.ONLY_LOG_ACCESS_ERRORS — a config variable that may be True in some environments (logging only) and False in others (full enforcement). The plan should call this out as a risk: enforcement state is environment-dependent. It should identify the only_for_identity decorator on /content/internal/reprocess and /content/internal/bulk-status as JWT-based auth (not GRASS/access_rules). It should list /content/audio/<int>/submit as an endpoint not explicitly named in access_rules.yml (only catch-all). It should note /public/meta-type as intentionally public.",
      "files": [
        "evals/files/ows-content/pyproject.toml",
        "evals/files/ows-content/content/*"
      ],
      "assertions": [
        "Output flags that access_log_only=config.ONLY_LOG_ACCESS_ERRORS means enforcement is environment-dependent",
        "Output identifies /hello/ and /public/meta-type as excluded paths that bypass access_rules enforcement",
        "Output does NOT include Detailed Findings sections for /hello/ or /public/meta-type",
        "Output identifies verify_grass_headers call on PUT /content/audio/<int> as an inline grass check",
        "Output identifies only_for_identity decorator on /content/internal/reprocess and /content/internal/bulk-status as JWT identity-based auth",
        "Output notes that /content/audio/<int>/submit is not explicitly named in access_rules.yml (only covered by catch-all)",
        "Output notes /public/meta-type as a public/excluded endpoint",
        "Phase 1 section includes integration test scaffolding that accounts for both enforced and log-only environments",
        "Output includes migration guidance or explicit human-review notes for the only_for_identity internal endpoints (/content/internal/reprocess and /content/internal/bulk-status)",
        "Output classifies PUT /content/audio/<int:content_id> as having two postures: Access rules (enforced, conditional on ONLY_LOG_ACCESS_ERRORS) from the middleware and Grass only (permissive) from the verify_grass_headers call which does not pass required=True",
        "Output classifies POST /content/audio/<int:content_id>/submit as covered by the catch-all rule in access_rules.yml, with enforcement conditional on ONLY_LOG_ACCESS_ERRORS (enforced in production, log-only in other environments)",
        "Output classifies POST /content/internal/reprocess and POST /content/internal/bulk-status as Needs human review due to the only_for_identity decorator",
        "The Endpoint Auth Posture table columns appear in the order: Posture, Endpoint, Handler, File, Evidence, Confidence — Posture is the first column"
      ]
    }
  ,
    {
      "id": "fastapi-mixed-migration-states",
      "name": "FastAPI service with mixed PP migration states",
      "service": "ows-product-staging",
      "prompt": "Scan the Python service at evals/files/ows-product-staging and generate a PP migration plan.",
      "expected_output": "A migration plan that recognizes ows-product-staging is a FastAPI service with no access_rules.yml or flask_request.setup middleware. It should identify GET /hello/ as excluded from JWT auth via JWTAuthenticationMiddleware exclude_paths and omit it from Detailed Findings. It should note that all other endpoints require a valid JWT. It should classify: POST /bulk-session as PP + legacy fallback (inline is_authorized_for_tenant + check_vendor_access_for_profiles); GET /bulk-session/{bulk_session_id} as PP + legacy fallback (via bulk_session_logic.assert_authorization); GET /bulk-session-ingestion/{bulk_session_ingestion_id}/products as legacy only / needs PP migration (assert_access calls only check_vendor_access_for_profiles); POST /bulk-session/products/dataloader as no resource-level auth. It should recommend migrating assert_access callers to assert_authorization as the primary migration path.",
      "files": [
        "evals/files/ows-product-staging/pyproject.toml",
        "evals/files/ows-product-staging/product_staging/api/*",
        "evals/files/ows-product-staging/product_staging/api/routers/*",
        "evals/files/ows-product-staging/product_staging/logic/*",
        "evals/files/ows-product-staging/tests/integration/conftest.py"
      ],
      "assertions": [
        "Output identifies this as a FastAPI service and notes there is no access_rules.yml or flask_request.setup middleware",
        "Output identifies GET /hello/ as excluded from JWT authentication via JWTAuthenticationMiddleware exclude_paths in main.py",
        "Output does NOT include a Detailed Findings section or Phase 1/2/3 migration steps for GET /hello/",
        "Output notes that JWT authentication is always required for all non-excluded endpoints",
        "Output classifies GET /bulk-session/{bulk_session_id} as PP + legacy fallback (bulk_session_logic.assert_authorization checks is_authorized_for_tenant first, then falls back to check_vendor_access_for_profiles)",
        "Output classifies POST /bulk-session as PP + legacy fallback (inline is_authorized_for_tenant check in handler, falls back to check_vendor_access_for_profiles)",
        "Output classifies GET /bulk-session-ingestion/{bulk_session_ingestion_id}/products as legacy only / needing PP migration (assert_access delegates to bulk_session_logic.assert_access which only calls check_vendor_access_for_profiles with no PP check)",
        "Output classifies /bulk-session/products/dataloader as having no resource-level auth (JWT required but no vendor/tenant ownership verified)",
        "Output recommends migrating assert_access callers to assert_authorization as the primary migration path (not rewriting from scratch)",
        "Phase 1 section references the existing conftest.py personas (client, subaccount_client, no_access_client) in its test cases",
        "Phase 1 section notes that <PP_role_name>_client and <PP_role_name>_subaccount_client fixtures must be added to conftest.py before the PP enforcement (Phase 3) test cases can be written",
        "Output does NOT reference verify_grass_access, access_rules.yml enforcement, or flask_request.setup as applicable to this service",
        "The Endpoint Auth Posture table columns appear in the order: Posture, Endpoint, Handler, File, Evidence, Confidence — Posture is the first column"
      ]
    }
  ,
    {
      "id": "standalone-access-rules",
      "name": "verify_access=False with verify_rules_access_standalone in every handler (Flask)",
      "service": "ows-track-standalone",
      "prompt": "Scan the Python service at evals/files/ows-track-standalone and generate a PP migration plan.",
      "expected_output": "A migration plan that recognizes the service uses standalone access-rules mode: flask_request.setup has verify_access=False, but every non-health endpoint calls verify_rules_access_standalone() directly in the handler. The plan should classify all non-excluded endpoints as Access rules (enforced) in standalone mode — NOT as Access rules (disabled). It should explain that standalone mode is the required prerequisite for adding pdp-sdk is_authorized() calls, because middleware-level enforcement would deny requests before the handler can reach pdp-sdk. The /hello/ path should be excluded from findings.",
      "files": [
        "evals/files/ows-track-standalone/pyproject.toml",
        "evals/files/ows-track-standalone/track/*"
      ],
      "assertions": [
        "Output does NOT classify any endpoint as Access rules (disabled) — the presence of verify_rules_access_standalone in every handler means rules are enforced in standalone mode",
        "Output recognizes that verify_access=False combined with verify_rules_access_standalone() calls in every handler means access rules are enforced in standalone (per-handler) mode",
        "Output classifies GET /track/<int>, PUT /track/<int>, DELETE /track/<int>, and POST /track/search as Access rules (enforced) or equivalent standalone-mode posture",
        "Output notes that standalone mode is required (or is the prerequisite) for adding pdp-sdk is_authorized() calls, because middleware-level enforcement would deny requests before the handler can call pdp-sdk",
        "Output does NOT include a Detailed Findings section or Phase 1/2/3 migration steps for /hello/",
        "The Endpoint Auth Posture table columns appear in the order: Posture, Endpoint, Handler, File, Evidence, Confidence — Posture is the first column"
      ]
    }
  ]
}