{
  "skill_name": "docker-image-scan",
  "evals": [
    {
      "id": 1,
      "prompt": "I need to scan the ows-permissions service container image for security vulnerabilities. The project is at {skill_dir}/evals/files/eval1-permissions — can you figure out which service to build and what command to run?",
      "expected_output": "The agent runs the build target identification script, finds that 'ows-permissions-service' has target: deploy (NOT the service named 'deploy', which is a Flyway migration runner), and plans to build with `docker compose build ows-permissions-service`.",
      "files": [
        "evals/files/eval1-permissions/docker-compose.yml",
        "evals/files/eval1-permissions/Dockerfile"
      ],
      "expectations": [
        "The agent runs the build target identification Python script (creates a venv, installs pyyaml, runs the detection logic)",
        "The agent identifies PATH=compose SERVICE=ows-permissions-service (not the service named 'deploy')",
        "The agent plans to build using `docker compose build ows-permissions-service`, not `docker compose build deploy`",
        "The agent does not confuse the Flyway 'deploy' service with the application build target"
      ]
    },
    {
      "id": 2,
      "prompt": "Please scan ows-assets for vulns. The project is at {skill_dir}/evals/files/eval7-secrets. Not sure how to get the npm token it needs",
      "expected_output": "After identifying the required GITHUB_NPM_TOKEN secret from the docker-compose.yml, the agent asks the user HOW to obtain the token — offering both options (run a command/script, or provide the value directly). It does NOT simply ask 'please provide the value of GITHUB_NPM_TOKEN'.",
      "files": [
        "evals/files/eval7-secrets/docker-compose.yml"
      ],
      "expectations": [
        "The agent reads the docker-compose.yml and identifies GITHUB_NPM_TOKEN as a required secret",
        "The agent asks the user how to obtain GITHUB_NPM_TOKEN rather than only asking for the raw value",
        "The agent explicitly offers both paths: running a command/script OR providing the value directly",
        "The agent does not export a placeholder or make up a token value"
      ]
    },
    {
      "id": 3,
      "prompt": "I ran a container image scan and got these results back. Can you look at this output and tell me how to fix each finding? Output is at {skill_dir}/evals/files/eval3-npm-routing/scan-output.txt",
      "expected_output": "The agent reads the scan output, recognizes that CVE-2024-29041 (express) is in the app's /var/app/node_modules/ (fixable via yarn upgrade) and CVE-2024-26485 (brace-expansion) is in /usr/local/lib/node_modules/npm/node_modules/ (bundled in npm, requires Node.js base image upgrade). It gives different remediation paths for each.",
      "files": [
        "evals/files/eval3-npm-routing/scan-output.txt"
      ],
      "assertions": [
        "The agent recommends bumping the express constraint and running yarn upgrade (or equivalent) for CVE-2024-29041 at /var/app/node_modules/",
        "The agent recommends upgrading the Node.js base image for CVE-2024-26485 at /usr/local/lib/node_modules/npm/node_modules/",
        "The agent does NOT recommend yarn upgrade or package.json changes for the brace-expansion finding",
        "The agent does NOT recommend a base-image upgrade as the sole fix for the express finding",
        "The agent references theorchard/docker-parent-images when recommending a base image upgrade"
      ]
    }
  ]
}