"""Agent runner: build a tool-augmented agent and run one eval case.
The agent gets read-only ``read_file`` / ``list_directory`` tools scoped to the
suite's ``file_root``. Depending on the artifact contract it may also get a single
``write_report`` tool (configurable name); in ``final_message`` mode there is no
report tool and the agent's last message is graded instead.
"""
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from langchain.agents import create_agent
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage
from langchain_core.tools import BaseTool, StructuredTool, tool
from skill_eval_runner.config import ArtifactConfig, EvalSuiteConfig, PromptsConfig
from skill_eval_runner.files import expand_files
from skill_eval_runner.suite import EvalCase
from skill_eval_runner.usage import sum_usage
# Key under which the report tool stashes its content for the runner to read back.
_ARTIFACT_KEY = "artifact"
DEFAULT_WITH_SKILL_PROMPT = (
"You are a software engineering assistant. "
"Follow the skill instructions below exactly before doing anything else.\n\n"
"\n{skill}\n\n\n"
"The fixture files are available via the read_file and list_directory tools.\n"
"{artifact}"
)
DEFAULT_WITHOUT_SKILL_PROMPT = (
"You are a software engineering assistant. "
"The fixture files are available via the read_file and list_directory tools.\n"
"{artifact}"
)
@dataclass
class AgentResult:
"""The graded artifact plus token/latency accounting for one agent run."""
response: str
input_tokens: int
output_tokens: int
duration_ms: int
def _artifact_instructions(artifact: ArtifactConfig) -> str:
"""Return the system-prompt sentence describing how to emit the final artifact."""
if artifact.mode == "tool":
return (
f"When you have finished, call {artifact.tool_name} once with the full "
"content. Do not print it as your text response — write it via the tool."
)
return "When you have finished, give the full result as your final message."
def build_system_prompt(
*,
with_skill: bool,
skill_md: str,
artifact: ArtifactConfig,
prompts: PromptsConfig,
) -> str:
"""Render the system prompt for a run, filling the ``{skill}``/``{artifact}`` slots."""
if with_skill:
template = prompts.with_skill or DEFAULT_WITH_SKILL_PROMPT
else:
template = prompts.without_skill or DEFAULT_WITHOUT_SKILL_PROMPT
# Literal replacement (not str.format) so braces in skill text are safe.
return template.replace("{skill}", skill_md).replace(
"{artifact}", _artifact_instructions(artifact)
)
def make_tools(
file_root: Path, artifact: ArtifactConfig, written: dict[str, str]
) -> list[BaseTool]:
"""Build the read-only file tools plus, in ``tool`` mode, the report tool."""
base = file_root.resolve()
@tool
def read_file(path: str) -> str:
"""Read a text file. Path is relative to the fixture root directory."""
target = (base / path).resolve()
if not target.is_relative_to(base):
return "Error: path is outside the allowed directory"
if not target.exists():
return f"Error: file not found: {path}"
if target.is_dir():
return f"Error: {path} is a directory — use list_directory instead"
return target.read_text()
@tool
def list_directory(path: str) -> str:
"""List the contents of a directory. Path is relative to the fixture root."""
target = (base / path).resolve()
if not target.is_relative_to(base):
return "Error: path is outside the allowed directory"
if not target.exists():
return f"Error: path not found: {path}"
lines = []
for entry in sorted(target.iterdir()):
kind = "dir " if entry.is_dir() else "file"
lines.append(f"{kind} {entry.relative_to(base)}")
return "\n".join(lines)
tools: list[BaseTool] = [read_file, list_directory]
if artifact.mode == "tool":
report_tool = _make_report_tool(artifact, written)
tools.append(report_tool)
return tools
def _make_report_tool(artifact: ArtifactConfig, written: dict[str, str]) -> BaseTool:
"""Create the single configurable ``write_report`` tool for ``tool`` mode."""
def _write(content: str) -> str:
written[_ARTIFACT_KEY] = content
return f"{artifact.filename} written ({len(content)} characters)."
return StructuredTool.from_function(
func=_write,
name=artifact.tool_name,
description=artifact.tool_description,
)
def _extract_response(written: dict[str, str], messages: list[Any]) -> str:
"""Prefer the artifact written via the report tool; else the last AI message."""
if _ARTIFACT_KEY in written:
return written[_ARTIFACT_KEY]
for msg in reversed(messages):
if isinstance(msg, AIMessage) and not msg.tool_calls:
content = msg.content
return content if isinstance(content, str) else str(content)
return ""
def run_agent(
case: EvalCase,
*,
with_skill: bool,
model: BaseChatModel,
config: EvalSuiteConfig,
) -> AgentResult:
"""Run one eval case in one configuration and return its artifact + accounting."""
skill_md = config.skill_file.read_text() if with_skill else ""
system = build_system_prompt(
with_skill=with_skill,
skill_md=skill_md,
artifact=config.artifact,
prompts=config.prompts,
)
written: dict[str, str] = {}
tools = make_tools(config.file_root, config.artifact, written)
agent = create_agent(model, tools, system_prompt=system)
user_content = case.prompt
if case.files:
injected = expand_files(case.files, config.file_root)
if injected:
sections = "\n\n".join(
f"### {path}\n```\n{content}\n```" for path, content in injected
)
user_content = f"{user_content}\n\n\n{sections}\n"
start = time.time()
result = agent.invoke(
{"messages": [HumanMessage(content=user_content)]},
config={"recursion_limit": 50},
)
duration_ms = int((time.time() - start) * 1000)
messages = list(result["messages"])
response = _extract_response(written, messages)
input_tokens, output_tokens = sum_usage(messages)
return AgentResult(
response=response,
input_tokens=input_tokens,
output_tokens=output_tokens,
duration_ms=duration_ms,
)