Skip to content

Commit 80b9095

Browse files
DavidKoleczekDavidKoleczek
authored
Terminal Bench Benchmarks and Analysis (#42)
* Benchmarks * update terminal-bench scripts for improved report generation and dashboard creation * update readme and revert dependency changes * update line ending
1 parent b5d02d1 commit 80b9095

File tree

6 files changed

+1584
-0
lines changed

6 files changed

+1584
-0
lines changed

‎tests/terminal_bench/README.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Amplifier Benchmarks
2+
3+
## Running Terminal-Bench
4+
5+
Run the terminal-bench benchmark suite with amplifier or baseline agent:
6+
7+
```bash
8+
uv run --with terminal-bench tests/terminal_bench/run_terminal_bench.py --agent baseline
9+
```
10+
11+
## Analyzing Benchmark Results
12+
13+
Generate failure analysis reports for a terminal-bench run:
14+
15+
```bash
16+
uv run tests/terminal_bench/generate_benchmark_report.py --run-dir "ai_working/tmp/2025-10-14__09-39-16"
17+
```
18+
19+
20+
## Common Issues
21+
22+
### Docker Compose Network Pool Exhaustion
23+
24+
**Error Message:**
25+
```
26+
Command '['docker', 'compose', '-p', 'task-name', '-f', '/path/to/docker-compose.yaml', 'up', '-d']' returned non-zero exit status 1.
27+
```
28+
29+
**Root Cause:**
30+
This error can occur when Docker runs out of available network address pools. Terminal-bench creates a new Docker network for each task run, and if these aren't cleaned up properly, Docker eventually exhausts its predefined address pools with the error:
31+
```
32+
failed to create network: Error response from daemon: all predefined address pools have been fully subnetted
33+
```
34+
35+
**Solution:**
36+
Clean up unused Docker networks:
37+
```bash
38+
# Remove all unused networks
39+
docker network prune -f
40+
41+
# Check how many networks exist (should be < 30)
42+
docker network ls | wc -l
43+
```

‎tests/terminal_bench/custom_agents.py‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
"""Extends terminal-bench's AbstractInstalledAgent to create a custom agent that uses amplifier."""
2+
3+
import os
4+
import shlex
5+
import tempfile
6+
from pathlib import Path
7+
8+
from terminal_bench.agents.agent_name import AgentName
9+
from terminal_bench.agents.installed_agents.abstract_installed_agent import AbstractInstalledAgent
10+
from terminal_bench.terminal.models import TerminalCommand
11+
12+
13+
class CustomAmplifierAgent(AbstractInstalledAgent):
14+
@staticmethod
15+
def name() -> str:
16+
return "amplifier"
17+
18+
ALLOWED_TOOLS = [
19+
"Bash",
20+
"mcp__deepwiki",
21+
"WebFetch",
22+
"TodoWrite",
23+
"Edit",
24+
"Write",
25+
"Read",
26+
"Glob",
27+
"Grep",
28+
"LS",
29+
"WebFetch",
30+
"NotebookEdit",
31+
"NotebookRead",
32+
"TodoRead",
33+
"Agent",
34+
"WebSearch",
35+
]
36+
37+
def __init__(self, model_name: str | None = None, *args, **kwargs):
38+
super().__init__(*args, **kwargs)
39+
self._model_name = model_name
40+
self._version = kwargs.get("version", "latest")
41+
42+
@property
43+
def _env(self) -> dict[str, str]:
44+
env = {
45+
"ANTHROPIC_API_KEY": os.environ["ANTHROPIC_API_KEY"],
46+
"FORCE_AUTO_BACKGROUND_TASKS": "1",
47+
"ENABLE_BACKGROUND_TASKS": "1",
48+
"BASH_DEFAULT_TIMEOUT_MS": "300000", # 5 minutes
49+
"BASH_MAX_TIMEOUT_MS": "600000", # 10 minutes
50+
}
51+
if self._model_name:
52+
env["ANTHROPIC_MODEL"] = self._model_name.removeprefix("anthropic/")
53+
elif "ANTHROPIC_MODEL" in os.environ:
54+
env["ANTHROPIC_MODEL"] = os.environ["ANTHROPIC_MODEL"]
55+
return env
56+
57+
@property
58+
def _install_agent_script_path(self) -> Path:
59+
"""Create the installation script for claude-code."""
60+
script_content = """#!/bin/bash
61+
62+
apt-get update
63+
apt-get install -y curl git make build-essential
64+
65+
# Install uv
66+
curl -LsSf https://astral.sh/uv/install.sh | sh
67+
export PATH="$HOME/.local/bin:$PATH"
68+
69+
# Install nvm and Node.js
70+
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash
71+
source "$HOME/.nvm/nvm.sh"
72+
73+
nvm install 22
74+
npm -v
75+
76+
# Install pnpm
77+
curl -fsSL https://get.pnpm.io/install.sh | sh -
78+
export PNPM_HOME="$HOME/.local/share/pnpm"
79+
export PATH="$PNPM_HOME:$PATH"
80+
81+
# Clone amplifier to temporary location
82+
git clone https://github.com/microsoft/amplifier.git /tmp/amplifier
83+
84+
# Copy all amplifier files into current working directory
85+
cp -r -n /tmp/amplifier/. .
86+
87+
make install
88+
89+
# Modify Claude settings to use acceptEdits mode instead of bypassPermissions
90+
if [ -f .claude/settings.json ]; then
91+
sed -i 's/"defaultMode": "bypassPermissions"/"defaultMode": "acceptEdits"/g' .claude/settings.json
92+
fi"""
93+
94+
with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as temp_file:
95+
temp_file.write(script_content)
96+
temp_path = Path(temp_file.name)
97+
98+
temp_path.chmod(0o755)
99+
return temp_path
100+
101+
def _run_agent_commands(self, instruction: str) -> list[TerminalCommand]:
102+
instruction = f"/ultrathink-task {instruction}"
103+
escaped_instruction = shlex.quote(instruction)
104+
return [
105+
TerminalCommand(
106+
command=f"claude --verbose --output-format stream-json "
107+
f"-p {escaped_instruction} --allowedTools "
108+
f"{' '.join(self.ALLOWED_TOOLS)}",
109+
min_timeout_sec=0.0,
110+
max_timeout_sec=float("inf"),
111+
block=True,
112+
append_enter=True,
113+
),
114+
]
115+
116+
117+
class ClaudeCodeAgent(AbstractInstalledAgent):
118+
@staticmethod
119+
def name() -> str:
120+
return AgentName.CLAUDE_CODE.value
121+
122+
ALLOWED_TOOLS = [
123+
"Bash",
124+
"Edit",
125+
"Write",
126+
"Read",
127+
"Glob",
128+
"Grep",
129+
"LS",
130+
"WebFetch",
131+
"NotebookEdit",
132+
"NotebookRead",
133+
"TodoRead",
134+
"TodoWrite",
135+
"Agent",
136+
"WebSearch",
137+
]
138+
139+
def __init__(self, model_name: str | None = None, *args, **kwargs):
140+
super().__init__(*args, **kwargs)
141+
self._model_name = model_name
142+
self._version = kwargs.get("version", "latest")
143+
144+
@property
145+
def _env(self) -> dict[str, str]:
146+
env = {
147+
"ANTHROPIC_API_KEY": os.environ["ANTHROPIC_API_KEY"],
148+
"FORCE_AUTO_BACKGROUND_TASKS": "1",
149+
"ENABLE_BACKGROUND_TASKS": "1",
150+
"BASH_DEFAULT_TIMEOUT_MS": "300000", # 5 minutes
151+
"BASH_MAX_TIMEOUT_MS": "600000", # 10 minutes
152+
}
153+
if self._model_name:
154+
env["ANTHROPIC_MODEL"] = self._model_name.removeprefix("anthropic/")
155+
elif "ANTHROPIC_MODEL" in os.environ:
156+
env["ANTHROPIC_MODEL"] = os.environ["ANTHROPIC_MODEL"]
157+
return env
158+
159+
@property
160+
def _install_agent_script_path(self) -> Path:
161+
"""Create the installation script for claude-code."""
162+
script_content = """#!/bin/bash
163+
164+
apt-get update
165+
apt-get install -y curl
166+
167+
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash
168+
169+
source "$HOME/.nvm/nvm.sh"
170+
171+
nvm install 22
172+
npm -v
173+
174+
npm install -g @anthropic-ai/claude-code"""
175+
176+
with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as temp_file:
177+
temp_file.write(script_content)
178+
temp_path = Path(temp_file.name)
179+
180+
temp_path.chmod(0o755)
181+
return temp_path
182+
183+
def _run_agent_commands(self, instruction: str) -> list[TerminalCommand]:
184+
escaped_instruction = shlex.quote(instruction)
185+
return [
186+
TerminalCommand(
187+
command=f"claude --verbose --output-format stream-json "
188+
f"-p {escaped_instruction} --allowedTools "
189+
f"{' '.join(self.ALLOWED_TOOLS)}",
190+
min_timeout_sec=0.0,
191+
max_timeout_sec=float("inf"),
192+
block=True,
193+
append_enter=True,
194+
),
195+
]

0 commit comments

Comments
 (0)