feat: add benchmark run-loop command (task 0029)
Add the maintainer-facing command that runs one chosen benchmark cell on demand, so only the token budget available at that moment is spent. runCells (bench/run-loop.ts) runs one (arm, task) cell for a batch of trials — defaulting to five with a reporting floor of three — by driving the existing single-cell runner and the append-only sample store rather than reimplementing orchestration. Re-running a cell deepens it: trial numbering continues past the highest trial the cell already holds and the new samples append, so a cell's sample size grows across sittings without overwriting prior runs. bench/run.ts is the command: parseRunArgs is the pure, unit-tested argument seam, and runBenchCommand is the live boundary that resolves host access, resolves the scored suite against the host's self-review support, selects the task, and drives the run loop. It is invoked via the new bench:run npm script, run under tsx (a new devDependency) because the harness's .js-specifier imports need a TypeScript-aware runner. The Claude Agent SDK is now declared as an optional peerDependency — documented but neither installed for package consumers nor pulled into CI. Every arm runs on the driver's single fixed model; the command exposes no per-cell model override that could break cross-arm comparability. The default store root bench/results/ is gitignored.
This commit was merged in pull request #30.
This commit is contained in:
28
package-lock.json
generated
28
package-lock.json
generated
@@ -19,11 +19,20 @@
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.19.0",
|
||||
"@vitest/coverage-v8": "^3.2.7",
|
||||
"tsx": "^4.23.1",
|
||||
"typescript": "^5.8.0",
|
||||
"vitest": "^3.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": ">=0.3.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@anthropic-ai/claude-agent-sdk": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@ampproject/remapping": {
|
||||
@@ -2229,6 +2238,25 @@
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tsx": {
|
||||
"version": "4.23.1",
|
||||
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.1.tgz",
|
||||
"integrity": "sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"esbuild": "~0.28.0"
|
||||
},
|
||||
"bin": {
|
||||
"tsx": "dist/cli.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "~2.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
|
||||
Reference in New Issue
Block a user