From 044457730d1bba63272eb0498a5a1e654b0b6453 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 19:05:55 +0000 Subject: [PATCH 1/3] eval: desk13 Exact (contract) Opus probe in our harness: 1, 2, 0 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWLtkg281f1NkGEDjCt --- .../results/desk13-exact-2026-11-13/README.md | 27 +++++++++++++++++++ .../contract-1.grade.txt | 7 +++++ .../contract-2.grade.txt | 8 ++++++ .../contract-3.grade.txt | 6 +++++ 4 files changed, 48 insertions(+) create mode 100644 eval/results/desk13-exact-2026-11-13/README.md create mode 100644 eval/results/desk13-exact-2026-11-13/contract-1.grade.txt create mode 100644 eval/results/desk13-exact-2026-11-13/contract-2.grade.txt create mode 100644 eval/results/desk13-exact-2026-11-13/contract-3.grade.txt diff --git a/eval/results/desk13-exact-2026-11-13/README.md b/eval/results/desk13-exact-2026-11-13/README.md new file mode 100644 index 0000000..0ebce6d --- /dev/null +++ b/eval/results/desk13-exact-2026-11-13/README.md @@ -0,0 +1,27 @@ +# desk9 in Exact, Opus, 3 apps, our harness (2026-11-13) + +Not preregistered. The `contract` arm (ported from expo/exact's bench patch, #59) run the way we ran the .rx probe: +Opus as general-purpose subagents, `--brief plain --inline-docs` (Exact's `docs/start-here.md` inlined, the rest of +its docs readable), `./check`/`./test` as the only build feedback (15 runs). Exact at main 8c742e6a, from a +`git archive` with `bench/` removed, so agents could not read Exact's own desk9 reference. + +| app | hidden | overlap | fuzz holes | defects | lines (non-blank, non-comment) | subagent tokens | minutes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| contract-1 | 77 | 37 | 0 | 1 | 605 | 155k | 12.4 | +| contract-2 | 76 | 37 | 0 | 2 | 568 | 145k | 11.4 | +| contract-3 | 77 | 38 | 0 | **0** | 605 | 168k | 10.5 | + +Mean **1.0**. For comparison, the last three .rx apps (rx-7..9, same harness, guide with the lifetime rule, Solid +next f00ce6f52) score 2, 0, 0 = **0.67**, at 99-107k tokens, about 5 minutes and 411-452 lines each. Three apps per +arm: neither difference means anything statistically. + +## The defects + +- **Settings save, contract-1 and contract-2** (overlap "a later save failing while an earlier one is out shows its + error and then the earlier result"). Every issue write is a `parallel` mutation, but both apps left + `mutation saved` on Exact's default mode, under which a later save replaces the earlier one, so the earlier + result never shows. contract-3 declares no such mutation for its save. +- **contract-2, "rows: row errors and detail errors leave each other alone"**: row actions and the detail share an + error path (agent). + +No fuzz holes in any Exact app. diff --git a/eval/results/desk13-exact-2026-11-13/contract-1.grade.txt b/eval/results/desk13-exact-2026-11-13/contract-1.grade.txt new file mode 100644 index 0000000..f89f7bd --- /dev/null +++ b/eval/results/desk13-exact-2026-11-13/contract-1.grade.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk13-contract-1: 0/20 seeds found a hole diff --git a/eval/results/desk13-exact-2026-11-13/contract-2.grade.txt b/eval/results/desk13-exact-2026-11-13/contract-2.grade.txt new file mode 100644 index 0000000..aefbe2c --- /dev/null +++ b/eval/results/desk13-exact-2026-11-13/contract-2.grade.txt @@ -0,0 +1,8 @@ +== hidden +✗ rows: row errors and detail errors leave each other alone +76/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk13-contract-2: 0/20 seeds found a hole diff --git a/eval/results/desk13-exact-2026-11-13/contract-3.grade.txt b/eval/results/desk13-exact-2026-11-13/contract-3.grade.txt new file mode 100644 index 0000000..66efee5 --- /dev/null +++ b/eval/results/desk13-exact-2026-11-13/contract-3.grade.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk13-contract-3: 0/20 seeds found a hole From 46ab71d3e99cbbc4cf25826c814d43e224a8418f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 19:06:09 +0000 Subject: [PATCH 2/3] eval: correct contract-3's save note Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWLtkg281f1NkGEDjCt --- eval/results/desk13-exact-2026-11-13/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/results/desk13-exact-2026-11-13/README.md b/eval/results/desk13-exact-2026-11-13/README.md index 0ebce6d..85b9b7b 100644 --- a/eval/results/desk13-exact-2026-11-13/README.md +++ b/eval/results/desk13-exact-2026-11-13/README.md @@ -20,7 +20,7 @@ arm: neither difference means anything statistically. - **Settings save, contract-1 and contract-2** (overlap "a later save failing while an earlier one is out shows its error and then the earlier result"). Every issue write is a `parallel` mutation, but both apps left `mutation saved` on Exact's default mode, under which a later save replaces the earlier one, so the earlier - result never shows. contract-3 declares no such mutation for its save. + result never shows. contract-3 made its save `parallel` too (`mutation saved … parallel refreshes settings`) and passes. - **contract-2, "rows: row errors and detail errors leave each other alone"**: row actions and the detail share an error path (agent). From b7815d69ebfc9bd3e85f9e66fd99c57e514cca07 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 19:11:10 +0000 Subject: [PATCH 3/3] eval: track time (tool time per ./check and ./test, time to first green, agent wall time from .agent.json) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWLtkg281f1NkGEDjCt --- eval/harness/agent-cli.mjs | 6 ++++-- eval/harness/score.mjs | 10 +++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/eval/harness/agent-cli.mjs b/eval/harness/agent-cli.mjs index 67f205b..8668b0d 100644 --- a/eval/harness/agent-cli.mjs +++ b/eval/harness/agent-cli.mjs @@ -1,5 +1,6 @@ // What the agent runs in its directory: `./check` (the arm's compiler feedback) and `./test` (the visible tests). -// Every run is appended to `.runlog` (one JSON line each), which the score reads for iterations and when green came. +// Every run is appended to `.runlog` (one JSON line each: `at` when it ended, `ms` how long it took), which the score +// reads for iterations, when green came, and how much of the agent's time went to these tools. // Usage: node harness/agent-cli.mjs import { appendFileSync } from "node:fs"; import path from "node:path"; @@ -8,7 +9,8 @@ import { runFile } from "./test.mjs"; const [cmd, dirArg] = process.argv.slice(2); const dir = path.resolve(dirArg); -const log = entry => appendFileSync(path.join(dir, ".runlog"), JSON.stringify({ at: new Date().toISOString(), cmd, ...entry }) + "\n"); +const started = Date.now(); +const log = entry => appendFileSync(path.join(dir, ".runlog"), JSON.stringify({ at: new Date().toISOString(), ms: Date.now() - started, cmd, ...entry }) + "\n"); if (cmd === "check") { const r = check(dir); diff --git a/eval/harness/score.mjs b/eval/harness/score.mjs index cd97569..424b539 100644 --- a/eval/harness/score.mjs +++ b/eval/harness/score.mjs @@ -1,5 +1,8 @@ // Scores finished runs: for each run directory (named --), the arm's check, the visible and the hidden // tests, and the agent's own log of `./check` and `./test` runs. Writes results.json and prints a summary table. +// Time: `toolMs` is the agent's time inside `./check` and `./test` (each `.runlog` entry's `ms`), `greenAfterMs` the time +// from its first run to its first passing `./test`; `agentMs`, the whole run's wall time, comes from `/.agent.json` +// (`{ "ms": … }`, written by whoever ran the agent) when there is one. The model's own time is agentMs - toolMs. // With --overlap, each task's hidden-overlap.mjs (round 3) runs too and counts toward the defect. // Tests named in `alongside` are run and reported but do not count toward the defect (round 4 pre-registration). // Usage: node harness/score.mjs [out.json] [--overlap] @@ -30,6 +33,10 @@ for (const name of readdirSync(runs).sort()) { const alongside = round5.has(task) ? round5Alongside : round4Alongside; const log = existsSync(path.join(dir, ".runlog")) ? readFileSync(path.join(dir, ".runlog"), "utf8").trim().split("\n").filter(Boolean).map(l => JSON.parse(l)) : []; const firstGreen = log.findIndex(e => e.cmd === "test" && e.ok); + const startOf = e => Date.parse(e.at) - (e.ms ?? 0); + const toolMs = log.every(e => e.ms != null) && log.length ? log.reduce((t, e) => t + e.ms, 0) : null; + const greenAfterMs = firstGreen < 0 || !log.length ? null : Date.parse(log[firstGreen].at) - startOf(log[0]); + const agent = existsSync(path.join(dir, ".agent.json")) ? JSON.parse(readFileSync(path.join(dir, ".agent.json"), "utf8")) : null; const checked = check(dir); const visible = await runFile(dir, path.join(tasksDir, task, "visible.mjs")); const hidden = await runFile(dir, path.join(tasksDir, task, "hidden.mjs")); @@ -41,6 +48,7 @@ for (const name of readdirSync(runs).sort()) { run: name, task, arm, n: Number(n), checks: log.filter(e => e.cmd === "check").length, tests: log.filter(e => e.cmd === "test").length, cyclesToGreen: firstGreen < 0 ? null : firstGreen + 1, + agentMs: agent?.ms ?? null, toolMs, greenAfterMs, checkOk: checked.ok, built: hidden.build.ok, visible: `${visible.results.filter(r => r.ok).length}/${visible.results.length}`, hidden: `${hidden.results.length - failedHidden.length}/${hidden.results.length}`, @@ -51,6 +59,6 @@ for (const name of readdirSync(runs).sort()) { failedVisible: visible.results.filter(r => !r.ok).map(r => ({ name: r.name, error: r.error })), }); const r = rows.at(-1); - console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)}${r.overlap ? ` overlap ${r.overlap}` : ""} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.shippedDefect ? " DEFECT" : ""}${r.failedAlongside.length ? " (alongside failed)" : ""}`); + console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)}${r.overlap ? ` overlap ${r.overlap}` : ""} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.agentMs != null ? ` agent ${(r.agentMs / 60000).toFixed(1)}m` : ""}${r.toolMs != null ? ` tools ${(r.toolMs / 60000).toFixed(1)}m` : ""}${r.shippedDefect ? " DEFECT" : ""}${r.failedAlongside.length ? " (alongside failed)" : ""}`); } writeFileSync(outArg ? path.resolve(outArg) : path.join(runs, "results.json"), JSON.stringify(rows, null, 2) + "\n");