eval: desk13 Exact (contract) Opus probe in our harness

eval/harness/agent-cli.mjs

@@ -1,5 +1,6 @@
 // What the agent runs in its directory: `./check` (the arm's compiler feedback) and `./test` (the visible tests).
-// Every run is appended to `.runlog` (one JSON line each), which the score reads for iterations and when green came.
+// Every run is appended to `.runlog` (one JSON line each: `at` when it ended, `ms` how long it took), which the score
+// reads for iterations, when green came, and how much of the agent's time went to these tools.
 // Usage: node harness/agent-cli.mjs <check|test> <dir>
 import { appendFileSync } from "node:fs";
 import path from "node:path";
@@ -8,7 +9,8 @@ import { runFile } from "./test.mjs";
 
 const [cmd, dirArg] = process.argv.slice(2);
 const dir = path.resolve(dirArg);
-const log = entry => appendFileSync(path.join(dir, ".runlog"), JSON.stringify({ at: new Date().toISOString(), cmd, ...entry }) + "\n");
+const started = Date.now();
+const log = entry => appendFileSync(path.join(dir, ".runlog"), JSON.stringify({ at: new Date().toISOString(), ms: Date.now() - started, cmd, ...entry }) + "\n");
 
 if (cmd === "check") {
   const r = check(dir);

eval/harness/score.mjs

@@ -1,5 +1,8 @@
 // Scores finished runs: for each run directory (named <task>-<arm>-<n>), the arm's check, the visible and the hidden
 // tests, and the agent's own log of `./check` and `./test` runs. Writes results.json and prints a summary table.
+// Time: `toolMs` is the agent's time inside `./check` and `./test` (each `.runlog` entry's `ms`), `greenAfterMs` the time
+// from its first run to its first passing `./test`; `agentMs`, the whole run's wall time, comes from `<run>/.agent.json`
+// (`{ "ms": … }`, written by whoever ran the agent) when there is one. The model's own time is agentMs - toolMs.
 // With --overlap, each task's hidden-overlap.mjs (round 3) runs too and counts toward the defect.
 // Tests named in `alongside` are run and reported but do not count toward the defect (round 4 pre-registration).
 // Usage: node harness/score.mjs <runs-dir> [out.json] [--overlap]
@@ -30,6 +33,10 @@ for (const name of readdirSync(runs).sort()) {
   const alongside = round5.has(task) ? round5Alongside : round4Alongside;
   const log = existsSync(path.join(dir, ".runlog")) ? readFileSync(path.join(dir, ".runlog"), "utf8").trim().split("\n").filter(Boolean).map(l => JSON.parse(l)) : [];
   const firstGreen = log.findIndex(e => e.cmd === "test" && e.ok);
+  const startOf = e => Date.parse(e.at) - (e.ms ?? 0);
+  const toolMs = log.every(e => e.ms != null) && log.length ? log.reduce((t, e) => t + e.ms, 0) : null;
+  const greenAfterMs = firstGreen < 0 || !log.length ? null : Date.parse(log[firstGreen].at) - startOf(log[0]);
+  const agent = existsSync(path.join(dir, ".agent.json")) ? JSON.parse(readFileSync(path.join(dir, ".agent.json"), "utf8")) : null;
   const checked = check(dir);
   const visible = await runFile(dir, path.join(tasksDir, task, "visible.mjs"));
   const hidden = await runFile(dir, path.join(tasksDir, task, "hidden.mjs"));
@@ -41,6 +48,7 @@ for (const name of readdirSync(runs).sort()) {
     run: name, task, arm, n: Number(n),
     checks: log.filter(e => e.cmd === "check").length, tests: log.filter(e => e.cmd === "test").length,
     cyclesToGreen: firstGreen < 0 ? null : firstGreen + 1,
+    agentMs: agent?.ms ?? null, toolMs, greenAfterMs,
     checkOk: checked.ok, built: hidden.build.ok,
     visible: `${visible.results.filter(r => r.ok).length}/${visible.results.length}`,
     hidden: `${hidden.results.length - failedHidden.length}/${hidden.results.length}`,
@@ -51,6 +59,6 @@ for (const name of readdirSync(runs).sort()) {
     failedVisible: visible.results.filter(r => !r.ok).map(r => ({ name: r.name, error: r.error })),
   });
   const r = rows.at(-1);
-  console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"}  visible ${r.visible}  hidden ${r.hidden.padEnd(5)}${r.overlap ? `  overlap ${r.overlap}` : ""}  cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.shippedDefect ? "  DEFECT" : ""}${r.failedAlongside.length ? "  (alongside failed)" : ""}`);
+  console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"}  visible ${r.visible}  hidden ${r.hidden.padEnd(5)}${r.overlap ? `  overlap ${r.overlap}` : ""}  cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.agentMs != null ? `  agent ${(r.agentMs / 60000).toFixed(1)}m` : ""}${r.toolMs != null ? `  tools ${(r.toolMs / 60000).toFixed(1)}m` : ""}${r.shippedDefect ? "  DEFECT" : ""}${r.failedAlongside.length ? "  (alongside failed)" : ""}`);
 }
 writeFileSync(outArg ? path.resolve(outArg) : path.join(runs, "results.json"), JSON.stringify(rows, null, 2) + "\n");

eval/results/desk13-exact-2026-11-13/README.md

@@ -0,0 +1,27 @@
+# desk9 in Exact, Opus, 3 apps, our harness (2026-11-13)
+
+Not preregistered. The `contract` arm (ported from expo/exact's bench patch, #59) run the way we ran the .rx probe:
+Opus as general-purpose subagents, `--brief plain --inline-docs` (Exact's `docs/start-here.md` inlined, the rest of
+its docs readable), `./check`/`./test` as the only build feedback (15 runs). Exact at main 8c742e6a, from a
+`git archive` with `bench/` removed, so agents could not read Exact's own desk9 reference.
+
+| app | hidden | overlap | fuzz holes | defects | lines (non-blank, non-comment) | subagent tokens | minutes |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| contract-1 | 77 | 37 | 0 | 1 | 605 | 155k | 12.4 |
+| contract-2 | 76 | 37 | 0 | 2 | 568 | 145k | 11.4 |
+| contract-3 | 77 | 38 | 0 | **0** | 605 | 168k | 10.5 |
+
+Mean **1.0**. For comparison, the last three .rx apps (rx-7..9, same harness, guide with the lifetime rule, Solid
+next f00ce6f52) score 2, 0, 0 = **0.67**, at 99-107k tokens, about 5 minutes and 411-452 lines each. Three apps per
+arm: neither difference means anything statistically.
+
+## The defects
+
+- **Settings save, contract-1 and contract-2** (overlap "a later save failing while an earlier one is out shows its
+  error and then the earlier result"). Every issue write is a `parallel` mutation, but both apps left
+  `mutation saved` on Exact's default mode, under which a later save replaces the earlier one, so the earlier
+  result never shows. contract-3 made its save `parallel` too (`mutation saved … parallel refreshes settings`) and passes.
+- **contract-2, "rows: row errors and detail errors leave each other alone"**: row actions and the detail share an
+  error path (agent).
+
+No fuzz holes in any Exact app.

eval/results/desk13-exact-2026-11-13/contract-1.grade.txt

@@ -0,0 +1,7 @@
+== hidden
+77/77 passed
+== overlap
+✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
+37/38 passed
+== fuzz
+desk13-contract-1: 0/20 seeds found a hole

eval/results/desk13-exact-2026-11-13/contract-2.grade.txt

@@ -0,0 +1,8 @@
+== hidden
+✗ rows: row errors and detail errors leave each other alone
+76/77 passed
+== overlap
+✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result
+37/38 passed
+== fuzz
+desk13-contract-2: 0/20 seeds found a hole

eval/results/desk13-exact-2026-11-13/contract-3.grade.txt

@@ -0,0 +1,6 @@
+== hidden
+77/77 passed
+== overlap
+38/38 passed
+== fuzz
+desk13-contract-3: 0/20 seeds found a hole