eval: desk13 Opus probe on the new guide; limits on module data

eval/harness/build.mjs

@@ -6,12 +6,14 @@
 //   vanilla arm: `src/main.ts` default-exports `mount(root: HTMLElement)`, called once with the `#app` container
 //   foldkit arm: `src/main.ts` exports Model, init, update and view; booted with Runtime.makeApplication and
 //              Runtime.run, as Foldkit's own `src/entry.ts` does
+//   contract arm: an Exact app, built by its own `exact.mjs web-build` (harness/contract.mjs)
 // The app's API is `src/api.js` over `src/flights.js`: every call is a flight the test lands or fails.
 // Usage: node harness/build.mjs <dir> [rx|react]   (the arm defaults to what `<dir>/arm` says)
 import { execFileSync } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
+import { buildContract } from "./contract.mjs";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 const evalDir = path.resolve(here, "..");
@@ -35,6 +37,7 @@ async function solidPlugin() {
 
 export async function build(dir, arm = armOf(dir)) {
   dir = realpathSync(path.resolve(dir));
+  if (arm === "contract") return buildContract(dir);
   const work = path.join(dir, ".build");
   rmSync(work, { recursive: true, force: true }); mkdirSync(work, { recursive: true });
   // the app's bare imports resolve from the harness's packages

eval/harness/check.mjs

@@ -4,12 +4,14 @@
 //   solid: tsc --noEmit -p . and eslint (eslint-plugin-solid's v2 config)
 //   vanilla: tsc --noEmit -p . (strict)
 //   foldkit: tsc --noEmit -p . with Foldkit's template tsconfig (its lint, oxlint + effect-tsgo, is not on npm)
+//   contract: `exact.mjs contract build app.contract`, which also type-checks app.ts
 // Usage: node harness/check.mjs <dir>
 import { execFileSync } from "node:child_process";
 import { existsSync, symlinkSync } from "node:fs";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { RXC, armOf } from "./build.mjs";
+import { checkContract } from "./contract.mjs";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 const evalDir = path.resolve(here, "..");
@@ -21,6 +23,7 @@ const run = (cmd, args, cwd) => {
 
 export function check(dir) {
   dir = path.resolve(dir);
+  if (armOf(dir) === "contract") return checkContract(dir);
   if (!existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules"));
   if (armOf(dir) === "rx") return run(RXC, ["check", "src/main.rx"], dir);
   const bin = n => path.join(evalDir, "node_modules/.bin", n);

eval/harness/contract.mjs

@@ -0,0 +1,50 @@
+// The contract arm: an Exact app (https://github.com/expo/exact), `app.contract` for the interface and `app.ts` for its
+// data module, which imports the task's `src/api.js`. A desk is the directory `exact new` makes, with the eval's files
+// beside it. EXACT2 names the Exact checkout the desk is made from; the desk's own `exact.mjs` keeps that path.
+//
+// The build is Exact's web JS target, a release build (`exact.mjs web-build --out`). The data module runs in the page's
+// realm, so `src/flights.js` puts `__flights` on the page as it does in every other arm, and the driver is unchanged.
+// BUN names the Bun to run `exact.mjs` with (default: `bun` on PATH).
+import { execFileSync, spawnSync } from "node:child_process";
+import { cpSync, existsSync, readFileSync, rmSync } from "node:fs";
+import path from "node:path";
+
+const bun = () => process.env.BUN ?? "bun";
+const exactEnv = () => ({ ...process.env, EXACT_FEEDBACK: "0", NO_COLOR: "1" });
+
+export function exact2() {
+  const dir = process.env.EXACT2;
+  if (!dir || !existsSync(path.join(dir, "scripts/exact.mjs"))) throw new Error("EXACT2 must name an Exact checkout (its scripts/exact.mjs)");
+  return path.resolve(dir);
+}
+
+/** `<dir>/dist`, built by the desk's `exact.mjs` (Exact keeps a build inside an app in `dist`, which its source scans
+ * skip); a compile error is the `compile` stage's output */
+export function buildContract(dir) {
+  const dist = path.join(dir, "dist");
+  const r = spawnSync(bun(), ["exact.mjs", "web-build", "--out", dist], { cwd: dir, env: exactEnv(), encoding: "utf8", maxBuffer: 64 << 20 });
+  if (r.status !== 0) return { ok: false, stage: "compile", output: `${r.stdout ?? ""}${r.stderr ?? ""}`.trim() };
+  return { ok: true, dist };
+}
+
+/** the compiler's diagnostics, warnings included, as the agent's `./check` shows them */
+export function checkContract(dir) {
+  const r = spawnSync(bun(), ["exact.mjs", "contract", "build", "app.contract"], { cwd: dir, env: exactEnv(), encoding: "utf8", maxBuffer: 64 << 20 });
+  return { ok: r.status === 0, out: `${r.stdout ?? ""}${r.stderr ?? ""}`.trim() || "ok" };
+}
+
+/** `exact new <dir>`, then the scaffold's own files go in beside it. The placeholder test is removed: `./test` is the
+ * desk's test command, as in every arm. */
+export function newContractDesk(dir) {
+  if (existsSync(dir)) throw new Error(`${dir} exists; a contract desk starts from \`exact new\``);
+  execFileSync(bun(), [path.join(exact2(), "scripts/exact.mjs"), "new", dir], { env: exactEnv(), stdio: "pipe" });
+  rmSync(path.join(dir, "app.test.contract"), { force: true });
+}
+
+/** Exact's one required guide, for the prompt's inline docs */
+export const startHere = () => readFileSync(path.join(exact2(), "docs/start-here.md"), "utf8");
+
+/** copies a reference app's files over a desk's placeholders */
+export function copyContractReference(from, dir) {
+  for (const f of ["app.contract", "app.ts"]) cpSync(path.join(from, f), path.join(dir, f));
+}

eval/harness/driver.mjs

@@ -23,7 +23,13 @@ export async function serve(dist) {
 }
 
 /** CHROMIUM=/path/to/chromium drives a system browser (NixOS and the like, where Playwright's download does not run). */
-export async function launch() { return chromium.launch(process.env.CHROMIUM ? { executablePath: process.env.CHROMIUM } : {}); }
+// Without CHROMIUM, a Playwright whose own browser is missing falls back to a preinstalled one (/opt/pw-browsers/chromium,
+// as in cloud containers), so an agent's ./test runs there too.
+export async function launch() {
+  if (process.env.CHROMIUM) return chromium.launch({ executablePath: process.env.CHROMIUM });
+  try { return await chromium.launch(); }
+  catch (e) { const alt = "/opt/pw-browsers/chromium"; if (existsSync(alt)) return chromium.launch({ executablePath: alt }); throw e; }
+}
 
 /** One test: a fresh page on the app, `run(app)` with the helpers below. Returns { ok, error }. */
 export async function runTest(browser, url, test) {

eval/harness/prompt-arms.json

@@ -58,5 +58,17 @@
     "LANG_RULE": "plain TypeScript and the browser DOM, with no UI framework or library: `src/main.ts` must default-export `function mount(root: HTMLElement): void`, which the harness calls once with the page's `#app` element. You may add files under `src/`. No packages are installed for you to use.",
     "API_TYPES": "`src/api.d.ts`",
     "CHECKER": "the TypeScript compiler (strict)"
+  },
+  "contract": {
+    "DOCS_RULE": ", except that you may read Exact's documentation in {{EXACT2}}/docs and look names up with `bun exact.mjs contract vocab <name>` and `bun exact.mjs docs <heading words>`",
+    "LANG_RULE": "Exact, with Contract for the interface and TypeScript for its data module. This directory is an Exact app that `exact new` made from the Exact checkout at {{EXACT2}}; the tests run its web build in a browser. Exact's guide is `{{EXACT2}}/docs/start-here.md`, the only required reading; the rest of `{{EXACT2}}/docs` is lookup. `app.contract` is the interface and `app.ts` its data module, whose sources call the backend: import it with `import * as api from './src/api.js'`. Give an element a test id with `testId=\"…\"`. The only commands that build, test, drive or serve the app are `./check` and `./test` below: do not run the `bun exact.mjs` verbs that do (`web`, `web-build`, `serve`, `test`, `verify`, `agent`, or a platform's).",
+    "API_TYPES": "`src/api.d.ts`",
+    "CHECKER": "the Contract compiler (`bun exact.mjs contract build app.contract`), which also type-checks `app.ts`"
+  },
+  "contract-inline": {
+    "DOCS_RULE": ", except that you may read Exact's documentation in {{EXACT2}}/docs and look names up with `bun exact.mjs contract vocab <name>` and `bun exact.mjs docs <heading words>`",
+    "LANG_RULE": "Exact, with Contract for the interface and TypeScript for its data module. This directory is an Exact app that `exact new` made from the Exact checkout at {{EXACT2}}; the tests run its web build in a browser. Exact's guide, `docs/start-here.md`, is at the end of this message: it is the only required reading, and the rest of `{{EXACT2}}/docs` is lookup. `app.contract` is the interface and `app.ts` its data module, whose sources call the backend: import it with `import * as api from './src/api.js'`. Give an element a test id with `testId=\"…\"`. The only commands that build, test, drive or serve the app are `./check` and `./test` below: do not run the `bun exact.mjs` verbs that do (`web`, `web-build`, `serve`, `test`, `verify`, `agent`, or a platform's).",
+    "API_TYPES": "`src/api.d.ts`",
+    "CHECKER": "the Contract compiler (`bun exact.mjs contract build app.contract`), which also type-checks `app.ts`"
   }
 }

eval/harness/scaffold.mjs

@@ -18,6 +18,7 @@
 import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, symlinkSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
+import { copyContractReference, exact2, newContractDesk, startHere } from "./contract.mjs";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 const evalDir = path.resolve(here, "..");
@@ -30,6 +31,7 @@ const statelyDocs = process.env.STATELY_DOCS ?? path.resolve(evalDir, "../../sta
 export function scaffold(task, arm, dir, { reference = false, brief = null, base = null, inline = false } = {}) {
   const t = path.join(evalDir, "tasks", task);
   dir = path.resolve(dir);
+  if (arm === "contract") newContractDesk(dir);
   mkdirSync(path.join(dir, "src"), { recursive: true }); mkdirSync(path.join(dir, "tests"), { recursive: true });
   writeFileSync(path.join(dir, "arm"), `${arm}\n`);
   cpSync(path.join(t, brief ? `brief-${brief}.md` : "brief.md"), path.join(dir, "BRIEF.md"));
@@ -69,7 +71,8 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base
   }
   // Solid's own docs ship in its package: the agent can read them there before its first build
   if (arm === "solid" && !existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules"));
-  if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true });
+  if (reference && arm === "contract") copyContractReference(path.join(evalDir, "reference", task, arm), dir);
+  else if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true });
   if (base) {
     cpSync(path.join(t, "BASE.md"), path.join(dir, "BASE.md"));
     // everything the earlier app had (a chain step's base is the previous agent's src/, which may hold more than
@@ -85,11 +88,13 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base
 export function prompt(arm, dir, { plain = false, change = false, inline = false } = {}) {
   const docs = inline && arm !== "react";
   const fill = { ...JSON.parse(readFileSync(path.join(here, "prompt-arms.json"), "utf8"))[docs ? `${arm}-inline` : arm] };
+  if (arm === "contract") fill.EXACT2 = exact2();
   fill.BACKEND_NOTE = plain ? "" : " whose answers the tests control";
   let text = readFileSync(path.join(here, change ? "prompt-change.md" : "prompt.md"), "utf8");
   for (let i = 0; i < 2; i++) for (const k of [...Object.keys(fill), "DIR"]) text = text.replaceAll(`{{${k}}}`, k === "DIR" ? dir : fill[k]);
   if (docs && arm === "rx") for (const f of ["language.md", "async.md", "reference.md"]) text += `\n---\n\n${readFileSync(path.join(evalDir, "../guide", f), "utf8")}`;
   if (docs && arm === "foldkit") text += `\n---\n\n${readFileSync(path.join(foldkitRepo, "packages/create-foldkit-app/templates/base/FOLDKIT.md"), "utf8")}`;
+  if (docs && arm === "contract") text += `\n---\n\n${startHere()}`;
   if (docs && arm === "xstate") text += `\n---\n\n${readFileSync(path.join(statelyDocs, "content/docs/cheatsheet.mdx"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}\n---\n\n${readFileSync(path.join(evalDir, "node_modules/@xstate/react/README.md"), "utf8")}`;
   if (docs && arm === "solid") text += `\n---\n\n${readFileSync(path.join(evalDir, "node_modules/solid-js/skills/solid-2.0/SKILL.md"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}`;
   return text;

eval/harness/score.mjs

@@ -24,7 +24,7 @@ const round5 = new Set(["desk", "labels", "related", "count", "chain1", "chain2"
 const rows = [];
 for (const name of readdirSync(runs).sort()) {
   const dir = path.join(runs, name);
-  const m = name.match(/^(\w+)-(rx|react|solid|foldkit|vanilla)-(\d+)$/);
+  const m = name.match(/^(\w+)-(rx|react|solid|foldkit|vanilla|contract)-(\d+)$/);
   if (!m) continue;
   const [, task, arm, n] = m;
   const alongside = round5.has(task) ? round5Alongside : round4Alongside;

eval/reference/desk9/contract/README.md

@@ -0,0 +1,11 @@
+# desk9 in Contract (Exact)
+
+Exact team's hand-written reference, from expo/exact `bench/reactivescript/harness.patch` (main 8c742e6a, 2026-11-13).
+MIT License, Copyright (c) 2026 650 Industries, Inc. (Expo).
+
+The `contract` arm (`harness/contract.mjs` and its hooks in build, check, scaffold, prompt-arms and score) comes from
+the same patch, ported to this harness. It needs `EXACT2` (an Exact checkout, `bun install` and `cargo fetch` run in
+it; the web build compiles Exact's Rust workspace once) and Bun on PATH.
+
+Graded here on 2026-11-13 (`harness/grade.sh`): 77/77 hidden, 38/38 overlap, 0/20 fuzz, 0 defects, as in Exact's own
+harness.