test: add WP7-02 controlled real contract gate

2026-08-04 14:06:52 +08:00
parent 623cad25b2
commit 28e6f66a1d
5 changed files with 636 additions and 9 deletions
@@ -0,0 +1,196 @@
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+import {
+  WP7_02_MODEL_IDS,
+  validateCandidateDependency,
+  validateIndependentEvidenceSet,
+} from "./lib/wp7-02-external-contract.mjs";
+
+const wp701Sha = "623cad25b2a2a9a003502c9a92ebd318dad06248";
+const candidateRunId = "wp7-01-candidate-20260804052447717";
+const runId = process.env.DADA_TDD_RUN_ID ?? `wp7-02-external-${new Date().toISOString().replace(/[^0-9]/g, "")}`;
+const runDirectory = resolve("artifacts", "tdd", runId);
+const caseDirectory = resolve(runDirectory, "cases", "TDD-WP7-EXT-001-three-real-models");
+const candidatePath = process.env.DADA_WP7_01_CANDIDATE_RECORD;
+process.on("uncaughtException", (error) => {
+  const message = error instanceof Error ? error.message : String(error);
+  const code = /^WP7_02_[A-Z0-9_:]+$/.test(message) ? message : "WP7_02_VALIDATION_FAILED";
+  console.error(JSON.stringify({ code, real_calls: 0, run_id: runId, status: "failed" }));
+  process.exit(1);
+});
+if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`);
+if (!candidatePath || !existsSync(candidatePath)) throw new Error("WP7_02_CANDIDATE_RECORD_REQUIRED");
+mkdirSync(caseDirectory, { recursive: true });
+
+function sha256(path) {
+  return createHash("sha256").update(readFileSync(path)).digest("hex").toUpperCase();
+}
+
+function gitOutput(args) {
+  const result = spawnSync("git", args, { encoding: "utf8", timeout: 60_000 });
+  if ((result.status ?? 1) !== 0) throw new Error(`WP7_02_GIT_COMMAND_FAILED:${args[0]}`);
+  return result.stdout.trim();
+}
+
+function verifyUpstream() {
+  const result = spawnSync("git", ["ls-remote", "--heads", "origin", "refs/heads/codex/wp7-01"], { encoding: "utf8", timeout: 60_000 });
+  if ((result.status ?? 1) !== 0) throw new Error("WP7_02_WP7_01_REMOTE_UNREADABLE");
+  const remoteSha = result.stdout.trim().split(/\s+/)[0];
+  if (remoteSha !== wp701Sha) throw new Error("WP7_02_WP7_01_REMOTE_SHA_MISMATCH");
+  const ancestry = spawnSync("git", ["merge-base", "--is-ancestor", wp701Sha, "HEAD"], { timeout: 30_000 });
+  if ((ancestry.status ?? 1) !== 0) throw new Error("WP7_02_WP7_01_NOT_ANCESTOR");
+  return remoteSha;
+}
+
+function run(name, command, args, expectedExitCode = 0) {
+  const started_at = new Date().toISOString();
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    env: process.env,
+    maxBuffer: 64 * 1024 * 1024,
+    timeout: 300_000,
+  });
+  if (result.stdout) process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+  return {
+    command: [command, ...args].join(" "),
+    exit_code: result.status ?? 1,
+    expected_exit_code: expectedExitCode,
+    finished_at: new Date().toISOString(),
+    name,
+    started_at,
+  };
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+const upstreamRemoteSha = verifyUpstream();
+const candidateRecord = readJson(candidatePath);
+const candidate = validateCandidateDependency(candidateRecord);
+const commands = [
+  run("contract-harness", process.execPath, ["--test", "tests/package/wp7-02-external-contract.test.mjs"]),
+  run("tdd-trace", process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm", process.platform === "win32"
+    ? ["/d", "/s", "/c", "pnpm validate:tdd-trace"] : ["validate:tdd-trace"]),
+  run("security", process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm", process.platform === "win32"
+    ? ["/d", "/s", "/c", "pnpm test:security"] : ["test:security"]),
+];
+commands[0].command = "node --test tests/package/wp7-02-external-contract.test.mjs";
+commands[1].command = "pnpm validate:tdd-trace";
+commands[2].command = "pnpm test:security";
+const firstAutomationFailure = commands.find((command) => command.exit_code !== command.expected_exit_code);
+if (firstAutomationFailure) {
+  writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`);
+  console.error(JSON.stringify({
+    code: "WP7_02_AUTOMATED_PREREQUISITE_FAILED",
+    command: firstAutomationFailure.command,
+    exit_code: firstAutomationFailure.exit_code,
+    real_calls: 0,
+    run_id: runId,
+    status: "failed",
+  }, null, 2));
+  process.exit(1);
+}
+
+const externalCommands = [];
+for (const modelId of WP7_02_MODEL_IDS) {
+  const modelDirectoryName = modelId.replaceAll(".", "_");
+  const modelDirectory = resolve(caseDirectory, modelDirectoryName);
+  const externalArguments = [
+    "--service", "ai-gateway-service-id", "--model", modelId,
+    "--run-id", runId, "--candidate-record", candidatePath, "--evidence-dir", modelDirectory, "--readiness-only",
+  ];
+  if (externalArguments.some((value) => /[\s"&|<>^%]/.test(value))) throw new Error("WP7_02_UNSAFE_EXTERNAL_ARGUMENT");
+  const command = process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm";
+  const args = process.platform === "win32"
+    ? ["/d", "/c", `pnpm validate:external -- ${externalArguments.join(" ")}`]
+    : ["validate:external", "--", ...externalArguments];
+  const result = run(`readiness-${modelId}`, command, args, 3);
+  result.command = `pnpm validate:external -- --service ai-gateway-service-id --model ${modelId} --run-id <run_id> --candidate-record <wp7-01-candidate> --evidence-dir <case/${modelDirectoryName}> --readiness-only`;
+  externalCommands.push(result);
+}
+commands.push(...externalCommands);
+
+const modelEvidence = WP7_02_MODEL_IDS.map((modelId) => {
+  const directoryName = modelId.replaceAll(".", "_");
+  const directory = resolve(caseDirectory, directoryName);
+  const readiness = readJson(resolve(directory, "readiness.json"));
+  return {
+    blockers: readiness.blockers,
+    candidate: readiness.candidate,
+    evidence_id: readiness.evidence_id,
+    external_calls: readJson(resolve(directory, "external-calls.json")),
+    manual_review: readJson(resolve(directory, "manual-review.json")),
+    matrix: readJson(resolve(directory, "contract-matrix.json")),
+    model_id: readiness.model_id,
+    redaction: readJson(resolve(directory, "redaction.json")),
+    run_id: readiness.run_id,
+    status: readiness.status,
+  };
+});
+validateIndependentEvidenceSet(modelEvidence);
+
+const requiredModelEvidence = WP7_02_MODEL_IDS.flatMap((modelId) => {
+  const directory = modelId.replaceAll(".", "_");
+  return ["contract-matrix.json", "external-calls.json", "manual-review.json", "readiness.json", "redaction.json"]
+    .map((name) => `${directory}/${name}`);
+});
+writeFileSync(resolve(caseDirectory, "candidate-dependency.json"), `${JSON.stringify({
+  candidate_run_id: candidateRunId,
+  record: candidate,
+  remote_branch: "codex/wp7-01",
+  remote_commit: upstreamRemoteSha,
+  status: "passed",
+}, null, 2)}\n`);
+writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`);
+
+const evidenceRefs = ["candidate-dependency.json", "commands.json", ...requiredModelEvidence];
+const missingEvidence = evidenceRefs.filter((path) => !existsSync(resolve(caseDirectory, path)));
+const automatedPassed = commands.slice(0, 3).every((command) => command.exit_code === 0);
+const readinessBlocked = externalCommands.every((command) => command.exit_code === 3);
+const blockersByModel = Object.fromEntries(modelEvidence.map((entry) => [entry.model_id, entry.blockers]));
+const status = automatedPassed && readinessBlocked && missingEvidence.length === 0 ? "externally_blocked" : "failed";
+const commit = gitOutput(["rev-parse", "HEAD"]);
+const dirty = gitOutput(["status", "--porcelain"]).length > 0;
+const result = {
+  acceptance_criteria: ["AC-40", "AC-41"],
+  automation: ["controlled_real", "manual_review"],
+  blockers_by_model: blockersByModel,
+  candidate_run_id: candidateRunId,
+  commit,
+  evidence_refs: evidenceRefs,
+  fixture_ids: [],
+  green_assertions: ["Each model independently passes the complete controlled-real matrix against its final config version."],
+  layer: ["EXT-REAL", "MANUAL"],
+  manifest: { path: "tasks.manifest.json", sha256: sha256("tasks.manifest.json") },
+  missing_evidence: missingEvidence,
+  phase: "controlled_real_readiness",
+  real_calls: 0,
+  red_reason: "任一模型缺独立真实契约证据",
+  release_gate: ["release:P0-A"],
+  requirements: ["GEN-13"],
+  run_id: runId,
+  status,
+  task_id: "TASK-WP7-02",
+  test_id: "TDD-WP7-EXT-001-three-real-models",
+  work_package: "WP-7",
+  worktree_under_test: dirty ? "uncommitted implementation" : "clean committed implementation",
+};
+writeFileSync(resolve(caseDirectory, "result.json"), `${JSON.stringify(result, null, 2)}\n`);
+writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify({
+  cases: [{ blockers_by_model: blockersByModel, missing_evidence: missingEvidence, status, test_id: result.test_id }],
+  candidate_run_id: candidateRunId,
+  commit,
+  phase: result.phase,
+  real_calls: 0,
+  redaction_scan: "passed",
+  run_id: runId,
+  status,
+  task_id: result.task_id,
+}, null, 2)}\n`);
+console.log(JSON.stringify({ blockers_by_model: blockersByModel, candidate_run_id: candidateRunId, real_calls: 0, run_id: runId, status }, null, 2));
+if (status === "failed") process.exit(1);