235 lines
11 KiB
JavaScript
235 lines
11 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
import { spawnSync } from "node:child_process";
|
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
|
|
import {
|
|
WP7_02_MODEL_IDS,
|
|
validateCandidateDependency,
|
|
validateIndependentEvidenceSet,
|
|
} from "./lib/wp7-02-external-contract.mjs";
|
|
import { validateSanitizedEvidence } from "./lib/wp7-02-controlled-executor.mjs";
|
|
|
|
const wp701Sha = "623cad25b2a2a9a003502c9a92ebd318dad06248";
|
|
const candidateRunId = "wp7-01-candidate-20260804052447717";
|
|
const controlledReal = process.argv.includes("--controlled-real");
|
|
const runId = process.env.DADA_TDD_RUN_ID ?? `wp7-02-${controlledReal ? "controlled" : "readiness"}-${new Date().toISOString().replace(/[^0-9]/g, "")}`;
|
|
const runDirectory = resolve("artifacts", "tdd", runId);
|
|
const caseDirectory = resolve(runDirectory, "cases", "TDD-WP7-EXT-001-three-real-models");
|
|
const candidatePath = process.env.DADA_WP7_01_CANDIDATE_RECORD;
|
|
const configPath = resolve(process.env.DADA_WP7_02_MODEL_CONFIG_MANIFEST ?? "config/wp7-02-oneapi-test.json");
|
|
|
|
process.on("uncaughtException", (error) => {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
const code = /^WP7_02_[A-Z0-9_:]+$/.test(message) ? message : "WP7_02_VALIDATION_FAILED";
|
|
console.error(JSON.stringify({ code, real_calls: 0, run_id: runId, status: "failed" }));
|
|
process.exit(1);
|
|
});
|
|
|
|
if (existsSync(runDirectory)) throw new Error("WP7_02_EVIDENCE_RUN_ALREADY_EXISTS");
|
|
if (!candidatePath || !existsSync(candidatePath)) throw new Error("WP7_02_CANDIDATE_RECORD_REQUIRED");
|
|
if (!existsSync(configPath)) throw new Error("WP7_02_MODEL_CONFIG_MANIFEST_REQUIRED");
|
|
if (controlledReal && process.env.DADA_WP7_02_CONTROLLED_REAL_CONFIRMATION !== "authorized-120") {
|
|
throw new Error("WP7_02_CONTROLLED_REAL_CONFIRMATION_REQUIRED");
|
|
}
|
|
mkdirSync(caseDirectory, { recursive: true });
|
|
|
|
function sha256(path) {
|
|
return createHash("sha256").update(readFileSync(path)).digest("hex").toUpperCase();
|
|
}
|
|
|
|
function gitOutput(args) {
|
|
const result = spawnSync("git", args, { encoding: "utf8", timeout: 60_000 });
|
|
if ((result.status ?? 1) !== 0) throw new Error(`WP7_02_GIT_COMMAND_FAILED:${args[0]}`);
|
|
return result.stdout.trim();
|
|
}
|
|
|
|
function remoteSha(branch) {
|
|
const result = spawnSync("git", ["ls-remote", "--heads", "origin", `refs/heads/${branch}`], { encoding: "utf8", timeout: 60_000 });
|
|
if ((result.status ?? 1) !== 0) throw new Error("WP7_02_REMOTE_UNREADABLE");
|
|
return result.stdout.trim().split(/\s+/)[0];
|
|
}
|
|
|
|
function verifyUpstream() {
|
|
const remote = remoteSha("codex/wp7-01");
|
|
if (remote !== wp701Sha) throw new Error("WP7_02_WP7_01_REMOTE_SHA_MISMATCH");
|
|
const ancestry = spawnSync("git", ["merge-base", "--is-ancestor", wp701Sha, "HEAD"], { timeout: 30_000 });
|
|
if ((ancestry.status ?? 1) !== 0) throw new Error("WP7_02_WP7_01_NOT_ANCESTOR");
|
|
return remote;
|
|
}
|
|
|
|
function run(name, command, args, options = {}) {
|
|
const started_at = new Date().toISOString();
|
|
const result = spawnSync(command, args, {
|
|
encoding: "utf8",
|
|
env: { ...process.env, ...(options.env ?? {}) },
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
timeout: options.timeout ?? 300_000,
|
|
windowsHide: true,
|
|
});
|
|
return {
|
|
command: options.logicalCommand ?? [command, ...args].join(" "),
|
|
exit_code: result.status ?? 1,
|
|
finished_at: new Date().toISOString(),
|
|
name,
|
|
started_at,
|
|
};
|
|
}
|
|
|
|
function pnpmRun(name, commandLine, options = {}) {
|
|
const command = process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm";
|
|
const args = process.platform === "win32" ? ["/d", "/c", commandLine] : commandLine.replace(/^pnpm\s+/, "").split(" ");
|
|
return run(name, command, args, { ...options, logicalCommand: commandLine });
|
|
}
|
|
|
|
function readJson(path) {
|
|
return validateSanitizedEvidence(JSON.parse(readFileSync(path, "utf8")));
|
|
}
|
|
|
|
const upstreamRemoteSha = verifyUpstream();
|
|
const candidate = validateCandidateDependency(JSON.parse(readFileSync(candidatePath, "utf8")));
|
|
const commands = [
|
|
run("contract-harness", process.execPath, ["--test", "tests/package/wp7-02-external-contract.test.mjs", "tests/package/wp7-02-controlled-executor.test.mjs"], {
|
|
logicalCommand: "node --test tests/package/wp7-02-external-contract.test.mjs tests/package/wp7-02-controlled-executor.test.mjs",
|
|
}),
|
|
pnpmRun("deterministic-state", "pnpm exec vitest run tests/integration/wp7-02-controlled-state.test.ts", {
|
|
env: { DADA_WP7_02_STATE_EVIDENCE_ROOT: caseDirectory }, timeout: 120_000,
|
|
}),
|
|
run("supervisor-build", "dotnet", ["build", "supervisor/Dada.Supervisor/Dada.Supervisor.csproj", "--no-restore"], {
|
|
logicalCommand: "dotnet build supervisor/Dada.Supervisor/Dada.Supervisor.csproj --no-restore", timeout: 120_000,
|
|
}),
|
|
pnpmRun("tdd-trace", "pnpm validate:tdd-trace"),
|
|
pnpmRun("security", "pnpm test:security"),
|
|
];
|
|
const firstAutomationFailure = commands.find((command) => command.exit_code !== 0);
|
|
if (firstAutomationFailure) {
|
|
writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`);
|
|
console.error(JSON.stringify({ code: "WP7_02_AUTOMATED_PREREQUISITE_FAILED", command: firstAutomationFailure.command, exit_code: firstAutomationFailure.exit_code, real_calls: 0, run_id: runId, status: "failed" }));
|
|
process.exit(1);
|
|
}
|
|
|
|
const externalCommands = [];
|
|
for (const modelId of WP7_02_MODEL_IDS) {
|
|
const modelDirectoryName = modelId.replaceAll(".", "_");
|
|
const modelDirectory = resolve(caseDirectory, modelDirectoryName);
|
|
const flags = controlledReal
|
|
? `--max-real-calls 120 --confirm-controlled-real --execute-controlled-real`
|
|
: "--confirm-controlled-real --readiness-only";
|
|
const commandLine = `pnpm validate:external -- --service ai-gateway-service-id --model ${modelId} --run-id ${runId} ${flags}`;
|
|
externalCommands.push(pnpmRun(`${controlledReal ? "controlled" : "readiness"}-${modelId}`, commandLine, {
|
|
env: {
|
|
DADA_WP7_01_CANDIDATE_RECORD: candidatePath,
|
|
DADA_WP7_02_EVIDENCE_DIR: modelDirectory,
|
|
DADA_WP7_02_MODEL_CONFIG_MANIFEST: configPath,
|
|
},
|
|
timeout: 20 * 60_000,
|
|
}));
|
|
}
|
|
commands.push(...externalCommands);
|
|
|
|
const externalExitCodesValid = controlledReal
|
|
? externalCommands.every((command) => command.exit_code === 0 || command.exit_code === 3)
|
|
: externalCommands.every((command) => command.exit_code === 3);
|
|
if (!externalExitCodesValid) {
|
|
writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`);
|
|
const failed = externalCommands.find((command) => ![0, 3].includes(command.exit_code));
|
|
console.error(JSON.stringify({ code: "WP7_02_EXTERNAL_COMMAND_FAILED", command: failed?.command, exit_code: failed?.exit_code, real_calls: 0, run_id: runId, status: "failed" }));
|
|
process.exit(1);
|
|
}
|
|
|
|
if (controlledReal && externalCommands.every((command) => command.exit_code === 0)) {
|
|
const manualConfirmed = process.env.DADA_WP7_02_MANUAL_REVIEW_CONFIRMATION === "confirmed";
|
|
const manual = run("manual-review", process.execPath, ["scripts/record-wp7-02-manual-review.mjs", ...(manualConfirmed ? ["--confirm-manual-review"] : [])], {
|
|
env: { DADA_TDD_RUN_ID: runId, DADA_WP7_02_CASE_DIR: caseDirectory },
|
|
logicalCommand: `pnpm review:wp7-02${manualConfirmed ? " -- --confirm-manual-review" : ""}`,
|
|
});
|
|
commands.push(manual);
|
|
}
|
|
|
|
const modelEvidence = WP7_02_MODEL_IDS.map((modelId) => {
|
|
const directory = resolve(caseDirectory, modelId.replaceAll(".", "_"));
|
|
const readiness = readJson(resolve(directory, "readiness.json"));
|
|
return {
|
|
blockers: readiness.blockers,
|
|
candidate: readiness.candidate,
|
|
evidence_id: readiness.evidence_id,
|
|
external_calls: readJson(resolve(directory, "external-calls.json")),
|
|
manual_review: readJson(resolve(directory, "manual-review.json")),
|
|
matrix: readJson(resolve(directory, "contract-matrix.json")),
|
|
model_id: readiness.model_id,
|
|
redaction: readJson(resolve(directory, "redaction.json")),
|
|
run_id: readiness.run_id,
|
|
status: readiness.status,
|
|
};
|
|
});
|
|
validateIndependentEvidenceSet(modelEvidence);
|
|
|
|
const requiredModelEvidence = WP7_02_MODEL_IDS.flatMap((modelId) => {
|
|
const directory = modelId.replaceAll(".", "_");
|
|
return ["contract-matrix.json", "deterministic-state.json", "external-calls.json", "manual-review.json", "readiness.json", "redaction.json"]
|
|
.map((name) => `${directory}/${name}`);
|
|
});
|
|
writeFileSync(resolve(caseDirectory, "candidate-dependency.json"), `${JSON.stringify({
|
|
candidate_run_id: candidateRunId,
|
|
record: candidate,
|
|
remote_branch: "codex/wp7-01",
|
|
remote_commit: upstreamRemoteSha,
|
|
status: "passed",
|
|
}, null, 2)}\n`);
|
|
writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`);
|
|
|
|
const evidenceRefs = ["candidate-dependency.json", "commands.json", ...requiredModelEvidence];
|
|
const missingEvidence = evidenceRefs.filter((path) => !existsSync(resolve(caseDirectory, path)));
|
|
const allModelsPassed = modelEvidence.every((entry) => entry.status === "passed" && entry.manual_review.status === "passed");
|
|
const commit = gitOutput(["rev-parse", "HEAD"]);
|
|
const remoteCommit = remoteSha("codex/wp7-02");
|
|
const dirty = gitOutput(["status", "--porcelain"]).length > 0;
|
|
const deliveryMatched = !dirty && commit === remoteCommit;
|
|
const status = missingEvidence.length > 0 ? "failed"
|
|
: allModelsPassed && deliveryMatched ? "passed"
|
|
: allModelsPassed ? "green"
|
|
: "externally_blocked";
|
|
const blockersByModel = Object.fromEntries(modelEvidence.map((entry) => [entry.model_id, entry.blockers]));
|
|
const realCalls = modelEvidence.reduce((total, entry) => total + entry.external_calls.real_calls, 0);
|
|
if (realCalls > 120) throw new Error("WP7_02_REAL_CALL_LIMIT_EXCEEDED");
|
|
const result = {
|
|
acceptance_criteria: ["AC-40", "AC-41"],
|
|
automation: ["controlled_real", "manual_review"],
|
|
blockers_by_model: blockersByModel,
|
|
candidate_run_id: candidateRunId,
|
|
commit,
|
|
evidence_refs: evidenceRefs,
|
|
fixture_ids: ["FX-WP7-CONTROLLED-REFERENCE"],
|
|
layer: ["EXT-REAL", "MANUAL"],
|
|
manifest: { path: "tasks.manifest.json", sha256: sha256("tasks.manifest.json") },
|
|
missing_evidence: missingEvidence,
|
|
phase: controlledReal ? "controlled_real" : "controlled_real_readiness",
|
|
real_calls: realCalls,
|
|
red_reason: "任一模型缺独立真实契约证据",
|
|
release_gate: ["release:P0-A"],
|
|
remote_branch: "codex/wp7-02",
|
|
remote_commit: remoteCommit,
|
|
requirements: ["GEN-13"],
|
|
run_id: runId,
|
|
status,
|
|
task_id: "TASK-WP7-02",
|
|
test_id: "TDD-WP7-EXT-001-three-real-models",
|
|
work_package: "WP-7",
|
|
worktree_under_test: dirty ? "uncommitted implementation" : "clean committed implementation",
|
|
};
|
|
writeFileSync(resolve(caseDirectory, "result.json"), `${JSON.stringify(result, null, 2)}\n`);
|
|
writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify({
|
|
cases: [{ blockers_by_model: blockersByModel, missing_evidence: missingEvidence, status, test_id: result.test_id }],
|
|
candidate_run_id: candidateRunId,
|
|
commit,
|
|
phase: result.phase,
|
|
real_calls: realCalls,
|
|
redaction_scan: "passed",
|
|
remote_commit: remoteCommit,
|
|
run_id: runId,
|
|
status,
|
|
task_id: result.task_id,
|
|
}, null, 2)}\n`);
|
|
console.log(JSON.stringify({ blockers_by_model: blockersByModel, candidate_run_id: candidateRunId, real_calls: realCalls, run_id: runId, status }));
|
|
if (status === "failed") process.exit(1);
|