feat: execute WP7-02 independent model matrices

scripts/validate-external.mjs

@@ -1,10 +1,20 @@
-import { existsSync, readFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
 
 import {
+  WP7_02_CONTROLLED_REAL_LIMIT,
+  validateSanitizedEvidence,
+} from "./lib/wp7-02-controlled-executor.mjs";
+import {
+  assembleControlledModelEvidence,
+  runControlledRealScenarios,
+} from "./lib/wp7-02-controlled-matrix.mjs";
+import {
+  AI_GATEWAY_CREDENTIAL_TARGET,
   WP7_02_MODEL_IDS,
   buildBlockedModelEvidence,
   inspectAiGatewayReadiness,
-  probeAiGatewayCredentialTargets,
   writeBlockedModelEvidence,
 } from "./lib/wp7-02-external-contract.mjs";
 
@@ -15,13 +25,21 @@ function argument(name) {
   return index >= 0 ? process.argv[index + 1] : undefined;
 }
 
+function output(value, error = false) {
+  const serialized = JSON.stringify(validateSanitizedEvidence(value));
+  if (error) console.error(serialized); else console.log(serialized);
+}
+
 const service = argument("--service");
 const runId = argument("--run-id");
 const model = argument("--model");
 const candidatePath = argument("--candidate-record") ?? process.env.DADA_WP7_01_CANDIDATE_RECORD;
 const configPath = argument("--config-manifest") ?? process.env.DADA_WP7_02_MODEL_CONFIG_MANIFEST;
-const evidenceDirectory = argument("--evidence-dir");
+const evidenceDirectory = argument("--evidence-dir") ?? process.env.DADA_WP7_02_EVIDENCE_DIR;
+const maxRealCalls = Number(argument("--max-real-calls"));
 const confirmed = process.argv.includes("--confirm-controlled-real");
+const executeControlledReal = process.argv.includes("--execute-controlled-real");
+const credentialStdin = process.argv.includes("--credential-stdin");
 const readinessOnly = process.argv.includes("--readiness-only");
 
 if (!allowedServices.has(service) || !runId || ((service === "ai" || service === "ai-gateway-service-id") && !WP7_02_MODEL_IDS.includes(model))) {
@@ -30,69 +48,176 @@ if (!allowedServices.has(service) || !runId || ((service === "ai" || service ===
 }
 
 if (service !== "ai" && service !== "ai-gateway-service-id") {
-  console.log(JSON.stringify({
-    blocker: undefined,
-    mode: "mock",
-    model,
-    real_calls: 0,
-    run_id: runId,
-    service,
-    status: "not_applicable_for_TASK-WP0-01",
-  }));
+  console.log(JSON.stringify({ mode: "mock", real_calls: 0, run_id: runId, service, status: "not_applicable_for_TASK-WP0-01" }));
   process.exit(0);
 }
 
-try {
+function parseInputs() {
   const candidateRecord = candidatePath && existsSync(candidatePath) ? JSON.parse(readFileSync(candidatePath, "utf8")) : undefined;
   const configManifest = configPath && existsSync(configPath) ? JSON.parse(readFileSync(configPath, "utf8")) : undefined;
   const modelConfig = Array.isArray(configManifest?.models)
     ? configManifest.models.find((entry) => entry?.model_id === model)
     : undefined;
+  return { candidateRecord, modelConfig };
+}
 
-  if (!candidateRecord) {
-    console.log(JSON.stringify({
-      blockers: ["candidate_record_absent", ...(confirmed ? [] : ["explicit_confirmation_absent"])],
-      mode: "controlled_real_not_executed", model, real_calls: 0, run_id: runId, service, status: "externally_blocked",
-    }));
-    process.exit(3);
+function delegateToSecureBroker() {
+  if (!candidatePath || !configPath || !evidenceDirectory || !confirmed || maxRealCalls !== WP7_02_CONTROLLED_REAL_LIMIT) {
+    output({ code: "WP7_02_CONTROLLED_EXECUTION_ARGUMENTS_REQUIRED", model, real_calls: 0, run_id: runId, service, status: "failed" }, true);
+    return 2;
   }
-
-  const readiness = inspectAiGatewayReadiness({
-    candidateRecord,
-    confirmed,
-    credentialTargets: probeAiGatewayCredentialTargets(),
-    modelConfig,
-    modelId: model,
+  const args = [
+    "run", "--no-build", "--project", "supervisor/Dada.Supervisor/Dada.Supervisor.csproj", "--",
+    "validate-external",
+    "--service", "ai-gateway-service-id",
+    "--model", model,
+    "--run-id", runId,
+    "--max-real-calls", String(maxRealCalls),
+    "--confirm-controlled-real",
+    "--execute-controlled-real",
+  ];
+  const result = spawnSync("dotnet", args, {
+    encoding: "utf8",
+    env: {
+      ...process.env,
+      DADA_WP7_01_CANDIDATE_RECORD: candidatePath,
+      DADA_WP7_02_EVIDENCE_DIR: evidenceDirectory,
+      DADA_WP7_02_MODEL_CONFIG_MANIFEST: configPath,
+    },
+    maxBuffer: 64 * 1024 * 1024,
+    timeout: 20 * 60_000,
+    windowsHide: true,
   });
-  if (!readinessOnly && readiness.blockers.length === 0) readiness.blockers.push("controlled_real_executor_unavailable");
-  readiness.status = readiness.blockers.length > 0 ? "externally_blocked" : "ready_for_controlled_execution";
-  if (readiness.blockers.length > 0 && evidenceDirectory) {
-    const evidence = buildBlockedModelEvidence({
-      blockers: readiness.blockers,
-      candidateRecord,
-      modelConfig: readiness.model_config,
-      modelId: model,
-      runId,
-    });
-    writeBlockedModelEvidence(evidenceDirectory, evidence);
+  const stdout = result.stdout?.trim() ?? "";
+  const stderr = result.stderr?.trim() ?? "";
+  const selected = stdout || stderr;
+  try {
+    if (!selected || (stdout && stderr)) throw new Error("invalid_output");
+    const parsed = validateSanitizedEvidence(JSON.parse(selected));
+    output(parsed, !stdout);
+  } catch {
+    output({ code: "WP7_02_SECURE_BROKER_FAILED", model, real_calls: 0, run_id: runId, service, status: "failed" }, true);
+    return 1;
   }
-  console.log(JSON.stringify({
-    blockers: readiness.blockers,
-    candidate_build_commit: readiness.candidate.build_commit,
-    mode: readinessOnly ? "readiness_only" : "controlled_real_not_executed",
-    model,
-    planned_provider_requests_max: readiness.plan.planned_provider_requests_max,
-    planned_request_breakdown: readiness.plan.planned_request_breakdown,
-    quota_impact: readiness.plan.quota_impact,
-    real_calls: 0,
-    run_id: runId,
-    service,
-    status: readiness.status,
-  }));
-  process.exit(readiness.blockers.length > 0 ? 3 : 0);
+  return result.status ?? 1;
+}
+
+async function readCredentialFromStdin() {
+  let serialized = "";
+  for await (const chunk of process.stdin) {
+    serialized += chunk.toString("utf8");
+    if (serialized.length > 16_384) throw new Error("WP7_02_CREDENTIAL_CHANNEL_INVALID");
+  }
+  const payload = JSON.parse(serialized);
+  serialized = "";
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)
+    || Object.keys(payload).length !== 1 || typeof payload[AI_GATEWAY_CREDENTIAL_TARGET] !== "string"
+    || payload[AI_GATEWAY_CREDENTIAL_TARGET].length < 8) {
+    throw new Error("WP7_02_CREDENTIAL_CHANNEL_INVALID");
+  }
+  const token = payload[AI_GATEWAY_CREDENTIAL_TARGET];
+  payload[AI_GATEWAY_CREDENTIAL_TARGET] = "";
+  return token;
+}
+
+function readDeterministicState() {
+  const path = evidenceDirectory && resolve(evidenceDirectory, "deterministic-state.json");
+  if (!path || !existsSync(path)) throw new Error("WP7_02_DETERMINISTIC_STATE_REQUIRED");
+  return validateSanitizedEvidence(JSON.parse(readFileSync(path, "utf8")));
+}
+
+function writeControlledEvidence(directory, evidence, readiness) {
+  mkdirSync(resolve(directory), { recursive: true });
+  const files = {
+    "contract-matrix.json": evidence.matrix,
+    "external-calls.json": evidence.external_calls,
+    "manual-review.json": evidence.manual_review,
+    "readiness.json": {
+      blockers: evidence.status === "passed" ? [] : evidence.external_calls.calls.filter((call) => call.status !== "passed").map((call) => call.error_code ?? call.scenario_id),
+      candidate: readiness.candidate,
+      evidence_id: evidence.evidence_id,
+      model_id: evidence.model_id,
+      run_id: evidence.run_id,
+      status: evidence.status,
+    },
+    "redaction.json": evidence.redaction,
+  };
+  for (const [name, value] of Object.entries(files)) {
+    writeFileSync(resolve(directory, name), `${JSON.stringify(validateSanitizedEvidence(value), null, 2)}\n`);
+  }
+}
+
+async function main() {
+  const { candidateRecord, modelConfig } = parseInputs();
+  if (!candidateRecord) {
+    output({ blockers: ["candidate_record_absent", ...(confirmed ? [] : ["explicit_confirmation_absent"])], mode: "controlled_real_not_executed", model, real_calls: 0, run_id: runId, service, status: "externally_blocked" });
+    return 3;
+  }
+
+  if (executeControlledReal && !credentialStdin) return delegateToSecureBroker();
+
+  let token = "";
+  try {
+    if (credentialStdin) token = await readCredentialFromStdin();
+    const readiness = inspectAiGatewayReadiness({
+      candidateRecord,
+      confirmed,
+      credentialTargets: credentialStdin ? [AI_GATEWAY_CREDENTIAL_TARGET] : [],
+      modelConfig,
+      modelId: model,
+    });
+    if (!credentialStdin) {
+      readiness.blockers = readiness.blockers.filter((blocker) => blocker !== "real_gateway_credentials_absent");
+      readiness.blockers.push("secure_credential_check_requires_execution");
+    }
+    readiness.status = readiness.blockers.length > 0 ? "externally_blocked" : "ready_for_controlled_execution";
+
+    if (!executeControlledReal || readinessOnly || readiness.blockers.length > 0) {
+      if (readiness.blockers.length > 0 && evidenceDirectory) {
+        writeBlockedModelEvidence(evidenceDirectory, buildBlockedModelEvidence({
+          blockers: readiness.blockers, candidateRecord, modelConfig: readiness.model_config, modelId: model, runId,
+        }));
+      }
+      output({
+        blockers: readiness.blockers,
+        candidate_build_commit: readiness.candidate.build_commit,
+        mode: readinessOnly ? "readiness_only" : "controlled_real_not_executed",
+        model,
+        planned_provider_requests_max: readiness.plan.planned_provider_requests_max,
+        planned_request_breakdown: readiness.plan.planned_request_breakdown,
+        real_calls: 0,
+        run_id: runId,
+        service,
+        status: readiness.status,
+      });
+      return readiness.blockers.length > 0 ? 3 : 0;
+    }
+
+    const deterministicState = readDeterministicState();
+    const realExecution = await runControlledRealScenarios({ maxRealCalls, modelConfig, token });
+    const evidence = assembleControlledModelEvidence({ deterministicState, modelConfig, realExecution, runId });
+    writeControlledEvidence(evidenceDirectory, evidence, readiness);
+    output({
+      blockers: realExecution.blockers,
+      config_version: modelConfig.config_version,
+      model,
+      planned_real_calls: realExecution.planned_real_calls,
+      real_calls: realExecution.real_calls,
+      run_id: runId,
+      service,
+      status: evidence.status === "passed" ? "controlled_real_passed_pending_manual_review" : "externally_blocked",
+    });
+    return evidence.status === "passed" ? 0 : 3;
+  } finally {
+    token = "";
+  }
+}
+
+try {
+  process.exitCode = await main();
 } catch (error) {
   const message = error instanceof Error ? error.message : String(error);
   const code = /^WP7_02_[A-Z0-9_:]+$/.test(message) ? message : "WP7_02_EXTERNAL_VALIDATION_FAILED";
-  console.error(JSON.stringify({ code, model, real_calls: 0, run_id: runId, service, status: "failed" }));
-  process.exit(1);
+  output({ code, model, real_calls: 0, run_id: runId, service, status: "failed" }, true);
+  process.exitCode = 1;
 }