224 lines
9.1 KiB
JavaScript
224 lines
9.1 KiB
JavaScript
import { spawnSync } from "node:child_process";
|
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
|
|
import {
|
|
WP7_02_CONTROLLED_REAL_LIMIT,
|
|
validateSanitizedEvidence,
|
|
} from "./lib/wp7-02-controlled-executor.mjs";
|
|
import {
|
|
assembleControlledModelEvidence,
|
|
runControlledRealScenarios,
|
|
} from "./lib/wp7-02-controlled-matrix.mjs";
|
|
import {
|
|
AI_GATEWAY_CREDENTIAL_TARGET,
|
|
WP7_02_MODEL_IDS,
|
|
buildBlockedModelEvidence,
|
|
inspectAiGatewayReadiness,
|
|
writeBlockedModelEvidence,
|
|
} from "./lib/wp7-02-external-contract.mjs";
|
|
|
|
const allowedServices = new Set(["ai", "ai-gateway-service-id", "resend", "amap"]);
|
|
|
|
function argument(name) {
|
|
const index = process.argv.indexOf(name);
|
|
return index >= 0 ? process.argv[index + 1] : undefined;
|
|
}
|
|
|
|
function output(value, error = false) {
|
|
const serialized = JSON.stringify(validateSanitizedEvidence(value));
|
|
if (error) console.error(serialized); else console.log(serialized);
|
|
}
|
|
|
|
const service = argument("--service");
|
|
const runId = argument("--run-id");
|
|
const model = argument("--model");
|
|
const candidatePath = argument("--candidate-record") ?? process.env.DADA_WP7_01_CANDIDATE_RECORD;
|
|
const configPath = argument("--config-manifest") ?? process.env.DADA_WP7_02_MODEL_CONFIG_MANIFEST;
|
|
const evidenceDirectory = argument("--evidence-dir") ?? process.env.DADA_WP7_02_EVIDENCE_DIR;
|
|
const maxRealCalls = Number(argument("--max-real-calls"));
|
|
const confirmed = process.argv.includes("--confirm-controlled-real");
|
|
const executeControlledReal = process.argv.includes("--execute-controlled-real");
|
|
const credentialStdin = process.argv.includes("--credential-stdin");
|
|
const readinessOnly = process.argv.includes("--readiness-only");
|
|
|
|
if (!allowedServices.has(service) || !runId || ((service === "ai" || service === "ai-gateway-service-id") && !WP7_02_MODEL_IDS.includes(model))) {
|
|
console.error("Usage: pnpm validate:external -- --service <ai|ai-gateway-service-id|resend|amap> --run-id <id> [--model <model-id>] [--readiness-only]");
|
|
process.exit(2);
|
|
}
|
|
|
|
if (service !== "ai" && service !== "ai-gateway-service-id") {
|
|
console.log(JSON.stringify({ mode: "mock", real_calls: 0, run_id: runId, service, status: "not_applicable_for_TASK-WP0-01" }));
|
|
process.exit(0);
|
|
}
|
|
|
|
function parseInputs() {
|
|
const candidateRecord = candidatePath && existsSync(candidatePath) ? JSON.parse(readFileSync(candidatePath, "utf8")) : undefined;
|
|
const configManifest = configPath && existsSync(configPath) ? JSON.parse(readFileSync(configPath, "utf8")) : undefined;
|
|
const modelConfig = Array.isArray(configManifest?.models)
|
|
? configManifest.models.find((entry) => entry?.model_id === model)
|
|
: undefined;
|
|
return { candidateRecord, modelConfig };
|
|
}
|
|
|
|
function delegateToSecureBroker() {
|
|
if (!candidatePath || !configPath || !evidenceDirectory || !confirmed || maxRealCalls !== WP7_02_CONTROLLED_REAL_LIMIT) {
|
|
output({ code: "WP7_02_CONTROLLED_EXECUTION_ARGUMENTS_REQUIRED", model, real_calls: 0, run_id: runId, service, status: "failed" }, true);
|
|
return 2;
|
|
}
|
|
const args = [
|
|
"run", "--no-build", "--project", "supervisor/Dada.Supervisor/Dada.Supervisor.csproj", "--",
|
|
"validate-external",
|
|
"--service", "ai-gateway-service-id",
|
|
"--model", model,
|
|
"--run-id", runId,
|
|
"--max-real-calls", String(maxRealCalls),
|
|
"--confirm-controlled-real",
|
|
"--execute-controlled-real",
|
|
];
|
|
const result = spawnSync("dotnet", args, {
|
|
encoding: "utf8",
|
|
env: {
|
|
...process.env,
|
|
DADA_WP7_01_CANDIDATE_RECORD: candidatePath,
|
|
DADA_WP7_02_EVIDENCE_DIR: evidenceDirectory,
|
|
DADA_WP7_02_MODEL_CONFIG_MANIFEST: configPath,
|
|
},
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
timeout: 20 * 60_000,
|
|
windowsHide: true,
|
|
});
|
|
const stdout = result.stdout?.trim() ?? "";
|
|
const stderr = result.stderr?.trim() ?? "";
|
|
const selected = stdout || stderr;
|
|
try {
|
|
if (!selected || (stdout && stderr)) throw new Error("invalid_output");
|
|
const parsed = validateSanitizedEvidence(JSON.parse(selected));
|
|
output(parsed, !stdout);
|
|
} catch {
|
|
output({ code: "WP7_02_SECURE_BROKER_FAILED", model, real_calls: 0, run_id: runId, service, status: "failed" }, true);
|
|
return 1;
|
|
}
|
|
return result.status ?? 1;
|
|
}
|
|
|
|
async function readCredentialFromStdin() {
|
|
let serialized = "";
|
|
for await (const chunk of process.stdin) {
|
|
serialized += chunk.toString("utf8");
|
|
if (serialized.length > 16_384) throw new Error("WP7_02_CREDENTIAL_CHANNEL_INVALID");
|
|
}
|
|
const payload = JSON.parse(serialized);
|
|
serialized = "";
|
|
if (!payload || typeof payload !== "object" || Array.isArray(payload)
|
|
|| Object.keys(payload).length !== 1 || typeof payload[AI_GATEWAY_CREDENTIAL_TARGET] !== "string"
|
|
|| payload[AI_GATEWAY_CREDENTIAL_TARGET].length < 8) {
|
|
throw new Error("WP7_02_CREDENTIAL_CHANNEL_INVALID");
|
|
}
|
|
const token = payload[AI_GATEWAY_CREDENTIAL_TARGET];
|
|
payload[AI_GATEWAY_CREDENTIAL_TARGET] = "";
|
|
return token;
|
|
}
|
|
|
|
function readDeterministicState() {
|
|
const path = evidenceDirectory && resolve(evidenceDirectory, "deterministic-state.json");
|
|
if (!path || !existsSync(path)) throw new Error("WP7_02_DETERMINISTIC_STATE_REQUIRED");
|
|
return validateSanitizedEvidence(JSON.parse(readFileSync(path, "utf8")));
|
|
}
|
|
|
|
function writeControlledEvidence(directory, evidence, readiness) {
|
|
mkdirSync(resolve(directory), { recursive: true });
|
|
const files = {
|
|
"contract-matrix.json": evidence.matrix,
|
|
"external-calls.json": evidence.external_calls,
|
|
"manual-review.json": evidence.manual_review,
|
|
"readiness.json": {
|
|
blockers: evidence.status === "passed" ? [] : evidence.external_calls.calls.filter((call) => call.status !== "passed").map((call) => call.error_code ?? call.scenario_id),
|
|
candidate: readiness.candidate,
|
|
evidence_id: evidence.evidence_id,
|
|
model_id: evidence.model_id,
|
|
run_id: evidence.run_id,
|
|
status: evidence.status,
|
|
},
|
|
"redaction.json": evidence.redaction,
|
|
};
|
|
for (const [name, value] of Object.entries(files)) {
|
|
writeFileSync(resolve(directory, name), `${JSON.stringify(validateSanitizedEvidence(value), null, 2)}\n`);
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const { candidateRecord, modelConfig } = parseInputs();
|
|
if (!candidateRecord) {
|
|
output({ blockers: ["candidate_record_absent", ...(confirmed ? [] : ["explicit_confirmation_absent"])], mode: "controlled_real_not_executed", model, real_calls: 0, run_id: runId, service, status: "externally_blocked" });
|
|
return 3;
|
|
}
|
|
|
|
if (executeControlledReal && !credentialStdin) return delegateToSecureBroker();
|
|
|
|
let token = "";
|
|
try {
|
|
if (credentialStdin) token = await readCredentialFromStdin();
|
|
const readiness = inspectAiGatewayReadiness({
|
|
candidateRecord,
|
|
confirmed,
|
|
credentialTargets: credentialStdin ? [AI_GATEWAY_CREDENTIAL_TARGET] : [],
|
|
modelConfig,
|
|
modelId: model,
|
|
});
|
|
if (!credentialStdin) {
|
|
readiness.blockers = readiness.blockers.filter((blocker) => blocker !== "real_gateway_credentials_absent");
|
|
readiness.blockers.push("secure_credential_check_requires_execution");
|
|
}
|
|
readiness.status = readiness.blockers.length > 0 ? "externally_blocked" : "ready_for_controlled_execution";
|
|
|
|
if (!executeControlledReal || readinessOnly || readiness.blockers.length > 0) {
|
|
if (readiness.blockers.length > 0 && evidenceDirectory) {
|
|
writeBlockedModelEvidence(evidenceDirectory, buildBlockedModelEvidence({
|
|
blockers: readiness.blockers, candidateRecord, modelConfig: readiness.model_config, modelId: model, runId,
|
|
}));
|
|
}
|
|
output({
|
|
blockers: readiness.blockers,
|
|
candidate_build_commit: readiness.candidate.build_commit,
|
|
mode: readinessOnly ? "readiness_only" : "controlled_real_not_executed",
|
|
model,
|
|
planned_provider_requests_max: readiness.plan.planned_provider_requests_max,
|
|
planned_request_breakdown: readiness.plan.planned_request_breakdown,
|
|
real_calls: 0,
|
|
run_id: runId,
|
|
service,
|
|
status: readiness.status,
|
|
});
|
|
return readiness.blockers.length > 0 ? 3 : 0;
|
|
}
|
|
|
|
const deterministicState = readDeterministicState();
|
|
const realExecution = await runControlledRealScenarios({ maxRealCalls, modelConfig, token });
|
|
const evidence = assembleControlledModelEvidence({ deterministicState, modelConfig, realExecution, runId });
|
|
writeControlledEvidence(evidenceDirectory, evidence, readiness);
|
|
output({
|
|
blockers: realExecution.blockers,
|
|
config_version: modelConfig.config_version,
|
|
model,
|
|
planned_real_calls: realExecution.planned_real_calls,
|
|
real_calls: realExecution.real_calls,
|
|
run_id: runId,
|
|
service,
|
|
status: evidence.status === "passed" ? "controlled_real_passed_pending_manual_review" : "externally_blocked",
|
|
});
|
|
return evidence.status === "passed" ? 0 : 3;
|
|
} finally {
|
|
token = "";
|
|
}
|
|
}
|
|
|
|
try {
|
|
process.exitCode = await main();
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
const code = /^WP7_02_[A-Z0-9_:]+$/.test(message) ? message : "WP7_02_EXTERNAL_VALIDATION_FAILED";
|
|
output({ code, model, real_calls: 0, run_id: runId, service, status: "failed" }, true);
|
|
process.exitCode = 1;
|
|
}
|