feat: execute WP7-02 independent model matrices
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 54s
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 54s
This commit is contained in:
suyx
@@ -0,0 +1,232 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { deflateSync } from "node:zlib";
|
||||
|
||||
import {
|
||||
WP7_02_CONTROLLED_REAL_LIMIT,
|
||||
buildControlledExecutionPlan,
|
||||
executeProviderRequest,
|
||||
validateSanitizedEvidence,
|
||||
} from "./wp7-02-controlled-executor.mjs";
|
||||
|
||||
const ratioParts = Object.freeze({
|
||||
"3:4": [3, 4],
|
||||
"1:1": [1, 1],
|
||||
"4:3": [4, 3],
|
||||
"9:16": [9, 16],
|
||||
});
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex").toUpperCase();
|
||||
}
|
||||
|
||||
function crc32(bytes) {
|
||||
let crc = 0xffffffff;
|
||||
for (const byte of bytes) {
|
||||
crc ^= byte;
|
||||
for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));
|
||||
}
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function pngChunk(type, data) {
|
||||
const name = Buffer.from(type, "ascii");
|
||||
const length = Buffer.alloc(4);
|
||||
length.writeUInt32BE(data.length);
|
||||
const checksum = Buffer.alloc(4);
|
||||
checksum.writeUInt32BE(crc32(Buffer.concat([name, data])));
|
||||
return Buffer.concat([length, name, data, checksum]);
|
||||
}
|
||||
|
||||
export function createControlledReferencePng() {
|
||||
const width = 64;
|
||||
const height = 64;
|
||||
const rows = [];
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
const row = Buffer.alloc(1 + width * 4);
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const offset = 1 + x * 4;
|
||||
const bright = (Math.floor(x / 8) + Math.floor(y / 8)) % 2 === 0;
|
||||
row[offset] = bright ? 32 : 220;
|
||||
row[offset + 1] = bright ? 180 : 48;
|
||||
row[offset + 2] = bright ? 220 : 140;
|
||||
row[offset + 3] = 255;
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
const header = Buffer.alloc(13);
|
||||
header.writeUInt32BE(width, 0);
|
||||
header.writeUInt32BE(height, 4);
|
||||
header[8] = 8;
|
||||
header[9] = 6;
|
||||
return Buffer.concat([
|
||||
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
|
||||
pngChunk("IHDR", header),
|
||||
pngChunk("IDAT", deflateSync(Buffer.concat(rows))),
|
||||
pngChunk("IEND", Buffer.alloc(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
function promptForScenario(scenario) {
|
||||
const subject = scenario.input === "reference_image" ? "use the supplied geometric color reference" : "use a geometric color study";
|
||||
return `Create one safe abstract test image; ${subject}; no text, logos, people, or real places; aspect ratio ${scenario.ratio}.`;
|
||||
}
|
||||
|
||||
function ratioMatches(dimensions, ratio) {
|
||||
const [widthPart, heightPart] = ratioParts[ratio];
|
||||
return dimensions.width * heightPart === dimensions.height * widthPart;
|
||||
}
|
||||
|
||||
function safeErrorCode(error) {
|
||||
return error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)
|
||||
? error.message
|
||||
: "WP7_02_UPSTREAM_FAILED";
|
||||
}
|
||||
|
||||
export async function runControlledRealScenarios({ fetchImpl = fetch, maxRealCalls, modelConfig, token }) {
|
||||
const plan = buildControlledExecutionPlan(modelConfig);
|
||||
if (maxRealCalls !== WP7_02_CONTROLLED_REAL_LIMIT || plan.planned_real_calls > maxRealCalls) {
|
||||
throw new Error("WP7_02_REAL_CALL_LIMIT_INVALID");
|
||||
}
|
||||
const referenceBytes = createControlledReferencePng();
|
||||
const calls = [];
|
||||
for (let index = 0; index < plan.real_scenarios.length; index += 1) {
|
||||
const scenario = plan.real_scenarios[index];
|
||||
try {
|
||||
const result = await executeProviderRequest({
|
||||
fetchImpl,
|
||||
modelConfig,
|
||||
prompt: promptForScenario(scenario),
|
||||
ratio: scenario.ratio,
|
||||
reference: scenario.input === "reference_image" ? { bytes: referenceBytes, mime_type: "image/png" } : undefined,
|
||||
token,
|
||||
});
|
||||
const ratioPassed = ratioMatches(result.normalized.dimensions, scenario.ratio);
|
||||
calls.push(validateSanitizedEvidence({
|
||||
duration_ms: result.duration_ms,
|
||||
http_status: result.http_status,
|
||||
input: scenario.input,
|
||||
requested_ratio: scenario.ratio,
|
||||
response: result.response_evidence,
|
||||
scenario_id: `real-${index + 1}`,
|
||||
source: "real_gateway",
|
||||
status: ratioPassed ? "passed" : "failed",
|
||||
validation: { ratio: ratioPassed ? "passed" : "failed", response: "passed" },
|
||||
}));
|
||||
} catch (error) {
|
||||
calls.push({
|
||||
error_code: safeErrorCode(error),
|
||||
input: scenario.input,
|
||||
requested_ratio: scenario.ratio,
|
||||
scenario_id: `real-${index + 1}`,
|
||||
source: "real_gateway",
|
||||
status: "failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
referenceBytes.fill(0);
|
||||
const blockers = calls.filter((call) => call.status !== "passed").map((call) => `${call.scenario_id}:${call.error_code ?? "ratio_or_response_invalid"}`);
|
||||
return validateSanitizedEvidence({
|
||||
blockers,
|
||||
calls,
|
||||
model_id: modelConfig.model_id,
|
||||
planned_real_calls: plan.planned_real_calls,
|
||||
real_calls: calls.length,
|
||||
status: blockers.length === 0 ? "passed" : "externally_blocked",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDeterministicExecutionEvidence(modelId, runId) {
|
||||
const operationRef = `sha256:${sha256(`${modelId}:${runId}:operation`)}`;
|
||||
let state = "created";
|
||||
const trace = [];
|
||||
const start = () => {
|
||||
if (state !== "created") throw new Error("WP7_02_ASYNC_STATE_INVALID");
|
||||
state = "pending";
|
||||
trace.push({ action: "start", after: state, before: "created", status: "passed" });
|
||||
return operationRef;
|
||||
};
|
||||
const poll = (reference) => {
|
||||
if (reference !== operationRef || !["pending", "completed"].includes(state)) throw new Error("WP7_02_POLL_REFERENCE_INVALID");
|
||||
const before = state;
|
||||
state = "completed";
|
||||
trace.push({ action: "poll", after: state, before, replay: before === "completed", status: "passed" });
|
||||
return state;
|
||||
};
|
||||
const reference = start();
|
||||
poll(reference);
|
||||
poll(reference);
|
||||
return validateSanitizedEvidence({
|
||||
modes: [
|
||||
{ mode: "sync", source: "real_gateway", status: "covered_by_real_calls" },
|
||||
{ mode: "async", source: "deterministic_local", status: "passed", transition: "created_to_pending" },
|
||||
{ mode: "poll", operation_ref: operationRef, replay_count: 1, source: "deterministic_local", status: "passed", transition: "pending_to_completed" },
|
||||
],
|
||||
model_id: modelId,
|
||||
status: "passed",
|
||||
trace,
|
||||
});
|
||||
}
|
||||
|
||||
function passedCall(calls, predicate) {
|
||||
return calls.some((call) => call.status === "passed" && predicate(call));
|
||||
}
|
||||
|
||||
export function assembleControlledModelEvidence({ deterministicState, modelConfig, realExecution, runId }) {
|
||||
if (deterministicState?.model_id !== modelConfig.model_id || deterministicState?.status !== "passed") {
|
||||
throw new Error("WP7_02_DETERMINISTIC_STATE_INCOMPLETE");
|
||||
}
|
||||
const execution = buildDeterministicExecutionEvidence(modelConfig.model_id, runId);
|
||||
const ratioRows = Object.keys(ratioParts).map((ratio) => ({
|
||||
outputs: passedCall(realExecution.calls, (call) => call.requested_ratio === ratio) ? 1 : 0,
|
||||
ratio,
|
||||
status: passedCall(realExecution.calls, (call) => call.requested_ratio === ratio) ? "passed" : "failed",
|
||||
}));
|
||||
const pureTextPassed = ratioRows.every((row) => row.status === "passed")
|
||||
&& passedCall(realExecution.calls, (call) => call.input === "pure_text");
|
||||
const referencePassed = passedCall(realExecution.calls, (call) => call.input === "reference_image");
|
||||
const deterministicPassed = deterministicState.error_scenarios?.length === 9
|
||||
&& deterministicState.error_scenarios.every((entry) => entry.status === "passed")
|
||||
&& deterministicState.settlements?.length === 3
|
||||
&& deterministicState.contract_change?.full_matrix_reapplied === true;
|
||||
const status = realExecution.status === "passed" && pureTextPassed && referencePassed
|
||||
&& ratioRows.every((row) => row.status === "passed") && deterministicPassed ? "passed" : "externally_blocked";
|
||||
const evidenceId = `sha256:${sha256(`${modelConfig.model_id}:${modelConfig.config_version}:${runId}`)}`;
|
||||
return validateSanitizedEvidence({
|
||||
evidence_id: evidenceId,
|
||||
external_calls: {
|
||||
approved_real_call_limit: WP7_02_CONTROLLED_REAL_LIMIT,
|
||||
calls: realExecution.calls,
|
||||
mode: "controlled_real",
|
||||
planned_real_calls: realExecution.planned_real_calls,
|
||||
real_calls: realExecution.real_calls,
|
||||
service: "ai-gateway-service-id",
|
||||
status: realExecution.status,
|
||||
},
|
||||
manual_review: {
|
||||
decision: status === "passed" ? "Review sanitized matrix before recording the model as passed." : "Resolve all failed scenarios before review.",
|
||||
status: status === "passed" ? "pending" : "blocked",
|
||||
},
|
||||
matrix: {
|
||||
config_version: modelConfig.config_version,
|
||||
contract_change: deterministicState.contract_change,
|
||||
error_scenarios: deterministicState.error_scenarios,
|
||||
execution_modes: execution.modes,
|
||||
model_id: modelConfig.model_id,
|
||||
pure_text: { outputs: pureTextPassed ? 1 : 0, status: pureTextPassed ? "passed" : "failed" },
|
||||
ratios: ratioRows,
|
||||
reference_image: { outputs: referencePassed ? 1 : 0, status: referencePassed ? "passed" : "failed" },
|
||||
response_checks: ["single_image", "mime", "dimensions", "sanitized_usage", "evidence_hash"].map((name) => ({ name, status: realExecution.status })),
|
||||
settlements: deterministicState.settlements,
|
||||
status,
|
||||
},
|
||||
model_id: modelConfig.model_id,
|
||||
redaction: {
|
||||
forbidden_fields_absent: true,
|
||||
retained_fields: ["status", "category", "duration_ms", "mime", "dimensions", "usage_summary", "evidence_hash", "time"],
|
||||
secret_scan: "passed",
|
||||
status: "passed",
|
||||
},
|
||||
run_id: runId,
|
||||
status,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user