260 lines
10 KiB
JavaScript
260 lines
10 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
import { deflateSync } from "node:zlib";
|
|
|
|
import {
|
|
WP7_02_CONTROLLED_REAL_LIMIT,
|
|
buildControlledExecutionPlan,
|
|
executeProviderRequest,
|
|
validateSanitizedEvidence,
|
|
} from "./wp7-02-controlled-executor.mjs";
|
|
|
|
const productDimensions = Object.freeze({
|
|
"3:4": { height: 1440, width: 1080 },
|
|
"1:1": { height: 1080, width: 1080 },
|
|
"4:3": { height: 1080, width: 1440 },
|
|
"9:16": { height: 1920, width: 1080 },
|
|
});
|
|
|
|
function sha256(value) {
|
|
return createHash("sha256").update(value).digest("hex").toUpperCase();
|
|
}
|
|
|
|
function crc32(bytes) {
|
|
let crc = 0xffffffff;
|
|
for (const byte of bytes) {
|
|
crc ^= byte;
|
|
for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));
|
|
}
|
|
return (crc ^ 0xffffffff) >>> 0;
|
|
}
|
|
|
|
function pngChunk(type, data) {
|
|
const name = Buffer.from(type, "ascii");
|
|
const length = Buffer.alloc(4);
|
|
length.writeUInt32BE(data.length);
|
|
const checksum = Buffer.alloc(4);
|
|
checksum.writeUInt32BE(crc32(Buffer.concat([name, data])));
|
|
return Buffer.concat([length, name, data, checksum]);
|
|
}
|
|
|
|
export function createControlledReferencePng() {
|
|
const width = 64;
|
|
const height = 64;
|
|
const rows = [];
|
|
for (let y = 0; y < height; y += 1) {
|
|
const row = Buffer.alloc(1 + width * 4);
|
|
for (let x = 0; x < width; x += 1) {
|
|
const offset = 1 + x * 4;
|
|
const bright = (Math.floor(x / 8) + Math.floor(y / 8)) % 2 === 0;
|
|
row[offset] = bright ? 32 : 220;
|
|
row[offset + 1] = bright ? 180 : 48;
|
|
row[offset + 2] = bright ? 220 : 140;
|
|
row[offset + 3] = 255;
|
|
}
|
|
rows.push(row);
|
|
}
|
|
const header = Buffer.alloc(13);
|
|
header.writeUInt32BE(width, 0);
|
|
header.writeUInt32BE(height, 4);
|
|
header[8] = 8;
|
|
header[9] = 6;
|
|
return Buffer.concat([
|
|
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
|
|
pngChunk("IHDR", header),
|
|
pngChunk("IDAT", deflateSync(Buffer.concat(rows))),
|
|
pngChunk("IEND", Buffer.alloc(0)),
|
|
]);
|
|
}
|
|
|
|
function promptForScenario(scenario) {
|
|
const subject = scenario.input === "reference_image" ? "use the supplied geometric color reference" : "use a geometric color study";
|
|
return `Create one safe abstract test image; ${subject}; no text, logos, people, or real places; aspect ratio ${scenario.ratio}.`;
|
|
}
|
|
|
|
function dimensionsMatch(dimensions, ratio) {
|
|
const expected = productDimensions[ratio];
|
|
return dimensions.width === expected.width && dimensions.height === expected.height;
|
|
}
|
|
|
|
function safeErrorCode(error) {
|
|
return error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)
|
|
? error.message
|
|
: "WP7_02_UPSTREAM_FAILED";
|
|
}
|
|
|
|
export async function runControlledRealScenarios({ fetchImpl = fetch, maxRealCalls, modelConfig, token }) {
|
|
const plan = buildControlledExecutionPlan(modelConfig);
|
|
if (maxRealCalls !== WP7_02_CONTROLLED_REAL_LIMIT || plan.planned_real_calls > maxRealCalls) {
|
|
throw new Error("WP7_02_REAL_CALL_LIMIT_INVALID");
|
|
}
|
|
const referenceBytes = createControlledReferencePng();
|
|
const attempts = [];
|
|
const calls = [];
|
|
let timeoutRetriesRemaining = 1;
|
|
let stop = false;
|
|
for (let index = 0; index < plan.real_scenarios.length; index += 1) {
|
|
const scenario = plan.real_scenarios[index];
|
|
const scenarioId = `real-${index + 1}`;
|
|
let attemptNo = 0;
|
|
while (true) {
|
|
attemptNo += 1;
|
|
try {
|
|
const result = await executeProviderRequest({
|
|
fetchImpl,
|
|
modelConfig,
|
|
prompt: promptForScenario(scenario),
|
|
ratio: scenario.ratio,
|
|
reference: scenario.input === "reference_image" ? { bytes: referenceBytes, mime_type: "image/png" } : undefined,
|
|
token,
|
|
});
|
|
attempts.push(validateSanitizedEvidence({
|
|
attempt_no: attemptNo, duration_ms: result.duration_ms, http_status: result.http_status,
|
|
scenario_id: scenarioId, status: "passed",
|
|
}));
|
|
const dimensionsPassed = dimensionsMatch(result.normalized.dimensions, scenario.ratio);
|
|
calls.push(validateSanitizedEvidence({
|
|
duration_ms: result.duration_ms,
|
|
http_status: result.http_status,
|
|
input: scenario.input,
|
|
requested_ratio: scenario.ratio,
|
|
response: result.response_evidence,
|
|
scenario_id: scenarioId,
|
|
source: "real_gateway",
|
|
status: dimensionsPassed ? "passed" : "failed",
|
|
validation: { dimensions: dimensionsPassed ? "passed" : "failed", response: "passed" },
|
|
}));
|
|
break;
|
|
} catch (error) {
|
|
const errorCode = safeErrorCode(error);
|
|
attempts.push(validateSanitizedEvidence({ attempt_no: attemptNo, error_code: errorCode, scenario_id: scenarioId, status: "failed" }));
|
|
if (errorCode === "WP7_02_UPSTREAM_TIMEOUT" && timeoutRetriesRemaining > 0) {
|
|
timeoutRetriesRemaining -= 1;
|
|
continue;
|
|
}
|
|
const failed = {
|
|
error_code: errorCode,
|
|
input: scenario.input,
|
|
requested_ratio: scenario.ratio,
|
|
scenario_id: scenarioId,
|
|
source: "real_gateway",
|
|
status: "failed",
|
|
...(error?.safe_response_shape ? { response_shape: error.safe_response_shape } : {}),
|
|
};
|
|
calls.push(validateSanitizedEvidence(failed));
|
|
if (["WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED", "WP7_02_RESPONSE_MEDIA_INVALID", "WP7_02_CREDENTIAL_INVALID",
|
|
"WP7_02_UPSTREAM_HTTP_401", "WP7_02_UPSTREAM_HTTP_403", "WP7_02_UPSTREAM_HTTP_404", "WP7_02_UPSTREAM_HTTP_429"].includes(failed.error_code)) stop = true;
|
|
break;
|
|
}
|
|
}
|
|
if (stop) break;
|
|
}
|
|
referenceBytes.fill(0);
|
|
const blockers = calls.filter((call) => call.status !== "passed").map((call) => `${call.scenario_id}:${call.error_code ?? "dimensions_or_response_invalid"}`);
|
|
return validateSanitizedEvidence({
|
|
blockers,
|
|
attempts,
|
|
calls,
|
|
maximum_real_calls: plan.planned_real_calls + 1,
|
|
model_id: modelConfig.model_id,
|
|
planned_real_calls: plan.planned_real_calls,
|
|
real_calls: attempts.length,
|
|
status: blockers.length === 0 ? "passed" : "externally_blocked",
|
|
});
|
|
}
|
|
|
|
export function buildDeterministicExecutionEvidence(modelId, runId) {
|
|
const operationRef = `sha256:${sha256(`${modelId}:${runId}:operation`)}`;
|
|
let state = "created";
|
|
const trace = [];
|
|
const start = () => {
|
|
if (state !== "created") throw new Error("WP7_02_ASYNC_STATE_INVALID");
|
|
state = "pending";
|
|
trace.push({ action: "start", after: state, before: "created", status: "passed" });
|
|
return operationRef;
|
|
};
|
|
const poll = (reference) => {
|
|
if (reference !== operationRef || !["pending", "completed"].includes(state)) throw new Error("WP7_02_POLL_REFERENCE_INVALID");
|
|
const before = state;
|
|
state = "completed";
|
|
trace.push({ action: "poll", after: state, before, replay: before === "completed", status: "passed" });
|
|
return state;
|
|
};
|
|
const reference = start();
|
|
poll(reference);
|
|
poll(reference);
|
|
return validateSanitizedEvidence({
|
|
modes: [
|
|
{ mode: "sync", source: "real_gateway", status: "covered_by_real_calls" },
|
|
{ mode: "async", source: "deterministic_local", status: "passed", transition: "created_to_pending" },
|
|
{ mode: "poll", operation_ref: operationRef, replay_count: 1, source: "deterministic_local", status: "passed", transition: "pending_to_completed" },
|
|
],
|
|
model_id: modelId,
|
|
status: "passed",
|
|
trace,
|
|
});
|
|
}
|
|
|
|
function passedCall(calls, predicate) {
|
|
return calls.some((call) => call.status === "passed" && predicate(call));
|
|
}
|
|
|
|
export function assembleControlledModelEvidence({ deterministicState, modelConfig, realExecution, runId }) {
|
|
if (deterministicState?.model_id !== modelConfig.model_id || deterministicState?.status !== "passed") {
|
|
throw new Error("WP7_02_DETERMINISTIC_STATE_INCOMPLETE");
|
|
}
|
|
const execution = buildDeterministicExecutionEvidence(modelConfig.model_id, runId);
|
|
const ratioRows = Object.keys(productDimensions).map((ratio) => ({
|
|
outputs: passedCall(realExecution.calls, (call) => call.requested_ratio === ratio) ? 1 : 0,
|
|
ratio,
|
|
status: passedCall(realExecution.calls, (call) => call.requested_ratio === ratio) ? "passed" : "failed",
|
|
}));
|
|
const pureTextPassed = ratioRows.every((row) => row.status === "passed")
|
|
&& passedCall(realExecution.calls, (call) => call.input === "pure_text");
|
|
const referencePassed = passedCall(realExecution.calls, (call) => call.input === "reference_image");
|
|
const deterministicPassed = deterministicState.error_scenarios?.length === 9
|
|
&& deterministicState.error_scenarios.every((entry) => entry.status === "passed")
|
|
&& deterministicState.settlements?.length === 3
|
|
&& deterministicState.contract_change?.full_matrix_reapplied === true;
|
|
const status = realExecution.status === "passed" && pureTextPassed && referencePassed
|
|
&& ratioRows.every((row) => row.status === "passed") && deterministicPassed ? "passed" : "externally_blocked";
|
|
const evidenceId = `sha256:${sha256(`${modelConfig.model_id}:${modelConfig.config_version}:${runId}`)}`;
|
|
return validateSanitizedEvidence({
|
|
evidence_id: evidenceId,
|
|
external_calls: {
|
|
approved_real_call_limit: WP7_02_CONTROLLED_REAL_LIMIT,
|
|
calls: realExecution.calls,
|
|
mode: "controlled_real",
|
|
planned_real_calls: realExecution.planned_real_calls,
|
|
real_calls: realExecution.real_calls,
|
|
service: "ai-gateway-service-id",
|
|
status: realExecution.status,
|
|
},
|
|
manual_review: {
|
|
decision: status === "passed" ? "Review sanitized matrix before recording the model as passed." : "Resolve all failed scenarios before review.",
|
|
status: status === "passed" ? "pending" : "blocked",
|
|
},
|
|
matrix: {
|
|
config_version: modelConfig.config_version,
|
|
contract_change: deterministicState.contract_change,
|
|
error_scenarios: deterministicState.error_scenarios,
|
|
execution_modes: execution.modes,
|
|
model_id: modelConfig.model_id,
|
|
pure_text: { outputs: pureTextPassed ? 1 : 0, status: pureTextPassed ? "passed" : "failed" },
|
|
ratios: ratioRows,
|
|
reference_image: { outputs: referencePassed ? 1 : 0, status: referencePassed ? "passed" : "failed" },
|
|
response_checks: ["single_image", "mime", "dimensions", "sanitized_usage", "evidence_hash"].map((name) => ({ name, status: realExecution.status })),
|
|
settlements: deterministicState.settlements,
|
|
status,
|
|
},
|
|
model_id: modelConfig.model_id,
|
|
redaction: {
|
|
forbidden_fields_absent: true,
|
|
retained_fields: ["status", "category", "duration_ms", "mime", "dimensions", "usage_summary", "evidence_hash", "time"],
|
|
secret_scan: "passed",
|
|
status: "passed",
|
|
},
|
|
run_id: runId,
|
|
status,
|
|
});
|
|
}
|