feat: execute WP7-02 independent model matrices
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 54s

This commit is contained in:
suyx
2026-08-04 16:01:31 +08:00
parent c2f89453a2
commit 470d243b5b
11 changed files with 811 additions and 142 deletions
@@ -10,6 +10,12 @@ import {
normalizeProviderResponse,
validateSanitizedEvidence,
} from "../../scripts/lib/wp7-02-controlled-executor.mjs";
import {
assembleControlledModelEvidence,
buildDeterministicExecutionEvidence,
createControlledReferencePng,
runControlledRealScenarios,
} from "../../scripts/lib/wp7-02-controlled-matrix.mjs";
const models = [
{
@@ -82,7 +88,7 @@ test("TDD-WP7-EXT-001 builds protocol-specific requests without auth in argument
model: "gpt-image-2",
prompt: "controlled fixture prompt",
response_format: "b64_json",
size: "1024x1536",
size: "1080x1920",
});
assert.equal("authorization" in openai.headers, false);
});
@@ -138,3 +144,65 @@ test("TDD-WP7-EXT-001 confines the credential to the request header and discards
return true;
});
});
test("TDD-WP7-EXT-001 executes only five real success probes per model and keeps failed ratios blocking", async () => {
let fetchCalls = 0;
const execution = await runControlledRealScenarios({
fetchImpl: async () => {
fetchCalls += 1;
return new Response(JSON.stringify({
candidates: [{ content: { parts: [{ inlineData: { data: onePixelPng.toString("base64"), mimeType: "image/png" } }] } }],
}), { status: 200 });
},
maxRealCalls: 120,
modelConfig: models[0],
token: "controlled-secret-value-for-test-only",
});
assert.equal(fetchCalls, 5);
assert.equal(execution.real_calls, 5);
assert.equal(execution.status, "externally_blocked");
assert.equal(execution.calls.filter((call) => call.status === "passed").length, 2);
assert.doesNotMatch(JSON.stringify(execution), /controlled-secret|fixture prompt|iVBOR/i);
await assert.rejects(() => runControlledRealScenarios({ maxRealCalls: 121, modelConfig: models[0], token: "not-used" }), /WP7_02_REAL_CALL_LIMIT_INVALID/);
});
test("TDD-WP7-EXT-001 assembles independent complete evidence without retaining reference bytes", () => {
const reference = createControlledReferencePng();
assert.equal(reference.subarray(0, 8).toString("hex"), "89504e470d0a1a0a");
const calls = ["3:4", "1:1", "4:3", "9:16"].map((ratio, index) => ({
duration_ms: 1,
http_status: 200,
input: "pure_text",
requested_ratio: ratio,
response: { dimensions: { height: 1, width: 1 }, evidence_hash: `sha256:${"A".repeat(64)}`, mime: "image/png", usage_summary: { input_units: 0, output_units: 0, total_units: 0 } },
scenario_id: `real-${index + 1}`,
source: "real_gateway",
status: "passed",
}));
calls.push({ ...calls[1], input: "reference_image", scenario_id: "real-5" });
const deterministicState = {
contract_change: { full_matrix_reapplied: true },
error_scenarios: Array.from({ length: 9 }, (_, index) => ({ category: `category-${index}`, status: "passed" })),
model_id: models[0].model_id,
settlements: ["succeeded", "failed", "rejected"].map((outcome) => ({ outcome })),
status: "passed",
};
const evidence = assembleControlledModelEvidence({
deterministicState,
modelConfig: models[0],
realExecution: { calls, planned_real_calls: 5, real_calls: 5, status: "passed" },
runId: "wp7-02-assembly-test",
});
assert.equal(evidence.status, "passed");
assert.equal(evidence.manual_review.status, "pending");
assert.equal(evidence.matrix.error_scenarios.length, 9);
assert.doesNotMatch(JSON.stringify(evidence), /iVBOR|image_bytes|raw_prompt|authorization/i);
const execution = buildDeterministicExecutionEvidence(models[0].model_id, "wp7-02-assembly-test");
assert.deepEqual(execution.modes.map((entry) => entry.mode), ["sync", "async", "poll"]);
assert.deepEqual(execution.trace.map((entry) => `${entry.action}:${entry.before}->${entry.after}`), [
"start:created->pending",
"poll:pending->completed",
"poll:completed->completed",
]);
assert.equal(execution.trace[2].replay, true);
});