feat: execute WP7-02 independent model matrices
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 54s
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 54s
This commit is contained in:
@@ -10,6 +10,12 @@ import {
|
||||
normalizeProviderResponse,
|
||||
validateSanitizedEvidence,
|
||||
} from "../../scripts/lib/wp7-02-controlled-executor.mjs";
|
||||
import {
|
||||
assembleControlledModelEvidence,
|
||||
buildDeterministicExecutionEvidence,
|
||||
createControlledReferencePng,
|
||||
runControlledRealScenarios,
|
||||
} from "../../scripts/lib/wp7-02-controlled-matrix.mjs";
|
||||
|
||||
const models = [
|
||||
{
|
||||
@@ -82,7 +88,7 @@ test("TDD-WP7-EXT-001 builds protocol-specific requests without auth in argument
|
||||
model: "gpt-image-2",
|
||||
prompt: "controlled fixture prompt",
|
||||
response_format: "b64_json",
|
||||
size: "1024x1536",
|
||||
size: "1080x1920",
|
||||
});
|
||||
assert.equal("authorization" in openai.headers, false);
|
||||
});
|
||||
@@ -138,3 +144,65 @@ test("TDD-WP7-EXT-001 confines the credential to the request header and discards
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test("TDD-WP7-EXT-001 executes only five real success probes per model and keeps failed ratios blocking", async () => {
|
||||
let fetchCalls = 0;
|
||||
const execution = await runControlledRealScenarios({
|
||||
fetchImpl: async () => {
|
||||
fetchCalls += 1;
|
||||
return new Response(JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ inlineData: { data: onePixelPng.toString("base64"), mimeType: "image/png" } }] } }],
|
||||
}), { status: 200 });
|
||||
},
|
||||
maxRealCalls: 120,
|
||||
modelConfig: models[0],
|
||||
token: "controlled-secret-value-for-test-only",
|
||||
});
|
||||
assert.equal(fetchCalls, 5);
|
||||
assert.equal(execution.real_calls, 5);
|
||||
assert.equal(execution.status, "externally_blocked");
|
||||
assert.equal(execution.calls.filter((call) => call.status === "passed").length, 2);
|
||||
assert.doesNotMatch(JSON.stringify(execution), /controlled-secret|fixture prompt|iVBOR/i);
|
||||
await assert.rejects(() => runControlledRealScenarios({ maxRealCalls: 121, modelConfig: models[0], token: "not-used" }), /WP7_02_REAL_CALL_LIMIT_INVALID/);
|
||||
});
|
||||
|
||||
test("TDD-WP7-EXT-001 assembles independent complete evidence without retaining reference bytes", () => {
|
||||
const reference = createControlledReferencePng();
|
||||
assert.equal(reference.subarray(0, 8).toString("hex"), "89504e470d0a1a0a");
|
||||
const calls = ["3:4", "1:1", "4:3", "9:16"].map((ratio, index) => ({
|
||||
duration_ms: 1,
|
||||
http_status: 200,
|
||||
input: "pure_text",
|
||||
requested_ratio: ratio,
|
||||
response: { dimensions: { height: 1, width: 1 }, evidence_hash: `sha256:${"A".repeat(64)}`, mime: "image/png", usage_summary: { input_units: 0, output_units: 0, total_units: 0 } },
|
||||
scenario_id: `real-${index + 1}`,
|
||||
source: "real_gateway",
|
||||
status: "passed",
|
||||
}));
|
||||
calls.push({ ...calls[1], input: "reference_image", scenario_id: "real-5" });
|
||||
const deterministicState = {
|
||||
contract_change: { full_matrix_reapplied: true },
|
||||
error_scenarios: Array.from({ length: 9 }, (_, index) => ({ category: `category-${index}`, status: "passed" })),
|
||||
model_id: models[0].model_id,
|
||||
settlements: ["succeeded", "failed", "rejected"].map((outcome) => ({ outcome })),
|
||||
status: "passed",
|
||||
};
|
||||
const evidence = assembleControlledModelEvidence({
|
||||
deterministicState,
|
||||
modelConfig: models[0],
|
||||
realExecution: { calls, planned_real_calls: 5, real_calls: 5, status: "passed" },
|
||||
runId: "wp7-02-assembly-test",
|
||||
});
|
||||
assert.equal(evidence.status, "passed");
|
||||
assert.equal(evidence.manual_review.status, "pending");
|
||||
assert.equal(evidence.matrix.error_scenarios.length, 9);
|
||||
assert.doesNotMatch(JSON.stringify(evidence), /iVBOR|image_bytes|raw_prompt|authorization/i);
|
||||
const execution = buildDeterministicExecutionEvidence(models[0].model_id, "wp7-02-assembly-test");
|
||||
assert.deepEqual(execution.modes.map((entry) => entry.mode), ["sync", "async", "poll"]);
|
||||
assert.deepEqual(execution.trace.map((entry) => `${entry.action}:${entry.before}->${entry.after}`), [
|
||||
"start:created->pending",
|
||||
"poll:pending->completed",
|
||||
"poll:completed->completed",
|
||||
]);
|
||||
assert.equal(execution.trace[2].replay, true);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user