test(wp7-02): require independent mixed evidence review
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 2m8s
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 2m8s
This commit is contained in:
@@ -256,11 +256,16 @@ test("TDD-WP7-EXT-001 assembles independent complete evidence without retaining
|
||||
const evidence = assembleControlledModelEvidence({
|
||||
deterministicState,
|
||||
modelConfig: models[0],
|
||||
realExecution: { calls, planned_real_calls: 5, real_calls: 5, status: "passed" },
|
||||
realExecution: {
|
||||
attempts: calls.map((call, index) => ({ attempt_no: 1, http_status: 200, scenario_id: call.scenario_id, status: "passed" })),
|
||||
calls, maximum_real_calls: 6, planned_real_calls: 5, real_calls: 5, status: "passed",
|
||||
},
|
||||
runId: "wp7-02-assembly-test",
|
||||
});
|
||||
assert.equal(evidence.status, "passed");
|
||||
assert.equal(evidence.manual_review.status, "pending");
|
||||
assert.equal(evidence.external_calls.attempts.length, 5);
|
||||
assert.equal(evidence.external_calls.maximum_real_calls, 6);
|
||||
assert.equal(evidence.matrix.error_scenarios.length, 9);
|
||||
assert.doesNotMatch(JSON.stringify(evidence), /iVBOR|image_bytes|raw_prompt|authorization/i);
|
||||
const execution = buildDeterministicExecutionEvidence(models[0].model_id, "wp7-02-assembly-test");
|
||||
|
||||
@@ -135,4 +135,14 @@ test("TDD-WP7-EXT-001 writes blocked evidence without mock or sensitive payloads
|
||||
const shared = structuredClone(evidence);
|
||||
shared[1].evidence_id = shared[0].evidence_id;
|
||||
assert.throws(() => validateIndependentEvidenceSet(shared), /WP7_02_SHARED_EVIDENCE_FORBIDDEN/);
|
||||
|
||||
const mixed = structuredClone(evidence);
|
||||
mixed[2].status = "passed";
|
||||
mixed[2].matrix = { model_id: mixed[2].model_id, status: "passed" };
|
||||
mixed[2].external_calls = { real_calls: 5, status: "passed" };
|
||||
mixed[2].manual_review = { status: "pending" };
|
||||
mixed[2].redaction = { status: "passed" };
|
||||
assert.doesNotThrow(() => validateIndependentEvidenceSet(mixed));
|
||||
mixed[2].manual_review = { status: "passed" };
|
||||
assert.doesNotThrow(() => validateIndependentEvidenceSet(mixed));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { reviewIndependentModelEvidence } from "../../scripts/lib/wp7-02-manual-review.mjs";
|
||||
|
||||
const modelIds = ["gemini-3.1-flash-image-preview", "gemini-3-pro-image-preview", "gpt-image-2"];
|
||||
const dimensions = { "3:4": [1080, 1440], "1:1": [1080, 1080], "4:3": [1440, 1080], "9:16": [1080, 1920] };
|
||||
|
||||
function entry(modelId, complete) {
|
||||
const calls = Object.entries(dimensions).map(([ratio, [width, height]], index) => ({
|
||||
input: "pure_text", requested_ratio: ratio, response: { dimensions: { height, width } },
|
||||
scenario_id: `real-${index + 1}`, source: "real_gateway", status: "passed",
|
||||
}));
|
||||
calls.push({ ...calls[1], input: "reference_image", scenario_id: "real-5" });
|
||||
return {
|
||||
externalCalls: complete ? {
|
||||
approved_real_call_limit: 120,
|
||||
attempts: calls.map((call) => ({ attempt_no: 1, scenario_id: call.scenario_id, status: "passed" })),
|
||||
calls, maximum_real_calls: 6, planned_real_calls: 5, real_calls: 5, status: "passed",
|
||||
} : { attempts: [{ attempt_no: 1, scenario_id: "real-1", status: "failed" }], calls: [], maximum_real_calls: 6, planned_real_calls: 5, real_calls: 1, status: "externally_blocked" },
|
||||
matrix: complete ? {
|
||||
config_version: 2,
|
||||
contract_change: { full_matrix_reapplied: true },
|
||||
error_scenarios: Array.from({ length: 9 }, () => ({ status: "passed" })),
|
||||
execution_modes: ["covered_by_real_calls", "passed", "passed"].map((status) => ({ status })),
|
||||
model_id: modelId,
|
||||
pure_text: { status: "passed" },
|
||||
ratios: Object.keys(dimensions).map((ratio) => ({ ratio, status: "passed" })),
|
||||
reference_image: { status: "passed" },
|
||||
settlements: [{}, {}, {}],
|
||||
status: "passed",
|
||||
} : { config_version: 2, model_id: modelId, status: "externally_blocked" },
|
||||
modelId,
|
||||
readiness: { evidence_id: `sha256:${modelId}`, status: complete ? "passed" : "externally_blocked" },
|
||||
redaction: { secret_scan: "passed", status: "passed" },
|
||||
};
|
||||
}
|
||||
|
||||
test("TDD-WP7-EXT-001 reviews each model independently when the set is mixed", () => {
|
||||
const result = reviewIndependentModelEvidence(modelIds.map((modelId, index) => entry(modelId, index === 2)), {
|
||||
reviewedAt: "2026-08-04T09:30:00.000Z",
|
||||
runId: "wp7-02-mixed-review",
|
||||
});
|
||||
assert.equal(result.status, "externally_blocked");
|
||||
assert.deepEqual(result.reviews.map((review) => [review.model_id, review.status]), [
|
||||
[modelIds[0], "blocked"], [modelIds[1], "blocked"], [modelIds[2], "passed"],
|
||||
]);
|
||||
});
|
||||
Reference in New Issue
Block a user