diff --git a/scripts/lib/wp7-02-controlled-executor.mjs b/scripts/lib/wp7-02-controlled-executor.mjs index 1b19a9d..6b11163 100644 --- a/scripts/lib/wp7-02-controlled-executor.mjs +++ b/scripts/lib/wp7-02-controlled-executor.mjs @@ -192,6 +192,27 @@ export function normalizeProviderResponse(modelConfig, response) { }; } +export function describeProviderResponseShape(value, depth = 0) { + if (depth >= 6) return { kind: "depth_limit" }; + if (Array.isArray(value)) { + return { + item: value.length > 0 ? describeProviderResponseShape(value[0], depth + 1) : { kind: "empty" }, + kind: "array", + length: value.length, + }; + } + if (value && typeof value === "object") { + return { + fields: Object.keys(value).toSorted().map((name) => ({ name, shape: describeProviderResponseShape(value[name], depth + 1) })), + kind: "object", + }; + } + if (typeof value === "string") return { kind: "string", size: value.length === 0 ? "empty" : value.length > 256 ? "large" : "small" }; + if (typeof value === "number") return { kind: "number" }; + if (typeof value === "boolean") return { kind: "boolean" }; + return { kind: value === null ? "null" : "undefined" }; +} + export function buildSanitizedResponseEvidence(normalized) { const evidence = { dimensions: structuredClone(normalized.dimensions), @@ -237,7 +258,16 @@ export async function executeProviderRequest({ fetchImpl = fetch, modelConfig, p }); const durationMs = Math.round(performance.now() - startedAt); if (!response.ok) throw new Error(`WP7_02_UPSTREAM_HTTP_${response.status}`); - const normalized = normalizeProviderResponse(modelConfig, await response.json()); + const providerResponse = await response.json(); + let normalized; + try { + normalized = normalizeProviderResponse(modelConfig, providerResponse); + } catch (error) { + if (error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)) { + error.safe_response_shape = describeProviderResponseShape(providerResponse); + } + throw error; + } return { duration_ms: durationMs, http_status: response.status, diff --git a/scripts/lib/wp7-02-controlled-matrix.mjs b/scripts/lib/wp7-02-controlled-matrix.mjs index 64a6066..ade9876 100644 --- a/scripts/lib/wp7-02-controlled-matrix.mjs +++ b/scripts/lib/wp7-02-controlled-matrix.mjs @@ -8,11 +8,11 @@ import { validateSanitizedEvidence, } from "./wp7-02-controlled-executor.mjs"; -const ratioParts = Object.freeze({ - "3:4": [3, 4], - "1:1": [1, 1], - "4:3": [4, 3], - "9:16": [9, 16], +const productDimensions = Object.freeze({ + "3:4": { height: 1440, width: 1080 }, + "1:1": { height: 1080, width: 1080 }, + "4:3": { height: 1080, width: 1440 }, + "9:16": { height: 1920, width: 1080 }, }); function sha256(value) { @@ -71,9 +71,9 @@ function promptForScenario(scenario) { return `Create one safe abstract test image; ${subject}; no text, logos, people, or real places; aspect ratio ${scenario.ratio}.`; } -function ratioMatches(dimensions, ratio) { - const [widthPart, heightPart] = ratioParts[ratio]; - return dimensions.width * heightPart === dimensions.height * widthPart; +function dimensionsMatch(dimensions, ratio) { + const expected = productDimensions[ratio]; + return dimensions.width === expected.width && dimensions.height === expected.height; } function safeErrorCode(error) { @@ -100,7 +100,7 @@ export async function runControlledRealScenarios({ fetchImpl = fetch, maxRealCal reference: scenario.input === "reference_image" ? { bytes: referenceBytes, mime_type: "image/png" } : undefined, token, }); - const ratioPassed = ratioMatches(result.normalized.dimensions, scenario.ratio); + const dimensionsPassed = dimensionsMatch(result.normalized.dimensions, scenario.ratio); calls.push(validateSanitizedEvidence({ duration_ms: result.duration_ms, http_status: result.http_status, @@ -109,22 +109,26 @@ export async function runControlledRealScenarios({ fetchImpl = fetch, maxRealCal response: result.response_evidence, scenario_id: `real-${index + 1}`, source: "real_gateway", - status: ratioPassed ? "passed" : "failed", - validation: { ratio: ratioPassed ? "passed" : "failed", response: "passed" }, + status: dimensionsPassed ? "passed" : "failed", + validation: { dimensions: dimensionsPassed ? "passed" : "failed", response: "passed" }, })); } catch (error) { - calls.push({ + const failed = { error_code: safeErrorCode(error), input: scenario.input, requested_ratio: scenario.ratio, scenario_id: `real-${index + 1}`, source: "real_gateway", status: "failed", - }); + ...(error?.safe_response_shape ? { response_shape: error.safe_response_shape } : {}), + }; + calls.push(validateSanitizedEvidence(failed)); + if (["WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED", "WP7_02_RESPONSE_MEDIA_INVALID", "WP7_02_CREDENTIAL_INVALID", + "WP7_02_UPSTREAM_HTTP_401", "WP7_02_UPSTREAM_HTTP_403", "WP7_02_UPSTREAM_HTTP_404", "WP7_02_UPSTREAM_HTTP_429"].includes(failed.error_code)) break; } } referenceBytes.fill(0); - const blockers = calls.filter((call) => call.status !== "passed").map((call) => `${call.scenario_id}:${call.error_code ?? "ratio_or_response_invalid"}`); + const blockers = calls.filter((call) => call.status !== "passed").map((call) => `${call.scenario_id}:${call.error_code ?? "dimensions_or_response_invalid"}`); return validateSanitizedEvidence({ blockers, calls, @@ -176,7 +180,7 @@ export function assembleControlledModelEvidence({ deterministicState, modelConfi throw new Error("WP7_02_DETERMINISTIC_STATE_INCOMPLETE"); } const execution = buildDeterministicExecutionEvidence(modelConfig.model_id, runId); - const ratioRows = Object.keys(ratioParts).map((ratio) => ({ + const ratioRows = Object.keys(productDimensions).map((ratio) => ({ outputs: passedCall(realExecution.calls, (call) => call.requested_ratio === ratio) ? 1 : 0, ratio, status: passedCall(realExecution.calls, (call) => call.requested_ratio === ratio) ? "passed" : "failed", diff --git a/scripts/record-wp7-02-manual-review.mjs b/scripts/record-wp7-02-manual-review.mjs index f1d6063..6147229 100644 --- a/scripts/record-wp7-02-manual-review.mjs +++ b/scripts/record-wp7-02-manual-review.mjs @@ -7,6 +7,7 @@ import { validateSanitizedEvidence } from "./lib/wp7-02-controlled-executor.mjs" const caseDirectory = process.env.DADA_WP7_02_CASE_DIR; const runId = process.env.DADA_TDD_RUN_ID; const confirmed = process.argv.includes("--confirm-manual-review"); +const productDimensions = { "3:4": [1080, 1440], "1:1": [1080, 1080], "4:3": [1440, 1080], "9:16": [1080, 1920] }; function readJson(path) { return validateSanitizedEvidence(JSON.parse(readFileSync(path, "utf8"))); @@ -39,6 +40,10 @@ try { && externalCalls.status === "passed" && externalCalls.real_calls === 5 && externalCalls.planned_real_calls === 5 && externalCalls.calls?.length === 5 && new Set(externalCalls.calls.map((entry) => entry.scenario_id)).size === 5 && externalCalls.calls.every((entry) => entry.status === "passed" && entry.source === "real_gateway") + && externalCalls.calls.every((entry) => { + const [width, height] = productDimensions[entry.requested_ratio] ?? []; + return entry.response?.dimensions?.width === width && entry.response?.dimensions?.height === height; + }) && externalCalls.approved_real_call_limit === 120 && readiness.status === "passed" && redaction.status === "passed" && redaction.secret_scan === "passed"; if (!complete || evidenceIds.has(readiness.evidence_id)) throw new Error("WP7_02_MANUAL_REVIEW_MATRIX_INCOMPLETE"); diff --git a/tests/package/wp7-02-controlled-executor.test.mjs b/tests/package/wp7-02-controlled-executor.test.mjs index 4f6e43a..3c6a8a7 100644 --- a/tests/package/wp7-02-controlled-executor.test.mjs +++ b/tests/package/wp7-02-controlled-executor.test.mjs @@ -6,6 +6,7 @@ import { buildControlledExecutionPlan, buildProviderRequest, buildSanitizedResponseEvidence, + describeProviderResponseShape, executeProviderRequest, normalizeProviderResponse, validateSanitizedEvidence, @@ -161,11 +162,29 @@ test("TDD-WP7-EXT-001 executes only five real success probes per model and keeps assert.equal(fetchCalls, 5); assert.equal(execution.real_calls, 5); assert.equal(execution.status, "externally_blocked"); - assert.equal(execution.calls.filter((call) => call.status === "passed").length, 2); + assert.equal(execution.calls.filter((call) => call.status === "passed").length, 0); assert.doesNotMatch(JSON.stringify(execution), /controlled-secret|fixture prompt|iVBOR/i); await assert.rejects(() => runControlledRealScenarios({ maxRealCalls: 121, modelConfig: models[0], token: "not-used" }), /WP7_02_REAL_CALL_LIMIT_INVALID/); }); +test("TDD-WP7-EXT-001 describes only protocol structure and stops repeated contract-shape calls", async () => { + let fetchCalls = 0; + const execution = await runControlledRealScenarios({ + fetchImpl: async () => { + fetchCalls += 1; + return new Response(JSON.stringify({ envelope: { outputs: [{ binary: "private-response-value" }] } }), { status: 200 }); + }, + maxRealCalls: 120, + modelConfig: models[0], + token: "controlled-secret-value-for-test-only", + }); + assert.equal(fetchCalls, 1); + assert.equal(execution.real_calls, 1); + assert.equal(execution.calls[0].error_code, "WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED"); + assert.deepEqual(execution.calls[0].response_shape, describeProviderResponseShape({ envelope: { outputs: [{ binary: "different-private-value" }] } })); + assert.doesNotMatch(JSON.stringify(execution.calls[0].response_shape), /private-response-value|different-private-value/); +}); + test("TDD-WP7-EXT-001 assembles independent complete evidence without retaining reference bytes", () => { const reference = createControlledReferencePng(); assert.equal(reference.subarray(0, 8).toString("hex"), "89504e470d0a1a0a");