diff --git a/scripts/lib/wp7-02-controlled-executor.mjs b/scripts/lib/wp7-02-controlled-executor.mjs index c94f9d4..c5fa552 100644 --- a/scripts/lib/wp7-02-controlled-executor.mjs +++ b/scripts/lib/wp7-02-controlled-executor.mjs @@ -285,6 +285,10 @@ export async function executeProviderRequest({ fetchImpl = fetch, modelConfig, p } catch (error) { if (error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)) { error.safe_response_shape = describeProviderResponseShape(providerResponse); + } else if (error instanceof Error && error.message === "image_output_media_invalid") { + throw new Error("WP7_02_RESPONSE_MEDIA_INVALID"); + } else if (error instanceof Error && /^image_output_(?:aspect_ratio_mismatch|dimensions_missing|normalization_failed)$/.test(error.message)) { + throw new Error("WP7_02_RESPONSE_DIMENSIONS_INVALID"); } throw error; } diff --git a/scripts/lib/wp7-02-controlled-matrix.mjs b/scripts/lib/wp7-02-controlled-matrix.mjs index ade9876..5b90645 100644 --- a/scripts/lib/wp7-02-controlled-matrix.mjs +++ b/scripts/lib/wp7-02-controlled-matrix.mjs @@ -88,53 +88,76 @@ export async function runControlledRealScenarios({ fetchImpl = fetch, maxRealCal throw new Error("WP7_02_REAL_CALL_LIMIT_INVALID"); } const referenceBytes = createControlledReferencePng(); + const attempts = []; const calls = []; + let timeoutRetriesRemaining = 1; + let stop = false; for (let index = 0; index < plan.real_scenarios.length; index += 1) { const scenario = plan.real_scenarios[index]; - try { - const result = await executeProviderRequest({ - fetchImpl, - modelConfig, - prompt: promptForScenario(scenario), - ratio: scenario.ratio, - reference: scenario.input === "reference_image" ? { bytes: referenceBytes, mime_type: "image/png" } : undefined, - token, - }); - const dimensionsPassed = dimensionsMatch(result.normalized.dimensions, scenario.ratio); - calls.push(validateSanitizedEvidence({ - duration_ms: result.duration_ms, - http_status: result.http_status, - input: scenario.input, - requested_ratio: scenario.ratio, - response: result.response_evidence, - scenario_id: `real-${index + 1}`, - source: "real_gateway", - status: dimensionsPassed ? "passed" : "failed", - validation: { dimensions: dimensionsPassed ? "passed" : "failed", response: "passed" }, - })); - } catch (error) { - const failed = { - error_code: safeErrorCode(error), - input: scenario.input, - requested_ratio: scenario.ratio, - scenario_id: `real-${index + 1}`, - source: "real_gateway", - status: "failed", - ...(error?.safe_response_shape ? { response_shape: error.safe_response_shape } : {}), - }; - calls.push(validateSanitizedEvidence(failed)); - if (["WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED", "WP7_02_RESPONSE_MEDIA_INVALID", "WP7_02_CREDENTIAL_INVALID", - "WP7_02_UPSTREAM_HTTP_401", "WP7_02_UPSTREAM_HTTP_403", "WP7_02_UPSTREAM_HTTP_404", "WP7_02_UPSTREAM_HTTP_429"].includes(failed.error_code)) break; + const scenarioId = `real-${index + 1}`; + let attemptNo = 0; + while (true) { + attemptNo += 1; + try { + const result = await executeProviderRequest({ + fetchImpl, + modelConfig, + prompt: promptForScenario(scenario), + ratio: scenario.ratio, + reference: scenario.input === "reference_image" ? { bytes: referenceBytes, mime_type: "image/png" } : undefined, + token, + }); + attempts.push(validateSanitizedEvidence({ + attempt_no: attemptNo, duration_ms: result.duration_ms, http_status: result.http_status, + scenario_id: scenarioId, status: "passed", + })); + const dimensionsPassed = dimensionsMatch(result.normalized.dimensions, scenario.ratio); + calls.push(validateSanitizedEvidence({ + duration_ms: result.duration_ms, + http_status: result.http_status, + input: scenario.input, + requested_ratio: scenario.ratio, + response: result.response_evidence, + scenario_id: scenarioId, + source: "real_gateway", + status: dimensionsPassed ? "passed" : "failed", + validation: { dimensions: dimensionsPassed ? "passed" : "failed", response: "passed" }, + })); + break; + } catch (error) { + const errorCode = safeErrorCode(error); + attempts.push(validateSanitizedEvidence({ attempt_no: attemptNo, error_code: errorCode, scenario_id: scenarioId, status: "failed" })); + if (errorCode === "WP7_02_UPSTREAM_TIMEOUT" && timeoutRetriesRemaining > 0) { + timeoutRetriesRemaining -= 1; + continue; + } + const failed = { + error_code: errorCode, + input: scenario.input, + requested_ratio: scenario.ratio, + scenario_id: scenarioId, + source: "real_gateway", + status: "failed", + ...(error?.safe_response_shape ? { response_shape: error.safe_response_shape } : {}), + }; + calls.push(validateSanitizedEvidence(failed)); + if (["WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED", "WP7_02_RESPONSE_MEDIA_INVALID", "WP7_02_CREDENTIAL_INVALID", + "WP7_02_UPSTREAM_HTTP_401", "WP7_02_UPSTREAM_HTTP_403", "WP7_02_UPSTREAM_HTTP_404", "WP7_02_UPSTREAM_HTTP_429"].includes(failed.error_code)) stop = true; + break; + } } + if (stop) break; } referenceBytes.fill(0); const blockers = calls.filter((call) => call.status !== "passed").map((call) => `${call.scenario_id}:${call.error_code ?? "dimensions_or_response_invalid"}`); return validateSanitizedEvidence({ blockers, + attempts, calls, + maximum_real_calls: plan.planned_real_calls + 1, model_id: modelConfig.model_id, planned_real_calls: plan.planned_real_calls, - real_calls: calls.length, + real_calls: attempts.length, status: blockers.length === 0 ? "passed" : "externally_blocked", }); } diff --git a/scripts/record-wp7-02-manual-review.mjs b/scripts/record-wp7-02-manual-review.mjs index 6147229..4325862 100644 --- a/scripts/record-wp7-02-manual-review.mjs +++ b/scripts/record-wp7-02-manual-review.mjs @@ -37,7 +37,8 @@ try { && matrix.execution_modes?.length === 3 && matrix.execution_modes.every((entry) => ["passed", "covered_by_real_calls"].includes(entry.status)) && matrix.error_scenarios?.length === 9 && matrix.error_scenarios.every((entry) => entry.status === "passed") && matrix.settlements?.length === 3 && matrix.contract_change?.full_matrix_reapplied === true - && externalCalls.status === "passed" && externalCalls.real_calls === 5 && externalCalls.planned_real_calls === 5 + && externalCalls.status === "passed" && externalCalls.real_calls >= 5 && externalCalls.real_calls <= 6 && externalCalls.planned_real_calls === 5 + && externalCalls.maximum_real_calls === 6 && externalCalls.attempts?.length === externalCalls.real_calls && externalCalls.calls?.length === 5 && new Set(externalCalls.calls.map((entry) => entry.scenario_id)).size === 5 && externalCalls.calls.every((entry) => entry.status === "passed" && entry.source === "real_gateway") && externalCalls.calls.every((entry) => { diff --git a/tests/package/wp7-02-controlled-executor.test.mjs b/tests/package/wp7-02-controlled-executor.test.mjs index 68f849f..eb63f28 100644 --- a/tests/package/wp7-02-controlled-executor.test.mjs +++ b/tests/package/wp7-02-controlled-executor.test.mjs @@ -170,6 +170,7 @@ test("TDD-WP7-EXT-001 executes only five real success probes per model and keeps }); assert.equal(fetchCalls, 5); assert.equal(execution.real_calls, 5); + assert.equal(execution.attempts.length, 5); assert.equal(execution.status, "externally_blocked"); assert.equal(execution.calls.filter((call) => call.status === "passed").length, 2); assert.doesNotMatch(JSON.stringify(execution), /controlled-secret|fixture prompt|iVBOR/i); @@ -200,7 +201,7 @@ test("TDD-WP7-EXT-001 retries one timeout once and records every real attempt", assert.equal(execution.attempts.length, 6); assert.deepEqual(execution.attempts.slice(0, 2).map((attempt) => [attempt.scenario_id, attempt.attempt_no, attempt.error_code ?? attempt.status]), [ ["real-1", 1, "WP7_02_UPSTREAM_TIMEOUT"], - ["real-1", 2, "failed"], + ["real-1", 2, "WP7_02_RESPONSE_DIMENSIONS_INVALID"], ]); });