fix(wp7-02): retry one transient image timeout
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 1m38s
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 1m38s
This commit is contained in:
@@ -285,6 +285,10 @@ export async function executeProviderRequest({ fetchImpl = fetch, modelConfig, p
|
||||
} catch (error) {
|
||||
if (error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)) {
|
||||
error.safe_response_shape = describeProviderResponseShape(providerResponse);
|
||||
} else if (error instanceof Error && error.message === "image_output_media_invalid") {
|
||||
throw new Error("WP7_02_RESPONSE_MEDIA_INVALID");
|
||||
} else if (error instanceof Error && /^image_output_(?:aspect_ratio_mismatch|dimensions_missing|normalization_failed)$/.test(error.message)) {
|
||||
throw new Error("WP7_02_RESPONSE_DIMENSIONS_INVALID");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -88,53 +88,76 @@ export async function runControlledRealScenarios({ fetchImpl = fetch, maxRealCal
|
||||
throw new Error("WP7_02_REAL_CALL_LIMIT_INVALID");
|
||||
}
|
||||
const referenceBytes = createControlledReferencePng();
|
||||
const attempts = [];
|
||||
const calls = [];
|
||||
let timeoutRetriesRemaining = 1;
|
||||
let stop = false;
|
||||
for (let index = 0; index < plan.real_scenarios.length; index += 1) {
|
||||
const scenario = plan.real_scenarios[index];
|
||||
try {
|
||||
const result = await executeProviderRequest({
|
||||
fetchImpl,
|
||||
modelConfig,
|
||||
prompt: promptForScenario(scenario),
|
||||
ratio: scenario.ratio,
|
||||
reference: scenario.input === "reference_image" ? { bytes: referenceBytes, mime_type: "image/png" } : undefined,
|
||||
token,
|
||||
});
|
||||
const dimensionsPassed = dimensionsMatch(result.normalized.dimensions, scenario.ratio);
|
||||
calls.push(validateSanitizedEvidence({
|
||||
duration_ms: result.duration_ms,
|
||||
http_status: result.http_status,
|
||||
input: scenario.input,
|
||||
requested_ratio: scenario.ratio,
|
||||
response: result.response_evidence,
|
||||
scenario_id: `real-${index + 1}`,
|
||||
source: "real_gateway",
|
||||
status: dimensionsPassed ? "passed" : "failed",
|
||||
validation: { dimensions: dimensionsPassed ? "passed" : "failed", response: "passed" },
|
||||
}));
|
||||
} catch (error) {
|
||||
const failed = {
|
||||
error_code: safeErrorCode(error),
|
||||
input: scenario.input,
|
||||
requested_ratio: scenario.ratio,
|
||||
scenario_id: `real-${index + 1}`,
|
||||
source: "real_gateway",
|
||||
status: "failed",
|
||||
...(error?.safe_response_shape ? { response_shape: error.safe_response_shape } : {}),
|
||||
};
|
||||
calls.push(validateSanitizedEvidence(failed));
|
||||
if (["WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED", "WP7_02_RESPONSE_MEDIA_INVALID", "WP7_02_CREDENTIAL_INVALID",
|
||||
"WP7_02_UPSTREAM_HTTP_401", "WP7_02_UPSTREAM_HTTP_403", "WP7_02_UPSTREAM_HTTP_404", "WP7_02_UPSTREAM_HTTP_429"].includes(failed.error_code)) break;
|
||||
const scenarioId = `real-${index + 1}`;
|
||||
let attemptNo = 0;
|
||||
while (true) {
|
||||
attemptNo += 1;
|
||||
try {
|
||||
const result = await executeProviderRequest({
|
||||
fetchImpl,
|
||||
modelConfig,
|
||||
prompt: promptForScenario(scenario),
|
||||
ratio: scenario.ratio,
|
||||
reference: scenario.input === "reference_image" ? { bytes: referenceBytes, mime_type: "image/png" } : undefined,
|
||||
token,
|
||||
});
|
||||
attempts.push(validateSanitizedEvidence({
|
||||
attempt_no: attemptNo, duration_ms: result.duration_ms, http_status: result.http_status,
|
||||
scenario_id: scenarioId, status: "passed",
|
||||
}));
|
||||
const dimensionsPassed = dimensionsMatch(result.normalized.dimensions, scenario.ratio);
|
||||
calls.push(validateSanitizedEvidence({
|
||||
duration_ms: result.duration_ms,
|
||||
http_status: result.http_status,
|
||||
input: scenario.input,
|
||||
requested_ratio: scenario.ratio,
|
||||
response: result.response_evidence,
|
||||
scenario_id: scenarioId,
|
||||
source: "real_gateway",
|
||||
status: dimensionsPassed ? "passed" : "failed",
|
||||
validation: { dimensions: dimensionsPassed ? "passed" : "failed", response: "passed" },
|
||||
}));
|
||||
break;
|
||||
} catch (error) {
|
||||
const errorCode = safeErrorCode(error);
|
||||
attempts.push(validateSanitizedEvidence({ attempt_no: attemptNo, error_code: errorCode, scenario_id: scenarioId, status: "failed" }));
|
||||
if (errorCode === "WP7_02_UPSTREAM_TIMEOUT" && timeoutRetriesRemaining > 0) {
|
||||
timeoutRetriesRemaining -= 1;
|
||||
continue;
|
||||
}
|
||||
const failed = {
|
||||
error_code: errorCode,
|
||||
input: scenario.input,
|
||||
requested_ratio: scenario.ratio,
|
||||
scenario_id: scenarioId,
|
||||
source: "real_gateway",
|
||||
status: "failed",
|
||||
...(error?.safe_response_shape ? { response_shape: error.safe_response_shape } : {}),
|
||||
};
|
||||
calls.push(validateSanitizedEvidence(failed));
|
||||
if (["WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED", "WP7_02_RESPONSE_MEDIA_INVALID", "WP7_02_CREDENTIAL_INVALID",
|
||||
"WP7_02_UPSTREAM_HTTP_401", "WP7_02_UPSTREAM_HTTP_403", "WP7_02_UPSTREAM_HTTP_404", "WP7_02_UPSTREAM_HTTP_429"].includes(failed.error_code)) stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (stop) break;
|
||||
}
|
||||
referenceBytes.fill(0);
|
||||
const blockers = calls.filter((call) => call.status !== "passed").map((call) => `${call.scenario_id}:${call.error_code ?? "dimensions_or_response_invalid"}`);
|
||||
return validateSanitizedEvidence({
|
||||
blockers,
|
||||
attempts,
|
||||
calls,
|
||||
maximum_real_calls: plan.planned_real_calls + 1,
|
||||
model_id: modelConfig.model_id,
|
||||
planned_real_calls: plan.planned_real_calls,
|
||||
real_calls: calls.length,
|
||||
real_calls: attempts.length,
|
||||
status: blockers.length === 0 ? "passed" : "externally_blocked",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -37,7 +37,8 @@ try {
|
||||
&& matrix.execution_modes?.length === 3 && matrix.execution_modes.every((entry) => ["passed", "covered_by_real_calls"].includes(entry.status))
|
||||
&& matrix.error_scenarios?.length === 9 && matrix.error_scenarios.every((entry) => entry.status === "passed")
|
||||
&& matrix.settlements?.length === 3 && matrix.contract_change?.full_matrix_reapplied === true
|
||||
&& externalCalls.status === "passed" && externalCalls.real_calls === 5 && externalCalls.planned_real_calls === 5
|
||||
&& externalCalls.status === "passed" && externalCalls.real_calls >= 5 && externalCalls.real_calls <= 6 && externalCalls.planned_real_calls === 5
|
||||
&& externalCalls.maximum_real_calls === 6 && externalCalls.attempts?.length === externalCalls.real_calls
|
||||
&& externalCalls.calls?.length === 5 && new Set(externalCalls.calls.map((entry) => entry.scenario_id)).size === 5
|
||||
&& externalCalls.calls.every((entry) => entry.status === "passed" && entry.source === "real_gateway")
|
||||
&& externalCalls.calls.every((entry) => {
|
||||
|
||||
@@ -170,6 +170,7 @@ test("TDD-WP7-EXT-001 executes only five real success probes per model and keeps
|
||||
});
|
||||
assert.equal(fetchCalls, 5);
|
||||
assert.equal(execution.real_calls, 5);
|
||||
assert.equal(execution.attempts.length, 5);
|
||||
assert.equal(execution.status, "externally_blocked");
|
||||
assert.equal(execution.calls.filter((call) => call.status === "passed").length, 2);
|
||||
assert.doesNotMatch(JSON.stringify(execution), /controlled-secret|fixture prompt|iVBOR/i);
|
||||
@@ -200,7 +201,7 @@ test("TDD-WP7-EXT-001 retries one timeout once and records every real attempt",
|
||||
assert.equal(execution.attempts.length, 6);
|
||||
assert.deepEqual(execution.attempts.slice(0, 2).map((attempt) => [attempt.scenario_id, attempt.attempt_no, attempt.error_code ?? attempt.status]), [
|
||||
["real-1", 1, "WP7_02_UPSTREAM_TIMEOUT"],
|
||||
["real-1", 2, "failed"],
|
||||
["real-1", 2, "WP7_02_RESPONSE_DIMENSIONS_INVALID"],
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user