From ae725a2d015c261fcc2d9b1d79b98347c17cb5b7 Mon Sep 17 00:00:00 2001 From: suyx Date: Tue, 4 Aug 2026 17:19:41 +0800 Subject: [PATCH] fix(wp7-02): normalize generated image dimensions --- apps/worker/package.json | 3 +- apps/worker/src/ai-adapter-gemini-flash.ts | 4 +- apps/worker/src/ai-adapter-gemini-pro.ts | 10 ++- apps/worker/src/ai-adapter-gpt-image.ts | 4 +- apps/worker/src/image-output-normalizer.mjs | 74 +++++++++++++++++++ apps/worker/tsconfig.json | 1 + pnpm-lock.yaml | 3 + scripts/frozen-versions.mjs | 1 + scripts/lib/wp7-02-controlled-executor.mjs | 33 +++++++-- .../wp7-02-controlled-executor.test.mjs | 7 +- .../wp7-02-image-output-normalizer.test.ts | 30 +++++++- 11 files changed, 153 insertions(+), 17 deletions(-) create mode 100644 apps/worker/src/image-output-normalizer.mjs diff --git a/apps/worker/package.json b/apps/worker/package.json index 9281266..bb95f77 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -9,7 +9,8 @@ }, "dependencies": { "better-sqlite3": "13.0.1", - "drizzle-orm": "0.45.2" + "drizzle-orm": "0.45.2", + "sharp": "0.35.3" }, "devDependencies": { "@types/better-sqlite3": "7.6.13", diff --git a/apps/worker/src/ai-adapter-gemini-flash.ts b/apps/worker/src/ai-adapter-gemini-flash.ts index 2021df5..6d6bc43 100644 --- a/apps/worker/src/ai-adapter-gemini-flash.ts +++ b/apps/worker/src/ai-adapter-gemini-flash.ts @@ -1,4 +1,5 @@ import type { GenerationAdapterRequest, NormalizedGenerationOutput } from "./ai-adapter-contract.js"; +import { normalizeImageOutputToRatio } from "./image-output-normalizer.mjs"; import { AdapterContractError, type AdapterStartResult, type AdapterTransport, balanceSignalFromResponse, classifyAdapterError, dimensionsForRatio, mockPngBytes, validateAdapterRequest, validateMockContract, @@ -41,7 +42,8 @@ export class GeminiFlashAdapter implements ModelAdapter { const classified = this.classifyError("error" in response ? response.error : undefined, (request.configSnapshot.error_mapping_profile as Record | undefined) ?? {}); return { ...classified, status: "failed" }; } - return { outputs: [this.normalizeOutput(response)], status: "completed" }; + const output = this.normalizeOutput(response); + return { outputs: [await normalizeImageOutputToRatio({ ...output, ratio: request.ratio })], status: "completed" }; } catch (error) { const classified = error instanceof AdapterContractError ? { category: "gateway_contract_invalid" as const, sourceCategory: error.sourceCategory } diff --git a/apps/worker/src/ai-adapter-gemini-pro.ts b/apps/worker/src/ai-adapter-gemini-pro.ts index fc0826e..9d1f844 100644 --- a/apps/worker/src/ai-adapter-gemini-pro.ts +++ b/apps/worker/src/ai-adapter-gemini-pro.ts @@ -1,4 +1,5 @@ import type { GenerationAdapterRequest, NormalizedGenerationOutput } from "./ai-adapter-contract.js"; +import { normalizeImageOutputToRatio } from "./image-output-normalizer.mjs"; import { AdapterContractError, type AdapterStartResult, type AdapterTransport, balanceSignalFromResponse, classifyAdapterError, dimensionsForRatio, mockPngBytes, validateAdapterRequest, validateMockContract, @@ -37,7 +38,7 @@ export class GeminiProAdapter implements ModelAdapter { try { validateAdapterRequest(request, this.modelId); const response = await this.transport.start({ operation: "start", modelId: this.modelId, prompt: request.prompt, ratio: request.ratio, referenceAssetIds: request.referenceAssetIds }); - return this.interpret(response, request.configSnapshot.error_mapping_profile as Record ?? {}); + return await this.interpret(response, request.configSnapshot.error_mapping_profile as Record ?? {}, request.ratio); } catch (error) { const classified = error instanceof AdapterContractError ? { category: "gateway_contract_invalid" as const, sourceCategory: error.sourceCategory } @@ -49,7 +50,7 @@ export class GeminiProAdapter implements ModelAdapter { async poll(upstreamJobReference: string): Promise { try { const response = await this.transport.poll({ operation: "poll", modelId: this.modelId, upstreamJobReference }); - return this.interpret(response, {}); + return await this.interpret(response, {}); } catch (error) { const classified = error instanceof AdapterContractError ? { category: "gateway_contract_invalid" as const, sourceCategory: error.sourceCategory } @@ -58,7 +59,7 @@ export class GeminiProAdapter implements ModelAdapter { } } - private interpret(response: unknown, mappingProfile: Readonly>): AdapterStartResult { + private async interpret(response: unknown, mappingProfile: Readonly>, ratio?: GenerationAdapterRequest["ratio"]): Promise { if (!response || typeof response !== "object" || !("operation" in response) || !response.operation || typeof response.operation !== "object") { return { category: "gateway_contract_invalid", sourceCategory: "response_shape_invalid", status: "failed" }; } @@ -72,7 +73,8 @@ export class GeminiProAdapter implements ModelAdapter { return reference ? { status: "pending", upstreamJobReference: reference } : { category: "gateway_contract_invalid", sourceCategory: "upstream_reference_missing", status: "failed" }; } try { - return { outputs: [this.normalizeOutput("response" in operation ? operation.response : undefined)], status: "completed" }; + const output = this.normalizeOutput("response" in operation ? operation.response : undefined); + return { outputs: [ratio ? await normalizeImageOutputToRatio({ ...output, ratio }) : output], status: "completed" }; } catch (error) { return { category: "gateway_contract_invalid", sourceCategory: error instanceof AdapterContractError ? error.sourceCategory : "response_shape_invalid", status: "failed" }; } diff --git a/apps/worker/src/ai-adapter-gpt-image.ts b/apps/worker/src/ai-adapter-gpt-image.ts index d7755db..398ed4e 100644 --- a/apps/worker/src/ai-adapter-gpt-image.ts +++ b/apps/worker/src/ai-adapter-gpt-image.ts @@ -1,4 +1,5 @@ import type { GenerationAdapterRequest, NormalizedGenerationOutput } from "./ai-adapter-contract.js"; +import { normalizeImageOutputToRatio } from "./image-output-normalizer.mjs"; import { AdapterContractError, type AdapterStartResult, type AdapterTransport, balanceSignalFromResponse, classifyAdapterError, dimensionsForRatio, mockPngBytes, validateAdapterRequest, validateMockContract, @@ -41,7 +42,8 @@ export class GptImageAdapter implements ModelAdapter { const classified = this.classifyError("error" in response ? response.error : undefined, (request.configSnapshot.error_mapping_profile as Record | undefined) ?? {}); return { ...classified, status: "failed" }; } - return { outputs: [this.normalizeOutput(response)], status: "completed" }; + const output = this.normalizeOutput(response); + return { outputs: [await normalizeImageOutputToRatio({ ...output, ratio: request.ratio })], status: "completed" }; } catch (error) { const classified = error instanceof AdapterContractError ? { category: "gateway_contract_invalid" as const, sourceCategory: error.sourceCategory } diff --git a/apps/worker/src/image-output-normalizer.mjs b/apps/worker/src/image-output-normalizer.mjs new file mode 100644 index 0000000..473d12f --- /dev/null +++ b/apps/worker/src/image-output-normalizer.mjs @@ -0,0 +1,74 @@ +import sharp from "sharp"; + +const productDimensions = Object.freeze({ + "3:4": Object.freeze({ pixelHeight: 1440, pixelWidth: 1080 }), + "1:1": Object.freeze({ pixelHeight: 1080, pixelWidth: 1080 }), + "4:3": Object.freeze({ pixelHeight: 1080, pixelWidth: 1440 }), + "9:16": Object.freeze({ pixelHeight: 1920, pixelWidth: 1080 }), +}); + +const gptImageRequestSizes = Object.freeze({ + "3:4": "1056x1408", + "1:1": "1088x1088", + "4:3": "1408x1056", + "9:16": "1008x1792", +}); + +const allowedMimeTypes = new Set(["image/jpeg", "image/png", "image/webp"]); +const maximumInputBytes = 20 * 1024 * 1024; + +function assertRatio(ratio) { + if (!(ratio in productDimensions)) throw new Error("image_output_ratio_unsupported"); + return ratio; +} + +export function productDimensionsForRatio(ratio) { + return { ...productDimensions[assertRatio(ratio)] }; +} + +export function gptImageRequestSizeForRatio(ratio) { + return gptImageRequestSizes[assertRatio(ratio)]; +} + +export async function normalizeImageOutputToRatio(input) { + const ratio = assertRatio(input?.ratio); + if (!Buffer.isBuffer(input?.bytes) || input.bytes.length === 0 || input.bytes.length > maximumInputBytes + || !allowedMimeTypes.has(input?.mimeType)) { + throw new Error("image_output_media_invalid"); + } + const target = productDimensions[ratio]; + if (input.pixelWidth === target.pixelWidth && input.pixelHeight === target.pixelHeight) { + return { + bytes: Buffer.from(input.bytes), + mimeType: input.mimeType, + normalized: false, + ...target, + upstreamPixelHeight: input.pixelHeight, + upstreamPixelWidth: input.pixelWidth, + }; + } + + const image = sharp(input.bytes, { failOn: "error", limitInputPixels: 40_000_000 }); + const metadata = await image.metadata(); + if (!metadata.width || !metadata.height) throw new Error("image_output_dimensions_missing"); + const requestedRatio = target.pixelWidth / target.pixelHeight; + const upstreamRatio = metadata.width / metadata.height; + if (Math.abs(upstreamRatio - requestedRatio) / requestedRatio > 0.02) { + throw new Error("image_output_aspect_ratio_mismatch"); + } + const { data, info } = await image + .resize(target.pixelWidth, target.pixelHeight, { fit: "fill", kernel: sharp.kernel.lanczos3 }) + .png({ compressionLevel: 9 }) + .toBuffer({ resolveWithObject: true }); + if (info.width !== target.pixelWidth || info.height !== target.pixelHeight || info.format !== "png") { + throw new Error("image_output_normalization_failed"); + } + return { + bytes: data, + mimeType: "image/png", + normalized: true, + ...target, + upstreamPixelHeight: metadata.height, + upstreamPixelWidth: metadata.width, + }; +} diff --git a/apps/worker/tsconfig.json b/apps/worker/tsconfig.json index 5425508..10feafa 100644 --- a/apps/worker/tsconfig.json +++ b/apps/worker/tsconfig.json @@ -1,6 +1,7 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { + "allowJs": true, "module": "NodeNext", "moduleResolution": "NodeNext", "lib": ["ES2024"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c78c144..73be884 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -133,6 +133,9 @@ importers: drizzle-orm: specifier: 0.45.2 version: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@13.0.1) + sharp: + specifier: 0.35.3 + version: 0.35.3(@types/node@24.13.3) devDependencies: '@types/better-sqlite3': specifier: 7.6.13 diff --git a/scripts/frozen-versions.mjs b/scripts/frozen-versions.mjs index 1af0fca..3e99470 100644 --- a/scripts/frozen-versions.mjs +++ b/scripts/frozen-versions.mjs @@ -44,6 +44,7 @@ export const frozenPackages = { dependencies: { "better-sqlite3": "13.0.1", "drizzle-orm": "0.45.2", + sharp: "0.35.3", }, devDependencies: { typescript: "7.0.2", diff --git a/scripts/lib/wp7-02-controlled-executor.mjs b/scripts/lib/wp7-02-controlled-executor.mjs index 4265a9d..c94f9d4 100644 --- a/scripts/lib/wp7-02-controlled-executor.mjs +++ b/scripts/lib/wp7-02-controlled-executor.mjs @@ -1,16 +1,14 @@ import { createHash } from "node:crypto"; +import { + gptImageRequestSizeForRatio, + normalizeImageOutputToRatio, +} from "../../apps/worker/src/image-output-normalizer.mjs"; import { WP7_02_MODEL_IDS, buildModelContractPlan } from "./wp7-02-external-contract.mjs"; export const WP7_02_CONTROLLED_REAL_LIMIT = 120; const ratios = ["3:4", "1:1", "4:3", "9:16"]; -const openAiSizes = Object.freeze({ - "3:4": "1080x1440", - "1:1": "1080x1080", - "4:3": "1440x1080", - "9:16": "1080x1920", -}); const allowedMimeTypes = new Set(["image/jpeg", "image/png", "image/webp"]); const forbiddenEvidenceKeys = /(?:^|_)(?:absolute_path|authorization|body|credential|credential_value|image_bytes|image_data|original_image|password|path|prompt|raw|raw_provider_payload|raw_prompt|secret|token)(?:_|$)/i; @@ -87,7 +85,7 @@ export function buildProviderRequest({ modelConfig, prompt, ratio, reference }) model: config.model_id, prompt, response_format: "b64_json", - size: openAiSizes[ratio], + size: gptImageRequestSizeForRatio(ratio), }; if (reference) body.image = `data:${reference.mime_type};base64,${reference.bytes.toString("base64")}`; return { body, headers, method: "POST", url: config.route_profile.endpoint }; @@ -221,6 +219,7 @@ export function buildSanitizedResponseEvidence(normalized) { dimensions: structuredClone(normalized.dimensions), evidence_hash: normalized.evidence_hash, mime: normalized.mime, + ...(normalized.normalization ? { normalization: structuredClone(normalized.normalization) } : {}), usage_summary: structuredClone(normalized.usage_summary), }; return validateSanitizedEvidence(evidence); @@ -264,7 +263,25 @@ export async function executeProviderRequest({ fetchImpl = fetch, modelConfig, p const providerResponse = await response.json(); let normalized; try { - normalized = normalizeProviderResponse(modelConfig, providerResponse); + const providerNormalized = normalizeProviderResponse(modelConfig, providerResponse); + const adapted = await normalizeImageOutputToRatio({ + bytes: providerNormalized.bytes, + mimeType: providerNormalized.mime, + pixelHeight: providerNormalized.dimensions.height, + pixelWidth: providerNormalized.dimensions.width, + ratio, + }); + normalized = { + ...providerNormalized, + bytes: adapted.bytes, + dimensions: { height: adapted.pixelHeight, width: adapted.pixelWidth }, + evidence_hash: `sha256:${sha256(adapted.bytes)}`, + mime: adapted.mimeType, + normalization: { + applied: adapted.normalized, + upstream_dimensions: { height: adapted.upstreamPixelHeight, width: adapted.upstreamPixelWidth }, + }, + }; } catch (error) { if (error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)) { error.safe_response_shape = describeProviderResponseShape(providerResponse); diff --git a/tests/package/wp7-02-controlled-executor.test.mjs b/tests/package/wp7-02-controlled-executor.test.mjs index baa438b..a0ebe1b 100644 --- a/tests/package/wp7-02-controlled-executor.test.mjs +++ b/tests/package/wp7-02-controlled-executor.test.mjs @@ -135,6 +135,11 @@ test("TDD-WP7-EXT-001 confines the credential to the request header and discards token: credentialMarker, }); assert.equal(success.http_status, 200); + assert.deepEqual(success.response_evidence.dimensions, { height: 1080, width: 1080 }); + assert.deepEqual(success.response_evidence.normalization, { + applied: true, + upstream_dimensions: { height: 1, width: 1 }, + }); assert.doesNotMatch(JSON.stringify({ ...success, normalized: undefined }), new RegExp(credentialMarker)); await assert.rejects(() => executeProviderRequest({ @@ -166,7 +171,7 @@ test("TDD-WP7-EXT-001 executes only five real success probes per model and keeps assert.equal(fetchCalls, 5); assert.equal(execution.real_calls, 5); assert.equal(execution.status, "externally_blocked"); - assert.equal(execution.calls.filter((call) => call.status === "passed").length, 0); + assert.equal(execution.calls.filter((call) => call.status === "passed").length, 2); assert.doesNotMatch(JSON.stringify(execution), /controlled-secret|fixture prompt|iVBOR/i); await assert.rejects(() => runControlledRealScenarios({ maxRealCalls: 121, modelConfig: models[0], token: "not-used" }), /WP7_02_REAL_CALL_LIMIT_INVALID/); }); diff --git a/tests/unit/wp7-02-image-output-normalizer.test.ts b/tests/unit/wp7-02-image-output-normalizer.test.ts index 3a5ddc8..d8c3fbf 100644 --- a/tests/unit/wp7-02-image-output-normalizer.test.ts +++ b/tests/unit/wp7-02-image-output-normalizer.test.ts @@ -1,10 +1,13 @@ import { describe, expect, it } from "vitest"; +import { GeminiFlashAdapter } from "../../apps/worker/src/ai-adapter-gemini-flash.js"; +import { GeminiProAdapter } from "../../apps/worker/src/ai-adapter-gemini-pro.js"; +import { GptImageAdapter } from "../../apps/worker/src/ai-adapter-gpt-image.js"; import { gptImageRequestSizeForRatio, normalizeImageOutputToRatio, productDimensionsForRatio, -} from "../../apps/worker/src/image-output-normalizer.js"; +} from "../../apps/worker/src/image-output-normalizer.mjs"; const onePixelPng = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", @@ -39,4 +42,29 @@ describe("TDD-WP7-EXT-001 exact image output normalization", () => { expect(output.bytes.subarray(0, 8).toString("hex")).toBe("89504e470d0a1a0a"); expect(productDimensionsForRatio("9:16")).toEqual({ pixelHeight: 1920, pixelWidth: 1080 }); }); + + it("is used by all three production adapter boundaries", async () => { + const encoded = onePixelPng.toString("base64"); + const adapters = [ + new GeminiFlashAdapter({ transport: { + async start() { return { candidates: [{ inline_data: { data: encoded, mime_type: "image/png" }, pixelHeight: 1, pixelWidth: 1 }] }; }, + async poll() { return {}; }, + } }), + new GeminiProAdapter({ transport: { + async start() { return { operation: { done: true, response: { candidates: [{ inline_data: { data: encoded, mime_type: "image/png" }, pixelHeight: 1, pixelWidth: 1 }] } } }; }, + async poll() { return {}; }, + } }), + new GptImageAdapter({ transport: { + async start() { return { data: [{ b64_json: encoded, pixelHeight: 1, pixelWidth: 1 }] }; }, + async poll() { return {}; }, + } }), + ]; + for (const adapter of adapters) { + const result = await adapter.start({ + configSnapshot: {}, generationId: `normalization-${adapter.modelId}`, modelId: adapter.modelId, + prompt: "sanitized fixture", ratio: "1:1", referenceAssetIds: [], + }); + expect(result).toMatchObject({ status: "completed", outputs: [{ mimeType: "image/png", pixelHeight: 1080, pixelWidth: 1080 }] }); + } + }); });