419 lines
18 KiB
JavaScript
419 lines
18 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
|
|
import {
|
|
gptImageRequestSizeForRatio,
|
|
normalizeImageOutputToRatio,
|
|
} from "../../apps/worker/src/image-output-normalizer.mjs";
|
|
import { WP7_02_MODEL_IDS, buildModelContractPlan } from "./wp7-02-external-contract.mjs";
|
|
|
|
export const WP7_02_CONTROLLED_REAL_LIMIT = 120;
|
|
|
|
const ratios = ["3:4", "1:1", "4:3", "9:16"];
|
|
const allowedMimeTypes = new Set(["image/jpeg", "image/png", "image/webp"]);
|
|
const forbiddenEvidenceKeys = /(?:^|_)(?:absolute_path|authorization|body|credential|credential_value|image_bytes|image_data|original_image|password|path|prompt|raw|raw_provider_payload|raw_prompt|secret|token)(?:_|$)/i;
|
|
|
|
function sha256(value) {
|
|
return createHash("sha256").update(value).digest("hex").toUpperCase();
|
|
}
|
|
|
|
function assertModelConfig(modelConfig) {
|
|
if (!modelConfig || typeof modelConfig !== "object" || !WP7_02_MODEL_IDS.includes(modelConfig.model_id)) {
|
|
throw new Error("WP7_02_MODEL_CONFIG_INVALID");
|
|
}
|
|
if (!Number.isSafeInteger(modelConfig.config_version) || modelConfig.config_version <= 0) {
|
|
throw new Error("WP7_02_MODEL_CONFIG_VERSION_INVALID");
|
|
}
|
|
const profile = modelConfig.route_profile;
|
|
if (!profile || typeof profile !== "object" || typeof profile.endpoint !== "string"
|
|
|| !profile.endpoint.startsWith("https://oneapi.intelligrow.cn/")
|
|
|| !["gemini-interactions-v1beta", "gemini-native-v1beta", "gemini-openai-chat-v1", "openai-images-v1"].includes(profile.protocol_version)) {
|
|
throw new Error("WP7_02_ROUTE_PROFILE_INVALID");
|
|
}
|
|
if (profile.protocol_version === "gemini-interactions-v1beta"
|
|
&& (profile.endpoint !== "https://oneapi.intelligrow.cn/v1beta/interactions"
|
|
|| profile.provider_model_id !== "gemini-3.1-flash-image")) {
|
|
throw new Error("WP7_02_ROUTE_PROFILE_INVALID");
|
|
}
|
|
if (profile.protocol_version === "gemini-openai-chat-v1"
|
|
&& (profile.endpoint !== "https://oneapi.intelligrow.cn/v1/chat/completions"
|
|
|| profile.provider_model_id !== "gemini-3.1-flash-image")) {
|
|
throw new Error("WP7_02_ROUTE_PROFILE_INVALID");
|
|
}
|
|
if (profile.protocol_version === "openai-images-v1"
|
|
&& (profile.reference_endpoint !== "https://oneapi.intelligrow.cn/v1/images/edits")) {
|
|
throw new Error("WP7_02_REFERENCE_ROUTE_PROFILE_INVALID");
|
|
}
|
|
if (profile.protocol_version === "openai-images-v1" && profile.provider_model_id !== undefined
|
|
&& profile.provider_model_id !== "gemini-3.1-flash-image") {
|
|
throw new Error("WP7_02_ROUTE_PROFILE_INVALID");
|
|
}
|
|
return modelConfig;
|
|
}
|
|
|
|
export function buildControlledExecutionPlan(modelConfig) {
|
|
const config = assertModelConfig(modelConfig);
|
|
const contractPlan = buildModelContractPlan(config.model_id);
|
|
const realScenarios = [
|
|
...ratios.map((ratio) => ({ input: "pure_text", ratio, source: "real_gateway" })),
|
|
{ input: "reference_image", ratio: "1:1", source: "real_gateway" },
|
|
];
|
|
return {
|
|
config_version: config.config_version,
|
|
error_scenarios: contractPlan.error_categories.map((name) => ({
|
|
expected: contractPlan.error_expectations[name], name, source: "deterministic_local",
|
|
})),
|
|
execution_modes: [
|
|
{ mode: "sync", source: "real_gateway" },
|
|
{ mode: "async", source: "deterministic_local" },
|
|
{ mode: "poll", source: "deterministic_local" },
|
|
],
|
|
model_id: config.model_id,
|
|
planned_real_calls: realScenarios.length,
|
|
quota_impact: "authorized_test_key_up_to_120_requests",
|
|
real_scenarios: realScenarios,
|
|
response_checks: ["single_image", "mime", "dimensions", "sanitized_usage", "evidence_hash"],
|
|
state_scenarios: contractPlan.state_checks.map((name) => ({ name, source: "deterministic_local" })),
|
|
};
|
|
}
|
|
|
|
export function buildProviderRequest({ modelConfig, prompt, ratio, reference }) {
|
|
const config = assertModelConfig(modelConfig);
|
|
if (typeof prompt !== "string" || !prompt.trim() || !ratios.includes(ratio)) throw new Error("WP7_02_REQUEST_FIXTURE_INVALID");
|
|
if (reference && (!Buffer.isBuffer(reference.bytes) || reference.bytes.length === 0 || !allowedMimeTypes.has(reference.mime_type))) {
|
|
throw new Error("WP7_02_REFERENCE_FIXTURE_INVALID");
|
|
}
|
|
const headers = { "content-type": "application/json" };
|
|
if (config.route_profile.protocol_version === "gemini-interactions-v1beta") {
|
|
const input = [{ text: prompt, type: "text" }];
|
|
if (reference) input.push({ data: reference.bytes.toString("base64"), mime_type: reference.mime_type, type: "image" });
|
|
return {
|
|
body: {
|
|
input,
|
|
model: config.route_profile.provider_model_id,
|
|
response_format: { aspect_ratio: ratio, image_size: "1K", type: "image" },
|
|
},
|
|
headers,
|
|
method: "POST",
|
|
url: config.route_profile.endpoint,
|
|
};
|
|
}
|
|
if (config.route_profile.protocol_version === "gemini-native-v1beta") {
|
|
const parts = [{ text: prompt }];
|
|
if (reference) parts.push({ inlineData: { data: reference.bytes.toString("base64"), mimeType: reference.mime_type } });
|
|
return {
|
|
body: {
|
|
contents: [{ parts, role: "user" }],
|
|
generationConfig: {
|
|
imageConfig: { aspectRatio: ratio, imageSize: "1K" },
|
|
responseModalities: ["IMAGE"],
|
|
},
|
|
},
|
|
headers,
|
|
method: "POST",
|
|
url: config.route_profile.endpoint,
|
|
};
|
|
}
|
|
if (config.route_profile.protocol_version === "gemini-openai-chat-v1") {
|
|
const content = reference
|
|
? [
|
|
{ text: prompt, type: "text" },
|
|
{
|
|
image_url: { url: `data:${reference.mime_type};base64,${reference.bytes.toString("base64")}` },
|
|
type: "image_url",
|
|
},
|
|
]
|
|
: prompt;
|
|
return {
|
|
body: {
|
|
extra_body: { google: { image_config: { aspect_ratio: ratio, image_size: "1K" } } },
|
|
messages: [{ content, role: "user" }],
|
|
model: config.route_profile.provider_model_id,
|
|
stream: false,
|
|
},
|
|
headers,
|
|
method: "POST",
|
|
url: config.route_profile.endpoint,
|
|
};
|
|
}
|
|
const providerModelId = config.route_profile.provider_model_id ?? config.model_id;
|
|
const body = {
|
|
model: providerModelId,
|
|
prompt,
|
|
response_format: "b64_json",
|
|
size: gptImageRequestSizeForRatio(ratio),
|
|
};
|
|
if (reference) {
|
|
const form = new FormData();
|
|
form.append("model", providerModelId);
|
|
form.append("prompt", prompt);
|
|
form.append("response_format", "b64_json");
|
|
form.append("size", gptImageRequestSizeForRatio(ratio));
|
|
form.append("image[]", new Blob([reference.bytes], { type: reference.mime_type }), "reference.png");
|
|
return { body: form, headers: {}, method: "POST", url: config.route_profile.reference_endpoint };
|
|
}
|
|
return { body, headers, method: "POST", url: config.route_profile.endpoint };
|
|
}
|
|
|
|
function pngDimensions(bytes) {
|
|
const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
|
|
if (bytes.length < 24 || !bytes.subarray(0, 8).equals(signature)) return undefined;
|
|
return { height: bytes.readUInt32BE(20), width: bytes.readUInt32BE(16) };
|
|
}
|
|
|
|
function jpegDimensions(bytes) {
|
|
if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined;
|
|
let offset = 2;
|
|
while (offset + 9 < bytes.length) {
|
|
if (bytes[offset] !== 0xff) { offset += 1; continue; }
|
|
const marker = bytes[offset + 1];
|
|
if ([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf].includes(marker)) {
|
|
return { height: bytes.readUInt16BE(offset + 5), width: bytes.readUInt16BE(offset + 7) };
|
|
}
|
|
if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) { offset += 2; continue; }
|
|
const length = bytes.readUInt16BE(offset + 2);
|
|
if (length < 2) return undefined;
|
|
offset += length + 2;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function webpDimensions(bytes) {
|
|
if (bytes.length < 30 || bytes.toString("ascii", 0, 4) !== "RIFF" || bytes.toString("ascii", 8, 12) !== "WEBP") return undefined;
|
|
const kind = bytes.toString("ascii", 12, 16);
|
|
if (kind === "VP8X") {
|
|
return {
|
|
height: 1 + bytes.readUIntLE(27, 3),
|
|
width: 1 + bytes.readUIntLE(24, 3),
|
|
};
|
|
}
|
|
if (kind === "VP8 " && bytes.length >= 30) return { height: bytes.readUInt16LE(28) & 0x3fff, width: bytes.readUInt16LE(26) & 0x3fff };
|
|
if (kind === "VP8L" && bytes.length >= 25) {
|
|
const bits = bytes.readUInt32LE(21);
|
|
return { height: 1 + ((bits >> 14) & 0x3fff), width: 1 + (bits & 0x3fff) };
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function inspectImage(bytes, declaredMime) {
|
|
const png = pngDimensions(bytes);
|
|
if (png && declaredMime === "image/png") return { ...png, mime: declaredMime };
|
|
const jpeg = jpegDimensions(bytes);
|
|
if (jpeg && declaredMime === "image/jpeg") return { ...jpeg, mime: declaredMime };
|
|
const webp = webpDimensions(bytes);
|
|
if (webp && declaredMime === "image/webp") return { ...webp, mime: declaredMime };
|
|
throw new Error("WP7_02_RESPONSE_MEDIA_INVALID");
|
|
}
|
|
|
|
function integerOrZero(value) {
|
|
return Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
}
|
|
|
|
function geminiUsage(response) {
|
|
const usage = response?.usageMetadata;
|
|
return {
|
|
input_units: integerOrZero(usage?.promptTokenCount),
|
|
output_units: integerOrZero(usage?.candidatesTokenCount),
|
|
total_units: integerOrZero(usage?.totalTokenCount),
|
|
};
|
|
}
|
|
|
|
function openAiUsage(response) {
|
|
const usage = response?.usage;
|
|
return {
|
|
input_units: integerOrZero(usage?.input_tokens ?? usage?.inputTokens ?? usage?.prompt_tokens ?? usage?.promptTokens),
|
|
output_units: integerOrZero(usage?.output_tokens ?? usage?.outputTokens ?? usage?.completion_tokens ?? usage?.completionTokens),
|
|
total_units: integerOrZero(usage?.total_tokens ?? usage?.totalTokens),
|
|
};
|
|
}
|
|
|
|
function openAiChatImage(response) {
|
|
const content = response?.choices?.[0]?.message?.content;
|
|
if (typeof content !== "string") throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
|
const matches = [...content.matchAll(/!\[[^\]]*\]\(\s*data:(image\/(?:jpeg|png|webp));base64,([A-Za-z0-9+/=\r\n]+)\s*\)/gi)];
|
|
if (matches.length !== 1) throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
|
return { data: matches[0][2], mime: matches[0][1].toLowerCase() };
|
|
}
|
|
|
|
function interactionUsage(response) {
|
|
const usage = response?.usage;
|
|
return {
|
|
input_units: integerOrZero(usage?.total_input_tokens),
|
|
output_units: integerOrZero(usage?.total_output_tokens),
|
|
total_units: integerOrZero(usage?.total_tokens),
|
|
};
|
|
}
|
|
|
|
export function normalizeProviderResponse(modelConfig, response) {
|
|
const config = assertModelConfig(modelConfig);
|
|
let bytes;
|
|
let mime;
|
|
let usageSummary;
|
|
if (config.route_profile.protocol_version === "gemini-interactions-v1beta") {
|
|
const stepImages = response?.steps?.flatMap((step) => step?.type === "model_output" ? step?.content ?? [] : [])
|
|
.filter((content) => content?.type === "image" && content?.data) ?? [];
|
|
const images = stepImages.length > 0
|
|
? stepImages
|
|
: [response?.output_image].filter((content) => content?.data);
|
|
if (images.length !== 1) throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
|
mime = images[0].mime_type ?? images[0].mimeType;
|
|
bytes = Buffer.from(images[0].data, "base64");
|
|
usageSummary = interactionUsage(response);
|
|
} else if (config.route_profile.protocol_version === "gemini-native-v1beta") {
|
|
const parts = response?.candidates?.flatMap((candidate) => candidate?.content?.parts ?? []) ?? [];
|
|
const images = parts.map((part) => part?.inlineData ?? part?.inline_data).filter((entry) => entry?.data);
|
|
if (images.length !== 1) throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
|
mime = images[0].mimeType ?? images[0].mime_type;
|
|
bytes = Buffer.from(images[0].data, "base64");
|
|
usageSummary = geminiUsage(response);
|
|
} else if (config.route_profile.protocol_version === "gemini-openai-chat-v1") {
|
|
const image = openAiChatImage(response);
|
|
bytes = Buffer.from(image.data, "base64");
|
|
mime = image.mime;
|
|
usageSummary = openAiUsage(response);
|
|
} else {
|
|
if (!Array.isArray(response?.data) || response.data.length !== 1 || typeof response.data[0]?.b64_json !== "string") {
|
|
throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
|
}
|
|
bytes = Buffer.from(response.data[0].b64_json, "base64");
|
|
mime = "image/png";
|
|
usageSummary = openAiUsage(response);
|
|
}
|
|
const media = inspectImage(bytes, mime);
|
|
return {
|
|
bytes,
|
|
dimensions: { height: media.height, width: media.width },
|
|
evidence_hash: `sha256:${sha256(bytes)}`,
|
|
mime: media.mime,
|
|
usage_summary: usageSummary,
|
|
};
|
|
}
|
|
|
|
export function describeProviderResponseShape(value, depth = 0) {
|
|
if (typeof value === "string") {
|
|
const trimmed = value.trim();
|
|
const representation = /^data:image\/(?:jpeg|png|webp);base64,/i.test(trimmed)
|
|
? "inline_media"
|
|
: /!\[[^\]]*\]\(\s*https?:\/\/[^)\s]+\s*\)/i.test(trimmed)
|
|
? "markdown_uri"
|
|
: /^https?:\/\/\S+$/i.test(trimmed)
|
|
? "uri"
|
|
: "plain_text";
|
|
return {
|
|
kind: "string",
|
|
representation,
|
|
size: value.length === 0 ? "empty" : value.length > 256 ? "large" : "small",
|
|
};
|
|
}
|
|
if (typeof value === "number") return { kind: "number" };
|
|
if (typeof value === "boolean") return { kind: "boolean" };
|
|
if (value === null || value === undefined) return { kind: value === null ? "null" : "undefined" };
|
|
if (depth >= 6) return { kind: "depth_limit" };
|
|
if (Array.isArray(value)) {
|
|
return {
|
|
item: value.length > 0 ? describeProviderResponseShape(value[0], depth + 1) : { kind: "empty" },
|
|
kind: "array",
|
|
length: value.length,
|
|
};
|
|
}
|
|
if (value && typeof value === "object") {
|
|
return {
|
|
fields: Object.keys(value).toSorted().map((name) => ({ name, shape: describeProviderResponseShape(value[name], depth + 1) })),
|
|
kind: "object",
|
|
};
|
|
}
|
|
return { kind: "undefined" };
|
|
}
|
|
|
|
export function buildSanitizedResponseEvidence(normalized) {
|
|
const evidence = {
|
|
dimensions: structuredClone(normalized.dimensions),
|
|
evidence_hash: normalized.evidence_hash,
|
|
mime: normalized.mime,
|
|
...(normalized.normalization ? { normalization: structuredClone(normalized.normalization) } : {}),
|
|
usage_summary: structuredClone(normalized.usage_summary),
|
|
};
|
|
return validateSanitizedEvidence(evidence);
|
|
}
|
|
|
|
function inspectEvidenceValue(value, seen = new Set()) {
|
|
if (value && typeof value === "object") {
|
|
if (seen.has(value)) throw new Error("WP7_02_EVIDENCE_CYCLE_FORBIDDEN");
|
|
seen.add(value);
|
|
for (const [key, entry] of Object.entries(value)) {
|
|
if (key === "verified") throw new Error("WP7_02_SHARED_VERIFIED_FORBIDDEN");
|
|
if (key !== "secret_scan" && forbiddenEvidenceKeys.test(key)) throw new Error(`WP7_02_SENSITIVE_EVIDENCE_FORBIDDEN:${key}`);
|
|
inspectEvidenceValue(entry, seen);
|
|
}
|
|
seen.delete(value);
|
|
} else if (typeof value === "string" && /[A-Za-z]:\\Users\\/i.test(value)) {
|
|
throw new Error("WP7_02_SENSITIVE_EVIDENCE_FORBIDDEN");
|
|
}
|
|
}
|
|
|
|
export function validateSanitizedEvidence(evidence) {
|
|
inspectEvidenceValue(evidence);
|
|
return evidence;
|
|
}
|
|
|
|
export async function executeProviderRequest({ fetchImpl = fetch, modelConfig, prompt, ratio, reference, token, timeoutMs = 180_000 }) {
|
|
if (typeof token !== "string" || token.length < 8) throw new Error("WP7_02_CREDENTIAL_INVALID");
|
|
const request = buildProviderRequest({ modelConfig, prompt, ratio, reference });
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
const startedAt = performance.now();
|
|
try {
|
|
const response = await fetchImpl(request.url, {
|
|
body: request.body instanceof FormData ? request.body : JSON.stringify(request.body),
|
|
headers: { ...request.headers, authorization: `Bearer ${token}` },
|
|
method: request.method,
|
|
signal: controller.signal,
|
|
});
|
|
const durationMs = Math.round(performance.now() - startedAt);
|
|
if (!response.ok) throw new Error(`WP7_02_UPSTREAM_HTTP_${response.status}`);
|
|
const providerResponse = await response.json();
|
|
let normalized;
|
|
try {
|
|
const providerNormalized = normalizeProviderResponse(modelConfig, providerResponse);
|
|
const adapted = await normalizeImageOutputToRatio({
|
|
bytes: providerNormalized.bytes,
|
|
mimeType: providerNormalized.mime,
|
|
pixelHeight: providerNormalized.dimensions.height,
|
|
pixelWidth: providerNormalized.dimensions.width,
|
|
ratio,
|
|
});
|
|
normalized = {
|
|
...providerNormalized,
|
|
bytes: adapted.bytes,
|
|
dimensions: { height: adapted.pixelHeight, width: adapted.pixelWidth },
|
|
evidence_hash: `sha256:${sha256(adapted.bytes)}`,
|
|
mime: adapted.mimeType,
|
|
normalization: {
|
|
applied: adapted.normalized,
|
|
upstream_dimensions: { height: adapted.upstreamPixelHeight, width: adapted.upstreamPixelWidth },
|
|
},
|
|
};
|
|
} catch (error) {
|
|
if (error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)) {
|
|
error.safe_response_shape = describeProviderResponseShape(providerResponse);
|
|
} else if (error instanceof Error && error.message === "image_output_media_invalid") {
|
|
throw new Error("WP7_02_RESPONSE_MEDIA_INVALID");
|
|
} else if (error instanceof Error && /^image_output_(?:aspect_ratio_mismatch|dimensions_missing|normalization_failed)$/.test(error.message)) {
|
|
throw new Error("WP7_02_RESPONSE_DIMENSIONS_INVALID");
|
|
}
|
|
throw error;
|
|
}
|
|
return {
|
|
duration_ms: durationMs,
|
|
http_status: response.status,
|
|
normalized,
|
|
response_evidence: buildSanitizedResponseEvidence(normalized),
|
|
};
|
|
} catch (error) {
|
|
if (error?.name === "AbortError") throw new Error("WP7_02_UPSTREAM_TIMEOUT");
|
|
if (error instanceof Error && /^WP7_02_[A-Z0-9_]+$/.test(error.message)) throw error;
|
|
throw new Error("WP7_02_UPSTREAM_FAILED");
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|