feat(wp7-02): support Gemini image interactions
Dada P0-A isolated Windows CI / validate-and-package (push) Canceled after 0s
Dada P0-A isolated Windows CI / validate-and-package (push) Canceled after 0s
This commit is contained in:
@@ -26,7 +26,12 @@ function assertModelConfig(modelConfig) {
|
|||||||
const profile = modelConfig.route_profile;
|
const profile = modelConfig.route_profile;
|
||||||
if (!profile || typeof profile !== "object" || typeof profile.endpoint !== "string"
|
if (!profile || typeof profile !== "object" || typeof profile.endpoint !== "string"
|
||||||
|| !profile.endpoint.startsWith("https://oneapi.intelligrow.cn/")
|
|| !profile.endpoint.startsWith("https://oneapi.intelligrow.cn/")
|
||||||
|| !["gemini-native-v1beta", "openai-images-v1"].includes(profile.protocol_version)) {
|
|| !["gemini-interactions-v1beta", "gemini-native-v1beta", "openai-images-v1"].includes(profile.protocol_version)) {
|
||||||
|
throw new Error("WP7_02_ROUTE_PROFILE_INVALID");
|
||||||
|
}
|
||||||
|
if (profile.protocol_version === "gemini-interactions-v1beta"
|
||||||
|
&& (profile.endpoint !== "https://oneapi.intelligrow.cn/v1beta/interactions"
|
||||||
|
|| profile.provider_model_id !== "gemini-3.1-flash-image")) {
|
||||||
throw new Error("WP7_02_ROUTE_PROFILE_INVALID");
|
throw new Error("WP7_02_ROUTE_PROFILE_INVALID");
|
||||||
}
|
}
|
||||||
if (profile.protocol_version === "openai-images-v1"
|
if (profile.protocol_version === "openai-images-v1"
|
||||||
@@ -69,6 +74,20 @@ export function buildProviderRequest({ modelConfig, prompt, ratio, reference })
|
|||||||
throw new Error("WP7_02_REFERENCE_FIXTURE_INVALID");
|
throw new Error("WP7_02_REFERENCE_FIXTURE_INVALID");
|
||||||
}
|
}
|
||||||
const headers = { "content-type": "application/json" };
|
const headers = { "content-type": "application/json" };
|
||||||
|
if (config.route_profile.protocol_version === "gemini-interactions-v1beta") {
|
||||||
|
const input = [{ text: prompt, type: "text" }];
|
||||||
|
if (reference) input.push({ data: reference.bytes.toString("base64"), mime_type: reference.mime_type, type: "image" });
|
||||||
|
return {
|
||||||
|
body: {
|
||||||
|
input,
|
||||||
|
model: config.route_profile.provider_model_id,
|
||||||
|
response_format: { aspect_ratio: ratio, image_size: "1K", type: "image" },
|
||||||
|
},
|
||||||
|
headers,
|
||||||
|
method: "POST",
|
||||||
|
url: config.route_profile.endpoint,
|
||||||
|
};
|
||||||
|
}
|
||||||
if (config.route_profile.protocol_version === "gemini-native-v1beta") {
|
if (config.route_profile.protocol_version === "gemini-native-v1beta") {
|
||||||
const parts = [{ text: prompt }];
|
const parts = [{ text: prompt }];
|
||||||
if (reference) parts.push({ inlineData: { data: reference.bytes.toString("base64"), mimeType: reference.mime_type } });
|
if (reference) parts.push({ inlineData: { data: reference.bytes.toString("base64"), mimeType: reference.mime_type } });
|
||||||
@@ -175,12 +194,31 @@ function openAiUsage(response) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function interactionUsage(response) {
|
||||||
|
const usage = response?.usage;
|
||||||
|
return {
|
||||||
|
input_units: integerOrZero(usage?.total_input_tokens),
|
||||||
|
output_units: integerOrZero(usage?.total_output_tokens),
|
||||||
|
total_units: integerOrZero(usage?.total_tokens),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function normalizeProviderResponse(modelConfig, response) {
|
export function normalizeProviderResponse(modelConfig, response) {
|
||||||
const config = assertModelConfig(modelConfig);
|
const config = assertModelConfig(modelConfig);
|
||||||
let bytes;
|
let bytes;
|
||||||
let mime;
|
let mime;
|
||||||
let usageSummary;
|
let usageSummary;
|
||||||
if (config.route_profile.protocol_version === "gemini-native-v1beta") {
|
if (config.route_profile.protocol_version === "gemini-interactions-v1beta") {
|
||||||
|
const stepImages = response?.steps?.flatMap((step) => step?.type === "model_output" ? step?.content ?? [] : [])
|
||||||
|
.filter((content) => content?.type === "image" && content?.data) ?? [];
|
||||||
|
const images = stepImages.length > 0
|
||||||
|
? stepImages
|
||||||
|
: [response?.output_image].filter((content) => content?.data);
|
||||||
|
if (images.length !== 1) throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
||||||
|
mime = images[0].mime_type ?? images[0].mimeType;
|
||||||
|
bytes = Buffer.from(images[0].data, "base64");
|
||||||
|
usageSummary = interactionUsage(response);
|
||||||
|
} else if (config.route_profile.protocol_version === "gemini-native-v1beta") {
|
||||||
const parts = response?.candidates?.flatMap((candidate) => candidate?.content?.parts ?? []) ?? [];
|
const parts = response?.candidates?.flatMap((candidate) => candidate?.content?.parts ?? []) ?? [];
|
||||||
const images = parts.map((part) => part?.inlineData ?? part?.inline_data).filter((entry) => entry?.data);
|
const images = parts.map((part) => part?.inlineData ?? part?.inline_data).filter((entry) => entry?.data);
|
||||||
if (images.length !== 1) throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
if (images.length !== 1) throw new Error("WP7_02_RESPONSE_SINGLE_IMAGE_REQUIRED");
|
||||||
|
|||||||
Reference in New Issue
Block a user