test: add WP7-02 controlled real contract gate
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 1m36s
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 1m36s
This commit is contained in:
suyx
@@ -0,0 +1,239 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
export const AI_GATEWAY_CREDENTIAL_TARGET = "Dada/P0A/worker/ai-gateway";
|
||||
export const WP7_02_MODEL_IDS = Object.freeze([
|
||||
"gemini-3.1-flash-image-preview",
|
||||
"gemini-3-pro-image-preview",
|
||||
"gpt-image-2",
|
||||
]);
|
||||
|
||||
const expectedCandidateCommit = "623cad25b2a2a9a003502c9a92ebd318dad06248";
|
||||
const expectedBrowsers = Object.freeze({
|
||||
"Google Chrome": "150.0.7871.187",
|
||||
"Microsoft Edge": "151.0.4129.59",
|
||||
});
|
||||
const ratios = Object.freeze(["3:4", "1:1", "4:3", "9:16"]);
|
||||
const errorCategories = Object.freeze([
|
||||
"upstream_timeout", "upstream_failed", "safety_rejected", "model_disabled",
|
||||
"gateway_balance_insufficient", "gateway_contract_invalid", "reference_invalid",
|
||||
"unknown_retryable", "unknown_non_retryable",
|
||||
]);
|
||||
const errorExpectations = Object.freeze({
|
||||
upstream_timeout: { credit_effect: "release_once", job_outcome: "failed", user_action: "retry_original_input" },
|
||||
upstream_failed: { credit_effect: "release_once", job_outcome: "failed", user_action: "retry_later" },
|
||||
safety_rejected: { credit_effect: "release_once", job_outcome: "rejected", user_action: "modify_prompt_or_reference" },
|
||||
model_disabled: { credit_effect: "no_reserve", job_outcome: "not_created", user_action: "choose_other_model_or_wait" },
|
||||
gateway_balance_insufficient: { credit_effect: "no_reserve_or_release_once", job_outcome: "not_created_or_failed", user_action: "choose_unaffected_model_or_contact_admin" },
|
||||
gateway_contract_invalid: { credit_effect: "no_reserve_or_release_once", job_outcome: "not_created_or_failed", user_action: "choose_other_model_or_contact_admin" },
|
||||
reference_invalid: { credit_effect: "no_reserve_or_release_once", job_outcome: "not_created_or_failed", user_action: "replace_or_remove_reference" },
|
||||
unknown_retryable: { credit_effect: "release_once", job_outcome: "failed", user_action: "retry_later" },
|
||||
unknown_non_retryable: { credit_effect: "release_once", job_outcome: "failed", user_action: "contact_admin" },
|
||||
});
|
||||
|
||||
function stableJson(value) {
|
||||
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(typeof value === "string" ? value : stableJson(value)).digest("hex").toUpperCase();
|
||||
}
|
||||
|
||||
function assertModelId(modelId) {
|
||||
if (!WP7_02_MODEL_IDS.includes(modelId)) throw new Error("WP7_02_MODEL_NOT_ALLOWED");
|
||||
return modelId;
|
||||
}
|
||||
|
||||
export function buildModelContractPlan(modelId) {
|
||||
assertModelId(modelId);
|
||||
const plannedRequestBreakdown = {
|
||||
contract_change_full_revalidation: 20,
|
||||
error_categories: 9,
|
||||
execution_modes_and_poll: 3,
|
||||
input_and_ratio_success: 6,
|
||||
settlement_boundaries: 2,
|
||||
};
|
||||
return {
|
||||
error_categories: [...errorCategories],
|
||||
error_expectations: structuredClone(errorExpectations),
|
||||
execution_modes: ["sync", "async", "poll"],
|
||||
inputs: ["pure_text", "reference_image"],
|
||||
model_id: modelId,
|
||||
planned_provider_requests_max: Object.values(plannedRequestBreakdown).reduce((total, count) => total + count, 0),
|
||||
planned_request_breakdown: plannedRequestBreakdown,
|
||||
quota_impact: "unknown_requires_operator_review",
|
||||
ratios: [...ratios],
|
||||
response_checks: ["single_image", "mime", "dimensions", "sanitized_usage"],
|
||||
state_checks: [
|
||||
"credit_commit_once", "credit_release_once_per_terminal_failure",
|
||||
"contract_change_invalidation", "full_revalidation",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function validateCandidateDependency(record) {
|
||||
if (!record || typeof record !== "object") throw new Error("WP7_02_CANDIDATE_RECORD_REQUIRED");
|
||||
if (record.final_release !== false || record.status !== "candidate_unvalidated"
|
||||
|| record.candidate_package?.release_status !== "candidate_unvalidated") {
|
||||
throw new Error("WP7_02_CANDIDATE_FINAL_RELEASE_FORBIDDEN");
|
||||
}
|
||||
if (record.build_commit !== expectedCandidateCommit || record.fixed_port !== 43121) {
|
||||
throw new Error("WP7_02_CANDIDATE_BASELINE_MISMATCH");
|
||||
}
|
||||
const browsers = Array.isArray(record.browsers) ? record.browsers : [];
|
||||
if (browsers.length !== 2 || Object.entries(expectedBrowsers).some(([brand, version]) => {
|
||||
const browser = browsers.find((entry) => entry?.brand === brand);
|
||||
return !browser || browser.full_version !== version || browser.major !== Number(version.split(".")[0])
|
||||
|| browser.source !== "installed_executable";
|
||||
})) throw new Error("WP7_02_CANDIDATE_BROWSER_MISMATCH");
|
||||
if (!/^[A-F0-9]{64}$/.test(record.candidate_package?.sha256 ?? "")) throw new Error("WP7_02_CANDIDATE_PACKAGE_HASH_INVALID");
|
||||
return {
|
||||
browsers: Object.entries(expectedBrowsers).map(([brand, full_version]) => ({ brand, full_version })),
|
||||
build_commit: record.build_commit,
|
||||
candidate_package_sha256: record.candidate_package.sha256,
|
||||
fixed_port: record.fixed_port,
|
||||
record_sha256: sha256(record),
|
||||
status: record.status,
|
||||
};
|
||||
}
|
||||
|
||||
function validateRealModelConfig(modelConfig, modelId) {
|
||||
if (!modelConfig || typeof modelConfig !== "object") return { blocker: "real_model_config_absent" };
|
||||
const endpoint = modelConfig.route_profile?.endpoint;
|
||||
const validEndpoint = typeof endpoint === "string" && endpoint.startsWith("https://")
|
||||
&& !/\.(?:invalid)(?:\/|$)/i.test(endpoint) && !/https:\/\/(?:localhost|127\.0\.0\.1)(?:[:/]|$)/i.test(endpoint);
|
||||
if (modelConfig.model_id !== modelId || !Number.isSafeInteger(modelConfig.config_version) || modelConfig.config_version <= 0
|
||||
|| !validEndpoint || typeof modelConfig.gateway_account_ref !== "string" || /mock/i.test(modelConfig.gateway_account_ref)) {
|
||||
return { blocker: "real_model_config_invalid" };
|
||||
}
|
||||
return {
|
||||
config: {
|
||||
config_version: modelConfig.config_version,
|
||||
endpoint_sha256: sha256(endpoint),
|
||||
gateway_account_ref_sha256: sha256(modelConfig.gateway_account_ref),
|
||||
model_id: modelId,
|
||||
route_profile_sha256: sha256(modelConfig.route_profile),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function inspectAiGatewayReadiness({ candidateRecord, confirmed, credentialTargets, modelConfig, modelId }) {
|
||||
const candidate = validateCandidateDependency(candidateRecord);
|
||||
assertModelId(modelId);
|
||||
const blockers = [];
|
||||
if (confirmed !== true) blockers.push("explicit_confirmation_absent");
|
||||
if (!Array.isArray(credentialTargets) || !credentialTargets.includes(AI_GATEWAY_CREDENTIAL_TARGET)) {
|
||||
blockers.push("real_gateway_credentials_absent");
|
||||
}
|
||||
const checkedConfig = validateRealModelConfig(modelConfig, modelId);
|
||||
if (checkedConfig.blocker) blockers.push(checkedConfig.blocker);
|
||||
return {
|
||||
blockers,
|
||||
candidate,
|
||||
model_config: checkedConfig.config ?? null,
|
||||
model_id: modelId,
|
||||
plan: buildModelContractPlan(modelId),
|
||||
real_calls: 0,
|
||||
status: blockers.length > 0 ? "externally_blocked" : "ready_for_controlled_execution",
|
||||
};
|
||||
}
|
||||
|
||||
function blockedScenarios(plan) {
|
||||
return [
|
||||
...plan.inputs.map((name) => ({ kind: "input", name, status: "not_run" })),
|
||||
...plan.ratios.map((name) => ({ kind: "ratio", name, status: "not_run" })),
|
||||
...plan.execution_modes.map((name) => ({ kind: "execution_mode", name, status: "not_run" })),
|
||||
...plan.response_checks.map((name) => ({ kind: "response_check", name, status: "not_run" })),
|
||||
...plan.error_categories.map((name) => ({
|
||||
expected: plan.error_expectations[name], kind: "error_category", name, status: "not_run",
|
||||
})),
|
||||
...plan.state_checks.map((name) => ({ kind: "state_check", name, status: "not_run" })),
|
||||
];
|
||||
}
|
||||
|
||||
export function buildBlockedModelEvidence({ blockers, candidateRecord, modelId, modelConfig = null, runId }) {
|
||||
const candidate = validateCandidateDependency(candidateRecord);
|
||||
const plan = buildModelContractPlan(modelId);
|
||||
if (!Array.isArray(blockers) || blockers.length === 0) throw new Error("WP7_02_EXTERNAL_BLOCKER_REQUIRED");
|
||||
const evidenceId = `sha256:${sha256({ model_id: modelId, run_id: runId })}`;
|
||||
return {
|
||||
blockers: [...new Set(blockers)],
|
||||
candidate,
|
||||
evidence_id: evidenceId,
|
||||
external_calls: {
|
||||
mode: "controlled_real_not_executed",
|
||||
planned_provider_requests_max: plan.planned_provider_requests_max,
|
||||
planned_request_breakdown: plan.planned_request_breakdown,
|
||||
quota_impact: plan.quota_impact,
|
||||
real_calls: 0,
|
||||
service: "ai-gateway-service-id",
|
||||
},
|
||||
manual_review: {
|
||||
decision: "Do not mark this model verified until every controlled-real scenario passes against the listed config version.",
|
||||
status: "blocked",
|
||||
},
|
||||
matrix: {
|
||||
config_version: modelConfig?.config_version ?? null,
|
||||
model_id: modelId,
|
||||
scenarios: blockedScenarios(plan),
|
||||
status: "not_run",
|
||||
},
|
||||
model_id: modelId,
|
||||
redaction: {
|
||||
retained_fields: ["status", "category", "duration_ms", "mime", "dimensions", "usage_summary", "evidence_hash", "time"],
|
||||
secret_scan: "passed",
|
||||
},
|
||||
run_id: runId,
|
||||
status: "externally_blocked",
|
||||
};
|
||||
}
|
||||
|
||||
export function validateIndependentEvidenceSet(evidence) {
|
||||
if (!Array.isArray(evidence) || evidence.length !== WP7_02_MODEL_IDS.length) throw new Error("WP7_02_THREE_MODEL_EVIDENCE_REQUIRED");
|
||||
const ids = evidence.map((entry) => entry.model_id).toSorted();
|
||||
if (JSON.stringify(ids) !== JSON.stringify([...WP7_02_MODEL_IDS].toSorted())) throw new Error("WP7_02_MODEL_EVIDENCE_SET_INVALID");
|
||||
if (new Set(evidence.map((entry) => entry.evidence_id)).size !== evidence.length) throw new Error("WP7_02_SHARED_EVIDENCE_FORBIDDEN");
|
||||
for (const entry of evidence) {
|
||||
if (entry.matrix?.model_id !== entry.model_id || entry.status !== "externally_blocked"
|
||||
|| entry.external_calls?.real_calls !== 0 || /\"verified\"\s*:/i.test(JSON.stringify(entry))) {
|
||||
throw new Error("WP7_02_BLOCKED_EVIDENCE_INVALID");
|
||||
}
|
||||
}
|
||||
return evidence;
|
||||
}
|
||||
|
||||
export function probeAiGatewayCredentialTargets() {
|
||||
if (process.platform !== "win32") return [];
|
||||
const result = spawnSync("cmdkey.exe", ["/list"], { encoding: "utf8", windowsHide: true });
|
||||
if ((result.status ?? 1) !== 0) throw new Error("WP7_02_CREDENTIAL_TARGET_PROBE_FAILED");
|
||||
return result.stdout.includes(AI_GATEWAY_CREDENTIAL_TARGET) ? [AI_GATEWAY_CREDENTIAL_TARGET] : [];
|
||||
}
|
||||
|
||||
export function writeBlockedModelEvidence(directory, evidence) {
|
||||
mkdirSync(resolve(directory), { recursive: true });
|
||||
const files = {
|
||||
"contract-matrix.json": evidence.matrix,
|
||||
"external-calls.json": evidence.external_calls,
|
||||
"manual-review.json": evidence.manual_review,
|
||||
"readiness.json": {
|
||||
blockers: evidence.blockers,
|
||||
candidate: evidence.candidate,
|
||||
evidence_id: evidence.evidence_id,
|
||||
model_id: evidence.model_id,
|
||||
run_id: evidence.run_id,
|
||||
status: evidence.status,
|
||||
},
|
||||
"redaction.json": evidence.redaction,
|
||||
};
|
||||
for (const [name, value] of Object.entries(files)) {
|
||||
writeFileSync(resolve(directory, name), `${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
return Object.keys(files);
|
||||
}
|
||||
Reference in New Issue
Block a user