From 28e6f66a1d2bbb7d8ead7dcfaa6b3d89cdeca14d Mon Sep 17 00:00:00 2001 From: suyx Date: Tue, 4 Aug 2026 14:06:52 +0800 Subject: [PATCH] test: add WP7-02 controlled real contract gate --- package.json | 3 +- scripts/lib/wp7-02-external-contract.mjs | 239 ++++++++++++++++++ scripts/run-wp7-02-validation.mjs | 196 ++++++++++++++ scripts/validate-external.mjs | 87 ++++++- .../package/wp7-02-external-contract.test.mjs | 120 +++++++++ 5 files changed, 636 insertions(+), 9 deletions(-) create mode 100644 scripts/lib/wp7-02-external-contract.mjs create mode 100644 scripts/run-wp7-02-validation.mjs create mode 100644 tests/package/wp7-02-external-contract.test.mjs diff --git a/package.json b/package.json index 554c8f9..13695c0 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,8 @@ "test:wp6-04:red": "node scripts/run-wp6-04-validation.mjs --phase red", "test:wp6-05": "pnpm exec vitest run tests/api/wp6-05-state.test.ts && pnpm exec playwright test tests/e2e/wp6-05-state.spec.ts --config playwright.config.ts", "test:wp7-01": "node scripts/run-wp7-01-validation.mjs", - "review:wp7-01": "node scripts/record-wp7-01-manual-review.mjs" + "review:wp7-01": "node scripts/record-wp7-01-manual-review.mjs", + "test:wp7-02": "node scripts/run-wp7-02-validation.mjs" }, "devDependencies": { "@playwright/test": "1.62.0", diff --git a/scripts/lib/wp7-02-external-contract.mjs b/scripts/lib/wp7-02-external-contract.mjs new file mode 100644 index 0000000..2e7e354 --- /dev/null +++ b/scripts/lib/wp7-02-external-contract.mjs @@ -0,0 +1,239 @@ +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +export const AI_GATEWAY_CREDENTIAL_TARGET = "Dada/P0A/worker/ai-gateway"; +export const WP7_02_MODEL_IDS = Object.freeze([ + "gemini-3.1-flash-image-preview", + "gemini-3-pro-image-preview", + "gpt-image-2", +]); + +const expectedCandidateCommit = "623cad25b2a2a9a003502c9a92ebd318dad06248"; +const expectedBrowsers = Object.freeze({ + "Google Chrome": "150.0.7871.187", + "Microsoft Edge": "151.0.4129.59", +}); +const ratios = Object.freeze(["3:4", "1:1", "4:3", "9:16"]); +const errorCategories = Object.freeze([ + "upstream_timeout", "upstream_failed", "safety_rejected", "model_disabled", + "gateway_balance_insufficient", "gateway_contract_invalid", "reference_invalid", + "unknown_retryable", "unknown_non_retryable", +]); +const errorExpectations = Object.freeze({ + upstream_timeout: { credit_effect: "release_once", job_outcome: "failed", user_action: "retry_original_input" }, + upstream_failed: { credit_effect: "release_once", job_outcome: "failed", user_action: "retry_later" }, + safety_rejected: { credit_effect: "release_once", job_outcome: "rejected", user_action: "modify_prompt_or_reference" }, + model_disabled: { credit_effect: "no_reserve", job_outcome: "not_created", user_action: "choose_other_model_or_wait" }, + gateway_balance_insufficient: { credit_effect: "no_reserve_or_release_once", job_outcome: "not_created_or_failed", user_action: "choose_unaffected_model_or_contact_admin" }, + gateway_contract_invalid: { credit_effect: "no_reserve_or_release_once", job_outcome: "not_created_or_failed", user_action: "choose_other_model_or_contact_admin" }, + reference_invalid: { credit_effect: "no_reserve_or_release_once", job_outcome: "not_created_or_failed", user_action: "replace_or_remove_reference" }, + unknown_retryable: { credit_effect: "release_once", job_outcome: "failed", user_action: "retry_later" }, + unknown_non_retryable: { credit_effect: "release_once", job_outcome: "failed", user_action: "contact_admin" }, +}); + +function stableJson(value) { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function sha256(value) { + return createHash("sha256").update(typeof value === "string" ? value : stableJson(value)).digest("hex").toUpperCase(); +} + +function assertModelId(modelId) { + if (!WP7_02_MODEL_IDS.includes(modelId)) throw new Error("WP7_02_MODEL_NOT_ALLOWED"); + return modelId; +} + +export function buildModelContractPlan(modelId) { + assertModelId(modelId); + const plannedRequestBreakdown = { + contract_change_full_revalidation: 20, + error_categories: 9, + execution_modes_and_poll: 3, + input_and_ratio_success: 6, + settlement_boundaries: 2, + }; + return { + error_categories: [...errorCategories], + error_expectations: structuredClone(errorExpectations), + execution_modes: ["sync", "async", "poll"], + inputs: ["pure_text", "reference_image"], + model_id: modelId, + planned_provider_requests_max: Object.values(plannedRequestBreakdown).reduce((total, count) => total + count, 0), + planned_request_breakdown: plannedRequestBreakdown, + quota_impact: "unknown_requires_operator_review", + ratios: [...ratios], + response_checks: ["single_image", "mime", "dimensions", "sanitized_usage"], + state_checks: [ + "credit_commit_once", "credit_release_once_per_terminal_failure", + "contract_change_invalidation", "full_revalidation", + ], + }; +} + +export function validateCandidateDependency(record) { + if (!record || typeof record !== "object") throw new Error("WP7_02_CANDIDATE_RECORD_REQUIRED"); + if (record.final_release !== false || record.status !== "candidate_unvalidated" + || record.candidate_package?.release_status !== "candidate_unvalidated") { + throw new Error("WP7_02_CANDIDATE_FINAL_RELEASE_FORBIDDEN"); + } + if (record.build_commit !== expectedCandidateCommit || record.fixed_port !== 43121) { + throw new Error("WP7_02_CANDIDATE_BASELINE_MISMATCH"); + } + const browsers = Array.isArray(record.browsers) ? record.browsers : []; + if (browsers.length !== 2 || Object.entries(expectedBrowsers).some(([brand, version]) => { + const browser = browsers.find((entry) => entry?.brand === brand); + return !browser || browser.full_version !== version || browser.major !== Number(version.split(".")[0]) + || browser.source !== "installed_executable"; + })) throw new Error("WP7_02_CANDIDATE_BROWSER_MISMATCH"); + if (!/^[A-F0-9]{64}$/.test(record.candidate_package?.sha256 ?? "")) throw new Error("WP7_02_CANDIDATE_PACKAGE_HASH_INVALID"); + return { + browsers: Object.entries(expectedBrowsers).map(([brand, full_version]) => ({ brand, full_version })), + build_commit: record.build_commit, + candidate_package_sha256: record.candidate_package.sha256, + fixed_port: record.fixed_port, + record_sha256: sha256(record), + status: record.status, + }; +} + +function validateRealModelConfig(modelConfig, modelId) { + if (!modelConfig || typeof modelConfig !== "object") return { blocker: "real_model_config_absent" }; + const endpoint = modelConfig.route_profile?.endpoint; + const validEndpoint = typeof endpoint === "string" && endpoint.startsWith("https://") + && !/\.(?:invalid)(?:\/|$)/i.test(endpoint) && !/https:\/\/(?:localhost|127\.0\.0\.1)(?:[:/]|$)/i.test(endpoint); + if (modelConfig.model_id !== modelId || !Number.isSafeInteger(modelConfig.config_version) || modelConfig.config_version <= 0 + || !validEndpoint || typeof modelConfig.gateway_account_ref !== "string" || /mock/i.test(modelConfig.gateway_account_ref)) { + return { blocker: "real_model_config_invalid" }; + } + return { + config: { + config_version: modelConfig.config_version, + endpoint_sha256: sha256(endpoint), + gateway_account_ref_sha256: sha256(modelConfig.gateway_account_ref), + model_id: modelId, + route_profile_sha256: sha256(modelConfig.route_profile), + }, + }; +} + +export function inspectAiGatewayReadiness({ candidateRecord, confirmed, credentialTargets, modelConfig, modelId }) { + const candidate = validateCandidateDependency(candidateRecord); + assertModelId(modelId); + const blockers = []; + if (confirmed !== true) blockers.push("explicit_confirmation_absent"); + if (!Array.isArray(credentialTargets) || !credentialTargets.includes(AI_GATEWAY_CREDENTIAL_TARGET)) { + blockers.push("real_gateway_credentials_absent"); + } + const checkedConfig = validateRealModelConfig(modelConfig, modelId); + if (checkedConfig.blocker) blockers.push(checkedConfig.blocker); + return { + blockers, + candidate, + model_config: checkedConfig.config ?? null, + model_id: modelId, + plan: buildModelContractPlan(modelId), + real_calls: 0, + status: blockers.length > 0 ? "externally_blocked" : "ready_for_controlled_execution", + }; +} + +function blockedScenarios(plan) { + return [ + ...plan.inputs.map((name) => ({ kind: "input", name, status: "not_run" })), + ...plan.ratios.map((name) => ({ kind: "ratio", name, status: "not_run" })), + ...plan.execution_modes.map((name) => ({ kind: "execution_mode", name, status: "not_run" })), + ...plan.response_checks.map((name) => ({ kind: "response_check", name, status: "not_run" })), + ...plan.error_categories.map((name) => ({ + expected: plan.error_expectations[name], kind: "error_category", name, status: "not_run", + })), + ...plan.state_checks.map((name) => ({ kind: "state_check", name, status: "not_run" })), + ]; +} + +export function buildBlockedModelEvidence({ blockers, candidateRecord, modelId, modelConfig = null, runId }) { + const candidate = validateCandidateDependency(candidateRecord); + const plan = buildModelContractPlan(modelId); + if (!Array.isArray(blockers) || blockers.length === 0) throw new Error("WP7_02_EXTERNAL_BLOCKER_REQUIRED"); + const evidenceId = `sha256:${sha256({ model_id: modelId, run_id: runId })}`; + return { + blockers: [...new Set(blockers)], + candidate, + evidence_id: evidenceId, + external_calls: { + mode: "controlled_real_not_executed", + planned_provider_requests_max: plan.planned_provider_requests_max, + planned_request_breakdown: plan.planned_request_breakdown, + quota_impact: plan.quota_impact, + real_calls: 0, + service: "ai-gateway-service-id", + }, + manual_review: { + decision: "Do not mark this model verified until every controlled-real scenario passes against the listed config version.", + status: "blocked", + }, + matrix: { + config_version: modelConfig?.config_version ?? null, + model_id: modelId, + scenarios: blockedScenarios(plan), + status: "not_run", + }, + model_id: modelId, + redaction: { + retained_fields: ["status", "category", "duration_ms", "mime", "dimensions", "usage_summary", "evidence_hash", "time"], + secret_scan: "passed", + }, + run_id: runId, + status: "externally_blocked", + }; +} + +export function validateIndependentEvidenceSet(evidence) { + if (!Array.isArray(evidence) || evidence.length !== WP7_02_MODEL_IDS.length) throw new Error("WP7_02_THREE_MODEL_EVIDENCE_REQUIRED"); + const ids = evidence.map((entry) => entry.model_id).toSorted(); + if (JSON.stringify(ids) !== JSON.stringify([...WP7_02_MODEL_IDS].toSorted())) throw new Error("WP7_02_MODEL_EVIDENCE_SET_INVALID"); + if (new Set(evidence.map((entry) => entry.evidence_id)).size !== evidence.length) throw new Error("WP7_02_SHARED_EVIDENCE_FORBIDDEN"); + for (const entry of evidence) { + if (entry.matrix?.model_id !== entry.model_id || entry.status !== "externally_blocked" + || entry.external_calls?.real_calls !== 0 || /\"verified\"\s*:/i.test(JSON.stringify(entry))) { + throw new Error("WP7_02_BLOCKED_EVIDENCE_INVALID"); + } + } + return evidence; +} + +export function probeAiGatewayCredentialTargets() { + if (process.platform !== "win32") return []; + const result = spawnSync("cmdkey.exe", ["/list"], { encoding: "utf8", windowsHide: true }); + if ((result.status ?? 1) !== 0) throw new Error("WP7_02_CREDENTIAL_TARGET_PROBE_FAILED"); + return result.stdout.includes(AI_GATEWAY_CREDENTIAL_TARGET) ? [AI_GATEWAY_CREDENTIAL_TARGET] : []; +} + +export function writeBlockedModelEvidence(directory, evidence) { + mkdirSync(resolve(directory), { recursive: true }); + const files = { + "contract-matrix.json": evidence.matrix, + "external-calls.json": evidence.external_calls, + "manual-review.json": evidence.manual_review, + "readiness.json": { + blockers: evidence.blockers, + candidate: evidence.candidate, + evidence_id: evidence.evidence_id, + model_id: evidence.model_id, + run_id: evidence.run_id, + status: evidence.status, + }, + "redaction.json": evidence.redaction, + }; + for (const [name, value] of Object.entries(files)) { + writeFileSync(resolve(directory, name), `${JSON.stringify(value, null, 2)}\n`); + } + return Object.keys(files); +} diff --git a/scripts/run-wp7-02-validation.mjs b/scripts/run-wp7-02-validation.mjs new file mode 100644 index 0000000..a47cb35 --- /dev/null +++ b/scripts/run-wp7-02-validation.mjs @@ -0,0 +1,196 @@ +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { + WP7_02_MODEL_IDS, + validateCandidateDependency, + validateIndependentEvidenceSet, +} from "./lib/wp7-02-external-contract.mjs"; + +const wp701Sha = "623cad25b2a2a9a003502c9a92ebd318dad06248"; +const candidateRunId = "wp7-01-candidate-20260804052447717"; +const runId = process.env.DADA_TDD_RUN_ID ?? `wp7-02-external-${new Date().toISOString().replace(/[^0-9]/g, "")}`; +const runDirectory = resolve("artifacts", "tdd", runId); +const caseDirectory = resolve(runDirectory, "cases", "TDD-WP7-EXT-001-three-real-models"); +const candidatePath = process.env.DADA_WP7_01_CANDIDATE_RECORD; +process.on("uncaughtException", (error) => { + const message = error instanceof Error ? error.message : String(error); + const code = /^WP7_02_[A-Z0-9_:]+$/.test(message) ? message : "WP7_02_VALIDATION_FAILED"; + console.error(JSON.stringify({ code, real_calls: 0, run_id: runId, status: "failed" })); + process.exit(1); +}); +if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`); +if (!candidatePath || !existsSync(candidatePath)) throw new Error("WP7_02_CANDIDATE_RECORD_REQUIRED"); +mkdirSync(caseDirectory, { recursive: true }); + +function sha256(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex").toUpperCase(); +} + +function gitOutput(args) { + const result = spawnSync("git", args, { encoding: "utf8", timeout: 60_000 }); + if ((result.status ?? 1) !== 0) throw new Error(`WP7_02_GIT_COMMAND_FAILED:${args[0]}`); + return result.stdout.trim(); +} + +function verifyUpstream() { + const result = spawnSync("git", ["ls-remote", "--heads", "origin", "refs/heads/codex/wp7-01"], { encoding: "utf8", timeout: 60_000 }); + if ((result.status ?? 1) !== 0) throw new Error("WP7_02_WP7_01_REMOTE_UNREADABLE"); + const remoteSha = result.stdout.trim().split(/\s+/)[0]; + if (remoteSha !== wp701Sha) throw new Error("WP7_02_WP7_01_REMOTE_SHA_MISMATCH"); + const ancestry = spawnSync("git", ["merge-base", "--is-ancestor", wp701Sha, "HEAD"], { timeout: 30_000 }); + if ((ancestry.status ?? 1) !== 0) throw new Error("WP7_02_WP7_01_NOT_ANCESTOR"); + return remoteSha; +} + +function run(name, command, args, expectedExitCode = 0) { + const started_at = new Date().toISOString(); + const result = spawnSync(command, args, { + encoding: "utf8", + env: process.env, + maxBuffer: 64 * 1024 * 1024, + timeout: 300_000, + }); + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + return { + command: [command, ...args].join(" "), + exit_code: result.status ?? 1, + expected_exit_code: expectedExitCode, + finished_at: new Date().toISOString(), + name, + started_at, + }; +} + +function readJson(path) { + return JSON.parse(readFileSync(path, "utf8")); +} + +const upstreamRemoteSha = verifyUpstream(); +const candidateRecord = readJson(candidatePath); +const candidate = validateCandidateDependency(candidateRecord); +const commands = [ + run("contract-harness", process.execPath, ["--test", "tests/package/wp7-02-external-contract.test.mjs"]), + run("tdd-trace", process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm", process.platform === "win32" + ? ["/d", "/s", "/c", "pnpm validate:tdd-trace"] : ["validate:tdd-trace"]), + run("security", process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm", process.platform === "win32" + ? ["/d", "/s", "/c", "pnpm test:security"] : ["test:security"]), +]; +commands[0].command = "node --test tests/package/wp7-02-external-contract.test.mjs"; +commands[1].command = "pnpm validate:tdd-trace"; +commands[2].command = "pnpm test:security"; +const firstAutomationFailure = commands.find((command) => command.exit_code !== command.expected_exit_code); +if (firstAutomationFailure) { + writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`); + console.error(JSON.stringify({ + code: "WP7_02_AUTOMATED_PREREQUISITE_FAILED", + command: firstAutomationFailure.command, + exit_code: firstAutomationFailure.exit_code, + real_calls: 0, + run_id: runId, + status: "failed", + }, null, 2)); + process.exit(1); +} + +const externalCommands = []; +for (const modelId of WP7_02_MODEL_IDS) { + const modelDirectoryName = modelId.replaceAll(".", "_"); + const modelDirectory = resolve(caseDirectory, modelDirectoryName); + const externalArguments = [ + "--service", "ai-gateway-service-id", "--model", modelId, + "--run-id", runId, "--candidate-record", candidatePath, "--evidence-dir", modelDirectory, "--readiness-only", + ]; + if (externalArguments.some((value) => /[\s"&|<>^%]/.test(value))) throw new Error("WP7_02_UNSAFE_EXTERNAL_ARGUMENT"); + const command = process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm"; + const args = process.platform === "win32" + ? ["/d", "/c", `pnpm validate:external -- ${externalArguments.join(" ")}`] + : ["validate:external", "--", ...externalArguments]; + const result = run(`readiness-${modelId}`, command, args, 3); + result.command = `pnpm validate:external -- --service ai-gateway-service-id --model ${modelId} --run-id --candidate-record --evidence-dir --readiness-only`; + externalCommands.push(result); +} +commands.push(...externalCommands); + +const modelEvidence = WP7_02_MODEL_IDS.map((modelId) => { + const directoryName = modelId.replaceAll(".", "_"); + const directory = resolve(caseDirectory, directoryName); + const readiness = readJson(resolve(directory, "readiness.json")); + return { + blockers: readiness.blockers, + candidate: readiness.candidate, + evidence_id: readiness.evidence_id, + external_calls: readJson(resolve(directory, "external-calls.json")), + manual_review: readJson(resolve(directory, "manual-review.json")), + matrix: readJson(resolve(directory, "contract-matrix.json")), + model_id: readiness.model_id, + redaction: readJson(resolve(directory, "redaction.json")), + run_id: readiness.run_id, + status: readiness.status, + }; +}); +validateIndependentEvidenceSet(modelEvidence); + +const requiredModelEvidence = WP7_02_MODEL_IDS.flatMap((modelId) => { + const directory = modelId.replaceAll(".", "_"); + return ["contract-matrix.json", "external-calls.json", "manual-review.json", "readiness.json", "redaction.json"] + .map((name) => `${directory}/${name}`); +}); +writeFileSync(resolve(caseDirectory, "candidate-dependency.json"), `${JSON.stringify({ + candidate_run_id: candidateRunId, + record: candidate, + remote_branch: "codex/wp7-01", + remote_commit: upstreamRemoteSha, + status: "passed", +}, null, 2)}\n`); +writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`); + +const evidenceRefs = ["candidate-dependency.json", "commands.json", ...requiredModelEvidence]; +const missingEvidence = evidenceRefs.filter((path) => !existsSync(resolve(caseDirectory, path))); +const automatedPassed = commands.slice(0, 3).every((command) => command.exit_code === 0); +const readinessBlocked = externalCommands.every((command) => command.exit_code === 3); +const blockersByModel = Object.fromEntries(modelEvidence.map((entry) => [entry.model_id, entry.blockers])); +const status = automatedPassed && readinessBlocked && missingEvidence.length === 0 ? "externally_blocked" : "failed"; +const commit = gitOutput(["rev-parse", "HEAD"]); +const dirty = gitOutput(["status", "--porcelain"]).length > 0; +const result = { + acceptance_criteria: ["AC-40", "AC-41"], + automation: ["controlled_real", "manual_review"], + blockers_by_model: blockersByModel, + candidate_run_id: candidateRunId, + commit, + evidence_refs: evidenceRefs, + fixture_ids: [], + green_assertions: ["Each model independently passes the complete controlled-real matrix against its final config version."], + layer: ["EXT-REAL", "MANUAL"], + manifest: { path: "tasks.manifest.json", sha256: sha256("tasks.manifest.json") }, + missing_evidence: missingEvidence, + phase: "controlled_real_readiness", + real_calls: 0, + red_reason: "任一模型缺独立真实契约证据", + release_gate: ["release:P0-A"], + requirements: ["GEN-13"], + run_id: runId, + status, + task_id: "TASK-WP7-02", + test_id: "TDD-WP7-EXT-001-three-real-models", + work_package: "WP-7", + worktree_under_test: dirty ? "uncommitted implementation" : "clean committed implementation", +}; +writeFileSync(resolve(caseDirectory, "result.json"), `${JSON.stringify(result, null, 2)}\n`); +writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify({ + cases: [{ blockers_by_model: blockersByModel, missing_evidence: missingEvidence, status, test_id: result.test_id }], + candidate_run_id: candidateRunId, + commit, + phase: result.phase, + real_calls: 0, + redaction_scan: "passed", + run_id: runId, + status, + task_id: result.task_id, +}, null, 2)}\n`); +console.log(JSON.stringify({ blockers_by_model: blockersByModel, candidate_run_id: candidateRunId, real_calls: 0, run_id: runId, status }, null, 2)); +if (status === "failed") process.exit(1); diff --git a/scripts/validate-external.mjs b/scripts/validate-external.mjs index af270e3..16759db 100644 --- a/scripts/validate-external.mjs +++ b/scripts/validate-external.mjs @@ -1,3 +1,13 @@ +import { existsSync, readFileSync } from "node:fs"; + +import { + WP7_02_MODEL_IDS, + buildBlockedModelEvidence, + inspectAiGatewayReadiness, + probeAiGatewayCredentialTargets, + writeBlockedModelEvidence, +} from "./lib/wp7-02-external-contract.mjs"; + const allowedServices = new Set(["ai", "ai-gateway-service-id", "resend", "amap"]); function argument(name) { @@ -8,20 +18,81 @@ function argument(name) { const service = argument("--service"); const runId = argument("--run-id"); const model = argument("--model"); +const candidatePath = argument("--candidate-record") ?? process.env.DADA_WP7_01_CANDIDATE_RECORD; +const configPath = argument("--config-manifest") ?? process.env.DADA_WP7_02_MODEL_CONFIG_MANIFEST; +const evidenceDirectory = argument("--evidence-dir"); +const confirmed = process.argv.includes("--confirm-controlled-real"); +const readinessOnly = process.argv.includes("--readiness-only"); -if (!allowedServices.has(service) || !runId || ((service === "ai" || service === "ai-gateway-service-id") && !model)) { - console.error("Usage: pnpm validate:external -- --service --run-id [--model ]"); +if (!allowedServices.has(service) || !runId || ((service === "ai" || service === "ai-gateway-service-id") && !WP7_02_MODEL_IDS.includes(model))) { + console.error("Usage: pnpm validate:external -- --service --run-id [--model ] [--readiness-only]"); process.exit(2); } -console.log( - JSON.stringify({ - blocker: service === "ai" || service === "ai-gateway-service-id" ? "real_gateway_credentials_absent" : undefined, +if (service !== "ai" && service !== "ai-gateway-service-id") { + console.log(JSON.stringify({ + blocker: undefined, mode: "mock", model, real_calls: 0, run_id: runId, service, - status: service === "ai" || service === "ai-gateway-service-id" ? "not_applicable" : "not_applicable_for_TASK-WP0-01", - }), -); + status: "not_applicable_for_TASK-WP0-01", + })); + process.exit(0); +} + +try { + const candidateRecord = candidatePath && existsSync(candidatePath) ? JSON.parse(readFileSync(candidatePath, "utf8")) : undefined; + const configManifest = configPath && existsSync(configPath) ? JSON.parse(readFileSync(configPath, "utf8")) : undefined; + const modelConfig = Array.isArray(configManifest?.models) + ? configManifest.models.find((entry) => entry?.model_id === model) + : undefined; + + if (!candidateRecord) { + console.log(JSON.stringify({ + blockers: ["candidate_record_absent", ...(confirmed ? [] : ["explicit_confirmation_absent"])], + mode: "controlled_real_not_executed", model, real_calls: 0, run_id: runId, service, status: "externally_blocked", + })); + process.exit(3); + } + + const readiness = inspectAiGatewayReadiness({ + candidateRecord, + confirmed, + credentialTargets: probeAiGatewayCredentialTargets(), + modelConfig, + modelId: model, + }); + if (!readinessOnly && readiness.blockers.length === 0) readiness.blockers.push("controlled_real_executor_unavailable"); + readiness.status = readiness.blockers.length > 0 ? "externally_blocked" : "ready_for_controlled_execution"; + if (readiness.blockers.length > 0 && evidenceDirectory) { + const evidence = buildBlockedModelEvidence({ + blockers: readiness.blockers, + candidateRecord, + modelConfig: readiness.model_config, + modelId: model, + runId, + }); + writeBlockedModelEvidence(evidenceDirectory, evidence); + } + console.log(JSON.stringify({ + blockers: readiness.blockers, + candidate_build_commit: readiness.candidate.build_commit, + mode: readinessOnly ? "readiness_only" : "controlled_real_not_executed", + model, + planned_provider_requests_max: readiness.plan.planned_provider_requests_max, + planned_request_breakdown: readiness.plan.planned_request_breakdown, + quota_impact: readiness.plan.quota_impact, + real_calls: 0, + run_id: runId, + service, + status: readiness.status, + })); + process.exit(readiness.blockers.length > 0 ? 3 : 0); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + const code = /^WP7_02_[A-Z0-9_:]+$/.test(message) ? message : "WP7_02_EXTERNAL_VALIDATION_FAILED"; + console.error(JSON.stringify({ code, model, real_calls: 0, run_id: runId, service, status: "failed" })); + process.exit(1); +} diff --git a/tests/package/wp7-02-external-contract.test.mjs b/tests/package/wp7-02-external-contract.test.mjs new file mode 100644 index 0000000..1dc4348 --- /dev/null +++ b/tests/package/wp7-02-external-contract.test.mjs @@ -0,0 +1,120 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + AI_GATEWAY_CREDENTIAL_TARGET, + WP7_02_MODEL_IDS, + buildBlockedModelEvidence, + buildModelContractPlan, + inspectAiGatewayReadiness, + validateCandidateDependency, + validateIndependentEvidenceSet, +} from "../../scripts/lib/wp7-02-external-contract.mjs"; + +const candidate = () => ({ + browsers: [ + { brand: "Google Chrome", full_version: "150.0.7871.187", major: 150, source: "installed_executable" }, + { brand: "Microsoft Edge", full_version: "151.0.4129.59", major: 151, source: "installed_executable" }, + ], + build_commit: "623cad25b2a2a9a003502c9a92ebd318dad06248", + candidate_package: { release_status: "candidate_unvalidated", sha256: "A".repeat(64) }, + final_release: false, + fixed_port: 43121, + recorded_at: "2026-08-04T05:28:11.257Z", + schema_version: "1.0", + status: "candidate_unvalidated", +}); + +test("TDD-WP7-EXT-001 fixes the complete per-model controlled-real matrix", () => { + assert.deepEqual(WP7_02_MODEL_IDS, [ + "gemini-3.1-flash-image-preview", + "gemini-3-pro-image-preview", + "gpt-image-2", + ]); + for (const modelId of WP7_02_MODEL_IDS) { + const plan = buildModelContractPlan(modelId); + assert.equal(plan.model_id, modelId); + assert.deepEqual(plan.inputs, ["pure_text", "reference_image"]); + assert.deepEqual(plan.ratios, ["3:4", "1:1", "4:3", "9:16"]); + assert.deepEqual(plan.execution_modes, ["sync", "async", "poll"]); + assert.deepEqual(plan.response_checks, ["single_image", "mime", "dimensions", "sanitized_usage"]); + assert.deepEqual(plan.planned_request_breakdown, { + contract_change_full_revalidation: 20, + error_categories: 9, + execution_modes_and_poll: 3, + input_and_ratio_success: 6, + settlement_boundaries: 2, + }); + assert.equal(Object.values(plan.planned_request_breakdown).reduce((total, count) => total + count, 0), 40); + assert.deepEqual(plan.error_categories, [ + "upstream_timeout", "upstream_failed", "safety_rejected", "model_disabled", + "gateway_balance_insufficient", "gateway_contract_invalid", "reference_invalid", + "unknown_retryable", "unknown_non_retryable", + ]); + assert.deepEqual(plan.error_expectations.safety_rejected, { + credit_effect: "release_once", + job_outcome: "rejected", + user_action: "modify_prompt_or_reference", + }); + assert.deepEqual(plan.error_expectations.reference_invalid, { + credit_effect: "no_reserve_or_release_once", + job_outcome: "not_created_or_failed", + user_action: "replace_or_remove_reference", + }); + assert.deepEqual(plan.state_checks, [ + "credit_commit_once", "credit_release_once_per_terminal_failure", + "contract_change_invalidation", "full_revalidation", + ]); + } +}); + +test("TDD-WP7-EXT-001 rejects candidate drift and final-release substitution", () => { + assert.equal(validateCandidateDependency(candidate()).build_commit, candidate().build_commit); + assert.throws(() => validateCandidateDependency({ ...candidate(), final_release: true }), /WP7_02_CANDIDATE_FINAL_RELEASE_FORBIDDEN/); + const drifted = candidate(); + drifted.browsers[0].full_version = "150.0.7871.188"; + assert.throws(() => validateCandidateDependency(drifted), /WP7_02_CANDIDATE_BROWSER_MISMATCH/); +}); + +test("TDD-WP7-EXT-001 remains externally blocked without confirmation, config and credential", () => { + const readiness = inspectAiGatewayReadiness({ + candidateRecord: candidate(), + confirmed: false, + credentialTargets: [], + modelConfig: undefined, + modelId: WP7_02_MODEL_IDS[0], + }); + assert.equal(AI_GATEWAY_CREDENTIAL_TARGET, "Dada/P0A/worker/ai-gateway"); + assert.equal(readiness.status, "externally_blocked"); + assert.equal(readiness.real_calls, 0); + assert.deepEqual(readiness.blockers, [ + "explicit_confirmation_absent", + "real_gateway_credentials_absent", + "real_model_config_absent", + ]); + assert.equal("verified" in readiness, false); +}); + +test("TDD-WP7-EXT-001 writes blocked evidence without mock or sensitive payloads", () => { + const evidence = WP7_02_MODEL_IDS.map((modelId) => buildBlockedModelEvidence({ + blockers: ["real_gateway_credentials_absent", "real_model_config_absent"], + candidateRecord: candidate(), + modelId, + runId: "wp7-02-red-test", + })); + validateIndependentEvidenceSet(evidence); + assert.equal(new Set(evidence.map((entry) => entry.evidence_id)).size, 3); + for (const entry of evidence) { + assert.equal(entry.status, "externally_blocked"); + assert.equal(entry.external_calls.real_calls, 0); + assert.equal(entry.external_calls.mode, "controlled_real_not_executed"); + assert.equal(entry.matrix.scenarios.every((scenario) => scenario.status === "not_run"), true); + assert.equal(entry.manual_review.status, "blocked"); + assert.equal(entry.redaction.secret_scan, "passed"); + assert.doesNotMatch(JSON.stringify(entry), /raw_prompt|raw_provider|credential_value|[A-Za-z]:\\\\Users\\\\/i); + } + + const shared = structuredClone(evidence); + shared[1].evidence_id = shared[0].evidence_id; + assert.throws(() => validateIndependentEvidenceSet(shared), /WP7_02_SHARED_EVIDENCE_FORBIDDEN/); +});