92 lines
5.6 KiB
TypeScript
92 lines
5.6 KiB
TypeScript
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
|
|
import { ModelConfigurationService } from "../../apps/api/src/model-configuration.js";
|
|
import { ModelContractEvidenceService } from "../../apps/api/src/model-contract-evidence.js";
|
|
import { RegistrationService } from "../../apps/api/src/registration.js";
|
|
import { MockResendAdapter } from "../../apps/api/src/resend-adapter.js";
|
|
|
|
const roots: string[] = [];
|
|
const services: RegistrationService[] = [];
|
|
const now = Date.parse("2026-08-02T16:00:00.000Z");
|
|
|
|
function harness() {
|
|
const root = mkdtempSync(join(tmpdir(), "dada-wp3-03-evidence-"));
|
|
roots.push(root);
|
|
const registration = new RegistrationService({
|
|
challengePepper: Buffer.alloc(32, 0xb1), clock: () => now,
|
|
currentPrivacyNoticeVersion: "p0a-registration-notice-v1", databasePath: join(root, "dada.sqlite3"),
|
|
invitePepper: Buffer.alloc(32, 0xb2), resend: new MockResendAdapter(), sessionPepper: Buffer.alloc(32, 0xb3),
|
|
});
|
|
services.push(registration);
|
|
const models = new ModelConfigurationService({ database: registration.database, clock: () => now });
|
|
return { evidence: new ModelContractEvidenceService({ clock: () => now, database: registration.database, models }), models, registration };
|
|
}
|
|
|
|
function matrix(modelId: string) {
|
|
return {
|
|
model_id: modelId,
|
|
protocol_version: "mock-v1",
|
|
pure_text: { status: "passed", outputs: 1 },
|
|
reference_image: { status: "passed", outputs: 1 },
|
|
ratios: ["3:4", "1:1", "4:3", "9:16"].map((ratio) => ({ ratio, status: "passed", outputs: 1 })),
|
|
execution_modes: ["sync", "async", "poll"],
|
|
error_mapping: ["upstream_timeout", "upstream_failed", "safety_rejected", "model_disabled", "gateway_balance_insufficient", "gateway_contract_invalid", "reference_invalid", "unknown_retryable", "unknown_non_retryable"],
|
|
};
|
|
}
|
|
|
|
function writeEvidence(relativePath: string, value: unknown) {
|
|
const root = process.env.DADA_EVIDENCE_DIR_CONTRACT;
|
|
if (!root) return;
|
|
const path = resolve(root, relativePath);
|
|
mkdirSync(join(path, ".."), { recursive: true });
|
|
writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
|
|
}
|
|
|
|
afterEach(() => {
|
|
for (const service of services.splice(0)) service.close();
|
|
for (const root of roots.splice(0)) rmSync(root, { force: true, recursive: true });
|
|
});
|
|
|
|
describe("TDD-WP3-ADP-002-invalidate-revalidate", () => {
|
|
it("records a complete per-model evidence set and leaves no raw provider payload", () => {
|
|
const { evidence, models, registration } = harness();
|
|
const first = evidence.recordVerified({
|
|
actorId: "wp3-03-evidence", expectedConfigSetVersion: 1,
|
|
evidence: { evidence_hash: "sha256:gemini-flash-mock", evidence_ref: "fixture:gemini-flash", matrix: matrix("gemini-3.1-flash-image-preview"), model_id: "gemini-3.1-flash-image-preview", verifier_ref: "mock-verifier", verified_at: new Date(now).toISOString() },
|
|
idempotencyKey: "wp3-03-evidence-0000000000000000000001",
|
|
});
|
|
expect(first.model.contract_validation_status).toBe("verified");
|
|
expect(first.model.config_version).toBe(2);
|
|
expect(models.read().models.find((model) => model.model_id === "gemini-3.1-flash-image-preview")).toMatchObject({ contract_validation_status: "verified", config_version: 2 });
|
|
expect(registration.database.prepare("SELECT COUNT(*) AS count FROM model_contract_evidence WHERE model_id = ?").get("gemini-3.1-flash-image-preview")).toEqual({ count: 1 });
|
|
expect(() => registration.database.prepare("UPDATE model_contract_evidence SET evidence_json = '{}' ").run()).toThrow();
|
|
|
|
const current = models.read();
|
|
const candidates = current.models.map(({ config_version: _configVersion, runtime_availability: _runtime, ...candidate }) => structuredClone(candidate));
|
|
const flash = candidates.find((candidate) => candidate.model_id === "gemini-3.1-flash-image-preview")!;
|
|
flash.route_profile = { ...flash.route_profile, endpoint: "https://mock.invalid/gemini/flash-v2" };
|
|
const changed = models.replace({ actorId: "wp3-03-route-change", expectedConfigSetVersion: 2, idempotencyKey: "wp3-03-route-change-0000000000000000000001", models: candidates });
|
|
expect(changed.config_set_version).toBe(3);
|
|
expect(changed.models.find((model) => model.model_id === flash.model_id)).toMatchObject({ contract_validation_status: "unverified", config_version: 3 });
|
|
expect(evidence.read(flash.model_id, 2)?.evidence_hash).toBe("sha256:gemini-flash-mock");
|
|
writeEvidence("db-diff.json", { before: { config_set_version: 2, contract_validation_status: "verified", config_version: 2 }, after: { config_set_version: 3, contract_validation_status: "unverified", config_version: 3 }, changed_fields: ["route_profile.endpoint"] });
|
|
writeEvidence("contract-matrix.json", matrix(flash.model_id));
|
|
writeEvidence("response.json", { model_id: flash.model_id, revalidation_required: true, output_count: 0 });
|
|
});
|
|
|
|
it("rejects an incomplete matrix before changing the config set", () => {
|
|
const { evidence, models } = harness();
|
|
expect(() => evidence.recordVerified({
|
|
actorId: "wp3-03-invalid", expectedConfigSetVersion: 1,
|
|
evidence: { evidence_hash: "sha256:invalid", evidence_ref: "fixture:invalid", matrix: { model_id: "gemini-3.1-flash-image-preview" }, model_id: "gemini-3.1-flash-image-preview", verifier_ref: "mock-verifier", verified_at: new Date(now).toISOString() },
|
|
idempotencyKey: "wp3-03-invalid-0000000000000000000001",
|
|
})).toThrow("contract_evidence_matrix_incomplete");
|
|
expect(models.read().config_set_version).toBe(1);
|
|
});
|
|
});
|