import { randomUUID } from "node:crypto"; import Database from "better-sqlite3"; import type BetterSqlite3 from "better-sqlite3"; import { configureWorkerDatabase } from "./sqlite-connection.js"; const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000; const recoveryCheckLifetimeMilliseconds = 5 * 60 * 1_000; const accountRefPattern = /^[a-z][a-z0-9_-]{2,119}$/; const sourceCategoryPattern = /^[a-z][a-z0-9_]{0,79}$/; const runtimeReasons = [ "available", "configured_disabled", "contract_unverified", "contract_blocked", "gateway_balance_insufficient", "gateway_paused", "worker_degraded", ] as const; type RuntimeReason = typeof runtimeReasons[number]; interface RuntimeRow { available_for_new_jobs: number; checked_at: number; model_id: string; reason: RuntimeReason; runtime_availability_version: number; } interface MappingRow { balance_blocked: number; gateway_account_ref: string; model_id: string; previous_runtime_reason: RuntimeReason | null; } interface BalanceRow { affected_model_ids_json: string; balance_status: "available" | "insufficient" | "unknown"; detected_at: number; gateway_account_ref: string; impact_scope: "model" | "account" | "unknown"; last_confirmed_at: number | null; recovery_status: "not_required" | "awaiting_confirmation" | "confirmed"; runtime_unavailable_model_ids_json: string; source_category: string; } export interface GatewayBalanceView { affectedModelIds: string[]; balanceStatus: "available" | "insufficient" | "unknown"; detectedAt: string; gatewayAccountRef: string; impactScope: "model" | "account" | "unknown"; lastConfirmedAt: string | null; recoveryStatus: "not_required" | "awaiting_confirmation" | "confirmed"; runtimeUnavailableModelIds: string[]; sourceCategory: string; } export interface GatewayRecoveryCheck { check_id: string; expires_at: string; gateway_account_ref: string; status: "passed"; checked_at: string; } export class GatewayBalanceRuntime { readonly database: BetterSqlite3.Database; private readonly clock: () => number; private readonly ownsDatabase: boolean; constructor(input: { clock?: () => number; database?: BetterSqlite3.Database; databasePath?: string }) { if (!input.database && !input.databasePath) throw new Error("gateway_balance_database_required"); this.database = input.database ?? new Database(input.databasePath!); this.ownsDatabase = !input.database; this.clock = input.clock ?? Date.now; if (this.ownsDatabase) configureWorkerDatabase(this.database); this.migrate(); } close() { if (this.ownsDatabase) this.database.close(); } seedModels(models: Array<{ gatewayAccountRef: string; modelId: string }>) { const now = this.clock(); this.immediate(() => { for (const model of models) { if (!accountRefPattern.test(model.gatewayAccountRef) || !model.modelId) throw new Error("gateway_balance_model_invalid"); this.database.prepare(` INSERT INTO model_runtime_availability ( model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version ) VALUES (?, 1, 'available', ?, 1) ON CONFLICT(model_id) DO NOTHING `).run(model.modelId, now); this.database.prepare(` INSERT INTO gateway_balance_affected_models (gateway_account_ref, model_id, balance_blocked, previous_runtime_reason) VALUES (?, ?, 0, NULL) ON CONFLICT(gateway_account_ref, model_id) DO NOTHING `).run(model.gatewayAccountRef, model.modelId); } }); } recordInsufficient(input: { eventId: string; gatewayAccountRef: string; impactScope: "model" | "account" | "unknown"; modelId: string; sourceCategory: string; }) { if (!input.eventId || !accountRefPattern.test(input.gatewayAccountRef) || !input.modelId || !sourceCategoryPattern.test(input.sourceCategory)) throw new Error("gateway_balance_signal_invalid"); return this.immediate(() => { const receipt = this.database.prepare("SELECT response_json FROM gateway_balance_event_receipts WHERE event_id = ?") .get(input.eventId) as { response_json: string } | undefined; if (receipt) return JSON.parse(receipt.response_json) as GatewayBalanceView; this.ensureModelMapping(input.gatewayAccountRef, input.modelId); const accountModels = this.database.prepare("SELECT model_id FROM gateway_balance_affected_models WHERE gateway_account_ref = ? ORDER BY model_id") .all(input.gatewayAccountRef) as Array<{ model_id: string }>; const newlyAffected = input.impactScope === "model" ? [input.modelId] : accountModels.map((row) => row.model_id); const existing = this.database.prepare("SELECT runtime_unavailable_model_ids_json FROM gateway_balance_states WHERE gateway_account_ref = ?") .get(input.gatewayAccountRef) as { runtime_unavailable_model_ids_json: string } | undefined; const unavailable = [...new Set([...(existing ? JSON.parse(existing.runtime_unavailable_model_ids_json) as string[] : []), ...newlyAffected])].toSorted(); const now = this.clock(); for (const modelId of unavailable) { const mapping = this.database.prepare("SELECT * FROM gateway_balance_affected_models WHERE gateway_account_ref = ? AND model_id = ?") .get(input.gatewayAccountRef, modelId) as MappingRow | undefined; const runtime = this.database.prepare("SELECT * FROM model_runtime_availability WHERE model_id = ?").get(modelId) as RuntimeRow | undefined; if (!mapping || !runtime) continue; if (mapping.balance_blocked === 0) { this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 1, previous_runtime_reason = ? WHERE gateway_account_ref = ? AND model_id = ?") .run(runtime.reason, input.gatewayAccountRef, modelId); } else { this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 1 WHERE gateway_account_ref = ? AND model_id = ?") .run(input.gatewayAccountRef, modelId); } const reason = runtime.reason === "configured_disabled" || runtime.reason === "contract_unverified" || runtime.reason === "contract_blocked" ? runtime.reason : "gateway_balance_insufficient"; this.updateRuntime(modelId, false, reason, now); } this.database.prepare(` INSERT INTO gateway_balance_states ( gateway_account_ref, balance_status, impact_scope, affected_model_ids_json, runtime_unavailable_model_ids_json, detected_at, last_confirmed_at, recovery_status, source_category ) VALUES (?, 'insufficient', ?, ?, ?, ?, NULL, 'awaiting_confirmation', ?) ON CONFLICT(gateway_account_ref) DO UPDATE SET balance_status = 'insufficient', impact_scope = excluded.impact_scope, affected_model_ids_json = excluded.affected_model_ids_json, runtime_unavailable_model_ids_json = excluded.runtime_unavailable_model_ids_json, detected_at = excluded.detected_at, last_confirmed_at = NULL, recovery_status = 'awaiting_confirmation', source_category = excluded.source_category `).run(input.gatewayAccountRef, input.impactScope, JSON.stringify(newlyAffected.toSorted()), JSON.stringify(unavailable), now, input.sourceCategory); this.database.prepare(` INSERT INTO admin_operation_logs ( log_id, actor_type, actor_ref, operation_type, target_type, target_ref, result, before_summary, after_summary, occurred_at, expires_at ) VALUES (?, 'system', 'generation_worker', 'gateway_balance_detected', 'gateway_balance_state', ?, 'succeeded', NULL, ?, ?, ?) `).run(randomUUID(), input.gatewayAccountRef, JSON.stringify({ impact_scope: input.impactScope, runtime_unavailable_model_ids: unavailable }), now, now + auditRetentionMilliseconds); const result = this.readState(input.gatewayAccountRef)!; this.database.prepare("INSERT INTO gateway_balance_event_receipts (event_id, gateway_account_ref, response_json, created_at) VALUES (?, ?, ?, ?)") .run(input.eventId, input.gatewayAccountRef, JSON.stringify(result), now); return result; }); } readState(gatewayAccountRef: string) { const row = this.database.prepare("SELECT * FROM gateway_balance_states WHERE gateway_account_ref = ?").get(gatewayAccountRef) as BalanceRow | undefined; return row ? this.view(row) : undefined; } listRuntime() { const rows = this.database.prepare(` SELECT r.*, COALESCE(m.gateway_account_ref, 'unknown') AS gateway_account_ref, COALESCE(m.balance_blocked, 0) AS balance_blocked FROM model_runtime_availability r LEFT JOIN gateway_balance_affected_models m ON m.model_id = r.model_id ORDER BY r.model_id `).all() as Array; return rows.map((row) => ({ availableForNewJobs: row.available_for_new_jobs === 1, balanceBlocked: row.balance_blocked === 1, gatewayAccountRef: row.gateway_account_ref, modelId: row.model_id, runtimeReason: row.reason, runtimeVersion: row.runtime_availability_version, updatedAt: new Date(row.checked_at).toISOString(), })); } runRecoveryCheck(gatewayAccountRef: string): GatewayRecoveryCheck { const state = this.readState(gatewayAccountRef); if (!state || state.recoveryStatus !== "awaiting_confirmation") throw new Error("gateway_balance_recovery_unconfirmed"); const now = this.clock(); const check = { check_id: randomUUID(), expires_at: new Date(now + recoveryCheckLifetimeMilliseconds).toISOString(), gateway_account_ref: gatewayAccountRef, status: "passed" as const, checked_at: new Date(now).toISOString(), }; this.immediate(() => { this.database.prepare(` INSERT INTO service_recovery_checks (check_id, service_name, target_ref, status, checked_at, expires_at, details_json) VALUES (?, 'gateway_balance', ?, 'passed', ?, ?, ?) `).run(check.check_id, gatewayAccountRef, now, now + recoveryCheckLifetimeMilliseconds, JSON.stringify({ non_sensitive: true })); }); return check; } confirmRecovery(input: { actorId: string; gatewayAccountRef: string; recoveryCheckId?: string }) { return this.immediate(() => { const state = this.database.prepare("SELECT * FROM gateway_balance_states WHERE gateway_account_ref = ?").get(input.gatewayAccountRef) as BalanceRow | undefined; if (!state || state.recovery_status !== "awaiting_confirmation") throw new Error("gateway_balance_recovery_unconfirmed"); const check = input.recoveryCheckId ? this.database.prepare("SELECT * FROM service_recovery_checks WHERE check_id = ? AND target_ref = ? AND service_name = 'gateway_balance'").get(input.recoveryCheckId, input.gatewayAccountRef) as { expires_at: number; status: string } | undefined : undefined; if (!check || check.status !== "passed" || check.expires_at < this.clock()) throw new Error("gateway_balance_recovery_unconfirmed"); const now = this.clock(); const rows = this.database.prepare("SELECT * FROM gateway_balance_affected_models WHERE gateway_account_ref = ? AND balance_blocked = 1") .all(input.gatewayAccountRef) as MappingRow[]; for (const row of rows) { const configuredReason = this.configuredRuntimeReason(row.model_id); const reason = configuredReason === "available" ? (row.previous_runtime_reason ?? "available") : configuredReason; this.updateRuntime(row.model_id, reason === "available", reason, now); this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 0, previous_runtime_reason = NULL WHERE gateway_account_ref = ? AND model_id = ?") .run(input.gatewayAccountRef, row.model_id); } this.database.prepare(` UPDATE gateway_balance_states SET balance_status = 'available', recovery_status = 'confirmed', last_confirmed_at = ? WHERE gateway_account_ref = ? `).run(now, input.gatewayAccountRef); this.database.prepare(` INSERT INTO admin_operation_logs ( log_id, actor_type, actor_ref, operation_type, target_type, target_ref, result, before_summary, after_summary, occurred_at, expires_at ) VALUES (?, 'super_admin', ?, 'gateway_balance_recovery', 'gateway_balance_state', ?, 'succeeded', ?, ?, ?, ?) `).run(randomUUID(), input.actorId, input.gatewayAccountRef, JSON.stringify({ recovery_status: "awaiting_confirmation" }), JSON.stringify({ recovery_status: "confirmed", restored_model_count: rows.length }), now, now + auditRetentionMilliseconds); return this.readState(input.gatewayAccountRef)!; }); } restoreWithoutConfirmedRecovery(gatewayAccountRef: string, _actorId: string) { const state = this.readState(gatewayAccountRef); if (!state || state.recoveryStatus !== "confirmed") throw new Error("gateway_balance_recovery_unconfirmed"); throw new Error("gateway_balance_recovery_owned_by_wp3"); } private ensureModelMapping(gatewayAccountRef: string, modelId: string) { this.database.prepare(` INSERT INTO model_runtime_availability (model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version) VALUES (?, 1, 'available', ?, 1) ON CONFLICT(model_id) DO NOTHING `).run(modelId, this.clock()); this.database.prepare(` INSERT INTO gateway_balance_affected_models (gateway_account_ref, model_id, balance_blocked, previous_runtime_reason) VALUES (?, ?, 0, NULL) ON CONFLICT(gateway_account_ref, model_id) DO NOTHING `).run(gatewayAccountRef, modelId); } private updateRuntime(modelId: string, available: boolean, reason: RuntimeReason, now: number) { this.database.prepare(` UPDATE model_runtime_availability SET available_for_new_jobs = ?, reason = ?, checked_at = ?, runtime_availability_version = runtime_availability_version + 1 WHERE model_id = ? `).run(available ? 1 : 0, reason, now, modelId); } private configuredRuntimeReason(modelId: string): RuntimeReason { const row = this.database.prepare(` SELECT m.enabled, v.contract_validation_status FROM model_config_current c JOIN model_config_set_members m ON m.config_set_id = c.config_set_id AND c.singleton = 1 JOIN model_config_versions v ON v.model_id = m.model_id AND v.config_version = m.config_version WHERE m.model_id = ? `).get(modelId) as { contract_validation_status: "blocked" | "unverified" | "verified"; enabled: number } | undefined; if (!row) return "available"; if (row.enabled !== 1) return "configured_disabled"; if (row.contract_validation_status === "blocked") return "contract_blocked"; if (row.contract_validation_status !== "verified") return "contract_unverified"; return "available"; } private view(row: BalanceRow): GatewayBalanceView { return { affectedModelIds: JSON.parse(row.affected_model_ids_json) as string[], balanceStatus: row.balance_status, detectedAt: new Date(row.detected_at).toISOString(), gatewayAccountRef: row.gateway_account_ref, impactScope: row.impact_scope, lastConfirmedAt: row.last_confirmed_at === null ? null : new Date(row.last_confirmed_at).toISOString(), recoveryStatus: row.recovery_status, runtimeUnavailableModelIds: JSON.parse(row.runtime_unavailable_model_ids_json) as string[], sourceCategory: row.source_category, }; } private immediate(action: () => T) { if (this.database.inTransaction) return action(); this.database.exec("BEGIN IMMEDIATE"); try { const result = action(); this.database.exec("COMMIT"); return result; } catch (error) { if (this.database.inTransaction) this.database.exec("ROLLBACK"); throw error; } } private migrate() { const existingRuntimeColumns = new Set((this.database.prepare("PRAGMA table_info(model_runtime_availability)").all() as Array<{ name: string }>).map((column) => column.name)); if (existingRuntimeColumns.has("gateway_account_ref")) { this.database.exec(` CREATE TABLE model_runtime_availability_v2 ( model_id TEXT PRIMARY KEY, available_for_new_jobs INTEGER NOT NULL CHECK (available_for_new_jobs IN (0, 1)), reason TEXT NOT NULL CHECK (reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')), checked_at INTEGER NOT NULL, runtime_availability_version INTEGER NOT NULL CHECK (runtime_availability_version >= 0) ); INSERT INTO model_runtime_availability_v2 (model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version) SELECT model_id, available_for_new_jobs, runtime_reason, updated_at, runtime_version FROM model_runtime_availability; DROP TABLE model_runtime_availability; ALTER TABLE model_runtime_availability_v2 RENAME TO model_runtime_availability; `); } else { this.database.exec(` CREATE TABLE IF NOT EXISTS model_runtime_availability ( model_id TEXT PRIMARY KEY, available_for_new_jobs INTEGER NOT NULL CHECK (available_for_new_jobs IN (0, 1)), reason TEXT NOT NULL CHECK (reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')), checked_at INTEGER NOT NULL, runtime_availability_version INTEGER NOT NULL CHECK (runtime_availability_version >= 0) ); `); } this.database.exec(` CREATE TABLE IF NOT EXISTS gateway_balance_affected_models ( gateway_account_ref TEXT NOT NULL, model_id TEXT NOT NULL, balance_blocked INTEGER NOT NULL CHECK (balance_blocked IN (0, 1)), previous_runtime_reason TEXT CHECK (previous_runtime_reason IS NULL OR previous_runtime_reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')), PRIMARY KEY (gateway_account_ref, model_id) ); CREATE TABLE IF NOT EXISTS gateway_balance_states ( gateway_account_ref TEXT PRIMARY KEY, balance_status TEXT NOT NULL CHECK (balance_status IN ('available', 'insufficient', 'unknown')), impact_scope TEXT NOT NULL CHECK (impact_scope IN ('model', 'account', 'unknown')), affected_model_ids_json TEXT NOT NULL CHECK (json_valid(affected_model_ids_json)), runtime_unavailable_model_ids_json TEXT NOT NULL CHECK (json_valid(runtime_unavailable_model_ids_json)), detected_at INTEGER NOT NULL, last_confirmed_at INTEGER, recovery_status TEXT NOT NULL CHECK (recovery_status IN ('not_required', 'awaiting_confirmation', 'confirmed')), source_category TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS gateway_balance_event_receipts ( event_id TEXT PRIMARY KEY, gateway_account_ref TEXT NOT NULL, response_json TEXT NOT NULL CHECK (json_valid(response_json)), created_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS service_recovery_checks ( check_id TEXT PRIMARY KEY, service_name TEXT NOT NULL, target_ref TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ('passed', 'failed')), checked_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, details_json TEXT NOT NULL CHECK (json_valid(details_json)) ); CREATE TABLE IF NOT EXISTS admin_operation_logs ( log_id TEXT PRIMARY KEY, actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')), actor_ref TEXT NOT NULL, operation_type TEXT NOT NULL, target_type TEXT NOT NULL, target_ref TEXT NOT NULL, result TEXT NOT NULL CHECK (result IN ('succeeded', 'failed')), before_summary TEXT, after_summary TEXT, occurred_at INTEGER NOT NULL, expires_at INTEGER NOT NULL ); CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_update BEFORE UPDATE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_delete BEFORE DELETE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; `); } }