391 lines
20 KiB
TypeScript
391 lines
20 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
|
|
import Database from "better-sqlite3";
|
|
import type BetterSqlite3 from "better-sqlite3";
|
|
|
|
import { configureWorkerDatabase } from "./sqlite-connection.js";
|
|
|
|
const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000;
|
|
const recoveryCheckLifetimeMilliseconds = 5 * 60 * 1_000;
|
|
const accountRefPattern = /^[a-z][a-z0-9_-]{2,119}$/;
|
|
const sourceCategoryPattern = /^[a-z][a-z0-9_]{0,79}$/;
|
|
const runtimeReasons = [
|
|
"available", "configured_disabled", "contract_unverified", "contract_blocked",
|
|
"gateway_balance_insufficient", "gateway_paused", "worker_degraded",
|
|
] as const;
|
|
type RuntimeReason = typeof runtimeReasons[number];
|
|
|
|
interface RuntimeRow {
|
|
available_for_new_jobs: number;
|
|
checked_at: number;
|
|
model_id: string;
|
|
reason: RuntimeReason;
|
|
runtime_availability_version: number;
|
|
}
|
|
|
|
interface MappingRow {
|
|
balance_blocked: number;
|
|
gateway_account_ref: string;
|
|
model_id: string;
|
|
previous_runtime_reason: RuntimeReason | null;
|
|
}
|
|
|
|
interface BalanceRow {
|
|
affected_model_ids_json: string;
|
|
balance_status: "available" | "insufficient" | "unknown";
|
|
detected_at: number;
|
|
gateway_account_ref: string;
|
|
impact_scope: "model" | "account" | "unknown";
|
|
last_confirmed_at: number | null;
|
|
recovery_status: "not_required" | "awaiting_confirmation" | "confirmed";
|
|
runtime_unavailable_model_ids_json: string;
|
|
source_category: string;
|
|
}
|
|
|
|
export interface GatewayBalanceView {
|
|
affectedModelIds: string[];
|
|
balanceStatus: "available" | "insufficient" | "unknown";
|
|
detectedAt: string;
|
|
gatewayAccountRef: string;
|
|
impactScope: "model" | "account" | "unknown";
|
|
lastConfirmedAt: string | null;
|
|
recoveryStatus: "not_required" | "awaiting_confirmation" | "confirmed";
|
|
runtimeUnavailableModelIds: string[];
|
|
sourceCategory: string;
|
|
}
|
|
|
|
export interface GatewayRecoveryCheck {
|
|
check_id: string;
|
|
expires_at: string;
|
|
gateway_account_ref: string;
|
|
status: "passed";
|
|
checked_at: string;
|
|
}
|
|
|
|
export class GatewayBalanceRuntime {
|
|
readonly database: BetterSqlite3.Database;
|
|
private readonly clock: () => number;
|
|
private readonly ownsDatabase: boolean;
|
|
|
|
constructor(input: { clock?: () => number; database?: BetterSqlite3.Database; databasePath?: string }) {
|
|
if (!input.database && !input.databasePath) throw new Error("gateway_balance_database_required");
|
|
this.database = input.database ?? new Database(input.databasePath!);
|
|
this.ownsDatabase = !input.database;
|
|
this.clock = input.clock ?? Date.now;
|
|
if (this.ownsDatabase) configureWorkerDatabase(this.database);
|
|
this.migrate();
|
|
}
|
|
|
|
close() {
|
|
if (this.ownsDatabase) this.database.close();
|
|
}
|
|
|
|
seedModels(models: Array<{ gatewayAccountRef: string; modelId: string }>) {
|
|
const now = this.clock();
|
|
this.immediate(() => {
|
|
for (const model of models) {
|
|
if (!accountRefPattern.test(model.gatewayAccountRef) || !model.modelId) throw new Error("gateway_balance_model_invalid");
|
|
this.database.prepare(`
|
|
INSERT INTO model_runtime_availability (
|
|
model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version
|
|
) VALUES (?, 1, 'available', ?, 1)
|
|
ON CONFLICT(model_id) DO NOTHING
|
|
`).run(model.modelId, now);
|
|
this.database.prepare(`
|
|
INSERT INTO gateway_balance_affected_models (gateway_account_ref, model_id, balance_blocked, previous_runtime_reason)
|
|
VALUES (?, ?, 0, NULL)
|
|
ON CONFLICT(gateway_account_ref, model_id) DO NOTHING
|
|
`).run(model.gatewayAccountRef, model.modelId);
|
|
}
|
|
});
|
|
}
|
|
|
|
recordInsufficient(input: {
|
|
eventId: string;
|
|
gatewayAccountRef: string;
|
|
impactScope: "model" | "account" | "unknown";
|
|
modelId: string;
|
|
sourceCategory: string;
|
|
}) {
|
|
if (!input.eventId || !accountRefPattern.test(input.gatewayAccountRef) || !input.modelId
|
|
|| !sourceCategoryPattern.test(input.sourceCategory)) throw new Error("gateway_balance_signal_invalid");
|
|
return this.immediate(() => {
|
|
const receipt = this.database.prepare("SELECT response_json FROM gateway_balance_event_receipts WHERE event_id = ?")
|
|
.get(input.eventId) as { response_json: string } | undefined;
|
|
if (receipt) return JSON.parse(receipt.response_json) as GatewayBalanceView;
|
|
|
|
this.ensureModelMapping(input.gatewayAccountRef, input.modelId);
|
|
const accountModels = this.database.prepare("SELECT model_id FROM gateway_balance_affected_models WHERE gateway_account_ref = ? ORDER BY model_id")
|
|
.all(input.gatewayAccountRef) as Array<{ model_id: string }>;
|
|
const newlyAffected = input.impactScope === "model" ? [input.modelId] : accountModels.map((row) => row.model_id);
|
|
const existing = this.database.prepare("SELECT runtime_unavailable_model_ids_json FROM gateway_balance_states WHERE gateway_account_ref = ?")
|
|
.get(input.gatewayAccountRef) as { runtime_unavailable_model_ids_json: string } | undefined;
|
|
const unavailable = [...new Set([...(existing ? JSON.parse(existing.runtime_unavailable_model_ids_json) as string[] : []), ...newlyAffected])].toSorted();
|
|
const now = this.clock();
|
|
for (const modelId of unavailable) {
|
|
const mapping = this.database.prepare("SELECT * FROM gateway_balance_affected_models WHERE gateway_account_ref = ? AND model_id = ?")
|
|
.get(input.gatewayAccountRef, modelId) as MappingRow | undefined;
|
|
const runtime = this.database.prepare("SELECT * FROM model_runtime_availability WHERE model_id = ?").get(modelId) as RuntimeRow | undefined;
|
|
if (!mapping || !runtime) continue;
|
|
if (mapping.balance_blocked === 0) {
|
|
this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 1, previous_runtime_reason = ? WHERE gateway_account_ref = ? AND model_id = ?")
|
|
.run(runtime.reason, input.gatewayAccountRef, modelId);
|
|
} else {
|
|
this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 1 WHERE gateway_account_ref = ? AND model_id = ?")
|
|
.run(input.gatewayAccountRef, modelId);
|
|
}
|
|
const reason = runtime.reason === "configured_disabled" || runtime.reason === "contract_unverified" || runtime.reason === "contract_blocked"
|
|
? runtime.reason : "gateway_balance_insufficient";
|
|
this.updateRuntime(modelId, false, reason, now);
|
|
}
|
|
this.database.prepare(`
|
|
INSERT INTO gateway_balance_states (
|
|
gateway_account_ref, balance_status, impact_scope, affected_model_ids_json,
|
|
runtime_unavailable_model_ids_json, detected_at, last_confirmed_at, recovery_status, source_category
|
|
) VALUES (?, 'insufficient', ?, ?, ?, ?, NULL, 'awaiting_confirmation', ?)
|
|
ON CONFLICT(gateway_account_ref) DO UPDATE SET
|
|
balance_status = 'insufficient', impact_scope = excluded.impact_scope,
|
|
affected_model_ids_json = excluded.affected_model_ids_json,
|
|
runtime_unavailable_model_ids_json = excluded.runtime_unavailable_model_ids_json,
|
|
detected_at = excluded.detected_at, last_confirmed_at = NULL,
|
|
recovery_status = 'awaiting_confirmation', source_category = excluded.source_category
|
|
`).run(input.gatewayAccountRef, input.impactScope, JSON.stringify(newlyAffected.toSorted()), JSON.stringify(unavailable), now, input.sourceCategory);
|
|
this.database.prepare(`
|
|
INSERT INTO admin_operation_logs (
|
|
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
|
|
result, before_summary, after_summary, occurred_at, expires_at
|
|
) VALUES (?, 'system', 'generation_worker', 'gateway_balance_detected', 'gateway_balance_state', ?, 'succeeded', NULL, ?, ?, ?)
|
|
`).run(randomUUID(), input.gatewayAccountRef, JSON.stringify({ impact_scope: input.impactScope, runtime_unavailable_model_ids: unavailable }), now, now + auditRetentionMilliseconds);
|
|
const result = this.readState(input.gatewayAccountRef)!;
|
|
this.database.prepare("INSERT INTO gateway_balance_event_receipts (event_id, gateway_account_ref, response_json, created_at) VALUES (?, ?, ?, ?)")
|
|
.run(input.eventId, input.gatewayAccountRef, JSON.stringify(result), now);
|
|
return result;
|
|
});
|
|
}
|
|
|
|
readState(gatewayAccountRef: string) {
|
|
const row = this.database.prepare("SELECT * FROM gateway_balance_states WHERE gateway_account_ref = ?").get(gatewayAccountRef) as BalanceRow | undefined;
|
|
return row ? this.view(row) : undefined;
|
|
}
|
|
|
|
listRuntime() {
|
|
const rows = this.database.prepare(`
|
|
SELECT r.*, COALESCE(m.gateway_account_ref, 'unknown') AS gateway_account_ref,
|
|
COALESCE(m.balance_blocked, 0) AS balance_blocked
|
|
FROM model_runtime_availability r
|
|
LEFT JOIN gateway_balance_affected_models m ON m.model_id = r.model_id
|
|
ORDER BY r.model_id
|
|
`).all() as Array<RuntimeRow & { balance_blocked: number; gateway_account_ref: string }>;
|
|
return rows.map((row) => ({
|
|
availableForNewJobs: row.available_for_new_jobs === 1,
|
|
balanceBlocked: row.balance_blocked === 1,
|
|
gatewayAccountRef: row.gateway_account_ref,
|
|
modelId: row.model_id,
|
|
runtimeReason: row.reason,
|
|
runtimeVersion: row.runtime_availability_version,
|
|
updatedAt: new Date(row.checked_at).toISOString(),
|
|
}));
|
|
}
|
|
|
|
runRecoveryCheck(gatewayAccountRef: string): GatewayRecoveryCheck {
|
|
const state = this.readState(gatewayAccountRef);
|
|
if (!state || state.recoveryStatus !== "awaiting_confirmation") throw new Error("gateway_balance_recovery_unconfirmed");
|
|
const now = this.clock();
|
|
const check = {
|
|
check_id: randomUUID(), expires_at: new Date(now + recoveryCheckLifetimeMilliseconds).toISOString(),
|
|
gateway_account_ref: gatewayAccountRef, status: "passed" as const, checked_at: new Date(now).toISOString(),
|
|
};
|
|
this.immediate(() => {
|
|
this.database.prepare(`
|
|
INSERT INTO service_recovery_checks (check_id, service_name, target_ref, status, checked_at, expires_at, details_json)
|
|
VALUES (?, 'gateway_balance', ?, 'passed', ?, ?, ?)
|
|
`).run(check.check_id, gatewayAccountRef, now, now + recoveryCheckLifetimeMilliseconds, JSON.stringify({ non_sensitive: true }));
|
|
});
|
|
return check;
|
|
}
|
|
|
|
confirmRecovery(input: { actorId: string; gatewayAccountRef: string; recoveryCheckId?: string }) {
|
|
return this.immediate(() => {
|
|
const state = this.database.prepare("SELECT * FROM gateway_balance_states WHERE gateway_account_ref = ?").get(input.gatewayAccountRef) as BalanceRow | undefined;
|
|
if (!state || state.recovery_status !== "awaiting_confirmation") throw new Error("gateway_balance_recovery_unconfirmed");
|
|
const check = input.recoveryCheckId
|
|
? this.database.prepare("SELECT * FROM service_recovery_checks WHERE check_id = ? AND target_ref = ? AND service_name = 'gateway_balance'").get(input.recoveryCheckId, input.gatewayAccountRef) as { expires_at: number; status: string } | undefined
|
|
: undefined;
|
|
if (!check || check.status !== "passed" || check.expires_at < this.clock()) throw new Error("gateway_balance_recovery_unconfirmed");
|
|
const now = this.clock();
|
|
const rows = this.database.prepare("SELECT * FROM gateway_balance_affected_models WHERE gateway_account_ref = ? AND balance_blocked = 1")
|
|
.all(input.gatewayAccountRef) as MappingRow[];
|
|
for (const row of rows) {
|
|
const configuredReason = this.configuredRuntimeReason(row.model_id);
|
|
const reason = configuredReason === "available" ? (row.previous_runtime_reason ?? "available") : configuredReason;
|
|
this.updateRuntime(row.model_id, reason === "available", reason, now);
|
|
this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 0, previous_runtime_reason = NULL WHERE gateway_account_ref = ? AND model_id = ?")
|
|
.run(input.gatewayAccountRef, row.model_id);
|
|
}
|
|
this.database.prepare(`
|
|
UPDATE gateway_balance_states SET balance_status = 'available', recovery_status = 'confirmed', last_confirmed_at = ?
|
|
WHERE gateway_account_ref = ?
|
|
`).run(now, input.gatewayAccountRef);
|
|
this.database.prepare(`
|
|
INSERT INTO admin_operation_logs (
|
|
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
|
|
result, before_summary, after_summary, occurred_at, expires_at
|
|
) VALUES (?, 'super_admin', ?, 'gateway_balance_recovery', 'gateway_balance_state', ?, 'succeeded', ?, ?, ?, ?)
|
|
`).run(randomUUID(), input.actorId, input.gatewayAccountRef,
|
|
JSON.stringify({ recovery_status: "awaiting_confirmation" }),
|
|
JSON.stringify({ recovery_status: "confirmed", restored_model_count: rows.length }), now, now + auditRetentionMilliseconds);
|
|
return this.readState(input.gatewayAccountRef)!;
|
|
});
|
|
}
|
|
|
|
restoreWithoutConfirmedRecovery(gatewayAccountRef: string, _actorId: string) {
|
|
const state = this.readState(gatewayAccountRef);
|
|
if (!state || state.recoveryStatus !== "confirmed") throw new Error("gateway_balance_recovery_unconfirmed");
|
|
throw new Error("gateway_balance_recovery_owned_by_wp3");
|
|
}
|
|
|
|
private ensureModelMapping(gatewayAccountRef: string, modelId: string) {
|
|
this.database.prepare(`
|
|
INSERT INTO model_runtime_availability (model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version)
|
|
VALUES (?, 1, 'available', ?, 1)
|
|
ON CONFLICT(model_id) DO NOTHING
|
|
`).run(modelId, this.clock());
|
|
this.database.prepare(`
|
|
INSERT INTO gateway_balance_affected_models (gateway_account_ref, model_id, balance_blocked, previous_runtime_reason)
|
|
VALUES (?, ?, 0, NULL)
|
|
ON CONFLICT(gateway_account_ref, model_id) DO NOTHING
|
|
`).run(gatewayAccountRef, modelId);
|
|
}
|
|
|
|
private updateRuntime(modelId: string, available: boolean, reason: RuntimeReason, now: number) {
|
|
this.database.prepare(`
|
|
UPDATE model_runtime_availability
|
|
SET available_for_new_jobs = ?, reason = ?, checked_at = ?, runtime_availability_version = runtime_availability_version + 1
|
|
WHERE model_id = ?
|
|
`).run(available ? 1 : 0, reason, now, modelId);
|
|
}
|
|
|
|
private configuredRuntimeReason(modelId: string): RuntimeReason {
|
|
const row = this.database.prepare(`
|
|
SELECT m.enabled, v.contract_validation_status
|
|
FROM model_config_current c
|
|
JOIN model_config_set_members m ON m.config_set_id = c.config_set_id AND c.singleton = 1
|
|
JOIN model_config_versions v ON v.model_id = m.model_id AND v.config_version = m.config_version
|
|
WHERE m.model_id = ?
|
|
`).get(modelId) as { contract_validation_status: "blocked" | "unverified" | "verified"; enabled: number } | undefined;
|
|
if (!row) return "available";
|
|
if (row.enabled !== 1) return "configured_disabled";
|
|
if (row.contract_validation_status === "blocked") return "contract_blocked";
|
|
if (row.contract_validation_status !== "verified") return "contract_unverified";
|
|
return "available";
|
|
}
|
|
|
|
private view(row: BalanceRow): GatewayBalanceView {
|
|
return {
|
|
affectedModelIds: JSON.parse(row.affected_model_ids_json) as string[],
|
|
balanceStatus: row.balance_status,
|
|
detectedAt: new Date(row.detected_at).toISOString(),
|
|
gatewayAccountRef: row.gateway_account_ref,
|
|
impactScope: row.impact_scope,
|
|
lastConfirmedAt: row.last_confirmed_at === null ? null : new Date(row.last_confirmed_at).toISOString(),
|
|
recoveryStatus: row.recovery_status,
|
|
runtimeUnavailableModelIds: JSON.parse(row.runtime_unavailable_model_ids_json) as string[],
|
|
sourceCategory: row.source_category,
|
|
};
|
|
}
|
|
|
|
private immediate<T>(action: () => T) {
|
|
if (this.database.inTransaction) return action();
|
|
this.database.exec("BEGIN IMMEDIATE");
|
|
try {
|
|
const result = action();
|
|
this.database.exec("COMMIT");
|
|
return result;
|
|
} catch (error) {
|
|
if (this.database.inTransaction) this.database.exec("ROLLBACK");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private migrate() {
|
|
const existingRuntimeColumns = new Set((this.database.prepare("PRAGMA table_info(model_runtime_availability)").all() as Array<{ name: string }>).map((column) => column.name));
|
|
if (existingRuntimeColumns.has("gateway_account_ref")) {
|
|
this.database.exec(`
|
|
CREATE TABLE model_runtime_availability_v2 (
|
|
model_id TEXT PRIMARY KEY,
|
|
available_for_new_jobs INTEGER NOT NULL CHECK (available_for_new_jobs IN (0, 1)),
|
|
reason TEXT NOT NULL CHECK (reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')),
|
|
checked_at INTEGER NOT NULL,
|
|
runtime_availability_version INTEGER NOT NULL CHECK (runtime_availability_version >= 0)
|
|
);
|
|
INSERT INTO model_runtime_availability_v2 (model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version)
|
|
SELECT model_id, available_for_new_jobs, runtime_reason, updated_at, runtime_version FROM model_runtime_availability;
|
|
DROP TABLE model_runtime_availability;
|
|
ALTER TABLE model_runtime_availability_v2 RENAME TO model_runtime_availability;
|
|
`);
|
|
} else {
|
|
this.database.exec(`
|
|
CREATE TABLE IF NOT EXISTS model_runtime_availability (
|
|
model_id TEXT PRIMARY KEY,
|
|
available_for_new_jobs INTEGER NOT NULL CHECK (available_for_new_jobs IN (0, 1)),
|
|
reason TEXT NOT NULL CHECK (reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')),
|
|
checked_at INTEGER NOT NULL,
|
|
runtime_availability_version INTEGER NOT NULL CHECK (runtime_availability_version >= 0)
|
|
);
|
|
`);
|
|
}
|
|
this.database.exec(`
|
|
CREATE TABLE IF NOT EXISTS gateway_balance_affected_models (
|
|
gateway_account_ref TEXT NOT NULL,
|
|
model_id TEXT NOT NULL,
|
|
balance_blocked INTEGER NOT NULL CHECK (balance_blocked IN (0, 1)),
|
|
previous_runtime_reason TEXT CHECK (previous_runtime_reason IS NULL OR previous_runtime_reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')),
|
|
PRIMARY KEY (gateway_account_ref, model_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS gateway_balance_states (
|
|
gateway_account_ref TEXT PRIMARY KEY,
|
|
balance_status TEXT NOT NULL CHECK (balance_status IN ('available', 'insufficient', 'unknown')),
|
|
impact_scope TEXT NOT NULL CHECK (impact_scope IN ('model', 'account', 'unknown')),
|
|
affected_model_ids_json TEXT NOT NULL CHECK (json_valid(affected_model_ids_json)),
|
|
runtime_unavailable_model_ids_json TEXT NOT NULL CHECK (json_valid(runtime_unavailable_model_ids_json)),
|
|
detected_at INTEGER NOT NULL,
|
|
last_confirmed_at INTEGER,
|
|
recovery_status TEXT NOT NULL CHECK (recovery_status IN ('not_required', 'awaiting_confirmation', 'confirmed')),
|
|
source_category TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS gateway_balance_event_receipts (
|
|
event_id TEXT PRIMARY KEY,
|
|
gateway_account_ref TEXT NOT NULL,
|
|
response_json TEXT NOT NULL CHECK (json_valid(response_json)),
|
|
created_at INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS service_recovery_checks (
|
|
check_id TEXT PRIMARY KEY,
|
|
service_name TEXT NOT NULL,
|
|
target_ref TEXT NOT NULL,
|
|
status TEXT NOT NULL CHECK (status IN ('passed', 'failed')),
|
|
checked_at INTEGER NOT NULL,
|
|
expires_at INTEGER NOT NULL,
|
|
details_json TEXT NOT NULL CHECK (json_valid(details_json))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS admin_operation_logs (
|
|
log_id TEXT PRIMARY KEY,
|
|
actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')),
|
|
actor_ref TEXT NOT NULL,
|
|
operation_type TEXT NOT NULL,
|
|
target_type TEXT NOT NULL,
|
|
target_ref TEXT NOT NULL,
|
|
result TEXT NOT NULL CHECK (result IN ('succeeded', 'failed')),
|
|
before_summary TEXT,
|
|
after_summary TEXT,
|
|
occurred_at INTEGER NOT NULL,
|
|
expires_at INTEGER NOT NULL
|
|
);
|
|
CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_update
|
|
BEFORE UPDATE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END;
|
|
CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_delete
|
|
BEFORE DELETE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END;
|
|
`);
|
|
}
|
|
}
|