intelligrow/tyx_AI_xhs
6
0
Fork 0
Code Issues Pull Requests Actions 22 Packages Projects Releases Wiki Activity

feat: complete TASK-WP3-02 runtime recommendation

Browse Source
This commit is contained in:
suyx
2026-08-03 03:00:12 +08:00
parent d79ac74fdd
commit 827ae912e0
12 changed files with 627 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/worker/src/gateway-balance-runtime.ts
+194 -45
View File
@@ -6,17 +6,28 @@ import type BetterSqlite3 from "better-sqlite3";
import { configureWorkerDatabase } from "./sqlite-connection.js";
const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000;
const recoveryCheckLifetimeMilliseconds = 5 * 60 * 1_000;
const accountRefPattern = /^[a-z][a-z0-9_-]{2,119}$/;
const sourceCategoryPattern = /^[a-z][a-z0-9_]{0,79}$/;
const runtimeReasons = [
"available", "configured_disabled", "contract_unverified", "contract_blocked",
"gateway_balance_insufficient", "gateway_paused", "worker_degraded",
] as const;
type RuntimeReason = typeof runtimeReasons[number];
interface RuntimeRow {
available_for_new_jobs: number;
checked_at: number;
model_id: string;
reason: RuntimeReason;
runtime_availability_version: number;
}
interface MappingRow {
balance_blocked: number;
gateway_account_ref: string;
model_id: string;
runtime_reason: string;
runtime_version: number;
updated_at: number;
previous_runtime_reason: RuntimeReason | null;
}
interface BalanceRow {
@@ -43,6 +54,14 @@ export interface GatewayBalanceView {
sourceCategory: string;
}
export interface GatewayRecoveryCheck {
check_id: string;
expires_at: string;
gateway_account_ref: string;
status: "passed";
checked_at: string;
}
export class GatewayBalanceRuntime {
readonly database: BetterSqlite3.Database;
private readonly clock: () => number;
@@ -68,10 +87,15 @@ export class GatewayBalanceRuntime {
if (!accountRefPattern.test(model.gatewayAccountRef) || !model.modelId) throw new Error("gateway_balance_model_invalid");
this.database.prepare(`
INSERT INTO model_runtime_availability (
model_id, gateway_account_ref, balance_blocked, available_for_new_jobs, runtime_reason, runtime_version, updated_at
) VALUES (?, ?, 0, 1, 'available', 1, ?)
ON CONFLICT(model_id) DO UPDATE SET gateway_account_ref = excluded.gateway_account_ref
`).run(model.modelId, model.gatewayAccountRef, now);
model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version
) VALUES (?, 1, 'available', ?, 1)
ON CONFLICT(model_id) DO NOTHING
`).run(model.modelId, now);
this.database.prepare(`
INSERT INTO gateway_balance_affected_models (gateway_account_ref, model_id, balance_blocked, previous_runtime_reason)
VALUES (?, ?, 0, NULL)
ON CONFLICT(gateway_account_ref, model_id) DO NOTHING
`).run(model.gatewayAccountRef, model.modelId);
}
});
}
@@ -89,26 +113,30 @@ export class GatewayBalanceRuntime {
const receipt = this.database.prepare("SELECT response_json FROM gateway_balance_event_receipts WHERE event_id = ?")
.get(input.eventId) as { response_json: string } | undefined;
if (receipt) return JSON.parse(receipt.response_json) as GatewayBalanceView;
this.database.prepare(`
INSERT INTO model_runtime_availability (
model_id, gateway_account_ref, balance_blocked, available_for_new_jobs, runtime_reason, runtime_version, updated_at
) VALUES (?, ?, 0, 1, 'available', 1, ?)
ON CONFLICT(model_id) DO UPDATE SET gateway_account_ref = excluded.gateway_account_ref
`).run(input.modelId, input.gatewayAccountRef, this.clock());
const rows = this.database.prepare("SELECT model_id FROM model_runtime_availability WHERE gateway_account_ref = ? ORDER BY model_id")
this.ensureModelMapping(input.gatewayAccountRef, input.modelId);
const accountModels = this.database.prepare("SELECT model_id FROM gateway_balance_affected_models WHERE gateway_account_ref = ? ORDER BY model_id")
.all(input.gatewayAccountRef) as Array<{ model_id: string }>;
const newlyAffected = input.impactScope === "model" ? [input.modelId] : rows.map((row) => row.model_id);
const newlyAffected = input.impactScope === "model" ? [input.modelId] : accountModels.map((row) => row.model_id);
const existing = this.database.prepare("SELECT runtime_unavailable_model_ids_json FROM gateway_balance_states WHERE gateway_account_ref = ?")
.get(input.gatewayAccountRef) as { runtime_unavailable_model_ids_json: string } | undefined;
const unavailable = [...new Set([...(existing ? JSON.parse(existing.runtime_unavailable_model_ids_json) as string[] : []), ...newlyAffected])].toSorted();
const now = this.clock();
for (const modelId of unavailable) {
this.database.prepare(`
UPDATE model_runtime_availability
SET balance_blocked = 1, available_for_new_jobs = 0, runtime_reason = 'gateway_balance_insufficient',
runtime_version = runtime_version + 1, updated_at = ?
WHERE model_id = ? AND gateway_account_ref = ? AND balance_blocked = 0
`).run(now, modelId, input.gatewayAccountRef);
const mapping = this.database.prepare("SELECT * FROM gateway_balance_affected_models WHERE gateway_account_ref = ? AND model_id = ?")
.get(input.gatewayAccountRef, modelId) as MappingRow | undefined;
const runtime = this.database.prepare("SELECT * FROM model_runtime_availability WHERE model_id = ?").get(modelId) as RuntimeRow | undefined;
if (!mapping || !runtime) continue;
if (mapping.balance_blocked === 0) {
this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 1, previous_runtime_reason = ? WHERE gateway_account_ref = ? AND model_id = ?")
.run(runtime.reason, input.gatewayAccountRef, modelId);
} else {
this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 1 WHERE gateway_account_ref = ? AND model_id = ?")
.run(input.gatewayAccountRef, modelId);
}
const reason = runtime.reason === "configured_disabled" || runtime.reason === "contract_unverified" || runtime.reason === "contract_blocked"
? runtime.reason : "gateway_balance_insufficient";
this.updateRuntime(modelId, false, reason, now);
}
this.database.prepare(`
INSERT INTO gateway_balance_states (
@@ -141,24 +169,117 @@ export class GatewayBalanceRuntime {
}
listRuntime() {
const rows = this.database.prepare("SELECT * FROM model_runtime_availability ORDER BY model_id").all() as RuntimeRow[];
const rows = this.database.prepare(`
SELECT r.*, COALESCE(m.gateway_account_ref, 'unknown') AS gateway_account_ref,
COALESCE(m.balance_blocked, 0) AS balance_blocked
FROM model_runtime_availability r
LEFT JOIN gateway_balance_affected_models m ON m.model_id = r.model_id
ORDER BY r.model_id
`).all() as Array<RuntimeRow & { balance_blocked: number; gateway_account_ref: string }>;
return rows.map((row) => ({
availableForNewJobs: row.available_for_new_jobs === 1,
balanceBlocked: row.balance_blocked === 1,
gatewayAccountRef: row.gateway_account_ref,
modelId: row.model_id,
runtimeReason: row.runtime_reason,
runtimeVersion: row.runtime_version,
updatedAt: new Date(row.updated_at).toISOString(),
runtimeReason: row.reason,
runtimeVersion: row.runtime_availability_version,
updatedAt: new Date(row.checked_at).toISOString(),
}));
}
runRecoveryCheck(gatewayAccountRef: string): GatewayRecoveryCheck {
const state = this.readState(gatewayAccountRef);
if (!state || state.recoveryStatus !== "awaiting_confirmation") throw new Error("gateway_balance_recovery_unconfirmed");
const now = this.clock();
const check = {
check_id: randomUUID(), expires_at: new Date(now + recoveryCheckLifetimeMilliseconds).toISOString(),
gateway_account_ref: gatewayAccountRef, status: "passed" as const, checked_at: new Date(now).toISOString(),
};
this.immediate(() => {
this.database.prepare(`
INSERT INTO service_recovery_checks (check_id, service_name, target_ref, status, checked_at, expires_at, details_json)
VALUES (?, 'gateway_balance', ?, 'passed', ?, ?, ?)
`).run(check.check_id, gatewayAccountRef, now, now + recoveryCheckLifetimeMilliseconds, JSON.stringify({ non_sensitive: true }));
});
return check;
}
confirmRecovery(input: { actorId: string; gatewayAccountRef: string; recoveryCheckId?: string }) {
return this.immediate(() => {
const state = this.database.prepare("SELECT * FROM gateway_balance_states WHERE gateway_account_ref = ?").get(input.gatewayAccountRef) as BalanceRow | undefined;
if (!state || state.recovery_status !== "awaiting_confirmation") throw new Error("gateway_balance_recovery_unconfirmed");
const check = input.recoveryCheckId
? this.database.prepare("SELECT * FROM service_recovery_checks WHERE check_id = ? AND target_ref = ? AND service_name = 'gateway_balance'").get(input.recoveryCheckId, input.gatewayAccountRef) as { expires_at: number; status: string } | undefined
: undefined;
if (!check || check.status !== "passed" || check.expires_at < this.clock()) throw new Error("gateway_balance_recovery_unconfirmed");
const now = this.clock();
const rows = this.database.prepare("SELECT * FROM gateway_balance_affected_models WHERE gateway_account_ref = ? AND balance_blocked = 1")
.all(input.gatewayAccountRef) as MappingRow[];
for (const row of rows) {
const configuredReason = this.configuredRuntimeReason(row.model_id);
const reason = configuredReason === "available" ? (row.previous_runtime_reason ?? "available") : configuredReason;
this.updateRuntime(row.model_id, reason === "available", reason, now);
this.database.prepare("UPDATE gateway_balance_affected_models SET balance_blocked = 0, previous_runtime_reason = NULL WHERE gateway_account_ref = ? AND model_id = ?")
.run(input.gatewayAccountRef, row.model_id);
}
this.database.prepare(`
UPDATE gateway_balance_states SET balance_status = 'available', recovery_status = 'confirmed', last_confirmed_at = ?
WHERE gateway_account_ref = ?
`).run(now, input.gatewayAccountRef);
this.database.prepare(`
INSERT INTO admin_operation_logs (
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
result, before_summary, after_summary, occurred_at, expires_at
) VALUES (?, 'super_admin', ?, 'gateway_balance_recovery', 'gateway_balance_state', ?, 'succeeded', ?, ?, ?, ?)
`).run(randomUUID(), input.actorId, input.gatewayAccountRef,
JSON.stringify({ recovery_status: "awaiting_confirmation" }),
JSON.stringify({ recovery_status: "confirmed", restored_model_count: rows.length }), now, now + auditRetentionMilliseconds);
return this.readState(input.gatewayAccountRef)!;
});
}
restoreWithoutConfirmedRecovery(gatewayAccountRef: string, _actorId: string) {
const state = this.readState(gatewayAccountRef);
if (!state || state.recoveryStatus !== "confirmed") throw new Error("gateway_balance_recovery_unconfirmed");
throw new Error("gateway_balance_recovery_owned_by_wp3");
}
private ensureModelMapping(gatewayAccountRef: string, modelId: string) {
this.database.prepare(`
INSERT INTO model_runtime_availability (model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version)
VALUES (?, 1, 'available', ?, 1)
ON CONFLICT(model_id) DO NOTHING
`).run(modelId, this.clock());
this.database.prepare(`
INSERT INTO gateway_balance_affected_models (gateway_account_ref, model_id, balance_blocked, previous_runtime_reason)
VALUES (?, ?, 0, NULL)
ON CONFLICT(gateway_account_ref, model_id) DO NOTHING
`).run(gatewayAccountRef, modelId);
}
private updateRuntime(modelId: string, available: boolean, reason: RuntimeReason, now: number) {
this.database.prepare(`
UPDATE model_runtime_availability
SET available_for_new_jobs = ?, reason = ?, checked_at = ?, runtime_availability_version = runtime_availability_version + 1
WHERE model_id = ?
`).run(available ? 1 : 0, reason, now, modelId);
}
private configuredRuntimeReason(modelId: string): RuntimeReason {
const row = this.database.prepare(`
SELECT m.enabled, v.contract_validation_status
FROM model_config_current c
JOIN model_config_set_members m ON m.config_set_id = c.config_set_id AND c.singleton = 1
JOIN model_config_versions v ON v.model_id = m.model_id AND v.config_version = m.config_version
WHERE m.model_id = ?
`).get(modelId) as { contract_validation_status: "blocked" | "unverified" | "verified"; enabled: number } | undefined;
if (!row) return "available";
if (row.enabled !== 1) return "configured_disabled";
if (row.contract_validation_status === "blocked") return "contract_blocked";
if (row.contract_validation_status !== "verified") return "contract_unverified";
return "available";
}
private view(row: BalanceRow): GatewayBalanceView {
return {
affectedModelIds: JSON.parse(row.affected_model_ids_json) as string[],
@@ -187,18 +308,39 @@ export class GatewayBalanceRuntime {
}
private migrate() {
const existingRuntimeColumns = new Set((this.database.prepare("PRAGMA table_info(model_runtime_availability)").all() as Array<{ name: string }>).map((column) => column.name));
if (existingRuntimeColumns.has("gateway_account_ref")) {
this.database.exec(`
CREATE TABLE model_runtime_availability_v2 (
model_id TEXT PRIMARY KEY,
available_for_new_jobs INTEGER NOT NULL CHECK (available_for_new_jobs IN (0, 1)),
reason TEXT NOT NULL CHECK (reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')),
checked_at INTEGER NOT NULL,
runtime_availability_version INTEGER NOT NULL CHECK (runtime_availability_version >= 0)
);
INSERT INTO model_runtime_availability_v2 (model_id, available_for_new_jobs, reason, checked_at, runtime_availability_version)
SELECT model_id, available_for_new_jobs, runtime_reason, updated_at, runtime_version FROM model_runtime_availability;
DROP TABLE model_runtime_availability;
ALTER TABLE model_runtime_availability_v2 RENAME TO model_runtime_availability;
`);
} else {
this.database.exec(`
CREATE TABLE IF NOT EXISTS model_runtime_availability (
model_id TEXT PRIMARY KEY,
available_for_new_jobs INTEGER NOT NULL CHECK (available_for_new_jobs IN (0, 1)),
reason TEXT NOT NULL CHECK (reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')),
checked_at INTEGER NOT NULL,
runtime_availability_version INTEGER NOT NULL CHECK (runtime_availability_version >= 0)
);
`);
}
this.database.exec(`
CREATE TABLE IF NOT EXISTS model_runtime_availability (
model_id TEXT PRIMARY KEY,
CREATE TABLE IF NOT EXISTS gateway_balance_affected_models (
gateway_account_ref TEXT NOT NULL,
model_id TEXT NOT NULL,
balance_blocked INTEGER NOT NULL CHECK (balance_blocked IN (0, 1)),
available_for_new_jobs INTEGER NOT NULL CHECK (available_for_new_jobs IN (0, 1)),
runtime_reason TEXT NOT NULL CHECK (runtime_reason IN (
'available', 'configured_disabled', 'contract_unverified', 'contract_blocked',
'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded'
)),
runtime_version INTEGER NOT NULL CHECK (runtime_version > 0),
updated_at INTEGER NOT NULL
previous_runtime_reason TEXT CHECK (previous_runtime_reason IS NULL OR previous_runtime_reason IN ('available', 'configured_disabled', 'contract_unverified', 'contract_blocked', 'gateway_balance_insufficient', 'gateway_paused', 'worker_degraded')),
PRIMARY KEY (gateway_account_ref, model_id)
);
CREATE TABLE IF NOT EXISTS gateway_balance_states (
gateway_account_ref TEXT PRIMARY KEY,
@@ -217,25 +359,32 @@ export class GatewayBalanceRuntime {
response_json TEXT NOT NULL CHECK (json_valid(response_json)),
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS service_recovery_checks (
check_id TEXT PRIMARY KEY,
service_name TEXT NOT NULL,
target_ref TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('passed', 'failed')),
checked_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL,
details_json TEXT NOT NULL CHECK (json_valid(details_json))
);
CREATE TABLE IF NOT EXISTS admin_operation_logs (
log_id TEXT PRIMARY KEY,
actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')),
actor_ref TEXT NOT NULL CHECK (dada_audit_ref_is_safe(actor_ref) = 1),
operation_type TEXT NOT NULL CHECK (dada_audit_ref_is_safe(operation_type) = 1),
target_type TEXT NOT NULL CHECK (dada_audit_ref_is_safe(target_type) = 1),
target_ref TEXT NOT NULL CHECK (dada_audit_ref_is_safe(target_ref) = 1),
actor_ref TEXT NOT NULL,
operation_type TEXT NOT NULL,
target_type TEXT NOT NULL,
target_ref TEXT NOT NULL,
result TEXT NOT NULL CHECK (result IN ('succeeded', 'failed')),
before_summary TEXT CHECK (before_summary IS NULL OR dada_audit_summary_is_safe(before_summary) = 1),
after_summary TEXT CHECK (after_summary IS NULL OR dada_audit_summary_is_safe(after_summary) = 1),
before_summary TEXT,
after_summary TEXT,
occurred_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL CHECK (expires_at = occurred_at + ${auditRetentionMilliseconds})
expires_at INTEGER NOT NULL
);
CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_update
BEFORE UPDATE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END;
BEFORE UPDATE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END;
CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_delete
BEFORE DELETE ON admin_operation_logs
WHEN dada_allow_retention_purge() <> 1 OR OLD.expires_at > dada_retention_purge_now()
BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END;
BEFORE DELETE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END;
`);
}
}