import { randomUUID } from "node:crypto"; import type BetterSqlite3 from "better-sqlite3"; import { serializeAuditSummary } from "./audit-policy.js"; export type ExternalServiceId = "resend_email" | "amap_web_service"; export type ExternalServicePeriodType = "daily" | "monthly"; export type ExternalServiceStatus = "active" | "paused_quota" | "paused_provider" | "disabled"; const retentionMilliseconds = 180 * 24 * 60 * 60 * 1_000; const recoveryCheckLifetimeMilliseconds = 15 * 60 * 1_000; const maximumHardLimits: Record>> = { resend_email: { daily: 80, monthly: 2_400 }, amap_web_service: { monthly: 1_000 }, }; export interface ExternalServiceUsageRow { serviceId: ExternalServiceId; periodType: ExternalServicePeriodType; periodStart: number; hardLimit: number; usedCount: number; status: ExternalServiceStatus; pauseReason: string | null; updatedAt: number; } export class ExternalServiceUsageError extends Error { constructor( readonly code: | "service_paused_quota" | "service_paused_provider" | "service_disabled" | "hard_limit_increase_forbidden" | "hard_limit_invalid" | "health_check_required" | "quota_exhausted" | "service_not_found", message = code, ) { super(message); this.name = "ExternalServiceUsageError"; } } interface ExternalServiceUsageOptions { clock?: () => number; database: BetterSqlite3.Database; } interface RawUsageRow { service_id: ExternalServiceId; period_type: ExternalServicePeriodType; period_start: number; hard_limit: number; used_count: number; service_status: ExternalServiceStatus; pause_reason: string | null; updated_at: number; } function periodStart(periodType: ExternalServicePeriodType, now: number) { const date = new Date(now); if (periodType === "daily") return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()); return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1); } function requiredPeriods(serviceId: ExternalServiceId): ExternalServicePeriodType[] { return serviceId === "resend_email" ? ["daily", "monthly"] : ["monthly"]; } function toPublic(row: RawUsageRow): ExternalServiceUsageRow { return { hardLimit: row.hard_limit, pauseReason: row.pause_reason, periodStart: row.period_start, periodType: row.period_type, serviceId: row.service_id, status: row.service_status, updatedAt: row.updated_at, usedCount: row.used_count, }; } export class ExternalServiceUsage { readonly database: BetterSqlite3.Database; readonly clock: () => number; constructor(options: ExternalServiceUsageOptions) { this.database = options.database; this.clock = options.clock ?? Date.now; this.ensureSchema(); this.runImmediate(() => { this.ensureCurrentRows(this.clock(), "resend_email"); this.ensureCurrentRows(this.clock(), "amap_web_service"); }); } claimResend(now = this.clock()) { return this.runImmediate(() => this.claimWithinTransaction("resend_email", now)); } claimResendWithinTransaction(now = this.clock()) { return this.claimWithinTransaction("resend_email", now); } claimAmap(now = this.clock()) { return this.runImmediate(() => this.claimWithinTransaction("amap_web_service", now)); } claimAmapWithinTransaction(now = this.clock()) { return this.claimWithinTransaction("amap_web_service", now); } markProviderFailure(input: { serviceId: ExternalServiceId; reason: string; now?: number }) { return this.runImmediate(() => this.markProviderFailureWithinTransaction(input)); } markProviderFailureWithinTransaction(input: { serviceId: ExternalServiceId; reason: string; now?: number }) { const now = input.now ?? this.clock(); const rows = this.ensureCurrentRows(now, input.serviceId); const reason = normalizeReason(input.reason); for (const row of rows) { if (row.service_status === "disabled") continue; this.database.prepare(` UPDATE external_service_usage SET service_status = 'paused_provider', pause_reason = ?, updated_at = ? WHERE service_id = ? AND period_type = ? AND period_start = ? `).run(reason, now, row.service_id, row.period_type, row.period_start); } this.recordAudit({ actorRef: "external_service_runtime", actorType: "system", afterSummary: { pause_reason: reason, status: "paused_provider" }, beforeSummary: { status: rows[0]?.service_status ?? "active" }, operationType: "service_provider_pause", result: "succeeded", targetRef: input.serviceId, targetType: "external_service", }, now); return this.read(input.serviceId); } recordHealthCheck(input: { serviceId: ExternalServiceId; available: boolean; reason?: string; now?: number }) { const now = input.now ?? this.clock(); const checkId = randomUUID(); const currentPeriod = periodStart(requiredPeriods(input.serviceId)[0]!, now); const result = this.runImmediate(() => { this.database.prepare(` INSERT INTO service_recovery_checks ( check_id, service_name, target_ref, status, checked_at, expires_at, details_json, service_id, period_start, available, check_reason ) VALUES (?, 'external_service', ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( checkId, input.serviceId, input.available ? "passed" : "failed", now, now + recoveryCheckLifetimeMilliseconds, JSON.stringify({ non_sensitive: true }), input.serviceId, currentPeriod, input.available ? 1 : 0, input.reason ? normalizeReason(input.reason) : null, ); return { checkId, available: input.available, checkedAt: now }; }); return result; } recover(input: { serviceId: ExternalServiceId; actorId: string; checkId: string; now?: number }) { const now = input.now ?? this.clock(); const result = this.runImmediate(() => { const check = this.database.prepare(` SELECT check_id, period_start, available, expires_at FROM service_recovery_checks WHERE check_id = ? AND service_id = ? AND service_name = 'external_service' `).get(input.checkId, input.serviceId) as { available: number; check_id: string; expires_at: number; period_start: number } | undefined; const currentPeriod = periodStart(requiredPeriods(input.serviceId)[0]!, now); if (!check?.available || check.period_start !== currentPeriod || check.expires_at <= now) { this.recordAudit({ actorRef: input.actorId, actorType: "super_admin", afterSummary: { reason: "health_check_required" }, beforeSummary: null, operationType: "service_recovery", result: "failed", targetRef: input.serviceId, targetType: "external_service", }, now); return { error: new ExternalServiceUsageError("health_check_required") }; } const rows = this.ensureCurrentRows(now, input.serviceId); if (rows.some((row) => row.used_count >= row.hard_limit)) { this.recordAudit({ actorRef: input.actorId, actorType: "super_admin", afterSummary: { reason: "quota_exhausted" }, beforeSummary: { status: rows[0]?.service_status ?? "paused_quota" }, operationType: "service_recovery", result: "failed", targetRef: input.serviceId, targetType: "external_service", }, now); return { error: new ExternalServiceUsageError("quota_exhausted") }; } for (const row of rows) { this.database.prepare(` UPDATE external_service_usage SET service_status = 'active', pause_reason = NULL, updated_at = ? WHERE service_id = ? AND period_type = ? AND period_start = ? `).run(now, row.service_id, row.period_type, row.period_start); } this.recordAudit({ actorRef: input.actorId, actorType: "super_admin", afterSummary: { check_id: input.checkId, status: "active" }, beforeSummary: { status: rows[0]?.service_status ?? "paused_provider" }, operationType: "service_recovery", result: "succeeded", targetRef: input.serviceId, targetType: "external_service", }, now); return { status: "active" as const }; }); if ("error" in result && result.error) throw result.error; return result; } setHardLimit(input: { serviceId: ExternalServiceId; periodType: ExternalServicePeriodType; hardLimit: number; actorId: string; now?: number; }) { const now = input.now ?? this.clock(); const result = this.runImmediate(() => { const maximum = maximumHardLimits[input.serviceId][input.periodType]; const current = this.ensureCurrentRows(now, input.serviceId).find((row) => row.period_type === input.periodType); if (current && input.hardLimit > current.hard_limit) { this.recordAudit({ actorRef: input.actorId, actorType: "super_admin", afterSummary: { reason: "hard_limit_increase_forbidden" }, beforeSummary: { hard_limit: current.hard_limit }, operationType: "service_hard_limit_update", result: "failed", targetRef: `${input.serviceId}:${input.periodType}`, targetType: "external_service_limit", }, now); return { error: new ExternalServiceUsageError("hard_limit_increase_forbidden") }; } const valid = maximum !== undefined && Number.isSafeInteger(input.hardLimit) && input.hardLimit >= 1 && input.hardLimit <= maximum; if (!valid || !current) { this.recordAudit({ actorRef: input.actorId, actorType: "super_admin", afterSummary: { reason: "hard_limit_invalid" }, beforeSummary: current ? { hard_limit: current.hard_limit } : null, operationType: "service_hard_limit_update", result: "failed", targetRef: `${input.serviceId}:${input.periodType}`, targetType: "external_service_limit", }, now); return { error: new ExternalServiceUsageError("hard_limit_invalid") }; } this.database.prepare(` UPDATE external_service_usage SET hard_limit = ?, service_status = CASE WHEN used_count >= ? THEN 'paused_quota' ELSE service_status END, pause_reason = CASE WHEN used_count >= ? THEN 'hard_limit_reached' ELSE pause_reason END, updated_at = ? WHERE service_id = ? AND period_type = ? AND period_start = ? `).run(input.hardLimit, input.hardLimit, input.hardLimit, now, current.service_id, current.period_type, current.period_start); this.recordAudit({ actorRef: input.actorId, actorType: "super_admin", afterSummary: { hard_limit: input.hardLimit }, beforeSummary: { hard_limit: current.hard_limit }, operationType: "service_hard_limit_update", result: "succeeded", targetRef: `${input.serviceId}:${input.periodType}`, targetType: "external_service_limit", }, now); return this.read(input.serviceId); }); if ("error" in result && result.error) throw result.error; return result; } read(serviceId?: ExternalServiceId) { const rows = (serviceId ? this.database.prepare("SELECT * FROM external_service_usage WHERE service_id = ? ORDER BY period_type").all(serviceId) : this.database.prepare("SELECT * FROM external_service_usage ORDER BY service_id, period_type").all()) as RawUsageRow[]; return rows.map(toPublic); } readCurrent() { const now = this.clock(); return this.read().filter((row) => row.periodStart === periodStart(row.periodType, now)); } readStatus(serviceId: ExternalServiceId) { const rows = this.read(serviceId).filter((row) => row.periodStart >= periodStart(row.periodType, this.clock())); const status = rows.some((row) => row.status === "disabled") ? "disabled" : rows.some((row) => row.status === "paused_provider") ? "paused_provider" : rows.some((row) => row.status === "paused_quota") ? "paused_quota" : "active"; return { serviceId, status, rows } as const; } private claimWithinTransaction(serviceId: ExternalServiceId, now: number) { const rows = this.ensureCurrentRows(now, serviceId); for (const row of rows) { if (row.service_status === "paused_quota") throw new ExternalServiceUsageError("service_paused_quota"); if (row.service_status === "paused_provider") throw new ExternalServiceUsageError("service_paused_provider"); if (row.service_status === "disabled") throw new ExternalServiceUsageError("service_disabled"); if (row.used_count >= row.hard_limit) { this.database.prepare(` UPDATE external_service_usage SET service_status = 'paused_quota', pause_reason = 'hard_limit_reached', updated_at = ? WHERE service_id = ? AND period_type = ? AND period_start = ? `).run(now, row.service_id, row.period_type, row.period_start); throw new ExternalServiceUsageError("service_paused_quota"); } } const updated = rows.map((row) => { const usedCount = row.used_count + 1; const status: ExternalServiceStatus = usedCount >= row.hard_limit ? "paused_quota" : "active"; this.database.prepare(` UPDATE external_service_usage SET used_count = ?, service_status = ?, pause_reason = CASE WHEN ? = 'active' THEN NULL ELSE 'hard_limit_reached' END, updated_at = ? WHERE service_id = ? AND period_type = ? AND period_start = ? `).run(usedCount, status, status, now, row.service_id, row.period_type, row.period_start); return { periodType: row.period_type, remaining: Math.max(0, row.hard_limit - usedCount), usedCount }; }); return { allowed: true as const, serviceId, allocations: updated, remaining: Math.min(...updated.map((item) => item.remaining)) }; } private ensureCurrentRows(now: number, serviceId: ExternalServiceId) { const periods = requiredPeriods(serviceId); for (const periodType of periods) { const start = periodStart(periodType, now); const current = this.database.prepare(` SELECT * FROM external_service_usage WHERE service_id = ? AND period_type = ? AND period_start = ? `).get(serviceId, periodType, start) as RawUsageRow | undefined; if (current) continue; const previous = this.database.prepare(` SELECT hard_limit FROM external_service_usage WHERE service_id = ? AND period_type = ? ORDER BY period_start DESC LIMIT 1 `).get(serviceId, periodType) as { hard_limit: number } | undefined; const maximum = maximumHardLimits[serviceId][periodType]; if (maximum === undefined) throw new ExternalServiceUsageError("service_not_found"); this.database.prepare(` INSERT INTO external_service_usage ( service_id, period_type, period_start, hard_limit, used_count, service_status, pause_reason, updated_at ) VALUES (?, ?, ?, ?, 0, ?, ?, ?) `).run(serviceId, periodType, start, previous?.hard_limit ?? maximum, previous ? "paused_quota" : "active", previous ? "period_confirmation_required" : null, now); } return this.database.prepare(` SELECT * FROM external_service_usage WHERE service_id = ? AND period_start IN (${periods.map(() => "?").join(",")}) ORDER BY period_type `).all(serviceId, ...periods.map((period) => periodStart(period, now))) as RawUsageRow[]; } private ensureSchema() { this.database.exec(` CREATE TABLE IF NOT EXISTS external_service_usage ( service_id TEXT NOT NULL CHECK (service_id IN ('resend_email', 'amap_web_service')), period_type TEXT NOT NULL CHECK (period_type IN ('daily', 'monthly')), period_start INTEGER NOT NULL, hard_limit INTEGER NOT NULL CHECK (hard_limit >= 1), used_count INTEGER NOT NULL CHECK (used_count >= 0 AND used_count <= hard_limit), service_status TEXT NOT NULL CHECK (service_status IN ('active', 'paused_quota', 'paused_provider', 'disabled')), pause_reason TEXT, updated_at INTEGER NOT NULL, PRIMARY KEY (service_id, period_type, period_start) ); CREATE TABLE IF NOT EXISTS service_recovery_checks ( check_id TEXT PRIMARY KEY, service_name TEXT NOT NULL DEFAULT 'external_service', target_ref TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'passed' CHECK (status IN ('passed', 'failed')), checked_at INTEGER NOT NULL DEFAULT 0, expires_at INTEGER NOT NULL DEFAULT 0, details_json TEXT NOT NULL DEFAULT '{}', service_id TEXT CHECK (service_id IS NULL OR service_id IN ('resend_email', 'amap_web_service')), period_start INTEGER, available INTEGER CHECK (available IS NULL OR available IN (0, 1)), check_reason TEXT ); `); const columns = new Set((this.database.prepare("PRAGMA table_info(service_recovery_checks)").all() as Array<{ name: string }>).map((column) => column.name)); const additions: Array<[string, string]> = [ ["service_name", "TEXT NOT NULL DEFAULT 'external_service'"], ["target_ref", "TEXT NOT NULL DEFAULT ''"], ["status", "TEXT NOT NULL DEFAULT 'passed'"], ["expires_at", "INTEGER NOT NULL DEFAULT 0"], ["details_json", "TEXT NOT NULL DEFAULT '{}'"], ["service_id", "TEXT"], ["period_start", "INTEGER"], ["available", "INTEGER"], ["check_reason", "TEXT"], ]; for (const [name, definition] of additions) { if (!columns.has(name)) this.database.exec(`ALTER TABLE service_recovery_checks ADD COLUMN ${name} ${definition}`); } } private runImmediate(action: () => T): T { const nested = this.database.inTransaction; if (!nested) this.database.exec("BEGIN IMMEDIATE"); try { const result = action(); if (!nested) this.database.exec("COMMIT"); return result; } catch (error) { if (!nested && this.database.inTransaction) this.database.exec("ROLLBACK"); throw error; } } private recordAudit(input: { actorRef: string; actorType: "system" | "super_admin"; afterSummary: Record | null; beforeSummary: Record | null; operationType: string; result: "succeeded" | "failed"; targetRef: string; targetType: string; }, now: number) { this.database.prepare(` INSERT INTO admin_operation_logs ( log_id, actor_type, actor_ref, operation_type, target_type, target_ref, result, before_summary, after_summary, occurred_at, expires_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( randomUUID(), input.actorType, input.actorRef, input.operationType, input.targetType, input.targetRef, input.result, serializeAuditSummary(input.beforeSummary), serializeAuditSummary(input.afterSummary), now, now + retentionMilliseconds, ); } } function normalizeReason(reason: string) { const normalized = reason.trim().toLowerCase().replace(/[^a-z0-9_.-]/g, "_").slice(0, 120); return normalized || "provider_unavailable"; }