460 lines
19 KiB
TypeScript
460 lines
19 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
|
|
import type BetterSqlite3 from "better-sqlite3";
|
|
|
|
import { serializeAuditSummary } from "./audit-policy.js";
|
|
|
|
export type ExternalServiceId = "resend_email" | "amap_web_service";
|
|
export type ExternalServicePeriodType = "daily" | "monthly";
|
|
export type ExternalServiceStatus = "active" | "paused_quota" | "paused_provider" | "disabled";
|
|
|
|
const retentionMilliseconds = 180 * 24 * 60 * 60 * 1_000;
|
|
const recoveryCheckLifetimeMilliseconds = 15 * 60 * 1_000;
|
|
const maximumHardLimits: Record<ExternalServiceId, Partial<Record<ExternalServicePeriodType, number>>> = {
|
|
resend_email: { daily: 80, monthly: 2_400 },
|
|
amap_web_service: { monthly: 1_000 },
|
|
};
|
|
|
|
export interface ExternalServiceUsageRow {
|
|
serviceId: ExternalServiceId;
|
|
periodType: ExternalServicePeriodType;
|
|
periodStart: number;
|
|
hardLimit: number;
|
|
usedCount: number;
|
|
status: ExternalServiceStatus;
|
|
pauseReason: string | null;
|
|
updatedAt: number;
|
|
}
|
|
|
|
export class ExternalServiceUsageError extends Error {
|
|
constructor(
|
|
readonly code:
|
|
| "service_paused_quota"
|
|
| "service_paused_provider"
|
|
| "service_disabled"
|
|
| "hard_limit_increase_forbidden"
|
|
| "hard_limit_invalid"
|
|
| "health_check_required"
|
|
| "quota_exhausted"
|
|
| "service_not_found",
|
|
message = code,
|
|
) {
|
|
super(message);
|
|
this.name = "ExternalServiceUsageError";
|
|
}
|
|
}
|
|
|
|
interface ExternalServiceUsageOptions {
|
|
clock?: () => number;
|
|
database: BetterSqlite3.Database;
|
|
}
|
|
|
|
interface RawUsageRow {
|
|
service_id: ExternalServiceId;
|
|
period_type: ExternalServicePeriodType;
|
|
period_start: number;
|
|
hard_limit: number;
|
|
used_count: number;
|
|
service_status: ExternalServiceStatus;
|
|
pause_reason: string | null;
|
|
updated_at: number;
|
|
}
|
|
|
|
function periodStart(periodType: ExternalServicePeriodType, now: number) {
|
|
const date = new Date(now);
|
|
if (periodType === "daily") return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
|
|
return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1);
|
|
}
|
|
|
|
function requiredPeriods(serviceId: ExternalServiceId): ExternalServicePeriodType[] {
|
|
return serviceId === "resend_email" ? ["daily", "monthly"] : ["monthly"];
|
|
}
|
|
|
|
function toPublic(row: RawUsageRow): ExternalServiceUsageRow {
|
|
return {
|
|
hardLimit: row.hard_limit,
|
|
pauseReason: row.pause_reason,
|
|
periodStart: row.period_start,
|
|
periodType: row.period_type,
|
|
serviceId: row.service_id,
|
|
status: row.service_status,
|
|
updatedAt: row.updated_at,
|
|
usedCount: row.used_count,
|
|
};
|
|
}
|
|
|
|
export class ExternalServiceUsage {
|
|
readonly database: BetterSqlite3.Database;
|
|
readonly clock: () => number;
|
|
|
|
constructor(options: ExternalServiceUsageOptions) {
|
|
this.database = options.database;
|
|
this.clock = options.clock ?? Date.now;
|
|
this.ensureSchema();
|
|
this.runImmediate(() => {
|
|
this.ensureCurrentRows(this.clock(), "resend_email");
|
|
this.ensureCurrentRows(this.clock(), "amap_web_service");
|
|
});
|
|
}
|
|
|
|
claimResend(now = this.clock()) {
|
|
return this.runImmediate(() => this.claimWithinTransaction("resend_email", now));
|
|
}
|
|
|
|
claimResendWithinTransaction(now = this.clock()) {
|
|
return this.claimWithinTransaction("resend_email", now);
|
|
}
|
|
|
|
claimAmap(now = this.clock()) {
|
|
return this.runImmediate(() => this.claimWithinTransaction("amap_web_service", now));
|
|
}
|
|
|
|
claimAmapWithinTransaction(now = this.clock()) {
|
|
return this.claimWithinTransaction("amap_web_service", now);
|
|
}
|
|
|
|
markProviderFailure(input: { serviceId: ExternalServiceId; reason: string; now?: number }) {
|
|
return this.runImmediate(() => this.markProviderFailureWithinTransaction(input));
|
|
}
|
|
|
|
markProviderFailureWithinTransaction(input: { serviceId: ExternalServiceId; reason: string; now?: number }) {
|
|
const now = input.now ?? this.clock();
|
|
const rows = this.ensureCurrentRows(now, input.serviceId);
|
|
const reason = normalizeReason(input.reason);
|
|
for (const row of rows) {
|
|
if (row.service_status === "disabled") continue;
|
|
this.database.prepare(`
|
|
UPDATE external_service_usage
|
|
SET service_status = 'paused_provider', pause_reason = ?, updated_at = ?
|
|
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
|
`).run(reason, now, row.service_id, row.period_type, row.period_start);
|
|
}
|
|
this.recordAudit({
|
|
actorRef: "external_service_runtime",
|
|
actorType: "system",
|
|
afterSummary: { pause_reason: reason, status: "paused_provider" },
|
|
beforeSummary: { status: rows[0]?.service_status ?? "active" },
|
|
operationType: "service_provider_pause",
|
|
result: "succeeded",
|
|
targetRef: input.serviceId,
|
|
targetType: "external_service",
|
|
}, now);
|
|
return this.read(input.serviceId);
|
|
}
|
|
|
|
recordHealthCheck(input: { serviceId: ExternalServiceId; available: boolean; reason?: string; now?: number }) {
|
|
const now = input.now ?? this.clock();
|
|
const checkId = randomUUID();
|
|
const currentPeriod = periodStart(requiredPeriods(input.serviceId)[0]!, now);
|
|
const result = this.runImmediate(() => {
|
|
this.database.prepare(`
|
|
INSERT INTO service_recovery_checks (
|
|
check_id, service_name, target_ref, status, checked_at, expires_at, details_json,
|
|
service_id, period_start, available, check_reason
|
|
) VALUES (?, 'external_service', ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`).run(
|
|
checkId,
|
|
input.serviceId,
|
|
input.available ? "passed" : "failed",
|
|
now,
|
|
now + recoveryCheckLifetimeMilliseconds,
|
|
JSON.stringify({ non_sensitive: true }),
|
|
input.serviceId,
|
|
currentPeriod,
|
|
input.available ? 1 : 0,
|
|
input.reason ? normalizeReason(input.reason) : null,
|
|
);
|
|
return { checkId, available: input.available, checkedAt: now };
|
|
});
|
|
return result;
|
|
}
|
|
|
|
recover(input: { serviceId: ExternalServiceId; actorId: string; checkId: string; now?: number }) {
|
|
const now = input.now ?? this.clock();
|
|
const result = this.runImmediate(() => {
|
|
const check = this.database.prepare(`
|
|
SELECT check_id, period_start, available, expires_at
|
|
FROM service_recovery_checks
|
|
WHERE check_id = ? AND service_id = ? AND service_name = 'external_service'
|
|
`).get(input.checkId, input.serviceId) as { available: number; check_id: string; expires_at: number; period_start: number } | undefined;
|
|
const currentPeriod = periodStart(requiredPeriods(input.serviceId)[0]!, now);
|
|
if (!check?.available || check.period_start !== currentPeriod || check.expires_at <= now) {
|
|
this.recordAudit({
|
|
actorRef: input.actorId,
|
|
actorType: "super_admin",
|
|
afterSummary: { reason: "health_check_required" },
|
|
beforeSummary: null,
|
|
operationType: "service_recovery",
|
|
result: "failed",
|
|
targetRef: input.serviceId,
|
|
targetType: "external_service",
|
|
}, now);
|
|
return { error: new ExternalServiceUsageError("health_check_required") };
|
|
}
|
|
const rows = this.ensureCurrentRows(now, input.serviceId);
|
|
if (rows.some((row) => row.used_count >= row.hard_limit)) {
|
|
this.recordAudit({
|
|
actorRef: input.actorId,
|
|
actorType: "super_admin",
|
|
afterSummary: { reason: "quota_exhausted" },
|
|
beforeSummary: { status: rows[0]?.service_status ?? "paused_quota" },
|
|
operationType: "service_recovery",
|
|
result: "failed",
|
|
targetRef: input.serviceId,
|
|
targetType: "external_service",
|
|
}, now);
|
|
return { error: new ExternalServiceUsageError("quota_exhausted") };
|
|
}
|
|
for (const row of rows) {
|
|
this.database.prepare(`
|
|
UPDATE external_service_usage
|
|
SET service_status = 'active', pause_reason = NULL, updated_at = ?
|
|
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
|
`).run(now, row.service_id, row.period_type, row.period_start);
|
|
}
|
|
this.recordAudit({
|
|
actorRef: input.actorId,
|
|
actorType: "super_admin",
|
|
afterSummary: { check_id: input.checkId, status: "active" },
|
|
beforeSummary: { status: rows[0]?.service_status ?? "paused_provider" },
|
|
operationType: "service_recovery",
|
|
result: "succeeded",
|
|
targetRef: input.serviceId,
|
|
targetType: "external_service",
|
|
}, now);
|
|
return { status: "active" as const };
|
|
});
|
|
if ("error" in result && result.error) throw result.error;
|
|
return result;
|
|
}
|
|
|
|
setHardLimit(input: {
|
|
serviceId: ExternalServiceId;
|
|
periodType: ExternalServicePeriodType;
|
|
hardLimit: number;
|
|
actorId: string;
|
|
now?: number;
|
|
}) {
|
|
const now = input.now ?? this.clock();
|
|
const result = this.runImmediate(() => {
|
|
const maximum = maximumHardLimits[input.serviceId][input.periodType];
|
|
const current = this.ensureCurrentRows(now, input.serviceId).find((row) => row.period_type === input.periodType);
|
|
if (current && input.hardLimit > current.hard_limit) {
|
|
this.recordAudit({
|
|
actorRef: input.actorId,
|
|
actorType: "super_admin",
|
|
afterSummary: { reason: "hard_limit_increase_forbidden" },
|
|
beforeSummary: { hard_limit: current.hard_limit },
|
|
operationType: "service_hard_limit_update",
|
|
result: "failed",
|
|
targetRef: `${input.serviceId}:${input.periodType}`,
|
|
targetType: "external_service_limit",
|
|
}, now);
|
|
return { error: new ExternalServiceUsageError("hard_limit_increase_forbidden") };
|
|
}
|
|
const valid = maximum !== undefined && Number.isSafeInteger(input.hardLimit) && input.hardLimit >= 1 && input.hardLimit <= maximum;
|
|
if (!valid || !current) {
|
|
this.recordAudit({
|
|
actorRef: input.actorId,
|
|
actorType: "super_admin",
|
|
afterSummary: { reason: "hard_limit_invalid" },
|
|
beforeSummary: current ? { hard_limit: current.hard_limit } : null,
|
|
operationType: "service_hard_limit_update",
|
|
result: "failed",
|
|
targetRef: `${input.serviceId}:${input.periodType}`,
|
|
targetType: "external_service_limit",
|
|
}, now);
|
|
return { error: new ExternalServiceUsageError("hard_limit_invalid") };
|
|
}
|
|
this.database.prepare(`
|
|
UPDATE external_service_usage
|
|
SET hard_limit = ?, service_status = CASE
|
|
WHEN used_count >= ? THEN 'paused_quota'
|
|
ELSE service_status
|
|
END, pause_reason = CASE
|
|
WHEN used_count >= ? THEN 'hard_limit_reached'
|
|
ELSE pause_reason
|
|
END, updated_at = ?
|
|
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
|
`).run(input.hardLimit, input.hardLimit, input.hardLimit, now, current.service_id, current.period_type, current.period_start);
|
|
this.recordAudit({
|
|
actorRef: input.actorId,
|
|
actorType: "super_admin",
|
|
afterSummary: { hard_limit: input.hardLimit },
|
|
beforeSummary: { hard_limit: current.hard_limit },
|
|
operationType: "service_hard_limit_update",
|
|
result: "succeeded",
|
|
targetRef: `${input.serviceId}:${input.periodType}`,
|
|
targetType: "external_service_limit",
|
|
}, now);
|
|
return this.read(input.serviceId);
|
|
});
|
|
if ("error" in result && result.error) throw result.error;
|
|
return result;
|
|
}
|
|
|
|
read(serviceId?: ExternalServiceId) {
|
|
const rows = (serviceId
|
|
? this.database.prepare("SELECT * FROM external_service_usage WHERE service_id = ? ORDER BY period_type").all(serviceId)
|
|
: this.database.prepare("SELECT * FROM external_service_usage ORDER BY service_id, period_type").all()) as RawUsageRow[];
|
|
return rows.map(toPublic);
|
|
}
|
|
|
|
readCurrent() {
|
|
const now = this.clock();
|
|
return this.read().filter((row) => row.periodStart === periodStart(row.periodType, now));
|
|
}
|
|
|
|
readStatus(serviceId: ExternalServiceId) {
|
|
const rows = this.read(serviceId).filter((row) => row.periodStart >= periodStart(row.periodType, this.clock()));
|
|
const status = rows.some((row) => row.status === "disabled")
|
|
? "disabled"
|
|
: rows.some((row) => row.status === "paused_provider")
|
|
? "paused_provider"
|
|
: rows.some((row) => row.status === "paused_quota")
|
|
? "paused_quota"
|
|
: "active";
|
|
return { serviceId, status, rows } as const;
|
|
}
|
|
|
|
private claimWithinTransaction(serviceId: ExternalServiceId, now: number) {
|
|
const rows = this.ensureCurrentRows(now, serviceId);
|
|
for (const row of rows) {
|
|
if (row.service_status === "paused_quota") throw new ExternalServiceUsageError("service_paused_quota");
|
|
if (row.service_status === "paused_provider") throw new ExternalServiceUsageError("service_paused_provider");
|
|
if (row.service_status === "disabled") throw new ExternalServiceUsageError("service_disabled");
|
|
if (row.used_count >= row.hard_limit) {
|
|
this.database.prepare(`
|
|
UPDATE external_service_usage
|
|
SET service_status = 'paused_quota', pause_reason = 'hard_limit_reached', updated_at = ?
|
|
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
|
`).run(now, row.service_id, row.period_type, row.period_start);
|
|
throw new ExternalServiceUsageError("service_paused_quota");
|
|
}
|
|
}
|
|
const updated = rows.map((row) => {
|
|
const usedCount = row.used_count + 1;
|
|
const status: ExternalServiceStatus = usedCount >= row.hard_limit ? "paused_quota" : "active";
|
|
this.database.prepare(`
|
|
UPDATE external_service_usage
|
|
SET used_count = ?, service_status = ?, pause_reason = CASE WHEN ? = 'active' THEN NULL ELSE 'hard_limit_reached' END, updated_at = ?
|
|
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
|
`).run(usedCount, status, status, now, row.service_id, row.period_type, row.period_start);
|
|
return { periodType: row.period_type, remaining: Math.max(0, row.hard_limit - usedCount), usedCount };
|
|
});
|
|
return { allowed: true as const, serviceId, allocations: updated, remaining: Math.min(...updated.map((item) => item.remaining)) };
|
|
}
|
|
|
|
private ensureCurrentRows(now: number, serviceId: ExternalServiceId) {
|
|
const periods = requiredPeriods(serviceId);
|
|
for (const periodType of periods) {
|
|
const start = periodStart(periodType, now);
|
|
const current = this.database.prepare(`
|
|
SELECT * FROM external_service_usage WHERE service_id = ? AND period_type = ? AND period_start = ?
|
|
`).get(serviceId, periodType, start) as RawUsageRow | undefined;
|
|
if (current) continue;
|
|
const previous = this.database.prepare(`
|
|
SELECT hard_limit FROM external_service_usage
|
|
WHERE service_id = ? AND period_type = ? ORDER BY period_start DESC LIMIT 1
|
|
`).get(serviceId, periodType) as { hard_limit: number } | undefined;
|
|
const maximum = maximumHardLimits[serviceId][periodType];
|
|
if (maximum === undefined) throw new ExternalServiceUsageError("service_not_found");
|
|
this.database.prepare(`
|
|
INSERT INTO external_service_usage (
|
|
service_id, period_type, period_start, hard_limit, used_count,
|
|
service_status, pause_reason, updated_at
|
|
) VALUES (?, ?, ?, ?, 0, ?, ?, ?)
|
|
`).run(serviceId, periodType, start, previous?.hard_limit ?? maximum, previous ? "paused_quota" : "active", previous ? "period_confirmation_required" : null, now);
|
|
}
|
|
return this.database.prepare(`
|
|
SELECT * FROM external_service_usage
|
|
WHERE service_id = ? AND period_start IN (${periods.map(() => "?").join(",")})
|
|
ORDER BY period_type
|
|
`).all(serviceId, ...periods.map((period) => periodStart(period, now))) as RawUsageRow[];
|
|
}
|
|
|
|
private ensureSchema() {
|
|
this.database.exec(`
|
|
CREATE TABLE IF NOT EXISTS external_service_usage (
|
|
service_id TEXT NOT NULL CHECK (service_id IN ('resend_email', 'amap_web_service')),
|
|
period_type TEXT NOT NULL CHECK (period_type IN ('daily', 'monthly')),
|
|
period_start INTEGER NOT NULL,
|
|
hard_limit INTEGER NOT NULL CHECK (hard_limit >= 1),
|
|
used_count INTEGER NOT NULL CHECK (used_count >= 0 AND used_count <= hard_limit),
|
|
service_status TEXT NOT NULL CHECK (service_status IN ('active', 'paused_quota', 'paused_provider', 'disabled')),
|
|
pause_reason TEXT,
|
|
updated_at INTEGER NOT NULL,
|
|
PRIMARY KEY (service_id, period_type, period_start)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS service_recovery_checks (
|
|
check_id TEXT PRIMARY KEY,
|
|
service_name TEXT NOT NULL DEFAULT 'external_service',
|
|
target_ref TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'passed' CHECK (status IN ('passed', 'failed')),
|
|
checked_at INTEGER NOT NULL DEFAULT 0,
|
|
expires_at INTEGER NOT NULL DEFAULT 0,
|
|
details_json TEXT NOT NULL DEFAULT '{}',
|
|
service_id TEXT CHECK (service_id IS NULL OR service_id IN ('resend_email', 'amap_web_service')),
|
|
period_start INTEGER,
|
|
available INTEGER CHECK (available IS NULL OR available IN (0, 1)),
|
|
check_reason TEXT
|
|
);
|
|
`);
|
|
const columns = new Set((this.database.prepare("PRAGMA table_info(service_recovery_checks)").all() as Array<{ name: string }>).map((column) => column.name));
|
|
const additions: Array<[string, string]> = [
|
|
["service_name", "TEXT NOT NULL DEFAULT 'external_service'"],
|
|
["target_ref", "TEXT NOT NULL DEFAULT ''"],
|
|
["status", "TEXT NOT NULL DEFAULT 'passed'"],
|
|
["expires_at", "INTEGER NOT NULL DEFAULT 0"],
|
|
["details_json", "TEXT NOT NULL DEFAULT '{}'"],
|
|
["service_id", "TEXT"],
|
|
["period_start", "INTEGER"],
|
|
["available", "INTEGER"],
|
|
["check_reason", "TEXT"],
|
|
];
|
|
for (const [name, definition] of additions) {
|
|
if (!columns.has(name)) this.database.exec(`ALTER TABLE service_recovery_checks ADD COLUMN ${name} ${definition}`);
|
|
}
|
|
}
|
|
|
|
private runImmediate<T>(action: () => T): T {
|
|
const nested = this.database.inTransaction;
|
|
if (!nested) this.database.exec("BEGIN IMMEDIATE");
|
|
try {
|
|
const result = action();
|
|
if (!nested) this.database.exec("COMMIT");
|
|
return result;
|
|
} catch (error) {
|
|
if (!nested && this.database.inTransaction) this.database.exec("ROLLBACK");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private recordAudit(input: {
|
|
actorRef: string;
|
|
actorType: "system" | "super_admin";
|
|
afterSummary: Record<string, unknown> | null;
|
|
beforeSummary: Record<string, unknown> | null;
|
|
operationType: string;
|
|
result: "succeeded" | "failed";
|
|
targetRef: string;
|
|
targetType: string;
|
|
}, now: number) {
|
|
this.database.prepare(`
|
|
INSERT INTO admin_operation_logs (
|
|
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
|
|
result, before_summary, after_summary, occurred_at, expires_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`).run(
|
|
randomUUID(), input.actorType, input.actorRef, input.operationType, input.targetType, input.targetRef,
|
|
input.result, serializeAuditSummary(input.beforeSummary), serializeAuditSummary(input.afterSummary), now,
|
|
now + retentionMilliseconds,
|
|
);
|
|
}
|
|
}
|
|
|
|
function normalizeReason(reason: string) {
|
|
const normalized = reason.trim().toLowerCase().replace(/[^a-z0-9_.-]/g, "_").slice(0, 120);
|
|
return normalized || "provider_unavailable";
|
|
}
|