feat: enforce WP6-03 external service hard stops
Dada P0-A isolated Windows CI / validate-and-package (push) Successful in 9m34s
Dada P0-A isolated Windows CI / validate-and-package (push) Successful in 9m34s
This commit is contained in:
suyx
@@ -0,0 +1,459 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import type BetterSqlite3 from "better-sqlite3";
|
||||
|
||||
import { serializeAuditSummary } from "./audit-policy.js";
|
||||
|
||||
export type ExternalServiceId = "resend_email" | "amap_web_service";
|
||||
export type ExternalServicePeriodType = "daily" | "monthly";
|
||||
export type ExternalServiceStatus = "active" | "paused_quota" | "paused_provider" | "disabled";
|
||||
|
||||
const retentionMilliseconds = 180 * 24 * 60 * 60 * 1_000;
|
||||
const recoveryCheckLifetimeMilliseconds = 15 * 60 * 1_000;
|
||||
const maximumHardLimits: Record<ExternalServiceId, Partial<Record<ExternalServicePeriodType, number>>> = {
|
||||
resend_email: { daily: 80, monthly: 2_400 },
|
||||
amap_web_service: { monthly: 1_000 },
|
||||
};
|
||||
|
||||
export interface ExternalServiceUsageRow {
|
||||
serviceId: ExternalServiceId;
|
||||
periodType: ExternalServicePeriodType;
|
||||
periodStart: number;
|
||||
hardLimit: number;
|
||||
usedCount: number;
|
||||
status: ExternalServiceStatus;
|
||||
pauseReason: string | null;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
export class ExternalServiceUsageError extends Error {
|
||||
constructor(
|
||||
readonly code:
|
||||
| "service_paused_quota"
|
||||
| "service_paused_provider"
|
||||
| "service_disabled"
|
||||
| "hard_limit_increase_forbidden"
|
||||
| "hard_limit_invalid"
|
||||
| "health_check_required"
|
||||
| "quota_exhausted"
|
||||
| "service_not_found",
|
||||
message = code,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ExternalServiceUsageError";
|
||||
}
|
||||
}
|
||||
|
||||
interface ExternalServiceUsageOptions {
|
||||
clock?: () => number;
|
||||
database: BetterSqlite3.Database;
|
||||
}
|
||||
|
||||
interface RawUsageRow {
|
||||
service_id: ExternalServiceId;
|
||||
period_type: ExternalServicePeriodType;
|
||||
period_start: number;
|
||||
hard_limit: number;
|
||||
used_count: number;
|
||||
service_status: ExternalServiceStatus;
|
||||
pause_reason: string | null;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
function periodStart(periodType: ExternalServicePeriodType, now: number) {
|
||||
const date = new Date(now);
|
||||
if (periodType === "daily") return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
|
||||
return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1);
|
||||
}
|
||||
|
||||
function requiredPeriods(serviceId: ExternalServiceId): ExternalServicePeriodType[] {
|
||||
return serviceId === "resend_email" ? ["daily", "monthly"] : ["monthly"];
|
||||
}
|
||||
|
||||
function toPublic(row: RawUsageRow): ExternalServiceUsageRow {
|
||||
return {
|
||||
hardLimit: row.hard_limit,
|
||||
pauseReason: row.pause_reason,
|
||||
periodStart: row.period_start,
|
||||
periodType: row.period_type,
|
||||
serviceId: row.service_id,
|
||||
status: row.service_status,
|
||||
updatedAt: row.updated_at,
|
||||
usedCount: row.used_count,
|
||||
};
|
||||
}
|
||||
|
||||
export class ExternalServiceUsage {
|
||||
readonly database: BetterSqlite3.Database;
|
||||
readonly clock: () => number;
|
||||
|
||||
constructor(options: ExternalServiceUsageOptions) {
|
||||
this.database = options.database;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.ensureSchema();
|
||||
this.runImmediate(() => {
|
||||
this.ensureCurrentRows(this.clock(), "resend_email");
|
||||
this.ensureCurrentRows(this.clock(), "amap_web_service");
|
||||
});
|
||||
}
|
||||
|
||||
claimResend(now = this.clock()) {
|
||||
return this.runImmediate(() => this.claimWithinTransaction("resend_email", now));
|
||||
}
|
||||
|
||||
claimResendWithinTransaction(now = this.clock()) {
|
||||
return this.claimWithinTransaction("resend_email", now);
|
||||
}
|
||||
|
||||
claimAmap(now = this.clock()) {
|
||||
return this.runImmediate(() => this.claimWithinTransaction("amap_web_service", now));
|
||||
}
|
||||
|
||||
claimAmapWithinTransaction(now = this.clock()) {
|
||||
return this.claimWithinTransaction("amap_web_service", now);
|
||||
}
|
||||
|
||||
markProviderFailure(input: { serviceId: ExternalServiceId; reason: string; now?: number }) {
|
||||
return this.runImmediate(() => this.markProviderFailureWithinTransaction(input));
|
||||
}
|
||||
|
||||
markProviderFailureWithinTransaction(input: { serviceId: ExternalServiceId; reason: string; now?: number }) {
|
||||
const now = input.now ?? this.clock();
|
||||
const rows = this.ensureCurrentRows(now, input.serviceId);
|
||||
const reason = normalizeReason(input.reason);
|
||||
for (const row of rows) {
|
||||
if (row.service_status === "disabled") continue;
|
||||
this.database.prepare(`
|
||||
UPDATE external_service_usage
|
||||
SET service_status = 'paused_provider', pause_reason = ?, updated_at = ?
|
||||
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
||||
`).run(reason, now, row.service_id, row.period_type, row.period_start);
|
||||
}
|
||||
this.recordAudit({
|
||||
actorRef: "external_service_runtime",
|
||||
actorType: "system",
|
||||
afterSummary: { pause_reason: reason, status: "paused_provider" },
|
||||
beforeSummary: { status: rows[0]?.service_status ?? "active" },
|
||||
operationType: "service_provider_pause",
|
||||
result: "succeeded",
|
||||
targetRef: input.serviceId,
|
||||
targetType: "external_service",
|
||||
}, now);
|
||||
return this.read(input.serviceId);
|
||||
}
|
||||
|
||||
recordHealthCheck(input: { serviceId: ExternalServiceId; available: boolean; reason?: string; now?: number }) {
|
||||
const now = input.now ?? this.clock();
|
||||
const checkId = randomUUID();
|
||||
const currentPeriod = periodStart(requiredPeriods(input.serviceId)[0]!, now);
|
||||
const result = this.runImmediate(() => {
|
||||
this.database.prepare(`
|
||||
INSERT INTO service_recovery_checks (
|
||||
check_id, service_name, target_ref, status, checked_at, expires_at, details_json,
|
||||
service_id, period_start, available, check_reason
|
||||
) VALUES (?, 'external_service', ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
checkId,
|
||||
input.serviceId,
|
||||
input.available ? "passed" : "failed",
|
||||
now,
|
||||
now + recoveryCheckLifetimeMilliseconds,
|
||||
JSON.stringify({ non_sensitive: true }),
|
||||
input.serviceId,
|
||||
currentPeriod,
|
||||
input.available ? 1 : 0,
|
||||
input.reason ? normalizeReason(input.reason) : null,
|
||||
);
|
||||
return { checkId, available: input.available, checkedAt: now };
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
recover(input: { serviceId: ExternalServiceId; actorId: string; checkId: string; now?: number }) {
|
||||
const now = input.now ?? this.clock();
|
||||
const result = this.runImmediate(() => {
|
||||
const check = this.database.prepare(`
|
||||
SELECT check_id, period_start, available, expires_at
|
||||
FROM service_recovery_checks
|
||||
WHERE check_id = ? AND service_id = ? AND service_name = 'external_service'
|
||||
`).get(input.checkId, input.serviceId) as { available: number; check_id: string; expires_at: number; period_start: number } | undefined;
|
||||
const currentPeriod = periodStart(requiredPeriods(input.serviceId)[0]!, now);
|
||||
if (!check?.available || check.period_start !== currentPeriod || check.expires_at <= now) {
|
||||
this.recordAudit({
|
||||
actorRef: input.actorId,
|
||||
actorType: "super_admin",
|
||||
afterSummary: { reason: "health_check_required" },
|
||||
beforeSummary: null,
|
||||
operationType: "service_recovery",
|
||||
result: "failed",
|
||||
targetRef: input.serviceId,
|
||||
targetType: "external_service",
|
||||
}, now);
|
||||
return { error: new ExternalServiceUsageError("health_check_required") };
|
||||
}
|
||||
const rows = this.ensureCurrentRows(now, input.serviceId);
|
||||
if (rows.some((row) => row.used_count >= row.hard_limit)) {
|
||||
this.recordAudit({
|
||||
actorRef: input.actorId,
|
||||
actorType: "super_admin",
|
||||
afterSummary: { reason: "quota_exhausted" },
|
||||
beforeSummary: { status: rows[0]?.service_status ?? "paused_quota" },
|
||||
operationType: "service_recovery",
|
||||
result: "failed",
|
||||
targetRef: input.serviceId,
|
||||
targetType: "external_service",
|
||||
}, now);
|
||||
return { error: new ExternalServiceUsageError("quota_exhausted") };
|
||||
}
|
||||
for (const row of rows) {
|
||||
this.database.prepare(`
|
||||
UPDATE external_service_usage
|
||||
SET service_status = 'active', pause_reason = NULL, updated_at = ?
|
||||
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
||||
`).run(now, row.service_id, row.period_type, row.period_start);
|
||||
}
|
||||
this.recordAudit({
|
||||
actorRef: input.actorId,
|
||||
actorType: "super_admin",
|
||||
afterSummary: { check_id: input.checkId, status: "active" },
|
||||
beforeSummary: { status: rows[0]?.service_status ?? "paused_provider" },
|
||||
operationType: "service_recovery",
|
||||
result: "succeeded",
|
||||
targetRef: input.serviceId,
|
||||
targetType: "external_service",
|
||||
}, now);
|
||||
return { status: "active" as const };
|
||||
});
|
||||
if ("error" in result && result.error) throw result.error;
|
||||
return result;
|
||||
}
|
||||
|
||||
setHardLimit(input: {
|
||||
serviceId: ExternalServiceId;
|
||||
periodType: ExternalServicePeriodType;
|
||||
hardLimit: number;
|
||||
actorId: string;
|
||||
now?: number;
|
||||
}) {
|
||||
const now = input.now ?? this.clock();
|
||||
const result = this.runImmediate(() => {
|
||||
const maximum = maximumHardLimits[input.serviceId][input.periodType];
|
||||
const current = this.ensureCurrentRows(now, input.serviceId).find((row) => row.period_type === input.periodType);
|
||||
if (current && input.hardLimit > current.hard_limit) {
|
||||
this.recordAudit({
|
||||
actorRef: input.actorId,
|
||||
actorType: "super_admin",
|
||||
afterSummary: { reason: "hard_limit_increase_forbidden" },
|
||||
beforeSummary: { hard_limit: current.hard_limit },
|
||||
operationType: "service_hard_limit_update",
|
||||
result: "failed",
|
||||
targetRef: `${input.serviceId}:${input.periodType}`,
|
||||
targetType: "external_service_limit",
|
||||
}, now);
|
||||
return { error: new ExternalServiceUsageError("hard_limit_increase_forbidden") };
|
||||
}
|
||||
const valid = maximum !== undefined && Number.isSafeInteger(input.hardLimit) && input.hardLimit >= 1 && input.hardLimit <= maximum;
|
||||
if (!valid || !current) {
|
||||
this.recordAudit({
|
||||
actorRef: input.actorId,
|
||||
actorType: "super_admin",
|
||||
afterSummary: { reason: "hard_limit_invalid" },
|
||||
beforeSummary: current ? { hard_limit: current.hard_limit } : null,
|
||||
operationType: "service_hard_limit_update",
|
||||
result: "failed",
|
||||
targetRef: `${input.serviceId}:${input.periodType}`,
|
||||
targetType: "external_service_limit",
|
||||
}, now);
|
||||
return { error: new ExternalServiceUsageError("hard_limit_invalid") };
|
||||
}
|
||||
this.database.prepare(`
|
||||
UPDATE external_service_usage
|
||||
SET hard_limit = ?, service_status = CASE
|
||||
WHEN used_count >= ? THEN 'paused_quota'
|
||||
ELSE service_status
|
||||
END, pause_reason = CASE
|
||||
WHEN used_count >= ? THEN 'hard_limit_reached'
|
||||
ELSE pause_reason
|
||||
END, updated_at = ?
|
||||
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
||||
`).run(input.hardLimit, input.hardLimit, input.hardLimit, now, current.service_id, current.period_type, current.period_start);
|
||||
this.recordAudit({
|
||||
actorRef: input.actorId,
|
||||
actorType: "super_admin",
|
||||
afterSummary: { hard_limit: input.hardLimit },
|
||||
beforeSummary: { hard_limit: current.hard_limit },
|
||||
operationType: "service_hard_limit_update",
|
||||
result: "succeeded",
|
||||
targetRef: `${input.serviceId}:${input.periodType}`,
|
||||
targetType: "external_service_limit",
|
||||
}, now);
|
||||
return this.read(input.serviceId);
|
||||
});
|
||||
if ("error" in result && result.error) throw result.error;
|
||||
return result;
|
||||
}
|
||||
|
||||
read(serviceId?: ExternalServiceId) {
|
||||
const rows = (serviceId
|
||||
? this.database.prepare("SELECT * FROM external_service_usage WHERE service_id = ? ORDER BY period_type").all(serviceId)
|
||||
: this.database.prepare("SELECT * FROM external_service_usage ORDER BY service_id, period_type").all()) as RawUsageRow[];
|
||||
return rows.map(toPublic);
|
||||
}
|
||||
|
||||
readCurrent() {
|
||||
const now = this.clock();
|
||||
return this.read().filter((row) => row.periodStart === periodStart(row.periodType, now));
|
||||
}
|
||||
|
||||
readStatus(serviceId: ExternalServiceId) {
|
||||
const rows = this.read(serviceId).filter((row) => row.periodStart >= periodStart(row.periodType, this.clock()));
|
||||
const status = rows.some((row) => row.status === "disabled")
|
||||
? "disabled"
|
||||
: rows.some((row) => row.status === "paused_provider")
|
||||
? "paused_provider"
|
||||
: rows.some((row) => row.status === "paused_quota")
|
||||
? "paused_quota"
|
||||
: "active";
|
||||
return { serviceId, status, rows } as const;
|
||||
}
|
||||
|
||||
private claimWithinTransaction(serviceId: ExternalServiceId, now: number) {
|
||||
const rows = this.ensureCurrentRows(now, serviceId);
|
||||
for (const row of rows) {
|
||||
if (row.service_status === "paused_quota") throw new ExternalServiceUsageError("service_paused_quota");
|
||||
if (row.service_status === "paused_provider") throw new ExternalServiceUsageError("service_paused_provider");
|
||||
if (row.service_status === "disabled") throw new ExternalServiceUsageError("service_disabled");
|
||||
if (row.used_count >= row.hard_limit) {
|
||||
this.database.prepare(`
|
||||
UPDATE external_service_usage
|
||||
SET service_status = 'paused_quota', pause_reason = 'hard_limit_reached', updated_at = ?
|
||||
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
||||
`).run(now, row.service_id, row.period_type, row.period_start);
|
||||
throw new ExternalServiceUsageError("service_paused_quota");
|
||||
}
|
||||
}
|
||||
const updated = rows.map((row) => {
|
||||
const usedCount = row.used_count + 1;
|
||||
const status: ExternalServiceStatus = usedCount >= row.hard_limit ? "paused_quota" : "active";
|
||||
this.database.prepare(`
|
||||
UPDATE external_service_usage
|
||||
SET used_count = ?, service_status = ?, pause_reason = CASE WHEN ? = 'active' THEN NULL ELSE 'hard_limit_reached' END, updated_at = ?
|
||||
WHERE service_id = ? AND period_type = ? AND period_start = ?
|
||||
`).run(usedCount, status, status, now, row.service_id, row.period_type, row.period_start);
|
||||
return { periodType: row.period_type, remaining: Math.max(0, row.hard_limit - usedCount), usedCount };
|
||||
});
|
||||
return { allowed: true as const, serviceId, allocations: updated, remaining: Math.min(...updated.map((item) => item.remaining)) };
|
||||
}
|
||||
|
||||
private ensureCurrentRows(now: number, serviceId: ExternalServiceId) {
|
||||
const periods = requiredPeriods(serviceId);
|
||||
for (const periodType of periods) {
|
||||
const start = periodStart(periodType, now);
|
||||
const current = this.database.prepare(`
|
||||
SELECT * FROM external_service_usage WHERE service_id = ? AND period_type = ? AND period_start = ?
|
||||
`).get(serviceId, periodType, start) as RawUsageRow | undefined;
|
||||
if (current) continue;
|
||||
const previous = this.database.prepare(`
|
||||
SELECT hard_limit FROM external_service_usage
|
||||
WHERE service_id = ? AND period_type = ? ORDER BY period_start DESC LIMIT 1
|
||||
`).get(serviceId, periodType) as { hard_limit: number } | undefined;
|
||||
const maximum = maximumHardLimits[serviceId][periodType];
|
||||
if (maximum === undefined) throw new ExternalServiceUsageError("service_not_found");
|
||||
this.database.prepare(`
|
||||
INSERT INTO external_service_usage (
|
||||
service_id, period_type, period_start, hard_limit, used_count,
|
||||
service_status, pause_reason, updated_at
|
||||
) VALUES (?, ?, ?, ?, 0, ?, ?, ?)
|
||||
`).run(serviceId, periodType, start, previous?.hard_limit ?? maximum, previous ? "paused_quota" : "active", previous ? "period_confirmation_required" : null, now);
|
||||
}
|
||||
return this.database.prepare(`
|
||||
SELECT * FROM external_service_usage
|
||||
WHERE service_id = ? AND period_start IN (${periods.map(() => "?").join(",")})
|
||||
ORDER BY period_type
|
||||
`).all(serviceId, ...periods.map((period) => periodStart(period, now))) as RawUsageRow[];
|
||||
}
|
||||
|
||||
private ensureSchema() {
|
||||
this.database.exec(`
|
||||
CREATE TABLE IF NOT EXISTS external_service_usage (
|
||||
service_id TEXT NOT NULL CHECK (service_id IN ('resend_email', 'amap_web_service')),
|
||||
period_type TEXT NOT NULL CHECK (period_type IN ('daily', 'monthly')),
|
||||
period_start INTEGER NOT NULL,
|
||||
hard_limit INTEGER NOT NULL CHECK (hard_limit >= 1),
|
||||
used_count INTEGER NOT NULL CHECK (used_count >= 0 AND used_count <= hard_limit),
|
||||
service_status TEXT NOT NULL CHECK (service_status IN ('active', 'paused_quota', 'paused_provider', 'disabled')),
|
||||
pause_reason TEXT,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (service_id, period_type, period_start)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS service_recovery_checks (
|
||||
check_id TEXT PRIMARY KEY,
|
||||
service_name TEXT NOT NULL DEFAULT 'external_service',
|
||||
target_ref TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'passed' CHECK (status IN ('passed', 'failed')),
|
||||
checked_at INTEGER NOT NULL DEFAULT 0,
|
||||
expires_at INTEGER NOT NULL DEFAULT 0,
|
||||
details_json TEXT NOT NULL DEFAULT '{}',
|
||||
service_id TEXT CHECK (service_id IS NULL OR service_id IN ('resend_email', 'amap_web_service')),
|
||||
period_start INTEGER,
|
||||
available INTEGER CHECK (available IS NULL OR available IN (0, 1)),
|
||||
check_reason TEXT
|
||||
);
|
||||
`);
|
||||
const columns = new Set((this.database.prepare("PRAGMA table_info(service_recovery_checks)").all() as Array<{ name: string }>).map((column) => column.name));
|
||||
const additions: Array<[string, string]> = [
|
||||
["service_name", "TEXT NOT NULL DEFAULT 'external_service'"],
|
||||
["target_ref", "TEXT NOT NULL DEFAULT ''"],
|
||||
["status", "TEXT NOT NULL DEFAULT 'passed'"],
|
||||
["expires_at", "INTEGER NOT NULL DEFAULT 0"],
|
||||
["details_json", "TEXT NOT NULL DEFAULT '{}'"],
|
||||
["service_id", "TEXT"],
|
||||
["period_start", "INTEGER"],
|
||||
["available", "INTEGER"],
|
||||
["check_reason", "TEXT"],
|
||||
];
|
||||
for (const [name, definition] of additions) {
|
||||
if (!columns.has(name)) this.database.exec(`ALTER TABLE service_recovery_checks ADD COLUMN ${name} ${definition}`);
|
||||
}
|
||||
}
|
||||
|
||||
private runImmediate<T>(action: () => T): T {
|
||||
const nested = this.database.inTransaction;
|
||||
if (!nested) this.database.exec("BEGIN IMMEDIATE");
|
||||
try {
|
||||
const result = action();
|
||||
if (!nested) this.database.exec("COMMIT");
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (!nested && this.database.inTransaction) this.database.exec("ROLLBACK");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private recordAudit(input: {
|
||||
actorRef: string;
|
||||
actorType: "system" | "super_admin";
|
||||
afterSummary: Record<string, unknown> | null;
|
||||
beforeSummary: Record<string, unknown> | null;
|
||||
operationType: string;
|
||||
result: "succeeded" | "failed";
|
||||
targetRef: string;
|
||||
targetType: string;
|
||||
}, now: number) {
|
||||
this.database.prepare(`
|
||||
INSERT INTO admin_operation_logs (
|
||||
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
|
||||
result, before_summary, after_summary, occurred_at, expires_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
randomUUID(), input.actorType, input.actorRef, input.operationType, input.targetType, input.targetRef,
|
||||
input.result, serializeAuditSummary(input.beforeSummary), serializeAuditSummary(input.afterSummary), now,
|
||||
now + retentionMilliseconds,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeReason(reason: string) {
|
||||
const normalized = reason.trim().toLowerCase().replace(/[^a-z0-9_.-]/g, "_").slice(0, 120);
|
||||
return normalized || "provider_unavailable";
|
||||
}
|
||||
Reference in New Issue
Block a user