feat: add admin state and diagnostics (TASK-WP6-05)

apps/api/src/admin-state.ts

@@ -0,0 +1,194 @@
+import type BetterSqlite3 from "better-sqlite3";
+
+import type { AdminDiagnosticsResponse, AdminServicesStorageResponse } from "@dada/shared-contracts";
+
+import type { BrowserSupportRelease } from "./browser-support.js";
+import type { ManagedStorage } from "./managed-storage.js";
+import type { ModelConfigurationService } from "./model-configuration.js";
+
+type AdminService = AdminServicesStorageResponse["services"][number];
+const adminServiceIds = ["resend", "amap", "ai_gateway", "worker", "api", "asset_root"] as const;
+const forbiddenDiagnosticPatterns = [
+  /\b(?:api[_ -]?key|secret|password|credential|authorization|bearer|session[_ -]?token|cookie|prompt|email|token)\b/i,
+  /[A-Z]:[\\/](?:Users|Documents|ProgramData|Windows)[\\/]/i,
+  /\\\\[^\\\s]+\\[^\s]+/,
+  /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i,
+  /https?:\/\//i,
+];
+const safePauseReasons = new Set([
+  "asset_root_state_missing", "balance_insufficient", "configured_disabled", "contract_blocked",
+  "contract_unverified", "gateway_balance_insufficient", "gateway_paused", "health_check_failed",
+  "model_disabled", "provider_unavailable", "quota_exhausted", "service_state_missing", "unknown",
+  "worker_degraded", "worker_state_missing", "worker_stopped",
+]);
+
+function iso(value: number | string | null | undefined) {
+  if (value === null || value === undefined) return null;
+  return typeof value === "number" ? new Date(value).toISOString() : value;
+}
+
+function tableExists(database: BetterSqlite3.Database, table: string) {
+  return Boolean(database.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = ?").get(table));
+}
+
+function safeService(
+  service_id: AdminService["service_id"],
+  status: AdminService["status"],
+  impact_scope: AdminService["impact_scope"],
+  configured: boolean,
+  checked_at: string | null,
+  pause_reason: string | null = null,
+): AdminService {
+  return { checked_at, configured, impact_scope, pause_reason, service_id, status };
+}
+
+function safeReason(value: unknown) {
+  return typeof value === "string" && safePauseReasons.has(value) ? value : null;
+}
+
+function serviceUsage(database: BetterSqlite3.Database, serviceName: string) {
+  if (!tableExists(database, "external_service_usage")) return undefined;
+  const columns = new Set((database.prepare("PRAGMA table_info(external_service_usage)").all() as Array<{ name: string }>).map((column) => column.name));
+  const serviceColumn = columns.has("service_name") ? "service_name" : columns.has("service_id") ? "service_id" : undefined;
+  if (!serviceColumn || !columns.has("service_status")) return undefined;
+  const row = database.prepare(`SELECT service_status, ${columns.has("checked_at") ? "checked_at" : "NULL AS checked_at"}, ${columns.has("pause_reason") ? "pause_reason" : "NULL AS pause_reason"} FROM external_service_usage WHERE ${serviceColumn} = ? ORDER BY rowid DESC LIMIT 1`).get(serviceName) as { service_status: string; checked_at: number | string | null; pause_reason: string | null } | undefined;
+  if (!row) return undefined;
+  const status = new Set<AdminService["status"]>(["active", "paused_quota", "paused_provider", "disabled"]).has(row.service_status as AdminService["status"])
+    ? row.service_status as AdminService["status"]
+    : "degraded";
+  return { status, checked_at: iso(row.checked_at), pause_reason: safeReason(row.pause_reason) };
+}
+
+function workerStatus(database: BetterSqlite3.Database) {
+  if (!tableExists(database, "worker_runtime_state")) return { status: "unavailable" as const, checked_at: null, pause_reason: "worker_state_missing" };
+  const row = database.prepare("SELECT status, reason, updated_at FROM worker_runtime_state WHERE singleton = 1").get() as { status: string; reason: string | null; updated_at: number | string | null } | undefined;
+  if (!row) return { status: "unavailable" as const, checked_at: null, pause_reason: "worker_state_missing" };
+  return {
+    status: row.status === "ready" ? "active" as const : row.status === "degraded" ? "degraded" as const : "unavailable" as const,
+    checked_at: iso(row.updated_at),
+    pause_reason: safeReason(row.reason),
+  };
+}
+
+export function createAdminServicesStorageProvider(input: {
+  database: BetterSqlite3.Database;
+  models?: ModelConfigurationService;
+  storage?: ManagedStorage;
+  assetRoot?: Pick<AdminService, "configured" | "status" | "checked_at" | "pause_reason">;
+  clock?: () => number;
+}): () => AdminServicesStorageResponse {
+  const clock = input.clock ?? Date.now;
+  return () => {
+    const generatedAt = new Date(clock()).toISOString();
+    const resend = serviceUsage(input.database, "resend");
+    const amap = serviceUsage(input.database, "amap");
+    const worker = workerStatus(input.database);
+    const modelRuntime = input.models?.read().models ?? [];
+    const unavailableModels = modelRuntime.filter((model) => !model.runtime_availability.available_for_new_jobs);
+    const gatewayReason = unavailableModels[0]?.runtime_availability.reason ?? null;
+    const storageState = input.storage?.getState();
+    const cleanupPendingCount = tableExists(input.database, "file_cleanup_queue")
+      ? (input.database.prepare("SELECT COUNT(*) AS count FROM file_cleanup_queue WHERE status IN ('pending', 'failed')").get() as { count: number }).count
+      : 0;
+    const services: AdminService[] = [
+      safeService("resend", resend?.status ?? "unavailable", "authentication", Boolean(resend), resend?.checked_at ?? null, resend?.pause_reason ?? "service_state_missing"),
+      safeService("amap", amap?.status ?? "unavailable", "location", Boolean(amap), amap?.checked_at ?? null, amap?.pause_reason ?? "service_state_missing"),
+      safeService("ai_gateway", unavailableModels.length > 0 ? "degraded" : modelRuntime.length > 0 ? "active" : "unavailable", "generation", modelRuntime.length > 0, generatedAt, safeReason(gatewayReason)),
+      safeService("worker", worker.status, "generation", worker.status !== "unavailable", worker.checked_at, worker.pause_reason),
+      safeService("api", "active", "api", true, generatedAt),
+      safeService(
+        "asset_root",
+        input.assetRoot?.status ?? "unavailable",
+        "storage",
+        input.assetRoot?.configured ?? false,
+        input.assetRoot?.checked_at ?? null,
+        input.assetRoot?.pause_reason ?? "asset_root_state_missing",
+      ),
+    ];
+    return assertSafeAdminServicesStorage({
+      generated_at: generatedAt,
+      services,
+      storage: {
+        capacity_notice_level: storageState?.capacity_notice_level ?? "normal",
+        cleanup_pending_count: cleanupPendingCount,
+        data_root_ref: "configured_local_data_root",
+        hard_limit_bytes: storageState?.hard_limit_bytes ?? 5_368_709_120,
+        last_measured_at: storageState?.measured_at ?? null,
+        managed_content_bytes: storageState?.managed_content_bytes ?? 0,
+        remeasurement_required: storageState?.storage_status === "unavailable",
+        status: storageState?.storage_status ?? "unavailable",
+        storage_backend: "local_filesystem",
+      },
+    });
+  };
+}
+
+function diagnosticText(input: AdminServicesStorageResponse, system: AdminDiagnosticsResponse["system"]) {
+  const lines = [
+    "Dada P0-A diagnostics",
+    `app_version=${system.app_version}`,
+    `api_status=${system.api_status}`,
+    `worker_status=${system.worker_status}`,
+    `storage_status=${input.storage.status}`,
+    `capacity_notice_level=${input.storage.capacity_notice_level}`,
+    `managed_content_bytes=${input.storage.managed_content_bytes}`,
+    `hard_limit_bytes=${input.storage.hard_limit_bytes}`,
+    `cleanup_pending_count=${input.storage.cleanup_pending_count}`,
+  ];
+  for (const service of input.services) lines.push(`service.${service.service_id}=${service.status}`);
+  return lines.join("\n");
+}
+
+export function createAdminDiagnosticsProvider(input: {
+  servicesStorage: () => AdminServicesStorageResponse;
+  browserSupportRelease?: BrowserSupportRelease;
+  appVersion?: string;
+  clock?: () => number;
+}): () => AdminDiagnosticsResponse {
+  const clock = input.clock ?? Date.now;
+  return () => {
+    const services = input.servicesStorage();
+    const system: AdminDiagnosticsResponse["system"] = {
+      api_status: "ready",
+      app_version: input.appVersion ?? input.browserSupportRelease?.appVersion ?? "0.0.0",
+      browser_support: (input.browserSupportRelease?.browsers ?? []).map((browser) => ({
+        brand: browser.brand,
+        major: Number.parseInt(browser.fullVersion.split(".")[0] ?? "0", 10),
+      })).filter((browser) => Number.isSafeInteger(browser.major) && browser.major > 0),
+      worker_status: services.services.find((service) => service.service_id === "worker")?.status === "active"
+        ? "ready"
+        : services.services.find((service) => service.service_id === "worker")?.status === "unavailable"
+          ? "unavailable"
+          : "degraded",
+    };
+    return assertSafeAdminDiagnostics({
+      generated_at: new Date(clock()).toISOString(),
+      diagnostic_text: diagnosticText(services, system),
+      services,
+      system,
+    });
+  };
+}
+
+export function assertSafeAdminServicesStorage(input: AdminServicesStorageResponse) {
+  const ids = input.services.map((service) => service.service_id);
+  if (ids.length !== adminServiceIds.length || new Set(ids).size !== adminServiceIds.length
+    || adminServiceIds.some((serviceId) => !ids.includes(serviceId))) {
+    throw new Error("admin_service_state_incomplete");
+  }
+  if (input.services.some((service) => service.pause_reason !== null && !safePauseReasons.has(service.pause_reason))) {
+    throw new Error("admin_services_redaction_failed");
+  }
+  if (forbiddenDiagnosticPatterns.some((pattern) => pattern.test(JSON.stringify(input)))) {
+    throw new Error("admin_services_redaction_failed");
+  }
+  return input;
+}
+
+export function assertSafeAdminDiagnostics(input: AdminDiagnosticsResponse) {
+  assertSafeAdminServicesStorage(input.services);
+  if (forbiddenDiagnosticPatterns.some((pattern) => pattern.test(input.diagnostic_text))) {
+    throw new Error("admin_diagnostics_redaction_failed");
+  }
+  return input;
+}

apps/api/src/app.ts

@@ -10,7 +10,9 @@ import {
   AccountProfileUpdateResponseSchema,
   AccountSettingsResponseSchema,
   AdminAuthenticatedUserSchema,
+  AdminDiagnosticsResponseSchema,
   AdminOverviewResponseSchema,
+  AdminServicesStorageResponseSchema,
   AdminCreditParamsSchema,
   AdminLoginCompleteRequestSchema,
   AdminLoginCompleteResponseSchema,
@@ -112,6 +114,8 @@ import {
   type AdminLoginCompleteRequest,
   type AdminLoginSendRequest,
   type AdminOverviewResponse,
+  type AdminDiagnosticsResponse,
+  type AdminServicesStorageResponse,
   type AccountDeletionCompleteRequest,
   type AccountProfileUpdateRequest,
   type AdminCreditParams,
@@ -180,6 +184,7 @@ import type { RecentAssetService } from "./recent-assets.js";
 import type { AmapAdapter } from "./amap-adapter.js";
 import { ModelConfigurationError } from "./model-configuration.js";
 import type { ModelConfigurationService } from "./model-configuration.js";
+import { assertSafeAdminDiagnostics, assertSafeAdminServicesStorage } from "./admin-state.js";
 
 const defaultBootstrap: BootstrapResponse = {
   app_version: "0.0.0",
@@ -194,7 +199,9 @@ const defaultBootstrap: BootstrapResponse = {
 };
 
 export interface CreateAppOptions {
+  adminDiagnostics?: () => AdminDiagnosticsResponse | Promise<AdminDiagnosticsResponse>;
   adminOverview?: () => AdminOverviewResponse | Promise<AdminOverviewResponse>;
+  adminServicesStorage?: () => AdminServicesStorageResponse | Promise<AdminServicesStorageResponse>;
   amap?: AmapAdapter;
   assetReleases?: AssetReleaseReader;
   bootstrap?: () => BootstrapResponse | Promise<BootstrapResponse>;
@@ -692,6 +699,8 @@ export async function createApp(options: CreateAppOptions = {}) {
     AdminLoginCompleteResponseSchema,
     AdminSessionResponseSchema,
     AdminOverviewResponseSchema,
+    AdminServicesStorageResponseSchema,
+    AdminDiagnosticsResponseSchema,
     CreditSummarySchema,
     CreditEntryTypeSchema,
     CreditEntryStatusSchema,
@@ -1236,6 +1245,60 @@ export async function createApp(options: CreateAppOptions = {}) {
     },
   );
 
+  app.get(
+    "/api/v1/admin/services-storage",
+    {
+      schema: {
+        operationId: "getAdminServicesStorage",
+        response: {
+          200: Type.Ref(AdminServicesStorageResponseSchema),
+          401: Type.Ref(ErrorEnvelopeSchema),
+          503: Type.Null(),
+        },
+        tags: ["Admin Operations"],
+      },
+    },
+    async (request, reply) => {
+      if (!options.registration) return reply.code(503).send(null);
+      const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName);
+      const session = token ? options.registration.readAdminSession(token) : undefined;
+      if (!session) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id }));
+      if (!options.adminServicesStorage) return reply.code(503).send(null);
+      try {
+        return assertSafeAdminServicesStorage(await options.adminServicesStorage());
+      } catch {
+        return reply.code(503).send(null);
+      }
+    },
+  );
+
+  app.get(
+    "/api/v1/admin/diagnostics",
+    {
+      schema: {
+        operationId: "getAdminDiagnostics",
+        response: {
+          200: Type.Ref(AdminDiagnosticsResponseSchema),
+          401: Type.Ref(ErrorEnvelopeSchema),
+          503: Type.Null(),
+        },
+        tags: ["Admin Operations"],
+      },
+    },
+    async (request, reply) => {
+      if (!options.registration) return reply.code(503).send(null);
+      const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName);
+      const session = token ? options.registration.readAdminSession(token) : undefined;
+      if (!session) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id }));
+      if (!options.adminDiagnostics) return reply.code(503).send(null);
+      try {
+        return assertSafeAdminDiagnostics(await options.adminDiagnostics());
+      } catch {
+        return reply.code(503).send(null);
+      }
+    },
+  );
+
   app.post(
     "/api/v1/auth/login/send",
     {

apps/api/src/main.ts

@@ -18,6 +18,7 @@ import { StructuredJsonlLogger } from "./structured-log.js";
 import { attachApiSupervisorControl, initializeApiCredentialClients, receiveApiCredentials } from "./supervisor-channel.js";
 import { ModelConfigurationService } from "./model-configuration.js";
 import { MockAmapAdapter } from "./amap-adapter.js";
+import { createAdminDiagnosticsProvider, createAdminServicesStorageProvider } from "./admin-state.js";
 
 const credentialChannelEnabled = process.argv.includes("--dada-credential-stdin");
 let registration: RegistrationService | undefined;
@@ -70,7 +71,22 @@ if (credentialChannelEnabled) {
 }
 
 const browserSupportRelease = readBrowserSupportRelease(resolve("RELEASE.json"));
+const adminServicesStorage = registration
+  ? createAdminServicesStorageProvider({
+    database: registration.database,
+    ...(models ? { models } : {}),
+    ...(storage ? { storage } : {}),
+  })
+  : undefined;
+const adminDiagnostics = adminServicesStorage
+  ? createAdminDiagnosticsProvider({
+    ...(browserSupportRelease ? { browserSupportRelease, appVersion: browserSupportRelease.appVersion } : {}),
+    servicesStorage: adminServicesStorage,
+  })
+  : undefined;
 const app = await createApp({
+  ...(adminServicesStorage ? { adminServicesStorage } : {}),
+  ...(adminDiagnostics ? { adminDiagnostics } : {}),
   amap: new MockAmapAdapter(),
   ...(browserSupportRelease ? { browserSupportRelease } : {}),
   ...(credits ? { credits } : {}),