From e12a83d6d3c7060d7ca5cc75a4e8829aac365535 Mon Sep 17 00:00:00 2001 From: suyx Date: Mon, 27 Jul 2026 19:10:57 +0800 Subject: [PATCH] feat: complete TASK-WP0-05 storage capacity --- apps/api/src/managed-storage.ts | 614 ++++++++++++++++++ apps/api/src/storage-policy.ts | 81 +++ apps/worker/src/storage-maintenance.ts | 10 + package.json | 8 +- scripts/run-wp0-05-validation.mjs | 119 ++++ tests/api/wp0-05-storage-capacity.test.ts | 30 + tests/e2e/fixtures/storage-capacity.html | 17 + tests/e2e/fixtures/storage-capacity.ts | 17 + tests/e2e/storage-capacity.spec.ts | 39 ++ .../wp0-05-managed-storage.test.ts | 265 ++++++++ tests/unit/wp0-05-storage-policy.test.ts | 80 +++ .../worker/wp0-05-storage-maintenance.test.ts | 17 + 12 files changed, 1294 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/managed-storage.ts create mode 100644 apps/api/src/storage-policy.ts create mode 100644 apps/worker/src/storage-maintenance.ts create mode 100644 scripts/run-wp0-05-validation.mjs create mode 100644 tests/api/wp0-05-storage-capacity.test.ts create mode 100644 tests/e2e/fixtures/storage-capacity.html create mode 100644 tests/e2e/fixtures/storage-capacity.ts create mode 100644 tests/e2e/storage-capacity.spec.ts create mode 100644 tests/integration/wp0-05-managed-storage.test.ts create mode 100644 tests/unit/wp0-05-storage-policy.test.ts create mode 100644 tests/worker/wp0-05-storage-maintenance.test.ts diff --git a/apps/api/src/managed-storage.ts b/apps/api/src/managed-storage.ts new file mode 100644 index 0000000..6dd08b7 --- /dev/null +++ b/apps/api/src/managed-storage.ts @@ -0,0 +1,614 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + createWriteStream, + existsSync, + mkdirSync, + readdirSync, + renameSync, + rmSync, + statSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { basename, dirname, extname, join, relative, resolve } from "node:path"; +import { Readable, Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +import type BetterSqlite3 from "better-sqlite3"; + +import { resolvePathWithinRoot } from "./local-data-root.js"; +import { + HARD_LIMIT_BYTES, + classifyCapacity, + decideStorageAction, + storageCapacityErrorDetails, + type StorageAction, +} from "./storage-policy.js"; + +const require = createRequire(import.meta.url); +const Database = require("better-sqlite3") as typeof BetterSqlite3; +const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const safeFileNamePattern = /^[A-Za-z0-9][A-Za-z0-9._-]{0,159}$/; + +export { HARD_LIMIT_BYTES } from "./storage-policy.js"; + +export type ManagedFileKind = + | "reference" + | "generated" + | "export" + | "derived" + | "sticker_original" + | "sticker_thumbnail"; +export type CommitFailurePoint = "after_staging" | "after_rename" | "after_database_commit"; + +interface StorageStateRow { + active_storage_reservations_bytes: number; + capacity_notice_level: "normal" | "warning" | "critical"; + data_root_ref: "configured_local_data_root"; + hard_limit_bytes: number; + managed_content_bytes: number; + measured_at: string; + storage_backend: "local_data_root"; + storage_status: "active" | "full" | "unavailable"; + version: number; +} + +interface ManagedFileRow { + byte_size: number; + file_id: string; + file_kind: ManagedFileKind; + relative_path: string; + status: "committed" | "purged"; +} + +interface CleanupQueueRow { + byte_size: number; + cleanup_id: string; + counts_toward_managed: 0 | 1; + managed_file_id: string | null; + relative_path: string; +} + +export class StorageCapacityError extends Error { + readonly code = "STORAGE_CAPACITY_EXCEEDED"; + readonly httpStatus = 507; + readonly details: ReturnType; + + constructor(input: { activeReservationBytes: number; managedContentBytes: number; projectedWriteBytes: number }) { + super("STORAGE_CAPACITY_EXCEEDED"); + this.details = storageCapacityErrorDetails(input); + } +} + +export class StorageUnavailableError extends Error { + readonly code = "storage_unavailable"; + + constructor() { + super("storage_unavailable"); + } +} + +class InjectedCrashError extends Error {} + +function now() { + return new Date().toISOString(); +} + +function validatePositiveBytes(value: number, name: string) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name}_invalid`); +} + +function sniffMime(prefix: Buffer) { + if (prefix.length >= 8 && prefix.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) { + return "image/png"; + } + if (prefix.length >= 3 && prefix[0] === 0xff && prefix[1] === 0xd8 && prefix[2] === 0xff) return "image/jpeg"; + return "application/octet-stream"; +} + +function listFiles(root: string): string[] { + if (!existsSync(root)) return []; + const files: string[] = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + const child = join(root, entry.name); + if (entry.isDirectory()) files.push(...listFiles(child)); + else if (entry.isFile()) files.push(child); + } + return files; +} + +export interface CommitStreamInput { + content: Readable; + expectedMimeType: "image/png" | "image/jpeg" | "application/octet-stream"; + expectedSha256?: string; + failurePoint?: CommitFailurePoint; + fileKind: ManagedFileKind; + fileName: string; + operationId: string; + ownerRef: string; + projectedWriteBytes: number; +} + +export class ManagedStorage { + readonly dataRoot: string; + readonly databasePath: string; + private readonly database: BetterSqlite3.Database; + private dataRootWritable = true; + private diskSpaceAvailable = true; + private sqliteWritable = true; + private measurementBaselineBytes = 0; + private recoveryRequiresRemeasure = false; + + constructor(input: { dataRoot: string; databasePath: string }) { + this.dataRoot = resolve(input.dataRoot); + this.databasePath = resolve(input.databasePath); + if (!this.databasePath.startsWith(`${this.dataRoot}${process.platform === "win32" ? "\\" : "/"}`)) { + throw new Error("database_outside_data_root"); + } + mkdirSync(dirname(this.databasePath), { recursive: true }); + this.database = new Database(this.databasePath); + this.database.pragma("journal_mode = WAL"); + this.database.pragma("foreign_keys = ON"); + this.database.pragma("busy_timeout = 5000"); + this.migrate(); + const state = this.database.prepare("SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1").get() as { managed_content_bytes: number }; + this.measurementBaselineBytes = Math.max(0, state.managed_content_bytes - this.physicalManagedBytes()); + } + + private migrate() { + this.database.exec(` + CREATE TABLE IF NOT EXISTS local_backend_storage_state ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + storage_backend TEXT NOT NULL CHECK (storage_backend = 'local_data_root'), + data_root_ref TEXT NOT NULL CHECK (data_root_ref = 'configured_local_data_root'), + hard_limit_bytes INTEGER NOT NULL, + managed_content_bytes INTEGER NOT NULL CHECK (managed_content_bytes >= 0), + capacity_notice_level TEXT NOT NULL CHECK (capacity_notice_level IN ('normal', 'warning', 'critical')), + storage_status TEXT NOT NULL CHECK (storage_status IN ('active', 'full', 'unavailable')), + measured_at TEXT NOT NULL, + version INTEGER NOT NULL CHECK (version >= 1) + ); + CREATE TABLE IF NOT EXISTS storage_reservations ( + reservation_id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL UNIQUE, + projected_bytes INTEGER NOT NULL CHECK (projected_bytes > 0), + status TEXT NOT NULL CHECK (status IN ('active', 'consumed', 'released')), + created_at TEXT NOT NULL, + resolved_at TEXT + ); + CREATE TABLE IF NOT EXISTS managed_files ( + file_id TEXT PRIMARY KEY, + file_kind TEXT NOT NULL CHECK (file_kind IN ('reference', 'generated', 'export', 'derived', 'sticker_original', 'sticker_thumbnail')), + relative_path TEXT NOT NULL UNIQUE, + byte_size INTEGER NOT NULL CHECK (byte_size > 0), + mime_type TEXT NOT NULL, + sha256 TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('committed', 'purged')), + created_at TEXT NOT NULL, + purged_at TEXT + ); + CREATE TABLE IF NOT EXISTS file_cleanup_queue ( + cleanup_id TEXT PRIMARY KEY, + managed_file_id TEXT, + relative_path TEXT NOT NULL UNIQUE, + byte_size INTEGER NOT NULL CHECK (byte_size >= 0), + counts_toward_managed INTEGER NOT NULL CHECK (counts_toward_managed IN (0, 1)), + reason TEXT NOT NULL CHECK (reason IN ('compensation', 'purge')), + status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'failed')), + created_at TEXT NOT NULL, + completed_at TEXT, + last_error TEXT + ); + CREATE TABLE IF NOT EXISTS project_asset_refs ( + reference_id TEXT PRIMARY KEY, + managed_file_id TEXT NOT NULL, + reference_type TEXT NOT NULL CHECK (reference_type IN ('project', 'release')), + created_at TEXT NOT NULL, + FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id) + ); + CREATE TABLE IF NOT EXISTS asset_cleanup_requests ( + request_id TEXT PRIMARY KEY, + status TEXT NOT NULL CHECK (status IN ('pending_confirmation', 'denied', 'queued', 'completed')), + created_at TEXT NOT NULL, + confirmed_at TEXT + ); + CREATE TABLE IF NOT EXISTS asset_cleanup_request_items ( + request_id TEXT NOT NULL, + managed_file_id TEXT NOT NULL, + PRIMARY KEY (request_id, managed_file_id), + FOREIGN KEY (request_id) REFERENCES asset_cleanup_requests(request_id), + FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id) + ); + CREATE TABLE IF NOT EXISTS admin_operation_logs ( + log_id TEXT PRIMARY KEY, + operation TEXT NOT NULL, + outcome TEXT NOT NULL, + target_ref TEXT NOT NULL, + created_at TEXT NOT NULL + ); + `); + const initial = classifyCapacity(0, 0); + this.database.prepare(` + INSERT OR IGNORE INTO local_backend_storage_state + (singleton, storage_backend, data_root_ref, hard_limit_bytes, managed_content_bytes, capacity_notice_level, storage_status, measured_at, version) + VALUES (1, 'local_data_root', 'configured_local_data_root', ?, 0, ?, ?, ?, 1) + `).run(HARD_LIMIT_BYTES, initial.capacity_notice_level, initial.storage_status, now()); + } + + close() { + this.database.close(); + } + + getState(): StorageStateRow { + const state = this.database.prepare("SELECT * FROM local_backend_storage_state WHERE singleton = 1").get() as Omit; + const withReservations = { ...state, active_storage_reservations_bytes: this.activeReservationBytes() }; + if (!this.dataRootWritable || !this.diskSpaceAvailable || !this.sqliteWritable) { + return { ...withReservations, storage_status: "unavailable" }; + } + return withReservations; + } + + private activeReservationBytes(excludingOperationId?: string) { + const row = this.database.prepare(` + SELECT COALESCE(SUM(projected_bytes), 0) AS bytes + FROM storage_reservations + WHERE status = 'active' AND (? IS NULL OR operation_id <> ?) + `).get(excludingOperationId ?? null, excludingOperationId ?? null) as { bytes: number }; + return row.bytes; + } + + private refreshState() { + if (!this.sqliteWritable) return; + const state = this.database.prepare("SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1").get() as { managed_content_bytes: number }; + const classification = classifyCapacity(state.managed_content_bytes, this.activeReservationBytes()); + const storageStatus = !this.dataRootWritable || !this.diskSpaceAvailable || this.recoveryRequiresRemeasure + ? "unavailable" + : classification.storage_status; + this.database.prepare(` + UPDATE local_backend_storage_state + SET capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1 + WHERE singleton = 1 + `).run(classification.capacity_notice_level, storageStatus, now()); + } + + applyControlledMeasurement(bytes: number) { + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error("managed_content_bytes_invalid"); + this.measurementBaselineBytes = Math.max(0, bytes - this.physicalManagedBytes()); + this.recoveryRequiresRemeasure = false; + const classification = classifyCapacity(bytes, this.activeReservationBytes()); + this.database.prepare(` + UPDATE local_backend_storage_state + SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1 + WHERE singleton = 1 + `).run(bytes, classification.capacity_notice_level, classification.storage_status, now()); + return this.getState(); + } + + setAvailability(input: { dataRootWritable: boolean; diskSpaceAvailable: boolean; sqliteWritable: boolean }) { + this.dataRootWritable = input.dataRootWritable; + this.diskSpaceAvailable = input.diskSpaceAvailable; + this.sqliteWritable = input.sqliteWritable; + if (!input.dataRootWritable || !input.diskSpaceAvailable || !input.sqliteWritable) this.recoveryRequiresRemeasure = true; + this.refreshState(); + } + + private physicalManagedBytes() { + return ["content", "managed-assets", "derived-assets"] + .flatMap((root) => listFiles(resolvePathWithinRoot(this.dataRoot, root))) + .reduce((sum, path) => sum + statSync(path).size, 0); + } + + private recordPhysicalMeasurement() { + const bytes = this.measurementBaselineBytes + this.physicalManagedBytes(); + this.recoveryRequiresRemeasure = false; + const classification = classifyCapacity(bytes, this.activeReservationBytes()); + this.database.prepare(` + UPDATE local_backend_storage_state + SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1 + WHERE singleton = 1 + `).run(bytes, classification.capacity_notice_level, classification.storage_status, now()); + } + + inspectAction(action: StorageAction) { + return decideStorageAction(this.getState().storage_status, this.sqliteWritable, action); + } + + private assertWritable() { + if (this.inspectAction("binary_write") === "reject_unavailable") throw new StorageUnavailableError(); + } + + private reserve(operationId: string, projectedWriteBytes: number) { + this.assertWritable(); + if (!uuidPattern.test(operationId)) throw new Error("operation_id_invalid"); + validatePositiveBytes(projectedWriteBytes, "projected_write_bytes"); + const transaction = this.database.transaction(() => { + const state = this.getState(); + const activeReservationBytes = this.activeReservationBytes(); + if (state.managed_content_bytes + activeReservationBytes + projectedWriteBytes > HARD_LIMIT_BYTES) { + throw new StorageCapacityError({ activeReservationBytes, managedContentBytes: state.managed_content_bytes, projectedWriteBytes }); + } + const reservationId = randomUUID(); + this.database.prepare(` + INSERT INTO storage_reservations (reservation_id, operation_id, projected_bytes, status, created_at) + VALUES (?, ?, ?, 'active', ?) + `).run(reservationId, operationId, projectedWriteBytes, now()); + this.refreshState(); + return reservationId; + }); + return transaction(); + } + + private releaseReservation(operationId: string) { + this.database.prepare(` + UPDATE storage_reservations SET status = 'released', resolved_at = ? + WHERE operation_id = ? AND status = 'active' + `).run(now(), operationId); + this.refreshState(); + } + + private destination(input: CommitStreamInput, fileId: string) { + if (!safeFileNamePattern.test(input.fileName) || basename(input.fileName) !== input.fileName || input.fileName.includes("..")) { + throw new Error("file_name_invalid"); + } + if (!uuidPattern.test(input.ownerRef)) throw new Error("owner_ref_invalid"); + const group = input.fileKind === "reference" + ? "content/references" + : input.fileKind === "generated" + ? "content/generated" + : input.fileKind === "export" + ? "content/exports" + : input.fileKind === "derived" + ? "derived-assets" + : input.fileKind === "sticker_original" + ? "managed-assets/stickers/original" + : "managed-assets/stickers/thumbnail"; + const extension = extname(input.fileName).toLowerCase(); + const relativePath = `${group}/${input.ownerRef}/${fileId}${extension}`; + return { absolutePath: resolvePathWithinRoot(this.dataRoot, relativePath), relativePath }; + } + + async commitStream(input: CommitStreamInput) { + const fileId = randomUUID(); + const destination = this.destination(input, fileId); + this.reserve(input.operationId, input.projectedWriteBytes); + const stagingDirectory = resolvePathWithinRoot(this.dataRoot, `staging/${input.operationId}`); + const stagingPath = resolvePathWithinRoot(this.dataRoot, `staging/${input.operationId}/payload.tmp`); + let renamed = false; + try { + mkdirSync(stagingDirectory, { recursive: true }); + const hash = createHash("sha256"); + let byteSize = 0; + let prefix = Buffer.alloc(0); + const inspect = new Transform({ + transform(chunk: Buffer | string, encoding, callback) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding); + byteSize += bytes.byteLength; + hash.update(bytes); + if (prefix.byteLength < 16) prefix = Buffer.concat([prefix, bytes.subarray(0, 16 - prefix.byteLength)]); + callback(null, bytes); + }, + }); + await pipeline(input.content, inspect, createWriteStream(stagingPath, { flags: "wx" })); + validatePositiveBytes(byteSize, "actual_write_bytes"); + const sha256 = hash.digest("hex"); + if (input.expectedSha256 && sha256.toLowerCase() !== input.expectedSha256.toLowerCase()) throw new Error("content_hash_invalid"); + if (sniffMime(prefix) !== input.expectedMimeType) throw new Error("content_mime_invalid"); + const state = this.getState(); + const otherReservations = this.activeReservationBytes(input.operationId); + if (state.managed_content_bytes + otherReservations + byteSize > HARD_LIMIT_BYTES) { + throw new StorageCapacityError({ activeReservationBytes: otherReservations, managedContentBytes: state.managed_content_bytes, projectedWriteBytes: byteSize }); + } + this.database.prepare(`UPDATE storage_reservations SET projected_bytes = ? WHERE operation_id = ? AND status = 'active'`).run(byteSize, input.operationId); + this.refreshState(); + if (input.failurePoint === "after_staging") throw new InjectedCrashError("injected_crash_after_staging"); + mkdirSync(dirname(destination.absolutePath), { recursive: true }); + renameSync(stagingPath, destination.absolutePath); + renamed = true; + rmSync(stagingDirectory, { force: true, recursive: true }); + if (input.failurePoint === "after_rename") throw new InjectedCrashError("injected_crash_after_rename"); + + const commit = this.database.transaction(() => { + this.database.prepare(` + INSERT INTO managed_files (file_id, file_kind, relative_path, byte_size, mime_type, sha256, status, created_at) + VALUES (?, ?, ?, ?, ?, ?, 'committed', ?) + `).run(fileId, input.fileKind, destination.relativePath, byteSize, input.expectedMimeType, sha256, now()); + this.database.prepare(` + UPDATE local_backend_storage_state SET managed_content_bytes = managed_content_bytes + ? WHERE singleton = 1 + `).run(byteSize); + this.database.prepare(` + UPDATE storage_reservations SET status = 'consumed', resolved_at = ? WHERE operation_id = ? AND status = 'active' + `).run(now(), input.operationId); + this.refreshState(); + }); + commit(); + if (input.failurePoint === "after_database_commit") throw new InjectedCrashError("injected_crash_after_database_commit"); + return { bytes: byteSize, file_id: fileId, relative_path: destination.relativePath, sha256 }; + } catch (error) { + if (error instanceof InjectedCrashError) throw error; + if (renamed && existsSync(destination.absolutePath)) { + try { + this.queueCompensation(destination.relativePath, statSync(destination.absolutePath).size); + } catch { + // Startup reconciliation is the final fallback if SQLite itself is unavailable. + } + } else { + rmSync(stagingDirectory, { force: true, recursive: true }); + } + this.releaseReservation(input.operationId); + throw error; + } + } + + async commitBufferFixture(fileKind: ManagedFileKind, fileName: string, bytes: Buffer) { + return this.commitStream({ + content: Readable.from(bytes), + expectedMimeType: sniffMime(bytes.subarray(0, 16)), + fileKind, + fileName, + operationId: randomUUID(), + ownerRef: randomUUID(), + projectedWriteBytes: bytes.byteLength, + }); + } + + private queueCompensation(relativePath: string, byteSize: number) { + this.database.prepare(` + INSERT OR IGNORE INTO file_cleanup_queue + (cleanup_id, managed_file_id, relative_path, byte_size, counts_toward_managed, reason, status, created_at) + VALUES (?, NULL, ?, ?, 0, 'compensation', 'pending', ?) + `).run(randomUUID(), relativePath, byteSize, now()); + } + + async reconcileStartup() { + let releasedReservations = 0; + const active = this.database.prepare("SELECT operation_id FROM storage_reservations WHERE status = 'active'").all() as Array<{ operation_id: string }>; + for (const reservation of active) { + this.database.prepare(`UPDATE storage_reservations SET status = 'released', resolved_at = ? WHERE operation_id = ?`).run(now(), reservation.operation_id); + releasedReservations += 1; + } + const stagingRoot = resolvePathWithinRoot(this.dataRoot, "staging"); + for (const entry of existsSync(stagingRoot) ? readdirSync(stagingRoot) : []) { + rmSync(join(stagingRoot, entry), { force: true, recursive: true }); + } + + const managedPaths = new Set((this.database.prepare("SELECT relative_path FROM managed_files").all() as Array<{ relative_path: string }>).map((row) => row.relative_path)); + const queuedPaths = new Set((this.database.prepare("SELECT relative_path FROM file_cleanup_queue WHERE status IN ('pending', 'failed')").all() as Array<{ relative_path: string }>).map((row) => row.relative_path)); + let orphaned = 0; + for (const root of ["content", "managed-assets", "derived-assets"]) { + for (const path of listFiles(resolvePathWithinRoot(this.dataRoot, root))) { + const relativePath = relative(this.dataRoot, path).replaceAll("\\", "/"); + if (!managedPaths.has(relativePath) && !queuedPaths.has(relativePath)) { + this.queueCompensation(relativePath, statSync(path).size); + orphaned += 1; + } + } + } + const missingManagedFile = (this.database.prepare("SELECT relative_path FROM managed_files WHERE status = 'committed'").all() as Array<{ relative_path: string }>).some( + (row) => !existsSync(resolvePathWithinRoot(this.dataRoot, row.relative_path)), + ); + if (missingManagedFile) this.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true }); + else this.recordPhysicalMeasurement(); + return { orphaned, released_reservations: releasedReservations }; + } + + inspectCounts() { + const count = (table: string, where = "1 = 1") => (this.database.prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${where}`).get() as { count: number }).count; + return { + active_reservations: count("storage_reservations", "status = 'active'"), + managed_files: count("managed_files"), + pending_cleanup: count("file_cleanup_queue", "status = 'pending'"), + cleanup_requests: count("asset_cleanup_requests"), + admin_logs: count("admin_operation_logs"), + }; + } + + resolveManagedFile(fileId: string) { + const row = this.database.prepare("SELECT relative_path FROM managed_files WHERE file_id = ? AND status = 'committed'").get(fileId) as { relative_path: string } | undefined; + return row ? resolvePathWithinRoot(this.dataRoot, row.relative_path) : undefined; + } + + addAssetReference(fileId: string, referenceType: "project" | "release") { + if (this.inspectAction("project_json_write") !== "allow") throw new StorageUnavailableError(); + this.database.prepare(`INSERT INTO project_asset_refs (reference_id, managed_file_id, reference_type, created_at) VALUES (?, ?, ?, ?)`) + .run(randomUUID(), fileId, referenceType, now()); + } + + removeAssetReferences(fileId: string) { + if (this.inspectAction("project_json_write") !== "allow") throw new StorageUnavailableError(); + this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(fileId); + } + + createCleanupIntent(fileIds: string[]) { + if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted"); + if (fileIds.length === 0 || new Set(fileIds).size !== fileIds.length) throw new Error("cleanup_candidates_invalid"); + const requestId = randomUUID(); + const transaction = this.database.transaction(() => { + for (const fileId of fileIds) { + const candidate = this.database.prepare(` + SELECT file_id FROM managed_files + WHERE file_id = ? AND status = 'committed' AND file_kind IN ('sticker_original', 'sticker_thumbnail') + `).get(fileId); + if (!candidate) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + } + this.database.prepare("INSERT INTO asset_cleanup_requests (request_id, status, created_at) VALUES (?, 'pending_confirmation', ?)").run(requestId, now()); + const insert = this.database.prepare("INSERT INTO asset_cleanup_request_items (request_id, managed_file_id) VALUES (?, ?)"); + for (const fileId of fileIds) insert.run(requestId, fileId); + }); + transaction(); + return { request_id: requestId }; + } + + confirmCleanupIntent(requestId: string) { + if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted"); + const transaction = this.database.transaction(() => { + const request = this.database.prepare("SELECT status FROM asset_cleanup_requests WHERE request_id = ?").get(requestId) as { status: string } | undefined; + if (request?.status !== "pending_confirmation") throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + const files = this.database.prepare(` + SELECT mf.* FROM asset_cleanup_request_items items + JOIN managed_files mf ON mf.file_id = items.managed_file_id + WHERE items.request_id = ? + `).all(requestId) as ManagedFileRow[]; + const conflict = files.some((file) => { + if (file.status !== "committed" || (file.file_kind !== "sticker_original" && file.file_kind !== "sticker_thumbnail")) return true; + const row = this.database.prepare("SELECT COUNT(*) AS count FROM project_asset_refs WHERE managed_file_id = ?").get(file.file_id) as { count: number }; + return row.count > 0; + }); + if (conflict) { + this.database.prepare("UPDATE asset_cleanup_requests SET status = 'denied', confirmed_at = ? WHERE request_id = ?").run(now(), requestId); + this.database.prepare("INSERT INTO admin_operation_logs (log_id, operation, outcome, target_ref, created_at) VALUES (?, 'asset_cleanup', 'denied_reference_conflict', ?, ?)") + .run(randomUUID(), requestId, now()); + return false; + } + for (const file of files) { + this.database.prepare("UPDATE managed_files SET status = 'purged', purged_at = ? WHERE file_id = ?").run(now(), file.file_id); + this.database.prepare(` + INSERT INTO file_cleanup_queue + (cleanup_id, managed_file_id, relative_path, byte_size, counts_toward_managed, reason, status, created_at) + VALUES (?, ?, ?, ?, 1, 'purge', 'pending', ?) + `).run(randomUUID(), file.file_id, file.relative_path, file.byte_size, now()); + } + this.database.prepare("UPDATE asset_cleanup_requests SET status = 'queued', confirmed_at = ? WHERE request_id = ?").run(now(), requestId); + this.database.prepare("INSERT INTO admin_operation_logs (log_id, operation, outcome, target_ref, created_at) VALUES (?, 'asset_cleanup', 'queued', ?, ?)") + .run(randomUUID(), requestId, now()); + return true; + }); + if (!transaction()) throw new Error("ASSET_HISTORY_REFERENCE_CONFLICT"); + } + + async processCleanupQueue() { + if (this.inspectAction("explicit_cleanup") !== "allow" || !this.dataRootWritable || !this.diskSpaceAvailable) { + return { completed: 0, failed: 0 }; + } + const rows = this.database.prepare("SELECT * FROM file_cleanup_queue WHERE status IN ('pending', 'failed') ORDER BY created_at").all() as CleanupQueueRow[]; + let completed = 0; + let failed = 0; + for (const row of rows) { + try { + const path = resolvePathWithinRoot(this.dataRoot, row.relative_path); + rmSync(path, { force: true }); + const finish = this.database.transaction(() => { + if (row.counts_toward_managed === 1 && row.managed_file_id) { + this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(row.managed_file_id); + const requests = this.database.prepare("SELECT request_id FROM asset_cleanup_request_items WHERE managed_file_id = ?").all(row.managed_file_id) as Array<{ request_id: string }>; + this.database.prepare("DELETE FROM asset_cleanup_request_items WHERE managed_file_id = ?").run(row.managed_file_id); + this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id); + for (const request of requests) { + const pendingItems = this.database.prepare("SELECT COUNT(*) AS count FROM asset_cleanup_request_items WHERE request_id = ?").get(request.request_id) as { count: number }; + if (pendingItems.count === 0) this.database.prepare("UPDATE asset_cleanup_requests SET status = 'completed' WHERE request_id = ? AND status = 'queued'").run(request.request_id); + } + } + this.database.prepare("UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL WHERE cleanup_id = ?").run(now(), row.cleanup_id); + this.database.prepare("INSERT INTO admin_operation_logs (log_id, operation, outcome, target_ref, created_at) VALUES (?, 'physical_file_cleanup', 'completed', ?, ?)") + .run(randomUUID(), row.cleanup_id, now()); + this.recordPhysicalMeasurement(); + }); + finish(); + completed += 1; + } catch (error) { + this.database.prepare("UPDATE file_cleanup_queue SET status = 'failed', last_error = ? WHERE cleanup_id = ?") + .run(error instanceof Error ? error.message : "cleanup_failed", row.cleanup_id); + failed += 1; + } + } + return { completed, failed }; + } +} diff --git a/apps/api/src/storage-policy.ts b/apps/api/src/storage-policy.ts new file mode 100644 index 0000000..c90af64 --- /dev/null +++ b/apps/api/src/storage-policy.ts @@ -0,0 +1,81 @@ +export const WARNING_LIMIT_BYTES = 4_294_967_296; +export const CRITICAL_LIMIT_BYTES = 4_831_838_208; +export const HARD_LIMIT_BYTES = 5_368_709_120; +export const CAPACITY_COUNTED_FILE_KINDS = [ + "reference", + "generated", + "export", + "derived", + "sticker_original", + "sticker_thumbnail", +] as const; +export const CAPACITY_EXCLUDED_STORAGE = [ + "read_only_assets", + "application_files", + "database", + "logs", + "audit", + "browser_cache", + "user_downloads", +] as const; + +export type CapacityNoticeLevel = "normal" | "warning" | "critical"; +export type StorageStatus = "active" | "full" | "unavailable"; +export type StorageAction = + | "project_json_write" + | "binary_write" + | "ai_call" + | "latest_export_write" + | "read" + | "download" + | "client_only_download" + | "permanent_delete" + | "explicit_cleanup"; +export type StorageActionDecision = "allow" | "reject_capacity" | "reject_unavailable" | "reject_uncommitted"; + +export function classifyCapacity(managedContentBytes: number, activeReservationBytes: number) { + if (!Number.isSafeInteger(managedContentBytes) || managedContentBytes < 0) throw new Error("managed_content_bytes_invalid"); + if (!Number.isSafeInteger(activeReservationBytes) || activeReservationBytes < 0) throw new Error("active_reservation_bytes_invalid"); + const capacity_notice_level: CapacityNoticeLevel = managedContentBytes < WARNING_LIMIT_BYTES + ? "normal" + : managedContentBytes < CRITICAL_LIMIT_BYTES + ? "warning" + : "critical"; + const storage_status: StorageStatus = managedContentBytes + activeReservationBytes >= HARD_LIMIT_BYTES + ? "full" + : "active"; + return { capacity_notice_level, storage_status }; +} + +export function decideStorageAction( + status: StorageStatus, + sqliteWritable: boolean, + action: StorageAction, +): StorageActionDecision { + if (status === "unavailable") { + if (action === "read" || action === "download" || action === "client_only_download") return "allow"; + if (action === "permanent_delete" || action === "explicit_cleanup") { + return sqliteWritable ? "allow" : "reject_uncommitted"; + } + return "reject_unavailable"; + } + if (status === "full" && (action === "binary_write" || action === "ai_call" || action === "latest_export_write")) { + return "reject_capacity"; + } + return "allow"; +} + +export function storageCapacityErrorDetails(input: { + activeReservationBytes: number; + managedContentBytes: number; + projectedWriteBytes: number; +}) { + const { capacity_notice_level, storage_status } = classifyCapacity( + input.managedContentBytes, + input.activeReservationBytes, + ); + return { + capacity_status: storage_status === "full" ? "full" as const : capacity_notice_level, + remaining_bytes: Math.max(0, HARD_LIMIT_BYTES - input.managedContentBytes - input.activeReservationBytes), + }; +} diff --git a/apps/worker/src/storage-maintenance.ts b/apps/worker/src/storage-maintenance.ts new file mode 100644 index 0000000..b7ec431 --- /dev/null +++ b/apps/worker/src/storage-maintenance.ts @@ -0,0 +1,10 @@ +export interface StorageMaintenanceTarget { + processCleanupQueue(): Promise<{ completed: number; failed: number }>; + reconcileStartup(): Promise<{ orphaned: number; released_reservations: number }>; +} + +export async function runStorageMaintenance(target: StorageMaintenanceTarget) { + const reconciliation = await target.reconcileStartup(); + const cleanup = await target.processCleanupQueue(); + return { cleanup, reconciliation }; +} diff --git a/package.json b/package.json index 9c2684b..2d0c936 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,8 @@ "test:unit": "pnpm --filter @dada/shared-contracts build && pnpm run test:unit:contract && vitest run tests/unit", "test:integration": "vitest run tests/integration", "test:api": "pnpm check:openapi && vitest run tests/api", - "test:worker": "pnpm --filter @dada/worker build && node scripts/worker-smoke.mjs", - "test:e2e": "pnpm check:openapi && playwright test tests/e2e/event-sync.spec.ts tests/e2e/support-gate.spec.ts tests/e2e/local-data-boundary.spec.ts --config playwright.config.ts", + "test:worker": "pnpm --filter @dada/worker build && node scripts/worker-smoke.mjs && vitest run tests/worker", + "test:e2e": "pnpm check:openapi && playwright test tests/e2e/event-sync.spec.ts tests/e2e/support-gate.spec.ts tests/e2e/local-data-boundary.spec.ts tests/e2e/storage-capacity.spec.ts --config playwright.config.ts", "test:visual": "node scripts/validate-layer-scope.mjs VISUAL", "test:performance": "node scripts/validate-layer-scope.mjs PERFORMANCE", "test:security": "node scripts/verify-frozen-dependencies.mjs && node scripts/redaction-scan.mjs", @@ -28,7 +28,9 @@ "test:wp0-02": "node scripts/run-wp0-02-validation.mjs", "test:wp0-03": "node scripts/run-wp0-03-validation.mjs", "test:wp0-04": "node scripts/run-wp0-04-validation.mjs", - "test:wp0-04:red": "node scripts/run-wp0-04-validation.mjs --phase red" + "test:wp0-04:red": "node scripts/run-wp0-04-validation.mjs --phase red", + "test:wp0-05": "node scripts/run-wp0-05-validation.mjs", + "test:wp0-05:red": "node scripts/run-wp0-05-validation.mjs --phase red" }, "devDependencies": { "@playwright/test": "1.62.0", diff --git a/scripts/run-wp0-05-validation.mjs b/scripts/run-wp0-05-validation.mjs new file mode 100644 index 0000000..ced4135 --- /dev/null +++ b/scripts/run-wp0-05-validation.mjs @@ -0,0 +1,119 @@ +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const phaseIndex = process.argv.indexOf("--phase"); +const phase = phaseIndex >= 0 ? process.argv[phaseIndex + 1] : "green"; +if (!["red", "green"].includes(phase)) throw new Error(`Unsupported phase: ${phase}`); +const runId = process.env.DADA_TDD_RUN_ID ?? `wp0-05-${phase}-${new Date().toISOString().replace(/[^0-9]/g, "")}`; +const runDirectory = resolve("artifacts", "tdd", runId); +const cases = [ + "TDD-WP0-FILE-001-atomic-commit", + "TDD-WP0-STO-001-byte-thresholds", + "TDD-WP0-STO-002-equal-limit", + "TDD-WP0-STO-002-over-limit", + "TDD-WP0-STO-003-cleanup-remeasure", + "TDD-WP0-STO-003-full-matrix", + "TDD-WP0-STO-003-unavailable-matrix", +]; +const directories = Object.fromEntries(cases.map((id) => [id, resolve(runDirectory, "cases", id)])); +const playwrightDirectory = resolve(runDirectory, "playwright"); +if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`); +for (const directory of Object.values(directories)) mkdirSync(directory, { recursive: true }); + +const redCommands = [ + ["pnpm exec vitest run tests/unit/wp0-05-storage-policy.test.ts", ["exec", "vitest", "run", "tests/unit/wp0-05-storage-policy.test.ts"]], + ["pnpm exec vitest run tests/integration/wp0-05-managed-storage.test.ts", ["exec", "vitest", "run", "tests/integration/wp0-05-managed-storage.test.ts"]], + ["pnpm exec vitest run tests/api/wp0-05-storage-capacity.test.ts", ["exec", "vitest", "run", "tests/api/wp0-05-storage-capacity.test.ts"]], + ["pnpm exec vitest run tests/worker/wp0-05-storage-maintenance.test.ts", ["exec", "vitest", "run", "tests/worker/wp0-05-storage-maintenance.test.ts"]], + ["pnpm exec playwright test tests/e2e/storage-capacity.spec.ts --config playwright.config.ts", ["exec", "playwright", "test", "tests/e2e/storage-capacity.spec.ts", "--config", "playwright.config.ts"]], +]; +const greenCommands = [ + ["pnpm test:integration", ["test:integration"]], + ["pnpm test:api", ["test:api"]], + ["pnpm test:worker", ["test:worker"]], + ["pnpm test:unit", ["test:unit"]], + ["pnpm test:e2e", ["test:e2e"]], + ["pnpm validate:tdd-trace", ["validate:tdd-trace"]], +]; +const env = { + ...process.env, + DADA_EVIDENCE_DIR_FILE_001: directories[cases[0]], + DADA_EVIDENCE_DIR_STO_001: directories[cases[1]], + DADA_EVIDENCE_DIR_STO_002_EQUAL: directories[cases[2]], + DADA_EVIDENCE_DIR_STO_002_OVER: directories[cases[3]], + DADA_EVIDENCE_DIR_STO_003_CLEANUP: directories[cases[4]], + DADA_EVIDENCE_DIR_STO_003_FULL: directories[cases[5]], + DADA_EVIDENCE_DIR_STO_003_UNAVAILABLE: directories[cases[6]], + DADA_PLAYWRIGHT_OUTPUT_DIR: playwrightDirectory, +}; +const startedAt = new Date().toISOString(); +const commands = []; +for (const [command, args] of phase === "red" ? redCommands : greenCommands) { + const started_at = new Date().toISOString(); + const executable = process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm"; + const actualArgs = process.platform === "win32" ? ["/d", "/s", "/c", `pnpm ${args.join(" ")}`] : args; + const execution = spawnSync(executable, actualArgs, { encoding: "utf8", env }); + if (execution.stdout) process.stdout.write(execution.stdout); + if (execution.stderr) process.stderr.write(execution.stderr); + commands.push({ command, exit_code: execution.status ?? 1, finished_at: new Date().toISOString(), started_at }); +} + +function find(root, name) { + if (!existsSync(root)) return []; + return readdirSync(root).flatMap((entry) => { + const child = resolve(root, entry); + return statSync(child).isDirectory() ? find(child, name) : entry === name ? [child] : []; + }); +} +if (phase === "green") { + const trace = find(playwrightDirectory, "trace.zip").find((path) => path.replaceAll("\\", "/").includes("storage-capacity")); + if (trace) { + copyFileSync(trace, resolve(directories[cases[5]], "trace.zip")); + copyFileSync(trace, resolve(directories[cases[6]], "trace.zip")); + } +} + +const commandEvidence = { commands, phase, run_id: runId, schema_version: "1.0" }; +for (const directory of Object.values(directories)) writeFileSync(resolve(directory, "commands.json"), `${JSON.stringify(commandEvidence, null, 2)}\n`); +const manifest = { path: "tasks.manifest.json", sha256: createHash("sha256").update(readFileSync("tasks.manifest.json")).digest("hex").toUpperCase() }; +const commit = spawnSync("git", ["rev-parse", "--short", "HEAD"], { encoding: "utf8" }).stdout.trim(); +const worktreeDirty = spawnSync("git", ["status", "--porcelain"], { encoding: "utf8" }).stdout.trim().length > 0; +const expectedEvidence = { + [cases[0]]: ["response.json", "db-diff.json", "fs-before.json", "fs-after.json", "crash-points.json"], + [cases[1]]: ["thresholds.json", "db-diff.json", "screenshots/capacity-bar.png"], + [cases[2]]: ["db-diff.json"], + [cases[3]]: [ + "reference-response.json", "reference-db-diff.json", "reference-fs-diff.json", "reference-external-calls.json", + "generated-response.json", "generated-db-diff.json", "generated-fs-diff.json", "generated-external-calls.json", + "export-response.json", "export-db-diff.json", "export-fs-diff.json", "export-external-calls.json", + "sticker_original-response.json", "sticker_original-db-diff.json", "sticker_original-fs-diff.json", "sticker_original-external-calls.json", + "sticker_thumbnail-response.json", "sticker_thumbnail-db-diff.json", "sticker_thumbnail-fs-diff.json", "sticker_thumbnail-external-calls.json", + ], + [cases[4]]: ["conflict-response.json", "queued-db-diff.json", "worker-result.json", "fs-diff.json"], + [cases[5]]: ["response.json", "db-diff.json", "external-calls.json", "trace.zip"], + [cases[6]]: ["matrix.json", "db-diff.json", "external-calls.json", "trace.zip", "screenshots/unavailable.png"], +}; +const commandState = phase === "red" ? commands.every((item) => item.exit_code !== 0) : commands.every((item) => item.exit_code === 0); +const results = cases.map((testId) => { + const evidence_refs = expectedEvidence[testId]; + const missing_evidence = phase === "green" ? evidence_refs.filter((path) => !existsSync(resolve(directories[testId], path))) : []; + const status = phase === "red" ? (commandState ? "red_confirmed" : "failed") : (commandState && missing_evidence.length === 0 ? "passed" : "failed"); + const result = { + acceptance_criteria: ["AC-31", "AC-35", "AC-44", "AC-55"], automation: ["automated"], commit, + environment: { arch: process.arch, node: process.version.slice(1), os: process.platform }, evidence_refs, + finished_at: new Date().toISOString(), layer: ["DB", "API", "WORKER", "E2E"], manifest, missing_evidence, + parent_family: testId.split("-").slice(0, -2).join("-"), phase, release_gate: ["work_package:WP-0", "release:P0-A"], + requirements: ["ADMIN-06", "EXPORT-06", "NFR-05", "PROJECT-04", "17.19"], run_id: runId, + schema_version: "1.0", started_at: startedAt, status, task_id: "TASK-WP0-05", test_id: testId, + work_package: "WP-0", worktree_under_test: worktreeDirty ? "uncommitted implementation" : "clean committed implementation", + }; + writeFileSync(resolve(directories[testId], "result.json"), `${JSON.stringify(result, null, 2)}\n`); + return result; +}); +const passed = results.every((result) => result.status === (phase === "red" ? "red_confirmed" : "passed")); +const summary = { cases: results.map(({ missing_evidence, status, test_id }) => ({ missing_evidence, status, test_id })), phase, run_id: runId, status: passed ? (phase === "red" ? "red_confirmed" : "passed") : "failed" }; +writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify(summary, null, 2)}\n`); +console.log(JSON.stringify(summary, null, 2)); +if (!passed) process.exit(1); diff --git a/tests/api/wp0-05-storage-capacity.test.ts b/tests/api/wp0-05-storage-capacity.test.ts new file mode 100644 index 0000000..cf03cd0 --- /dev/null +++ b/tests/api/wp0-05-storage-capacity.test.ts @@ -0,0 +1,30 @@ +import { randomUUID } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { storageCapacityErrorDetails } from "../../apps/api/src/storage-policy.js"; +import { createErrorEnvelope, isErrorEnvelope } from "../../packages/shared-contracts/src/index.js"; + +describe("TDD-WP0-STO-002-over-limit API envelope", () => { + it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"])( + "returns stable 507 details for %s", + (kind) => { + const envelope = createErrorEnvelope({ + code: "STORAGE_CAPACITY_EXCEEDED", + correlationId: randomUUID(), + details: storageCapacityErrorDetails({ + activeReservationBytes: 0, + managedContentBytes: 5_368_709_119, + projectedWriteBytes: 2, + }), + }); + expect(isErrorEnvelope(envelope)).toBe(true); + expect(envelope.error).toMatchObject({ + code: "STORAGE_CAPACITY_EXCEEDED", + details: { capacity_status: "critical", remaining_bytes: 1 }, + message_key: "STORAGE_CAPACITY_EXCEEDED", + }); + expect(kind).toBeTruthy(); + }, + ); +}); diff --git a/tests/e2e/fixtures/storage-capacity.html b/tests/e2e/fixtures/storage-capacity.html new file mode 100644 index 0000000..3637b22 --- /dev/null +++ b/tests/e2e/fixtures/storage-capacity.html @@ -0,0 +1,17 @@ + + + + + + Dada 容量状态验收探针 + + +
+ diff --git a/tests/e2e/fixtures/storage-capacity.ts b/tests/e2e/fixtures/storage-capacity.ts new file mode 100644 index 0000000..e1880b2 --- /dev/null +++ b/tests/e2e/fixtures/storage-capacity.ts @@ -0,0 +1,17 @@ +import { classifyCapacity, decideStorageAction } from "../../../apps/api/src/storage-policy.js"; + +const root = document.querySelector("#root"); +if (!root) throw new Error("Fixture root is missing."); +const capacity = classifyCapacity(5_368_709_120, 0); +const actions = ["project_json_write", "binary_write", "read", "download", "permanent_delete", "explicit_cleanup"] as const; +root.innerHTML = ` +

本机存储容量

+

${capacity.storage_status}

+
+ + ${actions.map((action) => ``).join("")} +
操作结果
${action}${decideStorageAction("full", true, action)}
+

存储不可用

+ + ${actions.map((action) => ``).join("")} +
操作SQLite 可写SQLite 不可写
${action}${decideStorageAction("unavailable", true, action)}${decideStorageAction("unavailable", false, action)}
`; diff --git a/tests/e2e/storage-capacity.spec.ts b/tests/e2e/storage-capacity.spec.ts new file mode 100644 index 0000000..9a7f4d5 --- /dev/null +++ b/tests/e2e/storage-capacity.spec.ts @@ -0,0 +1,39 @@ +import { mkdirSync } from "node:fs"; +import { resolve } from "node:path"; + +import { expect, test } from "@playwright/test"; +import { createServer, type ViteDevServer } from "vite"; + +let vite: ViteDevServer; +let webUrl: string; + +test.beforeAll(async () => { + vite = await createServer({ configFile: false, root: process.cwd(), server: { host: "127.0.0.1", port: 0 } }); + await vite.listen(); + const address = vite.httpServer?.address(); + if (!address || typeof address === "string") throw new Error("Vite did not expose a test port."); + webUrl = `http://127.0.0.1:${address.port}`; +}); + +test.afterAll(async () => vite.close()); + +test("TDD-WP0-STO-001 and STO-003 render the exact full action matrix", async ({ page }) => { + await page.goto(`${webUrl}/tests/e2e/fixtures/storage-capacity.html`); + await expect(page.getByRole("heading", { name: "本机存储容量" })).toBeVisible(); + await expect(page.locator("[data-status]")).toHaveText("full"); + await expect(page.getByRole("progressbar")).toHaveAttribute("aria-valuenow", "5368709120"); + await expect(page.locator("table").first().getByRole("row").filter({ hasText: "binary_write" })).toContainText("reject_capacity"); + await expect(page.locator("table").first().getByRole("row").filter({ hasText: "project_json_write" })).toContainText("allow"); + await expect(page.getByRole("heading", { name: "存储不可用" })).toBeVisible(); + await expect(page.locator("[data-unavailable] tr").filter({ hasText: "explicit_cleanup" })).toContainText("reject_uncommitted"); + const directory = process.env.DADA_EVIDENCE_DIR_STO_001; + if (directory) { + mkdirSync(resolve(directory, "screenshots"), { recursive: true }); + await page.screenshot({ path: resolve(directory, "screenshots", "capacity-bar.png") }); + } + const unavailableDirectory = process.env.DADA_EVIDENCE_DIR_STO_003_UNAVAILABLE; + if (unavailableDirectory) { + mkdirSync(resolve(unavailableDirectory, "screenshots"), { recursive: true }); + await page.screenshot({ path: resolve(unavailableDirectory, "screenshots", "unavailable.png"), fullPage: true }); + } +}); diff --git a/tests/integration/wp0-05-managed-storage.test.ts b/tests/integration/wp0-05-managed-storage.test.ts new file mode 100644 index 0000000..96243ca --- /dev/null +++ b/tests/integration/wp0-05-managed-storage.test.ts @@ -0,0 +1,265 @@ +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { Readable } from "node:stream"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + HARD_LIMIT_BYTES, + ManagedStorage, + StorageCapacityError, + type ManagedFileKind, +} from "../../apps/api/src/managed-storage.js"; + +const temporaryDirectories: string[] = []; + +function fixture() { + const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp0-05-")); + temporaryDirectories.push(dataRoot); + for (const directory of ["db", "content/references", "content/generated", "content/exports", "managed-assets", "derived-assets", "staging"]) { + mkdirSync(join(dataRoot, directory), { recursive: true }); + } + const storage = new ManagedStorage({ dataRoot, databasePath: join(dataRoot, "db", "dada.sqlite3") }); + return { dataRoot, storage }; +} + +function databaseCounts(storage: ManagedStorage) { + return storage.inspectCounts(); +} + +function tree(root: string) { + return readdirSync(root, { recursive: true }).map(String).sort(); +} + +function evidence(caseName: string, name: string, value: unknown) { + const directory = process.env[`DADA_EVIDENCE_DIR_${caseName}`]; + if (!directory) return; + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, name), `${JSON.stringify(value, null, 2)}\n`); +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + if (resolve(directory).startsWith(resolve(tmpdir()))) rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("TDD-WP0-STO-002 exact admission", () => { + it("allows equality, consumes its reservation and transitions to full atomically", async () => { + const { storage } = fixture(); + storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1); + const result = await storage.commitStream({ + content: Readable.from(Buffer.from("x")), + expectedMimeType: "application/octet-stream", + fileKind: "generated", + fileName: "asset.png", + operationId: randomUUID(), + ownerRef: randomUUID(), + projectedWriteBytes: 1, + }); + expect(result.bytes).toBe(1); + expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" }); + expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 1 }); + evidence("STO_002_EQUAL", "db-diff.json", { after: databaseCounts(storage), state: storage.getState() }); + storage.close(); + }); + + it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"] as ManagedFileKind[])( + "rejects %s when projected bytes are strictly over the limit without side effects", + async (fileKind) => { + const { dataRoot, storage } = fixture(); + storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1); + const beforeCounts = databaseCounts(storage); + const beforeTree = tree(dataRoot); + await expect(storage.commitStream({ + content: Readable.from(Buffer.from("xx")), + expectedMimeType: "image/png", + fileKind, + fileName: "asset.png", + operationId: randomUUID(), + ownerRef: randomUUID(), + projectedWriteBytes: 2, + })).rejects.toBeInstanceOf(StorageCapacityError); + expect(databaseCounts(storage)).toEqual(beforeCounts); + expect(tree(dataRoot)).toEqual(beforeTree); + expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 1, storage_status: "active" }); + evidence("STO_002_OVER", `${fileKind}-response.json`, { code: "STORAGE_CAPACITY_EXCEEDED", status: 507 }); + evidence("STO_002_OVER", `${fileKind}-db-diff.json`, { after: databaseCounts(storage), before: beforeCounts }); + evidence("STO_002_OVER", `${fileKind}-fs-diff.json`, { after: tree(dataRoot), before: beforeTree }); + evidence("STO_002_OVER", `${fileKind}-external-calls.json`, { calls: 0 }); + storage.close(); + }, + ); +}); + +describe("TDD-WP0-FILE-001 atomic commit", () => { + it("streams, validates and commits a managed file without a half-committed state", async () => { + const { dataRoot, storage } = fixture(); + const bytes = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.from("valid-image-payload")]); + const before = tree(dataRoot); + const committed = await storage.commitStream({ + content: Readable.from([bytes.subarray(0, 5), bytes.subarray(5)]), + expectedMimeType: "image/png", + expectedSha256: createHash("sha256").update(bytes).digest("hex"), + fileKind: "reference", + fileName: "reference.png", + operationId: randomUUID(), + ownerRef: randomUUID(), + projectedWriteBytes: bytes.byteLength, + }); + expect(existsSync(join(dataRoot, committed.relative_path))).toBe(true); + expect(readFileSync(join(dataRoot, committed.relative_path))).toEqual(bytes); + expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 0 }); + evidence("FILE_001", "fs-before.json", { entries: before }); + evidence("FILE_001", "fs-after.json", { entries: tree(dataRoot) }); + evidence("FILE_001", "db-diff.json", databaseCounts(storage)); + evidence("FILE_001", "response.json", { bytes: committed.bytes, file_id: committed.file_id, status: "committed" }); + storage.close(); + }); + + it("rejects invalid MIME, hash, traversal and symlink inputs before commit", async () => { + const { storage } = fixture(); + const base = { + content: Readable.from(Buffer.from("bad")), + expectedMimeType: "image/png" as const, + fileKind: "reference" as const, + ownerRef: randomUUID(), + projectedWriteBytes: 3, + }; + await expect(storage.commitStream({ ...base, fileName: "../bad.png", operationId: randomUUID() })).rejects.toThrow("file_name_invalid"); + await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.png", expectedSha256: "0".repeat(64), operationId: randomUUID() })).rejects.toThrow("content_hash_invalid"); + await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.jpg", expectedMimeType: "image/jpeg", operationId: randomUUID() })).rejects.toThrow("content_mime_invalid"); + const ownerRef = randomUUID(); + const target = join(storage.dataRoot, "symlink-target"); + mkdirSync(target); + symlinkSync(target, join(storage.dataRoot, "content", "references", ownerRef), "junction"); + await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.bin", expectedMimeType: "application/octet-stream", operationId: randomUUID(), ownerRef })).rejects.toThrow(/path_escape|symbolic_link/); + expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 0 }); + storage.close(); + }); + + it("reconciles each crash point into either a complete commit or compensation", async () => { + const { storage } = fixture(); + await expect(storage.commitStream({ + content: Readable.from(Buffer.from("staged")), expectedMimeType: "application/octet-stream", + failurePoint: "after_staging", fileKind: "generated", fileName: "staged.bin", + operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 6, + })).rejects.toThrow("injected_crash_after_staging"); + const afterStaging = await storage.reconcileStartup(); + expect(afterStaging).toMatchObject({ orphaned: 0, released_reservations: 1 }); + + await expect(storage.commitStream({ + content: Readable.from(Buffer.from("orphan")), + expectedMimeType: "application/octet-stream", + failurePoint: "after_rename", + fileKind: "generated", + fileName: "orphan.png", + operationId: randomUUID(), + ownerRef: randomUUID(), + projectedWriteBytes: 6, + })).rejects.toThrow("injected_crash_after_rename"); + const reconciled = await storage.reconcileStartup(); + expect(reconciled).toMatchObject({ orphaned: 1, released_reservations: 1 }); + expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, pending_cleanup: 1 }); + + await expect(storage.commitStream({ + content: Readable.from(Buffer.from("committed")), expectedMimeType: "application/octet-stream", + failurePoint: "after_database_commit", fileKind: "generated", fileName: "committed.bin", + operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 9, + })).rejects.toThrow("injected_crash_after_database_commit"); + const afterDatabaseCommit = await storage.reconcileStartup(); + expect(afterDatabaseCommit).toMatchObject({ orphaned: 0, released_reservations: 0 }); + expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 1 }); + evidence("FILE_001", "crash-points.json", { + after_database_commit: afterDatabaseCommit, + after_rename: reconciled, + after_staging: afterStaging, + }); + storage.close(); + }); +}); + +describe("TDD-WP0-STO-003 cleanup and availability", () => { + it("rejects a stale cleanup as one batch, then decrements only after physical deletion and remeasure", async () => { + const { dataRoot, storage } = fixture(); + const fsBefore = tree(dataRoot); + storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 2); + const first = await storage.commitBufferFixture("sticker_original", "a.png", Buffer.from("a")); + const second = await storage.commitBufferFixture("sticker_thumbnail", "b.png", Buffer.from("b")); + expect(storage.getState().storage_status).toBe("full"); + const stale = storage.createCleanupIntent([first.file_id, second.file_id]); + storage.addAssetReference(first.file_id, "release"); + expect(() => storage.confirmCleanupIntent(stale.request_id)).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT"); + expect(databaseCounts(storage)).toMatchObject({ pending_cleanup: 0 }); + evidence("STO_003_CLEANUP", "conflict-response.json", { code: "ASSET_HISTORY_REFERENCE_CONFLICT", partial_queue: false, status: 409 }); + + storage.removeAssetReferences(first.file_id); + const request = storage.createCleanupIntent([first.file_id, second.file_id]); + storage.confirmCleanupIntent(request.request_id); + expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" }); + expect(existsSync(join(dataRoot, first.relative_path))).toBe(true); + evidence("STO_003_CLEANUP", "queued-db-diff.json", { counts: databaseCounts(storage), state: storage.getState() }); + await storage.processCleanupQueue(); + expect(existsSync(join(dataRoot, first.relative_path))).toBe(false); + expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 2, storage_status: "active" }); + evidence("STO_003_CLEANUP", "worker-result.json", { counts: databaseCounts(storage), state: storage.getState() }); + evidence("STO_003_CLEANUP", "fs-diff.json", { after: tree(dataRoot), before: fsBefore }); + storage.close(); + }); + + it("keeps the full matrix read/write effects exact", () => { + const { storage } = fixture(); + storage.applyControlledMeasurement(HARD_LIMIT_BYTES); + const before = databaseCounts(storage); + const decisions = { + ai_call: storage.inspectAction("ai_call"), + binary_write: storage.inspectAction("binary_write"), + client_only_download: storage.inspectAction("client_only_download"), + download: storage.inspectAction("download"), + explicit_cleanup: storage.inspectAction("explicit_cleanup"), + latest_export_write: storage.inspectAction("latest_export_write"), + permanent_delete: storage.inspectAction("permanent_delete"), + project_json_write: storage.inspectAction("project_json_write"), + read: storage.inspectAction("read"), + }; + expect(decisions).toEqual({ + ai_call: "reject_capacity", binary_write: "reject_capacity", client_only_download: "allow", + download: "allow", explicit_cleanup: "allow", latest_export_write: "reject_capacity", + permanent_delete: "allow", project_json_write: "allow", read: "allow", + }); + expect(databaseCounts(storage)).toEqual(before); + evidence("STO_003_FULL", "response.json", decisions); + evidence("STO_003_FULL", "db-diff.json", { after: databaseCounts(storage), before }); + evidence("STO_003_FULL", "external-calls.json", { calls: 0 }); + storage.close(); + }); + + it("marks unavailable without mutating storage when the data root or SQLite cannot be written", async () => { + const { storage } = fixture(); + storage.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true }); + expect(storage.getState().storage_status).toBe("unavailable"); + expect(storage.inspectAction("project_json_write")).toBe("reject_unavailable"); + expect(storage.inspectAction("read")).toBe("allow"); + expect(storage.inspectAction("explicit_cleanup")).toBe("allow"); + storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: true }); + expect(storage.getState().storage_status).toBe("unavailable"); + storage.applyControlledMeasurement(0); + expect(storage.getState().storage_status).toBe("active"); + storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: false }); + expect(storage.inspectAction("explicit_cleanup")).toBe("reject_uncommitted"); + expect(() => storage.createCleanupIntent([randomUUID()])).toThrow("cleanup_uncommitted"); + const beforeMaintenance = databaseCounts(storage); + expect(await storage.processCleanupQueue()).toEqual({ completed: 0, failed: 0 }); + expect(databaseCounts(storage)).toEqual(beforeMaintenance); + evidence("STO_003_UNAVAILABLE", "matrix.json", { + read: storage.inspectAction("read"), + write: storage.inspectAction("project_json_write"), + cleanup: storage.inspectAction("explicit_cleanup"), + }); + evidence("STO_003_UNAVAILABLE", "db-diff.json", { counts: databaseCounts(storage), state: storage.getState() }); + evidence("STO_003_UNAVAILABLE", "external-calls.json", { calls: 0 }); + storage.close(); + }); +}); diff --git a/tests/unit/wp0-05-storage-policy.test.ts b/tests/unit/wp0-05-storage-policy.test.ts new file mode 100644 index 0000000..1776a03 --- /dev/null +++ b/tests/unit/wp0-05-storage-policy.test.ts @@ -0,0 +1,80 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + CAPACITY_COUNTED_FILE_KINDS, + CAPACITY_EXCLUDED_STORAGE, + HARD_LIMIT_BYTES, + classifyCapacity, + decideStorageAction, +} from "../../apps/api/src/storage-policy.js"; + +describe("TDD-WP0-STO-001-byte-thresholds", () => { + it.each([ + [4_294_967_295, "normal", "active"], + [4_294_967_296, "warning", "active"], + [4_831_838_207, "warning", "active"], + [4_831_838_208, "critical", "active"], + [5_368_709_119, "critical", "active"], + [5_368_709_120, "critical", "full"], + [5_368_709_121, "critical", "full"], + ] as const)("classifies %i bytes", (bytes, notice, status) => { + expect(classifyCapacity(bytes, 0)).toEqual({ capacity_notice_level: notice, storage_status: status }); + }); + + it("records the deterministic threshold result without deleting content", () => { + const values = [ + 4_294_967_295, + 4_294_967_296, + 4_831_838_207, + 4_831_838_208, + 5_368_709_119, + HARD_LIMIT_BYTES, + 5_368_709_121, + ].map((bytes) => ({ bytes, ...classifyCapacity(bytes, 0) })); + const directory = process.env.DADA_EVIDENCE_DIR_STO_001; + if (directory) { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "thresholds.json"), `${JSON.stringify({ automatic_cleanup: false, values }, null, 2)}\n`); + writeFileSync(join(directory, "db-diff.json"), `${JSON.stringify({ changed_fields: ["managed_content_bytes", "capacity_notice_level", "storage_status", "measured_at"], deleted_files: 0 }, null, 2)}\n`); + } + }); + + it("counts only managed binary classes and excludes infrastructure and downloads", () => { + expect(CAPACITY_COUNTED_FILE_KINDS).toEqual([ + "reference", "generated", "export", "derived", "sticker_original", "sticker_thumbnail", + ]); + expect(CAPACITY_EXCLUDED_STORAGE).toEqual([ + "read_only_assets", "application_files", "database", "logs", "audit", "browser_cache", "user_downloads", + ]); + }); +}); + +describe("TDD-WP0-STO-003 action matrices", () => { + it("allows only the frozen full-state actions", () => { + expect(decideStorageAction("full", true, "project_json_write")).toBe("allow"); + expect(decideStorageAction("full", true, "read")).toBe("allow"); + expect(decideStorageAction("full", true, "download")).toBe("allow"); + expect(decideStorageAction("full", true, "client_only_download")).toBe("allow"); + expect(decideStorageAction("full", true, "permanent_delete")).toBe("allow"); + expect(decideStorageAction("full", true, "explicit_cleanup")).toBe("allow"); + expect(decideStorageAction("full", true, "binary_write")).toBe("reject_capacity"); + expect(decideStorageAction("full", true, "ai_call")).toBe("reject_capacity"); + expect(decideStorageAction("full", true, "latest_export_write")).toBe("reject_capacity"); + }); + + it("blocks all new backend writes while unavailable and gates cleanup on SQLite", () => { + for (const action of ["project_json_write", "binary_write", "ai_call", "latest_export_write"] as const) { + expect(decideStorageAction("unavailable", true, action)).toBe("reject_unavailable"); + } + for (const action of ["read", "download", "client_only_download"] as const) { + expect(decideStorageAction("unavailable", false, action)).toBe("allow"); + } + expect(decideStorageAction("unavailable", true, "permanent_delete")).toBe("allow"); + expect(decideStorageAction("unavailable", false, "permanent_delete")).toBe("reject_uncommitted"); + expect(decideStorageAction("unavailable", true, "explicit_cleanup")).toBe("allow"); + expect(decideStorageAction("unavailable", false, "explicit_cleanup")).toBe("reject_uncommitted"); + }); +}); diff --git a/tests/worker/wp0-05-storage-maintenance.test.ts b/tests/worker/wp0-05-storage-maintenance.test.ts new file mode 100644 index 0000000..873dfcc --- /dev/null +++ b/tests/worker/wp0-05-storage-maintenance.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it, vi } from "vitest"; + +import { runStorageMaintenance } from "../../apps/worker/src/storage-maintenance.js"; + +describe("TDD-WP0-FILE-001 worker reconciliation", () => { + it("reconciles startup before processing physical cleanup", async () => { + const target = { + processCleanupQueue: vi.fn(async () => ({ completed: 1, failed: 0 })), + reconcileStartup: vi.fn(async () => ({ orphaned: 1, released_reservations: 1 })), + }; + await expect(runStorageMaintenance(target)).resolves.toEqual({ + cleanup: { completed: 1, failed: 0 }, + reconciliation: { orphaned: 1, released_reservations: 1 }, + }); + expect(target.reconcileStartup).toHaveBeenCalledBefore(target.processCleanupQueue); + }); +});