feat: complete TASK-WP0-05 storage capacity

2026-07-27 19:10:57 +08:00
parent b5a6c6629e
commit e12a83d6d3
12 changed files with 1294 additions and 3 deletions
@@ -0,0 +1,265 @@
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { Readable } from "node:stream";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  HARD_LIMIT_BYTES,
+  ManagedStorage,
+  StorageCapacityError,
+  type ManagedFileKind,
+} from "../../apps/api/src/managed-storage.js";
+
+const temporaryDirectories: string[] = [];
+
+function fixture() {
+  const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp0-05-"));
+  temporaryDirectories.push(dataRoot);
+  for (const directory of ["db", "content/references", "content/generated", "content/exports", "managed-assets", "derived-assets", "staging"]) {
+    mkdirSync(join(dataRoot, directory), { recursive: true });
+  }
+  const storage = new ManagedStorage({ dataRoot, databasePath: join(dataRoot, "db", "dada.sqlite3") });
+  return { dataRoot, storage };
+}
+
+function databaseCounts(storage: ManagedStorage) {
+  return storage.inspectCounts();
+}
+
+function tree(root: string) {
+  return readdirSync(root, { recursive: true }).map(String).sort();
+}
+
+function evidence(caseName: string, name: string, value: unknown) {
+  const directory = process.env[`DADA_EVIDENCE_DIR_${caseName}`];
+  if (!directory) return;
+  mkdirSync(directory, { recursive: true });
+  writeFileSync(join(directory, name), `${JSON.stringify(value, null, 2)}\n`);
+}
+
+afterEach(() => {
+  for (const directory of temporaryDirectories.splice(0)) {
+    if (resolve(directory).startsWith(resolve(tmpdir()))) rmSync(directory, { force: true, recursive: true });
+  }
+});
+
+describe("TDD-WP0-STO-002 exact admission", () => {
+  it("allows equality, consumes its reservation and transitions to full atomically", async () => {
+    const { storage } = fixture();
+    storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
+    const result = await storage.commitStream({
+      content: Readable.from(Buffer.from("x")),
+      expectedMimeType: "application/octet-stream",
+      fileKind: "generated",
+      fileName: "asset.png",
+      operationId: randomUUID(),
+      ownerRef: randomUUID(),
+      projectedWriteBytes: 1,
+    });
+    expect(result.bytes).toBe(1);
+    expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
+    expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 1 });
+    evidence("STO_002_EQUAL", "db-diff.json", { after: databaseCounts(storage), state: storage.getState() });
+    storage.close();
+  });
+
+  it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"] as ManagedFileKind[])(
+    "rejects %s when projected bytes are strictly over the limit without side effects",
+    async (fileKind) => {
+      const { dataRoot, storage } = fixture();
+      storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
+      const beforeCounts = databaseCounts(storage);
+      const beforeTree = tree(dataRoot);
+      await expect(storage.commitStream({
+        content: Readable.from(Buffer.from("xx")),
+        expectedMimeType: "image/png",
+        fileKind,
+        fileName: "asset.png",
+        operationId: randomUUID(),
+        ownerRef: randomUUID(),
+        projectedWriteBytes: 2,
+      })).rejects.toBeInstanceOf(StorageCapacityError);
+      expect(databaseCounts(storage)).toEqual(beforeCounts);
+      expect(tree(dataRoot)).toEqual(beforeTree);
+      expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 1, storage_status: "active" });
+      evidence("STO_002_OVER", `${fileKind}-response.json`, { code: "STORAGE_CAPACITY_EXCEEDED", status: 507 });
+      evidence("STO_002_OVER", `${fileKind}-db-diff.json`, { after: databaseCounts(storage), before: beforeCounts });
+      evidence("STO_002_OVER", `${fileKind}-fs-diff.json`, { after: tree(dataRoot), before: beforeTree });
+      evidence("STO_002_OVER", `${fileKind}-external-calls.json`, { calls: 0 });
+      storage.close();
+    },
+  );
+});
+
+describe("TDD-WP0-FILE-001 atomic commit", () => {
+  it("streams, validates and commits a managed file without a half-committed state", async () => {
+    const { dataRoot, storage } = fixture();
+    const bytes = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.from("valid-image-payload")]);
+    const before = tree(dataRoot);
+    const committed = await storage.commitStream({
+      content: Readable.from([bytes.subarray(0, 5), bytes.subarray(5)]),
+      expectedMimeType: "image/png",
+      expectedSha256: createHash("sha256").update(bytes).digest("hex"),
+      fileKind: "reference",
+      fileName: "reference.png",
+      operationId: randomUUID(),
+      ownerRef: randomUUID(),
+      projectedWriteBytes: bytes.byteLength,
+    });
+    expect(existsSync(join(dataRoot, committed.relative_path))).toBe(true);
+    expect(readFileSync(join(dataRoot, committed.relative_path))).toEqual(bytes);
+    expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 0 });
+    evidence("FILE_001", "fs-before.json", { entries: before });
+    evidence("FILE_001", "fs-after.json", { entries: tree(dataRoot) });
+    evidence("FILE_001", "db-diff.json", databaseCounts(storage));
+    evidence("FILE_001", "response.json", { bytes: committed.bytes, file_id: committed.file_id, status: "committed" });
+    storage.close();
+  });
+
+  it("rejects invalid MIME, hash, traversal and symlink inputs before commit", async () => {
+    const { storage } = fixture();
+    const base = {
+      content: Readable.from(Buffer.from("bad")),
+      expectedMimeType: "image/png" as const,
+      fileKind: "reference" as const,
+      ownerRef: randomUUID(),
+      projectedWriteBytes: 3,
+    };
+    await expect(storage.commitStream({ ...base, fileName: "../bad.png", operationId: randomUUID() })).rejects.toThrow("file_name_invalid");
+    await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.png", expectedSha256: "0".repeat(64), operationId: randomUUID() })).rejects.toThrow("content_hash_invalid");
+    await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.jpg", expectedMimeType: "image/jpeg", operationId: randomUUID() })).rejects.toThrow("content_mime_invalid");
+    const ownerRef = randomUUID();
+    const target = join(storage.dataRoot, "symlink-target");
+    mkdirSync(target);
+    symlinkSync(target, join(storage.dataRoot, "content", "references", ownerRef), "junction");
+    await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.bin", expectedMimeType: "application/octet-stream", operationId: randomUUID(), ownerRef })).rejects.toThrow(/path_escape|symbolic_link/);
+    expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 0 });
+    storage.close();
+  });
+
+  it("reconciles each crash point into either a complete commit or compensation", async () => {
+    const { storage } = fixture();
+    await expect(storage.commitStream({
+      content: Readable.from(Buffer.from("staged")), expectedMimeType: "application/octet-stream",
+      failurePoint: "after_staging", fileKind: "generated", fileName: "staged.bin",
+      operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 6,
+    })).rejects.toThrow("injected_crash_after_staging");
+    const afterStaging = await storage.reconcileStartup();
+    expect(afterStaging).toMatchObject({ orphaned: 0, released_reservations: 1 });
+
+    await expect(storage.commitStream({
+      content: Readable.from(Buffer.from("orphan")),
+      expectedMimeType: "application/octet-stream",
+      failurePoint: "after_rename",
+      fileKind: "generated",
+      fileName: "orphan.png",
+      operationId: randomUUID(),
+      ownerRef: randomUUID(),
+      projectedWriteBytes: 6,
+    })).rejects.toThrow("injected_crash_after_rename");
+    const reconciled = await storage.reconcileStartup();
+    expect(reconciled).toMatchObject({ orphaned: 1, released_reservations: 1 });
+    expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, pending_cleanup: 1 });
+
+    await expect(storage.commitStream({
+      content: Readable.from(Buffer.from("committed")), expectedMimeType: "application/octet-stream",
+      failurePoint: "after_database_commit", fileKind: "generated", fileName: "committed.bin",
+      operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 9,
+    })).rejects.toThrow("injected_crash_after_database_commit");
+    const afterDatabaseCommit = await storage.reconcileStartup();
+    expect(afterDatabaseCommit).toMatchObject({ orphaned: 0, released_reservations: 0 });
+    expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 1 });
+    evidence("FILE_001", "crash-points.json", {
+      after_database_commit: afterDatabaseCommit,
+      after_rename: reconciled,
+      after_staging: afterStaging,
+    });
+    storage.close();
+  });
+});
+
+describe("TDD-WP0-STO-003 cleanup and availability", () => {
+  it("rejects a stale cleanup as one batch, then decrements only after physical deletion and remeasure", async () => {
+    const { dataRoot, storage } = fixture();
+    const fsBefore = tree(dataRoot);
+    storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 2);
+    const first = await storage.commitBufferFixture("sticker_original", "a.png", Buffer.from("a"));
+    const second = await storage.commitBufferFixture("sticker_thumbnail", "b.png", Buffer.from("b"));
+    expect(storage.getState().storage_status).toBe("full");
+    const stale = storage.createCleanupIntent([first.file_id, second.file_id]);
+    storage.addAssetReference(first.file_id, "release");
+    expect(() => storage.confirmCleanupIntent(stale.request_id)).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT");
+    expect(databaseCounts(storage)).toMatchObject({ pending_cleanup: 0 });
+    evidence("STO_003_CLEANUP", "conflict-response.json", { code: "ASSET_HISTORY_REFERENCE_CONFLICT", partial_queue: false, status: 409 });
+
+    storage.removeAssetReferences(first.file_id);
+    const request = storage.createCleanupIntent([first.file_id, second.file_id]);
+    storage.confirmCleanupIntent(request.request_id);
+    expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
+    expect(existsSync(join(dataRoot, first.relative_path))).toBe(true);
+    evidence("STO_003_CLEANUP", "queued-db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
+    await storage.processCleanupQueue();
+    expect(existsSync(join(dataRoot, first.relative_path))).toBe(false);
+    expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 2, storage_status: "active" });
+    evidence("STO_003_CLEANUP", "worker-result.json", { counts: databaseCounts(storage), state: storage.getState() });
+    evidence("STO_003_CLEANUP", "fs-diff.json", { after: tree(dataRoot), before: fsBefore });
+    storage.close();
+  });
+
+  it("keeps the full matrix read/write effects exact", () => {
+    const { storage } = fixture();
+    storage.applyControlledMeasurement(HARD_LIMIT_BYTES);
+    const before = databaseCounts(storage);
+    const decisions = {
+      ai_call: storage.inspectAction("ai_call"),
+      binary_write: storage.inspectAction("binary_write"),
+      client_only_download: storage.inspectAction("client_only_download"),
+      download: storage.inspectAction("download"),
+      explicit_cleanup: storage.inspectAction("explicit_cleanup"),
+      latest_export_write: storage.inspectAction("latest_export_write"),
+      permanent_delete: storage.inspectAction("permanent_delete"),
+      project_json_write: storage.inspectAction("project_json_write"),
+      read: storage.inspectAction("read"),
+    };
+    expect(decisions).toEqual({
+      ai_call: "reject_capacity", binary_write: "reject_capacity", client_only_download: "allow",
+      download: "allow", explicit_cleanup: "allow", latest_export_write: "reject_capacity",
+      permanent_delete: "allow", project_json_write: "allow", read: "allow",
+    });
+    expect(databaseCounts(storage)).toEqual(before);
+    evidence("STO_003_FULL", "response.json", decisions);
+    evidence("STO_003_FULL", "db-diff.json", { after: databaseCounts(storage), before });
+    evidence("STO_003_FULL", "external-calls.json", { calls: 0 });
+    storage.close();
+  });
+
+  it("marks unavailable without mutating storage when the data root or SQLite cannot be written", async () => {
+    const { storage } = fixture();
+    storage.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true });
+    expect(storage.getState().storage_status).toBe("unavailable");
+    expect(storage.inspectAction("project_json_write")).toBe("reject_unavailable");
+    expect(storage.inspectAction("read")).toBe("allow");
+    expect(storage.inspectAction("explicit_cleanup")).toBe("allow");
+    storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: true });
+    expect(storage.getState().storage_status).toBe("unavailable");
+    storage.applyControlledMeasurement(0);
+    expect(storage.getState().storage_status).toBe("active");
+    storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: false });
+    expect(storage.inspectAction("explicit_cleanup")).toBe("reject_uncommitted");
+    expect(() => storage.createCleanupIntent([randomUUID()])).toThrow("cleanup_uncommitted");
+    const beforeMaintenance = databaseCounts(storage);
+    expect(await storage.processCleanupQueue()).toEqual({ completed: 0, failed: 0 });
+    expect(databaseCounts(storage)).toEqual(beforeMaintenance);
+    evidence("STO_003_UNAVAILABLE", "matrix.json", {
+      read: storage.inspectAction("read"),
+      write: storage.inspectAction("project_json_write"),
+      cleanup: storage.inspectAction("explicit_cleanup"),
+    });
+    evidence("STO_003_UNAVAILABLE", "db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
+    evidence("STO_003_UNAVAILABLE", "external-calls.json", { calls: 0 });
+    storage.close();
+  });
+});