import { createHash, randomUUID } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { Readable } from "node:stream"; import { afterEach, describe, expect, it } from "vitest"; import { HARD_LIMIT_BYTES, ManagedStorage, StorageCapacityError, type ManagedFileKind, } from "../../apps/api/src/managed-storage.js"; const temporaryDirectories: string[] = []; function fixture() { const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp0-05-")); temporaryDirectories.push(dataRoot); for (const directory of ["db", "content/references", "content/generated", "content/exports", "managed-assets", "derived-assets", "staging"]) { mkdirSync(join(dataRoot, directory), { recursive: true }); } const storage = new ManagedStorage({ dataRoot, databasePath: join(dataRoot, "db", "dada.sqlite3") }); return { dataRoot, storage }; } function databaseCounts(storage: ManagedStorage) { return storage.inspectCounts(); } function tree(root: string) { return readdirSync(root, { recursive: true }).map(String).sort(); } function evidence(caseName: string, name: string, value: unknown) { const directory = process.env[`DADA_EVIDENCE_DIR_${caseName}`]; if (!directory) return; mkdirSync(directory, { recursive: true }); writeFileSync(join(directory, name), `${JSON.stringify(value, null, 2)}\n`); } afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { if (resolve(directory).startsWith(resolve(tmpdir()))) rmSync(directory, { force: true, recursive: true }); } }); describe("TDD-WP0-STO-002 exact admission", () => { it("allows equality, consumes its reservation and transitions to full atomically", async () => { const { storage } = fixture(); storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1); const result = await storage.commitStream({ content: Readable.from(Buffer.from("x")), expectedMimeType: "application/octet-stream", fileKind: "generated", fileName: "asset.png", operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 1, }); expect(result.bytes).toBe(1); expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" }); expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 1 }); evidence("STO_002_EQUAL", "db-diff.json", { after: databaseCounts(storage), state: storage.getState() }); storage.close(); }); it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"] as ManagedFileKind[])( "rejects %s when projected bytes are strictly over the limit without side effects", async (fileKind) => { const { dataRoot, storage } = fixture(); storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1); const beforeCounts = databaseCounts(storage); const beforeTree = tree(dataRoot); await expect(storage.commitStream({ content: Readable.from(Buffer.from("xx")), expectedMimeType: "image/png", fileKind, fileName: "asset.png", operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 2, })).rejects.toBeInstanceOf(StorageCapacityError); expect(databaseCounts(storage)).toEqual(beforeCounts); expect(tree(dataRoot)).toEqual(beforeTree); expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 1, storage_status: "active" }); evidence("STO_002_OVER", `${fileKind}-response.json`, { code: "STORAGE_CAPACITY_EXCEEDED", status: 507 }); evidence("STO_002_OVER", `${fileKind}-db-diff.json`, { after: databaseCounts(storage), before: beforeCounts }); evidence("STO_002_OVER", `${fileKind}-fs-diff.json`, { after: tree(dataRoot), before: beforeTree }); evidence("STO_002_OVER", `${fileKind}-external-calls.json`, { calls: 0 }); storage.close(); }, ); }); describe("TDD-WP0-FILE-001 atomic commit", () => { it("streams, validates and commits a managed file without a half-committed state", async () => { const { dataRoot, storage } = fixture(); const bytes = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.from("valid-image-payload")]); const before = tree(dataRoot); const committed = await storage.commitStream({ content: Readable.from([bytes.subarray(0, 5), bytes.subarray(5)]), expectedMimeType: "image/png", expectedSha256: createHash("sha256").update(bytes).digest("hex"), fileKind: "reference", fileName: "reference.png", operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: bytes.byteLength, }); expect(existsSync(join(dataRoot, committed.relative_path))).toBe(true); expect(readFileSync(join(dataRoot, committed.relative_path))).toEqual(bytes); expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 0 }); evidence("FILE_001", "fs-before.json", { entries: before }); evidence("FILE_001", "fs-after.json", { entries: tree(dataRoot) }); evidence("FILE_001", "db-diff.json", databaseCounts(storage)); evidence("FILE_001", "response.json", { bytes: committed.bytes, file_id: committed.file_id, status: "committed" }); storage.close(); }); it("rejects invalid MIME, hash, traversal and symlink inputs before commit", async () => { const { storage } = fixture(); const base = { content: Readable.from(Buffer.from("bad")), expectedMimeType: "image/png" as const, fileKind: "reference" as const, ownerRef: randomUUID(), projectedWriteBytes: 3, }; await expect(storage.commitStream({ ...base, fileName: "../bad.png", operationId: randomUUID() })).rejects.toThrow("file_name_invalid"); await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.png", expectedSha256: "0".repeat(64), operationId: randomUUID() })).rejects.toThrow("content_hash_invalid"); await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.jpg", expectedMimeType: "image/jpeg", operationId: randomUUID() })).rejects.toThrow("content_mime_invalid"); const ownerRef = randomUUID(); const target = join(storage.dataRoot, "symlink-target"); mkdirSync(target); symlinkSync(target, join(storage.dataRoot, "content", "references", ownerRef), "junction"); await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.bin", expectedMimeType: "application/octet-stream", operationId: randomUUID(), ownerRef })).rejects.toThrow(/path_escape|symbolic_link/); expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 0 }); storage.close(); }); it("reconciles each crash point into either a complete commit or compensation", async () => { const { storage } = fixture(); await expect(storage.commitStream({ content: Readable.from(Buffer.from("staged")), expectedMimeType: "application/octet-stream", failurePoint: "after_staging", fileKind: "generated", fileName: "staged.bin", operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 6, })).rejects.toThrow("injected_crash_after_staging"); const afterStaging = await storage.reconcileStartup(); expect(afterStaging).toMatchObject({ orphaned: 0, released_reservations: 1 }); await expect(storage.commitStream({ content: Readable.from(Buffer.from("orphan")), expectedMimeType: "application/octet-stream", failurePoint: "after_rename", fileKind: "generated", fileName: "orphan.png", operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 6, })).rejects.toThrow("injected_crash_after_rename"); const reconciled = await storage.reconcileStartup(); expect(reconciled).toMatchObject({ orphaned: 1, released_reservations: 1 }); expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, pending_cleanup: 1 }); await expect(storage.commitStream({ content: Readable.from(Buffer.from("committed")), expectedMimeType: "application/octet-stream", failurePoint: "after_database_commit", fileKind: "generated", fileName: "committed.bin", operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 9, })).rejects.toThrow("injected_crash_after_database_commit"); const afterDatabaseCommit = await storage.reconcileStartup(); expect(afterDatabaseCommit).toMatchObject({ orphaned: 0, released_reservations: 0 }); expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 1 }); evidence("FILE_001", "crash-points.json", { after_database_commit: afterDatabaseCommit, after_rename: reconciled, after_staging: afterStaging, }); storage.close(); }); }); describe("TDD-WP0-STO-003 cleanup and availability", () => { it("rejects a stale cleanup as one batch, then decrements only after physical deletion and remeasure", async () => { const { dataRoot, storage } = fixture(); const fsBefore = tree(dataRoot); storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 2); const first = await storage.commitBufferFixture("sticker_original", "a.png", Buffer.from("a")); const second = await storage.commitBufferFixture("sticker_thumbnail", "b.png", Buffer.from("b")); expect(storage.getState().storage_status).toBe("full"); const stale = storage.createCleanupIntent([first.file_id, second.file_id]); storage.addAssetReference(first.file_id, "release"); expect(() => storage.confirmCleanupIntent(stale.request_id)).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT"); expect(databaseCounts(storage)).toMatchObject({ pending_cleanup: 0 }); evidence("STO_003_CLEANUP", "conflict-response.json", { code: "ASSET_HISTORY_REFERENCE_CONFLICT", partial_queue: false, status: 409 }); storage.removeAssetReferences(first.file_id); const request = storage.createCleanupIntent([first.file_id, second.file_id]); storage.confirmCleanupIntent(request.request_id); expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" }); expect(existsSync(join(dataRoot, first.relative_path))).toBe(true); evidence("STO_003_CLEANUP", "queued-db-diff.json", { counts: databaseCounts(storage), state: storage.getState() }); await storage.processCleanupQueue(); expect(existsSync(join(dataRoot, first.relative_path))).toBe(false); expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 2, storage_status: "active" }); evidence("STO_003_CLEANUP", "worker-result.json", { counts: databaseCounts(storage), state: storage.getState() }); evidence("STO_003_CLEANUP", "fs-diff.json", { after: tree(dataRoot), before: fsBefore }); storage.close(); }); it("keeps the full matrix read/write effects exact", () => { const { storage } = fixture(); storage.applyControlledMeasurement(HARD_LIMIT_BYTES); const before = databaseCounts(storage); const decisions = { ai_call: storage.inspectAction("ai_call"), binary_write: storage.inspectAction("binary_write"), client_only_download: storage.inspectAction("client_only_download"), download: storage.inspectAction("download"), explicit_cleanup: storage.inspectAction("explicit_cleanup"), latest_export_write: storage.inspectAction("latest_export_write"), permanent_delete: storage.inspectAction("permanent_delete"), project_json_write: storage.inspectAction("project_json_write"), read: storage.inspectAction("read"), }; expect(decisions).toEqual({ ai_call: "reject_capacity", binary_write: "reject_capacity", client_only_download: "allow", download: "allow", explicit_cleanup: "allow", latest_export_write: "reject_capacity", permanent_delete: "allow", project_json_write: "allow", read: "allow", }); expect(databaseCounts(storage)).toEqual(before); evidence("STO_003_FULL", "response.json", decisions); evidence("STO_003_FULL", "db-diff.json", { after: databaseCounts(storage), before }); evidence("STO_003_FULL", "external-calls.json", { calls: 0 }); storage.close(); }); it("marks unavailable without mutating storage when the data root or SQLite cannot be written", async () => { const { storage } = fixture(); storage.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true }); expect(storage.getState().storage_status).toBe("unavailable"); expect(storage.inspectAction("project_json_write")).toBe("reject_unavailable"); expect(storage.inspectAction("read")).toBe("allow"); expect(storage.inspectAction("explicit_cleanup")).toBe("allow"); storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: true }); expect(storage.getState().storage_status).toBe("unavailable"); storage.applyControlledMeasurement(0); expect(storage.getState().storage_status).toBe("active"); storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: false }); expect(storage.inspectAction("explicit_cleanup")).toBe("reject_uncommitted"); expect(() => storage.createCleanupIntent([randomUUID()])).toThrow("cleanup_uncommitted"); const beforeMaintenance = databaseCounts(storage); expect(await storage.processCleanupQueue()).toEqual({ completed: 0, failed: 0 }); expect(databaseCounts(storage)).toEqual(beforeMaintenance); evidence("STO_003_UNAVAILABLE", "matrix.json", { read: storage.inspectAction("read"), write: storage.inspectAction("project_json_write"), cleanup: storage.inspectAction("explicit_cleanup"), }); evidence("STO_003_UNAVAILABLE", "db-diff.json", { counts: databaseCounts(storage), state: storage.getState() }); evidence("STO_003_UNAVAILABLE", "external-calls.json", { calls: 0 }); storage.close(); }); });