266 lines
14 KiB
TypeScript
266 lines
14 KiB
TypeScript
import { createHash, randomUUID } from "node:crypto";
|
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import { Readable } from "node:stream";
|
|
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
HARD_LIMIT_BYTES,
|
|
ManagedStorage,
|
|
StorageCapacityError,
|
|
type ManagedFileKind,
|
|
} from "../../apps/api/src/managed-storage.js";
|
|
|
|
const temporaryDirectories: string[] = [];
|
|
|
|
function fixture() {
|
|
const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp0-05-"));
|
|
temporaryDirectories.push(dataRoot);
|
|
for (const directory of ["db", "content/references", "content/generated", "content/exports", "managed-assets", "derived-assets", "staging"]) {
|
|
mkdirSync(join(dataRoot, directory), { recursive: true });
|
|
}
|
|
const storage = new ManagedStorage({ dataRoot, databasePath: join(dataRoot, "db", "dada.sqlite3") });
|
|
return { dataRoot, storage };
|
|
}
|
|
|
|
function databaseCounts(storage: ManagedStorage) {
|
|
return storage.inspectCounts();
|
|
}
|
|
|
|
function tree(root: string) {
|
|
return readdirSync(root, { recursive: true }).map(String).sort();
|
|
}
|
|
|
|
function evidence(caseName: string, name: string, value: unknown) {
|
|
const directory = process.env[`DADA_EVIDENCE_DIR_${caseName}`];
|
|
if (!directory) return;
|
|
mkdirSync(directory, { recursive: true });
|
|
writeFileSync(join(directory, name), `${JSON.stringify(value, null, 2)}\n`);
|
|
}
|
|
|
|
afterEach(() => {
|
|
for (const directory of temporaryDirectories.splice(0)) {
|
|
if (resolve(directory).startsWith(resolve(tmpdir()))) rmSync(directory, { force: true, recursive: true });
|
|
}
|
|
});
|
|
|
|
describe("TDD-WP0-STO-002 exact admission", () => {
|
|
it("allows equality, consumes its reservation and transitions to full atomically", async () => {
|
|
const { storage } = fixture();
|
|
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
|
|
const result = await storage.commitStream({
|
|
content: Readable.from(Buffer.from("x")),
|
|
expectedMimeType: "application/octet-stream",
|
|
fileKind: "generated",
|
|
fileName: "asset.png",
|
|
operationId: randomUUID(),
|
|
ownerRef: randomUUID(),
|
|
projectedWriteBytes: 1,
|
|
});
|
|
expect(result.bytes).toBe(1);
|
|
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
|
|
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 1 });
|
|
evidence("STO_002_EQUAL", "db-diff.json", { after: databaseCounts(storage), state: storage.getState() });
|
|
storage.close();
|
|
});
|
|
|
|
it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"] as ManagedFileKind[])(
|
|
"rejects %s when projected bytes are strictly over the limit without side effects",
|
|
async (fileKind) => {
|
|
const { dataRoot, storage } = fixture();
|
|
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
|
|
const beforeCounts = databaseCounts(storage);
|
|
const beforeTree = tree(dataRoot);
|
|
await expect(storage.commitStream({
|
|
content: Readable.from(Buffer.from("xx")),
|
|
expectedMimeType: "image/png",
|
|
fileKind,
|
|
fileName: "asset.png",
|
|
operationId: randomUUID(),
|
|
ownerRef: randomUUID(),
|
|
projectedWriteBytes: 2,
|
|
})).rejects.toBeInstanceOf(StorageCapacityError);
|
|
expect(databaseCounts(storage)).toEqual(beforeCounts);
|
|
expect(tree(dataRoot)).toEqual(beforeTree);
|
|
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 1, storage_status: "active" });
|
|
evidence("STO_002_OVER", `${fileKind}-response.json`, { code: "STORAGE_CAPACITY_EXCEEDED", status: 507 });
|
|
evidence("STO_002_OVER", `${fileKind}-db-diff.json`, { after: databaseCounts(storage), before: beforeCounts });
|
|
evidence("STO_002_OVER", `${fileKind}-fs-diff.json`, { after: tree(dataRoot), before: beforeTree });
|
|
evidence("STO_002_OVER", `${fileKind}-external-calls.json`, { calls: 0 });
|
|
storage.close();
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("TDD-WP0-FILE-001 atomic commit", () => {
|
|
it("streams, validates and commits a managed file without a half-committed state", async () => {
|
|
const { dataRoot, storage } = fixture();
|
|
const bytes = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.from("valid-image-payload")]);
|
|
const before = tree(dataRoot);
|
|
const committed = await storage.commitStream({
|
|
content: Readable.from([bytes.subarray(0, 5), bytes.subarray(5)]),
|
|
expectedMimeType: "image/png",
|
|
expectedSha256: createHash("sha256").update(bytes).digest("hex"),
|
|
fileKind: "reference",
|
|
fileName: "reference.png",
|
|
operationId: randomUUID(),
|
|
ownerRef: randomUUID(),
|
|
projectedWriteBytes: bytes.byteLength,
|
|
});
|
|
expect(existsSync(join(dataRoot, committed.relative_path))).toBe(true);
|
|
expect(readFileSync(join(dataRoot, committed.relative_path))).toEqual(bytes);
|
|
expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 0 });
|
|
evidence("FILE_001", "fs-before.json", { entries: before });
|
|
evidence("FILE_001", "fs-after.json", { entries: tree(dataRoot) });
|
|
evidence("FILE_001", "db-diff.json", databaseCounts(storage));
|
|
evidence("FILE_001", "response.json", { bytes: committed.bytes, file_id: committed.file_id, status: "committed" });
|
|
storage.close();
|
|
});
|
|
|
|
it("rejects invalid MIME, hash, traversal and symlink inputs before commit", async () => {
|
|
const { storage } = fixture();
|
|
const base = {
|
|
content: Readable.from(Buffer.from("bad")),
|
|
expectedMimeType: "image/png" as const,
|
|
fileKind: "reference" as const,
|
|
ownerRef: randomUUID(),
|
|
projectedWriteBytes: 3,
|
|
};
|
|
await expect(storage.commitStream({ ...base, fileName: "../bad.png", operationId: randomUUID() })).rejects.toThrow("file_name_invalid");
|
|
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.png", expectedSha256: "0".repeat(64), operationId: randomUUID() })).rejects.toThrow("content_hash_invalid");
|
|
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.jpg", expectedMimeType: "image/jpeg", operationId: randomUUID() })).rejects.toThrow("content_mime_invalid");
|
|
const ownerRef = randomUUID();
|
|
const target = join(storage.dataRoot, "symlink-target");
|
|
mkdirSync(target);
|
|
symlinkSync(target, join(storage.dataRoot, "content", "references", ownerRef), "junction");
|
|
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.bin", expectedMimeType: "application/octet-stream", operationId: randomUUID(), ownerRef })).rejects.toThrow(/path_escape|symbolic_link/);
|
|
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 0 });
|
|
storage.close();
|
|
});
|
|
|
|
it("reconciles each crash point into either a complete commit or compensation", async () => {
|
|
const { storage } = fixture();
|
|
await expect(storage.commitStream({
|
|
content: Readable.from(Buffer.from("staged")), expectedMimeType: "application/octet-stream",
|
|
failurePoint: "after_staging", fileKind: "generated", fileName: "staged.bin",
|
|
operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 6,
|
|
})).rejects.toThrow("injected_crash_after_staging");
|
|
const afterStaging = await storage.reconcileStartup();
|
|
expect(afterStaging).toMatchObject({ orphaned: 0, released_reservations: 1 });
|
|
|
|
await expect(storage.commitStream({
|
|
content: Readable.from(Buffer.from("orphan")),
|
|
expectedMimeType: "application/octet-stream",
|
|
failurePoint: "after_rename",
|
|
fileKind: "generated",
|
|
fileName: "orphan.png",
|
|
operationId: randomUUID(),
|
|
ownerRef: randomUUID(),
|
|
projectedWriteBytes: 6,
|
|
})).rejects.toThrow("injected_crash_after_rename");
|
|
const reconciled = await storage.reconcileStartup();
|
|
expect(reconciled).toMatchObject({ orphaned: 1, released_reservations: 1 });
|
|
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, pending_cleanup: 1 });
|
|
|
|
await expect(storage.commitStream({
|
|
content: Readable.from(Buffer.from("committed")), expectedMimeType: "application/octet-stream",
|
|
failurePoint: "after_database_commit", fileKind: "generated", fileName: "committed.bin",
|
|
operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 9,
|
|
})).rejects.toThrow("injected_crash_after_database_commit");
|
|
const afterDatabaseCommit = await storage.reconcileStartup();
|
|
expect(afterDatabaseCommit).toMatchObject({ orphaned: 0, released_reservations: 0 });
|
|
expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 1 });
|
|
evidence("FILE_001", "crash-points.json", {
|
|
after_database_commit: afterDatabaseCommit,
|
|
after_rename: reconciled,
|
|
after_staging: afterStaging,
|
|
});
|
|
storage.close();
|
|
});
|
|
});
|
|
|
|
describe("TDD-WP0-STO-003 cleanup and availability", () => {
|
|
it("rejects a stale cleanup as one batch, then decrements only after physical deletion and remeasure", async () => {
|
|
const { dataRoot, storage } = fixture();
|
|
const fsBefore = tree(dataRoot);
|
|
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 2);
|
|
const first = await storage.commitBufferFixture("sticker_original", "a.png", Buffer.from("a"));
|
|
const second = await storage.commitBufferFixture("sticker_thumbnail", "b.png", Buffer.from("b"));
|
|
expect(storage.getState().storage_status).toBe("full");
|
|
const stale = storage.createCleanupIntent([first.file_id, second.file_id]);
|
|
storage.addAssetReference(first.file_id, "release");
|
|
expect(() => storage.confirmCleanupIntent(stale.request_id)).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT");
|
|
expect(databaseCounts(storage)).toMatchObject({ pending_cleanup: 0 });
|
|
evidence("STO_003_CLEANUP", "conflict-response.json", { code: "ASSET_HISTORY_REFERENCE_CONFLICT", partial_queue: false, status: 409 });
|
|
|
|
storage.removeAssetReferences(first.file_id);
|
|
const request = storage.createCleanupIntent([first.file_id, second.file_id]);
|
|
storage.confirmCleanupIntent(request.request_id);
|
|
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
|
|
expect(existsSync(join(dataRoot, first.relative_path))).toBe(true);
|
|
evidence("STO_003_CLEANUP", "queued-db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
|
|
await storage.processCleanupQueue();
|
|
expect(existsSync(join(dataRoot, first.relative_path))).toBe(false);
|
|
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 2, storage_status: "active" });
|
|
evidence("STO_003_CLEANUP", "worker-result.json", { counts: databaseCounts(storage), state: storage.getState() });
|
|
evidence("STO_003_CLEANUP", "fs-diff.json", { after: tree(dataRoot), before: fsBefore });
|
|
storage.close();
|
|
});
|
|
|
|
it("keeps the full matrix read/write effects exact", () => {
|
|
const { storage } = fixture();
|
|
storage.applyControlledMeasurement(HARD_LIMIT_BYTES);
|
|
const before = databaseCounts(storage);
|
|
const decisions = {
|
|
ai_call: storage.inspectAction("ai_call"),
|
|
binary_write: storage.inspectAction("binary_write"),
|
|
client_only_download: storage.inspectAction("client_only_download"),
|
|
download: storage.inspectAction("download"),
|
|
explicit_cleanup: storage.inspectAction("explicit_cleanup"),
|
|
latest_export_write: storage.inspectAction("latest_export_write"),
|
|
permanent_delete: storage.inspectAction("permanent_delete"),
|
|
project_json_write: storage.inspectAction("project_json_write"),
|
|
read: storage.inspectAction("read"),
|
|
};
|
|
expect(decisions).toEqual({
|
|
ai_call: "reject_capacity", binary_write: "reject_capacity", client_only_download: "allow",
|
|
download: "allow", explicit_cleanup: "allow", latest_export_write: "reject_capacity",
|
|
permanent_delete: "allow", project_json_write: "allow", read: "allow",
|
|
});
|
|
expect(databaseCounts(storage)).toEqual(before);
|
|
evidence("STO_003_FULL", "response.json", decisions);
|
|
evidence("STO_003_FULL", "db-diff.json", { after: databaseCounts(storage), before });
|
|
evidence("STO_003_FULL", "external-calls.json", { calls: 0 });
|
|
storage.close();
|
|
});
|
|
|
|
it("marks unavailable without mutating storage when the data root or SQLite cannot be written", async () => {
|
|
const { storage } = fixture();
|
|
storage.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true });
|
|
expect(storage.getState().storage_status).toBe("unavailable");
|
|
expect(storage.inspectAction("project_json_write")).toBe("reject_unavailable");
|
|
expect(storage.inspectAction("read")).toBe("allow");
|
|
expect(storage.inspectAction("explicit_cleanup")).toBe("allow");
|
|
storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: true });
|
|
expect(storage.getState().storage_status).toBe("unavailable");
|
|
storage.applyControlledMeasurement(0);
|
|
expect(storage.getState().storage_status).toBe("active");
|
|
storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: false });
|
|
expect(storage.inspectAction("explicit_cleanup")).toBe("reject_uncommitted");
|
|
expect(() => storage.createCleanupIntent([randomUUID()])).toThrow("cleanup_uncommitted");
|
|
const beforeMaintenance = databaseCounts(storage);
|
|
expect(await storage.processCleanupQueue()).toEqual({ completed: 0, failed: 0 });
|
|
expect(databaseCounts(storage)).toEqual(beforeMaintenance);
|
|
evidence("STO_003_UNAVAILABLE", "matrix.json", {
|
|
read: storage.inspectAction("read"),
|
|
write: storage.inspectAction("project_json_write"),
|
|
cleanup: storage.inspectAction("explicit_cleanup"),
|
|
});
|
|
evidence("STO_003_UNAVAILABLE", "db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
|
|
evidence("STO_003_UNAVAILABLE", "external-calls.json", { calls: 0 });
|
|
storage.close();
|
|
});
|
|
});
|