feat: complete TASK-WP0-05 storage capacity

This commit is contained in:
suyx
2026-07-27 19:10:57 +08:00
parent b5a6c6629e
commit e12a83d6d3
12 changed files with 1294 additions and 3 deletions
+30
View File
@@ -0,0 +1,30 @@
import { randomUUID } from "node:crypto";
import { describe, expect, it } from "vitest";
import { storageCapacityErrorDetails } from "../../apps/api/src/storage-policy.js";
import { createErrorEnvelope, isErrorEnvelope } from "../../packages/shared-contracts/src/index.js";
describe("TDD-WP0-STO-002-over-limit API envelope", () => {
it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"])(
"returns stable 507 details for %s",
(kind) => {
const envelope = createErrorEnvelope({
code: "STORAGE_CAPACITY_EXCEEDED",
correlationId: randomUUID(),
details: storageCapacityErrorDetails({
activeReservationBytes: 0,
managedContentBytes: 5_368_709_119,
projectedWriteBytes: 2,
}),
});
expect(isErrorEnvelope(envelope)).toBe(true);
expect(envelope.error).toMatchObject({
code: "STORAGE_CAPACITY_EXCEEDED",
details: { capacity_status: "critical", remaining_bytes: 1 },
message_key: "STORAGE_CAPACITY_EXCEEDED",
});
expect(kind).toBeTruthy();
},
);
});
+17
View File
@@ -0,0 +1,17 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Dada 容量状态验收探针</title>
<style>
body { font-family: system-ui, sans-serif; margin: 32px; color: #1f2328; }
main { max-width: 720px; }
.bar { height: 18px; background: #d0d7de; border-radius: 4px; overflow: hidden; }
.bar > span { display: block; height: 100%; background: #cf222e; width: 100%; }
table { border-collapse: collapse; margin-top: 24px; width: 100%; }
th, td { border-bottom: 1px solid #d8dee4; padding: 8px; text-align: left; }
</style>
</head>
<body><main id="root"></main><script type="module" src="/tests/e2e/fixtures/storage-capacity.ts"></script></body>
</html>
+17
View File
@@ -0,0 +1,17 @@
import { classifyCapacity, decideStorageAction } from "../../../apps/api/src/storage-policy.js";
const root = document.querySelector<HTMLElement>("#root");
if (!root) throw new Error("Fixture root is missing.");
const capacity = classifyCapacity(5_368_709_120, 0);
const actions = ["project_json_write", "binary_write", "read", "download", "permanent_delete", "explicit_cleanup"] as const;
root.innerHTML = `
<h1>本机存储容量</h1>
<p data-status>${capacity.storage_status}</p>
<div class="bar" role="progressbar" aria-valuemin="0" aria-valuemax="5368709120" aria-valuenow="5368709120"><span></span></div>
<table><thead><tr><th>操作</th><th>结果</th></tr></thead><tbody>
${actions.map((action) => `<tr><td>${action}</td><td>${decideStorageAction("full", true, action)}</td></tr>`).join("")}
</tbody></table>
<h2>存储不可用</h2>
<table data-unavailable><thead><tr><th>操作</th><th>SQLite 可写</th><th>SQLite 不可写</th></tr></thead><tbody>
${actions.map((action) => `<tr><td>${action}</td><td>${decideStorageAction("unavailable", true, action)}</td><td>${decideStorageAction("unavailable", false, action)}</td></tr>`).join("")}
</tbody></table>`;
+39
View File
@@ -0,0 +1,39 @@
import { mkdirSync } from "node:fs";
import { resolve } from "node:path";
import { expect, test } from "@playwright/test";
import { createServer, type ViteDevServer } from "vite";
let vite: ViteDevServer;
let webUrl: string;
test.beforeAll(async () => {
vite = await createServer({ configFile: false, root: process.cwd(), server: { host: "127.0.0.1", port: 0 } });
await vite.listen();
const address = vite.httpServer?.address();
if (!address || typeof address === "string") throw new Error("Vite did not expose a test port.");
webUrl = `http://127.0.0.1:${address.port}`;
});
test.afterAll(async () => vite.close());
test("TDD-WP0-STO-001 and STO-003 render the exact full action matrix", async ({ page }) => {
await page.goto(`${webUrl}/tests/e2e/fixtures/storage-capacity.html`);
await expect(page.getByRole("heading", { name: "本机存储容量" })).toBeVisible();
await expect(page.locator("[data-status]")).toHaveText("full");
await expect(page.getByRole("progressbar")).toHaveAttribute("aria-valuenow", "5368709120");
await expect(page.locator("table").first().getByRole("row").filter({ hasText: "binary_write" })).toContainText("reject_capacity");
await expect(page.locator("table").first().getByRole("row").filter({ hasText: "project_json_write" })).toContainText("allow");
await expect(page.getByRole("heading", { name: "存储不可用" })).toBeVisible();
await expect(page.locator("[data-unavailable] tr").filter({ hasText: "explicit_cleanup" })).toContainText("reject_uncommitted");
const directory = process.env.DADA_EVIDENCE_DIR_STO_001;
if (directory) {
mkdirSync(resolve(directory, "screenshots"), { recursive: true });
await page.screenshot({ path: resolve(directory, "screenshots", "capacity-bar.png") });
}
const unavailableDirectory = process.env.DADA_EVIDENCE_DIR_STO_003_UNAVAILABLE;
if (unavailableDirectory) {
mkdirSync(resolve(unavailableDirectory, "screenshots"), { recursive: true });
await page.screenshot({ path: resolve(unavailableDirectory, "screenshots", "unavailable.png"), fullPage: true });
}
});
@@ -0,0 +1,265 @@
import { createHash, randomUUID } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { Readable } from "node:stream";
import { afterEach, describe, expect, it } from "vitest";
import {
HARD_LIMIT_BYTES,
ManagedStorage,
StorageCapacityError,
type ManagedFileKind,
} from "../../apps/api/src/managed-storage.js";
const temporaryDirectories: string[] = [];
function fixture() {
const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp0-05-"));
temporaryDirectories.push(dataRoot);
for (const directory of ["db", "content/references", "content/generated", "content/exports", "managed-assets", "derived-assets", "staging"]) {
mkdirSync(join(dataRoot, directory), { recursive: true });
}
const storage = new ManagedStorage({ dataRoot, databasePath: join(dataRoot, "db", "dada.sqlite3") });
return { dataRoot, storage };
}
function databaseCounts(storage: ManagedStorage) {
return storage.inspectCounts();
}
function tree(root: string) {
return readdirSync(root, { recursive: true }).map(String).sort();
}
function evidence(caseName: string, name: string, value: unknown) {
const directory = process.env[`DADA_EVIDENCE_DIR_${caseName}`];
if (!directory) return;
mkdirSync(directory, { recursive: true });
writeFileSync(join(directory, name), `${JSON.stringify(value, null, 2)}\n`);
}
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
if (resolve(directory).startsWith(resolve(tmpdir()))) rmSync(directory, { force: true, recursive: true });
}
});
describe("TDD-WP0-STO-002 exact admission", () => {
it("allows equality, consumes its reservation and transitions to full atomically", async () => {
const { storage } = fixture();
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
const result = await storage.commitStream({
content: Readable.from(Buffer.from("x")),
expectedMimeType: "application/octet-stream",
fileKind: "generated",
fileName: "asset.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: 1,
});
expect(result.bytes).toBe(1);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 1 });
evidence("STO_002_EQUAL", "db-diff.json", { after: databaseCounts(storage), state: storage.getState() });
storage.close();
});
it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"] as ManagedFileKind[])(
"rejects %s when projected bytes are strictly over the limit without side effects",
async (fileKind) => {
const { dataRoot, storage } = fixture();
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
const beforeCounts = databaseCounts(storage);
const beforeTree = tree(dataRoot);
await expect(storage.commitStream({
content: Readable.from(Buffer.from("xx")),
expectedMimeType: "image/png",
fileKind,
fileName: "asset.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: 2,
})).rejects.toBeInstanceOf(StorageCapacityError);
expect(databaseCounts(storage)).toEqual(beforeCounts);
expect(tree(dataRoot)).toEqual(beforeTree);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 1, storage_status: "active" });
evidence("STO_002_OVER", `${fileKind}-response.json`, { code: "STORAGE_CAPACITY_EXCEEDED", status: 507 });
evidence("STO_002_OVER", `${fileKind}-db-diff.json`, { after: databaseCounts(storage), before: beforeCounts });
evidence("STO_002_OVER", `${fileKind}-fs-diff.json`, { after: tree(dataRoot), before: beforeTree });
evidence("STO_002_OVER", `${fileKind}-external-calls.json`, { calls: 0 });
storage.close();
},
);
});
describe("TDD-WP0-FILE-001 atomic commit", () => {
it("streams, validates and commits a managed file without a half-committed state", async () => {
const { dataRoot, storage } = fixture();
const bytes = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.from("valid-image-payload")]);
const before = tree(dataRoot);
const committed = await storage.commitStream({
content: Readable.from([bytes.subarray(0, 5), bytes.subarray(5)]),
expectedMimeType: "image/png",
expectedSha256: createHash("sha256").update(bytes).digest("hex"),
fileKind: "reference",
fileName: "reference.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: bytes.byteLength,
});
expect(existsSync(join(dataRoot, committed.relative_path))).toBe(true);
expect(readFileSync(join(dataRoot, committed.relative_path))).toEqual(bytes);
expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 0 });
evidence("FILE_001", "fs-before.json", { entries: before });
evidence("FILE_001", "fs-after.json", { entries: tree(dataRoot) });
evidence("FILE_001", "db-diff.json", databaseCounts(storage));
evidence("FILE_001", "response.json", { bytes: committed.bytes, file_id: committed.file_id, status: "committed" });
storage.close();
});
it("rejects invalid MIME, hash, traversal and symlink inputs before commit", async () => {
const { storage } = fixture();
const base = {
content: Readable.from(Buffer.from("bad")),
expectedMimeType: "image/png" as const,
fileKind: "reference" as const,
ownerRef: randomUUID(),
projectedWriteBytes: 3,
};
await expect(storage.commitStream({ ...base, fileName: "../bad.png", operationId: randomUUID() })).rejects.toThrow("file_name_invalid");
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.png", expectedSha256: "0".repeat(64), operationId: randomUUID() })).rejects.toThrow("content_hash_invalid");
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.jpg", expectedMimeType: "image/jpeg", operationId: randomUUID() })).rejects.toThrow("content_mime_invalid");
const ownerRef = randomUUID();
const target = join(storage.dataRoot, "symlink-target");
mkdirSync(target);
symlinkSync(target, join(storage.dataRoot, "content", "references", ownerRef), "junction");
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.bin", expectedMimeType: "application/octet-stream", operationId: randomUUID(), ownerRef })).rejects.toThrow(/path_escape|symbolic_link/);
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 0 });
storage.close();
});
it("reconciles each crash point into either a complete commit or compensation", async () => {
const { storage } = fixture();
await expect(storage.commitStream({
content: Readable.from(Buffer.from("staged")), expectedMimeType: "application/octet-stream",
failurePoint: "after_staging", fileKind: "generated", fileName: "staged.bin",
operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 6,
})).rejects.toThrow("injected_crash_after_staging");
const afterStaging = await storage.reconcileStartup();
expect(afterStaging).toMatchObject({ orphaned: 0, released_reservations: 1 });
await expect(storage.commitStream({
content: Readable.from(Buffer.from("orphan")),
expectedMimeType: "application/octet-stream",
failurePoint: "after_rename",
fileKind: "generated",
fileName: "orphan.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: 6,
})).rejects.toThrow("injected_crash_after_rename");
const reconciled = await storage.reconcileStartup();
expect(reconciled).toMatchObject({ orphaned: 1, released_reservations: 1 });
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, pending_cleanup: 1 });
await expect(storage.commitStream({
content: Readable.from(Buffer.from("committed")), expectedMimeType: "application/octet-stream",
failurePoint: "after_database_commit", fileKind: "generated", fileName: "committed.bin",
operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 9,
})).rejects.toThrow("injected_crash_after_database_commit");
const afterDatabaseCommit = await storage.reconcileStartup();
expect(afterDatabaseCommit).toMatchObject({ orphaned: 0, released_reservations: 0 });
expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 1 });
evidence("FILE_001", "crash-points.json", {
after_database_commit: afterDatabaseCommit,
after_rename: reconciled,
after_staging: afterStaging,
});
storage.close();
});
});
describe("TDD-WP0-STO-003 cleanup and availability", () => {
it("rejects a stale cleanup as one batch, then decrements only after physical deletion and remeasure", async () => {
const { dataRoot, storage } = fixture();
const fsBefore = tree(dataRoot);
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 2);
const first = await storage.commitBufferFixture("sticker_original", "a.png", Buffer.from("a"));
const second = await storage.commitBufferFixture("sticker_thumbnail", "b.png", Buffer.from("b"));
expect(storage.getState().storage_status).toBe("full");
const stale = storage.createCleanupIntent([first.file_id, second.file_id]);
storage.addAssetReference(first.file_id, "release");
expect(() => storage.confirmCleanupIntent(stale.request_id)).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT");
expect(databaseCounts(storage)).toMatchObject({ pending_cleanup: 0 });
evidence("STO_003_CLEANUP", "conflict-response.json", { code: "ASSET_HISTORY_REFERENCE_CONFLICT", partial_queue: false, status: 409 });
storage.removeAssetReferences(first.file_id);
const request = storage.createCleanupIntent([first.file_id, second.file_id]);
storage.confirmCleanupIntent(request.request_id);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
expect(existsSync(join(dataRoot, first.relative_path))).toBe(true);
evidence("STO_003_CLEANUP", "queued-db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
await storage.processCleanupQueue();
expect(existsSync(join(dataRoot, first.relative_path))).toBe(false);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 2, storage_status: "active" });
evidence("STO_003_CLEANUP", "worker-result.json", { counts: databaseCounts(storage), state: storage.getState() });
evidence("STO_003_CLEANUP", "fs-diff.json", { after: tree(dataRoot), before: fsBefore });
storage.close();
});
it("keeps the full matrix read/write effects exact", () => {
const { storage } = fixture();
storage.applyControlledMeasurement(HARD_LIMIT_BYTES);
const before = databaseCounts(storage);
const decisions = {
ai_call: storage.inspectAction("ai_call"),
binary_write: storage.inspectAction("binary_write"),
client_only_download: storage.inspectAction("client_only_download"),
download: storage.inspectAction("download"),
explicit_cleanup: storage.inspectAction("explicit_cleanup"),
latest_export_write: storage.inspectAction("latest_export_write"),
permanent_delete: storage.inspectAction("permanent_delete"),
project_json_write: storage.inspectAction("project_json_write"),
read: storage.inspectAction("read"),
};
expect(decisions).toEqual({
ai_call: "reject_capacity", binary_write: "reject_capacity", client_only_download: "allow",
download: "allow", explicit_cleanup: "allow", latest_export_write: "reject_capacity",
permanent_delete: "allow", project_json_write: "allow", read: "allow",
});
expect(databaseCounts(storage)).toEqual(before);
evidence("STO_003_FULL", "response.json", decisions);
evidence("STO_003_FULL", "db-diff.json", { after: databaseCounts(storage), before });
evidence("STO_003_FULL", "external-calls.json", { calls: 0 });
storage.close();
});
it("marks unavailable without mutating storage when the data root or SQLite cannot be written", async () => {
const { storage } = fixture();
storage.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true });
expect(storage.getState().storage_status).toBe("unavailable");
expect(storage.inspectAction("project_json_write")).toBe("reject_unavailable");
expect(storage.inspectAction("read")).toBe("allow");
expect(storage.inspectAction("explicit_cleanup")).toBe("allow");
storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: true });
expect(storage.getState().storage_status).toBe("unavailable");
storage.applyControlledMeasurement(0);
expect(storage.getState().storage_status).toBe("active");
storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: false });
expect(storage.inspectAction("explicit_cleanup")).toBe("reject_uncommitted");
expect(() => storage.createCleanupIntent([randomUUID()])).toThrow("cleanup_uncommitted");
const beforeMaintenance = databaseCounts(storage);
expect(await storage.processCleanupQueue()).toEqual({ completed: 0, failed: 0 });
expect(databaseCounts(storage)).toEqual(beforeMaintenance);
evidence("STO_003_UNAVAILABLE", "matrix.json", {
read: storage.inspectAction("read"),
write: storage.inspectAction("project_json_write"),
cleanup: storage.inspectAction("explicit_cleanup"),
});
evidence("STO_003_UNAVAILABLE", "db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
evidence("STO_003_UNAVAILABLE", "external-calls.json", { calls: 0 });
storage.close();
});
});
+80
View File
@@ -0,0 +1,80 @@
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
CAPACITY_COUNTED_FILE_KINDS,
CAPACITY_EXCLUDED_STORAGE,
HARD_LIMIT_BYTES,
classifyCapacity,
decideStorageAction,
} from "../../apps/api/src/storage-policy.js";
describe("TDD-WP0-STO-001-byte-thresholds", () => {
it.each([
[4_294_967_295, "normal", "active"],
[4_294_967_296, "warning", "active"],
[4_831_838_207, "warning", "active"],
[4_831_838_208, "critical", "active"],
[5_368_709_119, "critical", "active"],
[5_368_709_120, "critical", "full"],
[5_368_709_121, "critical", "full"],
] as const)("classifies %i bytes", (bytes, notice, status) => {
expect(classifyCapacity(bytes, 0)).toEqual({ capacity_notice_level: notice, storage_status: status });
});
it("records the deterministic threshold result without deleting content", () => {
const values = [
4_294_967_295,
4_294_967_296,
4_831_838_207,
4_831_838_208,
5_368_709_119,
HARD_LIMIT_BYTES,
5_368_709_121,
].map((bytes) => ({ bytes, ...classifyCapacity(bytes, 0) }));
const directory = process.env.DADA_EVIDENCE_DIR_STO_001;
if (directory) {
mkdirSync(directory, { recursive: true });
writeFileSync(join(directory, "thresholds.json"), `${JSON.stringify({ automatic_cleanup: false, values }, null, 2)}\n`);
writeFileSync(join(directory, "db-diff.json"), `${JSON.stringify({ changed_fields: ["managed_content_bytes", "capacity_notice_level", "storage_status", "measured_at"], deleted_files: 0 }, null, 2)}\n`);
}
});
it("counts only managed binary classes and excludes infrastructure and downloads", () => {
expect(CAPACITY_COUNTED_FILE_KINDS).toEqual([
"reference", "generated", "export", "derived", "sticker_original", "sticker_thumbnail",
]);
expect(CAPACITY_EXCLUDED_STORAGE).toEqual([
"read_only_assets", "application_files", "database", "logs", "audit", "browser_cache", "user_downloads",
]);
});
});
describe("TDD-WP0-STO-003 action matrices", () => {
it("allows only the frozen full-state actions", () => {
expect(decideStorageAction("full", true, "project_json_write")).toBe("allow");
expect(decideStorageAction("full", true, "read")).toBe("allow");
expect(decideStorageAction("full", true, "download")).toBe("allow");
expect(decideStorageAction("full", true, "client_only_download")).toBe("allow");
expect(decideStorageAction("full", true, "permanent_delete")).toBe("allow");
expect(decideStorageAction("full", true, "explicit_cleanup")).toBe("allow");
expect(decideStorageAction("full", true, "binary_write")).toBe("reject_capacity");
expect(decideStorageAction("full", true, "ai_call")).toBe("reject_capacity");
expect(decideStorageAction("full", true, "latest_export_write")).toBe("reject_capacity");
});
it("blocks all new backend writes while unavailable and gates cleanup on SQLite", () => {
for (const action of ["project_json_write", "binary_write", "ai_call", "latest_export_write"] as const) {
expect(decideStorageAction("unavailable", true, action)).toBe("reject_unavailable");
}
for (const action of ["read", "download", "client_only_download"] as const) {
expect(decideStorageAction("unavailable", false, action)).toBe("allow");
}
expect(decideStorageAction("unavailable", true, "permanent_delete")).toBe("allow");
expect(decideStorageAction("unavailable", false, "permanent_delete")).toBe("reject_uncommitted");
expect(decideStorageAction("unavailable", true, "explicit_cleanup")).toBe("allow");
expect(decideStorageAction("unavailable", false, "explicit_cleanup")).toBe("reject_uncommitted");
});
});
@@ -0,0 +1,17 @@
import { describe, expect, it, vi } from "vitest";
import { runStorageMaintenance } from "../../apps/worker/src/storage-maintenance.js";
describe("TDD-WP0-FILE-001 worker reconciliation", () => {
it("reconciles startup before processing physical cleanup", async () => {
const target = {
processCleanupQueue: vi.fn(async () => ({ completed: 1, failed: 0 })),
reconcileStartup: vi.fn(async () => ({ orphaned: 1, released_reservations: 1 })),
};
await expect(runStorageMaintenance(target)).resolves.toEqual({
cleanup: { completed: 1, failed: 0 },
reconciliation: { orphaned: 1, released_reservations: 1 },
});
expect(target.reconcileStartup).toHaveBeenCalledBefore(target.processCleanupQueue);
});
});