feat: complete TASK-WP0-05 storage capacity

This commit is contained in:
suyx
2026-07-27 19:10:57 +08:00
parent b5a6c6629e
commit e12a83d6d3
12 changed files with 1294 additions and 3 deletions
+614
View File
@@ -0,0 +1,614 @@
import { createHash, randomUUID } from "node:crypto";
import {
createWriteStream,
existsSync,
mkdirSync,
readdirSync,
renameSync,
rmSync,
statSync,
} from "node:fs";
import { createRequire } from "node:module";
import { basename, dirname, extname, join, relative, resolve } from "node:path";
import { Readable, Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
import type BetterSqlite3 from "better-sqlite3";
import { resolvePathWithinRoot } from "./local-data-root.js";
import {
HARD_LIMIT_BYTES,
classifyCapacity,
decideStorageAction,
storageCapacityErrorDetails,
type StorageAction,
} from "./storage-policy.js";
const require = createRequire(import.meta.url);
const Database = require("better-sqlite3") as typeof BetterSqlite3;
const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const safeFileNamePattern = /^[A-Za-z0-9][A-Za-z0-9._-]{0,159}$/;
export { HARD_LIMIT_BYTES } from "./storage-policy.js";
export type ManagedFileKind =
| "reference"
| "generated"
| "export"
| "derived"
| "sticker_original"
| "sticker_thumbnail";
export type CommitFailurePoint = "after_staging" | "after_rename" | "after_database_commit";
interface StorageStateRow {
active_storage_reservations_bytes: number;
capacity_notice_level: "normal" | "warning" | "critical";
data_root_ref: "configured_local_data_root";
hard_limit_bytes: number;
managed_content_bytes: number;
measured_at: string;
storage_backend: "local_data_root";
storage_status: "active" | "full" | "unavailable";
version: number;
}
interface ManagedFileRow {
byte_size: number;
file_id: string;
file_kind: ManagedFileKind;
relative_path: string;
status: "committed" | "purged";
}
interface CleanupQueueRow {
byte_size: number;
cleanup_id: string;
counts_toward_managed: 0 | 1;
managed_file_id: string | null;
relative_path: string;
}
export class StorageCapacityError extends Error {
readonly code = "STORAGE_CAPACITY_EXCEEDED";
readonly httpStatus = 507;
readonly details: ReturnType<typeof storageCapacityErrorDetails>;
constructor(input: { activeReservationBytes: number; managedContentBytes: number; projectedWriteBytes: number }) {
super("STORAGE_CAPACITY_EXCEEDED");
this.details = storageCapacityErrorDetails(input);
}
}
export class StorageUnavailableError extends Error {
readonly code = "storage_unavailable";
constructor() {
super("storage_unavailable");
}
}
class InjectedCrashError extends Error {}
function now() {
return new Date().toISOString();
}
function validatePositiveBytes(value: number, name: string) {
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name}_invalid`);
}
function sniffMime(prefix: Buffer) {
if (prefix.length >= 8 && prefix.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) {
return "image/png";
}
if (prefix.length >= 3 && prefix[0] === 0xff && prefix[1] === 0xd8 && prefix[2] === 0xff) return "image/jpeg";
return "application/octet-stream";
}
function listFiles(root: string): string[] {
if (!existsSync(root)) return [];
const files: string[] = [];
for (const entry of readdirSync(root, { withFileTypes: true })) {
const child = join(root, entry.name);
if (entry.isDirectory()) files.push(...listFiles(child));
else if (entry.isFile()) files.push(child);
}
return files;
}
export interface CommitStreamInput {
content: Readable;
expectedMimeType: "image/png" | "image/jpeg" | "application/octet-stream";
expectedSha256?: string;
failurePoint?: CommitFailurePoint;
fileKind: ManagedFileKind;
fileName: string;
operationId: string;
ownerRef: string;
projectedWriteBytes: number;
}
export class ManagedStorage {
readonly dataRoot: string;
readonly databasePath: string;
private readonly database: BetterSqlite3.Database;
private dataRootWritable = true;
private diskSpaceAvailable = true;
private sqliteWritable = true;
private measurementBaselineBytes = 0;
private recoveryRequiresRemeasure = false;
constructor(input: { dataRoot: string; databasePath: string }) {
this.dataRoot = resolve(input.dataRoot);
this.databasePath = resolve(input.databasePath);
if (!this.databasePath.startsWith(`${this.dataRoot}${process.platform === "win32" ? "\\" : "/"}`)) {
throw new Error("database_outside_data_root");
}
mkdirSync(dirname(this.databasePath), { recursive: true });
this.database = new Database(this.databasePath);
this.database.pragma("journal_mode = WAL");
this.database.pragma("foreign_keys = ON");
this.database.pragma("busy_timeout = 5000");
this.migrate();
const state = this.database.prepare("SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1").get() as { managed_content_bytes: number };
this.measurementBaselineBytes = Math.max(0, state.managed_content_bytes - this.physicalManagedBytes());
}
private migrate() {
this.database.exec(`
CREATE TABLE IF NOT EXISTS local_backend_storage_state (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
storage_backend TEXT NOT NULL CHECK (storage_backend = 'local_data_root'),
data_root_ref TEXT NOT NULL CHECK (data_root_ref = 'configured_local_data_root'),
hard_limit_bytes INTEGER NOT NULL,
managed_content_bytes INTEGER NOT NULL CHECK (managed_content_bytes >= 0),
capacity_notice_level TEXT NOT NULL CHECK (capacity_notice_level IN ('normal', 'warning', 'critical')),
storage_status TEXT NOT NULL CHECK (storage_status IN ('active', 'full', 'unavailable')),
measured_at TEXT NOT NULL,
version INTEGER NOT NULL CHECK (version >= 1)
);
CREATE TABLE IF NOT EXISTS storage_reservations (
reservation_id TEXT PRIMARY KEY,
operation_id TEXT NOT NULL UNIQUE,
projected_bytes INTEGER NOT NULL CHECK (projected_bytes > 0),
status TEXT NOT NULL CHECK (status IN ('active', 'consumed', 'released')),
created_at TEXT NOT NULL,
resolved_at TEXT
);
CREATE TABLE IF NOT EXISTS managed_files (
file_id TEXT PRIMARY KEY,
file_kind TEXT NOT NULL CHECK (file_kind IN ('reference', 'generated', 'export', 'derived', 'sticker_original', 'sticker_thumbnail')),
relative_path TEXT NOT NULL UNIQUE,
byte_size INTEGER NOT NULL CHECK (byte_size > 0),
mime_type TEXT NOT NULL,
sha256 TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('committed', 'purged')),
created_at TEXT NOT NULL,
purged_at TEXT
);
CREATE TABLE IF NOT EXISTS file_cleanup_queue (
cleanup_id TEXT PRIMARY KEY,
managed_file_id TEXT,
relative_path TEXT NOT NULL UNIQUE,
byte_size INTEGER NOT NULL CHECK (byte_size >= 0),
counts_toward_managed INTEGER NOT NULL CHECK (counts_toward_managed IN (0, 1)),
reason TEXT NOT NULL CHECK (reason IN ('compensation', 'purge')),
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'failed')),
created_at TEXT NOT NULL,
completed_at TEXT,
last_error TEXT
);
CREATE TABLE IF NOT EXISTS project_asset_refs (
reference_id TEXT PRIMARY KEY,
managed_file_id TEXT NOT NULL,
reference_type TEXT NOT NULL CHECK (reference_type IN ('project', 'release')),
created_at TEXT NOT NULL,
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
);
CREATE TABLE IF NOT EXISTS asset_cleanup_requests (
request_id TEXT PRIMARY KEY,
status TEXT NOT NULL CHECK (status IN ('pending_confirmation', 'denied', 'queued', 'completed')),
created_at TEXT NOT NULL,
confirmed_at TEXT
);
CREATE TABLE IF NOT EXISTS asset_cleanup_request_items (
request_id TEXT NOT NULL,
managed_file_id TEXT NOT NULL,
PRIMARY KEY (request_id, managed_file_id),
FOREIGN KEY (request_id) REFERENCES asset_cleanup_requests(request_id),
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
);
CREATE TABLE IF NOT EXISTS admin_operation_logs (
log_id TEXT PRIMARY KEY,
operation TEXT NOT NULL,
outcome TEXT NOT NULL,
target_ref TEXT NOT NULL,
created_at TEXT NOT NULL
);
`);
const initial = classifyCapacity(0, 0);
this.database.prepare(`
INSERT OR IGNORE INTO local_backend_storage_state
(singleton, storage_backend, data_root_ref, hard_limit_bytes, managed_content_bytes, capacity_notice_level, storage_status, measured_at, version)
VALUES (1, 'local_data_root', 'configured_local_data_root', ?, 0, ?, ?, ?, 1)
`).run(HARD_LIMIT_BYTES, initial.capacity_notice_level, initial.storage_status, now());
}
close() {
this.database.close();
}
getState(): StorageStateRow {
const state = this.database.prepare("SELECT * FROM local_backend_storage_state WHERE singleton = 1").get() as Omit<StorageStateRow, "active_storage_reservations_bytes">;
const withReservations = { ...state, active_storage_reservations_bytes: this.activeReservationBytes() };
if (!this.dataRootWritable || !this.diskSpaceAvailable || !this.sqliteWritable) {
return { ...withReservations, storage_status: "unavailable" };
}
return withReservations;
}
private activeReservationBytes(excludingOperationId?: string) {
const row = this.database.prepare(`
SELECT COALESCE(SUM(projected_bytes), 0) AS bytes
FROM storage_reservations
WHERE status = 'active' AND (? IS NULL OR operation_id <> ?)
`).get(excludingOperationId ?? null, excludingOperationId ?? null) as { bytes: number };
return row.bytes;
}
private refreshState() {
if (!this.sqliteWritable) return;
const state = this.database.prepare("SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1").get() as { managed_content_bytes: number };
const classification = classifyCapacity(state.managed_content_bytes, this.activeReservationBytes());
const storageStatus = !this.dataRootWritable || !this.diskSpaceAvailable || this.recoveryRequiresRemeasure
? "unavailable"
: classification.storage_status;
this.database.prepare(`
UPDATE local_backend_storage_state
SET capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1
WHERE singleton = 1
`).run(classification.capacity_notice_level, storageStatus, now());
}
applyControlledMeasurement(bytes: number) {
if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error("managed_content_bytes_invalid");
this.measurementBaselineBytes = Math.max(0, bytes - this.physicalManagedBytes());
this.recoveryRequiresRemeasure = false;
const classification = classifyCapacity(bytes, this.activeReservationBytes());
this.database.prepare(`
UPDATE local_backend_storage_state
SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1
WHERE singleton = 1
`).run(bytes, classification.capacity_notice_level, classification.storage_status, now());
return this.getState();
}
setAvailability(input: { dataRootWritable: boolean; diskSpaceAvailable: boolean; sqliteWritable: boolean }) {
this.dataRootWritable = input.dataRootWritable;
this.diskSpaceAvailable = input.diskSpaceAvailable;
this.sqliteWritable = input.sqliteWritable;
if (!input.dataRootWritable || !input.diskSpaceAvailable || !input.sqliteWritable) this.recoveryRequiresRemeasure = true;
this.refreshState();
}
private physicalManagedBytes() {
return ["content", "managed-assets", "derived-assets"]
.flatMap((root) => listFiles(resolvePathWithinRoot(this.dataRoot, root)))
.reduce((sum, path) => sum + statSync(path).size, 0);
}
private recordPhysicalMeasurement() {
const bytes = this.measurementBaselineBytes + this.physicalManagedBytes();
this.recoveryRequiresRemeasure = false;
const classification = classifyCapacity(bytes, this.activeReservationBytes());
this.database.prepare(`
UPDATE local_backend_storage_state
SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1
WHERE singleton = 1
`).run(bytes, classification.capacity_notice_level, classification.storage_status, now());
}
inspectAction(action: StorageAction) {
return decideStorageAction(this.getState().storage_status, this.sqliteWritable, action);
}
private assertWritable() {
if (this.inspectAction("binary_write") === "reject_unavailable") throw new StorageUnavailableError();
}
private reserve(operationId: string, projectedWriteBytes: number) {
this.assertWritable();
if (!uuidPattern.test(operationId)) throw new Error("operation_id_invalid");
validatePositiveBytes(projectedWriteBytes, "projected_write_bytes");
const transaction = this.database.transaction(() => {
const state = this.getState();
const activeReservationBytes = this.activeReservationBytes();
if (state.managed_content_bytes + activeReservationBytes + projectedWriteBytes > HARD_LIMIT_BYTES) {
throw new StorageCapacityError({ activeReservationBytes, managedContentBytes: state.managed_content_bytes, projectedWriteBytes });
}
const reservationId = randomUUID();
this.database.prepare(`
INSERT INTO storage_reservations (reservation_id, operation_id, projected_bytes, status, created_at)
VALUES (?, ?, ?, 'active', ?)
`).run(reservationId, operationId, projectedWriteBytes, now());
this.refreshState();
return reservationId;
});
return transaction();
}
private releaseReservation(operationId: string) {
this.database.prepare(`
UPDATE storage_reservations SET status = 'released', resolved_at = ?
WHERE operation_id = ? AND status = 'active'
`).run(now(), operationId);
this.refreshState();
}
private destination(input: CommitStreamInput, fileId: string) {
if (!safeFileNamePattern.test(input.fileName) || basename(input.fileName) !== input.fileName || input.fileName.includes("..")) {
throw new Error("file_name_invalid");
}
if (!uuidPattern.test(input.ownerRef)) throw new Error("owner_ref_invalid");
const group = input.fileKind === "reference"
? "content/references"
: input.fileKind === "generated"
? "content/generated"
: input.fileKind === "export"
? "content/exports"
: input.fileKind === "derived"
? "derived-assets"
: input.fileKind === "sticker_original"
? "managed-assets/stickers/original"
: "managed-assets/stickers/thumbnail";
const extension = extname(input.fileName).toLowerCase();
const relativePath = `${group}/${input.ownerRef}/${fileId}${extension}`;
return { absolutePath: resolvePathWithinRoot(this.dataRoot, relativePath), relativePath };
}
async commitStream(input: CommitStreamInput) {
const fileId = randomUUID();
const destination = this.destination(input, fileId);
this.reserve(input.operationId, input.projectedWriteBytes);
const stagingDirectory = resolvePathWithinRoot(this.dataRoot, `staging/${input.operationId}`);
const stagingPath = resolvePathWithinRoot(this.dataRoot, `staging/${input.operationId}/payload.tmp`);
let renamed = false;
try {
mkdirSync(stagingDirectory, { recursive: true });
const hash = createHash("sha256");
let byteSize = 0;
let prefix = Buffer.alloc(0);
const inspect = new Transform({
transform(chunk: Buffer | string, encoding, callback) {
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding);
byteSize += bytes.byteLength;
hash.update(bytes);
if (prefix.byteLength < 16) prefix = Buffer.concat([prefix, bytes.subarray(0, 16 - prefix.byteLength)]);
callback(null, bytes);
},
});
await pipeline(input.content, inspect, createWriteStream(stagingPath, { flags: "wx" }));
validatePositiveBytes(byteSize, "actual_write_bytes");
const sha256 = hash.digest("hex");
if (input.expectedSha256 && sha256.toLowerCase() !== input.expectedSha256.toLowerCase()) throw new Error("content_hash_invalid");
if (sniffMime(prefix) !== input.expectedMimeType) throw new Error("content_mime_invalid");
const state = this.getState();
const otherReservations = this.activeReservationBytes(input.operationId);
if (state.managed_content_bytes + otherReservations + byteSize > HARD_LIMIT_BYTES) {
throw new StorageCapacityError({ activeReservationBytes: otherReservations, managedContentBytes: state.managed_content_bytes, projectedWriteBytes: byteSize });
}
this.database.prepare(`UPDATE storage_reservations SET projected_bytes = ? WHERE operation_id = ? AND status = 'active'`).run(byteSize, input.operationId);
this.refreshState();
if (input.failurePoint === "after_staging") throw new InjectedCrashError("injected_crash_after_staging");
mkdirSync(dirname(destination.absolutePath), { recursive: true });
renameSync(stagingPath, destination.absolutePath);
renamed = true;
rmSync(stagingDirectory, { force: true, recursive: true });
if (input.failurePoint === "after_rename") throw new InjectedCrashError("injected_crash_after_rename");
const commit = this.database.transaction(() => {
this.database.prepare(`
INSERT INTO managed_files (file_id, file_kind, relative_path, byte_size, mime_type, sha256, status, created_at)
VALUES (?, ?, ?, ?, ?, ?, 'committed', ?)
`).run(fileId, input.fileKind, destination.relativePath, byteSize, input.expectedMimeType, sha256, now());
this.database.prepare(`
UPDATE local_backend_storage_state SET managed_content_bytes = managed_content_bytes + ? WHERE singleton = 1
`).run(byteSize);
this.database.prepare(`
UPDATE storage_reservations SET status = 'consumed', resolved_at = ? WHERE operation_id = ? AND status = 'active'
`).run(now(), input.operationId);
this.refreshState();
});
commit();
if (input.failurePoint === "after_database_commit") throw new InjectedCrashError("injected_crash_after_database_commit");
return { bytes: byteSize, file_id: fileId, relative_path: destination.relativePath, sha256 };
} catch (error) {
if (error instanceof InjectedCrashError) throw error;
if (renamed && existsSync(destination.absolutePath)) {
try {
this.queueCompensation(destination.relativePath, statSync(destination.absolutePath).size);
} catch {
// Startup reconciliation is the final fallback if SQLite itself is unavailable.
}
} else {
rmSync(stagingDirectory, { force: true, recursive: true });
}
this.releaseReservation(input.operationId);
throw error;
}
}
async commitBufferFixture(fileKind: ManagedFileKind, fileName: string, bytes: Buffer) {
return this.commitStream({
content: Readable.from(bytes),
expectedMimeType: sniffMime(bytes.subarray(0, 16)),
fileKind,
fileName,
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: bytes.byteLength,
});
}
private queueCompensation(relativePath: string, byteSize: number) {
this.database.prepare(`
INSERT OR IGNORE INTO file_cleanup_queue
(cleanup_id, managed_file_id, relative_path, byte_size, counts_toward_managed, reason, status, created_at)
VALUES (?, NULL, ?, ?, 0, 'compensation', 'pending', ?)
`).run(randomUUID(), relativePath, byteSize, now());
}
async reconcileStartup() {
let releasedReservations = 0;
const active = this.database.prepare("SELECT operation_id FROM storage_reservations WHERE status = 'active'").all() as Array<{ operation_id: string }>;
for (const reservation of active) {
this.database.prepare(`UPDATE storage_reservations SET status = 'released', resolved_at = ? WHERE operation_id = ?`).run(now(), reservation.operation_id);
releasedReservations += 1;
}
const stagingRoot = resolvePathWithinRoot(this.dataRoot, "staging");
for (const entry of existsSync(stagingRoot) ? readdirSync(stagingRoot) : []) {
rmSync(join(stagingRoot, entry), { force: true, recursive: true });
}
const managedPaths = new Set((this.database.prepare("SELECT relative_path FROM managed_files").all() as Array<{ relative_path: string }>).map((row) => row.relative_path));
const queuedPaths = new Set((this.database.prepare("SELECT relative_path FROM file_cleanup_queue WHERE status IN ('pending', 'failed')").all() as Array<{ relative_path: string }>).map((row) => row.relative_path));
let orphaned = 0;
for (const root of ["content", "managed-assets", "derived-assets"]) {
for (const path of listFiles(resolvePathWithinRoot(this.dataRoot, root))) {
const relativePath = relative(this.dataRoot, path).replaceAll("\\", "/");
if (!managedPaths.has(relativePath) && !queuedPaths.has(relativePath)) {
this.queueCompensation(relativePath, statSync(path).size);
orphaned += 1;
}
}
}
const missingManagedFile = (this.database.prepare("SELECT relative_path FROM managed_files WHERE status = 'committed'").all() as Array<{ relative_path: string }>).some(
(row) => !existsSync(resolvePathWithinRoot(this.dataRoot, row.relative_path)),
);
if (missingManagedFile) this.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true });
else this.recordPhysicalMeasurement();
return { orphaned, released_reservations: releasedReservations };
}
inspectCounts() {
const count = (table: string, where = "1 = 1") => (this.database.prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${where}`).get() as { count: number }).count;
return {
active_reservations: count("storage_reservations", "status = 'active'"),
managed_files: count("managed_files"),
pending_cleanup: count("file_cleanup_queue", "status = 'pending'"),
cleanup_requests: count("asset_cleanup_requests"),
admin_logs: count("admin_operation_logs"),
};
}
resolveManagedFile(fileId: string) {
const row = this.database.prepare("SELECT relative_path FROM managed_files WHERE file_id = ? AND status = 'committed'").get(fileId) as { relative_path: string } | undefined;
return row ? resolvePathWithinRoot(this.dataRoot, row.relative_path) : undefined;
}
addAssetReference(fileId: string, referenceType: "project" | "release") {
if (this.inspectAction("project_json_write") !== "allow") throw new StorageUnavailableError();
this.database.prepare(`INSERT INTO project_asset_refs (reference_id, managed_file_id, reference_type, created_at) VALUES (?, ?, ?, ?)`)
.run(randomUUID(), fileId, referenceType, now());
}
removeAssetReferences(fileId: string) {
if (this.inspectAction("project_json_write") !== "allow") throw new StorageUnavailableError();
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(fileId);
}
createCleanupIntent(fileIds: string[]) {
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
if (fileIds.length === 0 || new Set(fileIds).size !== fileIds.length) throw new Error("cleanup_candidates_invalid");
const requestId = randomUUID();
const transaction = this.database.transaction(() => {
for (const fileId of fileIds) {
const candidate = this.database.prepare(`
SELECT file_id FROM managed_files
WHERE file_id = ? AND status = 'committed' AND file_kind IN ('sticker_original', 'sticker_thumbnail')
`).get(fileId);
if (!candidate) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
}
this.database.prepare("INSERT INTO asset_cleanup_requests (request_id, status, created_at) VALUES (?, 'pending_confirmation', ?)").run(requestId, now());
const insert = this.database.prepare("INSERT INTO asset_cleanup_request_items (request_id, managed_file_id) VALUES (?, ?)");
for (const fileId of fileIds) insert.run(requestId, fileId);
});
transaction();
return { request_id: requestId };
}
confirmCleanupIntent(requestId: string) {
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
const transaction = this.database.transaction(() => {
const request = this.database.prepare("SELECT status FROM asset_cleanup_requests WHERE request_id = ?").get(requestId) as { status: string } | undefined;
if (request?.status !== "pending_confirmation") throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
const files = this.database.prepare(`
SELECT mf.* FROM asset_cleanup_request_items items
JOIN managed_files mf ON mf.file_id = items.managed_file_id
WHERE items.request_id = ?
`).all(requestId) as ManagedFileRow[];
const conflict = files.some((file) => {
if (file.status !== "committed" || (file.file_kind !== "sticker_original" && file.file_kind !== "sticker_thumbnail")) return true;
const row = this.database.prepare("SELECT COUNT(*) AS count FROM project_asset_refs WHERE managed_file_id = ?").get(file.file_id) as { count: number };
return row.count > 0;
});
if (conflict) {
this.database.prepare("UPDATE asset_cleanup_requests SET status = 'denied', confirmed_at = ? WHERE request_id = ?").run(now(), requestId);
this.database.prepare("INSERT INTO admin_operation_logs (log_id, operation, outcome, target_ref, created_at) VALUES (?, 'asset_cleanup', 'denied_reference_conflict', ?, ?)")
.run(randomUUID(), requestId, now());
return false;
}
for (const file of files) {
this.database.prepare("UPDATE managed_files SET status = 'purged', purged_at = ? WHERE file_id = ?").run(now(), file.file_id);
this.database.prepare(`
INSERT INTO file_cleanup_queue
(cleanup_id, managed_file_id, relative_path, byte_size, counts_toward_managed, reason, status, created_at)
VALUES (?, ?, ?, ?, 1, 'purge', 'pending', ?)
`).run(randomUUID(), file.file_id, file.relative_path, file.byte_size, now());
}
this.database.prepare("UPDATE asset_cleanup_requests SET status = 'queued', confirmed_at = ? WHERE request_id = ?").run(now(), requestId);
this.database.prepare("INSERT INTO admin_operation_logs (log_id, operation, outcome, target_ref, created_at) VALUES (?, 'asset_cleanup', 'queued', ?, ?)")
.run(randomUUID(), requestId, now());
return true;
});
if (!transaction()) throw new Error("ASSET_HISTORY_REFERENCE_CONFLICT");
}
async processCleanupQueue() {
if (this.inspectAction("explicit_cleanup") !== "allow" || !this.dataRootWritable || !this.diskSpaceAvailable) {
return { completed: 0, failed: 0 };
}
const rows = this.database.prepare("SELECT * FROM file_cleanup_queue WHERE status IN ('pending', 'failed') ORDER BY created_at").all() as CleanupQueueRow[];
let completed = 0;
let failed = 0;
for (const row of rows) {
try {
const path = resolvePathWithinRoot(this.dataRoot, row.relative_path);
rmSync(path, { force: true });
const finish = this.database.transaction(() => {
if (row.counts_toward_managed === 1 && row.managed_file_id) {
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(row.managed_file_id);
const requests = this.database.prepare("SELECT request_id FROM asset_cleanup_request_items WHERE managed_file_id = ?").all(row.managed_file_id) as Array<{ request_id: string }>;
this.database.prepare("DELETE FROM asset_cleanup_request_items WHERE managed_file_id = ?").run(row.managed_file_id);
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
for (const request of requests) {
const pendingItems = this.database.prepare("SELECT COUNT(*) AS count FROM asset_cleanup_request_items WHERE request_id = ?").get(request.request_id) as { count: number };
if (pendingItems.count === 0) this.database.prepare("UPDATE asset_cleanup_requests SET status = 'completed' WHERE request_id = ? AND status = 'queued'").run(request.request_id);
}
}
this.database.prepare("UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL WHERE cleanup_id = ?").run(now(), row.cleanup_id);
this.database.prepare("INSERT INTO admin_operation_logs (log_id, operation, outcome, target_ref, created_at) VALUES (?, 'physical_file_cleanup', 'completed', ?, ?)")
.run(randomUUID(), row.cleanup_id, now());
this.recordPhysicalMeasurement();
});
finish();
completed += 1;
} catch (error) {
this.database.prepare("UPDATE file_cleanup_queue SET status = 'failed', last_error = ? WHERE cleanup_id = ?")
.run(error instanceof Error ? error.message : "cleanup_failed", row.cleanup_id);
failed += 1;
}
}
return { completed, failed };
}
}
+81
View File
@@ -0,0 +1,81 @@
export const WARNING_LIMIT_BYTES = 4_294_967_296;
export const CRITICAL_LIMIT_BYTES = 4_831_838_208;
export const HARD_LIMIT_BYTES = 5_368_709_120;
export const CAPACITY_COUNTED_FILE_KINDS = [
"reference",
"generated",
"export",
"derived",
"sticker_original",
"sticker_thumbnail",
] as const;
export const CAPACITY_EXCLUDED_STORAGE = [
"read_only_assets",
"application_files",
"database",
"logs",
"audit",
"browser_cache",
"user_downloads",
] as const;
export type CapacityNoticeLevel = "normal" | "warning" | "critical";
export type StorageStatus = "active" | "full" | "unavailable";
export type StorageAction =
| "project_json_write"
| "binary_write"
| "ai_call"
| "latest_export_write"
| "read"
| "download"
| "client_only_download"
| "permanent_delete"
| "explicit_cleanup";
export type StorageActionDecision = "allow" | "reject_capacity" | "reject_unavailable" | "reject_uncommitted";
export function classifyCapacity(managedContentBytes: number, activeReservationBytes: number) {
if (!Number.isSafeInteger(managedContentBytes) || managedContentBytes < 0) throw new Error("managed_content_bytes_invalid");
if (!Number.isSafeInteger(activeReservationBytes) || activeReservationBytes < 0) throw new Error("active_reservation_bytes_invalid");
const capacity_notice_level: CapacityNoticeLevel = managedContentBytes < WARNING_LIMIT_BYTES
? "normal"
: managedContentBytes < CRITICAL_LIMIT_BYTES
? "warning"
: "critical";
const storage_status: StorageStatus = managedContentBytes + activeReservationBytes >= HARD_LIMIT_BYTES
? "full"
: "active";
return { capacity_notice_level, storage_status };
}
export function decideStorageAction(
status: StorageStatus,
sqliteWritable: boolean,
action: StorageAction,
): StorageActionDecision {
if (status === "unavailable") {
if (action === "read" || action === "download" || action === "client_only_download") return "allow";
if (action === "permanent_delete" || action === "explicit_cleanup") {
return sqliteWritable ? "allow" : "reject_uncommitted";
}
return "reject_unavailable";
}
if (status === "full" && (action === "binary_write" || action === "ai_call" || action === "latest_export_write")) {
return "reject_capacity";
}
return "allow";
}
export function storageCapacityErrorDetails(input: {
activeReservationBytes: number;
managedContentBytes: number;
projectedWriteBytes: number;
}) {
const { capacity_notice_level, storage_status } = classifyCapacity(
input.managedContentBytes,
input.activeReservationBytes,
);
return {
capacity_status: storage_status === "full" ? "full" as const : capacity_notice_level,
remaining_bytes: Math.max(0, HARD_LIMIT_BYTES - input.managedContentBytes - input.activeReservationBytes),
};
}
+10
View File
@@ -0,0 +1,10 @@
export interface StorageMaintenanceTarget {
processCleanupQueue(): Promise<{ completed: number; failed: number }>;
reconcileStartup(): Promise<{ orphaned: number; released_reservations: number }>;
}
export async function runStorageMaintenance(target: StorageMaintenanceTarget) {
const reconciliation = await target.reconcileStartup();
const cleanup = await target.processCleanupQueue();
return { cleanup, reconciliation };
}
+5 -3
View File
@@ -13,8 +13,8 @@
"test:unit": "pnpm --filter @dada/shared-contracts build && pnpm run test:unit:contract && vitest run tests/unit", "test:unit": "pnpm --filter @dada/shared-contracts build && pnpm run test:unit:contract && vitest run tests/unit",
"test:integration": "vitest run tests/integration", "test:integration": "vitest run tests/integration",
"test:api": "pnpm check:openapi && vitest run tests/api", "test:api": "pnpm check:openapi && vitest run tests/api",
"test:worker": "pnpm --filter @dada/worker build && node scripts/worker-smoke.mjs", "test:worker": "pnpm --filter @dada/worker build && node scripts/worker-smoke.mjs && vitest run tests/worker",
"test:e2e": "pnpm check:openapi && playwright test tests/e2e/event-sync.spec.ts tests/e2e/support-gate.spec.ts tests/e2e/local-data-boundary.spec.ts --config playwright.config.ts", "test:e2e": "pnpm check:openapi && playwright test tests/e2e/event-sync.spec.ts tests/e2e/support-gate.spec.ts tests/e2e/local-data-boundary.spec.ts tests/e2e/storage-capacity.spec.ts --config playwright.config.ts",
"test:visual": "node scripts/validate-layer-scope.mjs VISUAL", "test:visual": "node scripts/validate-layer-scope.mjs VISUAL",
"test:performance": "node scripts/validate-layer-scope.mjs PERFORMANCE", "test:performance": "node scripts/validate-layer-scope.mjs PERFORMANCE",
"test:security": "node scripts/verify-frozen-dependencies.mjs && node scripts/redaction-scan.mjs", "test:security": "node scripts/verify-frozen-dependencies.mjs && node scripts/redaction-scan.mjs",
@@ -28,7 +28,9 @@
"test:wp0-02": "node scripts/run-wp0-02-validation.mjs", "test:wp0-02": "node scripts/run-wp0-02-validation.mjs",
"test:wp0-03": "node scripts/run-wp0-03-validation.mjs", "test:wp0-03": "node scripts/run-wp0-03-validation.mjs",
"test:wp0-04": "node scripts/run-wp0-04-validation.mjs", "test:wp0-04": "node scripts/run-wp0-04-validation.mjs",
"test:wp0-04:red": "node scripts/run-wp0-04-validation.mjs --phase red" "test:wp0-04:red": "node scripts/run-wp0-04-validation.mjs --phase red",
"test:wp0-05": "node scripts/run-wp0-05-validation.mjs",
"test:wp0-05:red": "node scripts/run-wp0-05-validation.mjs --phase red"
}, },
"devDependencies": { "devDependencies": {
"@playwright/test": "1.62.0", "@playwright/test": "1.62.0",
+119
View File
@@ -0,0 +1,119 @@
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
const phaseIndex = process.argv.indexOf("--phase");
const phase = phaseIndex >= 0 ? process.argv[phaseIndex + 1] : "green";
if (!["red", "green"].includes(phase)) throw new Error(`Unsupported phase: ${phase}`);
const runId = process.env.DADA_TDD_RUN_ID ?? `wp0-05-${phase}-${new Date().toISOString().replace(/[^0-9]/g, "")}`;
const runDirectory = resolve("artifacts", "tdd", runId);
const cases = [
"TDD-WP0-FILE-001-atomic-commit",
"TDD-WP0-STO-001-byte-thresholds",
"TDD-WP0-STO-002-equal-limit",
"TDD-WP0-STO-002-over-limit",
"TDD-WP0-STO-003-cleanup-remeasure",
"TDD-WP0-STO-003-full-matrix",
"TDD-WP0-STO-003-unavailable-matrix",
];
const directories = Object.fromEntries(cases.map((id) => [id, resolve(runDirectory, "cases", id)]));
const playwrightDirectory = resolve(runDirectory, "playwright");
if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`);
for (const directory of Object.values(directories)) mkdirSync(directory, { recursive: true });
const redCommands = [
["pnpm exec vitest run tests/unit/wp0-05-storage-policy.test.ts", ["exec", "vitest", "run", "tests/unit/wp0-05-storage-policy.test.ts"]],
["pnpm exec vitest run tests/integration/wp0-05-managed-storage.test.ts", ["exec", "vitest", "run", "tests/integration/wp0-05-managed-storage.test.ts"]],
["pnpm exec vitest run tests/api/wp0-05-storage-capacity.test.ts", ["exec", "vitest", "run", "tests/api/wp0-05-storage-capacity.test.ts"]],
["pnpm exec vitest run tests/worker/wp0-05-storage-maintenance.test.ts", ["exec", "vitest", "run", "tests/worker/wp0-05-storage-maintenance.test.ts"]],
["pnpm exec playwright test tests/e2e/storage-capacity.spec.ts --config playwright.config.ts", ["exec", "playwright", "test", "tests/e2e/storage-capacity.spec.ts", "--config", "playwright.config.ts"]],
];
const greenCommands = [
["pnpm test:integration", ["test:integration"]],
["pnpm test:api", ["test:api"]],
["pnpm test:worker", ["test:worker"]],
["pnpm test:unit", ["test:unit"]],
["pnpm test:e2e", ["test:e2e"]],
["pnpm validate:tdd-trace", ["validate:tdd-trace"]],
];
const env = {
...process.env,
DADA_EVIDENCE_DIR_FILE_001: directories[cases[0]],
DADA_EVIDENCE_DIR_STO_001: directories[cases[1]],
DADA_EVIDENCE_DIR_STO_002_EQUAL: directories[cases[2]],
DADA_EVIDENCE_DIR_STO_002_OVER: directories[cases[3]],
DADA_EVIDENCE_DIR_STO_003_CLEANUP: directories[cases[4]],
DADA_EVIDENCE_DIR_STO_003_FULL: directories[cases[5]],
DADA_EVIDENCE_DIR_STO_003_UNAVAILABLE: directories[cases[6]],
DADA_PLAYWRIGHT_OUTPUT_DIR: playwrightDirectory,
};
const startedAt = new Date().toISOString();
const commands = [];
for (const [command, args] of phase === "red" ? redCommands : greenCommands) {
const started_at = new Date().toISOString();
const executable = process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "pnpm";
const actualArgs = process.platform === "win32" ? ["/d", "/s", "/c", `pnpm ${args.join(" ")}`] : args;
const execution = spawnSync(executable, actualArgs, { encoding: "utf8", env });
if (execution.stdout) process.stdout.write(execution.stdout);
if (execution.stderr) process.stderr.write(execution.stderr);
commands.push({ command, exit_code: execution.status ?? 1, finished_at: new Date().toISOString(), started_at });
}
function find(root, name) {
if (!existsSync(root)) return [];
return readdirSync(root).flatMap((entry) => {
const child = resolve(root, entry);
return statSync(child).isDirectory() ? find(child, name) : entry === name ? [child] : [];
});
}
if (phase === "green") {
const trace = find(playwrightDirectory, "trace.zip").find((path) => path.replaceAll("\\", "/").includes("storage-capacity"));
if (trace) {
copyFileSync(trace, resolve(directories[cases[5]], "trace.zip"));
copyFileSync(trace, resolve(directories[cases[6]], "trace.zip"));
}
}
const commandEvidence = { commands, phase, run_id: runId, schema_version: "1.0" };
for (const directory of Object.values(directories)) writeFileSync(resolve(directory, "commands.json"), `${JSON.stringify(commandEvidence, null, 2)}\n`);
const manifest = { path: "tasks.manifest.json", sha256: createHash("sha256").update(readFileSync("tasks.manifest.json")).digest("hex").toUpperCase() };
const commit = spawnSync("git", ["rev-parse", "--short", "HEAD"], { encoding: "utf8" }).stdout.trim();
const worktreeDirty = spawnSync("git", ["status", "--porcelain"], { encoding: "utf8" }).stdout.trim().length > 0;
const expectedEvidence = {
[cases[0]]: ["response.json", "db-diff.json", "fs-before.json", "fs-after.json", "crash-points.json"],
[cases[1]]: ["thresholds.json", "db-diff.json", "screenshots/capacity-bar.png"],
[cases[2]]: ["db-diff.json"],
[cases[3]]: [
"reference-response.json", "reference-db-diff.json", "reference-fs-diff.json", "reference-external-calls.json",
"generated-response.json", "generated-db-diff.json", "generated-fs-diff.json", "generated-external-calls.json",
"export-response.json", "export-db-diff.json", "export-fs-diff.json", "export-external-calls.json",
"sticker_original-response.json", "sticker_original-db-diff.json", "sticker_original-fs-diff.json", "sticker_original-external-calls.json",
"sticker_thumbnail-response.json", "sticker_thumbnail-db-diff.json", "sticker_thumbnail-fs-diff.json", "sticker_thumbnail-external-calls.json",
],
[cases[4]]: ["conflict-response.json", "queued-db-diff.json", "worker-result.json", "fs-diff.json"],
[cases[5]]: ["response.json", "db-diff.json", "external-calls.json", "trace.zip"],
[cases[6]]: ["matrix.json", "db-diff.json", "external-calls.json", "trace.zip", "screenshots/unavailable.png"],
};
const commandState = phase === "red" ? commands.every((item) => item.exit_code !== 0) : commands.every((item) => item.exit_code === 0);
const results = cases.map((testId) => {
const evidence_refs = expectedEvidence[testId];
const missing_evidence = phase === "green" ? evidence_refs.filter((path) => !existsSync(resolve(directories[testId], path))) : [];
const status = phase === "red" ? (commandState ? "red_confirmed" : "failed") : (commandState && missing_evidence.length === 0 ? "passed" : "failed");
const result = {
acceptance_criteria: ["AC-31", "AC-35", "AC-44", "AC-55"], automation: ["automated"], commit,
environment: { arch: process.arch, node: process.version.slice(1), os: process.platform }, evidence_refs,
finished_at: new Date().toISOString(), layer: ["DB", "API", "WORKER", "E2E"], manifest, missing_evidence,
parent_family: testId.split("-").slice(0, -2).join("-"), phase, release_gate: ["work_package:WP-0", "release:P0-A"],
requirements: ["ADMIN-06", "EXPORT-06", "NFR-05", "PROJECT-04", "17.19"], run_id: runId,
schema_version: "1.0", started_at: startedAt, status, task_id: "TASK-WP0-05", test_id: testId,
work_package: "WP-0", worktree_under_test: worktreeDirty ? "uncommitted implementation" : "clean committed implementation",
};
writeFileSync(resolve(directories[testId], "result.json"), `${JSON.stringify(result, null, 2)}\n`);
return result;
});
const passed = results.every((result) => result.status === (phase === "red" ? "red_confirmed" : "passed"));
const summary = { cases: results.map(({ missing_evidence, status, test_id }) => ({ missing_evidence, status, test_id })), phase, run_id: runId, status: passed ? (phase === "red" ? "red_confirmed" : "passed") : "failed" };
writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify(summary, null, 2)}\n`);
console.log(JSON.stringify(summary, null, 2));
if (!passed) process.exit(1);
+30
View File
@@ -0,0 +1,30 @@
import { randomUUID } from "node:crypto";
import { describe, expect, it } from "vitest";
import { storageCapacityErrorDetails } from "../../apps/api/src/storage-policy.js";
import { createErrorEnvelope, isErrorEnvelope } from "../../packages/shared-contracts/src/index.js";
describe("TDD-WP0-STO-002-over-limit API envelope", () => {
it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"])(
"returns stable 507 details for %s",
(kind) => {
const envelope = createErrorEnvelope({
code: "STORAGE_CAPACITY_EXCEEDED",
correlationId: randomUUID(),
details: storageCapacityErrorDetails({
activeReservationBytes: 0,
managedContentBytes: 5_368_709_119,
projectedWriteBytes: 2,
}),
});
expect(isErrorEnvelope(envelope)).toBe(true);
expect(envelope.error).toMatchObject({
code: "STORAGE_CAPACITY_EXCEEDED",
details: { capacity_status: "critical", remaining_bytes: 1 },
message_key: "STORAGE_CAPACITY_EXCEEDED",
});
expect(kind).toBeTruthy();
},
);
});
+17
View File
@@ -0,0 +1,17 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Dada 容量状态验收探针</title>
<style>
body { font-family: system-ui, sans-serif; margin: 32px; color: #1f2328; }
main { max-width: 720px; }
.bar { height: 18px; background: #d0d7de; border-radius: 4px; overflow: hidden; }
.bar > span { display: block; height: 100%; background: #cf222e; width: 100%; }
table { border-collapse: collapse; margin-top: 24px; width: 100%; }
th, td { border-bottom: 1px solid #d8dee4; padding: 8px; text-align: left; }
</style>
</head>
<body><main id="root"></main><script type="module" src="/tests/e2e/fixtures/storage-capacity.ts"></script></body>
</html>
+17
View File
@@ -0,0 +1,17 @@
import { classifyCapacity, decideStorageAction } from "../../../apps/api/src/storage-policy.js";
const root = document.querySelector<HTMLElement>("#root");
if (!root) throw new Error("Fixture root is missing.");
const capacity = classifyCapacity(5_368_709_120, 0);
const actions = ["project_json_write", "binary_write", "read", "download", "permanent_delete", "explicit_cleanup"] as const;
root.innerHTML = `
<h1>本机存储容量</h1>
<p data-status>${capacity.storage_status}</p>
<div class="bar" role="progressbar" aria-valuemin="0" aria-valuemax="5368709120" aria-valuenow="5368709120"><span></span></div>
<table><thead><tr><th>操作</th><th>结果</th></tr></thead><tbody>
${actions.map((action) => `<tr><td>${action}</td><td>${decideStorageAction("full", true, action)}</td></tr>`).join("")}
</tbody></table>
<h2>存储不可用</h2>
<table data-unavailable><thead><tr><th>操作</th><th>SQLite 可写</th><th>SQLite 不可写</th></tr></thead><tbody>
${actions.map((action) => `<tr><td>${action}</td><td>${decideStorageAction("unavailable", true, action)}</td><td>${decideStorageAction("unavailable", false, action)}</td></tr>`).join("")}
</tbody></table>`;
+39
View File
@@ -0,0 +1,39 @@
import { mkdirSync } from "node:fs";
import { resolve } from "node:path";
import { expect, test } from "@playwright/test";
import { createServer, type ViteDevServer } from "vite";
let vite: ViteDevServer;
let webUrl: string;
test.beforeAll(async () => {
vite = await createServer({ configFile: false, root: process.cwd(), server: { host: "127.0.0.1", port: 0 } });
await vite.listen();
const address = vite.httpServer?.address();
if (!address || typeof address === "string") throw new Error("Vite did not expose a test port.");
webUrl = `http://127.0.0.1:${address.port}`;
});
test.afterAll(async () => vite.close());
test("TDD-WP0-STO-001 and STO-003 render the exact full action matrix", async ({ page }) => {
await page.goto(`${webUrl}/tests/e2e/fixtures/storage-capacity.html`);
await expect(page.getByRole("heading", { name: "本机存储容量" })).toBeVisible();
await expect(page.locator("[data-status]")).toHaveText("full");
await expect(page.getByRole("progressbar")).toHaveAttribute("aria-valuenow", "5368709120");
await expect(page.locator("table").first().getByRole("row").filter({ hasText: "binary_write" })).toContainText("reject_capacity");
await expect(page.locator("table").first().getByRole("row").filter({ hasText: "project_json_write" })).toContainText("allow");
await expect(page.getByRole("heading", { name: "存储不可用" })).toBeVisible();
await expect(page.locator("[data-unavailable] tr").filter({ hasText: "explicit_cleanup" })).toContainText("reject_uncommitted");
const directory = process.env.DADA_EVIDENCE_DIR_STO_001;
if (directory) {
mkdirSync(resolve(directory, "screenshots"), { recursive: true });
await page.screenshot({ path: resolve(directory, "screenshots", "capacity-bar.png") });
}
const unavailableDirectory = process.env.DADA_EVIDENCE_DIR_STO_003_UNAVAILABLE;
if (unavailableDirectory) {
mkdirSync(resolve(unavailableDirectory, "screenshots"), { recursive: true });
await page.screenshot({ path: resolve(unavailableDirectory, "screenshots", "unavailable.png"), fullPage: true });
}
});
@@ -0,0 +1,265 @@
import { createHash, randomUUID } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { Readable } from "node:stream";
import { afterEach, describe, expect, it } from "vitest";
import {
HARD_LIMIT_BYTES,
ManagedStorage,
StorageCapacityError,
type ManagedFileKind,
} from "../../apps/api/src/managed-storage.js";
const temporaryDirectories: string[] = [];
function fixture() {
const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp0-05-"));
temporaryDirectories.push(dataRoot);
for (const directory of ["db", "content/references", "content/generated", "content/exports", "managed-assets", "derived-assets", "staging"]) {
mkdirSync(join(dataRoot, directory), { recursive: true });
}
const storage = new ManagedStorage({ dataRoot, databasePath: join(dataRoot, "db", "dada.sqlite3") });
return { dataRoot, storage };
}
function databaseCounts(storage: ManagedStorage) {
return storage.inspectCounts();
}
function tree(root: string) {
return readdirSync(root, { recursive: true }).map(String).sort();
}
function evidence(caseName: string, name: string, value: unknown) {
const directory = process.env[`DADA_EVIDENCE_DIR_${caseName}`];
if (!directory) return;
mkdirSync(directory, { recursive: true });
writeFileSync(join(directory, name), `${JSON.stringify(value, null, 2)}\n`);
}
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
if (resolve(directory).startsWith(resolve(tmpdir()))) rmSync(directory, { force: true, recursive: true });
}
});
describe("TDD-WP0-STO-002 exact admission", () => {
it("allows equality, consumes its reservation and transitions to full atomically", async () => {
const { storage } = fixture();
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
const result = await storage.commitStream({
content: Readable.from(Buffer.from("x")),
expectedMimeType: "application/octet-stream",
fileKind: "generated",
fileName: "asset.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: 1,
});
expect(result.bytes).toBe(1);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 1 });
evidence("STO_002_EQUAL", "db-diff.json", { after: databaseCounts(storage), state: storage.getState() });
storage.close();
});
it.each(["reference", "generated", "export", "sticker_original", "sticker_thumbnail"] as ManagedFileKind[])(
"rejects %s when projected bytes are strictly over the limit without side effects",
async (fileKind) => {
const { dataRoot, storage } = fixture();
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 1);
const beforeCounts = databaseCounts(storage);
const beforeTree = tree(dataRoot);
await expect(storage.commitStream({
content: Readable.from(Buffer.from("xx")),
expectedMimeType: "image/png",
fileKind,
fileName: "asset.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: 2,
})).rejects.toBeInstanceOf(StorageCapacityError);
expect(databaseCounts(storage)).toEqual(beforeCounts);
expect(tree(dataRoot)).toEqual(beforeTree);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 1, storage_status: "active" });
evidence("STO_002_OVER", `${fileKind}-response.json`, { code: "STORAGE_CAPACITY_EXCEEDED", status: 507 });
evidence("STO_002_OVER", `${fileKind}-db-diff.json`, { after: databaseCounts(storage), before: beforeCounts });
evidence("STO_002_OVER", `${fileKind}-fs-diff.json`, { after: tree(dataRoot), before: beforeTree });
evidence("STO_002_OVER", `${fileKind}-external-calls.json`, { calls: 0 });
storage.close();
},
);
});
describe("TDD-WP0-FILE-001 atomic commit", () => {
it("streams, validates and commits a managed file without a half-committed state", async () => {
const { dataRoot, storage } = fixture();
const bytes = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.from("valid-image-payload")]);
const before = tree(dataRoot);
const committed = await storage.commitStream({
content: Readable.from([bytes.subarray(0, 5), bytes.subarray(5)]),
expectedMimeType: "image/png",
expectedSha256: createHash("sha256").update(bytes).digest("hex"),
fileKind: "reference",
fileName: "reference.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: bytes.byteLength,
});
expect(existsSync(join(dataRoot, committed.relative_path))).toBe(true);
expect(readFileSync(join(dataRoot, committed.relative_path))).toEqual(bytes);
expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 0 });
evidence("FILE_001", "fs-before.json", { entries: before });
evidence("FILE_001", "fs-after.json", { entries: tree(dataRoot) });
evidence("FILE_001", "db-diff.json", databaseCounts(storage));
evidence("FILE_001", "response.json", { bytes: committed.bytes, file_id: committed.file_id, status: "committed" });
storage.close();
});
it("rejects invalid MIME, hash, traversal and symlink inputs before commit", async () => {
const { storage } = fixture();
const base = {
content: Readable.from(Buffer.from("bad")),
expectedMimeType: "image/png" as const,
fileKind: "reference" as const,
ownerRef: randomUUID(),
projectedWriteBytes: 3,
};
await expect(storage.commitStream({ ...base, fileName: "../bad.png", operationId: randomUUID() })).rejects.toThrow("file_name_invalid");
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.png", expectedSha256: "0".repeat(64), operationId: randomUUID() })).rejects.toThrow("content_hash_invalid");
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.jpg", expectedMimeType: "image/jpeg", operationId: randomUUID() })).rejects.toThrow("content_mime_invalid");
const ownerRef = randomUUID();
const target = join(storage.dataRoot, "symlink-target");
mkdirSync(target);
symlinkSync(target, join(storage.dataRoot, "content", "references", ownerRef), "junction");
await expect(storage.commitStream({ ...base, content: Readable.from(Buffer.from("bad")), fileName: "bad.bin", expectedMimeType: "application/octet-stream", operationId: randomUUID(), ownerRef })).rejects.toThrow(/path_escape|symbolic_link/);
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, managed_files: 0 });
storage.close();
});
it("reconciles each crash point into either a complete commit or compensation", async () => {
const { storage } = fixture();
await expect(storage.commitStream({
content: Readable.from(Buffer.from("staged")), expectedMimeType: "application/octet-stream",
failurePoint: "after_staging", fileKind: "generated", fileName: "staged.bin",
operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 6,
})).rejects.toThrow("injected_crash_after_staging");
const afterStaging = await storage.reconcileStartup();
expect(afterStaging).toMatchObject({ orphaned: 0, released_reservations: 1 });
await expect(storage.commitStream({
content: Readable.from(Buffer.from("orphan")),
expectedMimeType: "application/octet-stream",
failurePoint: "after_rename",
fileKind: "generated",
fileName: "orphan.png",
operationId: randomUUID(),
ownerRef: randomUUID(),
projectedWriteBytes: 6,
})).rejects.toThrow("injected_crash_after_rename");
const reconciled = await storage.reconcileStartup();
expect(reconciled).toMatchObject({ orphaned: 1, released_reservations: 1 });
expect(databaseCounts(storage)).toMatchObject({ active_reservations: 0, pending_cleanup: 1 });
await expect(storage.commitStream({
content: Readable.from(Buffer.from("committed")), expectedMimeType: "application/octet-stream",
failurePoint: "after_database_commit", fileKind: "generated", fileName: "committed.bin",
operationId: randomUUID(), ownerRef: randomUUID(), projectedWriteBytes: 9,
})).rejects.toThrow("injected_crash_after_database_commit");
const afterDatabaseCommit = await storage.reconcileStartup();
expect(afterDatabaseCommit).toMatchObject({ orphaned: 0, released_reservations: 0 });
expect(databaseCounts(storage)).toMatchObject({ managed_files: 1, pending_cleanup: 1 });
evidence("FILE_001", "crash-points.json", {
after_database_commit: afterDatabaseCommit,
after_rename: reconciled,
after_staging: afterStaging,
});
storage.close();
});
});
describe("TDD-WP0-STO-003 cleanup and availability", () => {
it("rejects a stale cleanup as one batch, then decrements only after physical deletion and remeasure", async () => {
const { dataRoot, storage } = fixture();
const fsBefore = tree(dataRoot);
storage.applyControlledMeasurement(HARD_LIMIT_BYTES - 2);
const first = await storage.commitBufferFixture("sticker_original", "a.png", Buffer.from("a"));
const second = await storage.commitBufferFixture("sticker_thumbnail", "b.png", Buffer.from("b"));
expect(storage.getState().storage_status).toBe("full");
const stale = storage.createCleanupIntent([first.file_id, second.file_id]);
storage.addAssetReference(first.file_id, "release");
expect(() => storage.confirmCleanupIntent(stale.request_id)).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT");
expect(databaseCounts(storage)).toMatchObject({ pending_cleanup: 0 });
evidence("STO_003_CLEANUP", "conflict-response.json", { code: "ASSET_HISTORY_REFERENCE_CONFLICT", partial_queue: false, status: 409 });
storage.removeAssetReferences(first.file_id);
const request = storage.createCleanupIntent([first.file_id, second.file_id]);
storage.confirmCleanupIntent(request.request_id);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES, storage_status: "full" });
expect(existsSync(join(dataRoot, first.relative_path))).toBe(true);
evidence("STO_003_CLEANUP", "queued-db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
await storage.processCleanupQueue();
expect(existsSync(join(dataRoot, first.relative_path))).toBe(false);
expect(storage.getState()).toMatchObject({ managed_content_bytes: HARD_LIMIT_BYTES - 2, storage_status: "active" });
evidence("STO_003_CLEANUP", "worker-result.json", { counts: databaseCounts(storage), state: storage.getState() });
evidence("STO_003_CLEANUP", "fs-diff.json", { after: tree(dataRoot), before: fsBefore });
storage.close();
});
it("keeps the full matrix read/write effects exact", () => {
const { storage } = fixture();
storage.applyControlledMeasurement(HARD_LIMIT_BYTES);
const before = databaseCounts(storage);
const decisions = {
ai_call: storage.inspectAction("ai_call"),
binary_write: storage.inspectAction("binary_write"),
client_only_download: storage.inspectAction("client_only_download"),
download: storage.inspectAction("download"),
explicit_cleanup: storage.inspectAction("explicit_cleanup"),
latest_export_write: storage.inspectAction("latest_export_write"),
permanent_delete: storage.inspectAction("permanent_delete"),
project_json_write: storage.inspectAction("project_json_write"),
read: storage.inspectAction("read"),
};
expect(decisions).toEqual({
ai_call: "reject_capacity", binary_write: "reject_capacity", client_only_download: "allow",
download: "allow", explicit_cleanup: "allow", latest_export_write: "reject_capacity",
permanent_delete: "allow", project_json_write: "allow", read: "allow",
});
expect(databaseCounts(storage)).toEqual(before);
evidence("STO_003_FULL", "response.json", decisions);
evidence("STO_003_FULL", "db-diff.json", { after: databaseCounts(storage), before });
evidence("STO_003_FULL", "external-calls.json", { calls: 0 });
storage.close();
});
it("marks unavailable without mutating storage when the data root or SQLite cannot be written", async () => {
const { storage } = fixture();
storage.setAvailability({ dataRootWritable: false, diskSpaceAvailable: true, sqliteWritable: true });
expect(storage.getState().storage_status).toBe("unavailable");
expect(storage.inspectAction("project_json_write")).toBe("reject_unavailable");
expect(storage.inspectAction("read")).toBe("allow");
expect(storage.inspectAction("explicit_cleanup")).toBe("allow");
storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: true });
expect(storage.getState().storage_status).toBe("unavailable");
storage.applyControlledMeasurement(0);
expect(storage.getState().storage_status).toBe("active");
storage.setAvailability({ dataRootWritable: true, diskSpaceAvailable: true, sqliteWritable: false });
expect(storage.inspectAction("explicit_cleanup")).toBe("reject_uncommitted");
expect(() => storage.createCleanupIntent([randomUUID()])).toThrow("cleanup_uncommitted");
const beforeMaintenance = databaseCounts(storage);
expect(await storage.processCleanupQueue()).toEqual({ completed: 0, failed: 0 });
expect(databaseCounts(storage)).toEqual(beforeMaintenance);
evidence("STO_003_UNAVAILABLE", "matrix.json", {
read: storage.inspectAction("read"),
write: storage.inspectAction("project_json_write"),
cleanup: storage.inspectAction("explicit_cleanup"),
});
evidence("STO_003_UNAVAILABLE", "db-diff.json", { counts: databaseCounts(storage), state: storage.getState() });
evidence("STO_003_UNAVAILABLE", "external-calls.json", { calls: 0 });
storage.close();
});
});
+80
View File
@@ -0,0 +1,80 @@
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
CAPACITY_COUNTED_FILE_KINDS,
CAPACITY_EXCLUDED_STORAGE,
HARD_LIMIT_BYTES,
classifyCapacity,
decideStorageAction,
} from "../../apps/api/src/storage-policy.js";
describe("TDD-WP0-STO-001-byte-thresholds", () => {
it.each([
[4_294_967_295, "normal", "active"],
[4_294_967_296, "warning", "active"],
[4_831_838_207, "warning", "active"],
[4_831_838_208, "critical", "active"],
[5_368_709_119, "critical", "active"],
[5_368_709_120, "critical", "full"],
[5_368_709_121, "critical", "full"],
] as const)("classifies %i bytes", (bytes, notice, status) => {
expect(classifyCapacity(bytes, 0)).toEqual({ capacity_notice_level: notice, storage_status: status });
});
it("records the deterministic threshold result without deleting content", () => {
const values = [
4_294_967_295,
4_294_967_296,
4_831_838_207,
4_831_838_208,
5_368_709_119,
HARD_LIMIT_BYTES,
5_368_709_121,
].map((bytes) => ({ bytes, ...classifyCapacity(bytes, 0) }));
const directory = process.env.DADA_EVIDENCE_DIR_STO_001;
if (directory) {
mkdirSync(directory, { recursive: true });
writeFileSync(join(directory, "thresholds.json"), `${JSON.stringify({ automatic_cleanup: false, values }, null, 2)}\n`);
writeFileSync(join(directory, "db-diff.json"), `${JSON.stringify({ changed_fields: ["managed_content_bytes", "capacity_notice_level", "storage_status", "measured_at"], deleted_files: 0 }, null, 2)}\n`);
}
});
it("counts only managed binary classes and excludes infrastructure and downloads", () => {
expect(CAPACITY_COUNTED_FILE_KINDS).toEqual([
"reference", "generated", "export", "derived", "sticker_original", "sticker_thumbnail",
]);
expect(CAPACITY_EXCLUDED_STORAGE).toEqual([
"read_only_assets", "application_files", "database", "logs", "audit", "browser_cache", "user_downloads",
]);
});
});
describe("TDD-WP0-STO-003 action matrices", () => {
it("allows only the frozen full-state actions", () => {
expect(decideStorageAction("full", true, "project_json_write")).toBe("allow");
expect(decideStorageAction("full", true, "read")).toBe("allow");
expect(decideStorageAction("full", true, "download")).toBe("allow");
expect(decideStorageAction("full", true, "client_only_download")).toBe("allow");
expect(decideStorageAction("full", true, "permanent_delete")).toBe("allow");
expect(decideStorageAction("full", true, "explicit_cleanup")).toBe("allow");
expect(decideStorageAction("full", true, "binary_write")).toBe("reject_capacity");
expect(decideStorageAction("full", true, "ai_call")).toBe("reject_capacity");
expect(decideStorageAction("full", true, "latest_export_write")).toBe("reject_capacity");
});
it("blocks all new backend writes while unavailable and gates cleanup on SQLite", () => {
for (const action of ["project_json_write", "binary_write", "ai_call", "latest_export_write"] as const) {
expect(decideStorageAction("unavailable", true, action)).toBe("reject_unavailable");
}
for (const action of ["read", "download", "client_only_download"] as const) {
expect(decideStorageAction("unavailable", false, action)).toBe("allow");
}
expect(decideStorageAction("unavailable", true, "permanent_delete")).toBe("allow");
expect(decideStorageAction("unavailable", false, "permanent_delete")).toBe("reject_uncommitted");
expect(decideStorageAction("unavailable", true, "explicit_cleanup")).toBe("allow");
expect(decideStorageAction("unavailable", false, "explicit_cleanup")).toBe("reject_uncommitted");
});
});
@@ -0,0 +1,17 @@
import { describe, expect, it, vi } from "vitest";
import { runStorageMaintenance } from "../../apps/worker/src/storage-maintenance.js";
describe("TDD-WP0-FILE-001 worker reconciliation", () => {
it("reconciles startup before processing physical cleanup", async () => {
const target = {
processCleanupQueue: vi.fn(async () => ({ completed: 1, failed: 0 })),
reconcileStartup: vi.fn(async () => ({ orphaned: 1, released_reservations: 1 })),
};
await expect(runStorageMaintenance(target)).resolves.toEqual({
cleanup: { completed: 1, failed: 0 },
reconciliation: { orphaned: 1, released_reservations: 1 },
});
expect(target.reconcileStartup).toHaveBeenCalledBefore(target.processCleanupQueue);
});
});