diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 4b9cdbe..a3f9546 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -180,6 +180,7 @@ import { ModelConfigurationError } from "./model-configuration.js"; import type { ModelConfigurationService } from "./model-configuration.js"; import { StickerReleaseError } from "./sticker-release-errors.js"; import type { StickerReleaseService } from "./sticker-releases.js"; +import type { ManagedStorage } from "./managed-storage.js"; const defaultBootstrap: BootstrapResponse = { app_version: "0.0.0", @@ -209,6 +210,7 @@ export interface CreateAppOptions { publicAssets?: PublicAssetResolver; recentAssets?: RecentAssetService; projects?: ProjectService; + storage?: ManagedStorage; previewAssetAuthorizer?: (input: { releaseVersion: string; resourceId: string; @@ -382,6 +384,19 @@ function stickerReleaseFailure(reply: FastifyReply, correlationId: string, error return latestExportFailure(reply, correlationId, error); } +function assetCleanupFailure(reply: FastifyReply, correlationId: string, error: unknown) { + const code = error instanceof Error ? error.message : ""; + if (code === "ASSET_HISTORY_REFERENCE_CONFLICT" || code === "ASSET_CLEANUP_CANDIDATE_STALE") { + return reply.code(409).send(createErrorEnvelope({ code, correlationId })); + } + if (code === "IDEMPOTENCY_KEY_CONFLICT") { + return reply.code(409).send(createErrorEnvelope({ code: "IDEMPOTENCY_KEY_CONFLICT", correlationId })); + } + if (code === "cleanup_candidates_invalid") return reply.code(400).send(null); + if (code === "cleanup_uncommitted") return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId })); + return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId })); +} + function generationTaskResponse(task: GenerationTaskView) { return { confirmed_credit_cost: task.confirmedCreditCost, @@ -965,6 +980,78 @@ export async function createApp(options: CreateAppOptions = {}) { }, ); + app.get( + "/api/v1/admin/assets/static-stickers/cleanup/candidates", + { schema: { hide: true } }, + async (request, reply) => { + if (!options.registration || !options.storage) { + return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId: request.id })); + } + const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName); + const session = token ? options.registration.readAdminSession(token) : undefined; + if (!session) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id })); + try { + reply.header("Cache-Control", "private, no-store"); + return options.storage.listAssetCleanupCandidates(); + } catch (error) { + return assetCleanupFailure(reply, request.id, error); + } + }, + ); + + app.post( + "/api/v1/admin/assets/static-stickers/cleanup/intents", + { schema: { hide: true } }, + async (request, reply) => { + if (!options.registration || !options.storage) { + return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId: request.id })); + } + const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName); + const csrfToken = headerValue(request.headers["x-csrf-token"]); + const idempotencyKey = headerValue(request.headers["idempotency-key"]); + if (!token || !csrfToken) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id })); + if (!idempotencyKey || !/^[A-Za-z0-9_-]{32,200}$/.test(idempotencyKey) || !/^[A-Za-z0-9_-]{43,64}$/.test(csrfToken)) return reply.code(400).send(null); + try { + const admin = options.registration.authorizeAdminMutation({ csrfToken, sessionToken: token }); + const body = request.body as { file_ids?: string[]; snapshot_version?: string } | undefined; + return reply.code(201).send(options.storage.createAssetCleanupIntent({ + actorId: admin.userId, + fileIds: body?.file_ids ?? [], + idempotencyKey, + snapshotVersion: body?.snapshot_version ?? "", + })); + } catch (error) { + return assetCleanupFailure(reply, request.id, error); + } + }, + ); + + app.post( + "/api/v1/admin/assets/static-stickers/cleanup/intents/:requestId/confirm", + { schema: { hide: true } }, + async (request, reply) => { + if (!options.registration || !options.storage) { + return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId: request.id })); + } + const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName); + const csrfToken = headerValue(request.headers["x-csrf-token"]); + if (!token || !csrfToken) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id })); + if (!/^[A-Za-z0-9_-]{43,64}$/.test(csrfToken)) return reply.code(400).send(null); + try { + const admin = options.registration.authorizeAdminMutation({ csrfToken, sessionToken: token }); + const body = request.body as { confirmation_token?: string } | undefined; + const requestId = (request.params as { requestId: string }).requestId; + return reply.send(options.storage.confirmAssetCleanupIntent({ + actorId: admin.userId, + confirmationToken: body?.confirmation_token ?? "", + requestId, + })); + } catch (error) { + return assetCleanupFailure(reply, request.id, error); + } + }, + ); + app.get( "/api/v1/assets/public/:resourceVersion/manifest", { schema: { hide: true } }, diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 6a9e0b0..ccbaea5 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -85,6 +85,7 @@ const app = await createApp({ ...(registration ? { registration } : {}), ...(recentAssets ? { recentAssets } : {}), ...(stickers ? { stickers } : {}), + ...(storage ? { storage } : {}), }); await app.listen({ diff --git a/apps/api/src/managed-storage.ts b/apps/api/src/managed-storage.ts index 27021cc..923bfeb 100644 --- a/apps/api/src/managed-storage.ts +++ b/apps/api/src/managed-storage.ts @@ -75,6 +75,31 @@ interface CleanupQueueRow { relative_path: string; } +export interface AssetCleanupCandidateView { + byte_size: number; + file_id: string; + file_kind: "original" | "thumbnail"; + hash_prefix: string; + reference_count: 0; + resource_version: string; + stable_id: string; +} + +export interface AssetCleanupCandidatesView { + candidate_snapshot_version: string; + expires_at: string; + items: AssetCleanupCandidateView[]; +} + +export interface AssetCleanupIntentView { + confirmation_token: string; + expires_at: string; + file_count: number; + request_id: string; + status: "pending_confirmation" | "denied" | "queued" | "completed"; + total_bytes: number; +} + export class StorageCapacityError extends Error { readonly code = "STORAGE_CAPACITY_EXCEEDED"; readonly httpStatus = 507; @@ -104,6 +129,10 @@ function auditExpiry(occurredAt: number) { return occurredAt + auditRetentionMilliseconds; } +function digest(value: string) { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + function validatePositiveBytes(value: number, name: string) { if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name}_invalid`); } @@ -255,6 +284,28 @@ export class ManagedStorage { FOREIGN KEY (request_id) REFERENCES asset_cleanup_requests(request_id), FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id) ); + CREATE TABLE IF NOT EXISTS asset_cleanup_candidate_snapshots ( + snapshot_version TEXT PRIMARY KEY, + items_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS sticker_managed_file_history ( + managed_file_id TEXT NOT NULL, + stable_id TEXT NOT NULL, + resource_version TEXT NOT NULL, + file_kind TEXT NOT NULL CHECK (file_kind IN ('original', 'thumbnail')), + created_at INTEGER NOT NULL, + PRIMARY KEY (managed_file_id, file_kind), + FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id) + ); + CREATE TABLE IF NOT EXISTS project_sticker_asset_refs ( + reference_id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + stable_id TEXT NOT NULL, + resource_version TEXT NOT NULL, + created_at INTEGER NOT NULL + ); CREATE TABLE IF NOT EXISTS admin_operation_logs ( log_id TEXT PRIMARY KEY, actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')), @@ -277,6 +328,49 @@ export class ManagedStorage { if (!managedFileColumns.some((column) => column.name === "owner_ref")) { this.database.exec("ALTER TABLE managed_files ADD COLUMN owner_ref TEXT"); } + if (!managedFileColumns.some((column) => column.name === "cleanup_status")) { + this.database.exec("ALTER TABLE managed_files ADD COLUMN cleanup_status TEXT"); + } + const cleanupRequestColumns = this.database.prepare("PRAGMA table_info(asset_cleanup_requests)").all() as Array<{ name: string }>; + const cleanupRequestAdditions: Array<[string, string]> = [ + ["created_by", "TEXT"], + ["confirmed_by", "TEXT"], + ["snapshot_version", "TEXT"], + ["expires_at", "INTEGER"], + ["confirmation_token_digest", "TEXT"], + ["idempotency_key_digest", "TEXT"], + ["request_hash", "TEXT"], + ["file_count", "INTEGER"], + ["total_bytes", "INTEGER"], + ["denied_reason", "TEXT"], + ]; + for (const [column, type] of cleanupRequestAdditions) { + if (!cleanupRequestColumns.some((item) => item.name === column)) { + this.database.exec(`ALTER TABLE asset_cleanup_requests ADD COLUMN ${column} ${type}`); + } + } + const cleanupItemColumns = this.database.prepare("PRAGMA table_info(asset_cleanup_request_items)").all() as Array<{ name: string }>; + const cleanupItemAdditions: Array<[string, string]> = [ + ["stable_id", "TEXT"], + ["resource_version", "TEXT"], + ["file_kind", "TEXT"], + ["byte_size", "INTEGER"], + ["sha256_prefix", "TEXT"], + ]; + for (const [column, type] of cleanupItemAdditions) { + if (!cleanupItemColumns.some((item) => item.name === column)) { + this.database.exec(`ALTER TABLE asset_cleanup_request_items ADD COLUMN ${column} ${type}`); + } + } + this.database.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS asset_cleanup_requests_actor_idempotency + ON asset_cleanup_requests (created_by, idempotency_key_digest) + WHERE created_by IS NOT NULL AND idempotency_key_digest IS NOT NULL; + CREATE INDEX IF NOT EXISTS sticker_managed_file_history_lookup + ON sticker_managed_file_history (stable_id, resource_version, file_kind); + CREATE INDEX IF NOT EXISTS asset_cleanup_candidate_snapshots_expiry + ON asset_cleanup_candidate_snapshots (expires_at); + `); ensureAdminOperationAuditSchema(this.database, Date.now()); const initial = classifyCapacity(0, 0); this.database.prepare(` @@ -299,6 +393,106 @@ export class ManagedStorage { return withReservations; } + private readAssetCleanupCandidates(): AssetCleanupCandidateView[] { + const releaseReferenceClause = this.tableExists("sticker_release_items") ? ` + AND NOT EXISTS ( + SELECT 1 FROM sticker_release_items release_items + WHERE release_items.original_file_id = mf.file_id OR release_items.thumbnail_file_id = mf.file_id + )` : ""; + return this.database.prepare(` + SELECT + mf.file_id, + mf.byte_size, + history.stable_id, + history.resource_version, + history.file_kind, + substr(mf.sha256, 1, 12) AS hash_prefix, + 0 AS reference_count + FROM sticker_managed_file_history history + JOIN managed_files mf ON mf.file_id = history.managed_file_id + WHERE mf.status = 'committed' + AND mf.cleanup_status IS NULL + AND mf.file_kind IN ('sticker_original', 'sticker_thumbnail') + AND NOT EXISTS ( + SELECT 1 FROM project_asset_refs refs WHERE refs.managed_file_id = mf.file_id + ) + AND NOT EXISTS ( + SELECT 1 FROM project_sticker_asset_refs project_refs + WHERE project_refs.stable_id = history.stable_id + AND project_refs.resource_version = history.resource_version + ) + ${releaseReferenceClause} + AND NOT EXISTS ( + SELECT 1 FROM asset_cleanup_request_items request_items + JOIN asset_cleanup_requests requests ON requests.request_id = request_items.request_id + WHERE request_items.managed_file_id = mf.file_id + AND requests.status IN ('pending_confirmation', 'queued') + ) + ORDER BY history.stable_id, history.resource_version, history.file_kind, mf.file_id + `).all() as AssetCleanupCandidateView[]; + } + + private assertActiveAdmin(actorId: string) { + const admin = this.database.prepare(` + SELECT 1 AS allowed FROM users u + JOIN admin_access access ON access.user_id = u.user_id + WHERE u.user_id = ? AND u.role = 'super_admin' AND u.status = 'active' AND access.allowed = 1 + `).get(actorId); + if (!admin) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + } + + private assetReferenceCount(fileId: string, requestId: string) { + const projectOrRelease = (this.database.prepare(` + SELECT COUNT(*) AS count FROM project_asset_refs WHERE managed_file_id = ? + `).get(fileId) as { count: number }).count; + const releaseItems = this.tableExists("sticker_release_items") + ? (this.database.prepare(` + SELECT COUNT(*) AS count FROM sticker_release_items + WHERE original_file_id = ? OR thumbnail_file_id = ? + `).get(fileId, fileId) as { count: number }).count + : 0; + const projectStickerRefs = (this.database.prepare(` + SELECT COUNT(*) AS count + FROM sticker_managed_file_history history + JOIN project_sticker_asset_refs refs + ON refs.stable_id = history.stable_id AND refs.resource_version = history.resource_version + WHERE history.managed_file_id = ? + `).get(fileId) as { count: number }).count; + const otherCleanup = (this.database.prepare(` + SELECT COUNT(*) AS count FROM asset_cleanup_request_items items + JOIN asset_cleanup_requests requests ON requests.request_id = items.request_id + WHERE items.managed_file_id = ? AND items.request_id <> ? + AND requests.status IN ('pending_confirmation', 'queued') + `).get(fileId, requestId) as { count: number }).count; + return projectOrRelease + releaseItems + projectStickerRefs + otherCleanup; + } + + private cleanupConfirmationToken(requestId: string, actorId: string, keyDigest: string) { + return digest(`Dada/P0A/asset-cleanup-confirm/v1:${requestId}:${actorId}:${keyDigest}`); + } + + private tableExists(name: string) { + return Boolean(this.database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name)); + } + + private insertCleanupAudit(input: { + actorRef: string; + afterSummary: Record; + operationType: string; + requestId: string; + result: "failed" | "succeeded"; + }, occurredAt: number) { + this.database.prepare(` + INSERT INTO admin_operation_logs ( + log_id, actor_type, actor_ref, operation_type, target_type, target_ref, + result, before_summary, after_summary, occurred_at, expires_at + ) VALUES (?, 'super_admin', ?, ?, 'asset_cleanup_request', ?, ?, NULL, ?, ?, ?) + `).run( + randomUUID(), input.actorRef, input.operationType, input.requestId, input.result, + serializeAuditSummary(input.afterSummary), occurredAt, auditExpiry(occurredAt), + ); + } + private activeReservationBytes(excludingOperationId?: string) { const row = this.database.prepare(` SELECT COALESCE(SUM(projected_bytes), 0) AS bytes @@ -692,6 +886,203 @@ export class ManagedStorage { this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(fileId); } + listAssetCleanupCandidates(): AssetCleanupCandidatesView { + const createdAt = Date.now(); + const expiresAt = createdAt + 5 * 60 * 1_000; + const items = this.readAssetCleanupCandidates(); + const snapshotVersion = digest(JSON.stringify({ + created_at: createdAt, + nonce: randomUUID(), + items: items.map((item) => ({ byte_size: item.byte_size, file_id: item.file_id, hash_prefix: item.hash_prefix })), + })); + this.database.prepare("DELETE FROM asset_cleanup_candidate_snapshots WHERE expires_at <= ?").run(createdAt); + this.database.prepare(` + INSERT INTO asset_cleanup_candidate_snapshots ( + snapshot_version, items_json, created_at, expires_at + ) VALUES (?, ?, ?, ?) + `).run(snapshotVersion, JSON.stringify(items), createdAt, expiresAt); + return { + candidate_snapshot_version: snapshotVersion, + expires_at: new Date(expiresAt).toISOString(), + items, + }; + } + + createAssetCleanupIntent(input: { + actorId: string; + fileIds: string[]; + idempotencyKey: string; + snapshotVersion: string; + }): AssetCleanupIntentView { + if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted"); + const fileIds = [...new Set(input.fileIds)].sort(); + if (!uuidPattern.test(input.actorId) || fileIds.length === 0 || fileIds.length !== input.fileIds.length + || fileIds.length > 100 || fileIds.some((fileId) => !uuidPattern.test(fileId)) + || !/^[A-Za-z0-9_-]{32,200}$/.test(input.idempotencyKey) + || !/^[0-9a-f]{64}$/.test(input.snapshotVersion)) { + throw new Error("cleanup_candidates_invalid"); + } + const keyDigest = digest(input.idempotencyKey); + const requestHash = digest(JSON.stringify({ file_ids: fileIds, snapshot_version: input.snapshotVersion })); + const existing = this.database.prepare(` + SELECT request_id, request_hash, expires_at, file_count, total_bytes, status + FROM asset_cleanup_requests + WHERE created_by = ? AND idempotency_key_digest = ? + `).get(input.actorId, keyDigest) as { + expires_at: number; file_count: number; request_hash: string; request_id: string; status: AssetCleanupIntentView["status"]; total_bytes: number; + } | undefined; + if (existing) { + if (existing.request_hash !== requestHash) throw new Error("IDEMPOTENCY_KEY_CONFLICT"); + return { + confirmation_token: this.cleanupConfirmationToken(existing.request_id, input.actorId, keyDigest), + expires_at: new Date(existing.expires_at).toISOString(), + file_count: existing.file_count, + request_id: existing.request_id, + status: existing.status, + total_bytes: existing.total_bytes, + }; + } + + const requestId = randomUUID(); + const confirmationToken = this.cleanupConfirmationToken(requestId, input.actorId, keyDigest); + const createdAt = Date.now(); + let view!: AssetCleanupIntentView; + const transaction = this.database.transaction(() => { + this.assertActiveAdmin(input.actorId); + const snapshot = this.database.prepare(` + SELECT items_json, expires_at FROM asset_cleanup_candidate_snapshots + WHERE snapshot_version = ? + `).get(input.snapshotVersion) as { expires_at: number; items_json: string } | undefined; + if (!snapshot || snapshot.expires_at <= createdAt) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + const snapshotItems = JSON.parse(snapshot.items_json) as AssetCleanupCandidateView[]; + const byId = new Map(snapshotItems.map((item) => [item.file_id, item])); + const selected = fileIds.map((fileId) => byId.get(fileId)); + if (selected.some((item) => !item)) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + const current = new Map(this.readAssetCleanupCandidates().map((item) => [item.file_id, item])); + if (fileIds.some((fileId) => !current.has(fileId))) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + const safeItems = selected as AssetCleanupCandidateView[]; + const totalBytes = safeItems.reduce((sum, item) => sum + item.byte_size, 0); + this.database.prepare(` + INSERT INTO asset_cleanup_requests ( + request_id, status, created_at, confirmed_at, created_by, confirmed_by, + snapshot_version, expires_at, confirmation_token_digest, + idempotency_key_digest, request_hash, file_count, total_bytes, denied_reason + ) VALUES (?, 'pending_confirmation', ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, NULL) + `).run( + requestId, new Date(createdAt).toISOString(), input.actorId, input.snapshotVersion, + snapshot.expires_at, digest(confirmationToken), keyDigest, requestHash, safeItems.length, totalBytes, + ); + const insert = this.database.prepare(` + INSERT INTO asset_cleanup_request_items ( + request_id, managed_file_id, stable_id, resource_version, file_kind, byte_size, sha256_prefix + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `); + for (const item of safeItems) { + insert.run(requestId, item.file_id, item.stable_id, item.resource_version, item.file_kind, item.byte_size, item.hash_prefix); + } + this.insertCleanupAudit({ + actorRef: input.actorId, + afterSummary: { file_count: safeItems.length, snapshot_version: input.snapshotVersion, total_bytes: totalBytes }, + operationType: "asset_cleanup_requested", + requestId, + result: "succeeded", + }, createdAt); + view = { + confirmation_token: confirmationToken, + expires_at: new Date(snapshot.expires_at).toISOString(), + file_count: safeItems.length, + request_id: requestId, + status: "pending_confirmation", + total_bytes: totalBytes, + }; + }); + transaction.immediate(); + return view; + } + + confirmAssetCleanupIntent(input: { actorId: string; confirmationToken: string; requestId: string }) { + if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted"); + if (!uuidPattern.test(input.actorId) || !uuidPattern.test(input.requestId) || !/^[0-9a-f]{64}$/.test(input.confirmationToken)) { + throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + } + const confirmedAt = Date.now(); + const outcome = this.database.transaction(() => { + this.assertActiveAdmin(input.actorId); + const request = this.database.prepare(` + SELECT status, created_by, expires_at, confirmation_token_digest, file_count, total_bytes + FROM asset_cleanup_requests WHERE request_id = ? + `).get(input.requestId) as { + confirmation_token_digest: string | null; created_by: string | null; expires_at: number | null; + file_count: number | null; status: string; total_bytes: number | null; + } | undefined; + if (!request || request.status !== "pending_confirmation" || request.created_by !== input.actorId + || !request.expires_at || request.expires_at <= confirmedAt + || request.confirmation_token_digest !== digest(input.confirmationToken)) { + throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + } + const files = this.database.prepare(` + SELECT mf.file_id, mf.file_kind, mf.relative_path, mf.byte_size, mf.status + FROM asset_cleanup_request_items items + JOIN managed_files mf ON mf.file_id = items.managed_file_id + WHERE items.request_id = ? ORDER BY mf.file_id + `).all(input.requestId) as ManagedFileRow[]; + if (files.length !== request.file_count) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE"); + const conflicted = files.some((file) => file.status !== "committed" + || !new Set(["sticker_original", "sticker_thumbnail"]).has(file.file_kind) + || this.assetReferenceCount(file.file_id, input.requestId) > 0); + if (conflicted) { + this.database.prepare(` + UPDATE asset_cleanup_requests + SET status = 'denied', confirmed_at = ?, confirmed_by = ?, denied_reason = 'reference_conflict' + WHERE request_id = ? + `).run(new Date(confirmedAt).toISOString(), input.actorId, input.requestId); + this.insertCleanupAudit({ + actorRef: input.actorId, + afterSummary: { file_count: files.length, reason: "reference_conflict", status: "denied" }, + operationType: "asset_cleanup_reference_denied", + requestId: input.requestId, + result: "failed", + }, confirmedAt); + return { conflict: true as const }; + } + + this.insertCleanupAudit({ + actorRef: input.actorId, + afterSummary: { file_count: files.length, status: "validated" }, + operationType: "asset_cleanup_validated", + requestId: input.requestId, + result: "succeeded", + }, confirmedAt); + for (const file of files) { + this.database.prepare(` + UPDATE managed_files SET status = 'purged', purged_at = ?, cleanup_status = 'pending_delete' + WHERE file_id = ? AND status = 'committed' + `).run(new Date(confirmedAt).toISOString(), file.file_id); + this.database.prepare(` + INSERT INTO file_cleanup_queue ( + cleanup_id, managed_file_id, relative_path, byte_size, counts_toward_managed, + reason, status, created_at, completed_at, last_error + ) VALUES (?, ?, ?, ?, 1, 'purge', 'pending', ?, NULL, NULL) + `).run(randomUUID(), file.file_id, file.relative_path, file.byte_size, new Date(confirmedAt).toISOString()); + } + this.database.prepare(` + UPDATE asset_cleanup_requests + SET status = 'queued', confirmed_at = ?, confirmed_by = ? + WHERE request_id = ? + `).run(new Date(confirmedAt).toISOString(), input.actorId, input.requestId); + this.insertCleanupAudit({ + actorRef: input.actorId, + afterSummary: { file_count: files.length, status: "queued", total_bytes: request.total_bytes ?? 0 }, + operationType: "asset_cleanup_scheduled", + requestId: input.requestId, + result: "succeeded", + }, confirmedAt + 1); + return { conflict: false as const, file_count: files.length, request_id: input.requestId, status: "queued" as const }; + }).immediate(); + if (outcome.conflict) throw new Error("ASSET_HISTORY_REFERENCE_CONFLICT"); + return outcome; + } + createCleanupIntent(fileIds: string[]) { if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted"); if (fileIds.length === 0 || new Set(fileIds).size !== fileIds.length) throw new Error("cleanup_candidates_invalid"); @@ -777,6 +1168,7 @@ export class ManagedStorage { this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(row.managed_file_id); const requests = this.database.prepare("SELECT request_id FROM asset_cleanup_request_items WHERE managed_file_id = ?").all(row.managed_file_id) as Array<{ request_id: string }>; this.database.prepare("DELETE FROM asset_cleanup_request_items WHERE managed_file_id = ?").run(row.managed_file_id); + this.database.prepare("DELETE FROM sticker_managed_file_history WHERE managed_file_id = ?").run(row.managed_file_id); this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id); for (const request of requests) { const pendingItems = this.database.prepare("SELECT COUNT(*) AS count FROM asset_cleanup_request_items WHERE request_id = ?").get(request.request_id) as { count: number }; diff --git a/apps/api/src/projects.ts b/apps/api/src/projects.ts index a65baf1..6d9a733 100644 --- a/apps/api/src/projects.ts +++ b/apps/api/src/projects.ts @@ -366,6 +366,7 @@ export class ProjectService { throw new ProjectError("project_state_conflict", latest); } this.insertProjectState({ canvasState: canvas, name, projectId: input.projectId, stateVersion: nextVersion }, now); + this.rebuildProjectStickerReferences(input.projectId, canvas, now); this.database.prepare(` INSERT INTO project_state_idempotency ( owner_id, project_id, idempotency_key, request_hash, response_state_version, created_at @@ -377,6 +378,25 @@ export class ProjectService { return result; } + private rebuildProjectStickerReferences(projectId: string, canvas: CanvasState, now: number) { + if (!this.tableExists("project_sticker_asset_refs")) return; + this.database.prepare("DELETE FROM project_sticker_asset_refs WHERE project_id = ?").run(projectId); + const insert = this.database.prepare(` + INSERT INTO project_sticker_asset_refs (reference_id, project_id, stable_id, resource_version, created_at) + VALUES (?, ?, ?, ?, ?) + `); + for (const element of canvas.elements) { + if (element.type !== "static_sticker") continue; + insert.run( + `project:${projectId}:sticker:${element.element_id}`, + projectId, + element.template_or_asset_id, + element.resource_version, + now, + ); + } + } + trashFailedEmpty(ownerId: string, projectIds: string[]) { const uniqueIds = [...new Set(projectIds)]; if (uniqueIds.length === 0 || uniqueIds.length > projectLimit) throw new ProjectError("generation_state_invalid"); @@ -774,6 +794,16 @@ export class ProjectService { FOREIGN KEY (project_id) REFERENCES projects(project_id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS project_resource_files_managed ON project_resource_files(managed_file_id, project_id); + CREATE TABLE IF NOT EXISTS project_sticker_asset_refs ( + reference_id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + stable_id TEXT NOT NULL, + resource_version TEXT NOT NULL, + created_at INTEGER NOT NULL, + FOREIGN KEY (project_id) REFERENCES projects(project_id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS project_sticker_asset_refs_lookup + ON project_sticker_asset_refs (stable_id, resource_version); CREATE TABLE IF NOT EXISTS latest_exports ( project_id TEXT NOT NULL, format TEXT NOT NULL CHECK (format IN ('jpg', 'png')), diff --git a/apps/api/src/sticker-releases.ts b/apps/api/src/sticker-releases.ts index f970b43..fc252a8 100644 --- a/apps/api/src/sticker-releases.ts +++ b/apps/api/src/sticker-releases.ts @@ -328,6 +328,14 @@ export class StickerReleaseService { input.width, input.height, input.expectedMimeType, input.original.sha256, input.original.fileId, input.original.bytes, input.thumbnail.fileId, input.thumbnail.relativePath, input.thumbnail.sha256, input.thumbnail.bytes, input.enabled ? 1 : 0, ); + this.database.prepare(` + INSERT OR IGNORE INTO sticker_managed_file_history ( + managed_file_id, stable_id, resource_version, file_kind, created_at + ) VALUES (?, ?, ?, 'original', ?), (?, ?, ?, 'thumbnail', ?) + `).run( + input.original.fileId, input.stableId, releaseVersion, this.clock(), + input.thumbnail.fileId, input.stableId, releaseVersion, this.clock(), + ); this.consumeStagedStorage([input.original, input.thumbnail]); this.database.prepare(` INSERT INTO sticker_upload_receipts (actor_id, idempotency_key_digest, request_hash, release_version, stable_id, created_at) @@ -510,6 +518,15 @@ export class StickerReleaseService { created_at TEXT NOT NULL, PRIMARY KEY (actor_id, idempotency_key_digest) ); + CREATE TABLE IF NOT EXISTS sticker_managed_file_history ( + managed_file_id TEXT NOT NULL, + stable_id TEXT NOT NULL, + resource_version TEXT NOT NULL, + file_kind TEXT NOT NULL CHECK (file_kind IN ('original', 'thumbnail')), + created_at INTEGER NOT NULL, + PRIMARY KEY (managed_file_id, file_kind), + FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id) + ); CREATE TRIGGER IF NOT EXISTS sticker_releases_no_update BEFORE UPDATE ON sticker_releases BEGIN SELECT RAISE(ABORT, 'sticker_releases_immutable'); END; CREATE TRIGGER IF NOT EXISTS sticker_releases_no_delete @@ -523,5 +540,17 @@ export class StickerReleaseService { WHEN EXISTS (SELECT 1 FROM sticker_releases WHERE release_version = OLD.release_version) BEGIN SELECT RAISE(ABORT, 'sticker_release_items_immutable'); END; `); + this.database.exec(` + INSERT OR IGNORE INTO sticker_managed_file_history ( + managed_file_id, stable_id, resource_version, file_kind, created_at + ) + SELECT original_file_id, stable_id, release_version, 'original', strftime('%s', 'now') * 1000 + FROM sticker_release_items; + INSERT OR IGNORE INTO sticker_managed_file_history ( + managed_file_id, stable_id, resource_version, file_kind, created_at + ) + SELECT thumbnail_file_id, stable_id, release_version, 'thumbnail', strftime('%s', 'now') * 1000 + FROM sticker_release_items; + `); } } diff --git a/apps/worker/src/project-purge-cleanup.ts b/apps/worker/src/project-purge-cleanup.ts index 7ad034a..76d0edb 100644 --- a/apps/worker/src/project-purge-cleanup.ts +++ b/apps/worker/src/project-purge-cleanup.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { rmSync } from "node:fs"; +import { existsSync, rmSync, statSync } from "node:fs"; import { createRequire } from "node:module"; import { isAbsolute, relative, resolve } from "node:path"; @@ -29,6 +29,40 @@ interface FileCleanupRow { const resourceScope = JSON.stringify([ "project_state", "generation", "generated_image", "reference", "location", "latest_export", ]); +const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000; +const auditRefPattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$/; +const forbiddenSummaryKeys = new Set([ + "absolute_path", "api_key", "body", "code_hmac", "content", "credential", "email", "image", + "image_content", "password", "path", "prompt", "secret", "session_token", "verification_code", "whitelist", +]); +const forbiddenSummaryFragments = ["content", "credential", "email", "image", "password", "path", "prompt", "secret", "token"]; + +function isSafeAuditRef(value: unknown) { + return typeof value === "string" && auditRefPattern.test(value) ? 1 : 0; +} + +function isSafeAuditSummaryJson(value: unknown) { + if (typeof value !== "string" || Buffer.byteLength(value, "utf8") > 2_048) return 0; + try { + const valid = (entry: unknown, depth: number): boolean => { + if (depth > 3) return false; + if (entry === null || typeof entry === "boolean") return true; + if (typeof entry === "number") return Number.isSafeInteger(entry); + if (typeof entry === "string") return /^[A-Za-z0-9_.:@-]{1,160}$/.test(entry) && !entry.includes("@"); + if (Array.isArray(entry)) return entry.length <= 20 && entry.every((item) => valid(item, depth + 1)); + if (!entry || typeof entry !== "object") return false; + return Object.entries(entry).length <= 32 && Object.entries(entry).every(([key, item]) => ( + auditRefPattern.test(key) + && !forbiddenSummaryKeys.has(key.toLowerCase()) + && !forbiddenSummaryFragments.some((fragment) => key.toLowerCase().includes(fragment)) + && valid(item, depth + 1) + )); + }; + return valid(JSON.parse(value), 0) ? 1 : 0; + } catch { + return 0; + } +} function iso(timestamp: number) { return new Date(timestamp).toISOString(); @@ -47,6 +81,12 @@ export class ProjectPurgeCleanup { this.database.pragma("journal_mode = WAL"); this.database.pragma("foreign_keys = ON"); this.database.pragma("busy_timeout = 5000"); + this.database.function("dada_audit_ref_is_safe", { deterministic: true }, isSafeAuditRef); + this.database.function("dada_audit_summary_is_safe", { deterministic: true }, isSafeAuditSummaryJson); + this.database.function("dada_allow_privacy_purge", { deterministic: false }, () => 0); + this.database.function("dada_privacy_purge_subject", { deterministic: false }, () => ""); + this.database.function("dada_allow_retention_purge", { deterministic: false }, () => 0); + this.database.function("dada_retention_purge_now", { deterministic: false }, () => 0); } close() { @@ -153,12 +193,16 @@ export class ProjectPurgeCleanup { if (pending.count === 0) { this.database.prepare("UPDATE asset_cleanup_requests SET status = 'completed' WHERE request_id = ? AND status = 'queued'") .run(request.request_id); + this.insertAssetCleanupAudit(request.request_id, row.byte_size, this.clock()); } } } } + if (this.tableExists("sticker_managed_file_history")) { + this.database.prepare("DELETE FROM sticker_managed_file_history WHERE managed_file_id = ?").run(row.managed_file_id); + } this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id); - if (this.tableExists("local_backend_storage_state")) this.decrementManagedCapacity(row.byte_size); + if (this.tableExists("local_backend_storage_state")) this.remeasureManagedCapacity(); } this.database.prepare(` UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL @@ -238,4 +282,46 @@ export class ProjectPurgeCleanup { WHERE singleton = 1 `).run(managed, notice, status, iso(this.clock())); } + + private insertAssetCleanupAudit(requestId: string, deletedBytes: number, occurredAt: number) { + if (!this.tableExists("admin_operation_logs")) return; + this.database.prepare(` + INSERT INTO admin_operation_logs ( + log_id, actor_type, actor_ref, operation_type, target_type, target_ref, + result, before_summary, after_summary, occurred_at, expires_at + ) VALUES (?, 'system', 'project_purge_worker', 'asset_cleanup_physical_completed', 'asset_cleanup', ?, 'succeeded', NULL, ?, ?, ?) + `).run( + randomUUID(), requestId, JSON.stringify({ deleted_bytes: deletedBytes, status: "completed" }), occurredAt, + occurredAt + auditRetentionMilliseconds, + ); + } + + private remeasureManagedCapacity() { + const state = this.database.prepare(` + SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1 + `).get() as { managed_content_bytes: number } | undefined; + if (!state) return; + const rows = this.database.prepare(` + SELECT relative_path FROM managed_files WHERE status = 'committed' + `).all() as Array<{ relative_path: string }>; + let managed = 0; + for (const row of rows) { + try { + const path = this.resolveManagedPath(row.relative_path); + if (existsSync(path)) managed += statSync(path).size; + } catch { + // A missing or invalid path is excluded from the measured physical total. + } + } + const reservations = this.tableExists("storage_reservations") + ? (this.database.prepare("SELECT COALESCE(SUM(projected_bytes), 0) AS bytes FROM storage_reservations WHERE status = 'active'").get() as { bytes: number }).bytes + : 0; + const notice = managed < 4_294_967_296 ? "normal" : managed < 4_831_838_208 ? "warning" : "critical"; + const status = managed + reservations >= 5_368_709_120 ? "full" : "active"; + this.database.prepare(` + UPDATE local_backend_storage_state + SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1 + WHERE singleton = 1 + `).run(managed, notice, status, iso(this.clock())); + } } diff --git a/tests/integration/wp5-06-sticker-cleanup.test.ts b/tests/integration/wp5-06-sticker-cleanup.test.ts new file mode 100644 index 0000000..b15ad1f --- /dev/null +++ b/tests/integration/wp5-06-sticker-cleanup.test.ts @@ -0,0 +1,139 @@ +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { ManagedStorage } from "../../apps/api/src/managed-storage.js"; +import { RegistrationService } from "../../apps/api/src/registration.js"; +import { MockResendAdapter } from "../../apps/api/src/resend-adapter.js"; +import { StickerReleaseService } from "../../apps/api/src/sticker-releases.js"; +import { ProjectPurgeCleanup } from "../../apps/worker/src/project-purge-cleanup.js"; + +const roots: string[] = []; +const closeables: Array<{ close(): void }> = []; + +function fixture() { + const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp5-06-cleanup-")); + roots.push(dataRoot); + mkdirSync(join(dataRoot, "db"), { recursive: true }); + const databasePath = join(dataRoot, "db", "dada.sqlite3"); + const registration = new RegistrationService({ + challengePepper: Buffer.alloc(32, 0x61), + currentPrivacyNoticeVersion: "p0a-registration-notice-v1", + databasePath, + invitePepper: Buffer.alloc(32, 0x62), + resend: new MockResendAdapter(), + sessionPepper: Buffer.alloc(32, 0x63), + }); + const storage = new ManagedStorage({ dataRoot, databasePath }); + const stickers = new StickerReleaseService({ databasePath, storage }); + closeables.push(stickers, storage, registration); + return { dataRoot, databasePath, database: registration.database, storage }; +} + +function seedAdmin(database: RegistrationService["database"]) { + const adminId = randomUUID(); + database.prepare(` + INSERT INTO users ( + user_id, normalized_email, role, status, counts_toward_stage_limit, + registration_id, created_at + ) VALUES (?, ?, 'super_admin', 'active', 0, ?, ?) + `).run(adminId, `${adminId}@example.invalid`, randomUUID(), Date.now()); + database.prepare("INSERT INTO admin_access (user_id, allowed) VALUES (?, 1)").run(adminId); + return adminId; +} + +async function seedHistoricalPair(test: ReturnType) { + const original = await test.storage.commitBufferFixture("sticker_original", "STK2401.png", Buffer.from("original-history")); + const thumbnail = await test.storage.commitBufferFixture("sticker_thumbnail", "STK2401-thumbnail.png", Buffer.from("thumbnail-history")); + const insert = test.database.prepare(` + INSERT INTO sticker_managed_file_history ( + managed_file_id, stable_id, resource_version, file_kind, created_at + ) VALUES (?, 'STK2401', 'asset-20260701.1', ?, ?) + `); + insert.run(original.file_id, "original", Date.now()); + insert.run(thumbnail.file_id, "thumbnail", Date.now()); + return { original, thumbnail }; +} + +afterEach(() => { + for (const value of closeables.splice(0).reverse()) value.close(); + for (const root of roots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); + +describe("TDD-WP5-CLN-001 sticker history cleanup", () => { + it("denies the whole batch when a reference appears after the candidate snapshot", async () => { + const test = fixture(); + const adminId = seedAdmin(test.database); + const files = await seedHistoricalPair(test); + const candidates = test.storage.listAssetCleanupCandidates(); + expect(candidates.items).toEqual(expect.arrayContaining([ + expect.objectContaining({ file_id: files.original.file_id, file_kind: "original", reference_count: 0, stable_id: "STK2401" }), + expect.objectContaining({ file_id: files.thumbnail.file_id, file_kind: "thumbnail", reference_count: 0, stable_id: "STK2401" }), + ])); + const intent = test.storage.createAssetCleanupIntent({ + actorId: adminId, + fileIds: [files.original.file_id, files.thumbnail.file_id], + idempotencyKey: `cleanup-${randomUUID()}-${randomUUID()}`, + snapshotVersion: candidates.candidate_snapshot_version, + }); + + test.storage.addAssetReference(files.original.file_id, "release"); + expect(() => test.storage.confirmAssetCleanupIntent({ + actorId: adminId, + confirmationToken: intent.confirmation_token, + requestId: intent.request_id, + })).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT"); + + expect(test.database.prepare("SELECT status FROM asset_cleanup_requests WHERE request_id = ?").get(intent.request_id)).toEqual({ status: "denied" }); + expect(test.database.prepare("SELECT COUNT(*) AS count FROM file_cleanup_queue WHERE status = 'pending'").get()).toEqual({ count: 0 }); + expect(test.database.prepare("SELECT COUNT(*) AS count FROM admin_operation_logs WHERE target_ref = ? AND result = 'failed'").get(intent.request_id)).toEqual({ count: 1 }); + }); + + it("requires the same admin, queues without reducing capacity, then remeasures after physical deletion", async () => { + const test = fixture(); + const adminId = seedAdmin(test.database); + const otherAdminId = seedAdmin(test.database); + const files = await seedHistoricalPair(test); + const bytesBefore = test.storage.getState().managed_content_bytes; + const candidates = test.storage.listAssetCleanupCandidates(); + const intent = test.storage.createAssetCleanupIntent({ + actorId: adminId, + fileIds: [files.original.file_id, files.thumbnail.file_id], + idempotencyKey: `cleanup-${randomUUID()}-${randomUUID()}`, + snapshotVersion: candidates.candidate_snapshot_version, + }); + + expect(() => test.storage.confirmAssetCleanupIntent({ + actorId: otherAdminId, + confirmationToken: intent.confirmation_token, + requestId: intent.request_id, + })).toThrow("ASSET_CLEANUP_CANDIDATE_STALE"); + const queued = test.storage.confirmAssetCleanupIntent({ + actorId: adminId, + confirmationToken: intent.confirmation_token, + requestId: intent.request_id, + }); + expect(queued).toMatchObject({ file_count: 2, status: "queued" }); + expect(test.storage.getState().managed_content_bytes).toBe(bytesBefore); + expect(existsSync(join(test.dataRoot, files.original.relative_path))).toBe(true); + expect(test.database.prepare("SELECT COUNT(*) AS count FROM file_cleanup_queue WHERE status = 'pending'").get()).toEqual({ count: 2 }); + + const worker = new ProjectPurgeCleanup({ dataRoot: test.dataRoot, databasePath: test.databasePath }); + const result = worker.processFileCleanup(); + worker.close(); + expect(result).toEqual({ completed: 2, failed: 0 }); + expect(existsSync(join(test.dataRoot, files.original.relative_path))).toBe(false); + expect(existsSync(join(test.dataRoot, files.thumbnail.relative_path))).toBe(false); + expect(test.storage.getState()).toMatchObject({ managed_content_bytes: 0, storage_status: "active" }); + expect(test.database.prepare("SELECT status FROM asset_cleanup_requests WHERE request_id = ?").get(intent.request_id)).toEqual({ status: "completed" }); + expect(test.database.prepare("SELECT operation_type, result FROM admin_operation_logs WHERE target_ref = ? ORDER BY occurred_at").all(intent.request_id)).toEqual([ + { operation_type: "asset_cleanup_requested", result: "succeeded" }, + { operation_type: "asset_cleanup_validated", result: "succeeded" }, + { operation_type: "asset_cleanup_scheduled", result: "succeeded" }, + { operation_type: "asset_cleanup_physical_completed", result: "succeeded" }, + ]); + }); +});