Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c2521f7208 | ||
|
|
b00500512e |
@@ -180,6 +180,7 @@ import { ModelConfigurationError } from "./model-configuration.js";
|
|||||||
import type { ModelConfigurationService } from "./model-configuration.js";
|
import type { ModelConfigurationService } from "./model-configuration.js";
|
||||||
import { StickerReleaseError } from "./sticker-release-errors.js";
|
import { StickerReleaseError } from "./sticker-release-errors.js";
|
||||||
import type { StickerReleaseService } from "./sticker-releases.js";
|
import type { StickerReleaseService } from "./sticker-releases.js";
|
||||||
|
import type { ManagedStorage } from "./managed-storage.js";
|
||||||
|
|
||||||
const defaultBootstrap: BootstrapResponse = {
|
const defaultBootstrap: BootstrapResponse = {
|
||||||
app_version: "0.0.0",
|
app_version: "0.0.0",
|
||||||
@@ -209,6 +210,7 @@ export interface CreateAppOptions {
|
|||||||
publicAssets?: PublicAssetResolver;
|
publicAssets?: PublicAssetResolver;
|
||||||
recentAssets?: RecentAssetService;
|
recentAssets?: RecentAssetService;
|
||||||
projects?: ProjectService;
|
projects?: ProjectService;
|
||||||
|
storage?: ManagedStorage;
|
||||||
previewAssetAuthorizer?: (input: {
|
previewAssetAuthorizer?: (input: {
|
||||||
releaseVersion: string;
|
releaseVersion: string;
|
||||||
resourceId: string;
|
resourceId: string;
|
||||||
@@ -382,6 +384,19 @@ function stickerReleaseFailure(reply: FastifyReply, correlationId: string, error
|
|||||||
return latestExportFailure(reply, correlationId, error);
|
return latestExportFailure(reply, correlationId, error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function assetCleanupFailure(reply: FastifyReply, correlationId: string, error: unknown) {
|
||||||
|
const code = error instanceof Error ? error.message : "";
|
||||||
|
if (code === "ASSET_HISTORY_REFERENCE_CONFLICT" || code === "ASSET_CLEANUP_CANDIDATE_STALE") {
|
||||||
|
return reply.code(409).send(createErrorEnvelope({ code, correlationId }));
|
||||||
|
}
|
||||||
|
if (code === "IDEMPOTENCY_KEY_CONFLICT") {
|
||||||
|
return reply.code(409).send(createErrorEnvelope({ code: "IDEMPOTENCY_KEY_CONFLICT", correlationId }));
|
||||||
|
}
|
||||||
|
if (code === "cleanup_candidates_invalid") return reply.code(400).send(null);
|
||||||
|
if (code === "cleanup_uncommitted") return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId }));
|
||||||
|
return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId }));
|
||||||
|
}
|
||||||
|
|
||||||
function generationTaskResponse(task: GenerationTaskView) {
|
function generationTaskResponse(task: GenerationTaskView) {
|
||||||
return {
|
return {
|
||||||
confirmed_credit_cost: task.confirmedCreditCost,
|
confirmed_credit_cost: task.confirmedCreditCost,
|
||||||
@@ -965,6 +980,78 @@ export async function createApp(options: CreateAppOptions = {}) {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
app.get(
|
||||||
|
"/api/v1/admin/assets/static-stickers/cleanup/candidates",
|
||||||
|
{ schema: { hide: true } },
|
||||||
|
async (request, reply) => {
|
||||||
|
if (!options.registration || !options.storage) {
|
||||||
|
return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId: request.id }));
|
||||||
|
}
|
||||||
|
const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName);
|
||||||
|
const session = token ? options.registration.readAdminSession(token) : undefined;
|
||||||
|
if (!session) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id }));
|
||||||
|
try {
|
||||||
|
reply.header("Cache-Control", "private, no-store");
|
||||||
|
return options.storage.listAssetCleanupCandidates();
|
||||||
|
} catch (error) {
|
||||||
|
return assetCleanupFailure(reply, request.id, error);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
app.post(
|
||||||
|
"/api/v1/admin/assets/static-stickers/cleanup/intents",
|
||||||
|
{ schema: { hide: true } },
|
||||||
|
async (request, reply) => {
|
||||||
|
if (!options.registration || !options.storage) {
|
||||||
|
return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId: request.id }));
|
||||||
|
}
|
||||||
|
const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName);
|
||||||
|
const csrfToken = headerValue(request.headers["x-csrf-token"]);
|
||||||
|
const idempotencyKey = headerValue(request.headers["idempotency-key"]);
|
||||||
|
if (!token || !csrfToken) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id }));
|
||||||
|
if (!idempotencyKey || !/^[A-Za-z0-9_-]{32,200}$/.test(idempotencyKey) || !/^[A-Za-z0-9_-]{43,64}$/.test(csrfToken)) return reply.code(400).send(null);
|
||||||
|
try {
|
||||||
|
const admin = options.registration.authorizeAdminMutation({ csrfToken, sessionToken: token });
|
||||||
|
const body = request.body as { file_ids?: string[]; snapshot_version?: string } | undefined;
|
||||||
|
return reply.code(201).send(options.storage.createAssetCleanupIntent({
|
||||||
|
actorId: admin.userId,
|
||||||
|
fileIds: body?.file_ids ?? [],
|
||||||
|
idempotencyKey,
|
||||||
|
snapshotVersion: body?.snapshot_version ?? "",
|
||||||
|
}));
|
||||||
|
} catch (error) {
|
||||||
|
return assetCleanupFailure(reply, request.id, error);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
app.post(
|
||||||
|
"/api/v1/admin/assets/static-stickers/cleanup/intents/:requestId/confirm",
|
||||||
|
{ schema: { hide: true } },
|
||||||
|
async (request, reply) => {
|
||||||
|
if (!options.registration || !options.storage) {
|
||||||
|
return reply.code(503).send(createErrorEnvelope({ code: "AUTH_SERVICE_UNAVAILABLE", correlationId: request.id }));
|
||||||
|
}
|
||||||
|
const token = cookieValue(headerValue(request.headers.cookie), adminSessionCookieName);
|
||||||
|
const csrfToken = headerValue(request.headers["x-csrf-token"]);
|
||||||
|
if (!token || !csrfToken) return reply.code(401).send(createErrorEnvelope({ code: "AUTH_SESSION_INVALID", correlationId: request.id }));
|
||||||
|
if (!/^[A-Za-z0-9_-]{43,64}$/.test(csrfToken)) return reply.code(400).send(null);
|
||||||
|
try {
|
||||||
|
const admin = options.registration.authorizeAdminMutation({ csrfToken, sessionToken: token });
|
||||||
|
const body = request.body as { confirmation_token?: string } | undefined;
|
||||||
|
const requestId = (request.params as { requestId: string }).requestId;
|
||||||
|
return reply.send(options.storage.confirmAssetCleanupIntent({
|
||||||
|
actorId: admin.userId,
|
||||||
|
confirmationToken: body?.confirmation_token ?? "",
|
||||||
|
requestId,
|
||||||
|
}));
|
||||||
|
} catch (error) {
|
||||||
|
return assetCleanupFailure(reply, request.id, error);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
app.get(
|
app.get(
|
||||||
"/api/v1/assets/public/:resourceVersion/manifest",
|
"/api/v1/assets/public/:resourceVersion/manifest",
|
||||||
{ schema: { hide: true } },
|
{ schema: { hide: true } },
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ const app = await createApp({
|
|||||||
...(registration ? { registration } : {}),
|
...(registration ? { registration } : {}),
|
||||||
...(recentAssets ? { recentAssets } : {}),
|
...(recentAssets ? { recentAssets } : {}),
|
||||||
...(stickers ? { stickers } : {}),
|
...(stickers ? { stickers } : {}),
|
||||||
|
...(storage ? { storage } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
await app.listen({
|
await app.listen({
|
||||||
|
|||||||
@@ -75,6 +75,31 @@ interface CleanupQueueRow {
|
|||||||
relative_path: string;
|
relative_path: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface AssetCleanupCandidateView {
|
||||||
|
byte_size: number;
|
||||||
|
file_id: string;
|
||||||
|
file_kind: "original" | "thumbnail";
|
||||||
|
hash_prefix: string;
|
||||||
|
reference_count: 0;
|
||||||
|
resource_version: string;
|
||||||
|
stable_id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AssetCleanupCandidatesView {
|
||||||
|
candidate_snapshot_version: string;
|
||||||
|
expires_at: string;
|
||||||
|
items: AssetCleanupCandidateView[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AssetCleanupIntentView {
|
||||||
|
confirmation_token: string;
|
||||||
|
expires_at: string;
|
||||||
|
file_count: number;
|
||||||
|
request_id: string;
|
||||||
|
status: "pending_confirmation" | "denied" | "queued" | "completed";
|
||||||
|
total_bytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
export class StorageCapacityError extends Error {
|
export class StorageCapacityError extends Error {
|
||||||
readonly code = "STORAGE_CAPACITY_EXCEEDED";
|
readonly code = "STORAGE_CAPACITY_EXCEEDED";
|
||||||
readonly httpStatus = 507;
|
readonly httpStatus = 507;
|
||||||
@@ -104,6 +129,10 @@ function auditExpiry(occurredAt: number) {
|
|||||||
return occurredAt + auditRetentionMilliseconds;
|
return occurredAt + auditRetentionMilliseconds;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function digest(value: string) {
|
||||||
|
return createHash("sha256").update(value, "utf8").digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
function validatePositiveBytes(value: number, name: string) {
|
function validatePositiveBytes(value: number, name: string) {
|
||||||
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name}_invalid`);
|
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name}_invalid`);
|
||||||
}
|
}
|
||||||
@@ -255,6 +284,28 @@ export class ManagedStorage {
|
|||||||
FOREIGN KEY (request_id) REFERENCES asset_cleanup_requests(request_id),
|
FOREIGN KEY (request_id) REFERENCES asset_cleanup_requests(request_id),
|
||||||
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
|
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
|
||||||
);
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS asset_cleanup_candidate_snapshots (
|
||||||
|
snapshot_version TEXT PRIMARY KEY,
|
||||||
|
items_json TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
expires_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS sticker_managed_file_history (
|
||||||
|
managed_file_id TEXT NOT NULL,
|
||||||
|
stable_id TEXT NOT NULL,
|
||||||
|
resource_version TEXT NOT NULL,
|
||||||
|
file_kind TEXT NOT NULL CHECK (file_kind IN ('original', 'thumbnail')),
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (managed_file_id, file_kind),
|
||||||
|
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS project_sticker_asset_refs (
|
||||||
|
reference_id TEXT PRIMARY KEY,
|
||||||
|
project_id TEXT NOT NULL,
|
||||||
|
stable_id TEXT NOT NULL,
|
||||||
|
resource_version TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
CREATE TABLE IF NOT EXISTS admin_operation_logs (
|
CREATE TABLE IF NOT EXISTS admin_operation_logs (
|
||||||
log_id TEXT PRIMARY KEY,
|
log_id TEXT PRIMARY KEY,
|
||||||
actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')),
|
actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')),
|
||||||
@@ -277,6 +328,49 @@ export class ManagedStorage {
|
|||||||
if (!managedFileColumns.some((column) => column.name === "owner_ref")) {
|
if (!managedFileColumns.some((column) => column.name === "owner_ref")) {
|
||||||
this.database.exec("ALTER TABLE managed_files ADD COLUMN owner_ref TEXT");
|
this.database.exec("ALTER TABLE managed_files ADD COLUMN owner_ref TEXT");
|
||||||
}
|
}
|
||||||
|
if (!managedFileColumns.some((column) => column.name === "cleanup_status")) {
|
||||||
|
this.database.exec("ALTER TABLE managed_files ADD COLUMN cleanup_status TEXT");
|
||||||
|
}
|
||||||
|
const cleanupRequestColumns = this.database.prepare("PRAGMA table_info(asset_cleanup_requests)").all() as Array<{ name: string }>;
|
||||||
|
const cleanupRequestAdditions: Array<[string, string]> = [
|
||||||
|
["created_by", "TEXT"],
|
||||||
|
["confirmed_by", "TEXT"],
|
||||||
|
["snapshot_version", "TEXT"],
|
||||||
|
["expires_at", "INTEGER"],
|
||||||
|
["confirmation_token_digest", "TEXT"],
|
||||||
|
["idempotency_key_digest", "TEXT"],
|
||||||
|
["request_hash", "TEXT"],
|
||||||
|
["file_count", "INTEGER"],
|
||||||
|
["total_bytes", "INTEGER"],
|
||||||
|
["denied_reason", "TEXT"],
|
||||||
|
];
|
||||||
|
for (const [column, type] of cleanupRequestAdditions) {
|
||||||
|
if (!cleanupRequestColumns.some((item) => item.name === column)) {
|
||||||
|
this.database.exec(`ALTER TABLE asset_cleanup_requests ADD COLUMN ${column} ${type}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const cleanupItemColumns = this.database.prepare("PRAGMA table_info(asset_cleanup_request_items)").all() as Array<{ name: string }>;
|
||||||
|
const cleanupItemAdditions: Array<[string, string]> = [
|
||||||
|
["stable_id", "TEXT"],
|
||||||
|
["resource_version", "TEXT"],
|
||||||
|
["file_kind", "TEXT"],
|
||||||
|
["byte_size", "INTEGER"],
|
||||||
|
["sha256_prefix", "TEXT"],
|
||||||
|
];
|
||||||
|
for (const [column, type] of cleanupItemAdditions) {
|
||||||
|
if (!cleanupItemColumns.some((item) => item.name === column)) {
|
||||||
|
this.database.exec(`ALTER TABLE asset_cleanup_request_items ADD COLUMN ${column} ${type}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.database.exec(`
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS asset_cleanup_requests_actor_idempotency
|
||||||
|
ON asset_cleanup_requests (created_by, idempotency_key_digest)
|
||||||
|
WHERE created_by IS NOT NULL AND idempotency_key_digest IS NOT NULL;
|
||||||
|
CREATE INDEX IF NOT EXISTS sticker_managed_file_history_lookup
|
||||||
|
ON sticker_managed_file_history (stable_id, resource_version, file_kind);
|
||||||
|
CREATE INDEX IF NOT EXISTS asset_cleanup_candidate_snapshots_expiry
|
||||||
|
ON asset_cleanup_candidate_snapshots (expires_at);
|
||||||
|
`);
|
||||||
ensureAdminOperationAuditSchema(this.database, Date.now());
|
ensureAdminOperationAuditSchema(this.database, Date.now());
|
||||||
const initial = classifyCapacity(0, 0);
|
const initial = classifyCapacity(0, 0);
|
||||||
this.database.prepare(`
|
this.database.prepare(`
|
||||||
@@ -299,6 +393,106 @@ export class ManagedStorage {
|
|||||||
return withReservations;
|
return withReservations;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private readAssetCleanupCandidates(): AssetCleanupCandidateView[] {
|
||||||
|
const releaseReferenceClause = this.tableExists("sticker_release_items") ? `
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM sticker_release_items release_items
|
||||||
|
WHERE release_items.original_file_id = mf.file_id OR release_items.thumbnail_file_id = mf.file_id
|
||||||
|
)` : "";
|
||||||
|
return this.database.prepare(`
|
||||||
|
SELECT
|
||||||
|
mf.file_id,
|
||||||
|
mf.byte_size,
|
||||||
|
history.stable_id,
|
||||||
|
history.resource_version,
|
||||||
|
history.file_kind,
|
||||||
|
substr(mf.sha256, 1, 12) AS hash_prefix,
|
||||||
|
0 AS reference_count
|
||||||
|
FROM sticker_managed_file_history history
|
||||||
|
JOIN managed_files mf ON mf.file_id = history.managed_file_id
|
||||||
|
WHERE mf.status = 'committed'
|
||||||
|
AND mf.cleanup_status IS NULL
|
||||||
|
AND mf.file_kind IN ('sticker_original', 'sticker_thumbnail')
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM project_asset_refs refs WHERE refs.managed_file_id = mf.file_id
|
||||||
|
)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM project_sticker_asset_refs project_refs
|
||||||
|
WHERE project_refs.stable_id = history.stable_id
|
||||||
|
AND project_refs.resource_version = history.resource_version
|
||||||
|
)
|
||||||
|
${releaseReferenceClause}
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM asset_cleanup_request_items request_items
|
||||||
|
JOIN asset_cleanup_requests requests ON requests.request_id = request_items.request_id
|
||||||
|
WHERE request_items.managed_file_id = mf.file_id
|
||||||
|
AND requests.status IN ('pending_confirmation', 'queued')
|
||||||
|
)
|
||||||
|
ORDER BY history.stable_id, history.resource_version, history.file_kind, mf.file_id
|
||||||
|
`).all() as AssetCleanupCandidateView[];
|
||||||
|
}
|
||||||
|
|
||||||
|
private assertActiveAdmin(actorId: string) {
|
||||||
|
const admin = this.database.prepare(`
|
||||||
|
SELECT 1 AS allowed FROM users u
|
||||||
|
JOIN admin_access access ON access.user_id = u.user_id
|
||||||
|
WHERE u.user_id = ? AND u.role = 'super_admin' AND u.status = 'active' AND access.allowed = 1
|
||||||
|
`).get(actorId);
|
||||||
|
if (!admin) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
}
|
||||||
|
|
||||||
|
private assetReferenceCount(fileId: string, requestId: string) {
|
||||||
|
const projectOrRelease = (this.database.prepare(`
|
||||||
|
SELECT COUNT(*) AS count FROM project_asset_refs WHERE managed_file_id = ?
|
||||||
|
`).get(fileId) as { count: number }).count;
|
||||||
|
const releaseItems = this.tableExists("sticker_release_items")
|
||||||
|
? (this.database.prepare(`
|
||||||
|
SELECT COUNT(*) AS count FROM sticker_release_items
|
||||||
|
WHERE original_file_id = ? OR thumbnail_file_id = ?
|
||||||
|
`).get(fileId, fileId) as { count: number }).count
|
||||||
|
: 0;
|
||||||
|
const projectStickerRefs = (this.database.prepare(`
|
||||||
|
SELECT COUNT(*) AS count
|
||||||
|
FROM sticker_managed_file_history history
|
||||||
|
JOIN project_sticker_asset_refs refs
|
||||||
|
ON refs.stable_id = history.stable_id AND refs.resource_version = history.resource_version
|
||||||
|
WHERE history.managed_file_id = ?
|
||||||
|
`).get(fileId) as { count: number }).count;
|
||||||
|
const otherCleanup = (this.database.prepare(`
|
||||||
|
SELECT COUNT(*) AS count FROM asset_cleanup_request_items items
|
||||||
|
JOIN asset_cleanup_requests requests ON requests.request_id = items.request_id
|
||||||
|
WHERE items.managed_file_id = ? AND items.request_id <> ?
|
||||||
|
AND requests.status IN ('pending_confirmation', 'queued')
|
||||||
|
`).get(fileId, requestId) as { count: number }).count;
|
||||||
|
return projectOrRelease + releaseItems + projectStickerRefs + otherCleanup;
|
||||||
|
}
|
||||||
|
|
||||||
|
private cleanupConfirmationToken(requestId: string, actorId: string, keyDigest: string) {
|
||||||
|
return digest(`Dada/P0A/asset-cleanup-confirm/v1:${requestId}:${actorId}:${keyDigest}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private tableExists(name: string) {
|
||||||
|
return Boolean(this.database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
|
||||||
|
}
|
||||||
|
|
||||||
|
private insertCleanupAudit(input: {
|
||||||
|
actorRef: string;
|
||||||
|
afterSummary: Record<string, unknown>;
|
||||||
|
operationType: string;
|
||||||
|
requestId: string;
|
||||||
|
result: "failed" | "succeeded";
|
||||||
|
}, occurredAt: number) {
|
||||||
|
this.database.prepare(`
|
||||||
|
INSERT INTO admin_operation_logs (
|
||||||
|
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
|
||||||
|
result, before_summary, after_summary, occurred_at, expires_at
|
||||||
|
) VALUES (?, 'super_admin', ?, ?, 'asset_cleanup_request', ?, ?, NULL, ?, ?, ?)
|
||||||
|
`).run(
|
||||||
|
randomUUID(), input.actorRef, input.operationType, input.requestId, input.result,
|
||||||
|
serializeAuditSummary(input.afterSummary), occurredAt, auditExpiry(occurredAt),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
private activeReservationBytes(excludingOperationId?: string) {
|
private activeReservationBytes(excludingOperationId?: string) {
|
||||||
const row = this.database.prepare(`
|
const row = this.database.prepare(`
|
||||||
SELECT COALESCE(SUM(projected_bytes), 0) AS bytes
|
SELECT COALESCE(SUM(projected_bytes), 0) AS bytes
|
||||||
@@ -692,6 +886,203 @@ export class ManagedStorage {
|
|||||||
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(fileId);
|
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(fileId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
listAssetCleanupCandidates(): AssetCleanupCandidatesView {
|
||||||
|
const createdAt = Date.now();
|
||||||
|
const expiresAt = createdAt + 5 * 60 * 1_000;
|
||||||
|
const items = this.readAssetCleanupCandidates();
|
||||||
|
const snapshotVersion = digest(JSON.stringify({
|
||||||
|
created_at: createdAt,
|
||||||
|
nonce: randomUUID(),
|
||||||
|
items: items.map((item) => ({ byte_size: item.byte_size, file_id: item.file_id, hash_prefix: item.hash_prefix })),
|
||||||
|
}));
|
||||||
|
this.database.prepare("DELETE FROM asset_cleanup_candidate_snapshots WHERE expires_at <= ?").run(createdAt);
|
||||||
|
this.database.prepare(`
|
||||||
|
INSERT INTO asset_cleanup_candidate_snapshots (
|
||||||
|
snapshot_version, items_json, created_at, expires_at
|
||||||
|
) VALUES (?, ?, ?, ?)
|
||||||
|
`).run(snapshotVersion, JSON.stringify(items), createdAt, expiresAt);
|
||||||
|
return {
|
||||||
|
candidate_snapshot_version: snapshotVersion,
|
||||||
|
expires_at: new Date(expiresAt).toISOString(),
|
||||||
|
items,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
createAssetCleanupIntent(input: {
|
||||||
|
actorId: string;
|
||||||
|
fileIds: string[];
|
||||||
|
idempotencyKey: string;
|
||||||
|
snapshotVersion: string;
|
||||||
|
}): AssetCleanupIntentView {
|
||||||
|
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
|
||||||
|
const fileIds = [...new Set(input.fileIds)].sort();
|
||||||
|
if (!uuidPattern.test(input.actorId) || fileIds.length === 0 || fileIds.length !== input.fileIds.length
|
||||||
|
|| fileIds.length > 100 || fileIds.some((fileId) => !uuidPattern.test(fileId))
|
||||||
|
|| !/^[A-Za-z0-9_-]{32,200}$/.test(input.idempotencyKey)
|
||||||
|
|| !/^[0-9a-f]{64}$/.test(input.snapshotVersion)) {
|
||||||
|
throw new Error("cleanup_candidates_invalid");
|
||||||
|
}
|
||||||
|
const keyDigest = digest(input.idempotencyKey);
|
||||||
|
const requestHash = digest(JSON.stringify({ file_ids: fileIds, snapshot_version: input.snapshotVersion }));
|
||||||
|
const existing = this.database.prepare(`
|
||||||
|
SELECT request_id, request_hash, expires_at, file_count, total_bytes, status
|
||||||
|
FROM asset_cleanup_requests
|
||||||
|
WHERE created_by = ? AND idempotency_key_digest = ?
|
||||||
|
`).get(input.actorId, keyDigest) as {
|
||||||
|
expires_at: number; file_count: number; request_hash: string; request_id: string; status: AssetCleanupIntentView["status"]; total_bytes: number;
|
||||||
|
} | undefined;
|
||||||
|
if (existing) {
|
||||||
|
if (existing.request_hash !== requestHash) throw new Error("IDEMPOTENCY_KEY_CONFLICT");
|
||||||
|
return {
|
||||||
|
confirmation_token: this.cleanupConfirmationToken(existing.request_id, input.actorId, keyDigest),
|
||||||
|
expires_at: new Date(existing.expires_at).toISOString(),
|
||||||
|
file_count: existing.file_count,
|
||||||
|
request_id: existing.request_id,
|
||||||
|
status: existing.status,
|
||||||
|
total_bytes: existing.total_bytes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const requestId = randomUUID();
|
||||||
|
const confirmationToken = this.cleanupConfirmationToken(requestId, input.actorId, keyDigest);
|
||||||
|
const createdAt = Date.now();
|
||||||
|
let view!: AssetCleanupIntentView;
|
||||||
|
const transaction = this.database.transaction(() => {
|
||||||
|
this.assertActiveAdmin(input.actorId);
|
||||||
|
const snapshot = this.database.prepare(`
|
||||||
|
SELECT items_json, expires_at FROM asset_cleanup_candidate_snapshots
|
||||||
|
WHERE snapshot_version = ?
|
||||||
|
`).get(input.snapshotVersion) as { expires_at: number; items_json: string } | undefined;
|
||||||
|
if (!snapshot || snapshot.expires_at <= createdAt) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
const snapshotItems = JSON.parse(snapshot.items_json) as AssetCleanupCandidateView[];
|
||||||
|
const byId = new Map(snapshotItems.map((item) => [item.file_id, item]));
|
||||||
|
const selected = fileIds.map((fileId) => byId.get(fileId));
|
||||||
|
if (selected.some((item) => !item)) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
const current = new Map(this.readAssetCleanupCandidates().map((item) => [item.file_id, item]));
|
||||||
|
if (fileIds.some((fileId) => !current.has(fileId))) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
const safeItems = selected as AssetCleanupCandidateView[];
|
||||||
|
const totalBytes = safeItems.reduce((sum, item) => sum + item.byte_size, 0);
|
||||||
|
this.database.prepare(`
|
||||||
|
INSERT INTO asset_cleanup_requests (
|
||||||
|
request_id, status, created_at, confirmed_at, created_by, confirmed_by,
|
||||||
|
snapshot_version, expires_at, confirmation_token_digest,
|
||||||
|
idempotency_key_digest, request_hash, file_count, total_bytes, denied_reason
|
||||||
|
) VALUES (?, 'pending_confirmation', ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, NULL)
|
||||||
|
`).run(
|
||||||
|
requestId, new Date(createdAt).toISOString(), input.actorId, input.snapshotVersion,
|
||||||
|
snapshot.expires_at, digest(confirmationToken), keyDigest, requestHash, safeItems.length, totalBytes,
|
||||||
|
);
|
||||||
|
const insert = this.database.prepare(`
|
||||||
|
INSERT INTO asset_cleanup_request_items (
|
||||||
|
request_id, managed_file_id, stable_id, resource_version, file_kind, byte_size, sha256_prefix
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`);
|
||||||
|
for (const item of safeItems) {
|
||||||
|
insert.run(requestId, item.file_id, item.stable_id, item.resource_version, item.file_kind, item.byte_size, item.hash_prefix);
|
||||||
|
}
|
||||||
|
this.insertCleanupAudit({
|
||||||
|
actorRef: input.actorId,
|
||||||
|
afterSummary: { file_count: safeItems.length, snapshot_version: input.snapshotVersion, total_bytes: totalBytes },
|
||||||
|
operationType: "asset_cleanup_requested",
|
||||||
|
requestId,
|
||||||
|
result: "succeeded",
|
||||||
|
}, createdAt);
|
||||||
|
view = {
|
||||||
|
confirmation_token: confirmationToken,
|
||||||
|
expires_at: new Date(snapshot.expires_at).toISOString(),
|
||||||
|
file_count: safeItems.length,
|
||||||
|
request_id: requestId,
|
||||||
|
status: "pending_confirmation",
|
||||||
|
total_bytes: totalBytes,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
transaction.immediate();
|
||||||
|
return view;
|
||||||
|
}
|
||||||
|
|
||||||
|
confirmAssetCleanupIntent(input: { actorId: string; confirmationToken: string; requestId: string }) {
|
||||||
|
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
|
||||||
|
if (!uuidPattern.test(input.actorId) || !uuidPattern.test(input.requestId) || !/^[0-9a-f]{64}$/.test(input.confirmationToken)) {
|
||||||
|
throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
}
|
||||||
|
const confirmedAt = Date.now();
|
||||||
|
const outcome = this.database.transaction(() => {
|
||||||
|
this.assertActiveAdmin(input.actorId);
|
||||||
|
const request = this.database.prepare(`
|
||||||
|
SELECT status, created_by, expires_at, confirmation_token_digest, file_count, total_bytes
|
||||||
|
FROM asset_cleanup_requests WHERE request_id = ?
|
||||||
|
`).get(input.requestId) as {
|
||||||
|
confirmation_token_digest: string | null; created_by: string | null; expires_at: number | null;
|
||||||
|
file_count: number | null; status: string; total_bytes: number | null;
|
||||||
|
} | undefined;
|
||||||
|
if (!request || request.status !== "pending_confirmation" || request.created_by !== input.actorId
|
||||||
|
|| !request.expires_at || request.expires_at <= confirmedAt
|
||||||
|
|| request.confirmation_token_digest !== digest(input.confirmationToken)) {
|
||||||
|
throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
}
|
||||||
|
const files = this.database.prepare(`
|
||||||
|
SELECT mf.file_id, mf.file_kind, mf.relative_path, mf.byte_size, mf.status
|
||||||
|
FROM asset_cleanup_request_items items
|
||||||
|
JOIN managed_files mf ON mf.file_id = items.managed_file_id
|
||||||
|
WHERE items.request_id = ? ORDER BY mf.file_id
|
||||||
|
`).all(input.requestId) as ManagedFileRow[];
|
||||||
|
if (files.length !== request.file_count) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
const conflicted = files.some((file) => file.status !== "committed"
|
||||||
|
|| !new Set(["sticker_original", "sticker_thumbnail"]).has(file.file_kind)
|
||||||
|
|| this.assetReferenceCount(file.file_id, input.requestId) > 0);
|
||||||
|
if (conflicted) {
|
||||||
|
this.database.prepare(`
|
||||||
|
UPDATE asset_cleanup_requests
|
||||||
|
SET status = 'denied', confirmed_at = ?, confirmed_by = ?, denied_reason = 'reference_conflict'
|
||||||
|
WHERE request_id = ?
|
||||||
|
`).run(new Date(confirmedAt).toISOString(), input.actorId, input.requestId);
|
||||||
|
this.insertCleanupAudit({
|
||||||
|
actorRef: input.actorId,
|
||||||
|
afterSummary: { file_count: files.length, reason: "reference_conflict", status: "denied" },
|
||||||
|
operationType: "asset_cleanup_reference_denied",
|
||||||
|
requestId: input.requestId,
|
||||||
|
result: "failed",
|
||||||
|
}, confirmedAt);
|
||||||
|
return { conflict: true as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
this.insertCleanupAudit({
|
||||||
|
actorRef: input.actorId,
|
||||||
|
afterSummary: { file_count: files.length, status: "validated" },
|
||||||
|
operationType: "asset_cleanup_validated",
|
||||||
|
requestId: input.requestId,
|
||||||
|
result: "succeeded",
|
||||||
|
}, confirmedAt);
|
||||||
|
for (const file of files) {
|
||||||
|
this.database.prepare(`
|
||||||
|
UPDATE managed_files SET status = 'purged', purged_at = ?, cleanup_status = 'pending_delete'
|
||||||
|
WHERE file_id = ? AND status = 'committed'
|
||||||
|
`).run(new Date(confirmedAt).toISOString(), file.file_id);
|
||||||
|
this.database.prepare(`
|
||||||
|
INSERT INTO file_cleanup_queue (
|
||||||
|
cleanup_id, managed_file_id, relative_path, byte_size, counts_toward_managed,
|
||||||
|
reason, status, created_at, completed_at, last_error
|
||||||
|
) VALUES (?, ?, ?, ?, 1, 'purge', 'pending', ?, NULL, NULL)
|
||||||
|
`).run(randomUUID(), file.file_id, file.relative_path, file.byte_size, new Date(confirmedAt).toISOString());
|
||||||
|
}
|
||||||
|
this.database.prepare(`
|
||||||
|
UPDATE asset_cleanup_requests
|
||||||
|
SET status = 'queued', confirmed_at = ?, confirmed_by = ?
|
||||||
|
WHERE request_id = ?
|
||||||
|
`).run(new Date(confirmedAt).toISOString(), input.actorId, input.requestId);
|
||||||
|
this.insertCleanupAudit({
|
||||||
|
actorRef: input.actorId,
|
||||||
|
afterSummary: { file_count: files.length, status: "queued", total_bytes: request.total_bytes ?? 0 },
|
||||||
|
operationType: "asset_cleanup_scheduled",
|
||||||
|
requestId: input.requestId,
|
||||||
|
result: "succeeded",
|
||||||
|
}, confirmedAt + 1);
|
||||||
|
return { conflict: false as const, file_count: files.length, request_id: input.requestId, status: "queued" as const };
|
||||||
|
}).immediate();
|
||||||
|
if (outcome.conflict) throw new Error("ASSET_HISTORY_REFERENCE_CONFLICT");
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
createCleanupIntent(fileIds: string[]) {
|
createCleanupIntent(fileIds: string[]) {
|
||||||
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
|
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
|
||||||
if (fileIds.length === 0 || new Set(fileIds).size !== fileIds.length) throw new Error("cleanup_candidates_invalid");
|
if (fileIds.length === 0 || new Set(fileIds).size !== fileIds.length) throw new Error("cleanup_candidates_invalid");
|
||||||
@@ -777,6 +1168,7 @@ export class ManagedStorage {
|
|||||||
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(row.managed_file_id);
|
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(row.managed_file_id);
|
||||||
const requests = this.database.prepare("SELECT request_id FROM asset_cleanup_request_items WHERE managed_file_id = ?").all(row.managed_file_id) as Array<{ request_id: string }>;
|
const requests = this.database.prepare("SELECT request_id FROM asset_cleanup_request_items WHERE managed_file_id = ?").all(row.managed_file_id) as Array<{ request_id: string }>;
|
||||||
this.database.prepare("DELETE FROM asset_cleanup_request_items WHERE managed_file_id = ?").run(row.managed_file_id);
|
this.database.prepare("DELETE FROM asset_cleanup_request_items WHERE managed_file_id = ?").run(row.managed_file_id);
|
||||||
|
this.database.prepare("DELETE FROM sticker_managed_file_history WHERE managed_file_id = ?").run(row.managed_file_id);
|
||||||
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
|
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
|
||||||
for (const request of requests) {
|
for (const request of requests) {
|
||||||
const pendingItems = this.database.prepare("SELECT COUNT(*) AS count FROM asset_cleanup_request_items WHERE request_id = ?").get(request.request_id) as { count: number };
|
const pendingItems = this.database.prepare("SELECT COUNT(*) AS count FROM asset_cleanup_request_items WHERE request_id = ?").get(request.request_id) as { count: number };
|
||||||
|
|||||||
@@ -366,6 +366,7 @@ export class ProjectService {
|
|||||||
throw new ProjectError("project_state_conflict", latest);
|
throw new ProjectError("project_state_conflict", latest);
|
||||||
}
|
}
|
||||||
this.insertProjectState({ canvasState: canvas, name, projectId: input.projectId, stateVersion: nextVersion }, now);
|
this.insertProjectState({ canvasState: canvas, name, projectId: input.projectId, stateVersion: nextVersion }, now);
|
||||||
|
this.rebuildProjectStickerReferences(input.projectId, canvas, now);
|
||||||
this.database.prepare(`
|
this.database.prepare(`
|
||||||
INSERT INTO project_state_idempotency (
|
INSERT INTO project_state_idempotency (
|
||||||
owner_id, project_id, idempotency_key, request_hash, response_state_version, created_at
|
owner_id, project_id, idempotency_key, request_hash, response_state_version, created_at
|
||||||
@@ -377,6 +378,25 @@ export class ProjectService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private rebuildProjectStickerReferences(projectId: string, canvas: CanvasState, now: number) {
|
||||||
|
if (!this.tableExists("project_sticker_asset_refs")) return;
|
||||||
|
this.database.prepare("DELETE FROM project_sticker_asset_refs WHERE project_id = ?").run(projectId);
|
||||||
|
const insert = this.database.prepare(`
|
||||||
|
INSERT INTO project_sticker_asset_refs (reference_id, project_id, stable_id, resource_version, created_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?)
|
||||||
|
`);
|
||||||
|
for (const element of canvas.elements) {
|
||||||
|
if (element.type !== "static_sticker") continue;
|
||||||
|
insert.run(
|
||||||
|
`project:${projectId}:sticker:${element.element_id}`,
|
||||||
|
projectId,
|
||||||
|
element.template_or_asset_id,
|
||||||
|
element.resource_version,
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
trashFailedEmpty(ownerId: string, projectIds: string[]) {
|
trashFailedEmpty(ownerId: string, projectIds: string[]) {
|
||||||
const uniqueIds = [...new Set(projectIds)];
|
const uniqueIds = [...new Set(projectIds)];
|
||||||
if (uniqueIds.length === 0 || uniqueIds.length > projectLimit) throw new ProjectError("generation_state_invalid");
|
if (uniqueIds.length === 0 || uniqueIds.length > projectLimit) throw new ProjectError("generation_state_invalid");
|
||||||
@@ -774,6 +794,16 @@ export class ProjectService {
|
|||||||
FOREIGN KEY (project_id) REFERENCES projects(project_id) ON DELETE CASCADE
|
FOREIGN KEY (project_id) REFERENCES projects(project_id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS project_resource_files_managed ON project_resource_files(managed_file_id, project_id);
|
CREATE INDEX IF NOT EXISTS project_resource_files_managed ON project_resource_files(managed_file_id, project_id);
|
||||||
|
CREATE TABLE IF NOT EXISTS project_sticker_asset_refs (
|
||||||
|
reference_id TEXT PRIMARY KEY,
|
||||||
|
project_id TEXT NOT NULL,
|
||||||
|
stable_id TEXT NOT NULL,
|
||||||
|
resource_version TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
FOREIGN KEY (project_id) REFERENCES projects(project_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS project_sticker_asset_refs_lookup
|
||||||
|
ON project_sticker_asset_refs (stable_id, resource_version);
|
||||||
CREATE TABLE IF NOT EXISTS latest_exports (
|
CREATE TABLE IF NOT EXISTS latest_exports (
|
||||||
project_id TEXT NOT NULL,
|
project_id TEXT NOT NULL,
|
||||||
format TEXT NOT NULL CHECK (format IN ('jpg', 'png')),
|
format TEXT NOT NULL CHECK (format IN ('jpg', 'png')),
|
||||||
|
|||||||
@@ -328,6 +328,14 @@ export class StickerReleaseService {
|
|||||||
input.width, input.height, input.expectedMimeType, input.original.sha256, input.original.fileId, input.original.bytes,
|
input.width, input.height, input.expectedMimeType, input.original.sha256, input.original.fileId, input.original.bytes,
|
||||||
input.thumbnail.fileId, input.thumbnail.relativePath, input.thumbnail.sha256, input.thumbnail.bytes, input.enabled ? 1 : 0,
|
input.thumbnail.fileId, input.thumbnail.relativePath, input.thumbnail.sha256, input.thumbnail.bytes, input.enabled ? 1 : 0,
|
||||||
);
|
);
|
||||||
|
this.database.prepare(`
|
||||||
|
INSERT OR IGNORE INTO sticker_managed_file_history (
|
||||||
|
managed_file_id, stable_id, resource_version, file_kind, created_at
|
||||||
|
) VALUES (?, ?, ?, 'original', ?), (?, ?, ?, 'thumbnail', ?)
|
||||||
|
`).run(
|
||||||
|
input.original.fileId, input.stableId, releaseVersion, this.clock(),
|
||||||
|
input.thumbnail.fileId, input.stableId, releaseVersion, this.clock(),
|
||||||
|
);
|
||||||
this.consumeStagedStorage([input.original, input.thumbnail]);
|
this.consumeStagedStorage([input.original, input.thumbnail]);
|
||||||
this.database.prepare(`
|
this.database.prepare(`
|
||||||
INSERT INTO sticker_upload_receipts (actor_id, idempotency_key_digest, request_hash, release_version, stable_id, created_at)
|
INSERT INTO sticker_upload_receipts (actor_id, idempotency_key_digest, request_hash, release_version, stable_id, created_at)
|
||||||
@@ -510,6 +518,15 @@ export class StickerReleaseService {
|
|||||||
created_at TEXT NOT NULL,
|
created_at TEXT NOT NULL,
|
||||||
PRIMARY KEY (actor_id, idempotency_key_digest)
|
PRIMARY KEY (actor_id, idempotency_key_digest)
|
||||||
);
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS sticker_managed_file_history (
|
||||||
|
managed_file_id TEXT NOT NULL,
|
||||||
|
stable_id TEXT NOT NULL,
|
||||||
|
resource_version TEXT NOT NULL,
|
||||||
|
file_kind TEXT NOT NULL CHECK (file_kind IN ('original', 'thumbnail')),
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (managed_file_id, file_kind),
|
||||||
|
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
|
||||||
|
);
|
||||||
CREATE TRIGGER IF NOT EXISTS sticker_releases_no_update
|
CREATE TRIGGER IF NOT EXISTS sticker_releases_no_update
|
||||||
BEFORE UPDATE ON sticker_releases BEGIN SELECT RAISE(ABORT, 'sticker_releases_immutable'); END;
|
BEFORE UPDATE ON sticker_releases BEGIN SELECT RAISE(ABORT, 'sticker_releases_immutable'); END;
|
||||||
CREATE TRIGGER IF NOT EXISTS sticker_releases_no_delete
|
CREATE TRIGGER IF NOT EXISTS sticker_releases_no_delete
|
||||||
@@ -523,5 +540,17 @@ export class StickerReleaseService {
|
|||||||
WHEN EXISTS (SELECT 1 FROM sticker_releases WHERE release_version = OLD.release_version)
|
WHEN EXISTS (SELECT 1 FROM sticker_releases WHERE release_version = OLD.release_version)
|
||||||
BEGIN SELECT RAISE(ABORT, 'sticker_release_items_immutable'); END;
|
BEGIN SELECT RAISE(ABORT, 'sticker_release_items_immutable'); END;
|
||||||
`);
|
`);
|
||||||
|
this.database.exec(`
|
||||||
|
INSERT OR IGNORE INTO sticker_managed_file_history (
|
||||||
|
managed_file_id, stable_id, resource_version, file_kind, created_at
|
||||||
|
)
|
||||||
|
SELECT original_file_id, stable_id, release_version, 'original', strftime('%s', 'now') * 1000
|
||||||
|
FROM sticker_release_items;
|
||||||
|
INSERT OR IGNORE INTO sticker_managed_file_history (
|
||||||
|
managed_file_id, stable_id, resource_version, file_kind, created_at
|
||||||
|
)
|
||||||
|
SELECT thumbnail_file_id, stable_id, release_version, 'thumbnail', strftime('%s', 'now') * 1000
|
||||||
|
FROM sticker_release_items;
|
||||||
|
`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { rmSync } from "node:fs";
|
import { existsSync, rmSync, statSync } from "node:fs";
|
||||||
import { createRequire } from "node:module";
|
import { createRequire } from "node:module";
|
||||||
import { isAbsolute, relative, resolve } from "node:path";
|
import { isAbsolute, relative, resolve } from "node:path";
|
||||||
|
|
||||||
@@ -29,6 +29,40 @@ interface FileCleanupRow {
|
|||||||
const resourceScope = JSON.stringify([
|
const resourceScope = JSON.stringify([
|
||||||
"project_state", "generation", "generated_image", "reference", "location", "latest_export",
|
"project_state", "generation", "generated_image", "reference", "location", "latest_export",
|
||||||
]);
|
]);
|
||||||
|
const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000;
|
||||||
|
const auditRefPattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$/;
|
||||||
|
const forbiddenSummaryKeys = new Set([
|
||||||
|
"absolute_path", "api_key", "body", "code_hmac", "content", "credential", "email", "image",
|
||||||
|
"image_content", "password", "path", "prompt", "secret", "session_token", "verification_code", "whitelist",
|
||||||
|
]);
|
||||||
|
const forbiddenSummaryFragments = ["content", "credential", "email", "image", "password", "path", "prompt", "secret", "token"];
|
||||||
|
|
||||||
|
function isSafeAuditRef(value: unknown) {
|
||||||
|
return typeof value === "string" && auditRefPattern.test(value) ? 1 : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSafeAuditSummaryJson(value: unknown) {
|
||||||
|
if (typeof value !== "string" || Buffer.byteLength(value, "utf8") > 2_048) return 0;
|
||||||
|
try {
|
||||||
|
const valid = (entry: unknown, depth: number): boolean => {
|
||||||
|
if (depth > 3) return false;
|
||||||
|
if (entry === null || typeof entry === "boolean") return true;
|
||||||
|
if (typeof entry === "number") return Number.isSafeInteger(entry);
|
||||||
|
if (typeof entry === "string") return /^[A-Za-z0-9_.:@-]{1,160}$/.test(entry) && !entry.includes("@");
|
||||||
|
if (Array.isArray(entry)) return entry.length <= 20 && entry.every((item) => valid(item, depth + 1));
|
||||||
|
if (!entry || typeof entry !== "object") return false;
|
||||||
|
return Object.entries(entry).length <= 32 && Object.entries(entry).every(([key, item]) => (
|
||||||
|
auditRefPattern.test(key)
|
||||||
|
&& !forbiddenSummaryKeys.has(key.toLowerCase())
|
||||||
|
&& !forbiddenSummaryFragments.some((fragment) => key.toLowerCase().includes(fragment))
|
||||||
|
&& valid(item, depth + 1)
|
||||||
|
));
|
||||||
|
};
|
||||||
|
return valid(JSON.parse(value), 0) ? 1 : 0;
|
||||||
|
} catch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function iso(timestamp: number) {
|
function iso(timestamp: number) {
|
||||||
return new Date(timestamp).toISOString();
|
return new Date(timestamp).toISOString();
|
||||||
@@ -47,6 +81,12 @@ export class ProjectPurgeCleanup {
|
|||||||
this.database.pragma("journal_mode = WAL");
|
this.database.pragma("journal_mode = WAL");
|
||||||
this.database.pragma("foreign_keys = ON");
|
this.database.pragma("foreign_keys = ON");
|
||||||
this.database.pragma("busy_timeout = 5000");
|
this.database.pragma("busy_timeout = 5000");
|
||||||
|
this.database.function("dada_audit_ref_is_safe", { deterministic: true }, isSafeAuditRef);
|
||||||
|
this.database.function("dada_audit_summary_is_safe", { deterministic: true }, isSafeAuditSummaryJson);
|
||||||
|
this.database.function("dada_allow_privacy_purge", { deterministic: false }, () => 0);
|
||||||
|
this.database.function("dada_privacy_purge_subject", { deterministic: false }, () => "");
|
||||||
|
this.database.function("dada_allow_retention_purge", { deterministic: false }, () => 0);
|
||||||
|
this.database.function("dada_retention_purge_now", { deterministic: false }, () => 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
close() {
|
close() {
|
||||||
@@ -153,12 +193,16 @@ export class ProjectPurgeCleanup {
|
|||||||
if (pending.count === 0) {
|
if (pending.count === 0) {
|
||||||
this.database.prepare("UPDATE asset_cleanup_requests SET status = 'completed' WHERE request_id = ? AND status = 'queued'")
|
this.database.prepare("UPDATE asset_cleanup_requests SET status = 'completed' WHERE request_id = ? AND status = 'queued'")
|
||||||
.run(request.request_id);
|
.run(request.request_id);
|
||||||
|
this.insertAssetCleanupAudit(request.request_id, row.byte_size, this.clock());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (this.tableExists("sticker_managed_file_history")) {
|
||||||
|
this.database.prepare("DELETE FROM sticker_managed_file_history WHERE managed_file_id = ?").run(row.managed_file_id);
|
||||||
|
}
|
||||||
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
|
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
|
||||||
if (this.tableExists("local_backend_storage_state")) this.decrementManagedCapacity(row.byte_size);
|
if (this.tableExists("local_backend_storage_state")) this.remeasureManagedCapacity();
|
||||||
}
|
}
|
||||||
this.database.prepare(`
|
this.database.prepare(`
|
||||||
UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL
|
UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL
|
||||||
@@ -238,4 +282,46 @@ export class ProjectPurgeCleanup {
|
|||||||
WHERE singleton = 1
|
WHERE singleton = 1
|
||||||
`).run(managed, notice, status, iso(this.clock()));
|
`).run(managed, notice, status, iso(this.clock()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private insertAssetCleanupAudit(requestId: string, deletedBytes: number, occurredAt: number) {
|
||||||
|
if (!this.tableExists("admin_operation_logs")) return;
|
||||||
|
this.database.prepare(`
|
||||||
|
INSERT INTO admin_operation_logs (
|
||||||
|
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
|
||||||
|
result, before_summary, after_summary, occurred_at, expires_at
|
||||||
|
) VALUES (?, 'system', 'project_purge_worker', 'asset_cleanup_physical_completed', 'asset_cleanup', ?, 'succeeded', NULL, ?, ?, ?)
|
||||||
|
`).run(
|
||||||
|
randomUUID(), requestId, JSON.stringify({ deleted_bytes: deletedBytes, status: "completed" }), occurredAt,
|
||||||
|
occurredAt + auditRetentionMilliseconds,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private remeasureManagedCapacity() {
|
||||||
|
const state = this.database.prepare(`
|
||||||
|
SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1
|
||||||
|
`).get() as { managed_content_bytes: number } | undefined;
|
||||||
|
if (!state) return;
|
||||||
|
const rows = this.database.prepare(`
|
||||||
|
SELECT relative_path FROM managed_files WHERE status = 'committed'
|
||||||
|
`).all() as Array<{ relative_path: string }>;
|
||||||
|
let managed = 0;
|
||||||
|
for (const row of rows) {
|
||||||
|
try {
|
||||||
|
const path = this.resolveManagedPath(row.relative_path);
|
||||||
|
if (existsSync(path)) managed += statSync(path).size;
|
||||||
|
} catch {
|
||||||
|
// A missing or invalid path is excluded from the measured physical total.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const reservations = this.tableExists("storage_reservations")
|
||||||
|
? (this.database.prepare("SELECT COALESCE(SUM(projected_bytes), 0) AS bytes FROM storage_reservations WHERE status = 'active'").get() as { bytes: number }).bytes
|
||||||
|
: 0;
|
||||||
|
const notice = managed < 4_294_967_296 ? "normal" : managed < 4_831_838_208 ? "warning" : "critical";
|
||||||
|
const status = managed + reservations >= 5_368_709_120 ? "full" : "active";
|
||||||
|
this.database.prepare(`
|
||||||
|
UPDATE local_backend_storage_state
|
||||||
|
SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1
|
||||||
|
WHERE singleton = 1
|
||||||
|
`).run(managed, notice, status, iso(this.clock()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -98,7 +98,11 @@ function copyRuntimeDependencies(sourceRoot, destinationRoot, rootNames) {
|
|||||||
try {
|
try {
|
||||||
entry = requireFrom.resolve(name);
|
entry = requireFrom.resolve(name);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(`Runtime dependency ${name} is unavailable from ${sourceRoot}.`, { cause: error });
|
try {
|
||||||
|
entry = requireFrom.resolve(`${name}/package`);
|
||||||
|
} catch {
|
||||||
|
throw new Error(`Runtime dependency ${name} is unavailable from ${sourceRoot}.`, { cause: error });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const root = packageRootFromEntry(entry, name);
|
const root = packageRootFromEntry(entry, name);
|
||||||
const manifest = json(join(root, "package.json"));
|
const manifest = json(join(root, "package.json"));
|
||||||
@@ -117,6 +121,14 @@ function copyRuntimeDependencies(sourceRoot, destinationRoot, rootNames) {
|
|||||||
for (const dependency of Object.keys(manifest.dependencies ?? {})) {
|
for (const dependency of Object.keys(manifest.dependencies ?? {})) {
|
||||||
copyResolved(dependency, nestedRequire, join(destination, "node_modules"), nestedAncestors);
|
copyResolved(dependency, nestedRequire, join(destination, "node_modules"), nestedAncestors);
|
||||||
}
|
}
|
||||||
|
for (const dependency of Object.keys(manifest.optionalDependencies ?? {})) {
|
||||||
|
try {
|
||||||
|
copyResolved(dependency, nestedRequire, join(destination, "node_modules"), nestedAncestors);
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.cause?.code !== "MODULE_NOT_FOUND") throw error;
|
||||||
|
debug(`skip unavailable optional dependency ${dependency}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const rootRequire = createRequire(join(sourceRoot, "package.json"));
|
const rootRequire = createRequire(join(sourceRoot, "package.json"));
|
||||||
for (const name of rootNames) copyResolved(name, rootRequire, join(destinationRoot, "node_modules"), new Set());
|
for (const name of rootNames) copyResolved(name, rootRequire, join(destinationRoot, "node_modules"), new Set());
|
||||||
@@ -306,7 +318,7 @@ export async function buildAndValidatePortablePackage({ evidenceDirectory, outpu
|
|||||||
|
|
||||||
const serverRoot = join(packageDirectory, "server");
|
const serverRoot = join(packageDirectory, "server");
|
||||||
debug("copy API application");
|
debug("copy API application");
|
||||||
const apiDependencies = copyApplication(join(repositoryRoot, "apps", "api"), join(serverRoot, "api"), ["@fastify/multipart", "@fastify/swagger", "@sinclair/typebox", "better-sqlite3", "fastify"]);
|
const apiDependencies = copyApplication(join(repositoryRoot, "apps", "api"), join(serverRoot, "api"), ["@fastify/multipart", "@fastify/swagger", "@sinclair/typebox", "better-sqlite3", "fastify", "sharp"]);
|
||||||
debug("copy Worker application");
|
debug("copy Worker application");
|
||||||
const workerDependencies = copyApplication(join(repositoryRoot, "apps", "worker"), join(serverRoot, "worker"), ["better-sqlite3"]);
|
const workerDependencies = copyApplication(join(repositoryRoot, "apps", "worker"), join(serverRoot, "worker"), ["better-sqlite3"]);
|
||||||
const sharedDestination = join(serverRoot, "api", "node_modules", "@dada", "shared-contracts");
|
const sharedDestination = join(serverRoot, "api", "node_modules", "@dada", "shared-contracts");
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { ManagedStorage } from "../../apps/api/src/managed-storage.js";
|
||||||
|
import { RegistrationService } from "../../apps/api/src/registration.js";
|
||||||
|
import { MockResendAdapter } from "../../apps/api/src/resend-adapter.js";
|
||||||
|
import { StickerReleaseService } from "../../apps/api/src/sticker-releases.js";
|
||||||
|
import { ProjectPurgeCleanup } from "../../apps/worker/src/project-purge-cleanup.js";
|
||||||
|
|
||||||
|
const roots: string[] = [];
|
||||||
|
const closeables: Array<{ close(): void }> = [];
|
||||||
|
|
||||||
|
function fixture() {
|
||||||
|
const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp5-06-cleanup-"));
|
||||||
|
roots.push(dataRoot);
|
||||||
|
mkdirSync(join(dataRoot, "db"), { recursive: true });
|
||||||
|
const databasePath = join(dataRoot, "db", "dada.sqlite3");
|
||||||
|
const registration = new RegistrationService({
|
||||||
|
challengePepper: Buffer.alloc(32, 0x61),
|
||||||
|
currentPrivacyNoticeVersion: "p0a-registration-notice-v1",
|
||||||
|
databasePath,
|
||||||
|
invitePepper: Buffer.alloc(32, 0x62),
|
||||||
|
resend: new MockResendAdapter(),
|
||||||
|
sessionPepper: Buffer.alloc(32, 0x63),
|
||||||
|
});
|
||||||
|
const storage = new ManagedStorage({ dataRoot, databasePath });
|
||||||
|
const stickers = new StickerReleaseService({ databasePath, storage });
|
||||||
|
closeables.push(stickers, storage, registration);
|
||||||
|
return { dataRoot, databasePath, database: registration.database, storage };
|
||||||
|
}
|
||||||
|
|
||||||
|
function seedAdmin(database: RegistrationService["database"]) {
|
||||||
|
const adminId = randomUUID();
|
||||||
|
database.prepare(`
|
||||||
|
INSERT INTO users (
|
||||||
|
user_id, normalized_email, role, status, counts_toward_stage_limit,
|
||||||
|
registration_id, created_at
|
||||||
|
) VALUES (?, ?, 'super_admin', 'active', 0, ?, ?)
|
||||||
|
`).run(adminId, `${adminId}@example.invalid`, randomUUID(), Date.now());
|
||||||
|
database.prepare("INSERT INTO admin_access (user_id, allowed) VALUES (?, 1)").run(adminId);
|
||||||
|
return adminId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function seedHistoricalPair(test: ReturnType<typeof fixture>) {
|
||||||
|
const original = await test.storage.commitBufferFixture("sticker_original", "STK2401.png", Buffer.from("original-history"));
|
||||||
|
const thumbnail = await test.storage.commitBufferFixture("sticker_thumbnail", "STK2401-thumbnail.png", Buffer.from("thumbnail-history"));
|
||||||
|
const insert = test.database.prepare(`
|
||||||
|
INSERT INTO sticker_managed_file_history (
|
||||||
|
managed_file_id, stable_id, resource_version, file_kind, created_at
|
||||||
|
) VALUES (?, 'STK2401', 'asset-20260701.1', ?, ?)
|
||||||
|
`);
|
||||||
|
insert.run(original.file_id, "original", Date.now());
|
||||||
|
insert.run(thumbnail.file_id, "thumbnail", Date.now());
|
||||||
|
return { original, thumbnail };
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const value of closeables.splice(0).reverse()) value.close();
|
||||||
|
for (const root of roots.splice(0)) rmSync(root, { force: true, recursive: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("TDD-WP5-CLN-001 sticker history cleanup", () => {
|
||||||
|
it("denies the whole batch when a reference appears after the candidate snapshot", async () => {
|
||||||
|
const test = fixture();
|
||||||
|
const adminId = seedAdmin(test.database);
|
||||||
|
const files = await seedHistoricalPair(test);
|
||||||
|
const candidates = test.storage.listAssetCleanupCandidates();
|
||||||
|
expect(candidates.items).toEqual(expect.arrayContaining([
|
||||||
|
expect.objectContaining({ file_id: files.original.file_id, file_kind: "original", reference_count: 0, stable_id: "STK2401" }),
|
||||||
|
expect.objectContaining({ file_id: files.thumbnail.file_id, file_kind: "thumbnail", reference_count: 0, stable_id: "STK2401" }),
|
||||||
|
]));
|
||||||
|
const intent = test.storage.createAssetCleanupIntent({
|
||||||
|
actorId: adminId,
|
||||||
|
fileIds: [files.original.file_id, files.thumbnail.file_id],
|
||||||
|
idempotencyKey: `cleanup-${randomUUID()}-${randomUUID()}`,
|
||||||
|
snapshotVersion: candidates.candidate_snapshot_version,
|
||||||
|
});
|
||||||
|
|
||||||
|
test.storage.addAssetReference(files.original.file_id, "release");
|
||||||
|
expect(() => test.storage.confirmAssetCleanupIntent({
|
||||||
|
actorId: adminId,
|
||||||
|
confirmationToken: intent.confirmation_token,
|
||||||
|
requestId: intent.request_id,
|
||||||
|
})).toThrow("ASSET_HISTORY_REFERENCE_CONFLICT");
|
||||||
|
|
||||||
|
expect(test.database.prepare("SELECT status FROM asset_cleanup_requests WHERE request_id = ?").get(intent.request_id)).toEqual({ status: "denied" });
|
||||||
|
expect(test.database.prepare("SELECT COUNT(*) AS count FROM file_cleanup_queue WHERE status = 'pending'").get()).toEqual({ count: 0 });
|
||||||
|
expect(test.database.prepare("SELECT COUNT(*) AS count FROM admin_operation_logs WHERE target_ref = ? AND result = 'failed'").get(intent.request_id)).toEqual({ count: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("requires the same admin, queues without reducing capacity, then remeasures after physical deletion", async () => {
|
||||||
|
const test = fixture();
|
||||||
|
const adminId = seedAdmin(test.database);
|
||||||
|
const otherAdminId = seedAdmin(test.database);
|
||||||
|
const files = await seedHistoricalPair(test);
|
||||||
|
const bytesBefore = test.storage.getState().managed_content_bytes;
|
||||||
|
const candidates = test.storage.listAssetCleanupCandidates();
|
||||||
|
const intent = test.storage.createAssetCleanupIntent({
|
||||||
|
actorId: adminId,
|
||||||
|
fileIds: [files.original.file_id, files.thumbnail.file_id],
|
||||||
|
idempotencyKey: `cleanup-${randomUUID()}-${randomUUID()}`,
|
||||||
|
snapshotVersion: candidates.candidate_snapshot_version,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(() => test.storage.confirmAssetCleanupIntent({
|
||||||
|
actorId: otherAdminId,
|
||||||
|
confirmationToken: intent.confirmation_token,
|
||||||
|
requestId: intent.request_id,
|
||||||
|
})).toThrow("ASSET_CLEANUP_CANDIDATE_STALE");
|
||||||
|
const queued = test.storage.confirmAssetCleanupIntent({
|
||||||
|
actorId: adminId,
|
||||||
|
confirmationToken: intent.confirmation_token,
|
||||||
|
requestId: intent.request_id,
|
||||||
|
});
|
||||||
|
expect(queued).toMatchObject({ file_count: 2, status: "queued" });
|
||||||
|
expect(test.storage.getState().managed_content_bytes).toBe(bytesBefore);
|
||||||
|
expect(existsSync(join(test.dataRoot, files.original.relative_path))).toBe(true);
|
||||||
|
expect(test.database.prepare("SELECT COUNT(*) AS count FROM file_cleanup_queue WHERE status = 'pending'").get()).toEqual({ count: 2 });
|
||||||
|
|
||||||
|
const worker = new ProjectPurgeCleanup({ dataRoot: test.dataRoot, databasePath: test.databasePath });
|
||||||
|
const result = worker.processFileCleanup();
|
||||||
|
worker.close();
|
||||||
|
expect(result).toEqual({ completed: 2, failed: 0 });
|
||||||
|
expect(existsSync(join(test.dataRoot, files.original.relative_path))).toBe(false);
|
||||||
|
expect(existsSync(join(test.dataRoot, files.thumbnail.relative_path))).toBe(false);
|
||||||
|
expect(test.storage.getState()).toMatchObject({ managed_content_bytes: 0, storage_status: "active" });
|
||||||
|
expect(test.database.prepare("SELECT status FROM asset_cleanup_requests WHERE request_id = ?").get(intent.request_id)).toEqual({ status: "completed" });
|
||||||
|
expect(test.database.prepare("SELECT operation_type, result FROM admin_operation_logs WHERE target_ref = ? ORDER BY occurred_at").all(intent.request_id)).toEqual([
|
||||||
|
{ operation_type: "asset_cleanup_requested", result: "succeeded" },
|
||||||
|
{ operation_type: "asset_cleanup_validated", result: "succeeded" },
|
||||||
|
{ operation_type: "asset_cleanup_scheduled", result: "succeeded" },
|
||||||
|
{ operation_type: "asset_cleanup_physical_completed", result: "succeeded" },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user