intelligrow/tyx_AI_xhs
6
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

feat: implement explicit sticker history cleanup (TASK-WP5-06)
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 3m16s
Details

Browse Source
This commit is contained in:
suyx
2026-08-04 10:40:11 +08:00
parent b00500512e
commit c2521f7208
7 changed files with 766 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/managed-storage.ts
+392
View File
@@ -75,6 +75,31 @@ interface CleanupQueueRow {
relative_path: string;
}
export interface AssetCleanupCandidateView {
byte_size: number;
file_id: string;
file_kind: "original" | "thumbnail";
hash_prefix: string;
reference_count: 0;
resource_version: string;
stable_id: string;
}
export interface AssetCleanupCandidatesView {
candidate_snapshot_version: string;
expires_at: string;
items: AssetCleanupCandidateView[];
}
export interface AssetCleanupIntentView {
confirmation_token: string;
expires_at: string;
file_count: number;
request_id: string;
status: "pending_confirmation" | "denied" | "queued" | "completed";
total_bytes: number;
}
export class StorageCapacityError extends Error {
readonly code = "STORAGE_CAPACITY_EXCEEDED";
readonly httpStatus = 507;
@@ -104,6 +129,10 @@ function auditExpiry(occurredAt: number) {
return occurredAt + auditRetentionMilliseconds;
}
function digest(value: string) {
return createHash("sha256").update(value, "utf8").digest("hex");
}
function validatePositiveBytes(value: number, name: string) {
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name}_invalid`);
}
@@ -255,6 +284,28 @@ export class ManagedStorage {
FOREIGN KEY (request_id) REFERENCES asset_cleanup_requests(request_id),
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
);
CREATE TABLE IF NOT EXISTS asset_cleanup_candidate_snapshots (
snapshot_version TEXT PRIMARY KEY,
items_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS sticker_managed_file_history (
managed_file_id TEXT NOT NULL,
stable_id TEXT NOT NULL,
resource_version TEXT NOT NULL,
file_kind TEXT NOT NULL CHECK (file_kind IN ('original', 'thumbnail')),
created_at INTEGER NOT NULL,
PRIMARY KEY (managed_file_id, file_kind),
FOREIGN KEY (managed_file_id) REFERENCES managed_files(file_id)
);
CREATE TABLE IF NOT EXISTS project_sticker_asset_refs (
reference_id TEXT PRIMARY KEY,
project_id TEXT NOT NULL,
stable_id TEXT NOT NULL,
resource_version TEXT NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS admin_operation_logs (
log_id TEXT PRIMARY KEY,
actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')),
@@ -277,6 +328,49 @@ export class ManagedStorage {
if (!managedFileColumns.some((column) => column.name === "owner_ref")) {
this.database.exec("ALTER TABLE managed_files ADD COLUMN owner_ref TEXT");
}
if (!managedFileColumns.some((column) => column.name === "cleanup_status")) {
this.database.exec("ALTER TABLE managed_files ADD COLUMN cleanup_status TEXT");
}
const cleanupRequestColumns = this.database.prepare("PRAGMA table_info(asset_cleanup_requests)").all() as Array<{ name: string }>;
const cleanupRequestAdditions: Array<[string, string]> = [
["created_by", "TEXT"],
["confirmed_by", "TEXT"],
["snapshot_version", "TEXT"],
["expires_at", "INTEGER"],
["confirmation_token_digest", "TEXT"],
["idempotency_key_digest", "TEXT"],
["request_hash", "TEXT"],
["file_count", "INTEGER"],
["total_bytes", "INTEGER"],
["denied_reason", "TEXT"],
];
for (const [column, type] of cleanupRequestAdditions) {
if (!cleanupRequestColumns.some((item) => item.name === column)) {
this.database.exec(`ALTER TABLE asset_cleanup_requests ADD COLUMN ${column} ${type}`);
}
}
const cleanupItemColumns = this.database.prepare("PRAGMA table_info(asset_cleanup_request_items)").all() as Array<{ name: string }>;
const cleanupItemAdditions: Array<[string, string]> = [
["stable_id", "TEXT"],
["resource_version", "TEXT"],
["file_kind", "TEXT"],
["byte_size", "INTEGER"],
["sha256_prefix", "TEXT"],
];
for (const [column, type] of cleanupItemAdditions) {
if (!cleanupItemColumns.some((item) => item.name === column)) {
this.database.exec(`ALTER TABLE asset_cleanup_request_items ADD COLUMN ${column} ${type}`);
}
}
this.database.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS asset_cleanup_requests_actor_idempotency
ON asset_cleanup_requests (created_by, idempotency_key_digest)
WHERE created_by IS NOT NULL AND idempotency_key_digest IS NOT NULL;
CREATE INDEX IF NOT EXISTS sticker_managed_file_history_lookup
ON sticker_managed_file_history (stable_id, resource_version, file_kind);
CREATE INDEX IF NOT EXISTS asset_cleanup_candidate_snapshots_expiry
ON asset_cleanup_candidate_snapshots (expires_at);
`);
ensureAdminOperationAuditSchema(this.database, Date.now());
const initial = classifyCapacity(0, 0);
this.database.prepare(`
@@ -299,6 +393,106 @@ export class ManagedStorage {
return withReservations;
}
private readAssetCleanupCandidates(): AssetCleanupCandidateView[] {
const releaseReferenceClause = this.tableExists("sticker_release_items") ? `
AND NOT EXISTS (
SELECT 1 FROM sticker_release_items release_items
WHERE release_items.original_file_id = mf.file_id OR release_items.thumbnail_file_id = mf.file_id
)` : "";
return this.database.prepare(`
SELECT
mf.file_id,
mf.byte_size,
history.stable_id,
history.resource_version,
history.file_kind,
substr(mf.sha256, 1, 12) AS hash_prefix,
0 AS reference_count
FROM sticker_managed_file_history history
JOIN managed_files mf ON mf.file_id = history.managed_file_id
WHERE mf.status = 'committed'
AND mf.cleanup_status IS NULL
AND mf.file_kind IN ('sticker_original', 'sticker_thumbnail')
AND NOT EXISTS (
SELECT 1 FROM project_asset_refs refs WHERE refs.managed_file_id = mf.file_id
)
AND NOT EXISTS (
SELECT 1 FROM project_sticker_asset_refs project_refs
WHERE project_refs.stable_id = history.stable_id
AND project_refs.resource_version = history.resource_version
)
${releaseReferenceClause}
AND NOT EXISTS (
SELECT 1 FROM asset_cleanup_request_items request_items
JOIN asset_cleanup_requests requests ON requests.request_id = request_items.request_id
WHERE request_items.managed_file_id = mf.file_id
AND requests.status IN ('pending_confirmation', 'queued')
)
ORDER BY history.stable_id, history.resource_version, history.file_kind, mf.file_id
`).all() as AssetCleanupCandidateView[];
}
private assertActiveAdmin(actorId: string) {
const admin = this.database.prepare(`
SELECT 1 AS allowed FROM users u
JOIN admin_access access ON access.user_id = u.user_id
WHERE u.user_id = ? AND u.role = 'super_admin' AND u.status = 'active' AND access.allowed = 1
`).get(actorId);
if (!admin) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
}
private assetReferenceCount(fileId: string, requestId: string) {
const projectOrRelease = (this.database.prepare(`
SELECT COUNT(*) AS count FROM project_asset_refs WHERE managed_file_id = ?
`).get(fileId) as { count: number }).count;
const releaseItems = this.tableExists("sticker_release_items")
? (this.database.prepare(`
SELECT COUNT(*) AS count FROM sticker_release_items
WHERE original_file_id = ? OR thumbnail_file_id = ?
`).get(fileId, fileId) as { count: number }).count
: 0;
const projectStickerRefs = (this.database.prepare(`
SELECT COUNT(*) AS count
FROM sticker_managed_file_history history
JOIN project_sticker_asset_refs refs
ON refs.stable_id = history.stable_id AND refs.resource_version = history.resource_version
WHERE history.managed_file_id = ?
`).get(fileId) as { count: number }).count;
const otherCleanup = (this.database.prepare(`
SELECT COUNT(*) AS count FROM asset_cleanup_request_items items
JOIN asset_cleanup_requests requests ON requests.request_id = items.request_id
WHERE items.managed_file_id = ? AND items.request_id <> ?
AND requests.status IN ('pending_confirmation', 'queued')
`).get(fileId, requestId) as { count: number }).count;
return projectOrRelease + releaseItems + projectStickerRefs + otherCleanup;
}
private cleanupConfirmationToken(requestId: string, actorId: string, keyDigest: string) {
return digest(`Dada/P0A/asset-cleanup-confirm/v1:${requestId}:${actorId}:${keyDigest}`);
}
private tableExists(name: string) {
return Boolean(this.database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
}
private insertCleanupAudit(input: {
actorRef: string;
afterSummary: Record<string, unknown>;
operationType: string;
requestId: string;
result: "failed" | "succeeded";
}, occurredAt: number) {
this.database.prepare(`
INSERT INTO admin_operation_logs (
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
result, before_summary, after_summary, occurred_at, expires_at
) VALUES (?, 'super_admin', ?, ?, 'asset_cleanup_request', ?, ?, NULL, ?, ?, ?)
`).run(
randomUUID(), input.actorRef, input.operationType, input.requestId, input.result,
serializeAuditSummary(input.afterSummary), occurredAt, auditExpiry(occurredAt),
);
}
private activeReservationBytes(excludingOperationId?: string) {
const row = this.database.prepare(`
SELECT COALESCE(SUM(projected_bytes), 0) AS bytes
@@ -692,6 +886,203 @@ export class ManagedStorage {
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(fileId);
}
listAssetCleanupCandidates(): AssetCleanupCandidatesView {
const createdAt = Date.now();
const expiresAt = createdAt + 5 * 60 * 1_000;
const items = this.readAssetCleanupCandidates();
const snapshotVersion = digest(JSON.stringify({
created_at: createdAt,
nonce: randomUUID(),
items: items.map((item) => ({ byte_size: item.byte_size, file_id: item.file_id, hash_prefix: item.hash_prefix })),
}));
this.database.prepare("DELETE FROM asset_cleanup_candidate_snapshots WHERE expires_at <= ?").run(createdAt);
this.database.prepare(`
INSERT INTO asset_cleanup_candidate_snapshots (
snapshot_version, items_json, created_at, expires_at
) VALUES (?, ?, ?, ?)
`).run(snapshotVersion, JSON.stringify(items), createdAt, expiresAt);
return {
candidate_snapshot_version: snapshotVersion,
expires_at: new Date(expiresAt).toISOString(),
items,
};
}
createAssetCleanupIntent(input: {
actorId: string;
fileIds: string[];
idempotencyKey: string;
snapshotVersion: string;
}): AssetCleanupIntentView {
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
const fileIds = [...new Set(input.fileIds)].sort();
if (!uuidPattern.test(input.actorId) || fileIds.length === 0 || fileIds.length !== input.fileIds.length
|| fileIds.length > 100 || fileIds.some((fileId) => !uuidPattern.test(fileId))
|| !/^[A-Za-z0-9_-]{32,200}$/.test(input.idempotencyKey)
|| !/^[0-9a-f]{64}$/.test(input.snapshotVersion)) {
throw new Error("cleanup_candidates_invalid");
}
const keyDigest = digest(input.idempotencyKey);
const requestHash = digest(JSON.stringify({ file_ids: fileIds, snapshot_version: input.snapshotVersion }));
const existing = this.database.prepare(`
SELECT request_id, request_hash, expires_at, file_count, total_bytes, status
FROM asset_cleanup_requests
WHERE created_by = ? AND idempotency_key_digest = ?
`).get(input.actorId, keyDigest) as {
expires_at: number; file_count: number; request_hash: string; request_id: string; status: AssetCleanupIntentView["status"]; total_bytes: number;
} | undefined;
if (existing) {
if (existing.request_hash !== requestHash) throw new Error("IDEMPOTENCY_KEY_CONFLICT");
return {
confirmation_token: this.cleanupConfirmationToken(existing.request_id, input.actorId, keyDigest),
expires_at: new Date(existing.expires_at).toISOString(),
file_count: existing.file_count,
request_id: existing.request_id,
status: existing.status,
total_bytes: existing.total_bytes,
};
}
const requestId = randomUUID();
const confirmationToken = this.cleanupConfirmationToken(requestId, input.actorId, keyDigest);
const createdAt = Date.now();
let view!: AssetCleanupIntentView;
const transaction = this.database.transaction(() => {
this.assertActiveAdmin(input.actorId);
const snapshot = this.database.prepare(`
SELECT items_json, expires_at FROM asset_cleanup_candidate_snapshots
WHERE snapshot_version = ?
`).get(input.snapshotVersion) as { expires_at: number; items_json: string } | undefined;
if (!snapshot || snapshot.expires_at <= createdAt) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
const snapshotItems = JSON.parse(snapshot.items_json) as AssetCleanupCandidateView[];
const byId = new Map(snapshotItems.map((item) => [item.file_id, item]));
const selected = fileIds.map((fileId) => byId.get(fileId));
if (selected.some((item) => !item)) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
const current = new Map(this.readAssetCleanupCandidates().map((item) => [item.file_id, item]));
if (fileIds.some((fileId) => !current.has(fileId))) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
const safeItems = selected as AssetCleanupCandidateView[];
const totalBytes = safeItems.reduce((sum, item) => sum + item.byte_size, 0);
this.database.prepare(`
INSERT INTO asset_cleanup_requests (
request_id, status, created_at, confirmed_at, created_by, confirmed_by,
snapshot_version, expires_at, confirmation_token_digest,
idempotency_key_digest, request_hash, file_count, total_bytes, denied_reason
) VALUES (?, 'pending_confirmation', ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, NULL)
`).run(
requestId, new Date(createdAt).toISOString(), input.actorId, input.snapshotVersion,
snapshot.expires_at, digest(confirmationToken), keyDigest, requestHash, safeItems.length, totalBytes,
);
const insert = this.database.prepare(`
INSERT INTO asset_cleanup_request_items (
request_id, managed_file_id, stable_id, resource_version, file_kind, byte_size, sha256_prefix
) VALUES (?, ?, ?, ?, ?, ?, ?)
`);
for (const item of safeItems) {
insert.run(requestId, item.file_id, item.stable_id, item.resource_version, item.file_kind, item.byte_size, item.hash_prefix);
}
this.insertCleanupAudit({
actorRef: input.actorId,
afterSummary: { file_count: safeItems.length, snapshot_version: input.snapshotVersion, total_bytes: totalBytes },
operationType: "asset_cleanup_requested",
requestId,
result: "succeeded",
}, createdAt);
view = {
confirmation_token: confirmationToken,
expires_at: new Date(snapshot.expires_at).toISOString(),
file_count: safeItems.length,
request_id: requestId,
status: "pending_confirmation",
total_bytes: totalBytes,
};
});
transaction.immediate();
return view;
}
confirmAssetCleanupIntent(input: { actorId: string; confirmationToken: string; requestId: string }) {
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
if (!uuidPattern.test(input.actorId) || !uuidPattern.test(input.requestId) || !/^[0-9a-f]{64}$/.test(input.confirmationToken)) {
throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
}
const confirmedAt = Date.now();
const outcome = this.database.transaction(() => {
this.assertActiveAdmin(input.actorId);
const request = this.database.prepare(`
SELECT status, created_by, expires_at, confirmation_token_digest, file_count, total_bytes
FROM asset_cleanup_requests WHERE request_id = ?
`).get(input.requestId) as {
confirmation_token_digest: string | null; created_by: string | null; expires_at: number | null;
file_count: number | null; status: string; total_bytes: number | null;
} | undefined;
if (!request || request.status !== "pending_confirmation" || request.created_by !== input.actorId
|| !request.expires_at || request.expires_at <= confirmedAt
|| request.confirmation_token_digest !== digest(input.confirmationToken)) {
throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
}
const files = this.database.prepare(`
SELECT mf.file_id, mf.file_kind, mf.relative_path, mf.byte_size, mf.status
FROM asset_cleanup_request_items items
JOIN managed_files mf ON mf.file_id = items.managed_file_id
WHERE items.request_id = ? ORDER BY mf.file_id
`).all(input.requestId) as ManagedFileRow[];
if (files.length !== request.file_count) throw new Error("ASSET_CLEANUP_CANDIDATE_STALE");
const conflicted = files.some((file) => file.status !== "committed"
|| !new Set(["sticker_original", "sticker_thumbnail"]).has(file.file_kind)
|| this.assetReferenceCount(file.file_id, input.requestId) > 0);
if (conflicted) {
this.database.prepare(`
UPDATE asset_cleanup_requests
SET status = 'denied', confirmed_at = ?, confirmed_by = ?, denied_reason = 'reference_conflict'
WHERE request_id = ?
`).run(new Date(confirmedAt).toISOString(), input.actorId, input.requestId);
this.insertCleanupAudit({
actorRef: input.actorId,
afterSummary: { file_count: files.length, reason: "reference_conflict", status: "denied" },
operationType: "asset_cleanup_reference_denied",
requestId: input.requestId,
result: "failed",
}, confirmedAt);
return { conflict: true as const };
}
this.insertCleanupAudit({
actorRef: input.actorId,
afterSummary: { file_count: files.length, status: "validated" },
operationType: "asset_cleanup_validated",
requestId: input.requestId,
result: "succeeded",
}, confirmedAt);
for (const file of files) {
this.database.prepare(`
UPDATE managed_files SET status = 'purged', purged_at = ?, cleanup_status = 'pending_delete'
WHERE file_id = ? AND status = 'committed'
`).run(new Date(confirmedAt).toISOString(), file.file_id);
this.database.prepare(`
INSERT INTO file_cleanup_queue (
cleanup_id, managed_file_id, relative_path, byte_size, counts_toward_managed,
reason, status, created_at, completed_at, last_error
) VALUES (?, ?, ?, ?, 1, 'purge', 'pending', ?, NULL, NULL)
`).run(randomUUID(), file.file_id, file.relative_path, file.byte_size, new Date(confirmedAt).toISOString());
}
this.database.prepare(`
UPDATE asset_cleanup_requests
SET status = 'queued', confirmed_at = ?, confirmed_by = ?
WHERE request_id = ?
`).run(new Date(confirmedAt).toISOString(), input.actorId, input.requestId);
this.insertCleanupAudit({
actorRef: input.actorId,
afterSummary: { file_count: files.length, status: "queued", total_bytes: request.total_bytes ?? 0 },
operationType: "asset_cleanup_scheduled",
requestId: input.requestId,
result: "succeeded",
}, confirmedAt + 1);
return { conflict: false as const, file_count: files.length, request_id: input.requestId, status: "queued" as const };
}).immediate();
if (outcome.conflict) throw new Error("ASSET_HISTORY_REFERENCE_CONFLICT");
return outcome;
}
createCleanupIntent(fileIds: string[]) {
if (this.inspectAction("explicit_cleanup") !== "allow") throw new Error("cleanup_uncommitted");
if (fileIds.length === 0 || new Set(fileIds).size !== fileIds.length) throw new Error("cleanup_candidates_invalid");
@@ -777,6 +1168,7 @@ export class ManagedStorage {
this.database.prepare("DELETE FROM project_asset_refs WHERE managed_file_id = ?").run(row.managed_file_id);
const requests = this.database.prepare("SELECT request_id FROM asset_cleanup_request_items WHERE managed_file_id = ?").all(row.managed_file_id) as Array<{ request_id: string }>;
this.database.prepare("DELETE FROM asset_cleanup_request_items WHERE managed_file_id = ?").run(row.managed_file_id);
this.database.prepare("DELETE FROM sticker_managed_file_history WHERE managed_file_id = ?").run(row.managed_file_id);
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
for (const request of requests) {
const pendingItems = this.database.prepare("SELECT COUNT(*) AS count FROM asset_cleanup_request_items WHERE request_id = ?").get(request.request_id) as { count: number };