feat: implement explicit sticker history cleanup (TASK-WP5-06)
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 3m16s

This commit is contained in:
suyx
2026-08-04 10:40:11 +08:00
parent b00500512e
commit c2521f7208
7 changed files with 766 additions and 2 deletions
+88 -2
View File
@@ -1,5 +1,5 @@
import { randomUUID } from "node:crypto";
import { rmSync } from "node:fs";
import { existsSync, rmSync, statSync } from "node:fs";
import { createRequire } from "node:module";
import { isAbsolute, relative, resolve } from "node:path";
@@ -29,6 +29,40 @@ interface FileCleanupRow {
const resourceScope = JSON.stringify([
"project_state", "generation", "generated_image", "reference", "location", "latest_export",
]);
const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000;
const auditRefPattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$/;
const forbiddenSummaryKeys = new Set([
"absolute_path", "api_key", "body", "code_hmac", "content", "credential", "email", "image",
"image_content", "password", "path", "prompt", "secret", "session_token", "verification_code", "whitelist",
]);
const forbiddenSummaryFragments = ["content", "credential", "email", "image", "password", "path", "prompt", "secret", "token"];
function isSafeAuditRef(value: unknown) {
return typeof value === "string" && auditRefPattern.test(value) ? 1 : 0;
}
function isSafeAuditSummaryJson(value: unknown) {
if (typeof value !== "string" || Buffer.byteLength(value, "utf8") > 2_048) return 0;
try {
const valid = (entry: unknown, depth: number): boolean => {
if (depth > 3) return false;
if (entry === null || typeof entry === "boolean") return true;
if (typeof entry === "number") return Number.isSafeInteger(entry);
if (typeof entry === "string") return /^[A-Za-z0-9_.:@-]{1,160}$/.test(entry) && !entry.includes("@");
if (Array.isArray(entry)) return entry.length <= 20 && entry.every((item) => valid(item, depth + 1));
if (!entry || typeof entry !== "object") return false;
return Object.entries(entry).length <= 32 && Object.entries(entry).every(([key, item]) => (
auditRefPattern.test(key)
&& !forbiddenSummaryKeys.has(key.toLowerCase())
&& !forbiddenSummaryFragments.some((fragment) => key.toLowerCase().includes(fragment))
&& valid(item, depth + 1)
));
};
return valid(JSON.parse(value), 0) ? 1 : 0;
} catch {
return 0;
}
}
function iso(timestamp: number) {
return new Date(timestamp).toISOString();
@@ -47,6 +81,12 @@ export class ProjectPurgeCleanup {
this.database.pragma("journal_mode = WAL");
this.database.pragma("foreign_keys = ON");
this.database.pragma("busy_timeout = 5000");
this.database.function("dada_audit_ref_is_safe", { deterministic: true }, isSafeAuditRef);
this.database.function("dada_audit_summary_is_safe", { deterministic: true }, isSafeAuditSummaryJson);
this.database.function("dada_allow_privacy_purge", { deterministic: false }, () => 0);
this.database.function("dada_privacy_purge_subject", { deterministic: false }, () => "");
this.database.function("dada_allow_retention_purge", { deterministic: false }, () => 0);
this.database.function("dada_retention_purge_now", { deterministic: false }, () => 0);
}
close() {
@@ -153,12 +193,16 @@ export class ProjectPurgeCleanup {
if (pending.count === 0) {
this.database.prepare("UPDATE asset_cleanup_requests SET status = 'completed' WHERE request_id = ? AND status = 'queued'")
.run(request.request_id);
this.insertAssetCleanupAudit(request.request_id, row.byte_size, this.clock());
}
}
}
}
if (this.tableExists("sticker_managed_file_history")) {
this.database.prepare("DELETE FROM sticker_managed_file_history WHERE managed_file_id = ?").run(row.managed_file_id);
}
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
if (this.tableExists("local_backend_storage_state")) this.decrementManagedCapacity(row.byte_size);
if (this.tableExists("local_backend_storage_state")) this.remeasureManagedCapacity();
}
this.database.prepare(`
UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL
@@ -238,4 +282,46 @@ export class ProjectPurgeCleanup {
WHERE singleton = 1
`).run(managed, notice, status, iso(this.clock()));
}
private insertAssetCleanupAudit(requestId: string, deletedBytes: number, occurredAt: number) {
if (!this.tableExists("admin_operation_logs")) return;
this.database.prepare(`
INSERT INTO admin_operation_logs (
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
result, before_summary, after_summary, occurred_at, expires_at
) VALUES (?, 'system', 'project_purge_worker', 'asset_cleanup_physical_completed', 'asset_cleanup', ?, 'succeeded', NULL, ?, ?, ?)
`).run(
randomUUID(), requestId, JSON.stringify({ deleted_bytes: deletedBytes, status: "completed" }), occurredAt,
occurredAt + auditRetentionMilliseconds,
);
}
private remeasureManagedCapacity() {
const state = this.database.prepare(`
SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1
`).get() as { managed_content_bytes: number } | undefined;
if (!state) return;
const rows = this.database.prepare(`
SELECT relative_path FROM managed_files WHERE status = 'committed'
`).all() as Array<{ relative_path: string }>;
let managed = 0;
for (const row of rows) {
try {
const path = this.resolveManagedPath(row.relative_path);
if (existsSync(path)) managed += statSync(path).size;
} catch {
// A missing or invalid path is excluded from the measured physical total.
}
}
const reservations = this.tableExists("storage_reservations")
? (this.database.prepare("SELECT COALESCE(SUM(projected_bytes), 0) AS bytes FROM storage_reservations WHERE status = 'active'").get() as { bytes: number }).bytes
: 0;
const notice = managed < 4_294_967_296 ? "normal" : managed < 4_831_838_208 ? "warning" : "critical";
const status = managed + reservations >= 5_368_709_120 ? "full" : "active";
this.database.prepare(`
UPDATE local_backend_storage_state
SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1
WHERE singleton = 1
`).run(managed, notice, status, iso(this.clock()));
}
}