feat: implement explicit sticker history cleanup (TASK-WP5-06)
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 3m16s
Dada P0-A isolated Windows CI / validate-and-package (push) Failing after 3m16s
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { rmSync } from "node:fs";
|
||||
import { existsSync, rmSync, statSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { isAbsolute, relative, resolve } from "node:path";
|
||||
|
||||
@@ -29,6 +29,40 @@ interface FileCleanupRow {
|
||||
const resourceScope = JSON.stringify([
|
||||
"project_state", "generation", "generated_image", "reference", "location", "latest_export",
|
||||
]);
|
||||
const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000;
|
||||
const auditRefPattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$/;
|
||||
const forbiddenSummaryKeys = new Set([
|
||||
"absolute_path", "api_key", "body", "code_hmac", "content", "credential", "email", "image",
|
||||
"image_content", "password", "path", "prompt", "secret", "session_token", "verification_code", "whitelist",
|
||||
]);
|
||||
const forbiddenSummaryFragments = ["content", "credential", "email", "image", "password", "path", "prompt", "secret", "token"];
|
||||
|
||||
function isSafeAuditRef(value: unknown) {
|
||||
return typeof value === "string" && auditRefPattern.test(value) ? 1 : 0;
|
||||
}
|
||||
|
||||
function isSafeAuditSummaryJson(value: unknown) {
|
||||
if (typeof value !== "string" || Buffer.byteLength(value, "utf8") > 2_048) return 0;
|
||||
try {
|
||||
const valid = (entry: unknown, depth: number): boolean => {
|
||||
if (depth > 3) return false;
|
||||
if (entry === null || typeof entry === "boolean") return true;
|
||||
if (typeof entry === "number") return Number.isSafeInteger(entry);
|
||||
if (typeof entry === "string") return /^[A-Za-z0-9_.:@-]{1,160}$/.test(entry) && !entry.includes("@");
|
||||
if (Array.isArray(entry)) return entry.length <= 20 && entry.every((item) => valid(item, depth + 1));
|
||||
if (!entry || typeof entry !== "object") return false;
|
||||
return Object.entries(entry).length <= 32 && Object.entries(entry).every(([key, item]) => (
|
||||
auditRefPattern.test(key)
|
||||
&& !forbiddenSummaryKeys.has(key.toLowerCase())
|
||||
&& !forbiddenSummaryFragments.some((fragment) => key.toLowerCase().includes(fragment))
|
||||
&& valid(item, depth + 1)
|
||||
));
|
||||
};
|
||||
return valid(JSON.parse(value), 0) ? 1 : 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function iso(timestamp: number) {
|
||||
return new Date(timestamp).toISOString();
|
||||
@@ -47,6 +81,12 @@ export class ProjectPurgeCleanup {
|
||||
this.database.pragma("journal_mode = WAL");
|
||||
this.database.pragma("foreign_keys = ON");
|
||||
this.database.pragma("busy_timeout = 5000");
|
||||
this.database.function("dada_audit_ref_is_safe", { deterministic: true }, isSafeAuditRef);
|
||||
this.database.function("dada_audit_summary_is_safe", { deterministic: true }, isSafeAuditSummaryJson);
|
||||
this.database.function("dada_allow_privacy_purge", { deterministic: false }, () => 0);
|
||||
this.database.function("dada_privacy_purge_subject", { deterministic: false }, () => "");
|
||||
this.database.function("dada_allow_retention_purge", { deterministic: false }, () => 0);
|
||||
this.database.function("dada_retention_purge_now", { deterministic: false }, () => 0);
|
||||
}
|
||||
|
||||
close() {
|
||||
@@ -153,12 +193,16 @@ export class ProjectPurgeCleanup {
|
||||
if (pending.count === 0) {
|
||||
this.database.prepare("UPDATE asset_cleanup_requests SET status = 'completed' WHERE request_id = ? AND status = 'queued'")
|
||||
.run(request.request_id);
|
||||
this.insertAssetCleanupAudit(request.request_id, row.byte_size, this.clock());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.tableExists("sticker_managed_file_history")) {
|
||||
this.database.prepare("DELETE FROM sticker_managed_file_history WHERE managed_file_id = ?").run(row.managed_file_id);
|
||||
}
|
||||
this.database.prepare("DELETE FROM managed_files WHERE file_id = ?").run(row.managed_file_id);
|
||||
if (this.tableExists("local_backend_storage_state")) this.decrementManagedCapacity(row.byte_size);
|
||||
if (this.tableExists("local_backend_storage_state")) this.remeasureManagedCapacity();
|
||||
}
|
||||
this.database.prepare(`
|
||||
UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL
|
||||
@@ -238,4 +282,46 @@ export class ProjectPurgeCleanup {
|
||||
WHERE singleton = 1
|
||||
`).run(managed, notice, status, iso(this.clock()));
|
||||
}
|
||||
|
||||
private insertAssetCleanupAudit(requestId: string, deletedBytes: number, occurredAt: number) {
|
||||
if (!this.tableExists("admin_operation_logs")) return;
|
||||
this.database.prepare(`
|
||||
INSERT INTO admin_operation_logs (
|
||||
log_id, actor_type, actor_ref, operation_type, target_type, target_ref,
|
||||
result, before_summary, after_summary, occurred_at, expires_at
|
||||
) VALUES (?, 'system', 'project_purge_worker', 'asset_cleanup_physical_completed', 'asset_cleanup', ?, 'succeeded', NULL, ?, ?, ?)
|
||||
`).run(
|
||||
randomUUID(), requestId, JSON.stringify({ deleted_bytes: deletedBytes, status: "completed" }), occurredAt,
|
||||
occurredAt + auditRetentionMilliseconds,
|
||||
);
|
||||
}
|
||||
|
||||
private remeasureManagedCapacity() {
|
||||
const state = this.database.prepare(`
|
||||
SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1
|
||||
`).get() as { managed_content_bytes: number } | undefined;
|
||||
if (!state) return;
|
||||
const rows = this.database.prepare(`
|
||||
SELECT relative_path FROM managed_files WHERE status = 'committed'
|
||||
`).all() as Array<{ relative_path: string }>;
|
||||
let managed = 0;
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const path = this.resolveManagedPath(row.relative_path);
|
||||
if (existsSync(path)) managed += statSync(path).size;
|
||||
} catch {
|
||||
// A missing or invalid path is excluded from the measured physical total.
|
||||
}
|
||||
}
|
||||
const reservations = this.tableExists("storage_reservations")
|
||||
? (this.database.prepare("SELECT COALESCE(SUM(projected_bytes), 0) AS bytes FROM storage_reservations WHERE status = 'active'").get() as { bytes: number }).bytes
|
||||
: 0;
|
||||
const notice = managed < 4_294_967_296 ? "normal" : managed < 4_831_838_208 ? "warning" : "critical";
|
||||
const status = managed + reservations >= 5_368_709_120 ? "full" : "active";
|
||||
this.database.prepare(`
|
||||
UPDATE local_backend_storage_state
|
||||
SET managed_content_bytes = ?, capacity_notice_level = ?, storage_status = ?, measured_at = ?, version = version + 1
|
||||
WHERE singleton = 1
|
||||
`).run(managed, notice, status, iso(this.clock()));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user