diff --git a/apps/api/src/audit-policy.ts b/apps/api/src/audit-policy.ts new file mode 100644 index 0000000..3a64c74 --- /dev/null +++ b/apps/api/src/audit-policy.ts @@ -0,0 +1,224 @@ +import type BetterSqlite3 from "better-sqlite3"; + +export const auditRetentionMilliseconds = 180 * 24 * 60 * 60 * 1_000; + +const auditRefPattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$/; +const forbiddenSummaryKeys = new Set([ + "absolute_path", "api_key", "body", "code_hmac", "content", "credential", + "email", "image", "image_content", "password", "path", "prompt", "secret", + "session_token", "verification_code", "whitelist", +]); +const forbiddenSummaryKeyFragments = [ + "content", "credential", "email", "image", "password", "path", "prompt", "secret", "token", +]; +const safeStringPattern = /^[A-Za-z0-9_.:@-]{1,160}$/; + +function isSafeSummaryValue(value: unknown, depth: number): boolean { + if (depth > 3) return false; + if (value === null || typeof value === "boolean") return true; + if (typeof value === "number") return Number.isSafeInteger(value); + if (typeof value === "string") return safeStringPattern.test(value) && !value.includes("@"); + if (Array.isArray(value)) return value.length <= 20 && value.every((entry) => isSafeSummaryValue(entry, depth + 1)); + if (!value || typeof value !== "object") return false; + const entries = Object.entries(value); + return entries.length <= 32 && entries.every(([key, entry]) => ( + auditRefPattern.test(key) + && !forbiddenSummaryKeys.has(key.toLowerCase()) + && !forbiddenSummaryKeyFragments.some((fragment) => key.toLowerCase().includes(fragment)) + && isSafeSummaryValue(entry, depth + 1) + )); +} + +export function isSafeAuditRef(value: unknown) { + return typeof value === "string" && auditRefPattern.test(value) ? 1 : 0; +} + +export function isSafeAuditSummaryJson(value: unknown) { + if (typeof value !== "string" || Buffer.byteLength(value, "utf8") > 2_048) return 0; + try { + return isSafeSummaryValue(JSON.parse(value), 0) ? 1 : 0; + } catch { + return 0; + } +} + +export function serializeAuditSummary(value: Record | null) { + if (value === null) return null; + const serialized = JSON.stringify(value); + if (isSafeAuditSummaryJson(serialized) !== 1) throw new Error("audit_summary_sensitive_or_invalid"); + return serialized; +} + +function toMilliseconds(value: string | number | null | undefined, fallback: number) { + if (typeof value === "number" && Number.isSafeInteger(value)) return value; + const parsed = Date.parse(String(value)); + return Number.isFinite(parsed) ? parsed : fallback; +} + +function createAdminOperationTable(database: BetterSqlite3.Database) { + database.exec(` + CREATE TABLE admin_operation_logs ( + log_id TEXT PRIMARY KEY, + actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')), + actor_ref TEXT NOT NULL CHECK (dada_audit_ref_is_safe(actor_ref) = 1), + operation_type TEXT NOT NULL CHECK (dada_audit_ref_is_safe(operation_type) = 1), + target_type TEXT NOT NULL CHECK (dada_audit_ref_is_safe(target_type) = 1), + target_ref TEXT NOT NULL CHECK (dada_audit_ref_is_safe(target_ref) = 1), + result TEXT NOT NULL CHECK (result IN ('succeeded', 'failed')), + before_summary TEXT CHECK (before_summary IS NULL OR dada_audit_summary_is_safe(before_summary) = 1), + after_summary TEXT CHECK (after_summary IS NULL OR dada_audit_summary_is_safe(after_summary) = 1), + occurred_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL CHECK (expires_at = occurred_at + ${auditRetentionMilliseconds}) + ); + `); +} + +function installAdminOperationTriggers(database: BetterSqlite3.Database) { + database.exec(` + DROP TRIGGER IF EXISTS admin_operation_logs_no_update; + DROP TRIGGER IF EXISTS admin_operation_logs_no_delete; + CREATE TRIGGER admin_operation_logs_no_update + BEFORE UPDATE ON admin_operation_logs + BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; + CREATE TRIGGER admin_operation_logs_no_delete + BEFORE DELETE ON admin_operation_logs + WHEN dada_allow_retention_purge() <> 1 OR OLD.expires_at > dada_retention_purge_now() + BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; + `); +} + +export function ensureAdminOperationAuditSchema(database: BetterSqlite3.Database, fallbackNow: number) { + const table = database.prepare(` + SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'admin_operation_logs' + `).get() as { sql: string } | undefined; + if (!table) { + createAdminOperationTable(database); + installAdminOperationTriggers(database); + return; + } + if (table.sql.includes("dada_audit_summary_is_safe") && table.sql.includes(String(auditRetentionMilliseconds))) { + installAdminOperationTriggers(database); + return; + } + + const columns = database.prepare("PRAGMA table_info(admin_operation_logs)").all() as Array<{ name: string }>; + const current = columns.some((column) => column.name === "actor_type"); + const rows = current + ? database.prepare("SELECT * FROM admin_operation_logs").all() as Array> + : database.prepare("SELECT log_id, operation, outcome, target_ref, created_at FROM admin_operation_logs").all() as Array>; + const migrate = database.transaction(() => { + database.exec(` + DROP TRIGGER IF EXISTS admin_operation_logs_no_update; + DROP TRIGGER IF EXISTS admin_operation_logs_no_delete; + ALTER TABLE admin_operation_logs RENAME TO admin_operation_logs_before_wp1_06; + `); + createAdminOperationTable(database); + const insert = database.prepare(` + INSERT INTO admin_operation_logs ( + log_id, actor_type, actor_ref, operation_type, target_type, target_ref, + result, before_summary, after_summary, occurred_at, expires_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + for (const row of rows) { + const occurredAt = toMilliseconds(current ? row.occurred_at! : row.created_at!, fallbackNow); + const expiresAt = current ? toMilliseconds(row.expires_at!, fallbackNow) : occurredAt + auditRetentionMilliseconds; + if (expiresAt !== occurredAt + auditRetentionMilliseconds) throw new Error("audit_retention_invalid"); + const beforeSummary = current && typeof row.before_summary === "string" + ? serializeAuditSummary(JSON.parse(row.before_summary) as Record) + : null; + const afterSummary = current + ? typeof row.after_summary === "string" ? serializeAuditSummary(JSON.parse(row.after_summary) as Record) : null + : serializeAuditSummary({ legacy_outcome: String(row.outcome) }); + insert.run( + row.log_id, + current ? row.actor_type : "system", + current ? row.actor_ref : "managed_storage_migration", + current ? row.operation_type : row.operation, + current ? row.target_type : "legacy_operation", + row.target_ref, + current ? row.result : String(row.outcome).startsWith("denied") ? "failed" : "succeeded", + beforeSummary, + afterSummary, + occurredAt, + expiresAt, + ); + } + database.exec("DROP TABLE admin_operation_logs_before_wp1_06"); + installAdminOperationTriggers(database); + }); + migrate(); +} + +function createPrivateAccessTable(database: BetterSqlite3.Database) { + database.exec(` + CREATE TABLE private_content_access_logs ( + log_id TEXT PRIMARY KEY, + actor_ref TEXT NOT NULL CHECK (length(actor_ref) = 36 AND dada_audit_ref_is_safe(actor_ref) = 1), + subject_ref TEXT NOT NULL CHECK (length(subject_ref) = 36 AND dada_audit_ref_is_safe(subject_ref) = 1), + target_ref TEXT NOT NULL CHECK (length(target_ref) = 36 AND dada_audit_ref_is_safe(target_ref) = 1), + content_type TEXT NOT NULL CHECK (content_type IN ('image', 'prompt')), + occurred_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL CHECK (expires_at = occurred_at + ${auditRetentionMilliseconds}) + ); + `); +} + +function installPrivateAccessTriggers(database: BetterSqlite3.Database) { + database.exec(` + DROP TRIGGER IF EXISTS private_content_access_logs_no_update; + DROP TRIGGER IF EXISTS private_content_access_logs_no_delete; + CREATE TRIGGER private_content_access_logs_no_update + BEFORE UPDATE ON private_content_access_logs + WHEN dada_allow_privacy_purge() <> 1 + OR OLD.subject_ref <> dada_privacy_purge_subject() + OR NEW.actor_ref <> OLD.actor_ref + OR NEW.content_type <> OLD.content_type + OR NEW.occurred_at <> OLD.occurred_at + OR NEW.expires_at <> OLD.expires_at + OR NEW.subject_ref = OLD.subject_ref + OR NEW.target_ref = OLD.target_ref + BEGIN SELECT RAISE(ABORT, 'private_content_access_logs_immutable'); END; + CREATE TRIGGER private_content_access_logs_no_delete + BEFORE DELETE ON private_content_access_logs + WHEN dada_allow_retention_purge() <> 1 OR OLD.expires_at > dada_retention_purge_now() + BEGIN SELECT RAISE(ABORT, 'private_content_access_logs_immutable'); END; + `); +} + +export function ensurePrivateAccessAuditSchema(database: BetterSqlite3.Database) { + const table = database.prepare(` + SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'private_content_access_logs' + `).get() as { sql: string } | undefined; + if (!table) { + createPrivateAccessTable(database); + installPrivateAccessTriggers(database); + return; + } + if (table.sql.includes(String(auditRetentionMilliseconds))) { + installPrivateAccessTriggers(database); + return; + } + const rows = database.prepare("SELECT * FROM private_content_access_logs").all() as Array>; + const migrate = database.transaction(() => { + database.exec(` + DROP TRIGGER IF EXISTS private_content_access_logs_no_update; + DROP TRIGGER IF EXISTS private_content_access_logs_no_delete; + ALTER TABLE private_content_access_logs RENAME TO private_content_access_logs_before_wp1_06; + `); + createPrivateAccessTable(database); + const insert = database.prepare(` + INSERT INTO private_content_access_logs ( + log_id, actor_ref, subject_ref, target_ref, content_type, occurred_at, expires_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `); + for (const row of rows) { + const occurredAt = toMilliseconds(row.occurred_at, 0); + const expiresAt = toMilliseconds(row.expires_at, 0); + if (expiresAt !== occurredAt + auditRetentionMilliseconds) throw new Error("audit_retention_invalid"); + insert.run(row.log_id, row.actor_ref, row.subject_ref, row.target_ref, row.content_type, occurredAt, expiresAt); + } + database.exec("DROP TABLE private_content_access_logs_before_wp1_06"); + installPrivateAccessTriggers(database); + }); + migrate(); +} diff --git a/apps/api/src/managed-storage.ts b/apps/api/src/managed-storage.ts index a74ebd3..916746e 100644 --- a/apps/api/src/managed-storage.ts +++ b/apps/api/src/managed-storage.ts @@ -15,6 +15,13 @@ import { pipeline } from "node:stream/promises"; import type BetterSqlite3 from "better-sqlite3"; +import { + auditRetentionMilliseconds, + ensureAdminOperationAuditSchema, + isSafeAuditRef, + isSafeAuditSummaryJson, + serializeAuditSummary, +} from "./audit-policy.js"; import { resolvePathWithinRoot } from "./local-data-root.js"; import { HARD_LIMIT_BYTES, @@ -93,8 +100,8 @@ function now() { return new Date().toISOString(); } -function auditExpiry(occurredAt: string) { - return new Date(Date.parse(occurredAt) + 180 * 24 * 60 * 60 * 1_000).toISOString(); +function auditExpiry(occurredAt: number) { + return occurredAt + auditRetentionMilliseconds; } function validatePositiveBytes(value: number, name: string) { @@ -155,6 +162,12 @@ export class ManagedStorage { this.database.pragma("journal_mode = WAL"); this.database.pragma("foreign_keys = ON"); this.database.pragma("busy_timeout = 5000"); + this.database.function("dada_audit_ref_is_safe", { deterministic: true }, isSafeAuditRef); + this.database.function("dada_audit_summary_is_safe", { deterministic: true }, isSafeAuditSummaryJson); + this.database.function("dada_allow_privacy_purge", { deterministic: false }, () => 0); + this.database.function("dada_privacy_purge_subject", { deterministic: false }, () => ""); + this.database.function("dada_allow_retention_purge", { deterministic: false }, () => 0); + this.database.function("dada_retention_purge_now", { deterministic: false }, () => 0); this.migrate(); const state = this.database.prepare("SELECT managed_content_bytes FROM local_backend_storage_state WHERE singleton = 1").get() as { managed_content_bytes: number }; this.measurementBaselineBytes = Math.max(0, state.managed_content_bytes - this.physicalManagedBytes()); @@ -247,7 +260,7 @@ export class ManagedStorage { if (!managedFileColumns.some((column) => column.name === "owner_ref")) { this.database.exec("ALTER TABLE managed_files ADD COLUMN owner_ref TEXT"); } - this.migrateLegacyAdminOperationLogs(); + ensureAdminOperationAuditSchema(this.database, Date.now()); const initial = classifyCapacity(0, 0); this.database.prepare(` INSERT OR IGNORE INTO local_backend_storage_state @@ -256,54 +269,6 @@ export class ManagedStorage { `).run(HARD_LIMIT_BYTES, initial.capacity_notice_level, initial.storage_status, now()); } - private migrateLegacyAdminOperationLogs() { - const columns = this.database.prepare("PRAGMA table_info(admin_operation_logs)").all() as Array<{ name: string }>; - if (columns.some((column) => column.name === "actor_type")) return; - const entries = this.database.prepare(` - SELECT log_id, operation, outcome, target_ref, created_at FROM admin_operation_logs - `).all() as Array<{ created_at: string; log_id: string; operation: string; outcome: string; target_ref: string }>; - this.database.exec(` - ALTER TABLE admin_operation_logs RENAME TO admin_operation_logs_legacy; - CREATE TABLE admin_operation_logs ( - log_id TEXT PRIMARY KEY, - actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')), - actor_ref TEXT NOT NULL, - operation_type TEXT NOT NULL, - target_type TEXT NOT NULL, - target_ref TEXT NOT NULL, - result TEXT NOT NULL CHECK (result IN ('succeeded', 'failed')), - before_summary TEXT, - after_summary TEXT, - occurred_at TEXT NOT NULL, - expires_at TEXT NOT NULL - ); - `); - const insert = this.database.prepare(` - INSERT INTO admin_operation_logs ( - log_id, actor_type, actor_ref, operation_type, target_type, target_ref, - result, before_summary, after_summary, occurred_at, expires_at - ) VALUES (?, 'system', 'managed_storage_migration', ?, 'legacy_operation', ?, ?, NULL, ?, ?, ?) - `); - for (const entry of entries) { - insert.run( - entry.log_id, - entry.operation, - entry.target_ref, - entry.outcome.startsWith("denied") ? "failed" : "succeeded", - JSON.stringify({ legacy_outcome: entry.outcome }), - entry.created_at, - auditExpiry(entry.created_at), - ); - } - this.database.exec(` - DROP TABLE admin_operation_logs_legacy; - CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_update - BEFORE UPDATE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; - CREATE TRIGGER IF NOT EXISTS admin_operation_logs_no_delete - BEFORE DELETE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; - `); - } - close() { this.database.close(); } @@ -629,14 +594,15 @@ export class ManagedStorage { return row.count > 0; }); if (conflict) { - const occurredAt = now(); - this.database.prepare("UPDATE asset_cleanup_requests SET status = 'denied', confirmed_at = ? WHERE request_id = ?").run(occurredAt, requestId); + const confirmedAt = now(); + const occurredAt = Date.now(); + this.database.prepare("UPDATE asset_cleanup_requests SET status = 'denied', confirmed_at = ? WHERE request_id = ?").run(confirmedAt, requestId); this.database.prepare(` INSERT INTO admin_operation_logs ( log_id, actor_type, actor_ref, operation_type, target_type, target_ref, result, before_summary, after_summary, occurred_at, expires_at ) VALUES (?, 'system', 'managed_storage', 'asset_cleanup', 'cleanup_request', ?, 'failed', NULL, ?, ?, ?) - `).run(randomUUID(), requestId, JSON.stringify({ reason: "reference_conflict" }), occurredAt, auditExpiry(occurredAt)); + `).run(randomUUID(), requestId, serializeAuditSummary({ reason: "reference_conflict" }), occurredAt, auditExpiry(occurredAt)); return false; } for (const file of files) { @@ -647,14 +613,15 @@ export class ManagedStorage { VALUES (?, ?, ?, ?, 1, 'purge', 'pending', ?) `).run(randomUUID(), file.file_id, file.relative_path, file.byte_size, now()); } - const occurredAt = now(); - this.database.prepare("UPDATE asset_cleanup_requests SET status = 'queued', confirmed_at = ? WHERE request_id = ?").run(occurredAt, requestId); + const confirmedAt = now(); + const occurredAt = Date.now(); + this.database.prepare("UPDATE asset_cleanup_requests SET status = 'queued', confirmed_at = ? WHERE request_id = ?").run(confirmedAt, requestId); this.database.prepare(` INSERT INTO admin_operation_logs ( log_id, actor_type, actor_ref, operation_type, target_type, target_ref, result, before_summary, after_summary, occurred_at, expires_at ) VALUES (?, 'system', 'managed_storage', 'asset_cleanup', 'cleanup_request', ?, 'succeeded', NULL, ?, ?, ?) - `).run(randomUUID(), requestId, JSON.stringify({ status: "queued" }), occurredAt, auditExpiry(occurredAt)); + `).run(randomUUID(), requestId, serializeAuditSummary({ status: "queued" }), occurredAt, auditExpiry(occurredAt)); return true; }); if (!transaction()) throw new Error("ASSET_HISTORY_REFERENCE_CONFLICT"); @@ -683,13 +650,13 @@ export class ManagedStorage { } } this.database.prepare("UPDATE file_cleanup_queue SET status = 'completed', completed_at = ?, last_error = NULL WHERE cleanup_id = ?").run(now(), row.cleanup_id); - const occurredAt = now(); + const occurredAt = Date.now(); this.database.prepare(` INSERT INTO admin_operation_logs ( log_id, actor_type, actor_ref, operation_type, target_type, target_ref, result, before_summary, after_summary, occurred_at, expires_at ) VALUES (?, 'system', 'managed_storage', 'physical_file_cleanup', 'cleanup_queue_item', ?, 'succeeded', NULL, ?, ?, ?) - `).run(randomUUID(), row.cleanup_id, JSON.stringify({ status: "completed" }), occurredAt, auditExpiry(occurredAt)); + `).run(randomUUID(), row.cleanup_id, serializeAuditSummary({ status: "completed" }), occurredAt, auditExpiry(occurredAt)); this.recordPhysicalMeasurement(); }); finish(); diff --git a/apps/api/src/registration.ts b/apps/api/src/registration.ts index 9a3e7b7..2b33d23 100644 --- a/apps/api/src/registration.ts +++ b/apps/api/src/registration.ts @@ -3,6 +3,13 @@ import { createRequire } from "node:module"; import type BetterSqlite3 from "better-sqlite3"; +import { + ensureAdminOperationAuditSchema, + ensurePrivateAccessAuditSchema, + isSafeAuditRef, + isSafeAuditSummaryJson, + serializeAuditSummary, +} from "./audit-policy.js"; import type { ResendAdapter } from "./resend-adapter.js"; import { RegistrationError, @@ -222,6 +229,7 @@ export class RegistrationService { readonly options: Required> & RegistrationServiceOptions; private adminAllowlistHashes = new Set(); private privacyPurgeActive = false; + private privacyPurgeSubject = ""; constructor(options: RegistrationServiceOptions) { assertSecret("invitePepper", options.invitePepper); @@ -239,8 +247,12 @@ export class RegistrationService { this.database.pragma("foreign_keys = ON"); this.database.pragma("synchronous = FULL"); this.database.pragma("busy_timeout = 5000"); + this.database.function("dada_audit_ref_is_safe", { deterministic: true }, isSafeAuditRef); + this.database.function("dada_audit_summary_is_safe", { deterministic: true }, isSafeAuditSummaryJson); this.database.function("dada_allow_privacy_purge", { deterministic: false }, () => this.privacyPurgeActive ? 1 : 0); + this.database.function("dada_privacy_purge_subject", { deterministic: false }, () => this.privacyPurgeSubject); this.database.function("dada_allow_retention_purge", { deterministic: false }, () => 0); + this.database.function("dada_retention_purge_now", { deterministic: false }, () => 0); this.migrate(); } @@ -1167,6 +1179,7 @@ export class RegistrationService { } this.privacyPurgeActive = true; + this.privacyPurgeSubject = session.user_id; try { this.database.prepare("DELETE FROM credit_ledger WHERE user_id = ?").run(session.user_id); this.database.prepare(` @@ -1174,6 +1187,7 @@ export class RegistrationService { `).run(randomUUID(), randomUUID(), session.user_id); } finally { this.privacyPurgeActive = false; + this.privacyPurgeSubject = ""; } this.queueOwnedManagedFiles(session.user_id, now); @@ -1468,59 +1482,8 @@ export class RegistrationService { singleton, applied_revision, allowlist_count, applied_at ) VALUES (1, 0, 0, 0); `); - this.migrateLegacyAdminOperationLogs(); - } - - private migrateLegacyAdminOperationLogs() { - const columns = this.database.prepare("PRAGMA table_info(admin_operation_logs)").all() as Array<{ name: string }>; - if (columns.some((column) => column.name === "actor_type")) return; - const legacy = this.database.prepare(` - SELECT log_id, operation, outcome, target_ref, created_at FROM admin_operation_logs - `).all() as Array<{ created_at: string | number; log_id: string; operation: string; outcome: string; target_ref: string }>; - this.database.exec(` - DROP TRIGGER IF EXISTS admin_operation_logs_no_update; - DROP TRIGGER IF EXISTS admin_operation_logs_no_delete; - ALTER TABLE admin_operation_logs RENAME TO admin_operation_logs_legacy; - CREATE TABLE admin_operation_logs ( - log_id TEXT PRIMARY KEY, - actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'super_admin')), - actor_ref TEXT NOT NULL, - operation_type TEXT NOT NULL, - target_type TEXT NOT NULL, - target_ref TEXT NOT NULL, - result TEXT NOT NULL CHECK (result IN ('succeeded', 'failed')), - before_summary TEXT, - after_summary TEXT, - occurred_at INTEGER NOT NULL, - expires_at INTEGER NOT NULL - ); - `); - const insert = this.database.prepare(` - INSERT INTO admin_operation_logs ( - log_id, actor_type, actor_ref, operation_type, target_type, target_ref, - result, before_summary, after_summary, occurred_at, expires_at - ) VALUES (?, 'system', 'managed_storage_migration', ?, 'legacy_operation', ?, ?, NULL, ?, ?, ?) - `); - for (const entry of legacy) { - const parsed = typeof entry.created_at === "number" ? entry.created_at : Date.parse(entry.created_at); - const occurredAt = Number.isFinite(parsed) ? parsed : this.options.clock(); - insert.run( - entry.log_id, - entry.operation, - entry.target_ref, - entry.outcome.startsWith("denied") ? "failed" : "succeeded", - JSON.stringify({ legacy_outcome: entry.outcome }), - occurredAt, - occurredAt + 180 * 24 * 60 * 60 * 1_000, - ); - } - this.database.exec(` - DROP TABLE admin_operation_logs_legacy; - CREATE TRIGGER admin_operation_logs_no_update - BEFORE UPDATE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; - CREATE TRIGGER admin_operation_logs_no_delete - BEFORE DELETE ON admin_operation_logs BEGIN SELECT RAISE(ABORT, 'admin_operation_logs_immutable'); END; - `); + ensureAdminOperationAuditSchema(this.database, this.options.clock()); + ensurePrivateAccessAuditSchema(this.database); } private runImmediate( @@ -1757,8 +1720,8 @@ export class RegistrationService { input.targetType, input.targetRef, input.result, - input.beforeSummary === null ? null : JSON.stringify(input.beforeSummary), - input.afterSummary === null ? null : JSON.stringify(input.afterSummary), + serializeAuditSummary(input.beforeSummary), + serializeAuditSummary(input.afterSummary), now, now + 180 * 24 * 60 * 60 * 1_000, ); diff --git a/apps/worker/src/retention-cleanup.ts b/apps/worker/src/retention-cleanup.ts index 795d50c..ab54e45 100644 --- a/apps/worker/src/retention-cleanup.ts +++ b/apps/worker/src/retention-cleanup.ts @@ -9,6 +9,7 @@ export class RetentionCleanup { private readonly clock: () => number; private readonly database: BetterSqlite3.Database; private retentionPurgeActive = false; + private retentionPurgeNow = 0; constructor(input: { clock?: () => number; databasePath: string }) { this.clock = input.clock ?? Date.now; @@ -18,6 +19,7 @@ export class RetentionCleanup { this.database.pragma("busy_timeout = 5000"); this.database.function("dada_allow_privacy_purge", { deterministic: false }, () => 0); this.database.function("dada_allow_retention_purge", { deterministic: false }, () => this.retentionPurgeActive ? 1 : 0); + this.database.function("dada_retention_purge_now", { deterministic: false }, () => this.retentionPurgeNow); } purgeExpired() { @@ -25,16 +27,21 @@ export class RetentionCleanup { this.database.exec("BEGIN IMMEDIATE"); try { this.retentionPurgeActive = true; + this.retentionPurgeNow = now; + const adminOperations = this.database.prepare("DELETE FROM admin_operation_logs WHERE expires_at <= ?").run(now); const privateAccess = this.database.prepare("DELETE FROM private_content_access_logs WHERE expires_at <= ?").run(now); const anonymous = this.database.prepare("DELETE FROM anonymous_retained_events WHERE expires_at <= ?").run(now); this.retentionPurgeActive = false; + this.retentionPurgeNow = 0; this.database.exec("COMMIT"); return { + admin_operation_logs: adminOperations.changes, anonymous_events: anonymous.changes, private_access_logs: privateAccess.changes, }; } catch (error) { this.retentionPurgeActive = false; + this.retentionPurgeNow = 0; if (this.database.inTransaction) this.database.exec("ROLLBACK"); throw error; } diff --git a/package.json b/package.json index 7452ae4..51be68d 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,9 @@ "test:wp1-04": "node scripts/run-wp1-04-validation.mjs", "test:wp1-04:red": "node scripts/run-wp1-04-validation.mjs --phase red", "test:wp1-05": "node scripts/run-wp1-05-validation.mjs", - "test:wp1-05:red": "node scripts/run-wp1-05-validation.mjs --phase red" + "test:wp1-05:red": "node scripts/run-wp1-05-validation.mjs --phase red", + "test:wp1-06": "node scripts/run-wp1-06-validation.mjs", + "test:wp1-06:red": "node scripts/run-wp1-06-validation.mjs --phase red" }, "devDependencies": { "@playwright/test": "1.62.0", diff --git a/scripts/run-wp1-06-validation.mjs b/scripts/run-wp1-06-validation.mjs new file mode 100644 index 0000000..08f7dea --- /dev/null +++ b/scripts/run-wp1-06-validation.mjs @@ -0,0 +1,61 @@ +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const phaseIndex = process.argv.indexOf("--phase"); +const phase = phaseIndex >= 0 ? process.argv[phaseIndex + 1] : "green"; +if (!["red", "green"].includes(phase)) throw new Error(`Unsupported phase: ${phase}`); +const runId = process.env.DADA_TDD_RUN_ID ?? `wp1-06-${phase}-${new Date().toISOString().replace(/[^0-9]/g, "")}`; +const testId = "TDD-WP1-AUD-001-immutable-triggers"; +const runDirectory = resolve("artifacts", "tdd", runId); +const caseDirectory = resolve(runDirectory, "cases", testId); +if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`); +mkdirSync(caseDirectory, { recursive: true }); + +const commandsToRun = phase === "red" + ? [["audit-integration", ["exec", "vitest", "run", "tests/integration/wp1-06-audit-immutability.test.ts"]]] + : [["integration", ["test:integration"]], ["tdd-trace", ["validate:tdd-trace"]]]; +const commandResults = []; +for (const [name, args] of commandsToRun) { + const command = `pnpm ${args.join(" ")}`; + const started_at = new Date().toISOString(); + const execution = spawnSync(process.env.ComSpec ?? "cmd.exe", ["/d", "/s", "/c", command], { + encoding: "utf8", + env: { ...process.env, DADA_EVIDENCE_DIR_AUDIT: caseDirectory }, + }); + if (execution.stdout) process.stdout.write(execution.stdout); + if (execution.stderr) process.stderr.write(execution.stderr); + commandResults.push({ command, exit_code: execution.status ?? 1, finished_at: new Date().toISOString(), name, started_at }); +} +writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands: commandResults, phase, run_id: runId }, null, 2)}\n`); +const expectedEvidence = phase === "red" + ? ["red-observation.json"] + : ["sql-results.json", "db-diff.json", "trigger-definition.json"]; +const commandState = phase === "red" + ? commandResults.every((result) => result.exit_code !== 0) + : commandResults.every((result) => result.exit_code === 0); +if (phase === "red") { + writeFileSync(resolve(caseDirectory, "red-observation.json"), `${JSON.stringify({ + expected_failure: "audit retention capability, exact expiry and summary constraints are incomplete", + status: commandState ? "red_confirmed" : "failed", + }, null, 2)}\n`); +} +const evidence_refs = expectedEvidence; +const missing_evidence = evidence_refs.filter((file) => !existsSync(resolve(caseDirectory, file))); +const targetStatus = phase === "red" ? "red_confirmed" : "passed"; +const status = commandState && missing_evidence.length === 0 ? targetStatus : "failed"; +const result = { + acceptance_criteria: ["AC-50"], automation: ["automated"], + commit: spawnSync("git", ["rev-parse", "--short", "HEAD"], { encoding: "utf8" }).stdout.trim(), + evidence_refs, layer: ["DB"], + manifest: { path: "tasks.manifest.json", sha256: createHash("sha256").update(readFileSync("tasks.manifest.json")).digest("hex").toUpperCase() }, + missing_evidence, phase, requirements: ["ADMIN-05", "ADMIN-09"], run_id: runId, status, + task_id: "TASK-WP1-06", test_id: testId, work_package: "WP-1", + worktree_under_test: spawnSync("git", ["status", "--porcelain"], { encoding: "utf8" }).stdout.trim() ? "uncommitted implementation" : "clean committed implementation", +}; +writeFileSync(resolve(caseDirectory, "result.json"), `${JSON.stringify(result, null, 2)}\n`); +const summary = { cases: [{ missing_evidence, status, test_id: testId }], phase, run_id: runId, status }; +writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify(summary, null, 2)}\n`); +console.log(JSON.stringify(summary, null, 2)); +if (status !== targetStatus) process.exit(1); diff --git a/tests/integration/wp1-05-account-deletion.test.ts b/tests/integration/wp1-05-account-deletion.test.ts index c298770..460779a 100644 --- a/tests/integration/wp1-05-account-deletion.test.ts +++ b/tests/integration/wp1-05-account-deletion.test.ts @@ -152,12 +152,14 @@ describe("TDD-WP1-DEL-002-anonymous-retention", () => { const harness = await createRegisteredHarness("retention@example.invalid"); const userId = harness.completed.user.userId; const csrfToken = harness.service.issueUserCsrfToken(harness.completed.sessionToken); - const accessExpiresAt = fixedNow + 30_000; + const accessActorRef = randomUUID(); + const accessTargetRef = randomUUID(); + const accessExpiresAt = fixedNow + 180 * 86_400_000; harness.service.database.prepare(` INSERT INTO private_content_access_logs ( log_id, actor_ref, subject_ref, target_ref, content_type, occurred_at, expires_at ) VALUES (?, ?, ?, ?, 'image', ?, ?) - `).run(randomUUID(), "admin-fixture", userId, "private-target-fixture", fixedNow - 1_000, accessExpiresAt); + `).run(randomUUID(), accessActorRef, userId, accessTargetRef, fixedNow, accessExpiresAt); const sent = await harness.service.sendAccountDeletionCode({ csrfToken, sessionToken: harness.completed.sessionToken }); harness.service.completeAccountDeletion({ @@ -182,12 +184,12 @@ describe("TDD-WP1-DEL-002-anonymous-retention", () => { }); expect(JSON.stringify(anonymous)).not.toContain(harness.email); expect(JSON.stringify(anonymous)).not.toContain(userId); - expect(JSON.stringify(anonymous)).not.toContain("private-target-fixture"); + expect(JSON.stringify(anonymous)).not.toContain(accessTargetRef); const access = harness.service.database.prepare("SELECT * FROM private_content_access_logs").get(); - expect(access).toMatchObject({ actor_ref: "admin-fixture", expires_at: accessExpiresAt }); + expect(access).toMatchObject({ actor_ref: accessActorRef, expires_at: accessExpiresAt }); expect(access.subject_ref).not.toBe(userId); - expect(access.target_ref).not.toBe("private-target-fixture"); + expect(access.target_ref).not.toBe(accessTargetRef); expect(access.subject_ref).not.toBe(anonymous[0].anonymous_subject_id); writeEvidence("DADA_EVIDENCE_DIR_RETENTION", "db-diff.json", { diff --git a/tests/integration/wp1-06-audit-immutability.test.ts b/tests/integration/wp1-06-audit-immutability.test.ts new file mode 100644 index 0000000..87049a2 --- /dev/null +++ b/tests/integration/wp1-06-audit-immutability.test.ts @@ -0,0 +1,136 @@ +import { randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { RegistrationService } from "../../apps/api/src/registration.js"; +import { MockResendAdapter } from "../../apps/api/src/resend-adapter.js"; +import { RetentionCleanup } from "../../apps/worker/src/retention-cleanup.js"; +import { registrationNotice } from "../../packages/shared-contracts/src/registration-notice.js"; + +const requireFromApi = createRequire(new URL("../../apps/api/package.json", import.meta.url)); +const Database = requireFromApi("better-sqlite3"); +const retentionMilliseconds = 180 * 24 * 60 * 60 * 1_000; +const baseNow = Date.now(); +const roots: string[] = []; +const services: RegistrationService[] = []; + +function createHarness() { + const root = mkdtempSync(join(tmpdir(), "dada-wp1-06-audit-")); + roots.push(root); + const databasePath = join(root, "dada.sqlite3"); + const service = new RegistrationService({ + challengePepper: Buffer.alloc(32, 0x41), clock: () => baseNow, + currentPrivacyNoticeVersion: registrationNotice.version, databasePath, + invitePepper: Buffer.alloc(32, 0x42), resend: new MockResendAdapter(), + sessionPepper: Buffer.alloc(32, 0x43), + }); + services.push(service); + return { databasePath, service }; +} + +function insertAuditPair(service: RegistrationService, occurredAt: number) { + const adminLogId = randomUUID(); + const privateLogId = randomUUID(); + const expiresAt = occurredAt + retentionMilliseconds; + service.database.prepare(` + INSERT INTO admin_operation_logs ( + log_id, actor_type, actor_ref, operation_type, target_type, target_ref, + result, before_summary, after_summary, occurred_at, expires_at + ) VALUES (?, 'system', 'backend_secure_config', 'secure_config_apply', + 'secure_config_revision', 'revision:fixture', 'succeeded', ?, ?, ?, ?) + `).run(adminLogId, JSON.stringify({ revision: 1 }), JSON.stringify({ revision: 2 }), occurredAt, expiresAt); + service.database.prepare(` + INSERT INTO private_content_access_logs ( + log_id, actor_ref, subject_ref, target_ref, content_type, occurred_at, expires_at + ) VALUES (?, ?, ?, ?, 'image', ?, ?) + `).run(privateLogId, randomUUID(), randomUUID(), randomUUID(), occurredAt, expiresAt); + return { adminLogId, expiresAt, privateLogId }; +} + +function writeEvidence(file: string, value: unknown) { + const directory = process.env.DADA_EVIDENCE_DIR_AUDIT; + if (!directory) return; + mkdirSync(directory, { recursive: true }); + writeFileSync(resolve(directory, file), `${JSON.stringify(value, null, 2)}\n`); +} + +afterEach(() => { + for (const service of services.splice(0)) service.close(); + for (const root of roots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); + +describe("TDD-WP1-AUD-001-immutable-triggers", () => { + it("blocks ordinary and forged-scope mutations but lets the Worker delete both audit classes at expiry", () => { + const { databasePath, service } = createHarness(); + const pair = insertAuditPair(service, baseNow); + const ordinary = new Database(databasePath); + ordinary.pragma("busy_timeout = 5000"); + try { + expect(() => ordinary.prepare("UPDATE admin_operation_logs SET result = 'failed' WHERE log_id = ?").run(pair.adminLogId)).toThrow(); + expect(() => ordinary.prepare("DELETE FROM admin_operation_logs WHERE log_id = ?").run(pair.adminLogId)).toThrow(); + expect(() => ordinary.prepare("UPDATE private_content_access_logs SET actor_ref = ? WHERE log_id = ?").run(randomUUID(), pair.privateLogId)).toThrow(); + expect(() => ordinary.prepare("DELETE FROM private_content_access_logs WHERE log_id = ?").run(pair.privateLogId)).toThrow(); + + ordinary.function("dada_allow_retention_purge", () => 1); + expect(() => ordinary.prepare("DELETE FROM private_content_access_logs WHERE log_id = ?").run(pair.privateLogId)).toThrow(); + ordinary.function("dada_allow_privacy_purge", () => 1); + expect(() => ordinary.prepare(` + UPDATE private_content_access_logs SET subject_ref = ?, target_ref = ? WHERE log_id = ? + `).run(randomUUID(), randomUUID(), pair.privateLogId)).toThrow(); + } finally { + ordinary.close(); + } + + const early = new RetentionCleanup({ clock: () => pair.expiresAt - 1, databasePath }); + expect(early.purgeExpired()).toEqual({ admin_operation_logs: 0, anonymous_events: 0, private_access_logs: 0 }); + early.close(); + const expired = new RetentionCleanup({ clock: () => pair.expiresAt, databasePath }); + expect(expired.purgeExpired()).toEqual({ admin_operation_logs: 1, anonymous_events: 0, private_access_logs: 1 }); + expired.close(); + + expect(service.database.prepare("SELECT COUNT(*) AS count FROM admin_operation_logs").get().count).toBe(0); + expect(service.database.prepare("SELECT COUNT(*) AS count FROM private_content_access_logs").get().count).toBe(0); + writeEvidence("sql-results.json", { + forged_scope_delete: "blocked", + forged_privacy_scope_update: "blocked", + ordinary_delete: "blocked", + ordinary_update: "blocked", + retention_delete_at_expiry: "allowed", + }); + writeEvidence("db-diff.json", { admin_operation_logs: 0, private_content_access_logs: 0 }); + }); + + it("enforces exact retention and rejects sensitive operation summaries", () => { + const { service } = createHarness(); + const badExpiry = baseNow + retentionMilliseconds + 1; + expect(() => service.database.prepare(` + INSERT INTO private_content_access_logs ( + log_id, actor_ref, subject_ref, target_ref, content_type, occurred_at, expires_at + ) VALUES (?, ?, ?, ?, 'prompt', ?, ?) + `).run(randomUUID(), randomUUID(), randomUUID(), randomUUID(), baseNow, badExpiry)).toThrow(); + expect(() => service.database.prepare(` + INSERT INTO admin_operation_logs ( + log_id, actor_type, actor_ref, operation_type, target_type, target_ref, + result, before_summary, after_summary, occurred_at, expires_at + ) VALUES (?, 'system', 'backend_secure_config', 'secure_config_apply', + 'secure_config_revision', 'revision:fixture', 'failed', ?, NULL, ?, ?) + `).run( + randomUUID(), + JSON.stringify({ relative_path: "private/fixture", session_token: "sensitive-fixture-token" }), + baseNow, + baseNow + retentionMilliseconds, + )).toThrow(); + expect(service.database.prepare("SELECT COUNT(*) AS count FROM admin_operation_logs").get().count).toBe(0); + expect(service.database.prepare("SELECT COUNT(*) AS count FROM private_content_access_logs").get().count).toBe(0); + + const triggers = service.database.prepare(` + SELECT name, sql FROM sqlite_master + WHERE type = 'trigger' AND name LIKE '%logs_%' ORDER BY name + `).all(); + writeEvidence("trigger-definition.json", { triggers }); + }); +}); diff --git a/tests/worker/wp1-05-retention-cleanup.test.ts b/tests/worker/wp1-05-retention-cleanup.test.ts index 48cf0f9..6a2ec7a 100644 --- a/tests/worker/wp1-05-retention-cleanup.test.ts +++ b/tests/worker/wp1-05-retention-cleanup.test.ts @@ -41,8 +41,8 @@ describe("TDD-WP1-DEL-002 retention worker", () => { const anonymousExpiry = baseNow + 2_000; service.database.prepare(` INSERT INTO private_content_access_logs (log_id, actor_ref, subject_ref, target_ref, content_type, occurred_at, expires_at) - VALUES (?, 'admin-fixture', ?, ?, 'prompt', ?, ?) - `).run(randomUUID(), randomUUID(), randomUUID(), baseNow - 10_000, accessExpiry); + VALUES (?, ?, ?, ?, 'prompt', ?, ?) + `).run(randomUUID(), randomUUID(), randomUUID(), randomUUID(), accessExpiry - 180 * 86_400_000, accessExpiry); service.database.prepare(` INSERT INTO anonymous_retained_events ( event_id, anonymous_subject_id, event_type, model_id, outcome, error_category, @@ -54,13 +54,13 @@ describe("TDD-WP1-DEL-002 retention worker", () => { expect(() => service.database.prepare("DELETE FROM private_content_access_logs").run()).toThrow(); const early = new RetentionCleanup({ clock: () => baseNow, databasePath }); - expect(early.purgeExpired()).toEqual({ anonymous_events: 0, private_access_logs: 0 }); + expect(early.purgeExpired()).toEqual({ admin_operation_logs: 0, anonymous_events: 0, private_access_logs: 0 }); early.close(); const firstExpiry = new RetentionCleanup({ clock: () => accessExpiry, databasePath }); - expect(firstExpiry.purgeExpired()).toEqual({ anonymous_events: 0, private_access_logs: 1 }); + expect(firstExpiry.purgeExpired()).toEqual({ admin_operation_logs: 0, anonymous_events: 0, private_access_logs: 1 }); firstExpiry.close(); const secondExpiry = new RetentionCleanup({ clock: () => anonymousExpiry, databasePath }); - expect(secondExpiry.purgeExpired()).toEqual({ anonymous_events: 1, private_access_logs: 0 }); + expect(secondExpiry.purgeExpired()).toEqual({ admin_operation_logs: 0, anonymous_events: 1, private_access_logs: 0 }); secondExpiry.close(); writeEvidence({