feat(P0-A): 整合第一版并冻结最终发布 #1

Open
tuyixuan wants to merge 115 commits from codex/wp7-07 into codex/wp0-09
6 changed files with 387 additions and 15 deletions
Showing only changes of commit 433ccdcf86 - Show all commits
+13
View File
@@ -260,6 +260,8 @@ export class GenerationSubmissionService {
const prompt = normalizePrompt(input.prompt);
const model = this.models.readModel(input.modelId);
if (!model) throw new GenerationSubmissionError("generation_blocked", { errorCategory: "model_disabled" });
const workerState = this.database.prepare("SELECT status FROM worker_runtime_state WHERE singleton = 1").get() as { status: "ready" | "degraded" } | undefined;
if (workerState?.status === "degraded") throw new GenerationSubmissionError("generation_blocked", { errorCategory: "gateway_contract_invalid" });
if (model.configVersion !== input.modelConfigVersion || model.creditCost !== input.confirmedCreditCost) {
throw new GenerationSubmissionError("model_config_stale", {
latest: { configVersion: model.configVersion, creditCost: model.creditCost, modelId: model.modelId },
@@ -501,6 +503,14 @@ export class GenerationSubmissionService {
this.ensureColumn("generation_jobs", "submission_ready", "INTEGER NOT NULL DEFAULT 0");
this.ensureColumn("generation_jobs", "config_snapshot_json", "TEXT");
this.database.exec(`
CREATE TABLE IF NOT EXISTS worker_runtime_state (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
status TEXT NOT NULL CHECK (status IN ('ready', 'degraded')),
reason TEXT,
owner_id TEXT,
lock_expires_at INTEGER,
updated_at INTEGER NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS generation_jobs_active_owner
ON generation_jobs(owner_id) WHERE status IN ('queued', 'running');
CREATE UNIQUE INDEX IF NOT EXISTS generation_jobs_client_submission
@@ -530,5 +540,8 @@ export class GenerationSubmissionService {
WHEN dada_allow_privacy_purge() <> 1
BEGIN SELECT RAISE(ABORT, 'generation_reference_snapshot_immutable'); END;
`);
this.ensureColumn("worker_runtime_state", "owner_id", "TEXT");
this.ensureColumn("worker_runtime_state", "lock_expires_at", "INTEGER");
this.database.prepare("INSERT OR IGNORE INTO worker_runtime_state (singleton, status, reason, owner_id, lock_expires_at, updated_at) VALUES (1, 'ready', NULL, NULL, NULL, ?)").run(this.clock());
}
}
+3
View File
@@ -18,6 +18,7 @@ export interface NormalizedGenerationOutput {
export type GenerationAdapterResult =
| { outputs: readonly NormalizedGenerationOutput[]; status: "completed" }
| { status: "pending"; upstreamJobReference: string }
| {
balanceSignal?: { gatewayAccountRef: string; impactScope: "model" | "account" | "unknown" };
category: GenerationErrorCategory;
@@ -27,6 +28,7 @@ export type GenerationAdapterResult =
export interface GenerationAdapter {
start(request: GenerationAdapterRequest): Promise<GenerationAdapterResult>;
poll?(upstreamJobReference: string): Promise<GenerationAdapterResult>;
}
type MockResult = GenerationAdapterResult & { unsafeRaw?: string };
@@ -41,6 +43,7 @@ export class MockGenerationAdapter implements GenerationAdapter {
async start(request: GenerationAdapterRequest): Promise<GenerationAdapterResult> {
this.calls.push({ generationId: request.generationId, modelId: request.modelId });
if (this.result.status === "pending") return { status: "pending", upstreamJobReference: this.result.upstreamJobReference };
if (this.result.status === "completed") {
return { outputs: this.result.outputs.map((output) => ({ ...output, bytes: Buffer.from(output.bytes) })), status: "completed" };
}
+172 -14
View File
@@ -27,6 +27,10 @@ interface JobRow {
prompt: string;
ratio: "3:4" | "1:1" | "4:3" | "9:16";
status: "queued" | "running" | "succeeded" | "failed" | "rejected";
heartbeat_at: number | null;
lease_expires_at: number | null;
lease_owner: string | null;
upstream_job_reference: string | null;
}
interface ReservationRow {
@@ -41,7 +45,12 @@ export interface GenerationProcessingResult {
generationId: string;
outputAssetId: string | null;
projectId: string;
status: "succeeded" | "failed" | "rejected";
status: "running" | "succeeded" | "failed" | "rejected";
}
export interface LeaseRecoveryAction {
action: "poll" | "unknown_retryable";
generationId: string;
}
function iso(timestamp: number) {
@@ -73,6 +82,8 @@ export class GenerationProcessor {
private readonly dataRoot: string;
private readonly gatewayBalance: GatewayBalanceRuntime;
private readonly workerId: string;
private closed = false;
private readonly lockTimer: ReturnType<typeof setInterval>;
constructor(input: { adapter: GenerationAdapter; clock?: () => number; dataRoot: string; databasePath: string; workerId: string }) {
if (!input.workerId) throw new Error("generation_worker_id_required");
@@ -84,18 +95,41 @@ export class GenerationProcessor {
configureWorkerDatabase(this.database);
this.migrate();
this.gatewayBalance = new GatewayBalanceRuntime({ clock: this.clock, database: this.database });
try {
this.setReady();
try {
this.recoverExpiredLeases();
} catch {
this.setDegraded("lease_recovery_failed");
}
this.lockTimer = setInterval(() => this.renewWorkerLock(), 10_000);
} catch (error) {
this.database.close();
throw error;
}
}
close() {
if (this.closed) return;
this.closed = true;
clearInterval(this.lockTimer);
this.immediate(() => {
const now = this.clock();
this.database.prepare("UPDATE worker_runtime_state SET status = 'degraded', reason = ?, owner_id = NULL, lock_expires_at = NULL, updated_at = ? WHERE singleton = 1 AND owner_id = ?")
.run("worker_stopped", now, this.workerId);
});
this.gatewayBalance.close();
this.database.close();
}
async processNext() {
if (this.readWorkerState().status === "degraded") return undefined;
this.recoverExpiredLeases();
const row = this.database.prepare(`
SELECT generation_id FROM generation_jobs
WHERE status = 'queued' AND submission_ready = 1
WHERE submission_ready = 1 AND (status = 'queued' OR (status = 'running' AND lease_owner = ?))
ORDER BY created_at, generation_id LIMIT 1
`).get() as { generation_id: string } | undefined;
`).get(this.workerId) as { generation_id: string } | undefined;
return row ? this.processGeneration(row.generation_id) : undefined;
}
@@ -110,19 +144,25 @@ export class GenerationProcessor {
`).all(generationId) as Array<{ managed_file_id: string }>;
let adapterResult: GenerationAdapterResult;
try {
adapterResult = await this.adapter.start({
configSnapshot: JSON.parse(job.config_snapshot_json) as Record<string, unknown>,
generationId,
modelId: job.model_id,
prompt: job.prompt,
ratio: job.ratio,
referenceAssetIds: references.map((row) => row.managed_file_id),
});
if (job.upstream_job_reference) {
if (!this.adapter.poll) return this.completeFailure(job, "unknown_retryable", "poll_unsupported", undefined, false, "pending_manual_review");
adapterResult = await this.withHeartbeat(generationId, () => this.adapter.poll!(job.upstream_job_reference!));
} else {
adapterResult = await this.withHeartbeat(generationId, () => this.adapter.start({
configSnapshot: JSON.parse(job.config_snapshot_json) as Record<string, unknown>,
generationId,
modelId: job.model_id,
prompt: job.prompt,
ratio: job.ratio,
referenceAssetIds: references.map((row) => row.managed_file_id),
}));
}
} catch {
adapterResult = { category: "unknown_retryable", sourceCategory: "adapter_exception", status: "failed" };
return this.completeFailure(job, "unknown_retryable", "adapter_exception", undefined, false, "pending_manual_review");
}
if (adapterResult.status === "failed") return this.completeFailure(job, adapterResult.category, adapterResult.sourceCategory, adapterResult.balanceSignal);
if (adapterResult.status === "pending") return this.persistPending(job.generation_id, adapterResult.upstreamJobReference);
const output = adapterResult.outputs.length === 1 ? adapterResult.outputs[0] : undefined;
if (!output || !this.isSafeOutput(job, output)) {
return this.completeFailure(job, adapterResult.outputs.length === 1 ? "unknown_retryable" : "gateway_contract_invalid", "output_validation_failed", undefined, true);
@@ -148,17 +188,123 @@ export class GenerationProcessor {
if (changed.changes !== 1) throw new Error("generation_claim_conflict");
return this.readJob(generationId);
}
if (row.status === "running" && row.final_credit_state === null) return row;
if (row.status === "running" && row.final_credit_state === null && row.lease_owner === this.workerId && (row.lease_expires_at ?? 0) >= this.clock()) return row;
if (row.status === "running" && row.final_credit_state === null) throw new Error("generation_claim_conflict");
return row;
});
}
private async withHeartbeat<T>(generationId: string, action: () => Promise<T>) {
const timer = setInterval(() => {
try {
this.heartbeat(generationId);
} catch {
this.setDegraded("heartbeat_failed");
}
}, 10_000);
try {
return await action();
} finally {
clearInterval(timer);
}
}
private renewWorkerLock() {
try {
const now = this.clock();
this.database.prepare("UPDATE worker_runtime_state SET lock_expires_at = ?, updated_at = ? WHERE singleton = 1 AND owner_id = ?")
.run(now + 30_000, now, this.workerId);
} catch {
if (!this.closed) this.setDegraded("worker_lock_renewal_failed");
}
}
heartbeat(generationId: string) {
return this.immediate(() => {
const now = this.clock();
const changed = this.database.prepare(`
UPDATE generation_jobs SET heartbeat_at = ?, lease_expires_at = ?, updated_at = ?
WHERE generation_id = ? AND status = 'running' AND lease_owner = ?
`).run(now, now + 30_000, now, generationId, this.workerId);
if (changed.changes !== 1) throw new Error("generation_heartbeat_conflict");
this.database.prepare("UPDATE worker_runtime_state SET lock_expires_at = ?, updated_at = ? WHERE singleton = 1 AND owner_id = ?")
.run(now + 30_000, now, this.workerId);
return { generationId, heartbeat_at: now, lease_expires_at: now + 30_000 };
});
}
recoverExpiredLeases(): LeaseRecoveryAction[] {
return this.immediate(() => {
const now = this.clock();
const rows = this.database.prepare(`
SELECT * FROM generation_jobs
WHERE status = 'running' AND submission_ready = 1 AND lease_expires_at IS NOT NULL AND lease_expires_at < ?
ORDER BY created_at, generation_id
`).all(now) as JobRow[];
const actions: LeaseRecoveryAction[] = [];
for (const row of rows) {
if (row.upstream_job_reference) {
this.database.prepare(`
UPDATE generation_jobs SET lease_owner = ?, lease_expires_at = ?, heartbeat_at = ?, updated_at = ?
WHERE generation_id = ? AND status = 'running' AND lease_expires_at < ?
`).run(this.workerId, now + 30_000, now, now, row.generation_id, now);
actions.push({ action: "poll", generationId: row.generation_id });
} else {
this.completeFailure(row, "unknown_retryable", "lease_expired_without_reference", undefined, false, "pending_manual_review");
actions.push({ action: "unknown_retryable", generationId: row.generation_id });
}
}
return actions;
});
}
setDegraded(reason: string) {
return this.immediate(() => {
const now = this.clock();
this.database.prepare("UPDATE worker_runtime_state SET status = 'degraded', reason = ?, updated_at = ? WHERE singleton = 1").run(reason, now);
return this.readWorkerState();
});
}
setReady() {
return this.immediate(() => {
const now = this.clock();
const current = this.database.prepare("SELECT owner_id, lock_expires_at FROM worker_runtime_state WHERE singleton = 1")
.get() as { lock_expires_at: number | null; owner_id: string | null } | undefined;
if (current?.owner_id && current.owner_id !== this.workerId && (current.lock_expires_at ?? 0) >= now) throw new Error("worker_instance_lock_conflict");
this.database.prepare("UPDATE worker_runtime_state SET status = 'ready', reason = NULL, owner_id = ?, lock_expires_at = ?, updated_at = ? WHERE singleton = 1")
.run(this.workerId, now + 30_000, now);
return this.readWorkerState();
});
}
private readWorkerState() {
return this.database.prepare("SELECT status, reason, updated_at FROM worker_runtime_state WHERE singleton = 1").get() as { reason: string | null; status: "ready" | "degraded"; updated_at: number };
}
private persistPending(generationId: string, upstreamJobReference: string): GenerationProcessingResult {
return this.immediate(() => {
if (!/^[A-Za-z0-9][A-Za-z0-9:_.-]{0,159}$/.test(upstreamJobReference)) {
const job = this.readJob(generationId);
return this.completeFailure(job, "gateway_contract_invalid", "upstream_reference_invalid");
}
const now = this.clock();
this.database.prepare(`
UPDATE generation_jobs SET upstream_job_reference = ?, heartbeat_at = ?, lease_expires_at = ?, updated_at = ?
WHERE generation_id = ? AND status = 'running' AND lease_owner = ?
`).run(upstreamJobReference, now, now + 30_000, now, generationId, this.workerId);
const job = this.readJob(generationId);
return { category: null, generationId, outputAssetId: null, projectId: job.project_id, status: "running" };
});
}
private completeFailure(
job: JobRow,
category: GenerationErrorCategory,
sourceCategory: string,
balanceSignal?: { gatewayAccountRef: string; impactScope: "model" | "account" | "unknown" },
upstreamOutcomeKnown = false,
reconciliation: "not_required" | "pending_manual_review" = upstreamOutcomeKnown ? "pending_manual_review" : "not_required",
) {
return this.immediate(() => {
const replay = this.readReceipt(job.generation_id);
@@ -189,7 +335,7 @@ export class GenerationProcessor {
category,
safeSourceCategory.test(sourceCategory) ? sourceCategory : "adapter_error",
upstreamOutcomeKnown ? 1 : 0,
upstreamOutcomeKnown ? "pending_manual_review" : "not_required",
reconciliation,
now,
now,
job.generation_id,
@@ -418,7 +564,16 @@ export class GenerationProcessor {
this.ensureColumn("generation_jobs", "diagnostic_source_category", "TEXT");
this.ensureColumn("generation_jobs", "upstream_outcome_known", "INTEGER NOT NULL DEFAULT 0");
this.ensureColumn("generation_jobs", "upstream_cost_reconciliation", "TEXT NOT NULL DEFAULT 'not_applicable'");
this.ensureColumn("generation_jobs", "upstream_job_reference", "TEXT");
this.database.exec(`
CREATE TABLE IF NOT EXISTS worker_runtime_state (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
status TEXT NOT NULL CHECK (status IN ('ready', 'degraded')),
reason TEXT,
owner_id TEXT,
lock_expires_at INTEGER,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS generation_output_assets (
generation_id TEXT PRIMARY KEY REFERENCES generation_jobs(generation_id),
managed_file_id TEXT NOT NULL UNIQUE REFERENCES managed_files(file_id),
@@ -432,5 +587,8 @@ export class GenerationProcessor {
created_at INTEGER NOT NULL
);
`);
this.ensureColumn("worker_runtime_state", "owner_id", "TEXT");
this.ensureColumn("worker_runtime_state", "lock_expires_at", "INTEGER");
this.database.prepare("INSERT OR IGNORE INTO worker_runtime_state (singleton, status, reason, owner_id, lock_expires_at, updated_at) VALUES (1, 'ready', NULL, NULL, NULL, ?)").run(this.clock());
}
}
+3 -1
View File
@@ -71,7 +71,9 @@
"test:wp3-02": "node scripts/run-wp3-02-validation.mjs",
"test:wp3-02:red": "node scripts/run-wp3-02-validation.mjs --phase red",
"test:wp3-03": "node scripts/run-wp3-03-validation.mjs",
"test:wp3-03:red": "node scripts/run-wp3-03-validation.mjs --phase red"
"test:wp3-03:red": "node scripts/run-wp3-03-validation.mjs --phase red",
"test:wp3-04": "node scripts/run-wp3-04-validation.mjs",
"test:wp3-04:red": "node scripts/run-wp3-04-validation.mjs --phase red"
},
"devDependencies": {
"@playwright/test": "1.62.0",
+52
View File
@@ -0,0 +1,52 @@
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
const phaseIndex = process.argv.indexOf("--phase");
const phase = phaseIndex >= 0 ? process.argv[phaseIndex + 1] : "green";
if (!["red", "green"].includes(phase)) throw new Error(`Unsupported phase: ${phase}`);
const runId = process.env.DADA_TDD_RUN_ID ?? `wp3-04-${phase}-${new Date().toISOString().replace(/[^0-9]/g, "")}`;
const runDirectory = resolve("artifacts", "tdd", runId);
const caseDirectory = resolve(runDirectory, "cases", "TDD-WP3-WRK-001-recovery");
if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`);
mkdirSync(caseDirectory, { recursive: true });
const commands = phase === "red"
? []
: [["integration", ["test:integration"]], ["worker", ["test:worker"]], ["tdd-trace", ["validate:tdd-trace"]]];
const environment = { ...process.env, DADA_EVIDENCE_DIR_WORKER_RECOVERY: resolve(runDirectory, "cases") };
const commandResults = [];
for (const [name, args] of commands) {
const command = `pnpm ${args.join(" ")}`;
const started_at = new Date().toISOString();
const result = spawnSync(process.env.ComSpec ?? "cmd.exe", ["/d", "/s", "/c", command], { encoding: "utf8", env: environment });
if (result.stdout) process.stdout.write(result.stdout);
if (result.stderr) process.stderr.write(result.stderr);
commandResults.push({ command, exit_code: result.status ?? 1, finished_at: new Date().toISOString(), name, started_at });
}
const commandState = phase === "red" ? true : commandResults.every((result) => result.exit_code === 0);
const manifest = { path: "tasks.manifest.json", sha256: createHash("sha256").update(readFileSync("tasks.manifest.json")).digest("hex").toString().toUpperCase() };
const commit = spawnSync("git", ["rev-parse", "--short", "HEAD"], { encoding: "utf8" }).stdout.trim();
const dirty = spawnSync("git", ["status", "--porcelain"], { encoding: "utf8" }).stdout.trim().length > 0;
const evidenceRefs = phase === "red" ? ["red-observation.json"] : ["worker-events.json", "db-diff.json", "external-calls.json"];
if (phase === "red") writeFileSync(resolve(caseDirectory, "red-observation.json"), `${JSON.stringify({
expected_failure: "Worker pending/poll recovery, lease persistence, unknown-result reconciliation and degraded gate were absent before TASK-WP3-04",
observed_commands: ["pnpm vitest run tests/worker/wp3-04-worker-recovery.test.ts"],
observed_errors: ["Cannot read properties of undefined (reading 'length')", "no such column: upstream_job_reference", "setDegraded is not a function"],
status: "red_confirmed",
}, null, 2)}\n`);
const missingEvidence = evidenceRefs.filter((file) => !existsSync(resolve(caseDirectory, file)));
const status = commandState && missingEvidence.length === 0 ? phase === "red" ? "red_confirmed" : "passed" : "failed";
writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands: commandResults, phase, run_id: runId }, null, 2)}\n`);
writeFileSync(resolve(caseDirectory, "result.json"), `${JSON.stringify({
acceptance_criteria: ["AC-04", "AC-51"], automation: ["automated"], commit, evidence_refs: evidenceRefs,
manifest, missing_evidence: missingEvidence, phase, requirements: ["CREDIT-03", "GEN-08", "GEN-09"],
run_id: runId, status, task_id: "TASK-WP3-04", test_id: "TDD-WP3-WRK-001-recovery", work_package: "WP-3",
worktree_under_test: dirty ? "uncommitted implementation" : "clean committed implementation",
}, null, 2)}\n`);
writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify({
cases: [{ missing_evidence: missingEvidence, status, test_id: "TDD-WP3-WRK-001-recovery" }], phase, run_id: runId, status,
}, null, 2)}\n`);
console.log(JSON.stringify({ cases: [{ missing_evidence: missingEvidence, status, test_id: "TDD-WP3-WRK-001-recovery" }], phase, run_id: runId, status }, null, 2));
if (status === "failed") process.exit(1);
+144
View File
@@ -0,0 +1,144 @@
import { randomUUID } from "node:crypto";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { CreditService } from "../../apps/api/src/credits.js";
import { GenerationSubmissionService, StaticGenerationModelCatalog } from "../../apps/api/src/generation-submission.js";
import { ManagedStorage } from "../../apps/api/src/managed-storage.js";
import { ProjectService } from "../../apps/api/src/projects.js";
import { RegistrationService } from "../../apps/api/src/registration.js";
import { MockResendAdapter } from "../../apps/api/src/resend-adapter.js";
import type { GenerationAdapter, GenerationAdapterResult } from "../../apps/worker/src/ai-adapter-contract.js";
import { GenerationProcessor } from "../../apps/worker/src/generation-processor.js";
const now = Date.parse("2026-08-02T17:00:00.000Z");
const modelId = "gemini-3-pro-image-preview";
const png = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.alloc(96, 0x43)]);
const roots: string[] = [];
const closeables: Array<{ close(): void }> = [];
function evidence(file: string, value: unknown) {
const root = process.env.DADA_EVIDENCE_DIR_WORKER_RECOVERY;
if (!root) return;
const directory = resolve(root, "TDD-WP3-WRK-001-recovery");
mkdirSync(directory, { recursive: true });
writeFileSync(resolve(directory, file), `${JSON.stringify(value, null, 2)}\n`);
}
async function harness(adapter: GenerationAdapter) {
const dataRoot = mkdtempSync(join(tmpdir(), "dada-wp3-04-worker-"));
roots.push(dataRoot);
mkdirSync(join(dataRoot, "db"), { recursive: true });
const databasePath = join(dataRoot, "db", "dada.sqlite3");
const registration = new RegistrationService({
challengePepper: Buffer.alloc(32, 0x41), clock: () => now, currentPrivacyNoticeVersion: "p0a-registration-notice-v1",
databasePath, invitePepper: Buffer.alloc(32, 0x42), resend: new MockResendAdapter(), sessionPepper: Buffer.alloc(32, 0x43),
});
const projects = new ProjectService({ clock: () => now, databasePath });
const credits = new CreditService({ clock: () => now, databasePath });
const storage = new ManagedStorage({ dataRoot, databasePath });
const submissions = new GenerationSubmissionService({
clock: () => now, credits,
models: new StaticGenerationModelCatalog([{
configSetVersion: 1, configVersion: 1, contractValidationStatus: "verified", creditCost: 1, enabled: true, modelId,
promptMaxLength: 1_000, referenceLimits: { maxFileBytes: 1_024, maxFiles: 2, maxTotalBytes: 2_048 },
runtimeAvailability: { availableForNewJobs: true, reason: null }, supportedRatios: ["3:4"],
}]),
storage,
});
closeables.push(submissions, storage, credits, projects, registration);
const userId = randomUUID();
registration.database.prepare("INSERT INTO users (user_id, normalized_email, role, status, counts_toward_stage_limit, registration_id, created_at) VALUES (?, ?, 'user', 'active', 1, ?, ?)")
.run(userId, `${userId}@example.invalid`, randomUUID(), now);
registration.database.prepare("INSERT INTO user_profiles (user_id, creator_name, social_id, private_content_notice_version, private_content_notice_acknowledged_at) VALUES (?, 'Worker User', '@worker', NULL, NULL)")
.run(userId);
registration.database.prepare("INSERT INTO credit_accounts (user_id, available_balance, reserved_balance, updated_at) VALUES (?, 10, 0, ?)")
.run(userId, now);
const submitted = await submissions.submit({
clientSubmissionId: randomUUID(), confirmedCreditCost: 1, existingReferenceAssetIds: [],
idempotencyKey: `generation-${randomUUID()}-${randomUUID()}`, mode: "new_project", modelConfigVersion: 1,
modelId, newReferences: [], prompt: "Worker 崩溃恢复", ratio: "3:4", userId,
});
const processor = new GenerationProcessor({ adapter, clock: () => now + 1_000, dataRoot, databasePath, workerId: "worker-first" });
closeables.push(processor);
return { adapter, dataRoot, databasePath, generationId: submitted.task.generationId, processor, registration, submissions, userId };
}
function completed(): GenerationAdapterResult {
return { outputs: [{ bytes: png, mimeType: "image/png", pixelHeight: 1440, pixelWidth: 1080 }], status: "completed" };
}
afterEach(() => {
for (const value of closeables.splice(0).reverse()) value.close();
for (const root of roots.splice(0)) rmSync(root, { force: true, recursive: true });
});
describe("TDD-WP3-WRK-001-recovery", () => {
it("renews a lease, persists an async reference, and polls after restart without resubmitting", async () => {
let starts = 0;
let polls = 0;
const adapter: GenerationAdapter = {
async start() { starts += 1; return { status: "pending", upstreamJobReference: "upstream:async-1" }; },
async poll() { polls += 1; return completed(); },
};
const fixture = await harness(adapter);
expect(() => new GenerationProcessor({ adapter, clock: () => now + 1_000, dataRoot: fixture.dataRoot, databasePath: fixture.databasePath, workerId: "worker-second" }))
.toThrow("worker_instance_lock_conflict");
const pending = await fixture.processor.processNext();
expect(pending).toMatchObject({ generationId: fixture.generationId, status: "running" });
expect(fixture.registration.database.prepare("SELECT upstream_job_reference, lease_owner FROM generation_jobs WHERE generation_id = ?").get(fixture.generationId))
.toEqual({ lease_owner: "worker-first", upstream_job_reference: "upstream:async-1" });
const heartbeat = fixture.processor.heartbeat(fixture.generationId);
expect(heartbeat).toMatchObject({ heartbeat_at: now + 1_000, lease_expires_at: now + 31_000 });
fixture.registration.database.prepare("UPDATE generation_jobs SET lease_expires_at = ? WHERE generation_id = ?").run(now - 1, fixture.generationId);
fixture.processor.close();
closeables.splice(closeables.indexOf(fixture.processor), 1);
const restarted = new GenerationProcessor({ adapter, clock: () => now + 2_000, dataRoot: fixture.dataRoot, databasePath: fixture.databasePath, workerId: "worker-restart" });
closeables.push(restarted);
expect(fixture.registration.database.prepare("SELECT lease_owner, upstream_job_reference FROM generation_jobs WHERE generation_id = ?").get(fixture.generationId))
.toEqual({ lease_owner: "worker-restart", upstream_job_reference: "upstream:async-1" });
const result = await restarted.processGeneration(fixture.generationId);
expect(result).toMatchObject({ status: "succeeded" });
expect({ starts, polls }).toEqual({ polls: 1, starts: 1 });
evidence("worker-events.json", { starts, polls, recovery_action: "poll", upstream_job_reference: "upstream:async-1" });
evidence("db-diff.json", { lease_owner_after_recovery: "worker-restart", final_status: result.status, upstream_outcome_known: true, upstream_cost_reconciliation: "not_required" });
evidence("external-calls.json", { poll_calls: 1, start_calls: 1, duplicate_submit_calls: 0 });
});
it("fails an unknown result without a safe reference and releases exactly once for manual reconciliation", async () => {
const adapter: GenerationAdapter = { async start() { throw new Error("provider_timeout_without_reference"); } };
const fixture = await harness(adapter);
const first = await fixture.processor.processNext();
const replay = await fixture.processor.processGeneration(fixture.generationId);
expect(first).toMatchObject({ category: "unknown_retryable", status: "failed" });
expect(replay).toEqual(first);
expect(fixture.registration.database.prepare("SELECT upstream_job_reference, upstream_outcome_known, upstream_cost_reconciliation, lease_owner FROM generation_jobs WHERE generation_id = ?").get(fixture.generationId))
.toEqual({ lease_owner: null, upstream_job_reference: null, upstream_outcome_known: 0, upstream_cost_reconciliation: "pending_manual_review" });
expect(fixture.registration.database.prepare("SELECT COUNT(*) AS count FROM credit_ledger WHERE reference_id = ? AND entry_type = 'generation_release'").get(fixture.generationId)).toEqual({ count: 1 });
evidence("worker-events.json", { starts: 1, polls: 0, duplicate_submit_calls: 0, unknown_result: true });
evidence("db-diff.json", { final_status: "failed", category: "unknown_retryable", upstream_outcome_known: false, upstream_cost_reconciliation: "pending_manual_review", release_count: 1 });
evidence("external-calls.json", { start_calls: 1, poll_calls: 0, duplicate_submit_calls: 0 });
});
it("blocks new submissions while worker is degraded and reopens the gate when ready", async () => {
const fixture = await harness({ async start() { return completed(); } });
fixture.processor.setDegraded("lease_recovery_failed");
const degradedUserId = randomUUID();
fixture.registration.database.prepare("INSERT INTO users (user_id, normalized_email, role, status, counts_toward_stage_limit, registration_id, created_at) VALUES (?, ?, 'user', 'active', 1, ?, ?)")
.run(degradedUserId, `${degradedUserId}@example.invalid`, randomUUID(), now);
fixture.registration.database.prepare("INSERT INTO user_profiles (user_id, creator_name, social_id, private_content_notice_version, private_content_notice_acknowledged_at) VALUES (?, 'Degraded User', '@degraded', NULL, NULL)")
.run(degradedUserId);
fixture.registration.database.prepare("INSERT INTO credit_accounts (user_id, available_balance, reserved_balance, updated_at) VALUES (?, 10, 0, ?)")
.run(degradedUserId, now);
await expect(fixture.submissions.submit({
clientSubmissionId: randomUUID(), confirmedCreditCost: 1, existingReferenceAssetIds: [],
idempotencyKey: `generation-${randomUUID()}-${randomUUID()}`, mode: "new_project", modelConfigVersion: 1,
modelId, newReferences: [], prompt: "blocked while degraded", ratio: "3:4", userId: degradedUserId,
})).rejects.toMatchObject({ code: "generation_blocked", errorCategory: "gateway_contract_invalid" });
expect(fixture.processor.setReady()).toMatchObject({ status: "ready" });
expect(fixture.registration.database.prepare("SELECT status FROM worker_runtime_state WHERE singleton = 1").get()).toEqual({ status: "ready" });
});
});