From cca0a38adaedf993aa7d5a36b0b782e91bbe7267 Mon Sep 17 00:00:00 2001 From: suyx Date: Tue, 4 Aug 2026 13:51:02 +0800 Subject: [PATCH] feat: add Resend release gate evidence validation (TASK-WP7-03) --- package.json | 4 +- scripts/lib/resend-release-gate.mjs | 145 ++++++++++++++++++ scripts/run-wp7-03-validation.mjs | 175 ++++++++++++++++++++++ tests/package/wp7-03-resend-gate.test.mjs | 76 ++++++++++ 4 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 scripts/lib/resend-release-gate.mjs create mode 100644 scripts/run-wp7-03-validation.mjs create mode 100644 tests/package/wp7-03-resend-gate.test.mjs diff --git a/package.json b/package.json index 554c8f9..6fbf378 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,9 @@ "test:wp6-04:red": "node scripts/run-wp6-04-validation.mjs --phase red", "test:wp6-05": "pnpm exec vitest run tests/api/wp6-05-state.test.ts && pnpm exec playwright test tests/e2e/wp6-05-state.spec.ts --config playwright.config.ts", "test:wp7-01": "node scripts/run-wp7-01-validation.mjs", - "review:wp7-01": "node scripts/record-wp7-01-manual-review.mjs" + "review:wp7-01": "node scripts/record-wp7-01-manual-review.mjs", + "test:wp7-03": "node scripts/run-wp7-03-validation.mjs --phase green", + "test:wp7-03:red": "node scripts/run-wp7-03-validation.mjs --phase red" }, "devDependencies": { "@playwright/test": "1.62.0", diff --git a/scripts/lib/resend-release-gate.mjs b/scripts/lib/resend-release-gate.mjs new file mode 100644 index 0000000..232c71e --- /dev/null +++ b/scripts/lib/resend-release-gate.mjs @@ -0,0 +1,145 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; + +import { validateReleaseCandidateRecord } from "./release-candidate.mjs"; + +export const RESEND_DAILY_LIMIT = 80; +export const RESEND_MONTHLY_LIMIT = 2_400; +export const DELIVERY_CATEGORIES = Object.freeze(["qq", "163", "enterprise"]); +export const DELIVERY_SAMPLE_SIZE = 20; +export const DELIVERY_MINIMUM_WITHIN_TWO_MINUTES = 19; +export const DELIVERY_WINDOW_SECONDS = 120; +export const EXPECTED_WP7_01_COMMIT = "623cad25b2a2a9a003502c9a92ebd318dad06248"; + +const emailPattern = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i; +const absolutePathPattern = /(?:[A-Z]:[\\/]|\\\\|\/Users\/|\/home\/)/i; +const sensitiveKeyPattern = /"(?:api[_ -]?key|secret|password|authorization|bearer|cookie|session[_ -]?token|verification[_ -]?code|private[_ -]?content|prompt|image)"\s*:/i; + +function object(value) { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value : undefined; +} + +function errorList(...values) { + return [...new Set(values.flat().filter((value) => typeof value === "string" && value.length > 0))]; +} + +export function validateDomainCheck(value) { + const item = object(value); + const freeRules = object(item?.free_rules); + const spf = object(item?.spf); + const dkim = object(item?.dkim); + const errors = []; + if (item?.schema_version !== "1.0") errors.push("schema_version"); + if (item?.service !== "resend") errors.push("service"); + if (item?.source !== "human_controlled_real") errors.push("source"); + if (item?.status !== "verified") errors.push("status"); + if (item?.domain_controlled !== true) errors.push("domain_controlled"); + if (spf?.status !== "pass") errors.push("spf"); + if (dkim?.status !== "pass") errors.push("dkim"); + if (freeRules?.status !== "verified") errors.push("free_rules.status"); + if (freeRules?.daily_limit !== RESEND_DAILY_LIMIT) errors.push("free_rules.daily_limit"); + if (freeRules?.monthly_limit !== RESEND_MONTHLY_LIMIT) errors.push("free_rules.monthly_limit"); + if (freeRules?.paid_fallback_enabled !== false) errors.push("free_rules.paid_fallback_enabled"); + return errorList(errors); +} + +export function validateDeliverySummary(value) { + const item = object(value); + const errors = []; + if (item?.schema_version !== "1.0") errors.push("schema_version"); + if (item?.service !== "resend") errors.push("service"); + if (item?.source !== "human_controlled_real") errors.push("source"); + if (item?.status !== "verified") errors.push("status"); + if (!Array.isArray(item?.categories) || item.categories.length !== DELIVERY_CATEGORIES.length) { + errors.push("categories"); + } else { + const categories = item.categories.map((entry) => entry?.category).sort(); + if (categories.join("|") !== DELIVERY_CATEGORIES.slice().sort().join("|")) errors.push("categories.names"); + for (const entry of item.categories) { + if (!Number.isInteger(entry?.sent_count) || entry.sent_count !== DELIVERY_SAMPLE_SIZE) errors.push(`${entry?.category ?? "unknown"}.sent_count`); + if (!Number.isInteger(entry?.delivered_within_120_seconds) + || entry.delivered_within_120_seconds < DELIVERY_MINIMUM_WITHIN_TWO_MINUTES + || entry.delivered_within_120_seconds > DELIVERY_SAMPLE_SIZE) { + errors.push(`${entry?.category ?? "unknown"}.delivered_within_120_seconds`); + } + if (!Number.isFinite(entry?.max_latency_seconds) || entry.max_latency_seconds > DELIVERY_WINDOW_SECONDS) errors.push(`${entry?.category ?? "unknown"}.max_latency_seconds`); + if (entry?.mock_used !== false) errors.push(`${entry?.category ?? "unknown"}.mock_used`); + if (entry?.preseeded_account_used !== false) errors.push(`${entry?.category ?? "unknown"}.preseeded_account_used`); + } + } + return errorList(errors); +} + +export function validateAuthResult(value) { + const item = object(value); + const ordinary = object(item?.ordinary); + const admin = object(item?.admin); + const errors = []; + if (item?.schema_version !== "1.0") errors.push("schema_version"); + if (item?.service !== "resend") errors.push("service"); + if (item?.source !== "human_controlled_real") errors.push("source"); + if (item?.status !== "verified") errors.push("status"); + if (item?.mock_used !== false) errors.push("mock_used"); + if (item?.preseeded_account_used !== false) errors.push("preseeded_account_used"); + for (const [name, auth] of [["ordinary", ordinary], ["admin", admin]]) { + if (auth?.status !== "passed") errors.push(`${name}.status`); + if (auth?.chain !== "formal") errors.push(`${name}.chain`); + if (auth?.verification_code_source !== "real_delivery") errors.push(`${name}.verification_code_source`); + } + return errorList(errors); +} + +export function validateRedaction(value, serializedEvidence = "") { + const item = object(value); + const errors = []; + if (item?.schema_version !== "1.0") errors.push("schema_version"); + if (item?.status !== "passed") errors.push("status"); + if (item?.forbidden_matches !== 0) errors.push("forbidden_matches"); + if (item?.credentials_in_evidence !== false) errors.push("credentials_in_evidence"); + if (item?.mailboxes_in_evidence !== false) errors.push("mailboxes_in_evidence"); + if (item?.private_content_in_evidence !== false) errors.push("private_content_in_evidence"); + if (item?.absolute_paths_in_evidence !== false) errors.push("absolute_paths_in_evidence"); + if (emailPattern.test(serializedEvidence)) errors.push("email_value"); + if (absolutePathPattern.test(serializedEvidence)) errors.push("absolute_path"); + if (sensitiveKeyPattern.test(serializedEvidence)) errors.push("sensitive_value"); + return errorList(errors); +} + +export function validateCandidateReference(record) { + try { + validateReleaseCandidateRecord(record); + } catch (error) { + return [error instanceof Error ? "candidate_record_invalid" : "candidate_record_invalid"]; + } + const errors = []; + if (record.build_commit !== EXPECTED_WP7_01_COMMIT) errors.push("candidate_build_commit"); + const versions = new Map((record.browsers ?? []).map((browser) => [browser.brand, browser.full_version])); + if (versions.get("Google Chrome") !== "150.0.7871.187") errors.push("chrome_full_version"); + if (versions.get("Microsoft Edge") !== "151.0.4129.59") errors.push("edge_full_version"); + return errorList(errors); +} + +export function readJson(path) { + if (!path || !existsSync(path)) return undefined; + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return undefined; + } +} + +export function fileSha256(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex").toUpperCase(); +} + +export function validateResendEvidence({ candidate, domainCheck, deliverySummary, authResult, redaction }) { + const serializedEvidence = JSON.stringify({ domainCheck, deliverySummary, authResult }); + const errors = [ + ...validateCandidateReference(candidate), + ...validateDomainCheck(domainCheck), + ...validateDeliverySummary(deliverySummary), + ...validateAuthResult(authResult), + ...validateRedaction(redaction, serializedEvidence), + ]; + return errorList(errors); +} diff --git a/scripts/run-wp7-03-validation.mjs b/scripts/run-wp7-03-validation.mjs new file mode 100644 index 0000000..efeb440 --- /dev/null +++ b/scripts/run-wp7-03-validation.mjs @@ -0,0 +1,175 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +import { + fileSha256, + readJson, + validateCandidateReference, + validateResendEvidence, +} from "./lib/resend-release-gate.mjs"; + +const phaseIndex = process.argv.indexOf("--phase"); +const phase = phaseIndex >= 0 ? process.argv[phaseIndex + 1] : "green"; +if (!new Set(["red", "green"]).has(phase)) throw new Error(`Unsupported phase: ${phase}`); + +const candidateRunId = process.env.DADA_WP7_01_RUN_ID ?? "wp7-01-candidate-20260804052447717"; +const runId = process.env.DADA_TDD_RUN_ID ?? `wp7-03-${phase}-${new Date().toISOString().replace(/[^0-9]/g, "")}`; +const runDirectory = resolve("artifacts", "tdd", runId); +const caseId = "TDD-WP7-EXT-002-real-resend"; +const caseDirectory = resolve(runDirectory, "cases", caseId); +if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`); +mkdirSync(caseDirectory, { recursive: true }); + +function writeJson(path, value) { + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function runCommand(name, command) { + const startedAt = new Date().toISOString(); + const result = spawnSync(process.env.ComSpec ?? "cmd.exe", ["/d", "/s", "/c", command], { + encoding: "utf8", + env: process.env, + maxBuffer: 40 * 1024 * 1024, + stdio: "inherit", + }); + return { + command, + exit_code: result.status ?? 1, + finished_at: new Date().toISOString(), + name, + started_at: startedAt, + }; +} + +const candidateEvidenceRoot = resolve( + process.env.DADA_WP7_01_EVIDENCE_DIR ?? join("artifacts", "tdd", candidateRunId), +); +const candidate = readJson(join(candidateEvidenceRoot, "release-candidate.json")); +const candidateErrors = candidate ? validateCandidateReference(candidate) : ["candidate_record_missing"]; +writeJson(resolve(caseDirectory, "candidate-reference.json"), candidate ? { + browsers: candidate.browsers.map(({ brand, full_version: fullVersion, major }) => ({ brand, full_version: fullVersion, major })), + build_commit: candidate.build_commit, + candidate_status: candidate.status, + final_release: candidate.final_release, + fixed_port: candidate.fixed_port, + run_id: candidateRunId, + schema_version: "1.0", +} : { + candidate_status: "not_available", + reason: "candidate_record_missing", + run_id: candidateRunId, + schema_version: "1.0", +}); + +const commands = phase === "red" ? [] : [ + ["workspace-build", "pnpm build:workspace-packages"], + ["integration", "pnpm test:integration"], + ["api", "pnpm check:openapi && pnpm exec vitest run tests/api --maxWorkers=1"], + ["security", "pnpm test:security"], + ["tdd-trace", "pnpm validate:tdd-trace"], +].map(([name, command]) => runCommand(name, command)); +const localGatePassed = phase === "red" || commands.length > 0 && commands.every(({ exit_code }) => exit_code === 0); + +const redaction = { + absolute_paths_in_evidence: false, + credentials_in_evidence: false, + forbidden_matches: 0, + mailboxes_in_evidence: false, + private_content_in_evidence: false, + schema_version: "1.0", + status: "passed", +}; +const externalRoot = process.env.DADA_RESEND_EVIDENCE_DIR ? resolve(process.env.DADA_RESEND_EVIDENCE_DIR) : undefined; +const realAuthorized = process.env.DADA_RESEND_REAL_AUTHORIZED === "1"; +const externalFiles = ["domain-check.json", "delivery-summary.json", "auth-result.json", "redaction.json"]; +const externalEvidence = externalRoot + ? Object.fromEntries(externalFiles.map((file) => [file, readJson(join(externalRoot, file))])) + : {}; +const externalEvidenceMissing = externalRoot ? externalFiles.filter((file) => !externalEvidence[file]) : externalFiles; +const externalErrors = externalRoot && externalEvidenceMissing.length === 0 && candidate + ? validateResendEvidence({ + authResult: externalEvidence["auth-result.json"], + candidate, + deliverySummary: externalEvidence["delivery-summary.json"], + domainCheck: externalEvidence["domain-check.json"], + redaction: externalEvidence["redaction.json"], + }) + : []; + +const externalBlockers = []; +if (candidateErrors.length > 0) externalBlockers.push("candidate_record_unavailable_or_drifted"); +if (!realAuthorized) externalBlockers.push("controlled_domain_or_real_mailbox_authorization_absent"); +if (!externalRoot) externalBlockers.push("real_resend_evidence_not_supplied"); +if (externalRoot && externalEvidenceMissing.length > 0) externalBlockers.push("real_resend_evidence_incomplete"); +if (externalErrors.length > 0) externalBlockers.push("real_resend_evidence_invalid"); + +if (phase === "red") { + writeJson(resolve(caseDirectory, "red-observation.json"), { + expected_failure: "Resend SPF/DKIM, free-rule, delivery, and formal authentication evidence is absent before TASK-WP7-03.", + observed_commands: ["pnpm test:wp7-03:red"], + red_reason: "TDD-WP7-EXT-002 requires controlled real domain/mailbox evidence; mock or pre-seeded accounts are not release evidence.", + status: "red_confirmed", + }); +} else if (externalRoot && externalEvidenceMissing.length === 0 && candidateErrors.length === 0 && realAuthorized && externalErrors.length === 0) { + for (const file of externalFiles) writeJson(resolve(caseDirectory, file), externalEvidence[file]); + writeJson(resolve(caseDirectory, "source-hashes.json"), { + files: Object.fromEntries(externalFiles.map((file) => [file, fileSha256(join(externalRoot, file))])), + schema_version: "1.0", + }); +} else { + writeJson(resolve(caseDirectory, "blocker.json"), { + blockers: externalErrors.length > 0 ? ["real_resend_evidence_invalid"] : externalBlockers, + mock_accepted_as_evidence: false, + paid_fallback_enabled: false, + preseeded_accounts_accepted_as_evidence: false, + real_calls_started: false, + schema_version: "1.0", + status: "externally_blocked", + }); + writeJson(resolve(caseDirectory, "redaction.json"), redaction); + writeJson(resolve(caseDirectory, "source-hashes.json"), { files: {}, schema_version: "1.0" }); +} + +const expectedEvidence = phase === "red" + ? ["candidate-reference.json", "red-observation.json"] + : externalRoot && externalEvidenceMissing.length === 0 && candidateErrors.length === 0 && realAuthorized && externalErrors.length === 0 + ? ["candidate-reference.json", "domain-check.json", "delivery-summary.json", "auth-result.json", "redaction.json", "source-hashes.json"] + : ["candidate-reference.json", "blocker.json", "redaction.json", "source-hashes.json"]; +const missingEvidence = expectedEvidence.filter((file) => !existsSync(resolve(caseDirectory, file))); +const status = phase === "red" + ? localGatePassed && missingEvidence.length === 0 ? "red_confirmed" : "failed" + : !localGatePassed || missingEvidence.length > 0 ? "failed" + : realAuthorized && (!externalRoot || externalEvidenceMissing.length > 0 || externalErrors.length > 0) ? "failed" + : externalRoot && externalEvidenceMissing.length === 0 && candidateErrors.length === 0 && realAuthorized && externalErrors.length === 0 ? "passed" + : "externally_blocked"; +const manifest = { + path: "tasks.manifest.json", + sha256: createHash("sha256").update(readFileSync("tasks.manifest.json")).digest("hex").toUpperCase(), +}; +const result = { + acceptance_criteria: ["AC-01", "AC-33", "AC-41", "AC-47", "AC-49"], + automation: ["controlled_real", "manual_review"], + candidate_run_id: candidateRunId, + commit: spawnSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).stdout.trim(), + evidence_refs: expectedEvidence, + external_blockers: phase === "green" && status === "externally_blocked" ? externalBlockers : [], + finished_at: new Date().toISOString(), + layer: ["EXT-REAL", "MANUAL"], + manifest, + missing_evidence: missingEvidence, + phase, + requirements: ["AUTH-01", "AUTH-02", "AUTH-07"], + run_id: runId, + schema_version: "1.0", + status, + task_id: "TASK-WP7-03", + test_id: caseId, + work_package: "WP-7", +}; +writeJson(resolve(caseDirectory, "commands.json"), { commands, phase, run_id: runId, schema_version: "1.0" }); +writeJson(resolve(caseDirectory, "result.json"), result); +writeJson(resolve(runDirectory, "evidence.json"), { cases: [{ external_blockers: result.external_blockers, missing_evidence: missingEvidence, status, test_id: caseId }], phase, run_id: runId, status }); +console.log(JSON.stringify({ external_blockers: result.external_blockers, phase, run_id: runId, status }, null, 2)); +if (status === "failed") process.exit(1); diff --git a/tests/package/wp7-03-resend-gate.test.mjs b/tests/package/wp7-03-resend-gate.test.mjs new file mode 100644 index 0000000..662a487 --- /dev/null +++ b/tests/package/wp7-03-resend-gate.test.mjs @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + validateAuthResult, + validateDeliverySummary, + validateDomainCheck, + validateRedaction, +} from "../../scripts/lib/resend-release-gate.mjs"; + +const realEvidence = { + source: "human_controlled_real", + service: "resend", + status: "verified", + schema_version: "1.0", +}; + +test("requires SPF, DKIM, exact free limits, and no paid fallback", () => { + assert.deepEqual(validateDomainCheck({ + ...realEvidence, + dkim: { status: "pass" }, + domain_controlled: true, + free_rules: { daily_limit: 80, monthly_limit: 2400, paid_fallback_enabled: false, status: "verified" }, + spf: { status: "pass" }, + }), []); + assert.ok(validateDomainCheck({ ...realEvidence, domain_controlled: true, spf: { status: "pass" }, dkim: { status: "fail" }, free_rules: { status: "unknown" } }).includes("dkim")); +}); + +test("requires exactly three 20-message delivery cohorts with 19 within 120 seconds", () => { + const summary = { + ...realEvidence, + categories: ["qq", "163", "enterprise"].map((category) => ({ + category, + delivered_within_120_seconds: 19, + max_latency_seconds: 120, + mock_used: false, + preseeded_account_used: false, + sent_count: 20, + })), + }; + assert.deepEqual(validateDeliverySummary(summary), []); + assert.ok(validateDeliverySummary({ ...summary, categories: summary.categories.map((item) => item.category === "163" ? { ...item, delivered_within_120_seconds: 18 } : item) }).some((error) => error.includes("163"))); + assert.ok(validateDeliverySummary({ ...summary, categories: summary.categories.map((item) => item.category === "enterprise" ? { ...item, max_latency_seconds: undefined } : item) }).some((error) => error.includes("enterprise"))); +}); + +test("requires formal ordinary and admin authentication chains", () => { + assert.deepEqual(validateAuthResult({ + ...realEvidence, + admin: { chain: "formal", status: "passed", verification_code_source: "real_delivery" }, + mock_used: false, + ordinary: { chain: "formal", status: "passed", verification_code_source: "real_delivery" }, + preseeded_account_used: false, + }), []); + assert.ok(validateAuthResult({ ...realEvidence, admin: { chain: "mock", status: "passed", verification_code_source: "fixture" }, ordinary: { status: "failed" } }).length > 0); +}); + +test("rejects credentials, mailboxes, private values, and paths from evidence", () => { + assert.deepEqual(validateRedaction({ + absolute_paths_in_evidence: false, + credentials_in_evidence: false, + forbidden_matches: 0, + mailboxes_in_evidence: false, + private_content_in_evidence: false, + schema_version: "1.0", + status: "passed", + }, JSON.stringify({ category: "qq", delivered_within_120_seconds: 20 })), []); + assert.ok(validateRedaction({ + absolute_paths_in_evidence: false, + credentials_in_evidence: false, + forbidden_matches: 0, + mailboxes_in_evidence: false, + private_content_in_evidence: false, + schema_version: "1.0", + status: "passed", + }, JSON.stringify({ mailbox: "recipient@example.invalid" })).includes("email_value")); +});