From a7772a7e926541aaae4adc7cb5a3125ddb94407e Mon Sep 17 00:00:00 2001 From: suyx Date: Tue, 4 Aug 2026 15:49:25 +0800 Subject: [PATCH] test: harden WP7-04 release gate evidence --- scripts/run-wp7-04-validation.mjs | 235 +++++++++++++++++++++++------- 1 file changed, 186 insertions(+), 49 deletions(-) diff --git a/scripts/run-wp7-04-validation.mjs b/scripts/run-wp7-04-validation.mjs index 6cede54..c06d80d 100644 --- a/scripts/run-wp7-04-validation.mjs +++ b/scripts/run-wp7-04-validation.mjs @@ -9,13 +9,13 @@ const caseDirectory = resolve(runDirectory, "cases", "TDD-WP7-EXT-003-real-amap" if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`); mkdirSync(caseDirectory, { recursive: true }); -function run(command, args) { +function run(command, args, evidenceCommand = [command, ...args].join(" ")) { const startedAt = new Date().toISOString(); const executable = process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : command; const actualArgs = process.platform === "win32" ? ["/d", "/s", "/c", [command, ...args].join(" ")] : args; const result = spawnSync(executable, actualArgs, { encoding: "utf8" }); return { - command: [command, ...args].join(" "), + command: evidenceCommand, exit_code: result.status ?? 1, finished_at: new Date().toISOString(), stderr: result.stderr ?? "", @@ -24,82 +24,219 @@ function run(command, args) { }; } +function skipped(command) { + const timestamp = new Date().toISOString(); + return { command, exit_code: 1, finished_at: timestamp, started_at: timestamp, stderr: "prerequisite_failed", stdout: "" }; +} + +function readConsoleReview() { + const raw = process.env.DADA_AMAP_CONSOLE_REVIEW_JSON; + const fallback = { + allowlist: "not_verified", + auto_scaling: "not_verified", + paid_fallback: "not_verified", + qps: "not_verified", + reviewed_at: null, + security_restriction: "not_verified", + service_binding: "not_verified", + source: "not_supplied", + valid: true, + }; + if (!raw) return fallback; + try { + const parsed = JSON.parse(raw); + const requiredKeys = ["allowlist", "auto_scaling", "paid_fallback", "qps", "reviewed_at", "security_restriction", "service_binding", "source"]; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || Object.keys(parsed).sort().join("|") !== requiredKeys.sort().join("|")) { + return { ...fallback, source: "invalid_input", valid: false }; + } + const statusFields = ["allowlist", "qps", "security_restriction", "service_binding"]; + const statusValid = statusFields.every((field) => ["failed", "not_verified", "passed"].includes(parsed[field])); + const disabledFieldsValid = ["auto_scaling", "paid_fallback"].every((field) => ["disabled", "not_verified"].includes(parsed[field])); + const reviewedAtValid = typeof parsed.reviewed_at === "string" && Number.isFinite(Date.parse(parsed.reviewed_at)); + if (!statusValid || !disabledFieldsValid || !reviewedAtValid || parsed.source !== "amap_console_manual_review") { + return { ...fallback, source: "invalid_input", valid: false }; + } + return { ...parsed, valid: true }; + } catch { + return { ...fallback, source: "invalid_input", valid: false }; + } +} + +function writeJson(path, value) { + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function probeCode(value) { + return typeof value === "string" && /^[a-z0-9_]{1,64}$/.test(value) ? value : "invalid_output"; +} + +function containsForbiddenEvidence(value) { + const serialized = JSON.stringify(value); + return [ + /[A-Za-z]:[\\/](?:Users|Documents)[\\/]/i, + /"(?:api[_-]?key|credential|secret_value|latitude|longitude|coordinates|email_address|ip_address)"\s*:/i, + /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/, + ].some((pattern) => pattern.test(serialized)); +} + +const consoleReview = readConsoleReview(); const build = run("pnpm", ["build:workspace-packages"]); -const contract = build.exit_code === 0 - ? run("pnpm", ["exec", "vitest", "run", "tests/api/wp4-04-location.test.ts", "tests/api/wp6-03-services.test.ts"]) - : { command: "pnpm exec vitest run tests/api/wp4-04-location.test.ts tests/api/wp6-03-services.test.ts", exit_code: 1, finished_at: new Date().toISOString(), started_at: new Date().toISOString() }; +const locationRegression = build.exit_code === 0 + ? run("pnpm", ["exec", "vitest", "run", "tests/api/wp4-04-location.test.ts"]) + : skipped("pnpm exec vitest run tests/api/wp4-04-location.test.ts"); +const serviceRegression = build.exit_code === 0 + ? run("pnpm", ["exec", "vitest", "run", "tests/api/wp6-03-services.test.ts"]) + : skipped("pnpm exec vitest run tests/api/wp6-03-services.test.ts"); +const localHardStop = build.exit_code === 0 + ? run("pnpm", ["exec", "vitest", "run", "tests/api/wp7-04-amap-release-gate.test.ts"]) + : skipped("pnpm exec vitest run tests/api/wp7-04-amap-release-gate.test.ts"); +const consentFlow = build.exit_code === 0 + ? run("pnpm", ["exec", "vitest", "run", "tests/unit/wp4-04-palette-dynamic.test.ts"]) + : skipped("pnpm exec vitest run tests/unit/wp4-04-palette-dynamic.test.ts"); const supervisorBuild = run("dotnet", ["build", "supervisor/Dada.Supervisor/Dada.Supervisor.csproj", "--configuration", "Release"]); const supervisorExe = resolve("supervisor", "Dada.Supervisor", "bin", "Release", "net8.0-windows", "Dada.Supervisor.exe"); const external = supervisorBuild.exit_code === 0 && existsSync(supervisorExe) - ? run(supervisorExe, ["secrets", "probe", "api-amap"]) - : { command: `${supervisorExe} secrets probe api-amap`, exit_code: 3, finished_at: new Date().toISOString(), started_at: new Date().toISOString(), stderr: "supervisor_probe_unavailable", stdout: "" }; + ? run(supervisorExe, ["secrets", "probe", "api-amap"], "Dada.Supervisor.exe secrets probe api-amap") + : skipped("Dada.Supervisor.exe secrets probe api-amap"); const trace = run("pnpm", ["validate:tdd-trace"]); -const commands = [build, contract, external, trace].map(({ command, exit_code, finished_at, started_at }) => ({ command, exit_code, finished_at, started_at })); +const security = run("pnpm", ["test:security"]); +const commands = [build, locationRegression, serviceRegression, localHardStop, consentFlow, supervisorBuild, external, trace, security] + .map(({ command, exit_code, finished_at, started_at }) => ({ command, exit_code, finished_at, started_at })); const parsedExternal = (() => { - try { return JSON.parse(external.stdout.trim().split(/\r?\n/).at(-1) ?? ""); } catch { return { status: "invalid_output" }; } + try { return JSON.parse(external.stdout.trim().split(/\r?\n/).at(-1) ?? ""); } catch { return {}; } })(); -const probePassed = external.exit_code === 0 && parsedExternal.code === "amap_probe_passed" && parsedExternal.real_calls === 2; -const blocker = probePassed ? "manual_acceptance_required" : (parsedExternal.code ?? "real_amap_validation_failed"); -const contractPassed = contract.exit_code === 0; -const tracePassed = trace.exit_code === 0; -writeFileSync(resolve(caseDirectory, "amap-contract.json"), `${JSON.stringify({ +const safeProbeCode = probeCode(parsedExternal.code); +const realCalls = Number.isSafeInteger(parsedExternal.real_calls) && parsedExternal.real_calls >= 0 && parsedExternal.real_calls <= 2 + ? parsedExternal.real_calls + : 0; +const probePassed = external.exit_code === 0 && safeProbeCode === "amap_probe_passed" && realCalls === 2; +const hardStopPassed = localHardStop.exit_code === 0; +const consentPassed = consentFlow.exit_code === 0; +const regressionPassed = locationRegression.exit_code === 0 && serviceRegression.exit_code === 0; +const automatedPassed = build.exit_code === 0 && hardStopPassed && consentPassed && regressionPassed && trace.exit_code === 0 && security.exit_code === 0; +const manualReviewPassed = consoleReview.valid + && consoleReview.service_binding === "passed" + && consoleReview.qps === "passed" + && consoleReview.allowlist === "passed" + && consoleReview.security_restriction === "passed" + && consoleReview.paid_fallback === "disabled" + && consoleReview.auto_scaling === "disabled"; + +const requiredBeforeRelease = []; +if (!probePassed) requiredBeforeRelease.push("real_location_and_reverse_geocode"); +if (consoleReview.service_binding !== "passed") requiredBeforeRelease.push("service_binding"); +if (consoleReview.qps !== "passed") requiredBeforeRelease.push("qps_confirmation"); +if (consoleReview.allowlist !== "passed") requiredBeforeRelease.push("allowlist_confirmation"); +if (consoleReview.security_restriction !== "passed") requiredBeforeRelease.push("security_restriction"); +if (consoleReview.paid_fallback !== "disabled") requiredBeforeRelease.push("paid_fallback_disabled"); +if (consoleReview.auto_scaling !== "disabled") requiredBeforeRelease.push("auto_scaling_disabled"); +if (!hardStopPassed) requiredBeforeRelease.push("monthly_1000_hard_stop"); +if (!consentPassed) requiredBeforeRelease.push("dyn004_confirmation"); + +const blocker = !automatedPassed + ? "automated_regression_failed" + : !probePassed + ? safeProbeCode + : !consoleReview.valid + ? "manual_review_invalid" + : consoleReview.allowlist === "failed" + ? "amap_allowlist_unconfigured" + : consoleReview.security_restriction === "failed" + ? "amap_security_restriction_unconfigured" + : !manualReviewPassed + ? "manual_acceptance_required" + : null; +const status = !automatedPassed ? "failed" : probePassed && manualReviewPassed ? "passed" : "externally_blocked"; + +const contractEvidence = { + blocker, checks: { + allowlist: consoleReview.allowlist === "passed" ? "manual_console_passed" : consoleReview.allowlist, + auto_scaling: consoleReview.auto_scaling, + dyn004_hard_stop: consentPassed ? "confirmation_contract_passed" : "failed", location_and_reverse_geocode: probePassed ? "real_probe_passed" : "not_verified", - qps_and_allowlist: "not_verified", - security_binding: "not_verified", - dyn004_hard_stop: "contract_passed", - monthly_hard_limit_1000: "not_verified", + monthly_hard_limit_1000: hardStopPassed ? "local_pre_egress_passed" : "failed", + paid_fallback: consoleReview.paid_fallback, + provider_qps: consoleReview.qps === "passed" ? "manual_console_passed" : consoleReview.qps, + security_binding: consoleReview.security_restriction === "passed" ? "manual_console_passed" : consoleReview.security_restriction, + service_binding: consoleReview.service_binding === "passed" ? "manual_console_passed" : consoleReview.service_binding, }, mode: probePassed ? "controlled_real" : "blocked", - real_calls: parsedExternal.real_calls ?? 0, - status: "externally_blocked", - blocker, - schema_version: "1.0", -}, null, 2)}\n`); -writeFileSync(resolve(caseDirectory, "external-calls.json"), `${JSON.stringify({ + real_calls: realCalls, + status, + schema_version: "1.1", +}; +const externalCallsEvidence = { blocker, mode: probePassed ? "controlled_real" : "blocked", - real_calls: parsedExternal.real_calls ?? 0, + real_calls: realCalls, + request_scope: ["geocoding", "reverse_geocoding"], service: "amap", - source_command_status: parsedExternal.code ?? "unknown", - status: "externally_blocked", - schema_version: "1.0", -}, null, 2)}\n`); -writeFileSync(resolve(caseDirectory, "redaction.json"), `${JSON.stringify({ - findings: [], - stored_fields: ["service", "status", "logical_limit", "evidence_sha256"], - forbidden_fields_present: false, - status: "passed", - schema_version: "1.0", -}, null, 2)}\n`); -writeFileSync(resolve(caseDirectory, "manual-review.json"), `${JSON.stringify({ + source_command_status: safeProbeCode, + status, + schema_version: "1.1", +}; +const manualReviewEvidence = { blocker, - required_before_release: probePassed ? ["security_binding", "allowlist_confirmation", "qps_and_1000_call_hard_stop"] : ["controlled_amap_key", "security_binding", "allowlist_confirmation", "real_location_and_reverse_geocode", "qps_and_1000_call_hard_stop"], - status: "externally_blocked", - schema_version: "1.0", -}, null, 2)}\n`); + checks: { + allowlist: consoleReview.allowlist, + auto_scaling: consoleReview.auto_scaling, + paid_fallback: consoleReview.paid_fallback, + qps: consoleReview.qps, + security_restriction: consoleReview.security_restriction, + service_binding: consoleReview.service_binding, + }, + required_before_release: requiredBeforeRelease, + reviewed_at: consoleReview.reviewed_at, + source: consoleReview.source, + status, + schema_version: "1.1", +}; const result = { acceptance_criteria: ["AC-17", "AC-41", "AC-47"], automation: ["controlled_real", "manual_review"], blocker, - contract_regression: contractPassed ? "passed" : "failed", - external_calls: parsedExternal.real_calls ?? 0, + contract_regression: automatedPassed ? "passed" : "failed", + external_calls: realCalls, finished_at: new Date().toISOString(), manifest: { path: "tasks.manifest.json", sha256: createHash("sha256").update(readFileSync("tasks.manifest.json")).digest("hex").toUpperCase() }, - missing_evidence: [], + missing_evidence: requiredBeforeRelease, parent_family: "TDD-WP7-EXT-003", phase: "green", release_gate: ["release:P0-A"], requirements: ["DYN-04", "PRIV-03"], run_id: runId, - schema_version: "1.0", - status: contractPassed && tracePassed ? "externally_blocked" : "failed", + schema_version: "1.1", + status, task_id: "TASK-WP7-04", test_id: "TDD-WP7-EXT-003-real-amap", work_package: "WP-7", }; -writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId, schema_version: "1.0" }, null, 2)}\n`); -writeFileSync(resolve(caseDirectory, "result.json"), `${JSON.stringify(result, null, 2)}\n`); -writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify({ cases: [{ status: result.status, test_id: result.test_id }], run_id: runId, status: result.status, schema_version: "1.0" }, null, 2)}\n`); +const commandsEvidence = { commands, run_id: runId, schema_version: "1.1" }; +const evidenceIndex = { cases: [{ status: result.status, test_id: result.test_id }], run_id: runId, status: result.status, schema_version: "1.1" }; +const redactionInputs = [contractEvidence, externalCallsEvidence, manualReviewEvidence, result, commandsEvidence, evidenceIndex]; +const evidenceRedactionPassed = !redactionInputs.some(containsForbiddenEvidence); +const redactionEvidence = { + findings: evidenceRedactionPassed ? [] : ["forbidden_evidence_field"], + forbidden_fields_present: !evidenceRedactionPassed, + source_security_scan: security.exit_code === 0 ? "passed" : "failed", + status: evidenceRedactionPassed && security.exit_code === 0 ? "passed" : "failed", + stored_fields: ["service", "status", "logical_limit", "manual_review_status"], + schema_version: "1.1", +}; +if (redactionEvidence.status !== "passed") { + result.status = "failed"; + result.blocker = "redaction_failed"; + evidenceIndex.status = "failed"; + evidenceIndex.cases[0].status = "failed"; +} + +writeJson(resolve(caseDirectory, "amap-contract.json"), contractEvidence); +writeJson(resolve(caseDirectory, "external-calls.json"), externalCallsEvidence); +writeJson(resolve(caseDirectory, "redaction.json"), redactionEvidence); +writeJson(resolve(caseDirectory, "manual-review.json"), manualReviewEvidence); +writeJson(resolve(caseDirectory, "commands.json"), commandsEvidence); +writeJson(resolve(caseDirectory, "result.json"), result); +writeJson(resolve(runDirectory, "evidence.json"), evidenceIndex); console.log(JSON.stringify(result, null, 2)); if (result.status === "failed") process.exit(1);