diff --git a/scripts/lib/wp7-02-controlled-matrix.mjs b/scripts/lib/wp7-02-controlled-matrix.mjs index 5b90645..dfa3475 100644 --- a/scripts/lib/wp7-02-controlled-matrix.mjs +++ b/scripts/lib/wp7-02-controlled-matrix.mjs @@ -222,7 +222,9 @@ export function assembleControlledModelEvidence({ deterministicState, modelConfi evidence_id: evidenceId, external_calls: { approved_real_call_limit: WP7_02_CONTROLLED_REAL_LIMIT, + attempts: realExecution.attempts, calls: realExecution.calls, + maximum_real_calls: realExecution.maximum_real_calls, mode: "controlled_real", planned_real_calls: realExecution.planned_real_calls, real_calls: realExecution.real_calls, diff --git a/scripts/lib/wp7-02-external-contract.mjs b/scripts/lib/wp7-02-external-contract.mjs index 38f22f8..2c637f0 100644 --- a/scripts/lib/wp7-02-external-contract.mjs +++ b/scripts/lib/wp7-02-external-contract.mjs @@ -206,7 +206,10 @@ export function validateIndependentEvidenceSet(evidence) { const passed = entry.status === "passed" && entry.matrix?.status === "passed" && entry.external_calls?.status === "passed" && entry.external_calls.real_calls > 0 && entry.manual_review?.status === "passed" && entry.redaction?.status === "passed"; - if (entry.matrix?.model_id !== entry.model_id || (!blocked && !passed) + const pendingReview = entry.status === "passed" && entry.matrix?.status === "passed" + && entry.external_calls?.status === "passed" && entry.external_calls.real_calls > 0 + && entry.manual_review?.status === "pending" && entry.redaction?.status === "passed"; + if (entry.matrix?.model_id !== entry.model_id || (!blocked && !passed && !pendingReview) || /\"verified\"\s*:/i.test(JSON.stringify(entry))) { throw new Error("WP7_02_BLOCKED_EVIDENCE_INVALID"); } diff --git a/scripts/lib/wp7-02-manual-review.mjs b/scripts/lib/wp7-02-manual-review.mjs new file mode 100644 index 0000000..84007e2 --- /dev/null +++ b/scripts/lib/wp7-02-manual-review.mjs @@ -0,0 +1,54 @@ +const productDimensions = Object.freeze({ + "3:4": [1080, 1440], + "1:1": [1080, 1080], + "4:3": [1440, 1080], + "9:16": [1080, 1920], +}); + +function modelEvidenceComplete(entry) { + const { externalCalls, matrix, modelId, readiness, redaction } = entry; + return matrix.model_id === modelId && Number.isSafeInteger(matrix.config_version) && matrix.config_version > 0 && matrix.status === "passed" + && matrix.pure_text?.status === "passed" && matrix.reference_image?.status === "passed" + && matrix.ratios?.length === 4 && matrix.ratios.every((row) => row.status === "passed") + && matrix.execution_modes?.length === 3 && matrix.execution_modes.every((row) => ["passed", "covered_by_real_calls"].includes(row.status)) + && matrix.error_scenarios?.length === 9 && matrix.error_scenarios.every((row) => row.status === "passed") + && matrix.settlements?.length === 3 && matrix.contract_change?.full_matrix_reapplied === true + && externalCalls.status === "passed" && externalCalls.real_calls >= 5 && externalCalls.real_calls <= 6 + && externalCalls.planned_real_calls === 5 && externalCalls.maximum_real_calls === 6 + && externalCalls.attempts?.length === externalCalls.real_calls + && externalCalls.calls?.length === 5 && new Set(externalCalls.calls.map((row) => row.scenario_id)).size === 5 + && externalCalls.calls.every((row) => row.status === "passed" && row.source === "real_gateway") + && externalCalls.calls.every((row) => { + const [width, height] = productDimensions[row.requested_ratio] ?? []; + return row.response?.dimensions?.width === width && row.response?.dimensions?.height === height; + }) + && externalCalls.approved_real_call_limit === 120 + && readiness.status === "passed" && redaction.status === "passed" && redaction.secret_scan === "passed"; +} + +export function reviewIndependentModelEvidence(entries, { reviewedAt, runId }) { + if (!Array.isArray(entries) || entries.length !== 3 || !runId || !reviewedAt) { + throw new Error("WP7_02_MANUAL_REVIEW_EVIDENCE_INVALID"); + } + const evidenceIds = entries.map((entry) => entry.readiness?.evidence_id); + if (evidenceIds.some((id) => typeof id !== "string") || new Set(evidenceIds).size !== entries.length) { + throw new Error("WP7_02_MANUAL_REVIEW_EVIDENCE_NOT_INDEPENDENT"); + } + const reviews = entries.map((entry) => modelEvidenceComplete(entry) ? { + basis: ["independent_model_evidence", "five_scenarios_bounded_attempts", "four_ratios", "reference_input", "nine_errors", "settlement", "contract_change", "redaction"], + decision: "Sanitized controlled-real and deterministic evidence is complete for this config version.", + model_id: entry.modelId, + reviewed_at: reviewedAt, + reviewer_role: "dada_editor_quality_group", + run_id: runId, + status: "passed", + } : { + decision: "Independent model evidence remains incomplete or externally blocked.", + model_id: entry.modelId, + reviewed_at: reviewedAt, + reviewer_role: "dada_editor_quality_group", + run_id: runId, + status: "blocked", + }); + return { reviews, status: reviews.every((review) => review.status === "passed") ? "passed" : "externally_blocked" }; +} diff --git a/scripts/record-wp7-02-manual-review.mjs b/scripts/record-wp7-02-manual-review.mjs index 4325862..b81fb5d 100644 --- a/scripts/record-wp7-02-manual-review.mjs +++ b/scripts/record-wp7-02-manual-review.mjs @@ -3,11 +3,11 @@ import { resolve } from "node:path"; import { WP7_02_MODEL_IDS } from "./lib/wp7-02-external-contract.mjs"; import { validateSanitizedEvidence } from "./lib/wp7-02-controlled-executor.mjs"; +import { reviewIndependentModelEvidence } from "./lib/wp7-02-manual-review.mjs"; const caseDirectory = process.env.DADA_WP7_02_CASE_DIR; const runId = process.env.DADA_TDD_RUN_ID; const confirmed = process.argv.includes("--confirm-manual-review"); -const productDimensions = { "3:4": [1080, 1440], "1:1": [1080, 1080], "4:3": [1440, 1080], "9:16": [1080, 1920] }; function readJson(path) { return validateSanitizedEvidence(JSON.parse(readFileSync(path, "utf8"))); @@ -20,8 +20,7 @@ function output(value, error = false) { try { if (!confirmed || !caseDirectory || !runId) throw new Error("WP7_02_MANUAL_REVIEW_CONFIRMATION_REQUIRED"); - const reviewed = []; - const evidenceIds = new Set(); + const entries = []; for (const modelId of WP7_02_MODEL_IDS) { const directory = resolve(caseDirectory, modelId.replaceAll(".", "_")); const paths = ["contract-matrix.json", "external-calls.json", "readiness.json", "redaction.json"] @@ -31,37 +30,15 @@ try { const externalCalls = readJson(paths[1]); const readiness = readJson(paths[2]); const redaction = readJson(paths[3]); - const complete = matrix.model_id === modelId && matrix.config_version === 1 && matrix.status === "passed" - && matrix.pure_text?.status === "passed" && matrix.reference_image?.status === "passed" - && matrix.ratios?.length === 4 && matrix.ratios.every((entry) => entry.status === "passed") - && matrix.execution_modes?.length === 3 && matrix.execution_modes.every((entry) => ["passed", "covered_by_real_calls"].includes(entry.status)) - && matrix.error_scenarios?.length === 9 && matrix.error_scenarios.every((entry) => entry.status === "passed") - && matrix.settlements?.length === 3 && matrix.contract_change?.full_matrix_reapplied === true - && externalCalls.status === "passed" && externalCalls.real_calls >= 5 && externalCalls.real_calls <= 6 && externalCalls.planned_real_calls === 5 - && externalCalls.maximum_real_calls === 6 && externalCalls.attempts?.length === externalCalls.real_calls - && externalCalls.calls?.length === 5 && new Set(externalCalls.calls.map((entry) => entry.scenario_id)).size === 5 - && externalCalls.calls.every((entry) => entry.status === "passed" && entry.source === "real_gateway") - && externalCalls.calls.every((entry) => { - const [width, height] = productDimensions[entry.requested_ratio] ?? []; - return entry.response?.dimensions?.width === width && entry.response?.dimensions?.height === height; - }) - && externalCalls.approved_real_call_limit === 120 - && readiness.status === "passed" && redaction.status === "passed" && redaction.secret_scan === "passed"; - if (!complete || evidenceIds.has(readiness.evidence_id)) throw new Error("WP7_02_MANUAL_REVIEW_MATRIX_INCOMPLETE"); - evidenceIds.add(readiness.evidence_id); - const review = validateSanitizedEvidence({ - basis: ["independent_model_evidence", "five_real_calls", "four_ratios", "reference_input", "nine_errors", "settlement", "contract_change", "redaction"], - decision: "Sanitized controlled-real and deterministic evidence is complete for this config version.", - model_id: modelId, - reviewed_at: new Date().toISOString(), - reviewer_role: "dada_editor_quality_group", - run_id: runId, - status: "passed", - }); - writeFileSync(resolve(directory, "manual-review.json"), `${JSON.stringify(review, null, 2)}\n`); - reviewed.push({ model_id: modelId, status: "passed" }); + entries.push({ directory, externalCalls, matrix, modelId, readiness, redaction }); } - output({ reviewed, run_id: runId, status: "passed" }); + const result = reviewIndependentModelEvidence(entries, { reviewedAt: new Date().toISOString(), runId }); + for (let index = 0; index < entries.length; index += 1) { + const review = validateSanitizedEvidence(result.reviews[index]); + writeFileSync(resolve(entries[index].directory, "manual-review.json"), `${JSON.stringify(review, null, 2)}\n`); + } + output({ reviewed: result.reviews.map(({ model_id, status }) => ({ model_id, status })), run_id: runId, status: result.status }, result.status !== "passed"); + if (result.status !== "passed") process.exitCode = 3; } catch (error) { const message = error instanceof Error ? error.message : String(error); const code = /^WP7_02_[A-Z0-9_:]+$/.test(message) ? message : "WP7_02_MANUAL_REVIEW_FAILED"; diff --git a/scripts/run-wp7-02-validation.mjs b/scripts/run-wp7-02-validation.mjs index 9068314..eeab26c 100644 --- a/scripts/run-wp7-02-validation.mjs +++ b/scripts/run-wp7-02-validation.mjs @@ -137,13 +137,18 @@ if (!externalExitCodesValid) { process.exit(1); } -if (controlledReal && externalCommands.every((command) => command.exit_code === 0)) { +if (controlledReal) { const manualConfirmed = process.env.DADA_WP7_02_MANUAL_REVIEW_CONFIRMATION === "confirmed"; const manual = run("manual-review", process.execPath, ["scripts/record-wp7-02-manual-review.mjs", ...(manualConfirmed ? ["--confirm-manual-review"] : [])], { env: { DADA_TDD_RUN_ID: runId, DADA_WP7_02_CASE_DIR: caseDirectory }, logicalCommand: `pnpm review:wp7-02${manualConfirmed ? " -- --confirm-manual-review" : ""}`, }); commands.push(manual); + if (![0, 3].includes(manual.exit_code)) { + writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands, run_id: runId }, null, 2)}\n`); + console.error(JSON.stringify({ code: "WP7_02_MANUAL_REVIEW_COMMAND_FAILED", command: manual.command, exit_code: manual.exit_code, real_calls: 0, run_id: runId, status: "failed" })); + process.exit(1); + } } const modelEvidence = WP7_02_MODEL_IDS.map((modelId) => {