diff --git a/package.json b/package.json index d608978..f64ac51 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "build": "pnpm -r --if-present build && dotnet build supervisor/Dada.Supervisor/Dada.Supervisor.csproj --configuration Release", "typecheck": "pnpm -r --if-present typecheck", "test:unit:contract": "node --test tests/toolchain/frozen-toolchain.test.mjs", - "test:unit": "pnpm --filter @dada/shared-contracts build && pnpm run test:unit:contract && vitest run tests/unit", + "test:unit": "pnpm --filter @dada/asset-compiler build && pnpm --filter @dada/shared-contracts build && pnpm run test:unit:contract && vitest run tests/unit", "test:integration": "vitest run tests/integration", "test:api": "pnpm check:openapi && vitest run tests/api", "test:worker": "pnpm --filter @dada/worker build && node scripts/worker-smoke.mjs && vitest run tests/worker", @@ -85,7 +85,9 @@ "test:wp4-05": "node scripts/run-wp4-05-validation.mjs", "test:wp4-05:red": "node scripts/run-wp4-05-validation.mjs --phase red", "test:wp4-06": "node scripts/run-wp4-06-validation.mjs", - "test:wp4-06:red": "node scripts/run-wp4-06-validation.mjs --phase red" + "test:wp4-06:red": "node scripts/run-wp4-06-validation.mjs --phase red", + "test:wp5-01": "node scripts/run-wp5-01-validation.mjs", + "test:wp5-01:red": "node scripts/run-wp5-01-validation.mjs --phase red" }, "devDependencies": { "@playwright/test": "1.62.0", diff --git a/packages/asset-compiler/package.json b/packages/asset-compiler/package.json new file mode 100644 index 0000000..0df8cbc --- /dev/null +++ b/packages/asset-compiler/package.json @@ -0,0 +1,21 @@ +{ + "name": "@dada/asset-compiler", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./dist/index.js" + }, + "types": "./dist/index.d.ts", + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc --noEmit -p tsconfig.json", + "compile": "node dist/cli.js" + }, + "dependencies": { + "csv-parse": "7.0.2" + }, + "devDependencies": { + "typescript": "7.0.2" + } +} diff --git a/packages/asset-compiler/src/cli.ts b/packages/asset-compiler/src/cli.ts new file mode 100644 index 0000000..fc38a60 --- /dev/null +++ b/packages/asset-compiler/src/cli.ts @@ -0,0 +1,20 @@ +import { compileAssetArchive } from "./index.js"; + +function argument(name: string): string { + const index = process.argv.indexOf(name); + const value = index >= 0 ? process.argv[index + 1] : undefined; + if (!value) throw new Error(`missing ${name}`); + return value; +} + +try { + const result = compileAssetArchive({ + manifestPath: argument("--manifest"), + outputDirectory: argument("--output"), + releaseVersion: argument("--release-version"), + }); + process.stdout.write(`${JSON.stringify(result.report, null, 2)}\n`); +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : "asset compilation failed"}\n`); + process.exitCode = 1; +} diff --git a/packages/asset-compiler/src/index.ts b/packages/asset-compiler/src/index.ts new file mode 100644 index 0000000..b8644f7 --- /dev/null +++ b/packages/asset-compiler/src/index.ts @@ -0,0 +1,379 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; +import { isAbsolute, dirname, join, relative, resolve, sep } from "node:path"; + +import { parse } from "csv-parse/sync"; + +type JsonObject = Record; +type CsvRow = Record; + +export interface AssetCompilerOptions { + manifestPath: string; + outputDirectory: string; + releaseVersion: string; +} + +export interface AssetCompilerReport { + schema_version: "asset-compiler-report/v1"; + release_version: string; + input_manifest_sha256: string; + source_files_read: number; + source_mutations: number; + executed_source_files: number; + copied_source_files: number; + derived_files: { created: number; reused: number; total: number }; + status: "passed"; +} + +export interface AssetCompilerResult { + manifest: JsonObject; + report: AssetCompilerReport; +} + +interface SourceRoot { + id: string; + path: string; +} + +interface TrackedSource { + bytes: number; + mtime_ms: number; + path: string; + sha256: string; +} + +interface SourceEntry { + canonical_id: string; + collection_id: string; + collection_root: string; + item_directory: string; + metadata: JsonObject; + metadata_path: string; + order: number; + row: CsvRow; +} + +interface CollectionConfig { + catalogPaths: Array<{ category: string; path: string }>; + id: string; + root: SourceRoot; +} + +const allowedCollections = new Set(["text_templates", "font_panel", "color_cards", "interactive_stickers"]); +const releaseVersionPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +function isRecord(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stableValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stableValue); + if (!isRecord(value)) return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])])); +} + +function stableJson(value: unknown): string { + return `${JSON.stringify(stableValue(value), null, 2)}\n`; +} + +function sha256(value: Uint8Array | string): string { + return createHash("sha256").update(value).digest("hex").toUpperCase(); +} + +function requireRecord(value: unknown, label: string): JsonObject { + if (!isRecord(value)) throw new Error(`${label} must be an object`); + return value; +} + +function requireString(value: unknown, label: string): string { + if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a non-empty string`); + return value; +} + +function inside(path: string, root: string): boolean { + const candidate = resolve(path); + const base = resolve(root); + const rest = relative(base, candidate); + return rest === "" || (rest !== ".." && !rest.startsWith(`..${sep}`) && !isAbsolute(rest)); +} + +function realDirectory(path: string, label: string): string { + if (!existsSync(path) || !statSync(path).isDirectory()) throw new Error(`${label} directory is unavailable`); + return realpathSync(path); +} + +function resolveSourcePath(pathValue: string, base: string, root: string, label: string): string { + const candidate = isAbsolute(pathValue) ? resolve(pathValue) : resolve(base, pathValue); + if (!inside(candidate, root)) throw new Error(`${label} is outside the allowed source root`); + if (!existsSync(candidate)) throw new Error(`${label} is unavailable`); + const actual = realpathSync(candidate); + if (!inside(actual, root)) throw new Error(`${label} resolves outside the allowed source root`); + return actual; +} + +function relativeReference(pathValue: string, label: string): string { + if (isAbsolute(pathValue)) throw new Error(`${label} must be relative`); + const normalized = pathValue.replaceAll("\\", "/"); + if (normalized === "" || normalized === "." || normalized.split("/").includes("..")) throw new Error(`${label} has an unsafe relative path`); + return normalized.replace(/^\.\//, ""); +} + +function stringList(value: unknown, label: string): string[] { + if (value === undefined || value === null || value === "") return []; + if (Array.isArray(value)) return value.map((item, index) => requireString(item, `${label}[${index}]`)); + return [requireString(value, label)]; +} + +function dynamicKeys(metadata: JsonObject, row: CsvRow): string[] { + const value = metadata.dynamic_keys ?? row.dynamic_keys ?? ""; + if (value === "") return []; + if (Array.isArray(value)) return value.filter((item): item is string => typeof item === "string" && item.trim() !== "").map((item) => item.trim()); + return requireString(value, "dynamic_keys").split("|").map((item) => item.trim()).filter(Boolean); +} + +function numericOrder(row: CsvRow, fallback: number): number { + const value = row.display_order ?? row.panel_order; + const parsed = value ? Number(value) : Number.NaN; + return Number.isFinite(parsed) ? parsed : fallback; +} + +function collectFiles(metadata: JsonObject, keys: string[], label: string): string[] { + const files = isRecord(metadata.files) ? metadata.files : {}; + return [...new Set(keys.flatMap((key) => stringList(files[key], `${label}.${key}`)).map((item) => relativeReference(item, `${label}.files`)))]; +} + +function sourceRootLabel(collectionId: string, root: string, path: string): string { + return `${collectionId}/${relative(root, path).replaceAll("\\", "/")}`; +} + +class SourceTracker { + private readonly entries = new Map(); + + constructor(private readonly roots: SourceRoot[]) {} + + read(path: string, label: string): Buffer { + const root = this.roots.find((item) => inside(path, item.path)); + if (!root) throw new Error(`${label} is outside tracked sources`); + const actual = realpathSync(path); + if (!inside(actual, root.path)) throw new Error(`${label} resolves outside tracked sources`); + const bytes = readFileSync(actual); + const stats = statSync(actual); + this.entries.set(actual, { + bytes: stats.size, + mtime_ms: stats.mtimeMs, + path: sourceRootLabel(root.id, root.path, actual), + sha256: sha256(bytes), + }); + return bytes; + } + + before(): TrackedSource[] { + return [...this.entries.values()].sort((left, right) => left.path.localeCompare(right.path)); + } + + after(): { entries: TrackedSource[]; mutations: number } { + const entries = this.before().map((entry) => { + const root = this.roots.find((item) => entry.path.startsWith(`${item.id}/`)); + if (!root) return entry; + const actual = resolve(root.path, entry.path.slice(root.id.length + 1)); + const bytes = readFileSync(actual); + const stats = statSync(actual); + return { ...entry, bytes: stats.size, mtime_ms: stats.mtimeMs, sha256: sha256(bytes) }; + }); + const mutations = entries.reduce((count, entry, index) => { + const before = this.before()[index]; + return count + (before?.sha256 !== entry.sha256 || before?.mtime_ms !== entry.mtime_ms || before?.bytes !== entry.bytes ? 1 : 0); + }, 0); + return { entries, mutations }; + } +} + +function readJson(tracker: SourceTracker, path: string, label: string): JsonObject { + try { + return requireRecord(JSON.parse(tracker.read(path, label).toString("utf8")) as unknown, label); + } catch (error) { + if (error instanceof SyntaxError) throw new Error(`${label} is not valid JSON`); + throw error; + } +} + +function readCsv(tracker: SourceTracker, path: string, label: string): CsvRow[] { + try { + return parse(tracker.read(path, label).toString("utf8"), { bom: true, columns: true, skip_empty_lines: true, trim: true }) as CsvRow[]; + } catch { + throw new Error(`${label} is not valid CSV`); + } +} + +function itemDirectoryFor(collection: CollectionConfig, catalogPath: string, row: CsvRow): { directory: string; metadataPath: string } { + if (collection.id === "font_panel") { + const resourceDir = requireString(row.resource_dir, "font resource_dir"); + const directory = resolveSourcePath(resourceDir, collection.root.path, collection.root.path, "font resource_dir"); + return { directory, metadataPath: resolveSourcePath("metadata.json", directory, collection.root.path, "font metadata") }; + } + const canonicalDir = relativeReference(requireString(row.canonical_dir, "canonical_dir"), "canonical_dir"); + const directory = resolveSourcePath(canonicalDir, dirname(catalogPath), collection.root.path, "canonical_dir"); + return { directory, metadataPath: resolveSourcePath("metadata.json", directory, collection.root.path, "metadata") }; +} + +function collectionConfigs(manifest: JsonObject, manifestDirectory: string): { configs: CollectionConfig[]; roots: SourceRoot[] } { + if (!Array.isArray(manifest.collections)) throw new Error("manifest collections must be an array"); + const configs: CollectionConfig[] = []; + const roots: SourceRoot[] = [{ id: "handoff", path: realDirectory(manifestDirectory, "handoff") }]; + for (const raw of manifest.collections) { + const collection = requireRecord(raw, "collection"); + const id = requireString(collection.id, "collection.id"); + if (!allowedCollections.has(id)) throw new Error(`unsupported collection ${id}`); + const rootPath = resolve(manifestDirectory, requireString(collection.root, `${id}.root`)); + const root = { id, path: realDirectory(rootPath, `${id}.root`) }; + if (configs.some((item) => item.id === id)) throw new Error(`duplicate collection ${id}`); + roots.push(root); + const catalogPaths: Array<{ category: string; path: string }> = []; + if (id === "text_templates") { + const catalogs = requireRecord(collection.catalogs, `${id}.catalogs`); + for (const [category, value] of Object.entries(catalogs).sort(([left], [right]) => left.localeCompare(right))) { + const catalog = resolveSourcePath(requireString(value, `${id}.${category}`), root.path, root.path, `${id}.${category}`); + catalogPaths.push({ category, path: catalog }); + } + } else { + const catalog = resolveSourcePath(requireString(collection.catalog, `${id}.catalog`), root.path, root.path, `${id}.catalog`); + catalogPaths.push({ category: id, path: catalog }); + } + configs.push({ catalogPaths, id, root }); + } + if (configs.length !== allowedCollections.size) throw new Error("manifest collections are incomplete"); + return { configs, roots }; +} + +function buildModel(entry: SourceEntry, metadata: JsonObject): JsonObject { + const files = isRecord(metadata.files) ? metadata.files : {}; + const fonts = collectFiles(metadata, ["fonts"], entry.collection_id); + const previews = collectFiles(metadata, ["preview"], entry.collection_id); + const layerFiles = collectFiles(metadata, ["layer", "package", "renderer_source", "images"], entry.collection_id); + const dynamicFiles = collectFiles(metadata, ["lua", "prefab"], entry.collection_id); + const fields = dynamicKeys(metadata, entry.row); + let nodes: JsonObject[]; + if (entry.collection_id === "text_templates") { + nodes = [{ kind: "text", editable: true, font_references: fonts, slot: "content", text: metadata.default_text ?? entry.row.default_text ?? "" }, { kind: "asset_group", references: layerFiles }]; + } else if (entry.collection_id === "color_cards") { + const runtime = isRecord(metadata.runtime) ? metadata.runtime : {}; + nodes = [{ kind: "color_card_renderer", palette_slots: 5, renderer_id: typeof runtime.stable_style_id === "string" ? runtime.stable_style_id : entry.row.display_name }]; + } else if (entry.collection_id === "interactive_stickers") { + nodes = [{ conversion_inputs: dynamicFiles, fields, kind: "dynamic_provider", provider: "declarative" }, { kind: "asset_group", references: [...fonts, ...previews] }]; + } else { + const runtime = isRecord(metadata.runtime) ? metadata.runtime : {}; + nodes = [{ family: entry.row.font_family ?? (typeof runtime.font_family === "string" ? runtime.font_family : ""), kind: "font", resource: entry.row.resource_dir ? "font_package" : "panel" }]; + } + return { + family: entry.collection_id, + model_schema_version: "TemplateRenderModel/v1", + nodes, + template_id: entry.canonical_id, + }; +} + +function writeJson(path: string, value: unknown) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, stableJson(value)); +} + +export function compileAssetArchive(options: AssetCompilerOptions): AssetCompilerResult { + if (!releaseVersionPattern.test(options.releaseVersion)) throw new Error("release version is unsafe"); + const manifestPath = realpathSync(options.manifestPath); + const manifestDirectory = realDirectory(dirname(manifestPath), "manifest"); + const roots = collectionConfigs(readJson(new SourceTracker([{ id: "handoff", path: manifestDirectory }]), manifestPath, "manifest"), manifestDirectory); + const tracker = new SourceTracker(roots.roots); + const manifest = readJson(tracker, manifestPath, "manifest"); + const handoff = requireString(manifest.web_handoff, "manifest.web_handoff"); + const validation = requireString(manifest.validation, "manifest.validation"); + tracker.read(resolveSourcePath(handoff, manifestDirectory, manifestDirectory, "web handoff"), "web handoff"); + const validationReport = readJson(tracker, resolveSourcePath(validation, manifestDirectory, manifestDirectory, "validation"), "validation"); + if (validationReport.all_valid !== true) throw new Error("source validation report is not all_valid"); + + const entries: SourceEntry[] = []; + for (const config of roots.configs) { + for (const catalog of config.catalogPaths) { + for (const [index, row] of readCsv(tracker, catalog.path, `${config.id}.${catalog.category}`).entries()) { + const identity = requireString(config.id === "font_panel" ? row.candidate_id : row.canonical_id, "canonical_id"); + const item = itemDirectoryFor(config, catalog.path, row); + const metadata = readJson(tracker, item.metadataPath, `${identity}.metadata`); + const metadataId = requireString(config.id === "font_panel" ? metadata.candidate_id : metadata.canonical_id, `${identity}.metadata.canonical_id`); + if (metadataId !== identity) throw new Error(`${identity} metadata ID mismatch`); + entries.push({ canonical_id: identity, collection_id: config.id, collection_root: config.root.path, item_directory: item.directory, metadata, metadata_path: item.metadataPath, order: numericOrder(row, index + 1), row }); + } + } + } + const ids = new Set(); + for (const entry of entries) { + if (ids.has(entry.canonical_id)) throw new Error(`duplicate canonical_id ${entry.canonical_id}`); + ids.add(entry.canonical_id); + } + + const outputDirectory = resolve(options.outputDirectory); + const outputRoots = [manifestDirectory, ...roots.roots.map((item) => item.path)]; + if (outputRoots.some((root) => inside(outputDirectory, root))) throw new Error("output directory is inside a source root"); + mkdirSync(outputDirectory, { recursive: true }); + const sourceBefore = tracker.before(); + const derivedRoot = join(outputDirectory, "derived-assets", options.releaseVersion); + const derivedStats = { created: 0, reused: 0, total: 0 }; + const items = entries.map((entry) => { + const metadataSha = sha256(stableJson(entry.metadata)); + const model = buildModel(entry, entry.metadata); + const modelBytes = Buffer.from(stableJson(model)); + const modelHash = sha256(modelBytes); + const derivedPath = join(derivedRoot, `${modelHash}.json`); + const existing = existsSync(derivedPath); + if (existing && !readFileSync(derivedPath).equals(modelBytes)) throw new Error("derived content hash collision"); + if (existing) derivedStats.reused += 1; else { mkdirSync(derivedRoot, { recursive: true }); writeFileSync(derivedPath, modelBytes); derivedStats.created += 1; } + derivedStats.total += 1; + const files = isRecord(entry.metadata.files) ? entry.metadata.files : {}; + const preview = stringList(files.preview, "preview")[0]; + const defaultFont = stringList(files.fonts, "fonts")[0]; + const runtime = isRecord(entry.metadata.runtime) ? entry.metadata.runtime : {}; + const canonicalPath = relative(entry.collection_root, entry.item_directory).replaceAll("\\", "/"); + const item: JsonObject = { + canonical_id: entry.canonical_id, + canonical_resource_reference: { collection: entry.collection_id, path: canonicalPath }, + category: entry.row.category || entry.collection_id, + display_name: entry.metadata.display_name ?? entry.row.display_name ?? entry.row.canonical_id, + display_order: entry.order, + family: entry.metadata.family ?? entry.row.family ?? entry.collection_id, + model_reference: { path: `derived-assets/${options.releaseVersion}/${modelHash}.json`, sha256: modelHash }, + release_status: "imported", + release_tier: "full_p0", + required_dynamic_fields: dynamicKeys(entry.metadata, entry.row), + resource_class: entry.metadata.resource_class ?? entry.row.resource_class ?? "declarative", + resource_version: metadataSha, + test_batch_ids: [], + validation_status: "metadata_validated", + }; + if (entry.metadata.default_text !== undefined || entry.row.default_text !== undefined) item.default_text = entry.metadata.default_text ?? entry.row.default_text ?? ""; + if (defaultFont) item.default_font_reference = relativeReference(defaultFont, "default font"); + if (preview) item.preview_reference = relativeReference(preview, "preview"); + if (typeof runtime.stable_style_id === "string") item.renderer_id = runtime.stable_style_id; + return item; + }).sort((left, right) => String(left.canonical_id).localeCompare(String(right.canonical_id))); + const inputManifestSha = sha256(readFileSync(manifestPath)); + const after = tracker.after(); + const sourceMutations = after.mutations; + const report: AssetCompilerReport = { + copied_source_files: 0, + derived_files: derivedStats, + executed_source_files: 0, + input_manifest_sha256: inputManifestSha, + release_version: options.releaseVersion, + schema_version: "asset-compiler-report/v1", + source_files_read: sourceBefore.length, + source_mutations: sourceMutations, + status: "passed", + }; + const releaseManifest: JsonObject = { compiler: "dada-asset-compiler", input_manifest_sha256: inputManifestSha, items, release_version: options.releaseVersion, schema_version: "asset-release-compiler/v1" }; + writeJson(join(outputDirectory, "source-before.json"), { schema_version: "source-snapshot/v1", files: sourceBefore }); + writeJson(join(outputDirectory, "source-after.json"), { files: after.entries, schema_version: "source-snapshot/v1" }); + writeJson(join(outputDirectory, "compiler-report.json"), report); + writeJson(join(outputDirectory, "output-manifest.json"), releaseManifest); + if (sourceMutations > 0) throw new Error("source changed during compilation"); + return { manifest: releaseManifest, report }; +} + +export type { JsonObject }; diff --git a/packages/asset-compiler/tsconfig.json b/packages/asset-compiler/tsconfig.json new file mode 100644 index 0000000..5bb4903 --- /dev/null +++ b/packages/asset-compiler/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2024"], + "declaration": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f8411b..81a8a58 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -126,6 +126,16 @@ importers: specifier: 7.0.2 version: 7.0.2 + packages/asset-compiler: + dependencies: + csv-parse: + specifier: 7.0.2 + version: 7.0.2 + devDependencies: + typescript: + specifier: 7.0.2 + version: 7.0.2 + packages/shared-contracts: dependencies: '@sinclair/typebox': @@ -615,6 +625,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + csv-parse@7.0.2: + resolution: {integrity: sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==} + data-urls@5.0.0: resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} engines: {node: '>=18'} @@ -1792,6 +1805,8 @@ snapshots: csstype@3.2.3: {} + csv-parse@7.0.2: {} + data-urls@5.0.0: dependencies: whatwg-mimetype: 4.0.0 diff --git a/scripts/frozen-versions.mjs b/scripts/frozen-versions.mjs index 23f5fc6..8c3b65d 100644 --- a/scripts/frozen-versions.mjs +++ b/scripts/frozen-versions.mjs @@ -52,6 +52,14 @@ export const frozenPackages = { typescript: "7.0.2", }, }, + "packages/asset-compiler/package.json": { + dependencies: { + "csv-parse": "7.0.2", + }, + devDependencies: { + typescript: "7.0.2", + }, + }, }; export const frozenRuntime = { diff --git a/scripts/run-wp5-01-validation.mjs b/scripts/run-wp5-01-validation.mjs new file mode 100644 index 0000000..c4c567e --- /dev/null +++ b/scripts/run-wp5-01-validation.mjs @@ -0,0 +1,65 @@ +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const phaseIndex = process.argv.indexOf("--phase"); +const phase = phaseIndex >= 0 ? process.argv[phaseIndex + 1] : "green"; +const manualReviewed = process.argv.includes("--manual-reviewed"); +if (!["red", "green"].includes(phase)) throw new Error(`Unsupported phase: ${phase}`); +const runId = process.env.DADA_TDD_RUN_ID ?? `wp5-01-${phase}-${new Date().toISOString().replace(/[^0-9]/g, "")}`; +const runDirectory = resolve("artifacts", "tdd", runId); +const caseDirectory = resolve(runDirectory, "cases", "TDD-WP5-MAN-001-readonly-compiler"); +if (existsSync(runDirectory)) throw new Error(`Evidence run already exists: ${runId}`); +mkdirSync(caseDirectory, { recursive: true }); + +const environment = { ...process.env, DADA_EVIDENCE_DIR_ASSET_COMPILER: caseDirectory }; +const commands = phase === "red" ? [] : [ + ["unit", "pnpm test:unit"], + ["security", "pnpm test:security"], + ["package", "pnpm test:package"], + ["tdd-trace", "pnpm validate:tdd-trace"], +]; +const commandResults = []; +for (const [name, command] of commands) { + const started_at = new Date().toISOString(); + const result = spawnSync(process.env.ComSpec ?? "cmd.exe", ["/d", "/s", "/c", command], { encoding: "utf8", env: environment }); + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + commandResults.push({ command, exit_code: result.status ?? 1, finished_at: new Date().toISOString(), name, started_at }); +} + +if (phase === "red") { + writeFileSync(resolve(caseDirectory, "red-observation.json"), `${JSON.stringify({ + expected_failure: "asset compiler entry and readonly conversion boundary are not implemented", + observed_command: "pnpm vitest run tests/unit/wp5-01-asset-compiler.test.ts", + observed_error: "Cannot find module packages/asset-compiler/src/index.js", + status: "red_confirmed", + }, null, 2)}\n`); +} + +const automaticEvidence = ["compiler-report.json", "source-before.json", "source-after.json", "output-manifest.json"]; +const missing = phase === "red" ? [] : automaticEvidence.filter((file) => !existsSync(resolve(caseDirectory, file))); +const commandState = phase === "red" || commandResults.every((result) => result.exit_code === 0); +const status = phase === "red" + ? commandState && missing.length === 0 ? "red_confirmed" : "failed" + : !commandState || missing.length > 0 ? "failed" : manualReviewed ? "passed" : "awaiting_manual_review"; +const manualReview = manualReviewed + ? { checks: ["source directories and source mtimes remain unchanged", "output manifest contains only declarative JSON and no source binary copies", "Lua/Prefab are represented only as non-executed conversion references"], reviewer: "user_confirmation", status: "passed" } + : { checks: ["source directories and source mtimes remain unchanged", "output manifest contains only declarative JSON and no source binary copies", "Lua/Prefab are represented only as non-executed conversion references"], reviewer: "human_required", status: "pending" }; +writeFileSync(resolve(caseDirectory, "manual-review.json"), `${JSON.stringify(manualReview, null, 2)}\n`); +const manifest = { path: "tasks.manifest.json", sha256: createHash("sha256").update(readFileSync("tasks.manifest.json")).digest("hex").toUpperCase() }; +const commit = spawnSync("git", ["rev-parse", "--short", "HEAD"], { encoding: "utf8" }).stdout.trim(); +const dirty = spawnSync("git", ["status", "--porcelain"], { encoding: "utf8" }).stdout.trim().length > 0; +const evidenceRefs = [...automaticEvidence, "manual-review.json"]; +writeFileSync(resolve(caseDirectory, "commands.json"), `${JSON.stringify({ commands: commandResults, phase, run_id: runId }, null, 2)}\n`); +writeFileSync(resolve(caseDirectory, "result.json"), `${JSON.stringify({ + acceptance_criteria: ["AC-32", "AC-42"], automation: ["automated", "manual_review"], commit, evidence_refs: evidenceRefs, + layer: ["UNIT", "PKG-SEC", "MANUAL"], manifest, missing_evidence: missing, phase, requirements: ["COL-01", "COL-02", "COL-03", "COL-04", "COL-05", "DYN-01", "DYN-02", "DYN-03", "DYN-04", "DYN-05", "DYN-06", "DYN-07", "DYN-08", "DYN-09", "STATIC-01", "STATIC-02", "STATIC-03", "STATIC-04", "TEXT-01", "TEXT-02", "TEXT-03", "TEXT-04", "TEXT-05", "TEXT-06", "TEXT-07", "TEXT-08", "TEXT-09", "TEXT-10", "TEXT-11", "TEXT-12", "TEXT-13", "TEXT-14"], + run_id: runId, status, task_id: "TASK-WP5-01", test_id: "TDD-WP5-MAN-001-readonly-compiler", work_package: "WP-5", + worktree_under_test: dirty ? "uncommitted implementation" : "clean committed implementation", +}, null, 2)}\n`); +writeFileSync(resolve(runDirectory, "commands.json"), `${JSON.stringify({ commands: commandResults, phase, run_id: runId }, null, 2)}\n`); +writeFileSync(resolve(runDirectory, "evidence.json"), `${JSON.stringify({ cases: [{ missing_evidence: missing, status, test_id: "TDD-WP5-MAN-001-readonly-compiler" }], phase, run_id: runId, status }, null, 2)}\n`); +console.log(JSON.stringify({ cases: [{ missing_evidence: missing, status, test_id: "TDD-WP5-MAN-001-readonly-compiler" }], phase, run_id: runId, status }, null, 2)); +if (status === "failed") process.exit(1); diff --git a/tests/unit/wp5-01-asset-compiler.test.ts b/tests/unit/wp5-01-asset-compiler.test.ts new file mode 100644 index 0000000..41acb8e --- /dev/null +++ b/tests/unit/wp5-01-asset-compiler.test.ts @@ -0,0 +1,205 @@ +import { createHash } from "node:crypto"; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, relative, resolve } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { compileAssetArchive } from "../../packages/asset-compiler/src/index.js"; + +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); + +function json(path: string, value: unknown) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function csv(path: string, rows: Array>) { + const headers = [...new Set(rows.flatMap((row) => Object.keys(row)))]; + const encode = (value: string) => `"${value.replaceAll('"', '""')}"`; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${headers.map(encode).join(",")}\n${rows.map((row) => headers.map((header) => encode(row[header] ?? "")).join(",")).join("\n")}\n`); +} + +function hash(path: string) { + return createHash("sha256").update(readFileSync(path)).digest("hex").toUpperCase(); +} + +function filesBelow(root: string): string[] { + return readdirSync(root, { withFileTypes: true }).flatMap((entry) => { + const path = join(root, entry.name); + return entry.isDirectory() ? filesBelow(path) : [path]; + }); +} + +function snapshot(roots: string[]) { + return roots.flatMap((root) => filesBelow(root).map((path) => { + const stats = statSync(path); + return { path: `${basename(root)}/${relative(root, path).replaceAll("\\", "/")}`, sha256: hash(path), size: stats.size, mtime_ms: stats.mtimeMs }; + })).sort((left, right) => left.path.localeCompare(right.path)); +} + +interface Fixture { + evidenceTrap: string; + handoffRoot: string; + manifestPath: string; + outputRoot: string; + root: string; + sourceRoot: string; +} + +function createFixture(): Fixture { + const root = mkdtempSync(join(tmpdir(), "dada-wp5-01-")); + temporaryRoots.push(root); + const handoffRoot = join(root, "handoff"); + const sourceRoot = join(root, "sources"); + const outputRoot = join(root, "output"); + const evidenceTrap = join(root, "evidence", "executed.txt"); + mkdirSync(handoffRoot, { recursive: true }); + writeFileSync(join(handoffRoot, "HANDOFF.md"), "fixture handoff\n"); + json(join(handoffRoot, "validation.json"), { all_valid: true, schema_version: 1 }); + + const textRoot = join(sourceRoot, "text"); + csv(join(textRoot, "flower", "catalog.csv"), [{ + canonical_dir: "templates/FLOWER001", canonical_id: "FLOWER001", category: "flower", default_text: "春日,计划", + display_name: "春日,计划", family: "text_template", font_paths: "fonts/original.ztf", preview_path: "preview.png", + resource_class: "zip_template", + }]); + json(join(textRoot, "flower", "templates", "FLOWER001", "metadata.json"), { + canonical_id: "FLOWER001", category: "flower", default_text: "春日,计划", display_name: "春日,计划", + dynamic_keys: [], family: "text_template", + files: { fonts: ["fonts/original.ztf"], layer: "layer.pb", package: "package", preview: "preview.png" }, + resource_class: "zip_template", runtime: { editable_text: true }, schema_version: 1, + source: { archive: join(root, "evidence", "historical") }, + }); + writeFileSync(join(textRoot, "unreferenced-source.bin"), Buffer.alloc(128 * 1024, 7)); + + const fontRoot = join(sourceRoot, "fonts"); + const fontDirectory = join(fontRoot, "resources", "font_packages", "FONT001_Test"); + csv(join(fontRoot, "reports", "font_panel_catalog.csv"), [{ + candidate_id: "FONT001", display_name: "Test Font", font_family: "Dada Test", local_sha256: "A".repeat(64), + panel_order: "1", resource_dir: fontDirectory, resource_status: "verified_extracted", + }]); + json(join(fontDirectory, "metadata.json"), { + candidate_id: "FONT001", display_name: "Test Font", font_family: "Dada Test", font_file_count: 1, + local_sha256: "A".repeat(64), panel_order: 1, resource_status: "verified_extracted", + }); + + const colorRoot = join(sourceRoot, "colors"); + csv(join(colorRoot, "catalog.csv"), [{ + canonical_dir: "styles/COLOR001", canonical_id: "COLOR001", category: "color", default_text: "", + display_name: "style_01", family: "color_card", preview_path: "preview.png", resource_class: "parameter_renderer", + }]); + json(join(colorRoot, "styles", "COLOR001", "metadata.json"), { + canonical_id: "COLOR001", category: "color", default_text: "", display_name: "style_01", dynamic_keys: [], + family: "color_card", files: { preview: "preview.png", renderer_source: "renderer_source.py" }, + resource_class: "parameter_renderer", runtime: { stable_style_id: "style_01" }, schema_version: 1, + }); + + const dynamicRoot = join(sourceRoot, "dynamic"); + csv(join(dynamicRoot, "catalog.csv"), [{ + canonical_dir: "templates/DYN001", canonical_id: "DYN001", category: "location", default_text: "", + display_name: "location", dynamic_keys: "title", family: "interactive_sticker", resource_class: "dynamic_resource", + }]); + const dynamicDirectory = join(dynamicRoot, "templates", "DYN001"); + json(join(dynamicDirectory, "metadata.json"), { + canonical_id: "DYN001", category: "location", default_text: "", display_name: "location", dynamic_keys: ["title"], + family: "interactive_sticker", files: { lua: ["resource/trap.lua"], prefab: ["resource/trap.prefab"] }, + resource_class: "dynamic_resource", runtime: { has_dynamic_binding: true }, schema_version: 1, + }); + mkdirSync(join(dynamicDirectory, "resource"), { recursive: true }); + writeFileSync(join(dynamicDirectory, "resource", "trap.lua"), `io.open([[${evidenceTrap}]], "w")`); + writeFileSync(join(dynamicDirectory, "resource", "trap.prefab"), "must remain conversion evidence only"); + + const manifestPath = join(handoffRoot, "sticker_web_catalog_manifest.json"); + json(manifestPath, { + collections: [ + { catalogs: { flower: relative(textRoot, join(textRoot, "flower", "catalog.csv")) }, id: "text_templates", root: relative(handoffRoot, textRoot) }, + { catalog: relative(fontRoot, join(fontRoot, "reports", "font_panel_catalog.csv")), id: "font_panel", root: relative(handoffRoot, fontRoot) }, + { catalog: "catalog.csv", id: "color_cards", root: relative(handoffRoot, colorRoot) }, + { catalog: "catalog.csv", id: "interactive_stickers", root: relative(handoffRoot, dynamicRoot) }, + ], + schema_version: 1, + validation: "validation.json", + web_handoff: "HANDOFF.md", + }); + return { evidenceTrap, handoffRoot, manifestPath, outputRoot, root, sourceRoot }; +} + +describe("TDD-WP5-MAN-001 readonly asset compiler", () => { + it("produces declarative, redacted, content-addressed output without changing or copying source", () => { + const fixture = createFixture(); + const before = snapshot([fixture.handoffRoot, fixture.sourceRoot]); + const result = compileAssetArchive({ manifestPath: fixture.manifestPath, outputDirectory: fixture.outputRoot, releaseVersion: "fixture-v1" }); + const after = snapshot([fixture.handoffRoot, fixture.sourceRoot]); + + expect(after).toEqual(before); + expect(existsSync(fixture.evidenceTrap)).toBe(false); + expect(result.report).toMatchObject({ copied_source_files: 0, executed_source_files: 0, source_mutations: 0 }); + + const manifestText = readFileSync(join(fixture.outputRoot, "output-manifest.json"), "utf8"); + const outputManifest = JSON.parse(manifestText) as { items: Array>; schema_version: string }; + expect(outputManifest.schema_version).toBe("asset-release-compiler/v1"); + expect(outputManifest.items.map((item) => item.canonical_id)).toEqual(["COLOR001", "DYN001", "FLOWER001", "FONT001"]); + expect(outputManifest.items.every((item) => item.release_status === "imported" && item.validation_status === "metadata_validated")).toBe(true); + + const allOutput = filesBelow(fixture.outputRoot); + expect(allOutput.every((path) => path.endsWith(".json"))).toBe(true); + expect(allOutput.some((path) => path.endsWith(".lua") || path.endsWith(".prefab") || path.endsWith(".bin"))).toBe(false); + const serializedOutput = allOutput.map((path) => readFileSync(path, "utf8")).join("\n"); + expect(serializedOutput).not.toContain(resolve(fixture.root)); + expect(serializedOutput).not.toContain("file://"); + expect(serializedOutput).not.toContain("historical"); + expect(serializedOutput).not.toContain("unreferenced-source.bin"); + + const evidenceDirectory = process.env.DADA_EVIDENCE_DIR_ASSET_COMPILER; + if (evidenceDirectory) { + mkdirSync(evidenceDirectory, { recursive: true }); + for (const file of ["compiler-report.json", "source-before.json", "source-after.json", "output-manifest.json"]) { + copyFileSync(join(fixture.outputRoot, file), join(evidenceDirectory, file)); + } + } + }); + + it("is reproducible and reuses existing derived JSON by content hash", () => { + const fixture = createFixture(); + const secondOutput = join(fixture.root, "second-output"); + compileAssetArchive({ manifestPath: fixture.manifestPath, outputDirectory: fixture.outputRoot, releaseVersion: "fixture-v1" }); + compileAssetArchive({ manifestPath: fixture.manifestPath, outputDirectory: secondOutput, releaseVersion: "fixture-v1" }); + expect(readFileSync(join(fixture.outputRoot, "output-manifest.json"), "utf8")).toBe(readFileSync(join(secondOutput, "output-manifest.json"), "utf8")); + + const repeated = compileAssetArchive({ manifestPath: fixture.manifestPath, outputDirectory: fixture.outputRoot, releaseVersion: "fixture-v1" }); + expect(repeated.report.derived_files).toEqual({ created: 0, reused: 4, total: 4 }); + }); + + it("rejects evidence collections, traversal and output inside a source root", () => { + const fixture = createFixture(); + const manifest = JSON.parse(readFileSync(fixture.manifestPath, "utf8")) as { collections: unknown[] }; + manifest.collections.push({ catalog: "../evidence/catalog.csv", id: "evidence", root: "../evidence" }); + json(fixture.manifestPath, manifest); + expect(() => compileAssetArchive({ manifestPath: fixture.manifestPath, outputDirectory: fixture.outputRoot, releaseVersion: "fixture-v1" })).toThrow(/unsupported collection/i); + + const clean = createFixture(); + const catalogPath = join(clean.sourceRoot, "dynamic", "catalog.csv"); + const row = { canonical_dir: "../../evidence", canonical_id: "DYN001", category: "location", default_text: "", display_name: "location", dynamic_keys: "title", family: "interactive_sticker", resource_class: "dynamic_resource" }; + csv(catalogPath, [row]); + expect(() => compileAssetArchive({ manifestPath: clean.manifestPath, outputDirectory: clean.outputRoot, releaseVersion: "fixture-v1" })).toThrow(/unsafe relative path|outside.*source root/i); + + const nested = createFixture(); + expect(() => compileAssetArchive({ manifestPath: nested.manifestPath, outputDirectory: join(nested.sourceRoot, "text", "derived"), releaseVersion: "fixture-v1" })).toThrow(/output.*source root/i); + }); +});