feat: complete TASK-WP5-01 asset compiler

2026-08-03 16:05:56 +08:00
parent a1b82ecdc4
commit 2b189e3186
9 changed files with 729 additions and 2 deletions
@@ -0,0 +1,20 @@
+import { compileAssetArchive } from "./index.js";
+
+function argument(name: string): string {
+  const index = process.argv.indexOf(name);
+  const value = index >= 0 ? process.argv[index + 1] : undefined;
+  if (!value) throw new Error(`missing ${name}`);
+  return value;
+}
+
+try {
+  const result = compileAssetArchive({
+    manifestPath: argument("--manifest"),
+    outputDirectory: argument("--output"),
+    releaseVersion: argument("--release-version"),
+  });
+  process.stdout.write(`${JSON.stringify(result.report, null, 2)}\n`);
+} catch (error) {
+  process.stderr.write(`${error instanceof Error ? error.message : "asset compilation failed"}\n`);
+  process.exitCode = 1;
+}
@@ -0,0 +1,379 @@
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
+import { isAbsolute, dirname, join, relative, resolve, sep } from "node:path";
+
+import { parse } from "csv-parse/sync";
+
+type JsonObject = Record<string, unknown>;
+type CsvRow = Record<string, string>;
+
+export interface AssetCompilerOptions {
+  manifestPath: string;
+  outputDirectory: string;
+  releaseVersion: string;
+}
+
+export interface AssetCompilerReport {
+  schema_version: "asset-compiler-report/v1";
+  release_version: string;
+  input_manifest_sha256: string;
+  source_files_read: number;
+  source_mutations: number;
+  executed_source_files: number;
+  copied_source_files: number;
+  derived_files: { created: number; reused: number; total: number };
+  status: "passed";
+}
+
+export interface AssetCompilerResult {
+  manifest: JsonObject;
+  report: AssetCompilerReport;
+}
+
+interface SourceRoot {
+  id: string;
+  path: string;
+}
+
+interface TrackedSource {
+  bytes: number;
+  mtime_ms: number;
+  path: string;
+  sha256: string;
+}
+
+interface SourceEntry {
+  canonical_id: string;
+  collection_id: string;
+  collection_root: string;
+  item_directory: string;
+  metadata: JsonObject;
+  metadata_path: string;
+  order: number;
+  row: CsvRow;
+}
+
+interface CollectionConfig {
+  catalogPaths: Array<{ category: string; path: string }>;
+  id: string;
+  root: SourceRoot;
+}
+
+const allowedCollections = new Set(["text_templates", "font_panel", "color_cards", "interactive_stickers"]);
+const releaseVersionPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+
+function isRecord(value: unknown): value is JsonObject {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function stableValue(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(stableValue);
+  if (!isRecord(value)) return value;
+  return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])]));
+}
+
+function stableJson(value: unknown): string {
+  return `${JSON.stringify(stableValue(value), null, 2)}\n`;
+}
+
+function sha256(value: Uint8Array | string): string {
+  return createHash("sha256").update(value).digest("hex").toUpperCase();
+}
+
+function requireRecord(value: unknown, label: string): JsonObject {
+  if (!isRecord(value)) throw new Error(`${label} must be an object`);
+  return value;
+}
+
+function requireString(value: unknown, label: string): string {
+  if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a non-empty string`);
+  return value;
+}
+
+function inside(path: string, root: string): boolean {
+  const candidate = resolve(path);
+  const base = resolve(root);
+  const rest = relative(base, candidate);
+  return rest === "" || (rest !== ".." && !rest.startsWith(`..${sep}`) && !isAbsolute(rest));
+}
+
+function realDirectory(path: string, label: string): string {
+  if (!existsSync(path) || !statSync(path).isDirectory()) throw new Error(`${label} directory is unavailable`);
+  return realpathSync(path);
+}
+
+function resolveSourcePath(pathValue: string, base: string, root: string, label: string): string {
+  const candidate = isAbsolute(pathValue) ? resolve(pathValue) : resolve(base, pathValue);
+  if (!inside(candidate, root)) throw new Error(`${label} is outside the allowed source root`);
+  if (!existsSync(candidate)) throw new Error(`${label} is unavailable`);
+  const actual = realpathSync(candidate);
+  if (!inside(actual, root)) throw new Error(`${label} resolves outside the allowed source root`);
+  return actual;
+}
+
+function relativeReference(pathValue: string, label: string): string {
+  if (isAbsolute(pathValue)) throw new Error(`${label} must be relative`);
+  const normalized = pathValue.replaceAll("\\", "/");
+  if (normalized === "" || normalized === "." || normalized.split("/").includes("..")) throw new Error(`${label} has an unsafe relative path`);
+  return normalized.replace(/^\.\//, "");
+}
+
+function stringList(value: unknown, label: string): string[] {
+  if (value === undefined || value === null || value === "") return [];
+  if (Array.isArray(value)) return value.map((item, index) => requireString(item, `${label}[${index}]`));
+  return [requireString(value, label)];
+}
+
+function dynamicKeys(metadata: JsonObject, row: CsvRow): string[] {
+  const value = metadata.dynamic_keys ?? row.dynamic_keys ?? "";
+  if (value === "") return [];
+  if (Array.isArray(value)) return value.filter((item): item is string => typeof item === "string" && item.trim() !== "").map((item) => item.trim());
+  return requireString(value, "dynamic_keys").split("|").map((item) => item.trim()).filter(Boolean);
+}
+
+function numericOrder(row: CsvRow, fallback: number): number {
+  const value = row.display_order ?? row.panel_order;
+  const parsed = value ? Number(value) : Number.NaN;
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function collectFiles(metadata: JsonObject, keys: string[], label: string): string[] {
+  const files = isRecord(metadata.files) ? metadata.files : {};
+  return [...new Set(keys.flatMap((key) => stringList(files[key], `${label}.${key}`)).map((item) => relativeReference(item, `${label}.files`)))];
+}
+
+function sourceRootLabel(collectionId: string, root: string, path: string): string {
+  return `${collectionId}/${relative(root, path).replaceAll("\\", "/")}`;
+}
+
+class SourceTracker {
+  private readonly entries = new Map<string, TrackedSource>();
+
+  constructor(private readonly roots: SourceRoot[]) {}
+
+  read(path: string, label: string): Buffer {
+    const root = this.roots.find((item) => inside(path, item.path));
+    if (!root) throw new Error(`${label} is outside tracked sources`);
+    const actual = realpathSync(path);
+    if (!inside(actual, root.path)) throw new Error(`${label} resolves outside tracked sources`);
+    const bytes = readFileSync(actual);
+    const stats = statSync(actual);
+    this.entries.set(actual, {
+      bytes: stats.size,
+      mtime_ms: stats.mtimeMs,
+      path: sourceRootLabel(root.id, root.path, actual),
+      sha256: sha256(bytes),
+    });
+    return bytes;
+  }
+
+  before(): TrackedSource[] {
+    return [...this.entries.values()].sort((left, right) => left.path.localeCompare(right.path));
+  }
+
+  after(): { entries: TrackedSource[]; mutations: number } {
+    const entries = this.before().map((entry) => {
+      const root = this.roots.find((item) => entry.path.startsWith(`${item.id}/`));
+      if (!root) return entry;
+      const actual = resolve(root.path, entry.path.slice(root.id.length + 1));
+      const bytes = readFileSync(actual);
+      const stats = statSync(actual);
+      return { ...entry, bytes: stats.size, mtime_ms: stats.mtimeMs, sha256: sha256(bytes) };
+    });
+    const mutations = entries.reduce((count, entry, index) => {
+      const before = this.before()[index];
+      return count + (before?.sha256 !== entry.sha256 || before?.mtime_ms !== entry.mtime_ms || before?.bytes !== entry.bytes ? 1 : 0);
+    }, 0);
+    return { entries, mutations };
+  }
+}
+
+function readJson(tracker: SourceTracker, path: string, label: string): JsonObject {
+  try {
+    return requireRecord(JSON.parse(tracker.read(path, label).toString("utf8")) as unknown, label);
+  } catch (error) {
+    if (error instanceof SyntaxError) throw new Error(`${label} is not valid JSON`);
+    throw error;
+  }
+}
+
+function readCsv(tracker: SourceTracker, path: string, label: string): CsvRow[] {
+  try {
+    return parse(tracker.read(path, label).toString("utf8"), { bom: true, columns: true, skip_empty_lines: true, trim: true }) as CsvRow[];
+  } catch {
+    throw new Error(`${label} is not valid CSV`);
+  }
+}
+
+function itemDirectoryFor(collection: CollectionConfig, catalogPath: string, row: CsvRow): { directory: string; metadataPath: string } {
+  if (collection.id === "font_panel") {
+    const resourceDir = requireString(row.resource_dir, "font resource_dir");
+    const directory = resolveSourcePath(resourceDir, collection.root.path, collection.root.path, "font resource_dir");
+    return { directory, metadataPath: resolveSourcePath("metadata.json", directory, collection.root.path, "font metadata") };
+  }
+  const canonicalDir = relativeReference(requireString(row.canonical_dir, "canonical_dir"), "canonical_dir");
+  const directory = resolveSourcePath(canonicalDir, dirname(catalogPath), collection.root.path, "canonical_dir");
+  return { directory, metadataPath: resolveSourcePath("metadata.json", directory, collection.root.path, "metadata") };
+}
+
+function collectionConfigs(manifest: JsonObject, manifestDirectory: string): { configs: CollectionConfig[]; roots: SourceRoot[] } {
+  if (!Array.isArray(manifest.collections)) throw new Error("manifest collections must be an array");
+  const configs: CollectionConfig[] = [];
+  const roots: SourceRoot[] = [{ id: "handoff", path: realDirectory(manifestDirectory, "handoff") }];
+  for (const raw of manifest.collections) {
+    const collection = requireRecord(raw, "collection");
+    const id = requireString(collection.id, "collection.id");
+    if (!allowedCollections.has(id)) throw new Error(`unsupported collection ${id}`);
+    const rootPath = resolve(manifestDirectory, requireString(collection.root, `${id}.root`));
+    const root = { id, path: realDirectory(rootPath, `${id}.root`) };
+    if (configs.some((item) => item.id === id)) throw new Error(`duplicate collection ${id}`);
+    roots.push(root);
+    const catalogPaths: Array<{ category: string; path: string }> = [];
+    if (id === "text_templates") {
+      const catalogs = requireRecord(collection.catalogs, `${id}.catalogs`);
+      for (const [category, value] of Object.entries(catalogs).sort(([left], [right]) => left.localeCompare(right))) {
+        const catalog = resolveSourcePath(requireString(value, `${id}.${category}`), root.path, root.path, `${id}.${category}`);
+        catalogPaths.push({ category, path: catalog });
+      }
+    } else {
+      const catalog = resolveSourcePath(requireString(collection.catalog, `${id}.catalog`), root.path, root.path, `${id}.catalog`);
+      catalogPaths.push({ category: id, path: catalog });
+    }
+    configs.push({ catalogPaths, id, root });
+  }
+  if (configs.length !== allowedCollections.size) throw new Error("manifest collections are incomplete");
+  return { configs, roots };
+}
+
+function buildModel(entry: SourceEntry, metadata: JsonObject): JsonObject {
+  const files = isRecord(metadata.files) ? metadata.files : {};
+  const fonts = collectFiles(metadata, ["fonts"], entry.collection_id);
+  const previews = collectFiles(metadata, ["preview"], entry.collection_id);
+  const layerFiles = collectFiles(metadata, ["layer", "package", "renderer_source", "images"], entry.collection_id);
+  const dynamicFiles = collectFiles(metadata, ["lua", "prefab"], entry.collection_id);
+  const fields = dynamicKeys(metadata, entry.row);
+  let nodes: JsonObject[];
+  if (entry.collection_id === "text_templates") {
+    nodes = [{ kind: "text", editable: true, font_references: fonts, slot: "content", text: metadata.default_text ?? entry.row.default_text ?? "" }, { kind: "asset_group", references: layerFiles }];
+  } else if (entry.collection_id === "color_cards") {
+    const runtime = isRecord(metadata.runtime) ? metadata.runtime : {};
+    nodes = [{ kind: "color_card_renderer", palette_slots: 5, renderer_id: typeof runtime.stable_style_id === "string" ? runtime.stable_style_id : entry.row.display_name }];
+  } else if (entry.collection_id === "interactive_stickers") {
+    nodes = [{ conversion_inputs: dynamicFiles, fields, kind: "dynamic_provider", provider: "declarative" }, { kind: "asset_group", references: [...fonts, ...previews] }];
+  } else {
+    const runtime = isRecord(metadata.runtime) ? metadata.runtime : {};
+    nodes = [{ family: entry.row.font_family ?? (typeof runtime.font_family === "string" ? runtime.font_family : ""), kind: "font", resource: entry.row.resource_dir ? "font_package" : "panel" }];
+  }
+  return {
+    family: entry.collection_id,
+    model_schema_version: "TemplateRenderModel/v1",
+    nodes,
+    template_id: entry.canonical_id,
+  };
+}
+
+function writeJson(path: string, value: unknown) {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, stableJson(value));
+}
+
+export function compileAssetArchive(options: AssetCompilerOptions): AssetCompilerResult {
+  if (!releaseVersionPattern.test(options.releaseVersion)) throw new Error("release version is unsafe");
+  const manifestPath = realpathSync(options.manifestPath);
+  const manifestDirectory = realDirectory(dirname(manifestPath), "manifest");
+  const roots = collectionConfigs(readJson(new SourceTracker([{ id: "handoff", path: manifestDirectory }]), manifestPath, "manifest"), manifestDirectory);
+  const tracker = new SourceTracker(roots.roots);
+  const manifest = readJson(tracker, manifestPath, "manifest");
+  const handoff = requireString(manifest.web_handoff, "manifest.web_handoff");
+  const validation = requireString(manifest.validation, "manifest.validation");
+  tracker.read(resolveSourcePath(handoff, manifestDirectory, manifestDirectory, "web handoff"), "web handoff");
+  const validationReport = readJson(tracker, resolveSourcePath(validation, manifestDirectory, manifestDirectory, "validation"), "validation");
+  if (validationReport.all_valid !== true) throw new Error("source validation report is not all_valid");
+
+  const entries: SourceEntry[] = [];
+  for (const config of roots.configs) {
+    for (const catalog of config.catalogPaths) {
+      for (const [index, row] of readCsv(tracker, catalog.path, `${config.id}.${catalog.category}`).entries()) {
+        const identity = requireString(config.id === "font_panel" ? row.candidate_id : row.canonical_id, "canonical_id");
+        const item = itemDirectoryFor(config, catalog.path, row);
+        const metadata = readJson(tracker, item.metadataPath, `${identity}.metadata`);
+        const metadataId = requireString(config.id === "font_panel" ? metadata.candidate_id : metadata.canonical_id, `${identity}.metadata.canonical_id`);
+        if (metadataId !== identity) throw new Error(`${identity} metadata ID mismatch`);
+        entries.push({ canonical_id: identity, collection_id: config.id, collection_root: config.root.path, item_directory: item.directory, metadata, metadata_path: item.metadataPath, order: numericOrder(row, index + 1), row });
+      }
+    }
+  }
+  const ids = new Set<string>();
+  for (const entry of entries) {
+    if (ids.has(entry.canonical_id)) throw new Error(`duplicate canonical_id ${entry.canonical_id}`);
+    ids.add(entry.canonical_id);
+  }
+
+  const outputDirectory = resolve(options.outputDirectory);
+  const outputRoots = [manifestDirectory, ...roots.roots.map((item) => item.path)];
+  if (outputRoots.some((root) => inside(outputDirectory, root))) throw new Error("output directory is inside a source root");
+  mkdirSync(outputDirectory, { recursive: true });
+  const sourceBefore = tracker.before();
+  const derivedRoot = join(outputDirectory, "derived-assets", options.releaseVersion);
+  const derivedStats = { created: 0, reused: 0, total: 0 };
+  const items = entries.map((entry) => {
+    const metadataSha = sha256(stableJson(entry.metadata));
+    const model = buildModel(entry, entry.metadata);
+    const modelBytes = Buffer.from(stableJson(model));
+    const modelHash = sha256(modelBytes);
+    const derivedPath = join(derivedRoot, `${modelHash}.json`);
+    const existing = existsSync(derivedPath);
+    if (existing && !readFileSync(derivedPath).equals(modelBytes)) throw new Error("derived content hash collision");
+    if (existing) derivedStats.reused += 1; else { mkdirSync(derivedRoot, { recursive: true }); writeFileSync(derivedPath, modelBytes); derivedStats.created += 1; }
+    derivedStats.total += 1;
+    const files = isRecord(entry.metadata.files) ? entry.metadata.files : {};
+    const preview = stringList(files.preview, "preview")[0];
+    const defaultFont = stringList(files.fonts, "fonts")[0];
+    const runtime = isRecord(entry.metadata.runtime) ? entry.metadata.runtime : {};
+    const canonicalPath = relative(entry.collection_root, entry.item_directory).replaceAll("\\", "/");
+    const item: JsonObject = {
+      canonical_id: entry.canonical_id,
+      canonical_resource_reference: { collection: entry.collection_id, path: canonicalPath },
+      category: entry.row.category || entry.collection_id,
+      display_name: entry.metadata.display_name ?? entry.row.display_name ?? entry.row.canonical_id,
+      display_order: entry.order,
+      family: entry.metadata.family ?? entry.row.family ?? entry.collection_id,
+      model_reference: { path: `derived-assets/${options.releaseVersion}/${modelHash}.json`, sha256: modelHash },
+      release_status: "imported",
+      release_tier: "full_p0",
+      required_dynamic_fields: dynamicKeys(entry.metadata, entry.row),
+      resource_class: entry.metadata.resource_class ?? entry.row.resource_class ?? "declarative",
+      resource_version: metadataSha,
+      test_batch_ids: [],
+      validation_status: "metadata_validated",
+    };
+    if (entry.metadata.default_text !== undefined || entry.row.default_text !== undefined) item.default_text = entry.metadata.default_text ?? entry.row.default_text ?? "";
+    if (defaultFont) item.default_font_reference = relativeReference(defaultFont, "default font");
+    if (preview) item.preview_reference = relativeReference(preview, "preview");
+    if (typeof runtime.stable_style_id === "string") item.renderer_id = runtime.stable_style_id;
+    return item;
+  }).sort((left, right) => String(left.canonical_id).localeCompare(String(right.canonical_id)));
+  const inputManifestSha = sha256(readFileSync(manifestPath));
+  const after = tracker.after();
+  const sourceMutations = after.mutations;
+  const report: AssetCompilerReport = {
+    copied_source_files: 0,
+    derived_files: derivedStats,
+    executed_source_files: 0,
+    input_manifest_sha256: inputManifestSha,
+    release_version: options.releaseVersion,
+    schema_version: "asset-compiler-report/v1",
+    source_files_read: sourceBefore.length,
+    source_mutations: sourceMutations,
+    status: "passed",
+  };
+  const releaseManifest: JsonObject = { compiler: "dada-asset-compiler", input_manifest_sha256: inputManifestSha, items, release_version: options.releaseVersion, schema_version: "asset-release-compiler/v1" };
+  writeJson(join(outputDirectory, "source-before.json"), { schema_version: "source-snapshot/v1", files: sourceBefore });
+  writeJson(join(outputDirectory, "source-after.json"), { files: after.entries, schema_version: "source-snapshot/v1" });
+  writeJson(join(outputDirectory, "compiler-report.json"), report);
+  writeJson(join(outputDirectory, "output-manifest.json"), releaseManifest);
+  if (sourceMutations > 0) throw new Error("source changed during compilation");
+  return { manifest: releaseManifest, report };
+}
+
+export type { JsonObject };