589 lines
29 KiB
TypeScript
589 lines
29 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, statSync, writeFileSync } from "node:fs";
|
|
import { isAbsolute, dirname, join, relative, resolve, sep } from "node:path";
|
|
|
|
import { parse } from "csv-parse/sync";
|
|
|
|
import type { StaticStickerCatalog, StaticStickerCatalogItem } from "@dada/static-sticker-catalog";
|
|
import { P0A_TEXT_TEMPLATE_IDS } from "@dada/template-registry";
|
|
|
|
type JsonObject = Record<string, unknown>;
|
|
type CsvRow = Record<string, string>;
|
|
|
|
export interface AssetCompilerOptions {
|
|
manifestPath: string;
|
|
outputDirectory: string;
|
|
releaseVersion: string;
|
|
}
|
|
|
|
export interface AssetCompilerReport {
|
|
schema_version: "asset-compiler-report/v1";
|
|
release_version: string;
|
|
input_manifest_sha256: string;
|
|
source_files_read: number;
|
|
source_mutations: number;
|
|
executed_source_files: number;
|
|
copied_source_files: number;
|
|
derived_files: { created: number; reused: number; total: number };
|
|
status: "passed";
|
|
}
|
|
|
|
export interface AssetCompilerResult {
|
|
manifest: JsonObject;
|
|
report: AssetCompilerReport;
|
|
}
|
|
|
|
export interface StaticStickerCatalogCompilerOptions {
|
|
sourceRoot: string;
|
|
outputDirectory: string;
|
|
releaseVersion: string;
|
|
expectedCount?: number;
|
|
}
|
|
|
|
export interface StaticStickerCatalogCompilerReport {
|
|
schema_version: "static-sticker-catalog-report/v1";
|
|
release_version: string;
|
|
source_files_read: number;
|
|
source_mutations: number;
|
|
copied_source_files: 0;
|
|
status: "passed";
|
|
count: number;
|
|
duplicate_sha256_groups: number;
|
|
}
|
|
|
|
interface SourceRoot {
|
|
id: string;
|
|
path: string;
|
|
}
|
|
|
|
interface TrackedSource {
|
|
bytes: number;
|
|
mtime_ms: number;
|
|
path: string;
|
|
sha256: string;
|
|
}
|
|
|
|
interface SourceEntry {
|
|
canonical_id: string;
|
|
collection_id: string;
|
|
collection_root: string;
|
|
item_directory: string;
|
|
metadata: JsonObject;
|
|
metadata_path: string;
|
|
order: number;
|
|
row: CsvRow;
|
|
}
|
|
|
|
interface CollectionConfig {
|
|
catalogPaths: Array<{ category: string; path: string }>;
|
|
id: string;
|
|
root: SourceRoot;
|
|
}
|
|
|
|
const allowedCollections = new Set(["text_templates", "font_panel", "color_cards", "interactive_stickers"]);
|
|
const releaseVersionPattern = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
|
|
function isRecord(value: unknown): value is JsonObject {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function stableValue(value: unknown): unknown {
|
|
if (Array.isArray(value)) return value.map(stableValue);
|
|
if (!isRecord(value)) return value;
|
|
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])]));
|
|
}
|
|
|
|
function stableJson(value: unknown): string {
|
|
return `${JSON.stringify(stableValue(value), null, 2)}\n`;
|
|
}
|
|
|
|
function sha256(value: Uint8Array | string): string {
|
|
return createHash("sha256").update(value).digest("hex").toUpperCase();
|
|
}
|
|
|
|
function requireRecord(value: unknown, label: string): JsonObject {
|
|
if (!isRecord(value)) throw new Error(`${label} must be an object`);
|
|
return value;
|
|
}
|
|
|
|
function requireString(value: unknown, label: string): string {
|
|
if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
function inside(path: string, root: string): boolean {
|
|
const candidate = resolve(path);
|
|
const base = resolve(root);
|
|
const rest = relative(base, candidate);
|
|
return rest === "" || (rest !== ".." && !rest.startsWith(`..${sep}`) && !isAbsolute(rest));
|
|
}
|
|
|
|
function realDirectory(path: string, label: string): string {
|
|
if (!existsSync(path) || !statSync(path).isDirectory()) throw new Error(`${label} directory is unavailable`);
|
|
return realpathSync(path);
|
|
}
|
|
|
|
function resolveSourcePath(pathValue: string, base: string, root: string, label: string): string {
|
|
const candidate = isAbsolute(pathValue) ? resolve(pathValue) : resolve(base, pathValue);
|
|
if (!inside(candidate, root)) throw new Error(`${label} is outside the allowed source root`);
|
|
if (!existsSync(candidate)) throw new Error(`${label} is unavailable`);
|
|
const actual = realpathSync(candidate);
|
|
if (!inside(actual, root)) throw new Error(`${label} resolves outside the allowed source root`);
|
|
return actual;
|
|
}
|
|
|
|
function relativeReference(pathValue: string, label: string): string {
|
|
if (isAbsolute(pathValue)) throw new Error(`${label} must be relative`);
|
|
const normalized = pathValue.replaceAll("\\", "/");
|
|
if (normalized === "" || normalized === "." || normalized.split("/").includes("..")) throw new Error(`${label} has an unsafe relative path`);
|
|
return normalized.replace(/^\.\//, "");
|
|
}
|
|
|
|
function stringList(value: unknown, label: string): string[] {
|
|
if (value === undefined || value === null || value === "") return [];
|
|
if (Array.isArray(value)) return value.map((item, index) => requireString(item, `${label}[${index}]`));
|
|
return [requireString(value, label)];
|
|
}
|
|
|
|
function dynamicKeys(metadata: JsonObject, row: CsvRow): string[] {
|
|
const value = metadata.dynamic_keys ?? row.dynamic_keys ?? "";
|
|
if (value === "") return [];
|
|
if (Array.isArray(value)) return value.filter((item): item is string => typeof item === "string" && item.trim() !== "").map((item) => item.trim());
|
|
return requireString(value, "dynamic_keys").split("|").map((item) => item.trim()).filter(Boolean);
|
|
}
|
|
|
|
function numericOrder(row: CsvRow, fallback: number): number {
|
|
const value = row.display_order ?? row.panel_order;
|
|
const parsed = value ? Number(value) : Number.NaN;
|
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
}
|
|
|
|
function collectFiles(metadata: JsonObject, keys: string[], label: string): string[] {
|
|
const files = isRecord(metadata.files) ? metadata.files : {};
|
|
return [...new Set(keys.flatMap((key) => stringList(files[key], `${label}.${key}`)).map((item) => relativeReference(item, `${label}.files`)))];
|
|
}
|
|
|
|
function sourceRootLabel(collectionId: string, root: string, path: string): string {
|
|
return `${collectionId}/${relative(root, path).replaceAll("\\", "/")}`;
|
|
}
|
|
|
|
class SourceTracker {
|
|
private readonly entries = new Map<string, TrackedSource>();
|
|
|
|
constructor(private readonly roots: SourceRoot[]) {}
|
|
|
|
read(path: string, label: string): Buffer {
|
|
const root = this.roots.find((item) => inside(path, item.path));
|
|
if (!root) throw new Error(`${label} is outside tracked sources`);
|
|
const actual = realpathSync(path);
|
|
if (!inside(actual, root.path)) throw new Error(`${label} resolves outside tracked sources`);
|
|
const bytes = readFileSync(actual);
|
|
const stats = statSync(actual);
|
|
this.entries.set(actual, {
|
|
bytes: stats.size,
|
|
mtime_ms: stats.mtimeMs,
|
|
path: sourceRootLabel(root.id, root.path, actual),
|
|
sha256: sha256(bytes),
|
|
});
|
|
return bytes;
|
|
}
|
|
|
|
before(): TrackedSource[] {
|
|
return [...this.entries.values()].sort((left, right) => left.path.localeCompare(right.path));
|
|
}
|
|
|
|
after(): { entries: TrackedSource[]; mutations: number } {
|
|
const entries = this.before().map((entry) => {
|
|
const root = this.roots.find((item) => entry.path.startsWith(`${item.id}/`));
|
|
if (!root) return entry;
|
|
const actual = resolve(root.path, entry.path.slice(root.id.length + 1));
|
|
const bytes = readFileSync(actual);
|
|
const stats = statSync(actual);
|
|
return { ...entry, bytes: stats.size, mtime_ms: stats.mtimeMs, sha256: sha256(bytes) };
|
|
});
|
|
const mutations = entries.reduce((count, entry, index) => {
|
|
const before = this.before()[index];
|
|
return count + (before?.sha256 !== entry.sha256 || before?.mtime_ms !== entry.mtime_ms || before?.bytes !== entry.bytes ? 1 : 0);
|
|
}, 0);
|
|
return { entries, mutations };
|
|
}
|
|
}
|
|
|
|
function readJson(tracker: SourceTracker, path: string, label: string): JsonObject {
|
|
try {
|
|
return requireRecord(JSON.parse(tracker.read(path, label).toString("utf8")) as unknown, label);
|
|
} catch (error) {
|
|
if (error instanceof SyntaxError) throw new Error(`${label} is not valid JSON`);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function readCsv(tracker: SourceTracker, path: string, label: string): CsvRow[] {
|
|
try {
|
|
return parse(tracker.read(path, label).toString("utf8"), { bom: true, columns: true, skip_empty_lines: true, trim: true }) as CsvRow[];
|
|
} catch {
|
|
throw new Error(`${label} is not valid CSV`);
|
|
}
|
|
}
|
|
|
|
function itemDirectoryFor(collection: CollectionConfig, catalogPath: string, row: CsvRow): { directory: string; metadataPath: string } {
|
|
if (collection.id === "font_panel") {
|
|
const configuredResourceDir = requireString(row.resource_dir, "font resource_dir");
|
|
const normalizedResourceDir = configuredResourceDir.replaceAll("\\", "/");
|
|
const relocationMarker = "/resources/font_packages/";
|
|
const markerIndex = normalizedResourceDir.lastIndexOf(relocationMarker);
|
|
const resourceDir = isAbsolute(configuredResourceDir) && !inside(configuredResourceDir, collection.root.path) && markerIndex >= 0
|
|
? relativeReference(normalizedResourceDir.slice(markerIndex + 1), "font resource_dir relocation")
|
|
: configuredResourceDir;
|
|
const directory = resolveSourcePath(resourceDir, collection.root.path, collection.root.path, "font resource_dir");
|
|
return { directory, metadataPath: resolveSourcePath("metadata.json", directory, collection.root.path, "font metadata") };
|
|
}
|
|
const canonicalDir = relativeReference(requireString(row.canonical_dir, "canonical_dir"), "canonical_dir");
|
|
const directory = resolveSourcePath(canonicalDir, dirname(catalogPath), collection.root.path, "canonical_dir");
|
|
return { directory, metadataPath: resolveSourcePath("metadata.json", directory, collection.root.path, "metadata") };
|
|
}
|
|
|
|
function collectionConfigs(manifest: JsonObject, manifestDirectory: string): { configs: CollectionConfig[]; roots: SourceRoot[] } {
|
|
if (!Array.isArray(manifest.collections)) throw new Error("manifest collections must be an array");
|
|
const configs: CollectionConfig[] = [];
|
|
const roots: SourceRoot[] = [{ id: "handoff", path: realDirectory(manifestDirectory, "handoff") }];
|
|
for (const raw of manifest.collections) {
|
|
const collection = requireRecord(raw, "collection");
|
|
const id = requireString(collection.id, "collection.id");
|
|
if (!allowedCollections.has(id)) throw new Error(`unsupported collection ${id}`);
|
|
const rootPath = resolve(manifestDirectory, requireString(collection.root, `${id}.root`));
|
|
const root = { id, path: realDirectory(rootPath, `${id}.root`) };
|
|
if (configs.some((item) => item.id === id)) throw new Error(`duplicate collection ${id}`);
|
|
roots.push(root);
|
|
const catalogPaths: Array<{ category: string; path: string }> = [];
|
|
if (id === "text_templates") {
|
|
const catalogs = requireRecord(collection.catalogs, `${id}.catalogs`);
|
|
for (const [category, value] of Object.entries(catalogs).sort(([left], [right]) => left.localeCompare(right))) {
|
|
const catalog = resolveSourcePath(requireString(value, `${id}.${category}`), root.path, root.path, `${id}.${category}`);
|
|
catalogPaths.push({ category, path: catalog });
|
|
}
|
|
} else {
|
|
const catalog = resolveSourcePath(requireString(collection.catalog, `${id}.catalog`), root.path, root.path, `${id}.catalog`);
|
|
catalogPaths.push({ category: id, path: catalog });
|
|
}
|
|
configs.push({ catalogPaths, id, root });
|
|
}
|
|
if (configs.length !== allowedCollections.size) throw new Error("manifest collections are incomplete");
|
|
return { configs, roots };
|
|
}
|
|
|
|
function buildModel(entry: SourceEntry, metadata: JsonObject): JsonObject {
|
|
const files = isRecord(metadata.files) ? metadata.files : {};
|
|
const fonts = collectFiles(metadata, ["fonts"], entry.collection_id);
|
|
const previews = collectFiles(metadata, ["preview"], entry.collection_id);
|
|
const layerFiles = collectFiles(metadata, ["layer", "package", "renderer_source", "images"], entry.collection_id);
|
|
const dynamicFiles = collectFiles(metadata, ["lua", "prefab"], entry.collection_id);
|
|
const fields = dynamicKeys(metadata, entry.row);
|
|
let nodes: JsonObject[];
|
|
if (entry.collection_id === "text_templates") {
|
|
nodes = [{ kind: "text", editable: true, font_references: fonts, slot: "content", text: metadata.default_text ?? entry.row.default_text ?? "" }, { kind: "asset_group", references: layerFiles }];
|
|
} else if (entry.collection_id === "color_cards") {
|
|
const runtime = isRecord(metadata.runtime) ? metadata.runtime : {};
|
|
nodes = [{ kind: "color_card_renderer", palette_slots: 5, renderer_id: typeof runtime.stable_style_id === "string" ? runtime.stable_style_id : entry.row.display_name }];
|
|
} else if (entry.collection_id === "interactive_stickers") {
|
|
nodes = [{ conversion_inputs: dynamicFiles, fields, kind: "dynamic_provider", provider: "declarative" }, { kind: "asset_group", references: [...fonts, ...previews] }];
|
|
} else {
|
|
const runtime = isRecord(metadata.runtime) ? metadata.runtime : {};
|
|
nodes = [{ family: entry.row.font_family ?? (typeof runtime.font_family === "string" ? runtime.font_family : ""), kind: "font", resource: entry.row.resource_dir ? "font_package" : "panel" }];
|
|
}
|
|
return {
|
|
family: entry.collection_id,
|
|
model_schema_version: "TemplateRenderModel/v1",
|
|
nodes,
|
|
template_id: entry.canonical_id,
|
|
};
|
|
}
|
|
|
|
function writeJson(path: string, value: unknown) {
|
|
mkdirSync(dirname(path), { recursive: true });
|
|
writeFileSync(path, stableJson(value));
|
|
}
|
|
|
|
function staticStickerId(index: number) {
|
|
return `STK${String(index).padStart(index < 1_000 ? 3 : 4, "0")}`;
|
|
}
|
|
|
|
function pngDimensions(bytes: Buffer, label: string): { width: number; height: number } {
|
|
const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
if (bytes.length < 45 || !bytes.subarray(0, 8).equals(signature)) throw new Error(`${label} is not a PNG`);
|
|
let offset = 8;
|
|
let sawHeader = false;
|
|
let sawEnd = false;
|
|
let width = 0;
|
|
let height = 0;
|
|
while (offset + 12 <= bytes.length) {
|
|
const length = bytes.readUInt32BE(offset);
|
|
const type = bytes.subarray(offset + 4, offset + 8).toString("ascii");
|
|
const chunkEnd = offset + 12 + length;
|
|
if (chunkEnd > bytes.length) throw new Error(`${label} has a truncated PNG chunk`);
|
|
if (!sawHeader && type !== "IHDR") throw new Error(`${label} has no PNG header`);
|
|
if (type === "IHDR") {
|
|
if (sawHeader || length !== 13) throw new Error(`${label} has an invalid PNG header`);
|
|
width = bytes.readUInt32BE(offset + 8);
|
|
height = bytes.readUInt32BE(offset + 12);
|
|
if (width < 1 || height < 1) throw new Error(`${label} has invalid PNG dimensions`);
|
|
sawHeader = true;
|
|
}
|
|
if (type === "IEND") {
|
|
if (length !== 0) throw new Error(`${label} has an invalid PNG end chunk`);
|
|
sawEnd = true;
|
|
break;
|
|
}
|
|
offset = chunkEnd;
|
|
}
|
|
if (!sawHeader || !sawEnd) throw new Error(`${label} is not a complete PNG`);
|
|
return { height, width };
|
|
}
|
|
|
|
function staticStickerSourceSnapshot(sourceRoot: string, path: string, bytes: Buffer) {
|
|
const stats = statSync(path);
|
|
return {
|
|
bytes: stats.size,
|
|
mtime_ms: stats.mtimeMs,
|
|
path: `sticker_source/${relative(sourceRoot, path).replaceAll("\\", "/")}`,
|
|
sha256: sha256(bytes),
|
|
};
|
|
}
|
|
|
|
export function compileStaticStickerCatalog(options: StaticStickerCatalogCompilerOptions): { catalog: StaticStickerCatalog; report: StaticStickerCatalogCompilerReport } {
|
|
if (!releaseVersionPattern.test(options.releaseVersion)) throw new Error("release version is unsafe");
|
|
const sourceRoot = realDirectory(options.sourceRoot, "sticker source");
|
|
const outputDirectory = resolve(options.outputDirectory);
|
|
if (inside(outputDirectory, sourceRoot)) throw new Error("output directory is inside a source root");
|
|
const items: StaticStickerCatalogItem[] = [];
|
|
const sourceBefore: Array<{ bytes: number; mtime_ms: number; path: string; sha256: string }> = [];
|
|
const partCounts: Record<string, number> = {};
|
|
for (let part = 1; part <= 25; part += 1) {
|
|
const partDirectory = resolve(sourceRoot, `sticker_part${part}`);
|
|
if (!existsSync(partDirectory) || !statSync(partDirectory).isDirectory()) throw new Error(`sticker_part${part} directory is unavailable`);
|
|
const actualPartDirectory = realpathSync(partDirectory);
|
|
if (!inside(actualPartDirectory, sourceRoot)) throw new Error(`sticker_part${part} resolves outside the source root`);
|
|
const files = readdirSync(actualPartDirectory, { withFileTypes: true })
|
|
.filter((entry) => entry.name.toLowerCase().endsWith(".png"))
|
|
.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
|
|
partCounts[String(part)] = files.length;
|
|
for (const [partOrder, entry] of files.entries()) {
|
|
if (!entry.isFile()) throw new Error(`sticker_part${part}/${entry.name} is not a regular PNG file`);
|
|
const path = resolve(actualPartDirectory, entry.name);
|
|
const actualPath = realpathSync(path);
|
|
if (!inside(actualPath, sourceRoot)) throw new Error(`sticker_part${part}/${entry.name} resolves outside the source root`);
|
|
const bytes = readFileSync(actualPath);
|
|
const dimensions = pngDimensions(bytes, `sticker_part${part}/${entry.name}`);
|
|
const stableId = staticStickerId(items.length + 1);
|
|
sourceBefore.push(staticStickerSourceSnapshot(sourceRoot, actualPath, bytes));
|
|
items.push({
|
|
enabled: true,
|
|
height: dimensions.height,
|
|
mime: "image/png",
|
|
mime_type: "image/png",
|
|
order: partOrder + 1,
|
|
original_filename: entry.name,
|
|
original_reference: `/api/v1/assets/public/${encodeURIComponent(options.releaseVersion)}/${encodeURIComponent(stableId)}`,
|
|
origin: "bundled_read_only",
|
|
part,
|
|
relative_path: `sticker_part${part}/${entry.name}`,
|
|
resource_version: options.releaseVersion,
|
|
sha256: sha256(bytes),
|
|
stable_id: stableId,
|
|
thumbnail_reference: {
|
|
media: "thumbnail",
|
|
resource_id: stableId,
|
|
resource_version: options.releaseVersion,
|
|
url: `/api/v1/assets/public/${encodeURIComponent(options.releaseVersion)}/${encodeURIComponent(stableId)}?variant=thumbnail`,
|
|
},
|
|
width: dimensions.width,
|
|
});
|
|
}
|
|
}
|
|
const duplicateSha256Groups = [...new Map(items.reduce((groups, item) => {
|
|
const ids = groups.get(item.sha256) ?? [];
|
|
ids.push(item.stable_id);
|
|
groups.set(item.sha256, ids);
|
|
return groups;
|
|
}, new Map<string, string[]>())).entries()]
|
|
.filter(([, stableIds]) => stableIds.length > 1)
|
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
.map(([hash, stableIds]) => ({ sha256: hash, stable_ids: stableIds }));
|
|
const payload = {
|
|
count: items.length,
|
|
duplicate_sha256_groups: duplicateSha256Groups,
|
|
items,
|
|
part_counts: partCounts,
|
|
release_version: options.releaseVersion,
|
|
schema_version: "StaticStickerCatalog/v1" as const,
|
|
};
|
|
const catalog: StaticStickerCatalog = { ...payload, manifest_sha256: sha256(stableJson(payload)) };
|
|
const sourceAfter = sourceBefore.map((entry) => {
|
|
const actualPath = resolve(sourceRoot, entry.path.slice("sticker_source/".length));
|
|
const bytes = readFileSync(actualPath);
|
|
return staticStickerSourceSnapshot(sourceRoot, actualPath, bytes);
|
|
});
|
|
const sourceMutations = sourceBefore.reduce((count, entry, index) => {
|
|
const after = sourceAfter[index];
|
|
return count + (after?.sha256 !== entry.sha256 || after?.mtime_ms !== entry.mtime_ms || after?.bytes !== entry.bytes ? 1 : 0);
|
|
}, 0);
|
|
if (options.expectedCount !== undefined && options.expectedCount !== items.length) throw new Error(`static sticker count mismatch: expected ${options.expectedCount}, received ${items.length}`);
|
|
if (sourceMutations > 0) throw new Error("sticker source changed during catalog compilation");
|
|
const duplicateGroups = duplicateSha256Groups.length;
|
|
const report: StaticStickerCatalogCompilerReport = {
|
|
copied_source_files: 0,
|
|
count: items.length,
|
|
duplicate_sha256_groups: duplicateGroups,
|
|
release_version: options.releaseVersion,
|
|
schema_version: "static-sticker-catalog-report/v1",
|
|
source_files_read: sourceBefore.length,
|
|
source_mutations: sourceMutations,
|
|
status: "passed",
|
|
};
|
|
mkdirSync(outputDirectory, { recursive: true });
|
|
writeJson(join(outputDirectory, "static-sticker-catalog.json"), catalog);
|
|
writeJson(join(outputDirectory, "catalog-validation.json"), {
|
|
count: items.length,
|
|
duplicate_sha256_groups: duplicateGroups,
|
|
expected_count: options.expectedCount ?? null,
|
|
order_valid: items.every((item, index) => item.stable_id === staticStickerId(index + 1)),
|
|
part_counts: partCounts,
|
|
schema_version: "static-sticker-catalog-validation/v1",
|
|
});
|
|
writeJson(join(outputDirectory, "source-before.json"), { files: sourceBefore, schema_version: "source-snapshot/v1" });
|
|
writeJson(join(outputDirectory, "source-after.json"), { files: sourceAfter, schema_version: "source-snapshot/v1" });
|
|
writeJson(join(outputDirectory, "compiler-report.json"), report);
|
|
writeJson(join(outputDirectory, "output-manifest.json"), {
|
|
catalog_reference: "static-sticker-catalog.json",
|
|
catalog_sha256: sha256(stableJson(catalog)),
|
|
release_version: options.releaseVersion,
|
|
schema_version: "asset-release-compiler/v1",
|
|
});
|
|
return { catalog, report };
|
|
}
|
|
|
|
export function compileAssetArchive(options: AssetCompilerOptions): AssetCompilerResult {
|
|
if (!releaseVersionPattern.test(options.releaseVersion)) throw new Error("release version is unsafe");
|
|
const manifestPath = realpathSync(options.manifestPath);
|
|
const manifestDirectory = realDirectory(dirname(manifestPath), "manifest");
|
|
const roots = collectionConfigs(readJson(new SourceTracker([{ id: "handoff", path: manifestDirectory }]), manifestPath, "manifest"), manifestDirectory);
|
|
const tracker = new SourceTracker(roots.roots);
|
|
const manifest = readJson(tracker, manifestPath, "manifest");
|
|
const handoff = requireString(manifest.web_handoff, "manifest.web_handoff");
|
|
const validation = requireString(manifest.validation, "manifest.validation");
|
|
tracker.read(resolveSourcePath(handoff, manifestDirectory, manifestDirectory, "web handoff"), "web handoff");
|
|
const validationReport = readJson(tracker, resolveSourcePath(validation, manifestDirectory, manifestDirectory, "validation"), "validation");
|
|
if (validationReport.all_valid !== true) throw new Error("source validation report is not all_valid");
|
|
|
|
const entries: SourceEntry[] = [];
|
|
for (const config of roots.configs) {
|
|
for (const catalog of config.catalogPaths) {
|
|
for (const [index, row] of readCsv(tracker, catalog.path, `${config.id}.${catalog.category}`).entries()) {
|
|
const identity = requireString(config.id === "font_panel" ? row.candidate_id : row.canonical_id, "canonical_id");
|
|
const item = itemDirectoryFor(config, catalog.path, row);
|
|
const metadata = readJson(tracker, item.metadataPath, `${identity}.metadata`);
|
|
const metadataId = requireString(config.id === "font_panel" ? metadata.candidate_id : metadata.canonical_id, `${identity}.metadata.canonical_id`);
|
|
if (metadataId !== identity) throw new Error(`${identity} metadata ID mismatch`);
|
|
entries.push({ canonical_id: identity, collection_id: config.id, collection_root: config.root.path, item_directory: item.directory, metadata, metadata_path: item.metadataPath, order: numericOrder(row, index + 1), row });
|
|
}
|
|
}
|
|
}
|
|
const ids = new Set<string>();
|
|
for (const entry of entries) {
|
|
if (ids.has(entry.canonical_id)) throw new Error(`duplicate canonical_id ${entry.canonical_id}`);
|
|
ids.add(entry.canonical_id);
|
|
}
|
|
|
|
const fontPanelIdBySha256 = new Map<string, string>();
|
|
for (const entry of entries.filter((item) => item.collection_id === "font_panel")) {
|
|
const value = entry.metadata.local_sha256;
|
|
if (typeof value === "string" && /^[0-9a-f]{64}$/i.test(value)) fontPanelIdBySha256.set(value.toUpperCase(), entry.canonical_id);
|
|
}
|
|
const p0aTextIds = new Set<string>(P0A_TEXT_TEMPLATE_IDS);
|
|
const fontRegistrations = new Map<string, { panelIds: string[]; sha256s: string[] }>();
|
|
for (const entry of entries.filter((item) => item.collection_id === "text_templates" && p0aTextIds.has(item.canonical_id))) {
|
|
const references = collectFiles(entry.metadata, ["fonts"], `${entry.canonical_id}.fonts`);
|
|
const sha256s = references.map((reference) => {
|
|
const path = resolveSourcePath(reference, entry.item_directory, entry.collection_root, `${entry.canonical_id} font`);
|
|
return sha256(tracker.read(path, `${entry.canonical_id} font`));
|
|
});
|
|
const panelIds = [...new Set(sha256s.map((hash) => fontPanelIdBySha256.get(hash)).filter((id): id is string => id !== undefined))]
|
|
.sort((left, right) => Number(left.slice(4)) - Number(right.slice(4)));
|
|
fontRegistrations.set(entry.canonical_id, { panelIds, sha256s });
|
|
}
|
|
|
|
const outputDirectory = resolve(options.outputDirectory);
|
|
const outputRoots = [manifestDirectory, ...roots.roots.map((item) => item.path)];
|
|
if (outputRoots.some((root) => inside(outputDirectory, root))) throw new Error("output directory is inside a source root");
|
|
mkdirSync(outputDirectory, { recursive: true });
|
|
const sourceBefore = tracker.before();
|
|
const derivedRoot = join(outputDirectory, "derived-assets", options.releaseVersion);
|
|
const derivedStats = { created: 0, reused: 0, total: 0 };
|
|
const items = entries.map((entry) => {
|
|
const metadataSha = sha256(stableJson(entry.metadata));
|
|
const model = buildModel(entry, entry.metadata);
|
|
const modelBytes = Buffer.from(stableJson(model));
|
|
const modelHash = sha256(modelBytes);
|
|
const derivedPath = join(derivedRoot, `${modelHash}.json`);
|
|
const existing = existsSync(derivedPath);
|
|
if (existing && !readFileSync(derivedPath).equals(modelBytes)) throw new Error("derived content hash collision");
|
|
if (existing) derivedStats.reused += 1; else { mkdirSync(derivedRoot, { recursive: true }); writeFileSync(derivedPath, modelBytes); derivedStats.created += 1; }
|
|
derivedStats.total += 1;
|
|
const files = isRecord(entry.metadata.files) ? entry.metadata.files : {};
|
|
const preview = stringList(files.preview, "preview")[0];
|
|
const defaultFont = stringList(files.fonts, "fonts")[0];
|
|
const runtime = isRecord(entry.metadata.runtime) ? entry.metadata.runtime : {};
|
|
const canonicalPath = relative(entry.collection_root, entry.item_directory).replaceAll("\\", "/");
|
|
const item: JsonObject = {
|
|
canonical_id: entry.canonical_id,
|
|
canonical_resource_reference: { collection: entry.collection_id, path: canonicalPath },
|
|
category: entry.row.category || entry.collection_id,
|
|
display_name: entry.metadata.display_name ?? entry.row.display_name ?? entry.row.canonical_id,
|
|
display_order: entry.order,
|
|
family: entry.metadata.family ?? entry.row.family ?? entry.collection_id,
|
|
model_reference: { path: `derived-assets/${options.releaseVersion}/${modelHash}.json`, sha256: modelHash },
|
|
release_status: "imported",
|
|
release_tier: "full_p0",
|
|
required_dynamic_fields: dynamicKeys(entry.metadata, entry.row),
|
|
resource_class: entry.metadata.resource_class ?? entry.row.resource_class ?? "declarative",
|
|
resource_version: metadataSha,
|
|
test_batch_ids: [],
|
|
validation_status: "metadata_validated",
|
|
};
|
|
if (entry.metadata.default_text !== undefined || entry.row.default_text !== undefined) item.default_text = entry.metadata.default_text ?? entry.row.default_text ?? "";
|
|
if (defaultFont) item.default_font_reference = relativeReference(defaultFont, "default font");
|
|
const fontRegistration = fontRegistrations.get(entry.canonical_id);
|
|
if (fontRegistration) {
|
|
item.font_panel_references = fontRegistration.panelIds;
|
|
item.font_resource_sha256s = fontRegistration.sha256s;
|
|
}
|
|
if (entry.collection_id === "font_panel" && typeof entry.metadata.local_sha256 === "string") item.source_sha256 = entry.metadata.local_sha256.toUpperCase();
|
|
if (preview) item.preview_reference = relativeReference(preview, "preview");
|
|
if (typeof runtime.stable_style_id === "string") item.renderer_id = runtime.stable_style_id;
|
|
return item;
|
|
}).sort((left, right) => String(left.canonical_id).localeCompare(String(right.canonical_id)));
|
|
const inputManifestSha = sha256(readFileSync(manifestPath));
|
|
const after = tracker.after();
|
|
const sourceMutations = after.mutations;
|
|
const report: AssetCompilerReport = {
|
|
copied_source_files: 0,
|
|
derived_files: derivedStats,
|
|
executed_source_files: 0,
|
|
input_manifest_sha256: inputManifestSha,
|
|
release_version: options.releaseVersion,
|
|
schema_version: "asset-compiler-report/v1",
|
|
source_files_read: sourceBefore.length,
|
|
source_mutations: sourceMutations,
|
|
status: "passed",
|
|
};
|
|
const releaseManifest: JsonObject = { compiler: "dada-asset-compiler", input_manifest_sha256: inputManifestSha, items, release_version: options.releaseVersion, schema_version: "asset-release-compiler/v1" };
|
|
writeJson(join(outputDirectory, "source-before.json"), { schema_version: "source-snapshot/v1", files: sourceBefore });
|
|
writeJson(join(outputDirectory, "source-after.json"), { files: after.entries, schema_version: "source-snapshot/v1" });
|
|
writeJson(join(outputDirectory, "compiler-report.json"), report);
|
|
writeJson(join(outputDirectory, "output-manifest.json"), releaseManifest);
|
|
if (sourceMutations > 0) throw new Error("source changed during compilation");
|
|
return { manifest: releaseManifest, report };
|
|
}
|
|
|
|
export type { JsonObject };
|