Report missing base game icons when exporting from headless Factorio

The headless build has no graphics, so export-data found none of the
base game's icon files. Plain icons were skipped silently and layered
icons were drawn blank and counted as exported.

Skip a layered icon when any of its layers is missing, and log one
warning with the number of base game icon files that were not found.
Add host.factorio_graphics_directory to read builtin mod graphics from
a full install when the instance runs the headless build.

Fixes #1068

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/configuration.md

@@ -339,6 +339,15 @@ This can either point to a Factorio server directory, or to a directory containi
 Defaults to "factorio".
 
 
+### host.factorio_graphics_directory
+
+Directory to read the base game's icons from when exporting data.
+The headless build of Factorio has no graphics, so instances running it export none of the base game's icons unless this points to a full install of Factorio.
+Like `host.factorio_directory` this can either point to a Factorio directory, or to a directory containing multiple versions, in which case the latest one matching the major and minor version of the instance is used.
+
+Defaults to null.
+
+
 ### host.mods_directory
 
 Directory the host stores mods downloaded from the controller in.

packages/host/src/Instance.ts

@@ -1469,7 +1469,7 @@ end`.replace(/\r?\n/g, " ");
 
 			this.logger.info("Exporting data .....");
 			await this.syncMods();
-			let zip = await exportData(this.server);
+			let zip = await exportData(this.server, this._host.config.get("host.factorio_graphics_directory"));
 
 			let url = new URL(this._host.config.get("host.controller_url"));
 			url.pathname += "api/upload-export";

packages/host/src/export.ts

@@ -6,7 +6,7 @@ import yazl from "yazl";
 
 import * as lib from "@clusterio/lib";
 import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js";
-import type { FactorioServer } from "./server.js";
+import { findVersion, type FactorioServer } from "./server.js";
 import { definesPrototypesFallback } from "./export_fallback.js";
 
 interface SimpleIconSpecification {
@@ -91,6 +91,13 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
 	return modZip;
 }
 
+/** Where icon files are read from during an export. */
+interface ExportFiles {
+	zipCache: ZipCache;
+	/** Data dir of a full install to fall back to for builtin mod files. */
+	graphicsDataDir?: string;
+}
+
 function closeZipCache(zipCache: ZipCache) {
 	for (const { zip } of zipCache.values()) {
 		zip.close();
@@ -103,7 +110,7 @@ function closeZipCache(zipCache: ZipCache) {
  *
  * @param server - The server to load the file from.
  * @param modVersions - Mapping of mod to version used.
- * @param zipCache - Mod zips opened by this export.
+ * @param files - Where to read files from.
  * @param modPath - Factorio style path to the file to load.
  * @returns The content of the file or null if not found.
  * @internal
@@ -111,7 +118,7 @@ function closeZipCache(zipCache: ZipCache) {
 async function loadFile(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	modPath: string,
 ) {
 	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
@@ -124,19 +131,25 @@ async function loadFile(
 
 	const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)];
 	if (builtinModNames.includes(mod)) {
-		try {
-			return await fs.readFile(server.dataPath(mod, filePath));
-		} catch (err: any) {
-			if (err.code === "ENOENT") {
-				return null;
-			}
-			throw err;
+		const dataDirs = [server.dataPath()];
+		if (files.graphicsDataDir) {
+			dataDirs.push(files.graphicsDataDir);
 		}
+		for (const dataDir of dataDirs) {
+			try {
+				return await fs.readFile(path.join(dataDir, mod, filePath));
+			} catch (err: any) {
+				if (err.code !== "ENOENT") {
+					throw err;
+				}
+			}
+		}
+		return null;
 	}
 
 	let modZip;
 	try {
-		modZip = await loadZip(server, modVersions, zipCache, mod);
+		modZip = await loadZip(server, modVersions, files.zipCache, mod);
 	} catch (err: any) {
 		if (err.code === "ENOENT") {
 			return null;
@@ -152,7 +165,7 @@ type IconCache = Map<string, Image | null>;
 async function loadIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	iconPath: string,
 	iconSize: number,
 	iconMipmaps: number,
@@ -160,7 +173,7 @@ async function loadIcon(
 ) {
 	let icon = iconCache.get(iconPath);
 	if (icon === undefined) {
-		let fileContent = await loadFile(server, modVersions, zipCache, iconPath);
+		let fileContent = await loadFile(server, modVersions, files, iconPath);
 		if (fileContent) {
 			icon = await Jimp.fromBuffer(fileContent) as Image;
 			icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize });
@@ -176,12 +189,12 @@ async function loadIcon(
 async function loadSimpleIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	item: SimpleIconSpecification,
 	size: number,
 	iconCache: IconCache,
 ): Promise<Image | null> {
-	let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache);
+	let icon = await loadIcon(server, modVersions, files, item.icon, item.icon_size ?? 64, 0, iconCache);
 	if (icon) {
 		let iconScale = size / (item.icon_size ?? 64);
 		if (iconScale !== 1) {
@@ -195,11 +208,11 @@ async function loadSimpleIcon(
 async function loadLayeredIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	item: LayeredIconSpecification,
 	size: number,
 	iconCache: IconCache,
-): Promise<Image> {
+): Promise<Image | null> {
 	let baseLayerSize = (item.icons[0].icon_size || item.icon_size) ?? 64;
 	let icon = new Jimp({ width: size, height: size });
 
@@ -212,11 +225,12 @@ async function loadLayeredIcon(
 	for (let layer of item.icons) {
 		let layerSize = (layer.icon_size || item.icon_size) ?? 64;
 		let iconLayer = await loadIcon(
-			server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
+			server, modVersions, files, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
 		);
 
+		// A partial icon would be wrong, skip it like a missing simple icon
 		if (!iconLayer) {
-			continue;
+			return null;
 		}
 
 		iconLayer = iconLayer.clone();
@@ -331,7 +345,7 @@ async function growSheet(state: SheetState) {
 async function exportIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	state: SheetState,
 	prototype: PrototypeWithIcon,
 ) {
@@ -342,9 +356,11 @@ async function exportIcon(
 
 	if (prototype.icons) {
 		icon = await loadLayeredIcon(
-			server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
+			server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache,
 		);
-		iconPos = state.pos;
+		if (icon) {
+			iconPos = state.pos;
+		}
 		iconPath = prototype.icons[0].icon;
 
 	} else if (prototype.icon) {
@@ -352,7 +368,7 @@ async function exportIcon(
 		iconPath = prototype.icon;
 		if (iconPos === undefined) {
 			icon = await loadSimpleIcon(
-				server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
+				server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache,
 			);
 			if (icon) {
 				iconPos = state.pos;
@@ -388,7 +404,7 @@ async function exportIcon(
  *
  * @param server - The server to load icons from.
  * @param modVersions - Mapping of mod name to version.
- * @param zipCache - Mod zips opened by this export.
+ * @param files - Where to read icon files from.
  * @param prototypes - Array of prototypes with icon to export for this base type.
  * @param baseName - name of base type (used in metadata).
  * @param state - Shared spritesheet state (mutated in place).
@@ -398,7 +414,7 @@ async function exportIcon(
 async function exportIcons(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	prototypes: PrototypeWithIcon[],
 	baseName: string,
 	state: SheetState,
@@ -407,7 +423,7 @@ async function exportIcons(
 	state.metadata[baseName] ??= metadata;
 	let count = 0;
 	for (let prototype of prototypes) {
-		const icon = await exportIcon(server, modVersions, zipCache, state, prototype);
+		const icon = await exportIcon(server, modVersions, files, state, prototype);
 		if (icon) {
 			count += 1;
 		}
@@ -498,19 +514,57 @@ async function exportLocale(
  * Export locale, icons and metadata for the given factorio server
  *
  * @param server - The server to export the data from.
+ * @param graphicsDirectory -
+ *     Full Factorio install, or directory of installs, to read builtin mod
+ *     graphics from when the server's install lacks them.
  * @returns zip file with exported data.
  */
-export async function exportData(server: FactorioServer) {
-	const zipCache: ZipCache = new Map();
+export async function exportData(server: FactorioServer, graphicsDirectory?: string | null) {
+	const files: ExportFiles = { zipCache: new Map() };
+	if (graphicsDirectory) {
+		const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion;
+		[files.graphicsDataDir] = await findVersion(graphicsDirectory, majorMinor);
+	}
 	try {
-		return await buildExport(server, zipCache);
+		return await buildExport(server, files);
 	} finally {
 		// Free up the zip files loaded during the export.
-		closeZipCache(zipCache);
+		closeZipCache(files.zipCache);
 	}
 }
 
-async function buildExport(server: FactorioServer, zipCache: ZipCache) {
+/**
+ * Warn about builtin mod icon files that could not be found
+ *
+ * @param server - The server the export ran on.
+ * @param files - Where icon files were read from.
+ * @param iconCache - Icons loaded by the export, null for missing files.
+ * @internal
+ */
+function warnMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, iconCache: IconCache) {
+	const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)];
+	let missing = 0;
+	for (const [iconPath, icon] of iconCache) {
+		const mod = /^__([^\/]+)__\//.exec(iconPath)?.[1];
+		if (icon === null && mod && builtinModNames.includes(mod)) {
+			missing += 1;
+		}
+	}
+	if (!missing) {
+		return;
+	}
+
+	const dirs = [server.dataPath(), files.graphicsDataDir].filter(Boolean).join(" or ");
+	const hint = files.graphicsDataDir
+		? "Check that host.factorio_graphics_directory points to a full Factorio install"
+		: "Set host.factorio_graphics_directory to a full Factorio install to read them from there";
+	server._logger.warn(
+		`${missing} icon files of the base game were not found in ${dirs}, icons using them were not exported. ` +
+		`The headless build of Factorio has no graphics. ${hint}, or export the data on a host with a full install.`
+	);
+}
+
+async function buildExport(server: FactorioServer, files: ExportFiles) {
 	await generateExportMod(server);
 
 	let settings: lib.ExportSettings = {};
@@ -590,7 +644,7 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) {
 		}
 	}
 
-	let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en");
+	let locale = await exportLocale(server, modVersions, files.zipCache, modOrder, "en");
 
 	let zip = new yazl.ZipFile();
 	const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name);
@@ -611,9 +665,10 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) {
 		if (prototypesWithIcons.length === 0) {
 			continue;
 		}
-		const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state);
+		const count = await exportIcons(server, modVersions, files, prototypesWithIcons, baseName, state);
 		server._logger.info(`Exported ${count} ${baseName} icons`);
 	}
+	warnMissingBuiltinIcons(server, files, state.iconCache);
 
 	// Crop sheet to actual used height and write single spritesheet + metadata.
 	const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
@@ -632,3 +687,4 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) {
 export const _exportLocale = exportLocale;
 export const _loadSimpleIcon = loadSimpleIcon;
 export const _loadLayeredIcon = loadLayeredIcon;
+export const _warnMissingBuiltinIcons = warnMissingBuiltinIcons;

packages/lib/src/config/definitions.ts

@@ -324,6 +324,7 @@ export interface HostConfigFields {
 	"host.name": string;
 	"host.id": number;
 	"host.factorio_directory": string;
+	"host.factorio_graphics_directory": string | null;
 	"host.mods_directory": string;
 	"host.instances_directory": string;
 	"host.controller_url": string;
@@ -372,6 +373,14 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			initialValue: "factorio",
 			validator: validators.filePath,
 		},
+		"host.factorio_graphics_directory": {
+			description:
+				"Path to a full Factorio install, or directory of installs, to read base game icons from when " +
+				"exporting data. Needed when running the headless build, which has no graphics.",
+			type: "string",
+			optional: true,
+			validator: validators.optional(validators.filePath),
+		},
 		"host.mods_directory": {
 			title: "Mods Directory",
 			description: "Path to directory where mods for instances are cached.",

test/host/export.js

@@ -2,7 +2,9 @@ import assert from "node:assert/strict";
 import path from "node:path";
 
 import { FactorioServer } from "@clusterio/host/dist/node/src/server.js";
-import { _exportLocale, _loadLayeredIcon, _loadSimpleIcon } from "@clusterio/host/dist/node/src/export.js";
+import {
+	_exportLocale, _loadLayeredIcon, _loadSimpleIcon, _warnMissingBuiltinIcons,
+} from "@clusterio/host/dist/node/src/export.js";
 
 
 describe("host/src/export", function() {
@@ -30,20 +32,37 @@ describe("host/src/export", function() {
 
 	// test-icon.png is a red 64px icon followed by a blue 32px mipmap
 	const red = 0xff0000ff;
+	// Stands in for the data dir of a full install
+	const graphicsDataDir = path.join("test", "file", "factorio-graphics");
 	describe("loadSimpleIcon()", function() {
 		it("defaults icon_size to 64", async function() {
 			const icon = await _loadSimpleIcon(
-				testServer, new Map(), new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map()
+				testServer, new Map(), { zipCache: new Map() },
+				{ icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map(),
 			);
 			assert.equal(icon.bitmap.width, 32);
 			assert.equal(icon.bitmap.height, 32);
 			assert.equal(icon.getPixelColor(31, 16), red);
 		});
+		it("returns null when the icon file is missing", async function() {
+			const icon = await _loadSimpleIcon(
+				testServer, new Map(), { zipCache: new Map() },
+				{ icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(),
+			);
+			assert.equal(icon, null);
+		});
+		it("falls back to graphicsDataDir for builtin mod files", async function() {
+			const icon = await _loadSimpleIcon(
+				testServer, new Map(), { zipCache: new Map(), graphicsDataDir },
+				{ icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(),
+			);
+			assert.equal(icon.getPixelColor(31, 16), red);
+		});
 	});
 	describe("loadLayeredIcon()", function() {
 		it("defaults icon_size to 64", async function() {
 			const icon = await _loadLayeredIcon(
-				testServer, new Map(), new Map(),
+				testServer, new Map(), { zipCache: new Map() },
 				{ icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map(),
 			);
 			assert.equal(icon.bitmap.width, 32);
@@ -51,5 +70,46 @@ describe("host/src/export", function() {
 			assert.equal(icon.getPixelColor(0, 0), red);
 			assert.equal(icon.getPixelColor(31, 31), red);
 		});
+		it("returns null when a layer is missing", async function() {
+			const icon = await _loadLayeredIcon(
+				testServer, new Map(), { zipCache: new Map() },
+				{ icons: [
+					{ icon: "__base__/graphics/icons/test-icon.png" },
+					{ icon: "__base__/graphics/icons/missing.png" },
+				] }, 32, new Map(),
+			);
+			assert.equal(icon, null);
+		});
+	});
+	describe("warnMissingBuiltinIcons()", function() {
+		let warnings;
+		let logger;
+		beforeEach(function() {
+			warnings = [];
+			logger = testServer._logger;
+			testServer._logger = { warn: msg => warnings.push(msg) };
+		});
+		afterEach(function() {
+			testServer._logger = logger;
+		});
+
+		it("warns once with the number of missing builtin files", function() {
+			_warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([
+				["__base__/graphics/icons/a.png", null],
+				["__core__/graphics/icons/b.png", null],
+				["__base__/graphics/icons/c.png", {}],
+				["__some_mod__/graphics/icons/d.png", null],
+			]));
+			assert.equal(warnings.length, 1);
+			assert.match(warnings[0], /^2 icon files of the base game were not found/);
+			assert.match(warnings[0], /headless/);
+		});
+		it("does not warn when no builtin files are missing", function() {
+			_warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([
+				["__base__/graphics/icons/c.png", {}],
+				["__some_mod__/graphics/icons/d.png", null],
+			]));
+			assert.deepEqual(warnings, []);
+		});
 	});
 });