Report missing base game icons when exporting from headless Factorio (#1153)

* Report missing base game icons when exporting from headless Factorio

The headless build has no graphics, so export-data found none of the
base game's icon files. Plain icons were skipped silently and layered
icons were drawn blank and counted as exported.

Skip a layered icon when any of its layers is missing, and log one
warning with the number of base game icon files that were not found.
Add host.factorio_graphics_directory to read builtin mod graphics from
a full install when the instance runs the headless build.

Fixes #1068

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Export without the graphics directory when it has no matching install

If host.factorio_graphics_directory had no install matching the major and
minor version of the instance, or did not exist, findVersion threw and the
whole export failed. Without the setting the same instance exports everything
except the base game icons, so it now logs a warning that names the setting
and exports without it.

* Find a full install in host.factorio_directory instead of a setting

Per review, drop host.factorio_graphics_directory. findVersion takes a
headless option that filters installs by build, telling them apart by
data/base/graphics/icons. When the instance runs the headless build the
export looks for a full install of the same major.minor version in the
host's factorio directory. Server startup is unchanged.

If none is found the export still runs, and the reason is included in
the missing icons warning.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Address review on findVersion and the export graphics lookup

findVersion takes headless as a plain optional argument, the helper is
checkInstallType, the warning is checkMissingBuiltinIcons and the full
install lookup is inlined into exportData.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Danielv123 <danielv@danielv.no>

docs/configuration.md

@@ -354,6 +354,7 @@ Defaults to a random 31 bit number.
 
 Directory to look for the Factorio server(s) in.
 This can either point to a Factorio server directory, or to a directory containing multiple versions of the Factorio server.
+The headless build has no graphics, so when exporting data from an instance running it the base game's icons are read from a full install of the same major and minor version in this directory instead, if there is one.
 
 Defaults to "factorio".
 

packages/host/src/export.ts

@@ -10,7 +10,7 @@ import yazl from "yazl";
 
 import * as lib from "@clusterio/lib";
 import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js";
-import type { FactorioServer } from "./server.js";
+import { findVersion, isHeadless, type FactorioServer } from "./server.js";
 import { definesPrototypesFallback } from "./export_fallback.js";
 
 interface SimpleIconSpecification {
@@ -95,6 +95,16 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
 	return modZip;
 }
 
+/** Where icon files are read from during an export. */
+interface ExportFiles {
+	/** Mod zips opened during the export, closed when it ends. */
+	zipCache: ZipCache;
+	/** Data dir of a full install to fall back to for builtin mod files. */
+	graphicsDataDir?: string;
+	/** Why no full install was found for a headless server. */
+	graphicsError?: string;
+}
+
 function closeZipCache(zipCache: ZipCache) {
 	for (const { zip } of zipCache.values()) {
 		zip.close();
@@ -107,7 +117,7 @@ function closeZipCache(zipCache: ZipCache) {
  *
  * @param server - The server to load the file from.
  * @param modVersions - Mapping of mod to version used.
- * @param zipCache - Mod zips opened by this export.
+ * @param files - Where to read files from.
  * @param modPath - Factorio style path to the file to load.
  * @returns The content of the file or null if not found.
  * @internal
@@ -115,7 +125,7 @@ function closeZipCache(zipCache: ZipCache) {
 async function loadFile(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	modPath: string,
 ) {
 	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
@@ -128,19 +138,25 @@ async function loadFile(
 
 	const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)];
 	if (builtinModNames.includes(mod)) {
-		try {
-			return await fs.readFile(server.dataPath(mod, filePath));
-		} catch (err: any) {
-			if (err.code === "ENOENT") {
-				return null;
-			}
-			throw err;
+		const dataDirs = [server.dataPath()];
+		if (files.graphicsDataDir) {
+			dataDirs.push(files.graphicsDataDir);
 		}
+		for (const dataDir of dataDirs) {
+			try {
+				return await fs.readFile(path.join(dataDir, mod, filePath));
+			} catch (err: any) {
+				if (err.code !== "ENOENT") {
+					throw err;
+				}
+			}
+		}
+		return null;
 	}
 
 	let modZip;
 	try {
-		modZip = await loadZip(server, modVersions, zipCache, mod);
+		modZip = await loadZip(server, modVersions, files.zipCache, mod);
 	} catch (err: any) {
 		if (err.code === "ENOENT") {
 			return null;
@@ -158,7 +174,7 @@ type IconCache = Map<string, Image | null>;
 async function loadIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	iconPath: string,
 	iconSize: number,
 	iconMipmaps: number,
@@ -166,7 +182,7 @@ async function loadIcon(
 ) {
 	let icon = iconCache.get(iconPath);
 	if (icon === undefined) {
-		let fileContent = await loadFile(server, modVersions, zipCache, iconPath);
+		let fileContent = await loadFile(server, modVersions, files, iconPath);
 		if (fileContent) {
 			icon = await Jimp.fromBuffer(fileContent) as Image;
 			icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize });
@@ -182,12 +198,12 @@ async function loadIcon(
 async function loadSimpleIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	item: SimpleIconSpecification,
 	size: number,
 	iconCache: IconCache,
 ): Promise<Image | null> {
-	let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache);
+	let icon = await loadIcon(server, modVersions, files, item.icon, item.icon_size ?? 64, 0, iconCache);
 	if (icon) {
 		let iconScale = size / (item.icon_size ?? 64);
 		if (iconScale !== 1) {
@@ -201,11 +217,11 @@ async function loadSimpleIcon(
 async function loadLayeredIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	item: LayeredIconSpecification,
 	size: number,
 	iconCache: IconCache,
-): Promise<Image> {
+): Promise<Image | null> {
 	let baseLayerSize = (item.icons[0].icon_size || item.icon_size) ?? 64;
 	let icon = new Jimp({ width: size, height: size });
 
@@ -218,11 +234,12 @@ async function loadLayeredIcon(
 	for (let layer of item.icons) {
 		let layerSize = (layer.icon_size || item.icon_size) ?? 64;
 		let iconLayer = await loadIcon(
-			server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
+			server, modVersions, files, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
 		);
 
+		// A partial icon would be wrong, skip it like a missing simple icon
 		if (!iconLayer) {
-			continue;
+			return null;
 		}
 
 		iconLayer = iconLayer.clone();
@@ -337,7 +354,7 @@ async function growSheet(state: SheetState) {
 async function exportIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	state: SheetState,
 	prototype: PrototypeWithIcon,
 ) {
@@ -348,9 +365,11 @@ async function exportIcon(
 
 	if (prototype.icons) {
 		icon = await loadLayeredIcon(
-			server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
+			server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache,
 		);
-		iconPos = state.pos;
+		if (icon) {
+			iconPos = state.pos;
+		}
 		iconPath = prototype.icons[0].icon;
 
 	} else if (prototype.icon) {
@@ -358,7 +377,7 @@ async function exportIcon(
 		iconPath = prototype.icon;
 		if (iconPos === undefined) {
 			icon = await loadSimpleIcon(
-				server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
+				server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache,
 			);
 			if (icon) {
 				iconPos = state.pos;
@@ -394,7 +413,7 @@ async function exportIcon(
  *
  * @param server - The server to load icons from.
  * @param modVersions - Mapping of mod name to version.
- * @param zipCache - Mod zips opened by this export.
+ * @param files - Where to read icon files from.
  * @param prototypes - Array of prototypes with icon to export for this base type.
  * @param baseName - name of base type (used in metadata).
  * @param state - Shared spritesheet state (mutated in place).
@@ -404,7 +423,7 @@ async function exportIcon(
 async function exportIcons(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
-	zipCache: ZipCache,
+	files: ExportFiles,
 	prototypes: PrototypeWithIcon[],
 	baseName: string,
 	state: SheetState,
@@ -415,7 +434,7 @@ async function exportIcons(
 	for (let prototype of prototypes) {
 		let icon: lib.ExportMetadataEntry["icon"];
 		try {
-			icon = await exportIcon(server, modVersions, zipCache, state, prototype);
+			icon = await exportIcon(server, modVersions, files, state, prototype);
 		} catch (err: any) {
 			// One broken icon should not stop the rest of the export
 			server._logger.warn(`Skipping icon of ${prototype.type} ${prototype.name}: ${err.message}`);
@@ -510,20 +529,67 @@ async function exportLocale(
 /**
  * Export locale, icons and metadata for the given factorio server
  *
+ * If the server runs the headless build, builtin mod graphics are read
+ * from a full install of the same major and minor version found next to
+ * it, if there is one.
+ *
  * @param server - The server to export the data from.
  * @returns zip file with exported data.
  */
 export async function exportData(server: FactorioServer) {
-	const zipCache: ZipCache = new Map();
+	const files: ExportFiles = { zipCache: new Map() };
+	if (await isHeadless(server.dataPath())) {
+		const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion;
+		try {
+			[files.graphicsDataDir] = await findVersion(server._factorioDir, majorMinor, false);
+		} catch (err: any) {
+			// Without it the export still has everything but the base game icons
+			files.graphicsError = err.message;
+		}
+	}
 	try {
-		return await buildExport(server, zipCache);
+		return await buildExport(server, files);
 	} finally {
 		// Free up the zip files loaded during the export.
-		closeZipCache(zipCache);
+		closeZipCache(files.zipCache);
 	}
 }
 
-async function buildExport(server: FactorioServer, zipCache: ZipCache) {
+/**
+ * Warn if any builtin mod icon files could not be found
+ *
+ * @param server - The server the export ran on.
+ * @param files - Where icon files were read from.
+ * @param iconCache - Icons loaded by the export, null for missing files.
+ * @internal
+ */
+function checkMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, iconCache: IconCache) {
+	const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)];
+	let missing = 0;
+	for (const [iconPath, icon] of iconCache) {
+		const mod = /^__([^\/]+)__\//.exec(iconPath)?.[1];
+		if (icon === null && mod && builtinModNames.includes(mod)) {
+			missing += 1;
+		}
+	}
+	if (!missing) {
+		return;
+	}
+
+	const dirs = [server.dataPath(), files.graphicsDataDir].filter(Boolean).join(" or ");
+	let message =
+		`${missing} icon files of the base game were not found in ${dirs}, icons using them were not exported.`;
+	if (files.graphicsError !== undefined) {
+		const majorMinor = server.version!.replace(/\.\d+$/, "");
+		message +=
+			` The headless build of Factorio has no graphics and no full install was found (${files.graphicsError}).` +
+			` Add a full install of Factorio ${majorMinor} to host.factorio_directory to read them from there,` +
+			" or export the data on a host with a full install.";
+	}
+	server._logger.warn(message);
+}
+
+async function buildExport(server: FactorioServer, files: ExportFiles) {
 	await generateExportMod(server);
 
 	let settings: lib.ExportSettings = {};
@@ -603,7 +669,7 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) {
 		}
 	}
 
-	let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en");
+	let locale = await exportLocale(server, modVersions, files.zipCache, modOrder, "en");
 
 	let zip = new yazl.ZipFile();
 	const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name);
@@ -624,9 +690,10 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) {
 		if (prototypesWithIcons.length === 0) {
 			continue;
 		}
-		const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state);
+		const count = await exportIcons(server, modVersions, files, prototypesWithIcons, baseName, state);
 		server._logger.info(`Exported ${count} ${baseName} icons`);
 	}
+	checkMissingBuiltinIcons(server, files, state.iconCache);
 
 	// Crop sheet to actual used height and write single spritesheet + metadata.
 	const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
@@ -647,3 +714,4 @@ export const _exportIcons = exportIcons;
 export const _exportLocale = exportLocale;
 export const _loadSimpleIcon = loadSimpleIcon;
 export const _loadLayeredIcon = loadLayeredIcon;
+export const _checkMissingBuiltinIcons = checkMissingBuiltinIcons;

packages/host/src/server.ts

@@ -262,15 +262,33 @@ export async function checkForUpdates(
  * @param targetVersion -
  *     Version to look for, supports the special value "latest" for the
  *     latest version available.
+ * @param headless -
+ *     Only consider headless installs if true, or only full installs if
+ *     false.
  * @returns Array with path to data dir and version found.
  */
 export async function findVersion(
 	factorioDir: string,
 	targetVersion: lib.TargetVersion,
+	headless?: boolean,
 ): Promise<[string, lib.FullVersion]> {
+	let kind = "";
+	if (headless !== undefined) {
+		kind = headless ? "headless " : "full ";
+	}
+	const checkInstallType = async (dir: string) => (
+		headless === undefined || await isHeadless(path.join(dir, "data")) === headless
+	);
+
 	// Check if this is a direct installation dir
 	const directVersion = await getFactorioVersion(factorioDir);
 	if (directVersion !== null) {
+		if (!await checkInstallType(factorioDir)) {
+			throw new Error(
+				`Unable to find a ${kind}install of Factorio ${targetVersion}: ${factorioDir} is a direct ` +
+				`(single-version) install of the ${headless ? "full" : "headless"} build.`
+			);
+		}
 		if (targetVersion === "latest" || directVersion === targetVersion || directVersion.startsWith(targetVersion)) {
 			return [path.join(factorioDir, "data"), directVersion];
 		}
@@ -287,7 +305,7 @@ export async function findVersion(
 		// Avoid any fancy directory detection in order to follow symlinks
 		const fullPath = path.join(factorioDir, name);
 		const version = await getFactorioVersion(fullPath);
-		if (version === null) {
+		if (version === null || !await checkInstallType(fullPath)) {
 			continue;
 		}
 
@@ -299,7 +317,7 @@ export async function findVersion(
 	}
 
 	if (!versions.size) {
-		throw new Error(`Unable to find any Factorio install in ${factorioDir}`);
+		throw new Error(`Unable to find any ${kind}Factorio install in ${factorioDir}`);
 	}
 
 	if (lib.isPartialVersion(targetVersion)) {
@@ -315,7 +333,26 @@ export async function findVersion(
 		return [path.join(factorioDir, versions.get(latest)!, "data"), latest];
 	}
 
-	throw new Error(`Unable to find Factorio version ${targetVersion}`);
+	throw new Error(`Unable to find ${kind}Factorio version ${targetVersion}`);
+}
+
+/**
+ * Check if a Factorio install is the headless build
+ *
+ * The headless build has no icons, which is what tells it apart.
+ *
+ * @param dataDir - Path to the data dir of the install.
+ */
+export async function isHeadless(dataDir: string) {
+	try {
+		await fs.access(path.join(dataDir, "base", "graphics", "icons"));
+		return false;
+	} catch (err: any) {
+		if (err.code === "ENOENT" || err.code === "ENOTDIR") {
+			return true;
+		}
+		throw err;
+	}
 }
 
 /**

test/file/factorio-mixed/full/data/changelog.txt

@@ -0,0 +1,3 @@
+---------------------------------------------------------------------------------------------------
+Version: 0.1.2
+About: Dummy changelog.txt for FactorioServer tests

test/file/factorio-mixed/headless/data/changelog.txt

@@ -0,0 +1,3 @@
+---------------------------------------------------------------------------------------------------
+Version: 0.1.2
+About: Dummy changelog.txt for FactorioServer tests

test/host/export.js

@@ -3,7 +3,7 @@ import path from "node:path";
 
 import { FactorioServer } from "@clusterio/host/dist/node/src/server.js";
 import {
-	_createSheetState, _exportIcons, _exportLocale, _loadLayeredIcon, _loadSimpleIcon,
+	_checkMissingBuiltinIcons, _createSheetState, _exportIcons, _exportLocale, _loadLayeredIcon, _loadSimpleIcon,
 } from "@clusterio/host/dist/node/src/export.js";
 
 
@@ -60,7 +60,7 @@ describe("host/src/export", function() {
 				{ type: "item", name: "unknown-mod", icon: "__not-a-mod__/graphics/icons/x.png", icon_size: 64 },
 				{ type: "item", name: "last", icon, icon_size: 64 },
 			];
-			const count = await _exportIcons(testServer, new Map(), new Map(), prototypes, "item", state);
+			const count = await _exportIcons(testServer, new Map(), { zipCache: new Map() }, prototypes, "item", state);
 			assert.equal(count, 2);
 			assert.deepEqual(
 				state.metadata.item.map(entry => [entry.name, Boolean(entry.icon)]),
@@ -72,20 +72,37 @@ describe("host/src/export", function() {
 		});
 	});
 
+	// Stands in for the data dir of a full install
+	const graphicsDataDir = path.join("test", "file", "factorio-mixed", "full", "data");
 	describe("loadSimpleIcon()", function() {
 		it("defaults icon_size to 64", async function() {
 			const icon = await _loadSimpleIcon(
-				testServer, new Map(), new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map()
+				testServer, new Map(), { zipCache: new Map() },
+				{ icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map(),
 			);
 			assert.equal(icon.bitmap.width, 32);
 			assert.equal(icon.bitmap.height, 32);
 			assert.equal(icon.getPixelColor(31, 16), red);
 		});
+		it("returns null when the icon file is missing", async function() {
+			const icon = await _loadSimpleIcon(
+				testServer, new Map(), { zipCache: new Map() },
+				{ icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(),
+			);
+			assert.equal(icon, null);
+		});
+		it("falls back to graphicsDataDir for builtin mod files", async function() {
+			const icon = await _loadSimpleIcon(
+				testServer, new Map(), { zipCache: new Map(), graphicsDataDir },
+				{ icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(),
+			);
+			assert.equal(icon.getPixelColor(31, 16), red);
+		});
 	});
 	describe("loadLayeredIcon()", function() {
 		it("defaults icon_size to 64", async function() {
 			const icon = await _loadLayeredIcon(
-				testServer, new Map(), new Map(),
+				testServer, new Map(), { zipCache: new Map() },
 				{ icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map(),
 			);
 			assert.equal(icon.bitmap.width, 32);
@@ -93,5 +110,43 @@ describe("host/src/export", function() {
 			assert.equal(icon.getPixelColor(0, 0), red);
 			assert.equal(icon.getPixelColor(31, 31), red);
 		});
+		it("returns null when a layer is missing", async function() {
+			const icon = await _loadLayeredIcon(
+				testServer, new Map(), { zipCache: new Map() },
+				{ icons: [
+					{ icon: "__base__/graphics/icons/test-icon.png" },
+					{ icon: "__base__/graphics/icons/missing.png" },
+				] }, 32, new Map(),
+			);
+			assert.equal(icon, null);
+		});
+	});
+	describe("checkMissingBuiltinIcons()", function() {
+		it("warns once with the number of missing builtin files", function() {
+			_checkMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([
+				["__base__/graphics/icons/a.png", null],
+				["__core__/graphics/icons/b.png", null],
+				["__base__/graphics/icons/c.png", {}],
+				["__some_mod__/graphics/icons/d.png", null],
+			]));
+			assert.equal(warnings.length, 1);
+			assert.match(warnings[0], /^2 icon files of the base game were not found/);
+			assert.doesNotMatch(warnings[0], /headless/);
+		});
+		it("explains the headless build when no full install was found", function() {
+			_checkMissingBuiltinIcons(testServer, { zipCache: new Map(), graphicsError: "Not found" }, new Map([
+				["__base__/graphics/icons/a.png", null],
+			]));
+			assert.equal(warnings.length, 1);
+			assert.match(warnings[0], /has no graphics and no full install was found \(Not found\)/);
+			assert.match(warnings[0], /Add a full install of Factorio 0\.1 to host\.factorio_directory/);
+		});
+		it("does not warn when no builtin files are missing", function() {
+			_checkMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([
+				["__base__/graphics/icons/c.png", {}],
+				["__some_mod__/graphics/icons/d.png", null],
+			]));
+			assert.deepEqual(warnings, []);
+		});
 	});
 });

test/host/server.js

@@ -105,6 +105,38 @@ describe("host/server", function() {
 				);
 			});
 		});
+		describe("headless argument", function() {
+			const mixedDir = path.join("test", "file", "factorio-mixed");
+			it("should find only full installs when false", async function() {
+				const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", false);
+				assert.equal(dir, path.join(mixedDir, "full", "data"));
+			});
+			it("should find only headless installs when true", async function() {
+				const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", true);
+				assert.equal(dir, path.join(mixedDir, "headless", "data"));
+			});
+			it("should skip installs of the other build when matching a partial version", async function() {
+				const installDir = path.join("test", "file", "factorio");
+				const [dir, version] = await hostServer.findVersion(installDir, "0.1", false);
+				assert.deepEqual([dir, version], [path.join(installDir, "0.1.2", "data"), "0.1.2"]);
+			});
+			it("should reject a direct install of the other build", async function() {
+				const installDir = path.join(mixedDir, "headless");
+				await assert.rejects(
+					hostServer.findVersion(installDir, "0.1.2", false),
+					new Error(
+						`Unable to find a full install of Factorio 0.1.2: ${installDir} is a direct ` +
+						"(single-version) install of the headless build."
+					)
+				);
+			});
+			it("should reject if no install of that build has the version", async function() {
+				await assert.rejects(
+					hostServer.findVersion(path.join("test", "file", "factorio"), "0.1.2", true),
+					new Error("Unable to find headless Factorio version 0.1.2")
+				);
+			});
+		});
 	});
 
 	describe("_listFactorioVersions()", function() {