diff --git a/docs/configuration.md b/docs/configuration.md index 1818a357..b7064174 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -339,6 +339,15 @@ This can either point to a Factorio server directory, or to a directory containi Defaults to "factorio". +### host.factorio_graphics_directory + +Directory to read the base game's icons from when exporting data. +The headless build of Factorio has no graphics, so instances running it export none of the base game's icons unless this points to a full install of Factorio. +Like `host.factorio_directory` this can either point to a Factorio directory, or to a directory containing multiple versions, in which case the latest one matching the major and minor version of the instance is used. + +Defaults to null. + + ### host.mods_directory Directory the host stores mods downloaded from the controller in. diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index f70080d3..7596a8fc 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -1469,7 +1469,7 @@ end`.replace(/\r?\n/g, " "); this.logger.info("Exporting data ....."); await this.syncMods(); - let zip = await exportData(this.server); + let zip = await exportData(this.server, this._host.config.get("host.factorio_graphics_directory")); let url = new URL(this._host.config.get("host.controller_url")); url.pathname += "api/upload-export"; diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index cb7f73d5..9554e5c5 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -6,7 +6,7 @@ import yazl from "yazl"; import * as lib from "@clusterio/lib"; import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js"; -import type { FactorioServer } from "./server.js"; +import { findVersion, type FactorioServer } from "./server.js"; import { definesPrototypesFallback } from "./export_fallback.js"; interface SimpleIconSpecification { @@ -91,6 +91,13 @@ async function loadZip(server: FactorioServer, modVersions: Map, return modZip; } +/** Where icon files are read from during an export. */ +interface ExportFiles { + zipCache: ZipCache; + /** Data dir of a full install to fall back to for builtin mod files. */ + graphicsDataDir?: string; +} + function closeZipCache(zipCache: ZipCache) { for (const { zip } of zipCache.values()) { zip.close(); @@ -103,7 +110,7 @@ function closeZipCache(zipCache: ZipCache) { * * @param server - The server to load the file from. * @param modVersions - Mapping of mod to version used. - * @param zipCache - Mod zips opened by this export. + * @param files - Where to read files from. * @param modPath - Factorio style path to the file to load. * @returns The content of the file or null if not found. * @internal @@ -111,7 +118,7 @@ function closeZipCache(zipCache: ZipCache) { async function loadFile( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, modPath: string, ) { let match = /^__([^\/]+)__\/(.*)$/.exec(modPath); @@ -124,19 +131,25 @@ async function loadFile( const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)]; if (builtinModNames.includes(mod)) { - try { - return await fs.readFile(server.dataPath(mod, filePath)); - } catch (err: any) { - if (err.code === "ENOENT") { - return null; - } - throw err; + const dataDirs = [server.dataPath()]; + if (files.graphicsDataDir) { + dataDirs.push(files.graphicsDataDir); } + for (const dataDir of dataDirs) { + try { + return await fs.readFile(path.join(dataDir, mod, filePath)); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + } + } + return null; } let modZip; try { - modZip = await loadZip(server, modVersions, zipCache, mod); + modZip = await loadZip(server, modVersions, files.zipCache, mod); } catch (err: any) { if (err.code === "ENOENT") { return null; @@ -152,7 +165,7 @@ type IconCache = Map; async function loadIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, iconPath: string, iconSize: number, iconMipmaps: number, @@ -160,7 +173,7 @@ async function loadIcon( ) { let icon = iconCache.get(iconPath); if (icon === undefined) { - let fileContent = await loadFile(server, modVersions, zipCache, iconPath); + let fileContent = await loadFile(server, modVersions, files, iconPath); if (fileContent) { icon = await Jimp.fromBuffer(fileContent) as Image; icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize }); @@ -176,12 +189,12 @@ async function loadIcon( async function loadSimpleIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, item: SimpleIconSpecification, size: number, iconCache: IconCache, ): Promise { - let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache); + let icon = await loadIcon(server, modVersions, files, item.icon, item.icon_size ?? 64, 0, iconCache); if (icon) { let iconScale = size / (item.icon_size ?? 64); if (iconScale !== 1) { @@ -195,11 +208,11 @@ async function loadSimpleIcon( async function loadLayeredIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, item: LayeredIconSpecification, size: number, iconCache: IconCache, -): Promise { +): Promise { let baseLayerSize = (item.icons[0].icon_size || item.icon_size) ?? 64; let icon = new Jimp({ width: size, height: size }); @@ -212,11 +225,12 @@ async function loadLayeredIcon( for (let layer of item.icons) { let layerSize = (layer.icon_size || item.icon_size) ?? 64; let iconLayer = await loadIcon( - server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache, + server, modVersions, files, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache, ); + // A partial icon would be wrong, skip it like a missing simple icon if (!iconLayer) { - continue; + return null; } iconLayer = iconLayer.clone(); @@ -331,7 +345,7 @@ async function growSheet(state: SheetState) { async function exportIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, state: SheetState, prototype: PrototypeWithIcon, ) { @@ -342,9 +356,11 @@ async function exportIcon( if (prototype.icons) { icon = await loadLayeredIcon( - server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache, + server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache, ); - iconPos = state.pos; + if (icon) { + iconPos = state.pos; + } iconPath = prototype.icons[0].icon; } else if (prototype.icon) { @@ -352,7 +368,7 @@ async function exportIcon( iconPath = prototype.icon; if (iconPos === undefined) { icon = await loadSimpleIcon( - server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache, + server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache, ); if (icon) { iconPos = state.pos; @@ -388,7 +404,7 @@ async function exportIcon( * * @param server - The server to load icons from. * @param modVersions - Mapping of mod name to version. - * @param zipCache - Mod zips opened by this export. + * @param files - Where to read icon files from. * @param prototypes - Array of prototypes with icon to export for this base type. * @param baseName - name of base type (used in metadata). * @param state - Shared spritesheet state (mutated in place). @@ -398,7 +414,7 @@ async function exportIcon( async function exportIcons( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, prototypes: PrototypeWithIcon[], baseName: string, state: SheetState, @@ -407,7 +423,7 @@ async function exportIcons( state.metadata[baseName] ??= metadata; let count = 0; for (let prototype of prototypes) { - const icon = await exportIcon(server, modVersions, zipCache, state, prototype); + const icon = await exportIcon(server, modVersions, files, state, prototype); if (icon) { count += 1; } @@ -498,19 +514,57 @@ async function exportLocale( * Export locale, icons and metadata for the given factorio server * * @param server - The server to export the data from. + * @param graphicsDirectory - + * Full Factorio install, or directory of installs, to read builtin mod + * graphics from when the server's install lacks them. * @returns zip file with exported data. */ -export async function exportData(server: FactorioServer) { - const zipCache: ZipCache = new Map(); +export async function exportData(server: FactorioServer, graphicsDirectory?: string | null) { + const files: ExportFiles = { zipCache: new Map() }; + if (graphicsDirectory) { + const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion; + [files.graphicsDataDir] = await findVersion(graphicsDirectory, majorMinor); + } try { - return await buildExport(server, zipCache); + return await buildExport(server, files); } finally { // Free up the zip files loaded during the export. - closeZipCache(zipCache); + closeZipCache(files.zipCache); } } -async function buildExport(server: FactorioServer, zipCache: ZipCache) { +/** + * Warn about builtin mod icon files that could not be found + * + * @param server - The server the export ran on. + * @param files - Where icon files were read from. + * @param iconCache - Icons loaded by the export, null for missing files. + * @internal + */ +function warnMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, iconCache: IconCache) { + const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)]; + let missing = 0; + for (const [iconPath, icon] of iconCache) { + const mod = /^__([^\/]+)__\//.exec(iconPath)?.[1]; + if (icon === null && mod && builtinModNames.includes(mod)) { + missing += 1; + } + } + if (!missing) { + return; + } + + const dirs = [server.dataPath(), files.graphicsDataDir].filter(Boolean).join(" or "); + const hint = files.graphicsDataDir + ? "Check that host.factorio_graphics_directory points to a full Factorio install" + : "Set host.factorio_graphics_directory to a full Factorio install to read them from there"; + server._logger.warn( + `${missing} icon files of the base game were not found in ${dirs}, icons using them were not exported. ` + + `The headless build of Factorio has no graphics. ${hint}, or export the data on a host with a full install.` + ); +} + +async function buildExport(server: FactorioServer, files: ExportFiles) { await generateExportMod(server); let settings: lib.ExportSettings = {}; @@ -590,7 +644,7 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) { } } - let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en"); + let locale = await exportLocale(server, modVersions, files.zipCache, modOrder, "en"); let zip = new yazl.ZipFile(); const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name); @@ -611,9 +665,10 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) { if (prototypesWithIcons.length === 0) { continue; } - const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state); + const count = await exportIcons(server, modVersions, files, prototypesWithIcons, baseName, state); server._logger.info(`Exported ${count} ${baseName} icons`); } + warnMissingBuiltinIcons(server, files, state.iconCache); // Crop sheet to actual used height and write single spritesheet + metadata. const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE)); @@ -632,3 +687,4 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) { export const _exportLocale = exportLocale; export const _loadSimpleIcon = loadSimpleIcon; export const _loadLayeredIcon = loadLayeredIcon; +export const _warnMissingBuiltinIcons = warnMissingBuiltinIcons; diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 5d19897a..93a7a43e 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -324,6 +324,7 @@ export interface HostConfigFields { "host.name": string; "host.id": number; "host.factorio_directory": string; + "host.factorio_graphics_directory": string | null; "host.mods_directory": string; "host.instances_directory": string; "host.controller_url": string; @@ -372,6 +373,14 @@ export class HostConfig extends classes.Config { initialValue: "factorio", validator: validators.filePath, }, + "host.factorio_graphics_directory": { + description: + "Path to a full Factorio install, or directory of installs, to read base game icons from when " + + "exporting data. Needed when running the headless build, which has no graphics.", + type: "string", + optional: true, + validator: validators.optional(validators.filePath), + }, "host.mods_directory": { title: "Mods Directory", description: "Path to directory where mods for instances are cached.", diff --git a/test/file/factorio-graphics/base/graphics/icons/graphics-icon.png b/test/file/factorio-graphics/base/graphics/icons/graphics-icon.png new file mode 100644 index 00000000..b2f06ccc Binary files /dev/null and b/test/file/factorio-graphics/base/graphics/icons/graphics-icon.png differ diff --git a/test/host/export.js b/test/host/export.js index 43109532..c371145f 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -2,7 +2,9 @@ import assert from "node:assert/strict"; import path from "node:path"; import { FactorioServer } from "@clusterio/host/dist/node/src/server.js"; -import { _exportLocale, _loadLayeredIcon, _loadSimpleIcon } from "@clusterio/host/dist/node/src/export.js"; +import { + _exportLocale, _loadLayeredIcon, _loadSimpleIcon, _warnMissingBuiltinIcons, +} from "@clusterio/host/dist/node/src/export.js"; describe("host/src/export", function() { @@ -30,20 +32,37 @@ describe("host/src/export", function() { // test-icon.png is a red 64px icon followed by a blue 32px mipmap const red = 0xff0000ff; + // Stands in for the data dir of a full install + const graphicsDataDir = path.join("test", "file", "factorio-graphics"); describe("loadSimpleIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadSimpleIcon( - testServer, new Map(), new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map() + testServer, new Map(), { zipCache: new Map() }, + { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map(), ); assert.equal(icon.bitmap.width, 32); assert.equal(icon.bitmap.height, 32); assert.equal(icon.getPixelColor(31, 16), red); }); + it("returns null when the icon file is missing", async function() { + const icon = await _loadSimpleIcon( + testServer, new Map(), { zipCache: new Map() }, + { icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(), + ); + assert.equal(icon, null); + }); + it("falls back to graphicsDataDir for builtin mod files", async function() { + const icon = await _loadSimpleIcon( + testServer, new Map(), { zipCache: new Map(), graphicsDataDir }, + { icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(), + ); + assert.equal(icon.getPixelColor(31, 16), red); + }); }); describe("loadLayeredIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadLayeredIcon( - testServer, new Map(), new Map(), + testServer, new Map(), { zipCache: new Map() }, { icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map(), ); assert.equal(icon.bitmap.width, 32); @@ -51,5 +70,46 @@ describe("host/src/export", function() { assert.equal(icon.getPixelColor(0, 0), red); assert.equal(icon.getPixelColor(31, 31), red); }); + it("returns null when a layer is missing", async function() { + const icon = await _loadLayeredIcon( + testServer, new Map(), { zipCache: new Map() }, + { icons: [ + { icon: "__base__/graphics/icons/test-icon.png" }, + { icon: "__base__/graphics/icons/missing.png" }, + ] }, 32, new Map(), + ); + assert.equal(icon, null); + }); + }); + describe("warnMissingBuiltinIcons()", function() { + let warnings; + let logger; + beforeEach(function() { + warnings = []; + logger = testServer._logger; + testServer._logger = { warn: msg => warnings.push(msg) }; + }); + afterEach(function() { + testServer._logger = logger; + }); + + it("warns once with the number of missing builtin files", function() { + _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ + ["__base__/graphics/icons/a.png", null], + ["__core__/graphics/icons/b.png", null], + ["__base__/graphics/icons/c.png", {}], + ["__some_mod__/graphics/icons/d.png", null], + ])); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /^2 icon files of the base game were not found/); + assert.match(warnings[0], /headless/); + }); + it("does not warn when no builtin files are missing", function() { + _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ + ["__base__/graphics/icons/c.png", {}], + ["__some_mod__/graphics/icons/d.png", null], + ])); + assert.deepEqual(warnings, []); + }); }); });