Move Factorio server implementation to host

Move Factorio related code in lib that's only used by the host into the
host package.
This commit is contained in:
Hornwitser committed 2023-07-28 16:03:29 +02:00
1 parent bc574ee1cb
commit 6af25e085c
13 files changed
+95 -100

No files matched your search

+7 -3
View File
@@ -9,6 +9,10 @@ const util = require("util");
const lib = require("@clusterio/lib");
const { PlayerStats } = lib;
const { FactorioServer } = require("./server");
const { patch } = require("./patch");
const { exportData } = require("./export");
const instanceRconCommandDuration = new lib.Histogram(
"clusterio_instance_rcon_command_duration_seconds",
@@ -143,7 +147,7 @@ class Instance extends lib.Link {
// Valid statuses are stopped, starting, running, stopping, creating_save and exporting_data.
this._status = "stopped";
this._loadedSave = null;
this.server = new lib.FactorioServer(
this.server = new FactorioServer(
factorioDir, this._dir, serverOptions
);
@@ -823,7 +827,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
}
}
await lib.patch(this.path("saves", saveName), [...modules.values()]);
await patch(this.path("saves", saveName), [...modules.values()]);
return saveName;
}
@@ -1082,7 +1086,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
this.logger.info("Exporting data .....");
await this.syncMods();
let zip = await lib.exportData(this.server);
let zip = await exportData(this.server);
let content = await zip.generateAsync({ type: "nodebuffer" });
let url = new URL(this._host.config.get("host.controller_url"));
+473
View File
@@ -0,0 +1,473 @@
// Export of item icons and locale
"use strict";
const fs = require("fs-extra");
const path = require("path");
const Jimp = require("jimp");
const JSZip = require("jszip");
const lib = require("@clusterio/lib");
const libBuildMod = require("@clusterio/lib/build_mod");
/**
* Generate the export mod needed for exportData
*
* Packs the lua export mod located in lua/export into the mods folder for
* the server given, with dependencies generated for all the mods present in
* the server's mods folder.
*
* @param {module:host/src/server.FactorioServer} server -
* The server to generate the export mod for.
* @memberof module:lib
*/
async function generateExportMod(server) {
let dependencies = [];
let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/;
for (let entry of await fs.readdir(server.writePath("mods"))) {
let match = splitter.exec(entry);
if (match && match[1] !== "export") {
dependencies.push(`? ${match[1]}`);
}
}
await libBuildMod.build({
clean: false,
build: true,
pack: true,
sourceDir: path.join(__dirname, "..", "lua", "export"),
outputDir: server.writePath("mods"),
bumpPatch: false,
factorioVersion: server.version.replace(/\.\d+$/, ""),
dependencies,
});
}
let zipCache = new Map();
async function loadZip(server, modVersions, mod) {
let modVersion = modVersions.get(mod);
if (!modVersion) {
throw new Error(`Got path for unknown mod ${mod}`);
}
let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`);
let zip = zipCache.get(zipPath);
if (!zip) {
zip = await JSZip.loadAsync(await fs.readFile(zipPath));
zipCache.set(zipPath, zip);
}
return zip.folder(lib.findRoot(zip));
}
/**
* Load the given Factorio file path into a Buffer
*
* @param {module:host/src/server.FactorioServer} server -
* The server to load the file from.
* @param {Map<string, string>} modVersions - Mapping of mod to version used.
* @param {string} modPath - Factorio style path to the file to load.
* @returns {Promise<?Buffer>} The content of the file or null if not found.
* @memberof module:lib
* @private
* @inner
*/
async function loadFile(server, modVersions, modPath) {
let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
if (!match) {
throw new Error(`Bad mod path ${match}`);
}
let [, mod, filePath] = match;
if (["core", "base"].includes(mod)) {
try {
return await fs.readFile(server.dataPath(mod, filePath));
} catch (err) {
if (err.code === "ENOENT") {
return null;
}
throw err;
}
}
let zip;
try {
zip = await loadZip(server, modVersions, mod);
} catch (err) {
if (err.code === "ENOENT") {
return null;
}
throw err;
}
let file = zip.file(filePath);
if (!file) {
return null;
}
return await file.async("nodebuffer");
}
async function loadIcon(server, modVersions, iconPath, iconSize, iconMipmaps, iconCache) {
let icon = iconCache.get(iconPath);
if (icon === undefined) {
let fileContent = await loadFile(server, modVersions, iconPath);
if (fileContent) {
icon = await Jimp.read(fileContent);
icon.crop(0, 0, iconSize, iconSize);
iconCache.set(iconPath, icon);
} else {
icon = null;
server._logger.warn(`${iconPath} not found`);
}
iconCache.set(iconPath, icon);
}
return icon;
}
async function loadSimpleIcon(server, modVersions, item, size, iconCache) {
let icon = await loadIcon(server, modVersions, item.icon, item.icon_size, 0, iconCache);
if (icon) {
let iconScale = size / item.icon_size;
if (iconScale !== 1) {
icon = icon.clone();
icon.scale(iconScale);
}
}
return icon;
}
async function loadLayeredIcon(server, modVersions, item, size, iconCache) {
let baseLayerSize = item.icons[0].icon_size || item.icon_size;
let icon = await Jimp.create(size, size);
// The scaling factor of the base layer
let baseLayerScale = item.icons[0].scale || 32 / baseLayerSize;
// The size in pixels of one unit
let baseUnit = size / (baseLayerSize * baseLayerScale);
for (let layer of item.icons) {
let layerSize = layer.icon_size || item.icon_size;
let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
if (!iconLayer) {
continue;
}
iconLayer = iconLayer.clone();
let tint;
if (layer.tint) {
let divisor = (layer.tint.r > 1 || layer.tint.g > 1 || layer.tint.b > 1) ? 255 : 1;
tint = {
r: (layer.tint.r || 0) / divisor,
g: (layer.tint.g || 0) / divisor,
b: (layer.tint.b || 0) / divisor,
a: layer.tint.a || 1,
};
} else {
tint = { r: 1, b: 1, g: 1, a: 1 };
}
let layerScale = layer.scale || 32 / layerSize;
let [xs, ys] = layer.shift || [0, 0];
let realScale = layerScale * baseUnit;
let sizeShift = (size - layerSize * realScale) / 2;
xs = xs * baseUnit + sizeShift;
ys = ys * baseUnit + sizeShift;
if (realScale !== 1) {
iconLayer.scale(realScale);
}
// eslint-disable-next-line no-loop-func
iconLayer.scan(0, 0, iconLayer.bitmap.width, iconLayer.bitmap.height, (x, y, sidx) => {
x += xs;
y += ys;
if (x < 0 || x >= size || y < 0 || y >= size) {
return;
}
let sdata = iconLayer.bitmap.data;
let ddata = icon.bitmap.data;
let didx = icon.getPixelIndex(x, y);
let sa = sdata[sidx + 3] / 255;
let da = ddata[didx + 3] / 255;
let db = da * (1 - sa * tint.a);
let cb = sa + da * (1 - sa);
ddata[didx + 0] = Math.min(255, (sdata[sidx + 0] * sa * tint.r + ddata[didx + 0] * db) / cb);
ddata[didx + 1] = Math.min(255, (sdata[sidx + 1] * sa * tint.g + ddata[didx + 1] * db) / cb);
ddata[didx + 2] = Math.min(255, (sdata[sidx + 2] * sa * tint.b + ddata[didx + 2] * db) / cb);
ddata[didx + 3] = Math.min(255, sdata[sidx + 3] + ddata[didx + 3] * (1 - sa));
});
}
return icon;
}
const itemTypes = new Set([
"item",
"ammo",
"capsule",
"gun",
"item-with-entity-data",
"item-with-label",
"item-with-inventory",
"blueprint-book",
"item-with-tags",
"selection-tool",
"blueprint",
"copy-paste-tool",
"deconstruction-item",
"upgrade-item",
"module",
"rail-planner",
"spidertron-remote",
"tool",
"armor",
"mining-tool",
"repair-tool",
// XXX Bad hack to get icons for fluids. These should be treated as a
// separate namespace from items as they may be named the same as items.
"fluid",
// TODO consider adding virtual signals and recipes
]);
function filterItems(prototypes) {
return Object.entries(prototypes)
.filter(([type, _]) => itemTypes.has(type))
.flatMap(([_, typePrototypes]) => Object.values(typePrototypes))
;
}
/**
* Export item icons and data
*
* Assembles and packs the icons for the item prototypes given into a single
* spritesheet and json file with meta data.
*
* @param {module:host/src/server.FactorioServer} server -
* The server to generate the export mod for.
* @param {Map<string, string>} modVersions -
* Mapping of mod name to versions to get icons from.
* @param {Array<Object>} items - Array of item prototypes.
* @return {Promise<{itemSheet: Jimp, itemData: Map<string,Object>}>}
* Item spritesheet and metadata.
* @memberof module:lib
* @private
* @inner
*/
async function exportItems(server, modVersions, items) {
// Size to render icons at
let size = 32;
// Width of spritesheet
let width = 1024;
let rows = Math.ceil(items.length / (width / size));
let iconSheet = await Jimp.create(width, rows * size);
let itemData = new Map();
let pos = 0;
let iconCache = new Map();
let simpleIcons = new Map();
for (let item of items) {
let icon;
let iconPos;
if (item.icons) {
icon = await loadLayeredIcon(server, modVersions, item, size, iconCache);
iconPos = pos;
} else {
iconPos = simpleIcons.get(item.icon);
if (iconPos === undefined) {
icon = await loadSimpleIcon(server, modVersions, item, size, iconCache);
if (icon) {
iconPos = pos;
simpleIcons.set(item.icon, pos);
}
}
}
if (iconPos !== undefined) {
itemData.set(item.name, {
x: iconPos * size % width,
y: Math.floor(iconPos / (width / size)) * size,
size,
localised_name: item.localised_name,
localised_description: item.localised_description,
});
}
if (icon) {
iconSheet.composite(icon, pos * size % width, Math.floor(pos / (width / size)) * size);
pos += 1;
}
}
iconSheet.crop(0, 0, width, Math.ceil(pos / (width / size)) * size);
return { iconSheet, itemData };
}
/**
* Exports the locale files for the base game and the given mods
*
* Parses and merges all the locales for the all the mods given through
* `modVersions` and `modOrder`.
*
* @param {module:host/src/server.FactorioServer} server -
* The server to export the locale from.
* @param {Map<string, string>} modVersions -
* Mapping of mod name to version to export locale from.
* @param {Array<string>} modOrder - Load order of the mods.
* @param {string} languageCode - Language to export locale for.
* @returns {Promise<Map<string, string>>} merged locale information
* @memberof module:lib
* @private
* @inner
*/
async function exportLocale(server, modVersions, modOrder, languageCode) {
let mergedLocales = new Map();
function mergeLocale(locale) {
for (let [category, entries] of Object.entries(locale)) {
if (typeof entries === "string") {
mergedLocales.set(category, entries);
} else {
for (let [key, value] of Object.entries(entries)) {
mergedLocales.set(`${category}.${key}`, value);
}
}
}
}
let baseLocaleFilePath = server.dataPath("base", "locale", languageCode, "base.cfg");
mergeLocale(lib.parse(await fs.readFile(baseLocaleFilePath, "utf8")));
for (let mod of modOrder) {
if (["core", "base", "export"].includes(mod)) {
continue;
}
let zip;
try {
zip = await loadZip(server, modVersions, mod);
} catch (err) {
if (err.code === "ENOENT") {
continue;
}
throw err;
}
for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) {
let content = await file.async("nodebuffer");
mergeLocale(lib.parse(content.toString("utf8")));
}
}
return mergedLocales;
}
/**
* Export the locale and item icons for the given factorio server
*
* @param {module:host/src/server.FactorioServer} server -
* The server to export the data from.
* @returns {Promise<JSZip>} zip file with exported data.
* @memberof module:lib
*/
async function exportData(server) {
await generateExportMod(server);
let settings = {};
let prototypes = {};
let modVersions = new Map();
let modOrder = [];
function add(obj, prototype) {
if (!Object.prototype.hasOwnProperty.call(obj, prototype.type)) {
obj[prototype.type] = {};
}
obj[prototype.type][prototype.name] = prototype;
}
server.on("ipc-prototype_export", data => add(prototypes, data));
server.on("ipc-settings_export", data => add(settings, data));
server.on("ipc-mod_setting_mod", ({ name, mod }) => {
for (let type of Object.values(settings)) {
if (Object.prototype.hasOwnProperty.call(type, name)) {
type[name].mod = mod;
return;
}
}
server._logger.error(`Unable to find ${name} in settings prototypes`);
});
server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); });
server.on("output", parsed => {
if (parsed.format === "seconds" && parsed.type === "generic") {
let match = /^Checksum of (.*): \d+$/.exec(parsed.message);
if (match) {
modOrder.push(match[1]);
}
}
});
try {
await server.startScenario("base/freeplay");
await server.kill();
} finally {
await fs.unlink(server.writePath("mods", "export_0.0.0.zip"));
}
if (!Object.keys(prototypes).length) {
throw new Error("No prototypes got exported");
}
// Some mod authors put leading zeros into the versions of their zip files.
let splitter = /^(.*)_(\d+)\.(\d+)\.(\d+)\.zip?$/;
for (let entry of await fs.readdir(server.writePath("mods"))) {
let match = splitter.exec(entry);
if (!match) {
continue;
}
let modVersion = `${match[2]}.${match[3]}.${match[4]}`;
let normalizedVersion =
`${Number.parseInt(match[2], 10)}.${Number.parseInt(match[3], 10)}.${Number.parseInt(match[4], 10)}`
;
if (modVersion === normalizedVersion) {
continue;
}
if (modVersions.get(match[1]) === normalizedVersion) {
modVersions.set(match[1], modVersion);
}
}
let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes));
let locale = await exportLocale(server, modVersions, modOrder, "en");
// Free up the memory used by zip files loaded during the export.
zipCache.clear();
let zip = new JSZip();
zip.file("export/settings.json", JSON.stringify(settings));
zip.file("export/prototypes.json", JSON.stringify(prototypes));
zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
return zip;
}
module.exports = {
exportData,
// For testing only
_exportLocale: exportLocale,
};
+270
View File
@@ -0,0 +1,270 @@
// Library for patching Factorio saves with scenario code.
"use strict";
const events = require("events");
const fs = require("fs-extra");
const JSZip = require("jszip");
const path = require("path");
const semver = require("semver");
const lib = require("@clusterio/lib");
const knownScenarios = {
// First seen in 0.17.63
"4e866186ebe297f1038fd325b09df1a1f5e2fdd1": {
"name": "freeplay",
"modules": [
"freeplay",
"silo-script",
],
},
};
/**
* Generates control.lua code for loading the Clusterio modules
*
* @param {Object} patchInfo - The patch info files's json content
* @returns {string} Generated control.lua code.
* @memberof module:lib
* @private
* @inner
*/
function generateLoader(patchInfo) {
let lines = [
"-- Auto generated scenario module loader created by Clusterio",
"-- Modifications to this file will be lost when loaded in Clusterio",
`clusterio_patch_number = ${patchInfo.patch_number}`,
"",
'local event_handler = require("event_handler")',
"",
"-- Scenario modules",
];
for (let moduleName of patchInfo["scenario"]["modules"]) {
lines.push(`event_handler.add_lib(require("${moduleName}"))`);
}
lines.push(...[
"",
"-- Clusterio modules",
]);
for (let module of patchInfo["modules"]) {
for (let file of module["files"]) {
let requirePath = `modules/${module.name}/${file["path"].slice(0, -4)}`;
if (file["load"]) {
lines.push(`event_handler.add_lib(require("${requirePath}"))`);
}
if (file["require"]) {
lines.push(`require("${requirePath}")`);
}
}
}
// End last line with a newline
lines.push("");
return lines.join("\n");
}
/**
* Reorders modules to satisfy their dependencies
*
* Looks through and reorders the array of module definitions in order to
* satisfy the property that dependencies are earlier in the array than
* their dependents. Throws an error if this is not possible.
*
* @param {Array<Object>} modules - Array of modules to reorder
* @memberof module:lib
* @private
* @inner
*/
function reorderDependencies(modules) {
let index = 0;
let present = new Map();
let hold = new Map();
reorder: while (index < modules.length) {
let module = modules[index];
if (semver.valid(module.version) === null) {
throw new Error(`Invalid version '${module.version}' for module ${module.name}`);
}
for (let [dependency, requirement] of Object.entries(module.dependencies)) {
if (semver.validRange(requirement) === null) {
throw new Error(
`Invalid version range '${requirement}' for dependency ${dependency} on module ${module.name}`
);
}
if (present.has(dependency)) {
if (!semver.satisfies(present.get(dependency), requirement)) {
throw new Error(`Module ${module.name} requires ${dependency} ${requirement}`);
}
// We have an unmet dependency, take it out and continue
} else {
if (hold.has(dependency)) {
hold.get(dependency).push(module);
} else {
hold.set(dependency, [module]);
}
modules.splice(index, 1);
continue reorder;
}
}
// No unmet dependencies, record and continue
present.set(module.name, module.version);
index += 1;
if (hold.has(module.name)) {
modules.splice(index, 0, ...hold.get(module.name));
hold.delete(module.name);
}
}
if (!hold.size) {
return;
}
// There are three reasons for a module to end up being held: The module depends
// on a module that is missing, the module is part of a dependency loop, or the
// the module depends on a module that satisfy any of these conditions.
let remaining = new Map();
for (let heldModules of hold.values()) {
for (let module of heldModules) {
remaining.set(module.name, module);
}
}
// Start with a random module from the remaining modules
for (let module of remaining.values()) {
let cycle = [];
while (true) {
// Find an unmet dependency
let dependency = Object.keys(module.dependencies).find(name => !present.has(name));
if (!remaining.has(dependency)) {
// There's no module being held up by this dependency, the
// dependency is missing.
throw new Error(`Missing dependency ${dependency} for module ${module.name}`);
}
if (cycle.includes(module.name)) {
cycle.push(module.name);
cycle.splice(0, cycle.indexOf(module.name));
throw new Error(`Module dependency loop detected: ${cycle.join(" -> ")}`);
}
cycle.push(module.name);
module = remaining.get(dependency);
}
}
}
/**
* Patch a save with the given modules
*
* Adds the modules given by the modules parameter to the save located
* at savePath and rewrites the control.lua in the save to load the
* modules that were added. Will also remove any previous module
* located in the save.
*
* @param {string} savePath - Path to the Factorio save to patch.
* @param {Array<Object>} modules - Description of the modules to patch.
* @memberof module:lib
*/
async function patch(savePath, modules) {
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
let root = zip.folder(lib.findRoot(zip));
let patchInfoFile = root.file("clusterio.json");
let patchInfo;
if (patchInfoFile !== null) {
let content = await patchInfoFile.async("string");
patchInfo = JSON.parse(content);
// No info file present, try to detect if it's a known compatible scenario.
} else {
let controlStream = root.file("control.lua").nodeStream("nodebuffer");
let controlHash = await lib.hashStream(controlStream);
if (controlHash in knownScenarios) {
patchInfo = {
"scenario": knownScenarios[controlHash],
};
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
// Increment patch number
patchInfo["patch_number"] = (patchInfo["patch_number"] || 0) + 1;
// Remove any existing modules from the save
patchInfo["modules"] = [];
for (let file of root.file(/^modules\//)) {
zip.remove(file.name);
}
reorderDependencies(modules);
// Add the modules to the save.
for (let module of modules) {
let moduleDir = root.folder(`modules/${module.name}`);
let moduleInfo = {
"name": module.name,
"files": [],
};
let dirs = [[module.path, ""]];
while (dirs.length) {
let [dir, relativeDir] = dirs.pop();
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
let fsPath = path.join(dir, entry.name);
let relativePath = path.posix.join(relativeDir, entry.name);
if (entry.isFile()) {
moduleDir.file(relativePath, await fs.readFile(fsPath));
moduleInfo["files"].push({
"path": relativePath,
"load": module.load.includes(relativePath),
"require": module.require.includes(relativePath),
});
} else if (entry.isDirectory()) {
dirs.push([fsPath, relativePath]);
}
}
}
patchInfo["modules"].push(moduleInfo);
}
// Add loading code and patch info
root.file("control.lua", generateLoader(patchInfo));
root.file("clusterio.json", JSON.stringify(patchInfo, null, 4));
// Write back the save
let tempSavePath = `${savePath}.tmp`;
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
let fd = await fs.open(tempSavePath, "w");
try {
let pipe = stream.pipe(fs.createWriteStream("", { fd, autoClose: false }));
await events.once(pipe, "finish");
await fs.fsync(fd);
} finally {
await fs.close(fd);
}
await fs.rename(tempSavePath, savePath);
}
module.exports = {
patch,
// For testing only
_generateLoader: generateLoader,
_reorderDependencies: reorderDependencies,
};
File diff suppressed because it is too large. Load diff