// Library for patching Factorio saves with scenario code. import fs from "node:fs/promises"; import path from "path"; import semver from "semver"; import { pipeline } from "node:stream/promises"; import { Type, Static } from "@sinclair/typebox"; import * as lib from "@clusterio/lib"; /** * Describes a module that can be patched into a save */ export class SaveModule { constructor( /** Module */ public info: lib.ModuleInfo, /** Files and their content that will be patched into the save */ public files = new Map(), ) { } static moduleFilePath(filePath: string, moduleName: string) { // Map locale files to the save's locale folder if (filePath.startsWith("locale/")) { const slashes = filePath.match(/\//g)!.length; if (slashes > 1) { const secondSlash = filePath.indexOf("/", "locale/".length); return `${filePath.slice(0, secondSlash)}/${moduleName}-${filePath.slice(secondSlash + 1)}`; } const period = `${filePath}.`.indexOf("."); return `${filePath.slice(0, period)}/${moduleName}${filePath.slice(period)}`; } // Map all other files into modules/name in the save. return path.posix.join("modules", moduleName, filePath); } async loadFiles(moduleDirectory: string) { let dirs: [string, string][] = [[moduleDirectory, ""]]; while (dirs.length) { let [dir, relativeDir] = dirs.pop()!; for (let entry of await fs.readdir(dir, { withFileTypes: true })) { let fsPath = path.join(dir, entry.name); let relativePath = path.posix.join(relativeDir, entry.name); if (entry.isFile()) { let savePath = SaveModule.moduleFilePath(relativePath, this.info.name); this.files.set(savePath, await fs.readFile(fsPath)); if (relativePath === "module_exports.lua") { this.files.set( `modules/${this.info.name}.lua`, Buffer.from(`return require("modules/${this.info.name}/module_exports")`, "utf-8") ); } } else if (entry.isDirectory()) { dirs.push([fsPath, relativePath]); } } } } static jsonSchema = Type.Object({ ...lib.ModuleInfo.jsonSchema.properties, "files": Type.Array(Type.String()), }); toJSON() { return { ...this.info.toJSON(), files: [...this.files.keys()], }; } static async fromSave(json: Static, zip: lib.ZipArchive, root: string) { const module = new this(lib.ModuleInfo.fromJSON(json)); for (const filename of json.files) { const content = await zip.readFile(path.posix.join(root, filename)); if (content === null) { lib.logger.warn(`Missing file ${filename} in save`); continue; } module.files.set(filename, content); } return module; } static async fromPluginInfo(pluginInfo: lib.PluginNodeEnvInfo) { let moduleDirectory = path.join(path.dirname(pluginInfo.packagePath), "module"); try { await fs.access(moduleDirectory); } catch (err: any) { return null; } let moduleJsonPath = path.join(moduleDirectory, "module.json"); let moduleJson; try { moduleJson = { name: pluginInfo.name, version: pluginInfo.version, dependencies: { "clusterio": "*" }, ...JSON.parse(await fs.readFile(moduleJsonPath, "utf8")), }; } catch (err: any) { throw new Error(`Loading module/module.json in plugin ${pluginInfo.name} failed: ${err.message}`); } if (!lib.ModuleInfo.validate(moduleJson)) { throw new Error( `module/module.json in plugin ${pluginInfo.name} failed validation:\n` + `${JSON.stringify(lib.ModuleInfo.validate.errors, null, "\t")}` ); } let module = new SaveModule(lib.ModuleInfo.fromJSON(moduleJson)); await module.loadFiles(moduleDirectory); return module; } static async fromDirectory(moduleDirectory: string) { let name = path.basename(moduleDirectory); let moduleJsonPath = path.join(moduleDirectory, "module.json"); let moduleJson; try { moduleJson = JSON.parse(await fs.readFile(moduleJsonPath, "utf8")); } catch (err: any) { throw new Error(`Loading ${name}/module.json failed: ${err.message}`); } if (!lib.ModuleInfo.validate(moduleJson)) { throw new Error( `${name}/module.json failed validation:\n` + `${JSON.stringify(lib.ModuleInfo.validate.errors, null, "\t")}` ); } if (moduleJson.name !== name) { throw new Error(`Expected name of module ${moduleJson.name} to match the directory name ${name}`); } let module = new SaveModule(lib.ModuleInfo.fromJSON(moduleJson)); await module.loadFiles(moduleDirectory); return module; } } /** * Load the modules to patch into a save * * Collects the modules provided by the given plugins together with the * modules bundled with the host package. * * @param pluginInfos - Plugins to load modules from. * @returns Map of module name to module. */ export async function loadModules(pluginInfos: Iterable) { let modules: Map = new Map(); for (let pluginInfo of pluginInfos) { let module = await SaveModule.fromPluginInfo(pluginInfo); if (!module) { continue; } modules.set(module.info.name, module); } // Find stand alone modules to load // XXX for now only the included clusterio module is loaded let modulesDirectory = path.join(import.meta.dirname, "..", "..", "..", "modules"); for (let entry of await fs.readdir(modulesDirectory, { withFileTypes: true })) { if (entry.isDirectory()) { if (modules.has(entry.name)) { throw new Error(`Module with name ${entry.name} already exists in a plugin`); } let module = await SaveModule.fromDirectory(path.join(modulesDirectory, entry.name)); modules.set(module.info.name, module); } } return modules; } export class PatchInfo { static currentVersion = 1; constructor( public patchNumber: number, public scenario: lib.ModuleInfo, public modules: SaveModule[], public version = PatchInfo.currentVersion, ) { } static jsonSchema = Type.Object({ "version": Type.Number(), "patch_number": Type.Number(), "scenario": lib.ModuleInfo.jsonSchema, "modules": Type.Array(SaveModule.jsonSchema), }); toJSON() { return { version: PatchInfo.currentVersion, patch_number: this.patchNumber, scenario: this.scenario.toJSON(), modules: this.modules.map(m => m.toJSON()), }; } static async fromSave(json: Static, zip: lib.ZipArchive, root: string) { if (json.version === undefined) { interface ScenarioInfoV0 { name: string; modules: string[]; } interface PatchInfoV0 { patch_number: number; scenario: ScenarioInfoV0; modules: { name: string, files: { path: string, load: boolean, require: boolean }[], }[]; } const info = json as unknown as PatchInfoV0; return new this( info.patch_number, new lib.ModuleInfo(info.scenario.name, "0.0.0", info.scenario.modules), [], 0, ); } return new this( json.patch_number, lib.ModuleInfo.fromJSON(json.scenario), await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))), json.version, ); } } /** * Generates control.lua code for loading the Clusterio modules * * @param patchInfo - The patch info files's json content * @returns Generated control.lua code. * @internal */ function generateLoader(patchInfo: PatchInfo) { let lines = [ "-- Auto generated scenario module loader created by Clusterio", "-- Modifications to this file will be lost when loaded in Clusterio", `clusterio_patch_number = ${patchInfo.patchNumber}`, "", 'local event_handler = require("event_handler")', "", "-- Scenario modules", ]; for (let requirePath of patchInfo.scenario.load) { lines.push(`event_handler.add_lib(require("${requirePath}"))`); } for (let requirePath of patchInfo.scenario.require) { lines.push(`require("${requirePath}")`); } lines.push(...[ "", "-- Clusterio modules", ]); for (let module of patchInfo.modules) { for (let file of module.info.load) { let requirePath = `modules/${module.info.name}/${file.replace(/\.lua$/i, "")}`; lines.push(`event_handler.add_lib(require("${requirePath}"))`); } for (let file of module.info.require) { let requirePath = `modules/${module.info.name}/${file.replace(/\.lua$/i, "")}`; lines.push(`require("${requirePath}")`); } } // End last line with a newline lines.push(""); return lines.join("\n"); } /** * Reorders modules to satisfy their dependencies * * Looks through and reorders the array of module definitions in order to * satisfy the property that dependencies are earlier in the array than * their dependents. Throws an error if this is not possible. * * @param modules - Array of modules to reorder * @internal */ function reorderDependencies(modules: SaveModule[]) { let index = 0; let present = new Map(); let hold = new Map(); reorder: while (index < modules.length) { let module = modules[index]; if (semver.valid(module.info.version) === null) { throw new Error(`Invalid version '${module.info.version}' for module ${module.info.name}`); } for (let [dependency, requirement] of module.info.dependencies) { if (semver.validRange(requirement) === null) { throw new Error( `Invalid version range '${requirement}' for dependency ${dependency} on module ${module.info.name}` ); } if (present.has(dependency)) { if (!semver.satisfies(present.get(dependency)!, requirement)) { throw new Error(`Module ${module.info.name} requires ${dependency} ${requirement}`); } // We have an unmet dependency, take it out and continue } else { if (hold.has(dependency)) { hold.get(dependency)!.push(module); } else { hold.set(dependency, [module]); } modules.splice(index, 1); continue reorder; } } // No unmet dependencies, record and continue present.set(module.info.name, module.info.version); index += 1; if (hold.has(module.info.name)) { modules.splice(index, 0, ...hold.get(module.info.name)!); hold.delete(module.info.name); } } if (!hold.size) { return; } // There are three reasons for a module to end up being held: The module depends // on a module that is missing, the module is part of a dependency loop, or the // the module depends on a module that satisfy any of these conditions. let remaining = new Map(); for (let heldModules of hold.values()) { for (let module of heldModules) { remaining.set(module.info.name, module); } } // Start with a random module from the remaining modules for (let module of remaining.values()) { let cycle: string[] = []; while (true) { // Find an unmet dependency let dependency = [...module.info.dependencies.keys()].find(name => !present.has(name))!; if (!remaining.has(dependency)) { // There's no module being held up by this dependency, the // dependency is missing. throw new Error(`Missing dependency ${dependency} for module ${module.info.name}`); } if (cycle.includes(module.info.name)) { cycle.push(module.info.name); cycle.splice(0, cycle.indexOf(module.info.name)); throw new Error(`Module dependency loop detected: ${cycle.join(" -> ")}`); } cycle.push(module.info.name); module = remaining.get(dependency)!; } } } const knownScenarios: Record = { // First seen in 0.17.63 "4e866186ebe297f1038fd325b09df1a1f5e2fdd1": new lib.ModuleInfo("freeplay", "0.17.63", [], ["scenario"]), // First seen in 2.0 "bcbdde18ce4ec16ebfd93bd694cd9b12ef969c9a": new lib.ModuleInfo("freeplay", "2.0.0", [], ["scenario"]), // First seen in 2.0.29 "3af547d2f4db3728b75ac5d1b3a4f47830dc48e7": new lib.ModuleInfo("freeplay", "2.0.29", [], ["scenario"]), }; /** * Names of the zip entries added by patching for the modules in the patch info */ function moduleEntries(zip: lib.ZipArchive, patchInfo: PatchInfo, root: string) { const entries = new Set(); if (patchInfo.version === 0) { const modulesPath = path.posix.join(root, "modules/"); for (const name of zip.entries.keys()) { if (name.startsWith(modulesPath)) { entries.add(name); } } } else { for (const module of patchInfo.modules) { for (const filepath of module.files.keys()) { entries.add(path.posix.join(root, filepath)); } } } return entries; } /** * Write a copy of the save next to it with entries removed and added * * Keeps the order of the entries in the original. * * @returns Path to the temporary file written. */ async function rewriteSave( zip: lib.ZipArchive, savePath: string, removed: Set, added: Map ) { added = new Map(added); const output = new lib.ZipWriter(); const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath); const writing = pipeline(output.outputStream, writeStream); // Awaited below, but a write error can reject it before then. writing.catch(() => {}); try { for (const entry of zip.entries.values()) { const name = entry.fileName; if (removed.has(name)) { continue; } const contents = added.get(name); if (contents !== undefined) { await output.addBuffer(contents, name); added.delete(name); } else { await output.addEntry(zip, entry); } } for (const [name, contents] of added) { await output.addBuffer(contents, name); } await output.end(); await writing; } catch (err) { output.outputStream.destroy(); await writing.catch(() => {}); await fs.rm(tempSavePath, { force: true }); throw err; } return tempSavePath; } /** * Patch a save with the given modules * * Adds the modules given by the modules parameter to the save located * at savePath and rewrites the control.lua in the save to load the * modules that were added. Will also remove any previous module * located in the save. * * @param savePath - Path to the Factorio save to patch. * @param modules - Description of the modules to patch. */ export async function patch(savePath: string, modules: SaveModule[]) { const zip = await lib.ZipArchive.fromFile(savePath); let tempSavePath: string; try { const root = lib.findRoot(zip); const rootPath = (relativePath: string) => path.posix.join(root, relativePath); // Files to add or replace in the save, relative to the root const files = new Map(); let patchInfo: PatchInfo; const patchInfoContent = await zip.readFile(rootPath("clusterio.json")); if (patchInfoContent !== null) { patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root); if (patchInfo.version > PatchInfo.currentVersion) { throw new Error( `Save patch version ${patchInfo.version} is newer than the patch version this ` + `version of Clusterio can load (${PatchInfo.currentVersion})` ); } // Unpatching put the original control.lua back in place of scenario.lua const scenarioRequired = [...patchInfo.scenario.load, ...patchInfo.scenario.require].includes("scenario"); if (scenarioRequired && !zip.file(rootPath("scenario.lua"))) { const control = await zip.readFile(rootPath("control.lua")); if (control === null) { throw new Error("Unable to patch save, missing control.lua file."); } files.set("scenario.lua", control); } // No info file present, try to detect if it's a known compatible scenario. } else { const controlFile = zip.file(rootPath("control.lua")); if (!controlFile) { throw new Error("Unable to patch save, missing control.lua file."); } const controlHash = await lib.hashStream(await zip.openStream(controlFile)); if (controlHash in knownScenarios) { patchInfo = new PatchInfo(0, knownScenarios[controlHash], []); files.set("scenario.lua", await zip.readEntry(controlFile)); } else { throw new Error(`Unable to patch save, unknown scenario (${controlHash})`); } } // Increment patch number patchInfo.patchNumber = patchInfo.patchNumber + 1; // Remove any existing modules from the save const removed = moduleEntries(zip, patchInfo, root); patchInfo.modules = []; reorderDependencies(modules); // Add the modules to the save. for (const module of modules) { for (const [relativePath, contents] of module.files) { files.set(relativePath, contents); } patchInfo.modules.push(module); } // Add loading code and patch info files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8")); files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8")); const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents])); tempSavePath = await rewriteSave(zip, savePath, removed, added); } finally { zip.close(); } await fs.rename(tempSavePath, savePath); } /** * Remove the modules patched into a save * * Moves the original control.lua that patching kept as scenario.lua back * and removes the module files. The patch info is kept with no modules so that * the patch number keeps counting if the save is patched again. * * @param savePath - Path to the Factorio save to unpatch. * @returns true if the save was changed. */ export async function unpatch(savePath: string) { const zip = await lib.ZipArchive.fromFile(savePath); let tempSavePath: string; try { const root = lib.findRoot(zip); const rootPath = (relativePath: string) => path.posix.join(root, relativePath); const patchInfoContent = await zip.readFile(rootPath("clusterio.json")); if (patchInfoContent === null) { return false; } const patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root); if (patchInfo.version > PatchInfo.currentVersion) { throw new Error( `Save patch version ${patchInfo.version} is newer than the patch version this ` + `version of Clusterio can unpatch (${PatchInfo.currentVersion})` ); } if (patchInfo.version !== 0 && !patchInfo.modules.length) { return false; } const removed = moduleEntries(zip, patchInfo, root); patchInfo.modules = []; const scenario = await zip.readFile(rootPath("scenario.lua")); if (scenario !== null) { removed.add(rootPath("scenario.lua")); } // Scenarios made for Clusterio have no scenario.lua, keep loading them without modules const control = scenario ?? Buffer.from(generateLoader(patchInfo), "utf8"); tempSavePath = await rewriteSave(zip, savePath, removed, new Map([ [rootPath("control.lua"), control], [rootPath("clusterio.json"), Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8")], ])); } finally { zip.close(); } await fs.rename(tempSavePath, savePath); return true; } // For testing only export const _generateLoader = generateLoader; export const _reorderDependencies = reorderDependencies;