From c419f3e9b2c1b5083e157e45d346efd98d863993 Mon Sep 17 00:00:00 2001 From: bbassie Date: Thu, 10 Sep 2026 00:16:29 +0000 Subject: [PATCH] Add create-scenario command to clusteriohost Creates a scenario directory with the Clusterio modules patched into it so the modules can be used outside of Clusterio. The command reuses the save patching by zipping the scenario, running patch() on it and extracting the result. Defaults to the freeplay scenario from the Factorio install in the host config and all plugins enabled there. Module loading in Instance.prepareSave is moved to patch.loadModules so the command and the instance share it, and findVersion is exported from server.ts so the command can find the freeplay scenario. Zipped scenarios were tried as an output form but Factorio does not pick them up from the scenarios folder, so only directories are written. Co-Authored-By: Claude Fable 5.1 --- docs/how-it-works.md | 1 + packages/host/README.md | 21 ++++ packages/host/host.ts | 11 ++ packages/host/src/Instance.ts | 26 +---- packages/host/src/create_scenario.ts | 149 +++++++++++++++++++++++++++ packages/host/src/patch.ts | 48 ++++++++- packages/host/src/server.ts | 7 +- test/host/create_scenario.js | 93 +++++++++++++++++ test/host/patch.js | 16 +++ test/host/server.js | 18 ++-- 10 files changed, 349 insertions(+), 41 deletions(-) create mode 100644 packages/host/src/create_scenario.ts create mode 100644 test/host/create_scenario.js diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 6a943554..0bc0021b 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -5,6 +5,7 @@ The save patching is based on the built in [event_handler](https://github.com/wu Mods on the other hand work without modifications. This can be disabled with the `factorio.enable_save_patching` config on a per instance basis, however disabling it means that modules are not loaded into the game and most plugins will not function. +To use the modules outside of Clusterio the `npx clusteriohost create-scenario` command creates a scenario with the modules patched into it. 1: Before version 2.0 of Clusterio this was done at runtime with a patcher called Hotpatch, but due to the difficulty in supporting run time patching and the lack of documentation it was replace it with save patching instead. diff --git a/packages/host/README.md b/packages/host/README.md index 418e36ac..9fc74d22 100644 --- a/packages/host/README.md +++ b/packages/host/README.md @@ -78,6 +78,27 @@ See docs/configuration.md in the main repositiory for the available configuratio Shows the value for a single config entry. +### `create-scenario ` + +Create a scenario with the Clusterio modules patched into it and write it to the `output` directory. +This is the same patching that is applied to saves when an instance starts, see docs/how-it-works.md in the main repository. +The result can be used as a regular scenario outside of Clusterio by copying it to the `scenarios` folder of Factorio or into a mod. + +By default the freeplay scenario from the latest Factorio install in `host.factorio_directory` is patched with the modules from all plugins that are enabled in the host config. + + * `--scenario ` Scenario directory, zipped scenario or save to patch instead of freeplay. + This has the same restrictions as save patching, the scenario must either use the event_handler library and have been seen by Clusterio before, or already have been patched by Clusterio. + + * `--factorio-version ` Factorio version to take the freeplay scenario from. + Defaults to `latest`. + + * `--plugins ...` Only patch in modules from the given plugins. + +For example, creating a scenario with only the inventory_sync module: + + npx clusteriohost create-scenario ~/.factorio/scenarios/inventory_sync --plugins inventory_sync + + #### `config list` Lists up all configuration entries with their currently configured values. diff --git a/packages/host/host.ts b/packages/host/host.ts index a2b882fb..8794e49d 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -24,6 +24,7 @@ import * as lib from "@clusterio/lib"; import { ConsoleTransport, levels, logger } from "@clusterio/lib"; import Host from "./src/Host"; +import { createScenarioCommand, handleCreateScenarioCommand } from "./src/create_scenario"; let host: Host | undefined; @@ -113,6 +114,11 @@ async function startHost() { .option("bypass-lock-file", { hidden: true, type: "boolean", nargs: 0, default: false }) .command("plugin", "Manage available plugins", lib.pluginCommand) .command("config", "Manage Host config", lib.configCommand) + .command( + "create-scenario ", + "Create a scenario patched with modules for use outside Clusterio", + createScenarioCommand, + ) .command("run", "Run host", yargs => { yargs.option("can-restart", { type: "boolean", nargs: 0, default: false, @@ -194,6 +200,11 @@ async function startHost() { return; } + if (command === "create-scenario") { + await handleCreateScenarioCommand(args, hostConfig, pluginInfos); + return; + } + // From this point on the host should run await hostConfigLock.acquire(); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 3e45f04d..0a658048 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -10,7 +10,7 @@ const execAsync = util.promisify(exec); import * as lib from "@clusterio/lib"; import { FactorioServer } from "./server"; -import { SaveModule, patch } from "./patch"; +import { loadModules, patch } from "./patch"; import { exportData } from "./export"; import type Host from "./Host"; import BaseInstancePlugin from "./BaseInstancePlugin"; @@ -951,29 +951,7 @@ end`.replace(/\r?\n/g, " "); // Patch save with lua modules from plugins this.logger.verbose("Patching save"); - // Find plugin modules to patch in - let modules: Map = new Map(); - for (let plugin of this.plugins.values()) { - let module = await SaveModule.fromPlugin(plugin); - if (!module) { - continue; - } - modules.set(module.info.name, module); - } - - // Find stand alone modules to load - // XXX for now only the included clusterio module is loaded - let modulesDirectory = path.join(__dirname, "..", "..", "..", "modules"); - for (let entry of await fs.readdir(modulesDirectory, { withFileTypes: true })) { - if (entry.isDirectory()) { - if (modules.has(entry.name)) { - throw new Error(`Module with name ${entry.name} already exists in a plugin`); - } - let module = await SaveModule.fromDirectory(path.join(modulesDirectory, entry.name)); - modules.set(module.info.name, module); - } - } - + let modules = await loadModules([...this.plugins.values()].map(plugin => plugin.info)); await patch(this.path("saves", saveName), [...modules.values()]); return saveName; } diff --git a/packages/host/src/create_scenario.ts b/packages/host/src/create_scenario.ts new file mode 100644 index 00000000..4aff341c --- /dev/null +++ b/packages/host/src/create_scenario.ts @@ -0,0 +1,149 @@ +// Implementation of the create-scenario command +import fs from "node:fs/promises"; +import os from "node:os"; +import JSZip from "jszip"; +import path from "path"; + +import * as lib from "@clusterio/lib"; +import { logger } from "@clusterio/lib"; + +import { loadModules, patch, SaveModule } from "./patch"; +import { findVersion } from "./server"; + + +/** + * Create a scenario patched with the given modules + * + * Runs the save patching on the scenario at scenarioPath and writes the + * result to the directory at outputPath. The scenario can be a directory, + * a zipped scenario or a save. + * + * @param scenarioPath - Path to the scenario to patch. + * @param outputPath - Directory to write the patched scenario to. + * @param modules - Description of the modules to patch. + */ +export async function createScenario(scenarioPath: string, outputPath: string, modules: SaveModule[]) { + if (await fs.access(outputPath).then(() => true, () => false)) { + throw new Error(`${outputPath} already exists`); + } + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-scenario-")); + try { + const tempPath = path.join(tempDir, "scenario.zip"); + if ((await fs.stat(scenarioPath)).isDirectory()) { + await zipDirectory(scenarioPath, tempPath); + } else { + await fs.copyFile(scenarioPath, tempPath); + } + + await patch(tempPath, modules); + await extractZip(tempPath, outputPath); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +} + +async function zipDirectory(dirPath: string, zipPath: string) { + const root = path.basename(path.resolve(dirPath)); + const zip = new JSZip(); + let dirs: [string, string][] = [[dirPath, root]]; + while (dirs.length) { + let [dir, relativeDir] = dirs.pop()!; + for (let entry of await fs.readdir(dir, { withFileTypes: true })) { + let fsPath = path.join(dir, entry.name); + let relativePath = path.posix.join(relativeDir, entry.name); + if (entry.isFile()) { + zip.file(relativePath, await fs.readFile(fsPath)); + } else if (entry.isDirectory()) { + dirs.push([fsPath, relativePath]); + } + } + } + await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" })); +} + +async function extractZip(zipPath: string, dirPath: string) { + const zip = await JSZip.loadAsync(await fs.readFile(zipPath)); + const root = lib.findRoot(zip); + for (let file of Object.values(zip.files)) { + if (file.dir || !file.name.startsWith(`${root}/`)) { + continue; + } + const fsPath = path.join(dirPath, file.name.slice(root.length + 1)); + await fs.mkdir(path.dirname(fsPath), { recursive: true }); + await fs.writeFile(fsPath, await file.async("nodebuffer")); + } +} + +/** + * Yargs create-scenario command + * + * @param yargs - yargs command builder. + */ +export function createScenarioCommand(yargs: any) { + yargs + .positional("output", { + describe: "Directory to write the scenario to", + type: "string", + }) + .option("scenario", { + nargs: 1, + describe: "Scenario directory, zipped scenario or save to patch, defaults to freeplay", + type: "string", + }) + .option("factorio-version", { + nargs: 1, + describe: "Factorio version to take the freeplay scenario from", + default: "latest", + type: "string", + }) + .option("plugins", { + describe: "Plugins to patch modules from, defaults to all plugins enabled in the host config", + type: "array", + }) + ; +} + +/** + * Handle yargs create-scenario command + * + * @param args - yargs args object. + * @param hostConfig - Host config. + * @param pluginInfos - Plugins available to the host. + */ +export async function handleCreateScenarioCommand( + args: Record, + hostConfig: lib.HostConfig, + pluginInfos: lib.PluginNodeEnvInfo[], +) { + let selected: lib.PluginNodeEnvInfo[]; + if (args.plugins) { + selected = (args.plugins as string[]).map(name => { + const pluginInfo = pluginInfos.find(info => info.name === name); + if (!pluginInfo) { + throw new Error(`Plugin ${name} is not in the plugin list`); + } + return pluginInfo; + }); + } else { + selected = pluginInfos.filter( + info => info.instanceEntrypoint && hostConfig.get(`${info.name}.load_plugin`) + ); + } + + let scenarioPath = args.scenario as string | undefined; + if (scenarioPath === undefined) { + const factorioDir = hostConfig.get("host.factorio_directory"); + const version = args.factorioVersion as lib.TargetVersion; + if (!lib.isTargetVersion(version)) { + throw new Error(`Invalid Factorio version ${version}`); + } + const [dataDir] = await findVersion(factorioDir, version); + scenarioPath = path.join(dataDir, "base", "scenarios", "freeplay"); + } + + const modules = await loadModules(selected); + logger.info(`Patching ${scenarioPath} with modules ${[...modules.keys()].join(", ")}`); + await createScenario(scenarioPath, args.output as string, [...modules.values()]); + logger.info(`Created scenario ${args.output}`); +} diff --git a/packages/host/src/patch.ts b/packages/host/src/patch.ts index 639ce65f..031d1565 100644 --- a/packages/host/src/patch.ts +++ b/packages/host/src/patch.ts @@ -91,7 +91,11 @@ export class SaveModule { } static async fromPlugin(plugin: BaseInstancePlugin) { - let pluginPackagePath = require.resolve(path.posix.join(plugin.info.requirePath, "package.json")); + return SaveModule.fromPluginInfo(plugin.info); + } + + static async fromPluginInfo(info: lib.PluginNodeEnvInfo) { + let pluginPackagePath = require.resolve(path.posix.join(info.requirePath, "package.json")); let moduleDirectory = path.join(path.dirname(pluginPackagePath), "module"); try { await fs.access(moduleDirectory); @@ -103,17 +107,17 @@ export class SaveModule { let moduleJson; try { moduleJson = { - name: plugin.info.name, - version: plugin.info.version, + name: info.name, + version: info.version, dependencies: { "clusterio": "*" }, ...JSON.parse(await fs.readFile(moduleJsonPath, "utf8")), }; } catch (err: any) { - throw new Error(`Loading module/module.json in plugin ${plugin.info.name} failed: ${err.message}`); + throw new Error(`Loading module/module.json in plugin ${info.name} failed: ${err.message}`); } if (!lib.ModuleInfo.validate(moduleJson)) { throw new Error( - `module/module.json in plugin ${plugin.info.name} failed validation:\n` + + `module/module.json in plugin ${info.name} failed validation:\n` + `${JSON.stringify(lib.ModuleInfo.validate.errors, null, "\t")}` ); } @@ -149,6 +153,40 @@ export class SaveModule { } } +/** + * Load the modules to patch into a save + * + * Collects the modules provided by the given plugins together with the + * modules bundled with the host package. + * + * @param pluginInfos - Plugins to load modules from. + * @returns Map of module name to module. + */ +export async function loadModules(pluginInfos: Iterable) { + let modules: Map = new Map(); + for (let pluginInfo of pluginInfos) { + let module = await SaveModule.fromPluginInfo(pluginInfo); + if (!module) { + continue; + } + modules.set(module.info.name, module); + } + + // Find stand alone modules to load + // XXX for now only the included clusterio module is loaded + let modulesDirectory = path.join(__dirname, "..", "..", "..", "modules"); + for (let entry of await fs.readdir(modulesDirectory, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (modules.has(entry.name)) { + throw new Error(`Module with name ${entry.name} already exists in a plugin`); + } + let module = await SaveModule.fromDirectory(path.join(modulesDirectory, entry.name)); + modules.set(module.info.name, module); + } + } + return modules; +} + export class PatchInfo { static currentVersion = 1; diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index 32d65b52..309a6044 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -181,9 +181,11 @@ async function downloadAndExtractTar(url: string, targetDir: string) { * Version to look for, supports the special value "latest" for the * latest version available. * @returns Array with path to data dir and version found. - * @internal */ -async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion): Promise<[string, lib.FullVersion]> { +export async function findVersion( + factorioDir: string, + targetVersion: lib.TargetVersion, +): Promise<[string, lib.FullVersion]> { // Check if this is a direct installation dir const directVersion = await getFactorioVersion(factorioDir); if (directVersion !== null) { @@ -1580,7 +1582,6 @@ export class FactorioServer extends events.EventEmitter { // For testing only export const _getFactorioVersion = getFactorioVersion; export const _versionOrder = versionOrder; -export const _findVersion = findVersion; export const _listFactorioVersions = listFactorioVersions; export const _downloadAndExtractZip = downloadAndExtractZip; export const _downloadAndExtractTar = downloadAndExtractTar; diff --git a/test/host/create_scenario.js b/test/host/create_scenario.js new file mode 100644 index 00000000..4609d732 --- /dev/null +++ b/test/host/create_scenario.js @@ -0,0 +1,93 @@ +"use strict"; +const assert = require("assert").strict; +const fs = require("node:fs/promises"); +const JSZip = require("jszip"); +const path = require("path"); + +const lib = require("@clusterio/lib"); +const { createScenario } = require("@clusterio/host/dist/node/src/create_scenario"); +const { SaveModule, PatchInfo } = require("@clusterio/host/dist/node/src/patch"); + + +describe("host/create_scenario", function() { + const tempDir = path.join("temp", "test", "create_scenario"); + const patchInfo = new PatchInfo(0, new lib.ModuleInfo("test", "1.0.0", [], ["scenario"]), []); + const scenarioFiles = new Map([ + ["scenario.lua", "-- scenario\n"], + ["description.json", "{}\n"], + ["locale/en/test.cfg", "test=Test\n"], + ["clusterio.json", JSON.stringify(patchInfo)], + ]); + const module = new SaveModule( + new lib.ModuleInfo("spam", "1.0.0", ["spam.lua"]), + new Map([["modules/spam/spam.lua", Buffer.from("return {}\n")]]), + ); + + async function readDir(dirPath) { + const files = new Map(); + for (const entry of await fs.readdir(dirPath, { recursive: true, withFileTypes: true })) { + if (entry.isFile()) { + const fsPath = path.join(entry.parentPath, entry.name); + files.set(path.relative(dirPath, fsPath).split(path.sep).join("/"), await fs.readFile(fsPath, "utf8")); + } + } + return files; + } + + function checkOutput(files) { + assert(files.get("control.lua").includes('event_handler.add_lib(require("modules/spam/spam"))')); + assert.equal(JSON.parse(files.get("clusterio.json")).patch_number, 1); + assert.equal(files.get("modules/spam/spam.lua"), "return {}\n"); + for (const name of ["scenario.lua", "description.json", "locale/en/test.cfg"]) { + assert.equal(files.get(name), scenarioFiles.get(name)); + } + assert.equal(files.size, scenarioFiles.size + 2); + } + + before(async function() { + await fs.rm(tempDir, { recursive: true, force: true }); + await fs.mkdir(path.join(tempDir, "scenario"), { recursive: true }); + const zip = new JSZip(); + for (const [name, content] of scenarioFiles) { + const fsPath = path.join(tempDir, "scenario", name); + await fs.mkdir(path.dirname(fsPath), { recursive: true }); + await fs.writeFile(fsPath, content); + zip.file(`scenario/${name}`, content); + } + await fs.writeFile(path.join(tempDir, "scenario.zip"), await zip.generateAsync({ type: "nodebuffer" })); + }); + + it("should patch a scenario directory", async function() { + const output = path.join(tempDir, "from_dir"); + await createScenario(path.join(tempDir, "scenario"), output, [module]); + checkOutput(await readDir(output)); + }); + + it("should patch a zipped scenario", async function() { + const output = path.join(tempDir, "from_zip"); + await createScenario(path.join(tempDir, "scenario.zip"), output, [module]); + checkOutput(await readDir(output)); + }); + + it("should not modify the input scenario", async function() { + const files = await readDir(path.join(tempDir, "scenario")); + assert.deepEqual(files, scenarioFiles); + }); + + it("should throw if the output already exists", async function() { + const output = path.join(tempDir, "from_dir"); + await assert.rejects( + createScenario(path.join(tempDir, "scenario"), output, [module]), + new Error(`${output} already exists`), + ); + }); + + it("should throw on unknown scenario", async function() { + await fs.mkdir(path.join(tempDir, "unknown")); + await fs.writeFile(path.join(tempDir, "unknown", "control.lua"), "-- unknown\n"); + await assert.rejects( + createScenario(path.join(tempDir, "unknown"), path.join(tempDir, "from_unknown"), [module]), + new Error("Unable to patch save, unknown scenario (3acc3be3861144e55604f5ac2f2555071885ebc4)") + ); + }); +}); diff --git a/test/host/patch.js b/test/host/patch.js index 3df1ca5e..53c4aa43 100644 --- a/test/host/patch.js +++ b/test/host/patch.js @@ -226,6 +226,22 @@ describe("host/patch", function() { }); }); + describe("loadModules()", function() { + it("should load the bundled clusterio module", async function() { + const modules = await patch.loadModules([]); + assert.deepEqual([...modules.keys()], ["clusterio"]); + assert(modules.get("clusterio").files.has("modules/clusterio/api.lua")); + }); + it("should load modules from plugins", async function() { + const pluginInfos = await lib.loadPluginInfos( + new Map([["research_sync", path.resolve("plugins/research_sync")]]) + ); + const modules = await patch.loadModules(pluginInfos); + assert.deepEqual([...modules.keys()], ["research_sync", "clusterio"]); + assert(modules.get("research_sync").files.has("modules/research_sync/sync.lua")); + }); + }); + describe("patch()", function() { it("should throw on unknown scenario", async function() { let zip = new JSZip(); diff --git a/test/host/server.js b/test/host/server.js index a18e6058..2321d527 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -42,26 +42,26 @@ describe("host/server", function() { describe("direct install", function() { it("should search given directory for latest Factorio install", async function() { const installDir = path.join("test", "file", "factorio", "0.1.2"); - const [dir, version] = await hostServer._findVersion(installDir, "latest"); + const [dir, version] = await hostServer.findVersion(installDir, "latest"); assert.equal(dir, path.join(installDir, "data")); assert.equal(version, "0.1.2"); }); it("should search given directory for given Factorio install", async function() { const installDir = path.join("test", "file", "factorio", "0.1.1"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1.1"); assert.equal(dir, path.join(installDir, "data")); assert.equal(version, "0.1.1"); }); it("should search given directory for partly given Factorio install", async function() { const installDir = path.join("test", "file", "factorio", "0.1.2"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1"); assert.equal(dir, path.join(installDir, "data")); assert.equal(version, "0.1.2"); }); it("should reject if the version does not match", async function() { let installDir = path.join("test", "file", "factorio", "0.1.1"); await assert.rejects( - hostServer._findVersion(installDir, "0.1.2"), + hostServer.findVersion(installDir, "0.1.2"), new Error( `Unable to find Factorio version 0.1.2: ${installDir} is a direct (single-version) ` + `install of 0.1.1. Use a versioned layout, where ${installDir} contains a subdirectory ` + @@ -74,33 +74,33 @@ describe("host/server", function() { it("should reject if no factorio install with the given version was found", async function() { let installDir = path.join("test", "file", "factorio"); await assert.rejects( - hostServer._findVersion(installDir, "0.1.3"), + hostServer.findVersion(installDir, "0.1.3"), new Error("Unable to find Factorio version 0.1.3") ); }); it("should search given directory for given Factorio install", async function() { const installDir = path.join("test", "file", "factorio"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1.1"); assert.equal(dir, path.join(installDir, "0.1.1", "data")); assert.equal(version, "0.1.1"); }); it("should search given directory for partly given Factorio install", async function() { const installDir = path.join("test", "file", "factorio"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1"); assert.equal(dir, path.join(installDir, "0.1.2", "data")); assert.equal(version, "0.1.2"); }); it("should reject if no factorio install with the given version was found", async function() { let installDir = path.join("test", "file", "factorio"); await assert.rejects( - hostServer._findVersion(installDir, "0.1.3"), + hostServer.findVersion(installDir, "0.1.3"), new Error("Unable to find Factorio version 0.1.3") ); }); it("should reject if no factorio install was found", async function() { let installDir = path.join("test", "file"); await assert.rejects( - hostServer._findVersion(installDir, "0.0.0"), + hostServer.findVersion(installDir, "0.0.0"), new Error(`Unable to find any Factorio install in ${installDir}`) ); });