diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 6a943554..0bc0021b 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -5,6 +5,7 @@ The save patching is based on the built in [event_handler](https://github.com/wu Mods on the other hand work without modifications. This can be disabled with the `factorio.enable_save_patching` config on a per instance basis, however disabling it means that modules are not loaded into the game and most plugins will not function. +To use the modules outside of Clusterio the `npx clusteriohost create-scenario` command creates a scenario with the modules patched into it. 1: Before version 2.0 of Clusterio this was done at runtime with a patcher called Hotpatch, but due to the difficulty in supporting run time patching and the lack of documentation it was replace it with save patching instead. diff --git a/packages/host/README.md b/packages/host/README.md index 418e36ac..9fc74d22 100644 --- a/packages/host/README.md +++ b/packages/host/README.md @@ -78,6 +78,27 @@ See docs/configuration.md in the main repositiory for the available configuratio Shows the value for a single config entry. +### `create-scenario ` + +Create a scenario with the Clusterio modules patched into it and write it to the `output` directory. +This is the same patching that is applied to saves when an instance starts, see docs/how-it-works.md in the main repository. +The result can be used as a regular scenario outside of Clusterio by copying it to the `scenarios` folder of Factorio or into a mod. + +By default the freeplay scenario from the latest Factorio install in `host.factorio_directory` is patched with the modules from all plugins that are enabled in the host config. + + * `--scenario ` Scenario directory, zipped scenario or save to patch instead of freeplay. + This has the same restrictions as save patching, the scenario must either use the event_handler library and have been seen by Clusterio before, or already have been patched by Clusterio. + + * `--factorio-version ` Factorio version to take the freeplay scenario from. + Defaults to `latest`. + + * `--plugins ...` Only patch in modules from the given plugins. + +For example, creating a scenario with only the inventory_sync module: + + npx clusteriohost create-scenario ~/.factorio/scenarios/inventory_sync --plugins inventory_sync + + #### `config list` Lists up all configuration entries with their currently configured values. diff --git a/packages/host/host.ts b/packages/host/host.ts index a2b882fb..8794e49d 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -24,6 +24,7 @@ import * as lib from "@clusterio/lib"; import { ConsoleTransport, levels, logger } from "@clusterio/lib"; import Host from "./src/Host"; +import { createScenarioCommand, handleCreateScenarioCommand } from "./src/create_scenario"; let host: Host | undefined; @@ -113,6 +114,11 @@ async function startHost() { .option("bypass-lock-file", { hidden: true, type: "boolean", nargs: 0, default: false }) .command("plugin", "Manage available plugins", lib.pluginCommand) .command("config", "Manage Host config", lib.configCommand) + .command( + "create-scenario ", + "Create a scenario patched with modules for use outside Clusterio", + createScenarioCommand, + ) .command("run", "Run host", yargs => { yargs.option("can-restart", { type: "boolean", nargs: 0, default: false, @@ -194,6 +200,11 @@ async function startHost() { return; } + if (command === "create-scenario") { + await handleCreateScenarioCommand(args, hostConfig, pluginInfos); + return; + } + // From this point on the host should run await hostConfigLock.acquire(); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 3e45f04d..0a658048 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -10,7 +10,7 @@ const execAsync = util.promisify(exec); import * as lib from "@clusterio/lib"; import { FactorioServer } from "./server"; -import { SaveModule, patch } from "./patch"; +import { loadModules, patch } from "./patch"; import { exportData } from "./export"; import type Host from "./Host"; import BaseInstancePlugin from "./BaseInstancePlugin"; @@ -951,29 +951,7 @@ end`.replace(/\r?\n/g, " "); // Patch save with lua modules from plugins this.logger.verbose("Patching save"); - // Find plugin modules to patch in - let modules: Map = new Map(); - for (let plugin of this.plugins.values()) { - let module = await SaveModule.fromPlugin(plugin); - if (!module) { - continue; - } - modules.set(module.info.name, module); - } - - // Find stand alone modules to load - // XXX for now only the included clusterio module is loaded - let modulesDirectory = path.join(__dirname, "..", "..", "..", "modules"); - for (let entry of await fs.readdir(modulesDirectory, { withFileTypes: true })) { - if (entry.isDirectory()) { - if (modules.has(entry.name)) { - throw new Error(`Module with name ${entry.name} already exists in a plugin`); - } - let module = await SaveModule.fromDirectory(path.join(modulesDirectory, entry.name)); - modules.set(module.info.name, module); - } - } - + let modules = await loadModules([...this.plugins.values()].map(plugin => plugin.info)); await patch(this.path("saves", saveName), [...modules.values()]); return saveName; } diff --git a/packages/host/src/create_scenario.ts b/packages/host/src/create_scenario.ts new file mode 100644 index 00000000..4aff341c --- /dev/null +++ b/packages/host/src/create_scenario.ts @@ -0,0 +1,149 @@ +// Implementation of the create-scenario command +import fs from "node:fs/promises"; +import os from "node:os"; +import JSZip from "jszip"; +import path from "path"; + +import * as lib from "@clusterio/lib"; +import { logger } from "@clusterio/lib"; + +import { loadModules, patch, SaveModule } from "./patch"; +import { findVersion } from "./server"; + + +/** + * Create a scenario patched with the given modules + * + * Runs the save patching on the scenario at scenarioPath and writes the + * result to the directory at outputPath. The scenario can be a directory, + * a zipped scenario or a save. + * + * @param scenarioPath - Path to the scenario to patch. + * @param outputPath - Directory to write the patched scenario to. + * @param modules - Description of the modules to patch. + */ +export async function createScenario(scenarioPath: string, outputPath: string, modules: SaveModule[]) { + if (await fs.access(outputPath).then(() => true, () => false)) { + throw new Error(`${outputPath} already exists`); + } + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-scenario-")); + try { + const tempPath = path.join(tempDir, "scenario.zip"); + if ((await fs.stat(scenarioPath)).isDirectory()) { + await zipDirectory(scenarioPath, tempPath); + } else { + await fs.copyFile(scenarioPath, tempPath); + } + + await patch(tempPath, modules); + await extractZip(tempPath, outputPath); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +} + +async function zipDirectory(dirPath: string, zipPath: string) { + const root = path.basename(path.resolve(dirPath)); + const zip = new JSZip(); + let dirs: [string, string][] = [[dirPath, root]]; + while (dirs.length) { + let [dir, relativeDir] = dirs.pop()!; + for (let entry of await fs.readdir(dir, { withFileTypes: true })) { + let fsPath = path.join(dir, entry.name); + let relativePath = path.posix.join(relativeDir, entry.name); + if (entry.isFile()) { + zip.file(relativePath, await fs.readFile(fsPath)); + } else if (entry.isDirectory()) { + dirs.push([fsPath, relativePath]); + } + } + } + await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" })); +} + +async function extractZip(zipPath: string, dirPath: string) { + const zip = await JSZip.loadAsync(await fs.readFile(zipPath)); + const root = lib.findRoot(zip); + for (let file of Object.values(zip.files)) { + if (file.dir || !file.name.startsWith(`${root}/`)) { + continue; + } + const fsPath = path.join(dirPath, file.name.slice(root.length + 1)); + await fs.mkdir(path.dirname(fsPath), { recursive: true }); + await fs.writeFile(fsPath, await file.async("nodebuffer")); + } +} + +/** + * Yargs create-scenario command + * + * @param yargs - yargs command builder. + */ +export function createScenarioCommand(yargs: any) { + yargs + .positional("output", { + describe: "Directory to write the scenario to", + type: "string", + }) + .option("scenario", { + nargs: 1, + describe: "Scenario directory, zipped scenario or save to patch, defaults to freeplay", + type: "string", + }) + .option("factorio-version", { + nargs: 1, + describe: "Factorio version to take the freeplay scenario from", + default: "latest", + type: "string", + }) + .option("plugins", { + describe: "Plugins to patch modules from, defaults to all plugins enabled in the host config", + type: "array", + }) + ; +} + +/** + * Handle yargs create-scenario command + * + * @param args - yargs args object. + * @param hostConfig - Host config. + * @param pluginInfos - Plugins available to the host. + */ +export async function handleCreateScenarioCommand( + args: Record, + hostConfig: lib.HostConfig, + pluginInfos: lib.PluginNodeEnvInfo[], +) { + let selected: lib.PluginNodeEnvInfo[]; + if (args.plugins) { + selected = (args.plugins as string[]).map(name => { + const pluginInfo = pluginInfos.find(info => info.name === name); + if (!pluginInfo) { + throw new Error(`Plugin ${name} is not in the plugin list`); + } + return pluginInfo; + }); + } else { + selected = pluginInfos.filter( + info => info.instanceEntrypoint && hostConfig.get(`${info.name}.load_plugin`) + ); + } + + let scenarioPath = args.scenario as string | undefined; + if (scenarioPath === undefined) { + const factorioDir = hostConfig.get("host.factorio_directory"); + const version = args.factorioVersion as lib.TargetVersion; + if (!lib.isTargetVersion(version)) { + throw new Error(`Invalid Factorio version ${version}`); + } + const [dataDir] = await findVersion(factorioDir, version); + scenarioPath = path.join(dataDir, "base", "scenarios", "freeplay"); + } + + const modules = await loadModules(selected); + logger.info(`Patching ${scenarioPath} with modules ${[...modules.keys()].join(", ")}`); + await createScenario(scenarioPath, args.output as string, [...modules.values()]); + logger.info(`Created scenario ${args.output}`); +} diff --git a/packages/host/src/patch.ts b/packages/host/src/patch.ts index 639ce65f..031d1565 100644 --- a/packages/host/src/patch.ts +++ b/packages/host/src/patch.ts @@ -91,7 +91,11 @@ export class SaveModule { } static async fromPlugin(plugin: BaseInstancePlugin) { - let pluginPackagePath = require.resolve(path.posix.join(plugin.info.requirePath, "package.json")); + return SaveModule.fromPluginInfo(plugin.info); + } + + static async fromPluginInfo(info: lib.PluginNodeEnvInfo) { + let pluginPackagePath = require.resolve(path.posix.join(info.requirePath, "package.json")); let moduleDirectory = path.join(path.dirname(pluginPackagePath), "module"); try { await fs.access(moduleDirectory); @@ -103,17 +107,17 @@ export class SaveModule { let moduleJson; try { moduleJson = { - name: plugin.info.name, - version: plugin.info.version, + name: info.name, + version: info.version, dependencies: { "clusterio": "*" }, ...JSON.parse(await fs.readFile(moduleJsonPath, "utf8")), }; } catch (err: any) { - throw new Error(`Loading module/module.json in plugin ${plugin.info.name} failed: ${err.message}`); + throw new Error(`Loading module/module.json in plugin ${info.name} failed: ${err.message}`); } if (!lib.ModuleInfo.validate(moduleJson)) { throw new Error( - `module/module.json in plugin ${plugin.info.name} failed validation:\n` + + `module/module.json in plugin ${info.name} failed validation:\n` + `${JSON.stringify(lib.ModuleInfo.validate.errors, null, "\t")}` ); } @@ -149,6 +153,40 @@ export class SaveModule { } } +/** + * Load the modules to patch into a save + * + * Collects the modules provided by the given plugins together with the + * modules bundled with the host package. + * + * @param pluginInfos - Plugins to load modules from. + * @returns Map of module name to module. + */ +export async function loadModules(pluginInfos: Iterable) { + let modules: Map = new Map(); + for (let pluginInfo of pluginInfos) { + let module = await SaveModule.fromPluginInfo(pluginInfo); + if (!module) { + continue; + } + modules.set(module.info.name, module); + } + + // Find stand alone modules to load + // XXX for now only the included clusterio module is loaded + let modulesDirectory = path.join(__dirname, "..", "..", "..", "modules"); + for (let entry of await fs.readdir(modulesDirectory, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (modules.has(entry.name)) { + throw new Error(`Module with name ${entry.name} already exists in a plugin`); + } + let module = await SaveModule.fromDirectory(path.join(modulesDirectory, entry.name)); + modules.set(module.info.name, module); + } + } + return modules; +} + export class PatchInfo { static currentVersion = 1; diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index 32d65b52..309a6044 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -181,9 +181,11 @@ async function downloadAndExtractTar(url: string, targetDir: string) { * Version to look for, supports the special value "latest" for the * latest version available. * @returns Array with path to data dir and version found. - * @internal */ -async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion): Promise<[string, lib.FullVersion]> { +export async function findVersion( + factorioDir: string, + targetVersion: lib.TargetVersion, +): Promise<[string, lib.FullVersion]> { // Check if this is a direct installation dir const directVersion = await getFactorioVersion(factorioDir); if (directVersion !== null) { @@ -1580,7 +1582,6 @@ export class FactorioServer extends events.EventEmitter { // For testing only export const _getFactorioVersion = getFactorioVersion; export const _versionOrder = versionOrder; -export const _findVersion = findVersion; export const _listFactorioVersions = listFactorioVersions; export const _downloadAndExtractZip = downloadAndExtractZip; export const _downloadAndExtractTar = downloadAndExtractTar; diff --git a/test/host/create_scenario.js b/test/host/create_scenario.js new file mode 100644 index 00000000..4609d732 --- /dev/null +++ b/test/host/create_scenario.js @@ -0,0 +1,93 @@ +"use strict"; +const assert = require("assert").strict; +const fs = require("node:fs/promises"); +const JSZip = require("jszip"); +const path = require("path"); + +const lib = require("@clusterio/lib"); +const { createScenario } = require("@clusterio/host/dist/node/src/create_scenario"); +const { SaveModule, PatchInfo } = require("@clusterio/host/dist/node/src/patch"); + + +describe("host/create_scenario", function() { + const tempDir = path.join("temp", "test", "create_scenario"); + const patchInfo = new PatchInfo(0, new lib.ModuleInfo("test", "1.0.0", [], ["scenario"]), []); + const scenarioFiles = new Map([ + ["scenario.lua", "-- scenario\n"], + ["description.json", "{}\n"], + ["locale/en/test.cfg", "test=Test\n"], + ["clusterio.json", JSON.stringify(patchInfo)], + ]); + const module = new SaveModule( + new lib.ModuleInfo("spam", "1.0.0", ["spam.lua"]), + new Map([["modules/spam/spam.lua", Buffer.from("return {}\n")]]), + ); + + async function readDir(dirPath) { + const files = new Map(); + for (const entry of await fs.readdir(dirPath, { recursive: true, withFileTypes: true })) { + if (entry.isFile()) { + const fsPath = path.join(entry.parentPath, entry.name); + files.set(path.relative(dirPath, fsPath).split(path.sep).join("/"), await fs.readFile(fsPath, "utf8")); + } + } + return files; + } + + function checkOutput(files) { + assert(files.get("control.lua").includes('event_handler.add_lib(require("modules/spam/spam"))')); + assert.equal(JSON.parse(files.get("clusterio.json")).patch_number, 1); + assert.equal(files.get("modules/spam/spam.lua"), "return {}\n"); + for (const name of ["scenario.lua", "description.json", "locale/en/test.cfg"]) { + assert.equal(files.get(name), scenarioFiles.get(name)); + } + assert.equal(files.size, scenarioFiles.size + 2); + } + + before(async function() { + await fs.rm(tempDir, { recursive: true, force: true }); + await fs.mkdir(path.join(tempDir, "scenario"), { recursive: true }); + const zip = new JSZip(); + for (const [name, content] of scenarioFiles) { + const fsPath = path.join(tempDir, "scenario", name); + await fs.mkdir(path.dirname(fsPath), { recursive: true }); + await fs.writeFile(fsPath, content); + zip.file(`scenario/${name}`, content); + } + await fs.writeFile(path.join(tempDir, "scenario.zip"), await zip.generateAsync({ type: "nodebuffer" })); + }); + + it("should patch a scenario directory", async function() { + const output = path.join(tempDir, "from_dir"); + await createScenario(path.join(tempDir, "scenario"), output, [module]); + checkOutput(await readDir(output)); + }); + + it("should patch a zipped scenario", async function() { + const output = path.join(tempDir, "from_zip"); + await createScenario(path.join(tempDir, "scenario.zip"), output, [module]); + checkOutput(await readDir(output)); + }); + + it("should not modify the input scenario", async function() { + const files = await readDir(path.join(tempDir, "scenario")); + assert.deepEqual(files, scenarioFiles); + }); + + it("should throw if the output already exists", async function() { + const output = path.join(tempDir, "from_dir"); + await assert.rejects( + createScenario(path.join(tempDir, "scenario"), output, [module]), + new Error(`${output} already exists`), + ); + }); + + it("should throw on unknown scenario", async function() { + await fs.mkdir(path.join(tempDir, "unknown")); + await fs.writeFile(path.join(tempDir, "unknown", "control.lua"), "-- unknown\n"); + await assert.rejects( + createScenario(path.join(tempDir, "unknown"), path.join(tempDir, "from_unknown"), [module]), + new Error("Unable to patch save, unknown scenario (3acc3be3861144e55604f5ac2f2555071885ebc4)") + ); + }); +}); diff --git a/test/host/patch.js b/test/host/patch.js index 3df1ca5e..53c4aa43 100644 --- a/test/host/patch.js +++ b/test/host/patch.js @@ -226,6 +226,22 @@ describe("host/patch", function() { }); }); + describe("loadModules()", function() { + it("should load the bundled clusterio module", async function() { + const modules = await patch.loadModules([]); + assert.deepEqual([...modules.keys()], ["clusterio"]); + assert(modules.get("clusterio").files.has("modules/clusterio/api.lua")); + }); + it("should load modules from plugins", async function() { + const pluginInfos = await lib.loadPluginInfos( + new Map([["research_sync", path.resolve("plugins/research_sync")]]) + ); + const modules = await patch.loadModules(pluginInfos); + assert.deepEqual([...modules.keys()], ["research_sync", "clusterio"]); + assert(modules.get("research_sync").files.has("modules/research_sync/sync.lua")); + }); + }); + describe("patch()", function() { it("should throw on unknown scenario", async function() { let zip = new JSZip(); diff --git a/test/host/server.js b/test/host/server.js index a18e6058..2321d527 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -42,26 +42,26 @@ describe("host/server", function() { describe("direct install", function() { it("should search given directory for latest Factorio install", async function() { const installDir = path.join("test", "file", "factorio", "0.1.2"); - const [dir, version] = await hostServer._findVersion(installDir, "latest"); + const [dir, version] = await hostServer.findVersion(installDir, "latest"); assert.equal(dir, path.join(installDir, "data")); assert.equal(version, "0.1.2"); }); it("should search given directory for given Factorio install", async function() { const installDir = path.join("test", "file", "factorio", "0.1.1"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1.1"); assert.equal(dir, path.join(installDir, "data")); assert.equal(version, "0.1.1"); }); it("should search given directory for partly given Factorio install", async function() { const installDir = path.join("test", "file", "factorio", "0.1.2"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1"); assert.equal(dir, path.join(installDir, "data")); assert.equal(version, "0.1.2"); }); it("should reject if the version does not match", async function() { let installDir = path.join("test", "file", "factorio", "0.1.1"); await assert.rejects( - hostServer._findVersion(installDir, "0.1.2"), + hostServer.findVersion(installDir, "0.1.2"), new Error( `Unable to find Factorio version 0.1.2: ${installDir} is a direct (single-version) ` + `install of 0.1.1. Use a versioned layout, where ${installDir} contains a subdirectory ` + @@ -74,33 +74,33 @@ describe("host/server", function() { it("should reject if no factorio install with the given version was found", async function() { let installDir = path.join("test", "file", "factorio"); await assert.rejects( - hostServer._findVersion(installDir, "0.1.3"), + hostServer.findVersion(installDir, "0.1.3"), new Error("Unable to find Factorio version 0.1.3") ); }); it("should search given directory for given Factorio install", async function() { const installDir = path.join("test", "file", "factorio"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1.1"); assert.equal(dir, path.join(installDir, "0.1.1", "data")); assert.equal(version, "0.1.1"); }); it("should search given directory for partly given Factorio install", async function() { const installDir = path.join("test", "file", "factorio"); - const [dir, version] = await hostServer._findVersion(installDir, "0.1"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1"); assert.equal(dir, path.join(installDir, "0.1.2", "data")); assert.equal(version, "0.1.2"); }); it("should reject if no factorio install with the given version was found", async function() { let installDir = path.join("test", "file", "factorio"); await assert.rejects( - hostServer._findVersion(installDir, "0.1.3"), + hostServer.findVersion(installDir, "0.1.3"), new Error("Unable to find Factorio version 0.1.3") ); }); it("should reject if no factorio install was found", async function() { let installDir = path.join("test", "file"); await assert.rejects( - hostServer._findVersion(installDir, "0.0.0"), + hostServer.findVersion(installDir, "0.0.0"), new Error(`Unable to find any Factorio install in ${installDir}`) ); });