From fb4ffaffed772c2983cb2d75b5d71c6f179f9a15 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 15 Oct 2022 17:53:22 +0200 Subject: [PATCH] Implement mod pack loading for shared mods Wipe and reconstruct the instances' mods/ folder using the configured ModPack for the case were the mods folder is shared with the master server. This assumes the master server will check the mod checksums before sending the start request. --- .dockerignore | 3 +- .github/workflows/node.js.yml | 4 +- .gitignore | 1 - Dockerfile | 8 +- packages/lib/config/definitions.js | 20 +++ packages/master/src/BaseConnection.js | 9 ++ packages/master/src/ControlConnection.js | 9 -- packages/master/src/SlaveConnection.js | 5 - packages/slave/slave.js | 2 +- packages/slave/src/Instance.js | 149 +++++++++--------- .../web_ui/src/components/ModPackViewPage.jsx | 2 + test/integration/clusterio.js | 1 + test/integration/index.js | 9 ++ test/slave.js | 67 -------- 14 files changed, 124 insertions(+), 165 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2ad21c71..74f895db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,8 +13,7 @@ factorio factorio.tar.gz # We build internal mods during build, mods from local test env should not be applied -sharedMods -sharedModsDisabled +mods # These files should never be required during the build .git diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index ee732740..c3a3923d 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -26,8 +26,8 @@ jobs: node-version: ${{ matrix.node-version }} - run: pnpm i --no-optional --no-frozen-lockfile - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz - - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods - - run: wget -q -O temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip + - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/mods + - run: wget -q -O temp/test/mods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip - name: Run tests if: ${{ matrix.node-version != '12.x' }} run: pnpm test diff --git a/.gitignore b/.gitignore index a769fb6c..37bf1a49 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,6 @@ node_modules Factorio* /factorio /instances -/sharedMods /mods /database /plugin-list.json diff --git a/Dockerfile b/Dockerfile index 71008b26..3eb36c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,15 +22,15 @@ RUN pnpm install #RUN pnpm install @clusterio/plugin-subspace_storage #RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage -COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/sharedMods/ +COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/mods/ # Build Lua library mod RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \ -&& mv dist/* sharedMods/ \ +&& mv dist/* mods/ \ && mkdir temp \ && mkdir temp/test \ -&& cp sharedMods/ temp/test/ -r \ -&& ls sharedMods +&& cp mods/ temp/test/ -r \ +&& ls mods # Remove node_modules RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \; diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index 01d74f78..b0926270 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -160,6 +160,20 @@ SlaveGroup.define({ type: "string", initial_value: "factorio", }); +SlaveGroup.define({ + name: "mods_directory", + title: "Mods Directory", + description: "Path to directory where mods for instances are cached.", + type: "string", + initial_value: "mods", +}); +SlaveGroup.define({ + name: "mods_directory_is_shared", + title: "Mods Directory is Shared", + description: "True if the mods directory is shared with the master server.", + type: "boolean", + initial_value: true, +}); SlaveGroup.define({ name: "instances_directory", description: "Path to directory to store instances in.", @@ -280,6 +294,12 @@ FactorioGroup.define({ type: "number", initial_value: 5, }); +FactorioGroup.define({ + name: "mod_pack", + description: "Mod pack to use on this server", + type: "number", + optional: true, +}); FactorioGroup.define({ name: "enable_save_patching", description: "Patch saves with Lua code. Required for Clusterio integrations, lua modules, and most plugins.", diff --git a/packages/master/src/BaseConnection.js b/packages/master/src/BaseConnection.js index e7bca6fa..f22094cf 100644 --- a/packages/master/src/BaseConnection.js +++ b/packages/master/src/BaseConnection.js @@ -84,6 +84,15 @@ class BaseConnection extends libLink.Link { get connected() { return !this._disconnecting && this.connector.connected; } + + async getModPackRequestHandler(message) { + let { id } = message.data; + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } } module.exports = BaseConnection; diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index 48834a90..9856b33b 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -474,15 +474,6 @@ class ControlConnection extends BaseConnection { return { save: result.save }; } - async getModPackRequestHandler(message) { - let { id } = message.data; - let modPack = this._master.modPacks.get(id); - if (!modPack) { - throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); - } - return { mod_pack: modPack.toJSON() }; - } - async listModPacksRequestHandler(message) { return { list: [...this._master.modPacks.values()].map(pack => pack.toJSON()) }; } diff --git a/packages/master/src/SlaveConnection.js b/packages/master/src/SlaveConnection.js index 4f9203ed..35564264 100644 --- a/packages/master/src/SlaveConnection.js +++ b/packages/master/src/SlaveConnection.js @@ -3,7 +3,6 @@ const libConfig = require("@clusterio/lib/config"); const libLink = require("@clusterio/lib/link"); const { logger } = require("@clusterio/lib/logging"); const libPlugin = require("@clusterio/lib/plugin"); -const libErrors = require("@clusterio/lib/errors"); const PlayerStats = require("@clusterio/lib/PlayerStats"); const BaseConnection = require("./BaseConnection"); @@ -168,10 +167,6 @@ class SlaveConnection extends BaseConnection { this._master.saveListUpdate(message.data); } - async getModPackRequestHandler(message) { - throw new libErrors.RequestError("Not implemented"); - } - async logMessageEventHandler(message) { this._master.clusterLogger.log({ ...message.data.info, diff --git a/packages/slave/slave.js b/packages/slave/slave.js index 9c4fb8c6..8f9f70e7 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -177,7 +177,7 @@ async function startSlave() { // If we get here the command was run await fs.ensureDir(slaveConfig.get("slave.instances_directory")); - await fs.ensureDir("sharedMods"); + await fs.ensureDir(slaveConfig.get("slave.mods_directory")); await fs.ensureDir("modules"); // Set the process title, shows up as the title of the CMD window on windows diff --git a/packages/slave/src/Instance.js b/packages/slave/src/Instance.js index a5a717a6..2148206e 100644 --- a/packages/slave/src/Instance.js +++ b/packages/slave/src/Instance.js @@ -6,6 +6,7 @@ const phin = require("phin"); const util = require("util"); // internal libraries +const libData = require("@clusterio/lib/data"); const libFileOps = require("@clusterio/lib/file_ops"); const libFactorio = require("@clusterio/lib/factorio"); const libLink = require("@clusterio/lib/link"); @@ -94,71 +95,6 @@ const serverSettingsActions = { }, }; -/** - * Create and update symlinks for shared mods in an instance - * - * Creates symlinks for .zip and .dat files that are not present in the - * instance mods directory but is present in the sharedMods directory, - * and removes any symlinks that don't point to a file in the instance - * mods directory. If the instance mods directory doesn't exist it will - * be created. - * - * Note that on Windows this creates hard links instead of symbolic - * links as the latter requires elevated privileges. This unfortunately - * means the removal of mods from the shared mods dir can't be detected. - * - * @param {Instance} instance - Instance to link mods for - * @param {string} sharedMods - Path to folder to link mods from. - * @private - */ -async function symlinkMods(instance, sharedMods) { - await fs.ensureDir(instance.path("mods")); - - // Remove broken symlinks in instance mods. - for (let entry of await fs.readdir(instance.path("mods"), { withFileTypes: true })) { - if (entry.isSymbolicLink()) { - if (!await fs.pathExists(instance.path("mods", entry.name))) { - instance.logger.verbose(`Removing broken symlink ${entry.name}`); - await fs.unlink(instance.path("mods", entry.name)); - } - } - } - - // Link entries that are in sharedMods but not in instance mods. - let instanceModsEntries = new Set(await fs.readdir(instance.path("mods"))); - for (let entry of await fs.readdir(sharedMods, { withFileTypes: true })) { - if (entry.isFile()) { - if ([".zip", ".dat", ".json"].includes(path.extname(entry.name))) { - if (!instanceModsEntries.has(entry.name)) { - instance.logger.verbose(`linking ${entry.name} from ${sharedMods}`); - let target = path.join(sharedMods, entry.name); - let link = instance.path("mods", entry.name); - - /* eslint-disable max-depth */ - if (process.platform !== "win32") { - await fs.symlink(path.relative(path.dirname(link), target), link); - - // On Windows symlinks require elevated privileges, which is - // not something we want to have. For this reason the mods - // are hard linked instead. This has the drawback of not - // being able to identify when mods are removed from the - // sharedMods directory, or which mods are linked. - } else { - await fs.link(target, link); - } - /* eslint-enable max-depth */ - } - - } else { - instance.logger.warn(`Warning: ignoring file '${entry.name}' in sharedMods`); - } - - } else { - instance.logger.warn(`Warning: ignoring non-file '${entry.name}' in sharedMods`); - } - } -} - /** * Keeps track of the runtime parameters of an instance @@ -643,6 +579,77 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await fs.ensureDir(path.join(instanceDir, "saves")); } + /** + * Sync instance mods directory with configured mod pack + * + * Adds, deletes, and updates all files in the instance mods folder to + * match up with the mod pack that's configured. If no mod pack is + * configured then an empty mod pack is used, esentially turning it into + * a vanilla mods folder. If the instance mods directory does't exist + * then it will be created. + * + * On Linux this creates symlinks to mods in the slave's mods folder, on + * Windows hard links are used instead due to symlinks being privileged. + */ + async syncMods() { + const modPackId = this.config.get("factorio.mod_pack"); + let modPack; + if (modPackId === null) { + modPack = new libData.ModPack(); + } else { + const response = await libLink.messages.getModPack.send(this, { id: modPackId }); + modPack = new libData.ModPack(response.mod_pack); + } + + // TODO validate factorioVersion + + if (!this._slave.config.get("slave.mods_directory_is_shared")) { + throw new Error("Fetching mods is not implemented"); + } + + await fs.ensureDir(this.path("mods")); + + // Remove all files + for (let entry of await fs.readdir(this.path("mods"), { withFileTypes: true })) { + if (entry.isDirectory()) { + this.logger.warn( + `Found unexpected directory ${entry.name} in mods folder, it may break Clusterio's mod syncing` + ); + continue; + } + + if (entry.isFile() || entry.isSymbolicLink()) { + await fs.unlink(this.path("mods", entry.name)); + } + } + + // Add mods from mod the pack + const modsDir = this._slave.config.get("slave.mods_directory"); + for (let mod of modPack.mods.values()) { + const modFile = `${mod.name}_${mod.version}.zip`; + const target = path.join(modsDir, modFile); + const link = this.path("mods", modFile); + + if (process.platform !== "win32") { + await fs.symlink(path.relative(path.dirname(link), target), link); + + // On Windows symlinks require elevated privileges, which is + // not something we want to have. For this reason the mods + // are hard linked instead. + } else { + await fs.link(target, link); + } + } + + // Write mod-list.json + await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({ + mods: [{ name: "base" }, ...modPack.mods.values()].map(m => ({ name: m.name, enabled: true })), + }, null, 2)); + + // Write mod-settings.dat + await fs.outputFile(this.path("mods", "mod-settings.dat"), modPack.toModSettingsDat()); + } + /** * Prepare instance for starting * @@ -678,8 +685,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); ); } - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } /** @@ -1030,8 +1036,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.verbose("Creating save ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } catch (err) { this.notifyExit(); @@ -1053,8 +1058,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.info("Exporting data ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); let zip = await libFactorio.exportData(this.server); let content = await zip.generateAsync({ type: "nodebuffer" }); @@ -1116,6 +1120,3 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); } module.exports = Instance; - -// For testing only -module.exports._symlinkMods = symlinkMods; diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 115560a3..dfce060d 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -368,9 +368,11 @@ function SettingsTable(props) { {controls("startup", pickValue(props.modPack.settings["startup"]))} Map + These only apply for new saves. {controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))} Per Player + Default settings for new players, these only apply for new saves. {controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))} ; } diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index e5da8157..6ab91fa9 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -265,6 +265,7 @@ describe("Integration of Clusterio", function() { ); await execCtl(`instance config set-prop test factorio.settings visibility "'${value}'"`); await execCtl("instance config set-prop test factorio.settings require_user_verification false"); + await execCtl("instance config set test factorio.mod_pack 12"); }); }); diff --git a/test/integration/index.js b/test/integration/index.js index 2830e39f..b604bb1f 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -8,6 +8,7 @@ const phin = require("phin"); const util = require("util"); const events = require("events"); +const libData = require("@clusterio/lib/data"); const libLink = require("@clusterio/lib/link"); const { LineSplitter } = require("@clusterio/lib/stream"); const { ConsoleTransport, logger } = require("@clusterio/lib/logging"); @@ -220,6 +221,14 @@ before(async function() { controlConnector.token = controlToken; control = new TestControl(controlConnector); await controlConnector.connect(); + + const testPack = new libData.ModPack(); + testPack.id = 12; + testPack.name = "subspace_storage-pack"; + testPack.factorioVersion = "1.1.0"; + testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); + testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); + libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); }); after(async function() { diff --git a/test/slave.js b/test/slave.js index 0cb23498..d8d34788 100644 --- a/test/slave.js +++ b/test/slave.js @@ -1,11 +1,8 @@ "use strict"; const assert = require("assert").strict; const path = require("path"); -const fs = require("fs-extra"); -const libLink = require("@clusterio/lib/link"); const libConfig = require("@clusterio/lib/config"); -const Instance = require("@clusterio/slave/src/Instance"); const Slave = require("@clusterio/slave/src/Slave"); describe("Slave testing", function() { @@ -13,70 +10,6 @@ describe("Slave testing", function() { libConfig.InstanceConfig.finalize(); }); - describe("symlinkMods()", function() { - let testDir = path.join("temp", "test", "symlink"); - - let instance; - before(async function() { - // Remove previous test output - await fs.remove(testDir); - - // Add some test mods - await fs.outputFile(path.join(testDir, "shared", "mod_a.zip"), "a"); - await fs.outputFile(path.join(testDir, "shared", "mod_b.zip"), "b"); - await fs.outputFile(path.join(testDir, "shared", "mod.dat"), "c"); - await fs.outputFile(path.join(testDir, "shared", "mod-list.json"), "d"); - - let instanceConfig = new libConfig.InstanceConfig("slave"); - await instanceConfig.init(); - instanceConfig.set("instance.name", "test"); - instance = new Instance( - {}, new libLink.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig - ); - await fs.outputFile(instance.path("mods", "mod_i.zip"), "i"); - }); - - it("should link mods and data files", async function() { - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert.equal(await fs.readFile(instance.path("mods", "mod_a.zip"), "utf-8"), "a"); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - - it("should ignore directories", async function() { - await fs.ensureDir(path.join(testDir, "shared", "dir")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "dir"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should ignore files", async function() { - await fs.outputFile(path.join(testDir, "shared", "file"), "f"); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "file"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should unlink removed mods", async function() { - // This does not work on Windows - if (process.platform === "win32") { - this.skip(); - } - - await fs.unlink(path.join(testDir, "shared", "mod_a.zip")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - await assert.rejects(fs.lstat(instance.path("mods", "mod_a.zip")), { code: "ENOENT" }); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - }); - describe("discoverInstances()", function() { it("should discover test instance", async function() { let instancePath = path.join("test", "file", "instances");