diff --git a/.dockerignore b/.dockerignore index 2ad21c71..74f895db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,8 +13,7 @@ factorio factorio.tar.gz # We build internal mods during build, mods from local test env should not be applied -sharedMods -sharedModsDisabled +mods # These files should never be required during the build .git diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index ee732740..c3a3923d 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -26,8 +26,8 @@ jobs: node-version: ${{ matrix.node-version }} - run: pnpm i --no-optional --no-frozen-lockfile - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz - - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods - - run: wget -q -O temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip + - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/mods + - run: wget -q -O temp/test/mods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip - name: Run tests if: ${{ matrix.node-version != '12.x' }} run: pnpm test diff --git a/.gitignore b/.gitignore index a769fb6c..37bf1a49 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,6 @@ node_modules Factorio* /factorio /instances -/sharedMods /mods /database /plugin-list.json diff --git a/Dockerfile b/Dockerfile index 71008b26..3eb36c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,15 +22,15 @@ RUN pnpm install #RUN pnpm install @clusterio/plugin-subspace_storage #RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage -COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/sharedMods/ +COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/mods/ # Build Lua library mod RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \ -&& mv dist/* sharedMods/ \ +&& mv dist/* mods/ \ && mkdir temp \ && mkdir temp/test \ -&& cp sharedMods/ temp/test/ -r \ -&& ls sharedMods +&& cp mods/ temp/test/ -r \ +&& ls mods # Remove node_modules RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \; diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index 01d74f78..b0926270 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -160,6 +160,20 @@ SlaveGroup.define({ type: "string", initial_value: "factorio", }); +SlaveGroup.define({ + name: "mods_directory", + title: "Mods Directory", + description: "Path to directory where mods for instances are cached.", + type: "string", + initial_value: "mods", +}); +SlaveGroup.define({ + name: "mods_directory_is_shared", + title: "Mods Directory is Shared", + description: "True if the mods directory is shared with the master server.", + type: "boolean", + initial_value: true, +}); SlaveGroup.define({ name: "instances_directory", description: "Path to directory to store instances in.", @@ -280,6 +294,12 @@ FactorioGroup.define({ type: "number", initial_value: 5, }); +FactorioGroup.define({ + name: "mod_pack", + description: "Mod pack to use on this server", + type: "number", + optional: true, +}); FactorioGroup.define({ name: "enable_save_patching", description: "Patch saves with Lua code. Required for Clusterio integrations, lua modules, and most plugins.", diff --git a/packages/master/src/BaseConnection.js b/packages/master/src/BaseConnection.js index e7bca6fa..f22094cf 100644 --- a/packages/master/src/BaseConnection.js +++ b/packages/master/src/BaseConnection.js @@ -84,6 +84,15 @@ class BaseConnection extends libLink.Link { get connected() { return !this._disconnecting && this.connector.connected; } + + async getModPackRequestHandler(message) { + let { id } = message.data; + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } } module.exports = BaseConnection; diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index 48834a90..9856b33b 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -474,15 +474,6 @@ class ControlConnection extends BaseConnection { return { save: result.save }; } - async getModPackRequestHandler(message) { - let { id } = message.data; - let modPack = this._master.modPacks.get(id); - if (!modPack) { - throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); - } - return { mod_pack: modPack.toJSON() }; - } - async listModPacksRequestHandler(message) { return { list: [...this._master.modPacks.values()].map(pack => pack.toJSON()) }; } diff --git a/packages/master/src/SlaveConnection.js b/packages/master/src/SlaveConnection.js index 4f9203ed..35564264 100644 --- a/packages/master/src/SlaveConnection.js +++ b/packages/master/src/SlaveConnection.js @@ -3,7 +3,6 @@ const libConfig = require("@clusterio/lib/config"); const libLink = require("@clusterio/lib/link"); const { logger } = require("@clusterio/lib/logging"); const libPlugin = require("@clusterio/lib/plugin"); -const libErrors = require("@clusterio/lib/errors"); const PlayerStats = require("@clusterio/lib/PlayerStats"); const BaseConnection = require("./BaseConnection"); @@ -168,10 +167,6 @@ class SlaveConnection extends BaseConnection { this._master.saveListUpdate(message.data); } - async getModPackRequestHandler(message) { - throw new libErrors.RequestError("Not implemented"); - } - async logMessageEventHandler(message) { this._master.clusterLogger.log({ ...message.data.info, diff --git a/packages/slave/slave.js b/packages/slave/slave.js index 9c4fb8c6..8f9f70e7 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -177,7 +177,7 @@ async function startSlave() { // If we get here the command was run await fs.ensureDir(slaveConfig.get("slave.instances_directory")); - await fs.ensureDir("sharedMods"); + await fs.ensureDir(slaveConfig.get("slave.mods_directory")); await fs.ensureDir("modules"); // Set the process title, shows up as the title of the CMD window on windows diff --git a/packages/slave/src/Instance.js b/packages/slave/src/Instance.js index a5a717a6..2148206e 100644 --- a/packages/slave/src/Instance.js +++ b/packages/slave/src/Instance.js @@ -6,6 +6,7 @@ const phin = require("phin"); const util = require("util"); // internal libraries +const libData = require("@clusterio/lib/data"); const libFileOps = require("@clusterio/lib/file_ops"); const libFactorio = require("@clusterio/lib/factorio"); const libLink = require("@clusterio/lib/link"); @@ -94,71 +95,6 @@ const serverSettingsActions = { }, }; -/** - * Create and update symlinks for shared mods in an instance - * - * Creates symlinks for .zip and .dat files that are not present in the - * instance mods directory but is present in the sharedMods directory, - * and removes any symlinks that don't point to a file in the instance - * mods directory. If the instance mods directory doesn't exist it will - * be created. - * - * Note that on Windows this creates hard links instead of symbolic - * links as the latter requires elevated privileges. This unfortunately - * means the removal of mods from the shared mods dir can't be detected. - * - * @param {Instance} instance - Instance to link mods for - * @param {string} sharedMods - Path to folder to link mods from. - * @private - */ -async function symlinkMods(instance, sharedMods) { - await fs.ensureDir(instance.path("mods")); - - // Remove broken symlinks in instance mods. - for (let entry of await fs.readdir(instance.path("mods"), { withFileTypes: true })) { - if (entry.isSymbolicLink()) { - if (!await fs.pathExists(instance.path("mods", entry.name))) { - instance.logger.verbose(`Removing broken symlink ${entry.name}`); - await fs.unlink(instance.path("mods", entry.name)); - } - } - } - - // Link entries that are in sharedMods but not in instance mods. - let instanceModsEntries = new Set(await fs.readdir(instance.path("mods"))); - for (let entry of await fs.readdir(sharedMods, { withFileTypes: true })) { - if (entry.isFile()) { - if ([".zip", ".dat", ".json"].includes(path.extname(entry.name))) { - if (!instanceModsEntries.has(entry.name)) { - instance.logger.verbose(`linking ${entry.name} from ${sharedMods}`); - let target = path.join(sharedMods, entry.name); - let link = instance.path("mods", entry.name); - - /* eslint-disable max-depth */ - if (process.platform !== "win32") { - await fs.symlink(path.relative(path.dirname(link), target), link); - - // On Windows symlinks require elevated privileges, which is - // not something we want to have. For this reason the mods - // are hard linked instead. This has the drawback of not - // being able to identify when mods are removed from the - // sharedMods directory, or which mods are linked. - } else { - await fs.link(target, link); - } - /* eslint-enable max-depth */ - } - - } else { - instance.logger.warn(`Warning: ignoring file '${entry.name}' in sharedMods`); - } - - } else { - instance.logger.warn(`Warning: ignoring non-file '${entry.name}' in sharedMods`); - } - } -} - /** * Keeps track of the runtime parameters of an instance @@ -643,6 +579,77 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await fs.ensureDir(path.join(instanceDir, "saves")); } + /** + * Sync instance mods directory with configured mod pack + * + * Adds, deletes, and updates all files in the instance mods folder to + * match up with the mod pack that's configured. If no mod pack is + * configured then an empty mod pack is used, esentially turning it into + * a vanilla mods folder. If the instance mods directory does't exist + * then it will be created. + * + * On Linux this creates symlinks to mods in the slave's mods folder, on + * Windows hard links are used instead due to symlinks being privileged. + */ + async syncMods() { + const modPackId = this.config.get("factorio.mod_pack"); + let modPack; + if (modPackId === null) { + modPack = new libData.ModPack(); + } else { + const response = await libLink.messages.getModPack.send(this, { id: modPackId }); + modPack = new libData.ModPack(response.mod_pack); + } + + // TODO validate factorioVersion + + if (!this._slave.config.get("slave.mods_directory_is_shared")) { + throw new Error("Fetching mods is not implemented"); + } + + await fs.ensureDir(this.path("mods")); + + // Remove all files + for (let entry of await fs.readdir(this.path("mods"), { withFileTypes: true })) { + if (entry.isDirectory()) { + this.logger.warn( + `Found unexpected directory ${entry.name} in mods folder, it may break Clusterio's mod syncing` + ); + continue; + } + + if (entry.isFile() || entry.isSymbolicLink()) { + await fs.unlink(this.path("mods", entry.name)); + } + } + + // Add mods from mod the pack + const modsDir = this._slave.config.get("slave.mods_directory"); + for (let mod of modPack.mods.values()) { + const modFile = `${mod.name}_${mod.version}.zip`; + const target = path.join(modsDir, modFile); + const link = this.path("mods", modFile); + + if (process.platform !== "win32") { + await fs.symlink(path.relative(path.dirname(link), target), link); + + // On Windows symlinks require elevated privileges, which is + // not something we want to have. For this reason the mods + // are hard linked instead. + } else { + await fs.link(target, link); + } + } + + // Write mod-list.json + await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({ + mods: [{ name: "base" }, ...modPack.mods.values()].map(m => ({ name: m.name, enabled: true })), + }, null, 2)); + + // Write mod-settings.dat + await fs.outputFile(this.path("mods", "mod-settings.dat"), modPack.toModSettingsDat()); + } + /** * Prepare instance for starting * @@ -678,8 +685,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); ); } - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } /** @@ -1030,8 +1036,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.verbose("Creating save ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } catch (err) { this.notifyExit(); @@ -1053,8 +1058,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.info("Exporting data ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); let zip = await libFactorio.exportData(this.server); let content = await zip.generateAsync({ type: "nodebuffer" }); @@ -1116,6 +1120,3 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); } module.exports = Instance; - -// For testing only -module.exports._symlinkMods = symlinkMods; diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 115560a3..dfce060d 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -368,9 +368,11 @@ function SettingsTable(props) { {controls("startup", pickValue(props.modPack.settings["startup"]))} Map + These only apply for new saves. {controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))} Per Player + Default settings for new players, these only apply for new saves. {controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))} ; } diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index e5da8157..6ab91fa9 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -265,6 +265,7 @@ describe("Integration of Clusterio", function() { ); await execCtl(`instance config set-prop test factorio.settings visibility "'${value}'"`); await execCtl("instance config set-prop test factorio.settings require_user_verification false"); + await execCtl("instance config set test factorio.mod_pack 12"); }); }); diff --git a/test/integration/index.js b/test/integration/index.js index 2830e39f..b604bb1f 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -8,6 +8,7 @@ const phin = require("phin"); const util = require("util"); const events = require("events"); +const libData = require("@clusterio/lib/data"); const libLink = require("@clusterio/lib/link"); const { LineSplitter } = require("@clusterio/lib/stream"); const { ConsoleTransport, logger } = require("@clusterio/lib/logging"); @@ -220,6 +221,14 @@ before(async function() { controlConnector.token = controlToken; control = new TestControl(controlConnector); await controlConnector.connect(); + + const testPack = new libData.ModPack(); + testPack.id = 12; + testPack.name = "subspace_storage-pack"; + testPack.factorioVersion = "1.1.0"; + testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); + testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); + libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); }); after(async function() { diff --git a/test/slave.js b/test/slave.js index 0cb23498..d8d34788 100644 --- a/test/slave.js +++ b/test/slave.js @@ -1,11 +1,8 @@ "use strict"; const assert = require("assert").strict; const path = require("path"); -const fs = require("fs-extra"); -const libLink = require("@clusterio/lib/link"); const libConfig = require("@clusterio/lib/config"); -const Instance = require("@clusterio/slave/src/Instance"); const Slave = require("@clusterio/slave/src/Slave"); describe("Slave testing", function() { @@ -13,70 +10,6 @@ describe("Slave testing", function() { libConfig.InstanceConfig.finalize(); }); - describe("symlinkMods()", function() { - let testDir = path.join("temp", "test", "symlink"); - - let instance; - before(async function() { - // Remove previous test output - await fs.remove(testDir); - - // Add some test mods - await fs.outputFile(path.join(testDir, "shared", "mod_a.zip"), "a"); - await fs.outputFile(path.join(testDir, "shared", "mod_b.zip"), "b"); - await fs.outputFile(path.join(testDir, "shared", "mod.dat"), "c"); - await fs.outputFile(path.join(testDir, "shared", "mod-list.json"), "d"); - - let instanceConfig = new libConfig.InstanceConfig("slave"); - await instanceConfig.init(); - instanceConfig.set("instance.name", "test"); - instance = new Instance( - {}, new libLink.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig - ); - await fs.outputFile(instance.path("mods", "mod_i.zip"), "i"); - }); - - it("should link mods and data files", async function() { - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert.equal(await fs.readFile(instance.path("mods", "mod_a.zip"), "utf-8"), "a"); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - - it("should ignore directories", async function() { - await fs.ensureDir(path.join(testDir, "shared", "dir")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "dir"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should ignore files", async function() { - await fs.outputFile(path.join(testDir, "shared", "file"), "f"); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "file"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should unlink removed mods", async function() { - // This does not work on Windows - if (process.platform === "win32") { - this.skip(); - } - - await fs.unlink(path.join(testDir, "shared", "mod_a.zip")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - await assert.rejects(fs.lstat(instance.path("mods", "mod_a.zip")), { code: "ENOENT" }); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - }); - describe("discoverInstances()", function() { it("should discover test instance", async function() { let instancePath = path.join("test", "file", "instances");