Implement mod pack loading for shared mods

Wipe and reconstruct the instances' mods/ folder using the configured
ModPack for the case were the mods folder is shared with the master
server.  This assumes the master server will check the mod checksums
before sending the start request.

.dockerignore

@@ -13,8 +13,7 @@ factorio
 factorio.tar.gz
 
 # We build internal mods during build, mods from local test env should not be applied
-sharedMods
-sharedModsDisabled
+mods
 
 # These files should never be required during the build
 .git

.github/workflows/node.js.yml

@@ -26,8 +26,8 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - run: pnpm i --no-optional --no-frozen-lockfile
     - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
-    - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods
-    - run: wget -q -O temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip
+    - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/mods
+    - run: wget -q -O temp/test/mods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip
     - name: Run tests
       if: ${{ matrix.node-version != '12.x' }}
       run: pnpm test

.gitignore

@@ -32,7 +32,6 @@ node_modules
 Factorio*
 /factorio
 /instances
-/sharedMods
 /mods
 /database
 /plugin-list.json

Dockerfile

@@ -22,15 +22,15 @@ RUN pnpm install
 #RUN pnpm install @clusterio/plugin-subspace_storage
 #RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
 
-COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/sharedMods/
+COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/mods/
 
 # Build Lua library mod
 RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \
-&& mv dist/* sharedMods/ \
+&& mv dist/* mods/ \
 && mkdir temp \
 && mkdir temp/test \
-&& cp sharedMods/ temp/test/ -r \
-&& ls sharedMods
+&& cp mods/ temp/test/ -r \
+&& ls mods
 
 # Remove node_modules
 RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \;

packages/lib/config/definitions.js

@@ -160,6 +160,20 @@ SlaveGroup.define({
 	type: "string",
 	initial_value: "factorio",
 });
+SlaveGroup.define({
+	name: "mods_directory",
+	title: "Mods Directory",
+	description: "Path to directory where mods for instances are cached.",
+	type: "string",
+	initial_value: "mods",
+});
+SlaveGroup.define({
+	name: "mods_directory_is_shared",
+	title: "Mods Directory is Shared",
+	description: "True if the mods directory is shared with the master server.",
+	type: "boolean",
+	initial_value: true,
+});
 SlaveGroup.define({
 	name: "instances_directory",
 	description: "Path to directory to store instances in.",
@@ -280,6 +294,12 @@ FactorioGroup.define({
 	type: "number",
 	initial_value: 5,
 });
+FactorioGroup.define({
+	name: "mod_pack",
+	description: "Mod pack to use on this server",
+	type: "number",
+	optional: true,
+});
 FactorioGroup.define({
 	name: "enable_save_patching",
 	description: "Patch saves with Lua code. Required for Clusterio integrations, lua modules, and most plugins.",

packages/master/src/BaseConnection.js

@@ -84,6 +84,15 @@ class BaseConnection extends libLink.Link {
 	get connected() {
 		return !this._disconnecting && this.connector.connected;
 	}
+
+	async getModPackRequestHandler(message) {
+		let { id } = message.data;
+		let modPack = this._master.modPacks.get(id);
+		if (!modPack) {
+			throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`);
+		}
+		return { mod_pack: modPack.toJSON() };
+	}
 }
 
 module.exports = BaseConnection;

packages/master/src/ControlConnection.js

@@ -474,15 +474,6 @@ class ControlConnection extends BaseConnection {
 		return { save: result.save };
 	}
 
-	async getModPackRequestHandler(message) {
-		let { id } = message.data;
-		let modPack = this._master.modPacks.get(id);
-		if (!modPack) {
-			throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`);
-		}
-		return { mod_pack: modPack.toJSON() };
-	}
-
 	async listModPacksRequestHandler(message) {
 		return { list: [...this._master.modPacks.values()].map(pack => pack.toJSON()) };
 	}

packages/master/src/SlaveConnection.js

@@ -3,7 +3,6 @@ const libConfig = require("@clusterio/lib/config");
 const libLink = require("@clusterio/lib/link");
 const { logger } = require("@clusterio/lib/logging");
 const libPlugin = require("@clusterio/lib/plugin");
-const libErrors = require("@clusterio/lib/errors");
 const PlayerStats = require("@clusterio/lib/PlayerStats");
 
 const BaseConnection = require("./BaseConnection");
@@ -168,10 +167,6 @@ class SlaveConnection extends BaseConnection {
 		this._master.saveListUpdate(message.data);
 	}
 
-	async getModPackRequestHandler(message) {
-		throw new libErrors.RequestError("Not implemented");
-	}
-
 	async logMessageEventHandler(message) {
 		this._master.clusterLogger.log({
 			...message.data.info,

packages/slave/slave.js

@@ -177,7 +177,7 @@ async function startSlave() {
 	// If we get here the command was run
 
 	await fs.ensureDir(slaveConfig.get("slave.instances_directory"));
-	await fs.ensureDir("sharedMods");
+	await fs.ensureDir(slaveConfig.get("slave.mods_directory"));
 	await fs.ensureDir("modules");
 
 	// Set the process title, shows up as the title of the CMD window on windows

packages/slave/src/Instance.js

@@ -6,6 +6,7 @@ const phin = require("phin");
 const util = require("util");
 
 // internal libraries
+const libData = require("@clusterio/lib/data");
 const libFileOps = require("@clusterio/lib/file_ops");
 const libFactorio = require("@clusterio/lib/factorio");
 const libLink = require("@clusterio/lib/link");
@@ -94,71 +95,6 @@ const serverSettingsActions = {
 	},
 };
 
-/**
- * Create and update symlinks for shared mods in an instance
- *
- * Creates symlinks for .zip and .dat files that are not present in the
- * instance mods directory but is present in the sharedMods directory,
- * and removes any symlinks that don't point to a file in the instance
- * mods directory.  If the instance mods directory doesn't exist it will
- * be created.
- *
- * Note that on Windows this creates hard links instead of symbolic
- * links as the latter requires elevated privileges.  This unfortunately
- * means the removal of mods from the shared mods dir can't be detected.
- *
- * @param {Instance} instance - Instance to link mods for
- * @param {string} sharedMods - Path to folder to link mods from.
- * @private
- */
-async function symlinkMods(instance, sharedMods) {
-	await fs.ensureDir(instance.path("mods"));
-
-	// Remove broken symlinks in instance mods.
-	for (let entry of await fs.readdir(instance.path("mods"), { withFileTypes: true })) {
-		if (entry.isSymbolicLink()) {
-			if (!await fs.pathExists(instance.path("mods", entry.name))) {
-				instance.logger.verbose(`Removing broken symlink ${entry.name}`);
-				await fs.unlink(instance.path("mods", entry.name));
-			}
-		}
-	}
-
-	// Link entries that are in sharedMods but not in instance mods.
-	let instanceModsEntries = new Set(await fs.readdir(instance.path("mods")));
-	for (let entry of await fs.readdir(sharedMods, { withFileTypes: true })) {
-		if (entry.isFile()) {
-			if ([".zip", ".dat", ".json"].includes(path.extname(entry.name))) {
-				if (!instanceModsEntries.has(entry.name)) {
-					instance.logger.verbose(`linking ${entry.name} from ${sharedMods}`);
-					let target = path.join(sharedMods, entry.name);
-					let link = instance.path("mods", entry.name);
-
-					/* eslint-disable max-depth */
-					if (process.platform !== "win32") {
-						await fs.symlink(path.relative(path.dirname(link), target), link);
-
-					// On Windows symlinks require elevated privileges, which is
-					// not something we want to have.  For this reason the mods
-					// are hard linked instead.  This has the drawback of not
-					// being able to identify when mods are removed from the
-					// sharedMods directory, or which mods are linked.
-					} else {
-						await fs.link(target, link);
-					}
-					/* eslint-enable max-depth */
-				}
-
-			} else {
-				instance.logger.warn(`Warning: ignoring file '${entry.name}' in sharedMods`);
-			}
-
-		} else {
-			instance.logger.warn(`Warning: ignoring non-file '${entry.name}' in sharedMods`);
-		}
-	}
-}
-
 
 /**
  * Keeps track of the runtime parameters of an instance
@@ -643,6 +579,77 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 		await fs.ensureDir(path.join(instanceDir, "saves"));
 	}
 
+	/**
+	 * Sync instance mods directory with configured mod pack
+	 *
+	 * Adds, deletes, and updates all files in the instance mods folder to
+	 * match up with the mod pack that's configured. If no mod pack is
+	 * configured then an empty mod pack is used, esentially turning it into
+	 * a vanilla mods folder.  If the instance mods directory does't exist
+	 * then it will be created.
+	 *
+	 * On Linux this creates symlinks to mods in the slave's mods folder, on
+	 * Windows hard links are used instead due to symlinks being privileged.
+	 */
+	async syncMods() {
+		const modPackId = this.config.get("factorio.mod_pack");
+		let modPack;
+		if (modPackId === null) {
+			modPack = new libData.ModPack();
+		} else {
+			const response = await libLink.messages.getModPack.send(this, { id: modPackId });
+			modPack = new libData.ModPack(response.mod_pack);
+		}
+
+		// TODO validate factorioVersion
+
+		if (!this._slave.config.get("slave.mods_directory_is_shared")) {
+			throw new Error("Fetching mods is not implemented");
+		}
+
+		await fs.ensureDir(this.path("mods"));
+
+		// Remove all files
+		for (let entry of await fs.readdir(this.path("mods"), { withFileTypes: true })) {
+			if (entry.isDirectory()) {
+				this.logger.warn(
+					`Found unexpected directory ${entry.name} in mods folder, it may break Clusterio's mod syncing`
+				);
+				continue;
+			}
+
+			if (entry.isFile() || entry.isSymbolicLink()) {
+				await fs.unlink(this.path("mods", entry.name));
+			}
+		}
+
+		// Add mods from mod the pack
+		const modsDir = this._slave.config.get("slave.mods_directory");
+		for (let mod of modPack.mods.values()) {
+			const modFile = `${mod.name}_${mod.version}.zip`;
+			const target = path.join(modsDir, modFile);
+			const link = this.path("mods", modFile);
+
+			if (process.platform !== "win32") {
+				await fs.symlink(path.relative(path.dirname(link), target), link);
+
+			// On Windows symlinks require elevated privileges, which is
+			// not something we want to have.  For this reason the mods
+			// are hard linked instead.
+			} else {
+				await fs.link(target, link);
+			}
+		}
+
+		// Write mod-list.json
+		await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({
+			mods: [{ name: "base" }, ...modPack.mods.values()].map(m => ({ name: m.name, enabled: true })),
+		}, null, 2));
+
+		// Write mod-settings.dat
+		await fs.outputFile(this.path("mods", "mod-settings.dat"), modPack.toModSettingsDat());
+	}
+
 	/**
 	 * Prepare instance for starting
 	 *
@@ -678,8 +685,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			);
 		}
 
-		// eslint-disable-next-line no-use-before-define
-		await symlinkMods(this, "sharedMods");
+		await this.syncMods();
 	}
 
 	/**
@@ -1030,8 +1036,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			await this.writeServerSettings();
 
 			this.logger.verbose("Creating save .....");
-			// eslint-disable-next-line no-use-before-define
-			await symlinkMods(this, "sharedMods");
+			await this.syncMods();
 
 		} catch (err) {
 			this.notifyExit();
@@ -1053,8 +1058,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			await this.writeServerSettings();
 
 			this.logger.info("Exporting data .....");
-			// eslint-disable-next-line no-use-before-define
-			await symlinkMods(this, "sharedMods");
+			await this.syncMods();
 			let zip = await libFactorio.exportData(this.server);
 
 			let content = await zip.generateAsync({ type: "nodebuffer" });
@@ -1116,6 +1120,3 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 }
 
 module.exports = Instance;
-
-// For testing only
-module.exports._symlinkMods = symlinkMods;

packages/web_ui/src/components/ModPackViewPage.jsx

@@ -368,9 +368,11 @@ function SettingsTable(props) {
 		{controls("startup", pickValue(props.modPack.settings["startup"]))}
 
 		<Typography.Title level={5}>Map</Typography.Title>
+		<Typography.Paragraph>These only apply for new saves.</Typography.Paragraph>
 		{controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))}
 
 		<Typography.Title level={5}>Per Player</Typography.Title>
+		<Typography.Paragraph>Default settings for new players, these only apply for new saves.</Typography.Paragraph>
 		{controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))}
 	</>;
 }

test/integration/clusterio.js

@@ -265,6 +265,7 @@ describe("Integration of Clusterio", function() {
 				);
 				await execCtl(`instance config set-prop test factorio.settings visibility "'${value}'"`);
 				await execCtl("instance config set-prop test factorio.settings require_user_verification false");
+				await execCtl("instance config set test factorio.mod_pack 12");
 			});
 		});
 

test/integration/index.js

@@ -8,6 +8,7 @@ const phin = require("phin");
 const util = require("util");
 const events = require("events");
 
+const libData = require("@clusterio/lib/data");
 const libLink = require("@clusterio/lib/link");
 const { LineSplitter } = require("@clusterio/lib/stream");
 const { ConsoleTransport, logger } = require("@clusterio/lib/logging");
@@ -220,6 +221,14 @@ before(async function() {
 	controlConnector.token = controlToken;
 	control = new TestControl(controlConnector);
 	await controlConnector.connect();
+
+	const testPack = new libData.ModPack();
+	testPack.id = 12;
+	testPack.name = "subspace_storage-pack";
+	testPack.factorioVersion = "1.1.0";
+	testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" });
+	testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" });
+	libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() });
 });
 
 after(async function() {

test/slave.js

@@ -1,11 +1,8 @@
 "use strict";
 const assert = require("assert").strict;
 const path = require("path");
-const fs = require("fs-extra");
 
-const libLink = require("@clusterio/lib/link");
 const libConfig = require("@clusterio/lib/config");
-const Instance = require("@clusterio/slave/src/Instance");
 const Slave = require("@clusterio/slave/src/Slave");
 
 describe("Slave testing", function() {
@@ -13,70 +10,6 @@ describe("Slave testing", function() {
 		libConfig.InstanceConfig.finalize();
 	});
 
-	describe("symlinkMods()", function() {
-		let testDir = path.join("temp", "test", "symlink");
-
-		let instance;
-		before(async function() {
-			// Remove previous test output
-			await fs.remove(testDir);
-
-			// Add some test mods
-			await fs.outputFile(path.join(testDir, "shared", "mod_a.zip"), "a");
-			await fs.outputFile(path.join(testDir, "shared", "mod_b.zip"), "b");
-			await fs.outputFile(path.join(testDir, "shared", "mod.dat"), "c");
-			await fs.outputFile(path.join(testDir, "shared", "mod-list.json"), "d");
-
-			let instanceConfig = new libConfig.InstanceConfig("slave");
-			await instanceConfig.init();
-			instanceConfig.set("instance.name", "test");
-			instance = new Instance(
-				{}, new libLink.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig
-			);
-			await fs.outputFile(instance.path("mods", "mod_i.zip"), "i");
-		});
-
-		it("should link mods and data files", async function() {
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			assert.equal(await fs.readFile(instance.path("mods", "mod_a.zip"), "utf-8"), "a");
-			assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b");
-			assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c");
-			assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d");
-			assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i");
-		});
-
-		it("should ignore directories", async function() {
-			await fs.ensureDir(path.join(testDir, "shared", "dir"));
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			assert(!await fs.exists(instance.path("mods", "dir"), "utf-8"), "dir was unxpectedly linked");
-		});
-
-		it("should ignore files", async function() {
-			await fs.outputFile(path.join(testDir, "shared", "file"), "f");
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			assert(!await fs.exists(instance.path("mods", "file"), "utf-8"), "dir was unxpectedly linked");
-		});
-
-		it("should unlink removed mods", async function() {
-			// This does not work on Windows
-			if (process.platform === "win32") {
-				this.skip();
-			}
-
-			await fs.unlink(path.join(testDir, "shared", "mod_a.zip"));
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			await assert.rejects(fs.lstat(instance.path("mods", "mod_a.zip")), { code: "ENOENT" });
-			assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b");
-			assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c");
-			assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d");
-			assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i");
-		});
-	});
-
 	describe("discoverInstances()", function() {
 		it("should discover test instance", async function() {
 			let instancePath = path.join("test", "file", "instances");