Merge pull request #478 from Hornwitser/mod-management

Add mod pack oriented mod management

.dockerignore

@@ -13,8 +13,7 @@ factorio
 factorio.tar.gz
 
 # We build internal mods during build, mods from local test env should not be applied
-sharedMods
-sharedModsDisabled
+mods
 
 # These files should never be required during the build
 .git

.github/workflows/node.js.yml

@@ -26,8 +26,8 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - run: pnpm i --no-optional --no-frozen-lockfile
     - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
-    - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods
-    - run: wget -q -O temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip
+    - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/mods
+    - run: wget -q -O temp/test/mods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip
     - name: Run tests
       if: ${{ matrix.node-version != '12.x' }}
       run: pnpm test

.gitignore

@@ -32,7 +32,7 @@ node_modules
 Factorio*
 /factorio
 /instances
-/sharedMods
+/mods
 /database
 /plugin-list.json
 /config-master.json

Dockerfile

@@ -22,15 +22,15 @@ RUN pnpm install
 #RUN pnpm install @clusterio/plugin-subspace_storage
 #RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
 
-COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/sharedMods/
+COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/mods/
 
 # Build Lua library mod
 RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \
-&& mv dist/* sharedMods/ \
+&& mv dist/* mods/ \
 && mkdir temp \
 && mkdir temp/test \
-&& cp sharedMods/ temp/test/ -r \
-&& ls sharedMods
+&& cp mods/ temp/test/ -r \
+&& ls mods
 
 # Remove node_modules
 RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \;

packages/ctl/ctl.js

@@ -89,6 +89,10 @@ class Control extends libLink.Link {
 
 	async saveListUpdateEventHandler() { }
 
+	async modPackUpdateEventHandler() { }
+
+	async modUpdateEventHandler() { }
+
 	async userUpdateEventHandler() { }
 
 	async setLogSubscriptions({

packages/ctl/src/commands.js

@@ -14,6 +14,7 @@ const libLink = require("@clusterio/lib/link");
 const libErrors = require("@clusterio/lib/errors");
 const libConfig = require("@clusterio/lib/config");
 const libCommand = require("@clusterio/lib/command");
+const libData = require("@clusterio/lib/data");
 const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
 const libHelpers = require("@clusterio/lib/helpers");
@@ -860,6 +861,465 @@ instanceCommands.add(new libCommand.Command({
 	},
 }));
 
+const modPackCommands = new libCommand.CommandTree({ name: "mod-pack", description: "Mod Pack" });
+modPackCommands.add(new libCommand.Command({
+	definition: ["show <mod-pack>", "Show details of mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to show", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let mods = new Map(
+			(await libLink.messages.listMods.send(control)).list.map(m => [`${m.name}_${m.version}`, m])
+		);
+		let response = await libLink.messages.getModPack.send(control, {
+			id: await libCommand.resolveModPack(control, args.modPack),
+		});
+
+		for (let [field, value] of Object.entries(response.mod_pack)) {
+			if (field === "mods") {
+				print(`${field}:`);
+				for (let entry of value) {
+					const mod = mods.get(`${entry.name}_${entry.version}`);
+					const missing = !mod && "? ";
+					const badChecksum = mod && entry.sha1 && entry.sha1 !== mod.sha1 && "! ";
+					const warn = missing || badChecksum || "";
+					const enabled = entry.enabled ? "" : "(disabled) ";
+					print(`  ${warn}${enabled}${entry.name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`);
+				}
+			} else if (field === "settings") {
+				print(`${field}:`);
+				for (let [scope, settings] of Object.entries(value)) {
+					print(`  ${scope}:`);
+					for (let [setting, settingValue] of Object.entries(settings)) {
+						print(`    ${setting}: ${JSON.stringify(settingValue.value)}`);
+					}
+				}
+			} else if (field === "export_manifest") {
+				print(`${field}:`);
+				print("  assets:");
+				for (let [name, fileName] of Object.entries(value.assets)) {
+					print(`    ${name}: ${fileName}`);
+				}
+
+			} else {
+				print(`${field}: ${value}`);
+			}
+		}
+	},
+}));
+
+modPackCommands.add(new libCommand.Command({
+	definition: [["list", "l"], "List mod packs in the cluster"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listModPacks.send(control);
+		let fields = ["id", "name", "factorio_version"];
+		for (let entry of response.list) {
+			for (let field of Object.keys(entry)) {
+				if (!fields.includes(field)) {
+					delete entry[field];
+				}
+			}
+		}
+		print(asTable(response.list));
+	},
+}));
+
+function setModPackSettings(modPack, args) {
+	function doSettings(settings, cast) {
+		for (let i = 0; i + 2 < settings.length; i += 3) {
+			const [scope, name, value] = settings.slice(i, i + 3);
+			if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
+				throw new libErrors.CommandError(
+					`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
+				);
+			}
+			modPack.settings[scope].set(name, { value: cast(value) });
+		}
+	}
+
+	doSettings(args.boolSetting || [], value => {
+		if (!["true", "false"].includes(value)) {
+			throw new libErrors.CommandError(`boolean value must be one of true or false, not ${value}`);
+		}
+		return value === "true";
+	});
+	doSettings(args.intSetting || [], value => {
+		// eslint-disable-next-line radix
+		let number = Number.parseInt(value);
+		if (isNaN(number)) {
+			throw new libErrors.CommandError(`int value must be an integer, not ${value}`);
+		}
+		return number;
+	});
+	doSettings(args.doubleSetting || [], value => {
+		let number = Number.parseFloat(value);
+		if (isNaN(number)) {
+			throw new libErrors.CommandError(`double value must be a number, not ${value}`);
+		}
+		return number;
+	});
+	doSettings(args.stringSetting || [], value => value);
+}
+
+function setModPackMods(modPack, mods) {
+	for (let mod of mods || []) {
+		const [name, version, sha1] = mod.split(":");
+		if (!version) {
+			throw new libErrors.CommandError("Added mod must be formatted as name:version or name:version:sha1");
+		}
+		if (!/^\d+\.\d+\.\d+$/.test(version)) {
+			throw new libErrors.CommandError("version must match the format digit.digit.digit");
+		}
+		if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) {
+			throw new libErrors.CommandError("sha1 must be a 40 digit lower case hex string");
+		}
+		modPack.mods.set(name, { name, enabled: true, version, sha1 });
+	}
+}
+
+function setModPackModsEnabled(modPack, mods, enabled) {
+	for (let mod of mods || []) {
+		if (!modPack.mods.has(mod)) {
+			throw new libErrors.CommandError(`Mod named ${mod} does not exist in the mod pack`);
+		}
+		modPack.mods.get(mod).enabled = enabled;
+	}
+}
+
+modPackCommands.add(new libCommand.Command({
+	definition: ["create <name> <factorio-version>", "Create mod pack", (yargs) => {
+		yargs.positional("name", { describe: "Name of mod pack to create", type: "string" });
+		yargs.positional("factorio-version", { describe: "Version of factorio the mod pack is for", type: "string" });
+		yargs.options({
+			"description": { describe: "Description for mod pack", type: "string" },
+			"mods": { describe: "Mods in the form of name:version[:sha1]", array: true, type: "string" },
+			"disabled-mods": { describe: "Mods that are in the pack but not enabled", array: true, type: "string" },
+			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
+			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
+			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
+			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
+		});
+	}],
+	handler: async function(args, control) {
+		const modPack = new libData.ModPack();
+		modPack.name = args.name;
+		if (args.description) { modPack.description = args.description; }
+		if (args.factorioVersion) {
+			if (!/^\d+\.\d+\.\d+?$/.test(args.factorioVersion)) {
+				throw new libErrors.CommandError("factorio-version must match the format digit.digit.digit");
+			}
+			modPack.factorioVersion = args.factorioVersion;
+		}
+		setModPackMods(modPack, args.mods);
+		setModPackModsEnabled(modPack, args.disabledMods, false);
+		setModPackSettings(modPack, args);
+		await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() });
+		print(`Created mod pack ${modPack.name} (${modPack.id})`);
+	},
+}));
+
+modPackCommands.add(new libCommand.Command({
+	definition: ["import <string>", "Import mod pack string", (yargs) => {
+		yargs.positional("string", { describe: "Mod pack string to import", type: "string" });
+	}],
+	handler: async function(args, control) {
+		const modPack = libData.ModPack.fromModPackString(args.string);
+		await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() });
+		print(`Created mod pack ${modPack.name} (${modPack.id})`);
+	},
+}));
+
+modPackCommands.add(new libCommand.Command({
+	definition: ["export <mod-pack>", "Export mod pack string", (yargs) => {
+		yargs.positional("string", { describe: "Mod pack to export", type: "string" });
+	}],
+	handler: async function(args, control) {
+		const response = await libLink.messages.getModPack.send(control, {
+			id: await libCommand.resolveModPack(control, args.modPack),
+		});
+		const modPack = new libData.ModPack(response.mod_pack);
+		print(modPack.toModPackString());
+	},
+}));
+
+modPackCommands.add(new libCommand.Command({
+	definition: ["edit <mod-pack>", "Edit mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" });
+		yargs.options({
+			"name": { describe: "New name for mod pack", type: "string" },
+			"description": { describe: "New description for mod pack", type: "string" },
+			"factorio-version": { describe: "Set version of factorio the mod pack is for", type: "string" },
+			"add-mods": { describe: "Mods in the form of name:version[:sha1] to add", array: true, type: "string" },
+			"enable-mods": { describe: "Mods to set as enabled", array: true, type: "string" },
+			"disable-mods": { describe: "Mods to set as disabled", array: true, type: "string" },
+			"remove-mods": { describe: "Name of mods to remove", array: true, type: "string" },
+			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
+			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
+			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
+			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
+			"remove-setting": { describe: "Remove a setting", array: true, nargs: 2, type: "string" },
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.getModPack.send(control, {
+			id: await libCommand.resolveModPack(control, args.modPack),
+		});
+		let modPack = new libData.ModPack(response.mod_pack);
+
+		if (args.name) { modPack.name = args.name; }
+		if (args.description) { modPack.description = args.description; }
+		if (args.factorioVersion) {
+			if (!/^\d+\.\d+\.\d+?$/.test(args.factorioVersion)) {
+				throw new libErrors.CommandError("factorio-version must match the format digit.digit.digit");
+			}
+			modPack.factorioVersion = args.factorioVersion;
+		}
+
+		setModPackMods(modPack, args.addMods);
+		for (let name of args.removeMods || []) {
+			if (modPack.mods.has(name)) {
+				modPack.mods.delete(name);
+			} else {
+				logger.warn(`Mod ${name} did not exist on ${modPack.name}`);
+			}
+		}
+		setModPackModsEnabled(modPack, args.disableMods, false);
+		setModPackModsEnabled(modPack, args.enableMods, true);
+
+		setModPackSettings(modPack, args);
+		if (args.removeSetting) {
+			for (let i = 0; i + 1 < args.removeSetting.length; i += 2) {
+				const [scope, name] = settings.slice(i, i + 2);
+				if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
+					throw new libErrors.CommandError(
+						`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
+					);
+				}
+				if (modPack.settings[scope].has(name)) {
+					modPack.settings[scope].delete(name);
+				} else {
+					logger.warn(`Mod setting ${scope} ${name} did not exist on ${modPack.name}`);
+				}
+			}
+		}
+		await libLink.messages.updateModPack.send(control, { mod_pack: modPack.toJSON() });
+	},
+}));
+
+modPackCommands.add(new libCommand.Command({
+	definition: ["delete <mod-pack>", "Delete mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to delete", type: "string" });
+	}],
+	handler: async function(args, control) {
+		const id = await libCommand.resolveModPack(control, args.modPack);
+		await libLink.messages.deleteModPack.send(control, { id });
+	},
+}));
+
+const modCommands = new libCommand.CommandTree({ name: "mod", description: "Manage uploaded mods" });
+modCommands.add(new libCommand.Command({
+	definition: ["show <name> <mod-version>", "Show details for a mod stored in the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Mod name to show details for", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of the mod", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.getMod.send(control, { name: args.name, version: args.modVersion });
+		for (let [field, value] of Object.entries(response.mod)) {
+			if (value instanceof Array) {
+				print(`${field}:`);
+				for (let entry of value) {
+					print(`  ${entry}`);
+				}
+			} else {
+				print(`${field}: ${value}`);
+			}
+		}
+	},
+}));
+
+modCommands.add(new libCommand.Command({
+	definition: [["list", "l"], "List mods stored in the cluster", (yargs) => {
+		yargs.options({
+			"fields": {
+				describe: "Fields to show, supports 'all'.",
+				array: true,
+				type: "string",
+				default: ["name", "version", "title", "factorio_version"],
+			},
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listMods.send(control);
+		if (!args.fields.includes("all")) {
+			for (let entry of response.list) {
+				for (let field of Object.keys(entry)) {
+					if (!args.fields.includes(field)) {
+						delete entry[field];
+					}
+				}
+			}
+		}
+		print(asTable(response.list));
+	},
+}));
+
+modCommands.add(new libCommand.Command({
+	definition: ["search <factorio-version> [query]", "Search mods stored in the cluster", (yargs) => {
+		yargs.positional("factorio-version", { describe: "Major version of Factorio to search for", type: "string" });
+		yargs.positional("query", { describe: "Search query", type: "string", default: "" });
+		yargs.options({
+			"page": {
+				describe: "Result page to show",
+				type: "number",
+				default: 1,
+			},
+			"page-size": {
+				describe: "Results per page to show",
+				type: "number",
+				default: 10,
+			},
+			"sort": {
+				describe: "sort results by given field",
+				type: "string",
+			},
+			"sort-order": {
+				describe: "order to sort results in (asc/desc)",
+				type: "string",
+				default: "asc",
+			},
+			"fields": {
+				describe: "Fields to show, supports 'all'.",
+				array: true,
+				type: "string",
+				default: ["name", "version", "title", "factorio_version"],
+			},
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.searchMods.send(control, {
+			"query": args.query,
+			"factorio_version": args.factorioVersion,
+			"page_size": args.pageSize,
+			"page": args.page,
+			"sort": args.sort,
+			"sort_order": args.sortOrder,
+		});
+		let results = response.results.flatMap(result => result.versions);
+		if (!args.fields.includes("all")) {
+			for (let entry of results) {
+				for (let field of Object.keys(entry)) {
+					if (!args.fields.includes(field)) {
+						delete entry[field];
+					}
+				}
+			}
+		}
+		for (let issue of response.query_issues) {
+			print(issue);
+		}
+		print(`page ${args.page} of ${response.page_count} (${response.result_count} results)`);
+		print(asTable(results));
+	},
+}));
+
+modCommands.add(new libCommand.Command({
+	definition: ["upload <file>", "Upload mod to the cluster", (yargs) => {
+		yargs.positional("file", { describe: "File to upload", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let filename = path.basename(args.file);
+		if (!filename.endsWith(".zip")) {
+			throw new libErrors.CommandError("Mod filename must end with .zip");
+		}
+		// phin doesn't support streaming requests :(
+		let content = await fs.readFile(args.file);
+
+		let url = new URL(control.config.get("control.master_url"));
+		url.pathname += "api/upload-mod";
+		url.searchParams.append("filename", filename);
+
+		let result = await phin({
+			url, method: "POST",
+			headers: {
+				"X-Access-Token": control.config.get("control.master_token"),
+				"Content-Type": "application/zip",
+			},
+			core: { ca: control.tlsCa },
+			data: content,
+			parse: "json",
+		});
+
+		for (let error of result.body.errors || []) {
+			logger.error(error);
+		}
+
+		for (let requestError of result.body.request_errors || []) {
+			logger.error(requestError);
+		}
+
+		if (result.body.mods && result.body.mods.length) {
+			const mod = new libData.ModInfo(result.body.mods[0]);
+			logger.info(`Successfully uploaded ${mod.filename}`);
+		}
+
+		if ((result.body.errors || []).length || (result.body.request_errors || []).length) {
+			throw new libErrors.CommandError("Uploading mod failed");
+		}
+	},
+}));
+
+modCommands.add(new libCommand.Command({
+	definition: ["download <name> <mod-version>", "Download a mod from the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Internal name of mod to download", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of mod to download", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let result = await libLink.messages.downloadMod.send(control, {
+			name: args.name,
+			version: args.modVersion,
+		});
+
+		let url = new URL(control.config.get("control.master_url"));
+		url.pathname += `api/stream/${result.stream_id}`;
+		let response = await phin({
+			url, method: "GET",
+			core: { ca: control.tlsCa },
+			stream: true,
+		});
+
+		let writeStream;
+		let filename = `${args.name}_${args.modVersion}.zip`;
+		let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
+		while (true) {
+			try {
+				writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
+				await events.once(writeStream, "open");
+				break;
+			} catch (err) {
+				if (err.code === "EEXIST") {
+					tempFilename = await libFileOps.findUnusedName(".", tempFilename, ".tmp.zip");
+				} else {
+					throw err;
+				}
+			}
+		}
+		response.pipe(writeStream);
+		await finished(writeStream);
+		await fs.rename(tempFilename, filename);
+
+		logger.info(`Downloaded ${filename}`);
+	},
+}));
+
+modCommands.add(new libCommand.Command({
+	definition: ["delete <name> <mod-version>", "Delete a mod stored in the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Name of mod to delete", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of mod to delete", type: "string" });
+	}],
+	handler: async function(args, control) {
+		await libLink.messages.deleteMod.send(control, { name: args.name, version: args.modVersion });
+	},
+}));
+
 const permissionCommands = new libCommand.CommandTree({ name: "permission", description: "Permission inspection" });
 permissionCommands.add(new libCommand.Command({
 	definition: [["list", "l"], "List permissions in the cluster"],
@@ -1194,6 +1654,8 @@ async function registerCommands(controlPlugins, yargs) {
 	rootCommands.add(masterCommands);
 	rootCommands.add(slaveCommands);
 	rootCommands.add(instanceCommands);
+	rootCommands.add(modPackCommands);
+	rootCommands.add(modCommands);
 	rootCommands.add(permissionCommands);
 	rootCommands.add(roleCommands);
 	rootCommands.add(userCommands);

packages/lib/browser.js

@@ -1,4 +1,5 @@
 export { default as libConfig } from "./config";
+export { default as libData } from "./data";
 export { default as libErrors } from "./errors";
 export { default as libFactorio } from "./factorio/exchange_string";
 export { default as libHelpers } from "./helpers";

packages/lib/command.js

@@ -198,6 +198,39 @@ async function resolveInstance(client, instanceName) {
 	return instanceId;
 }
 
+/**
+ * Resolve a string to a mod pack ID
+ *
+ * Resolevs a string with either a mod pack name or an id into an integer
+ * with the mod pack ID.
+ *
+ * @param {module:lib/link.Link} client -
+ *     link to master server to query mod pack on.
+ * @param {string} modPackName - string with name or id of mod pack.
+ * @returns {Promise<number>} mod pack ID.
+ * @static
+ */
+async function resolveModPack(client, modPackName) {
+	let modPackId;
+	if (/^-?\d+/.test(modPackName)) {
+		modPackId = parseInt(modPackName, 10);
+	} else {
+		let response = await libLink.messages.listModPacks.send(client);
+		for (let mod_pack of response.list) {
+			if (mod_pack["name"] === modPackName) {
+				modPackId = mod_pack["id"];
+				break;
+			}
+		}
+
+		if (modPackId === undefined) {
+			throw new libErrors.CommandError(`No mod pack named ${modPackName}`);
+		}
+	}
+
+	return modPackId;
+}
+
 /**
  * Retrieve role object from string
  *
@@ -244,5 +277,6 @@ module.exports = {
 
 	resolveSlave,
 	resolveInstance,
+	resolveModPack,
 	retrieveRole,
 };

packages/lib/config/definitions.js

@@ -19,6 +19,13 @@ MasterGroup.define({
 	type: "string",
 	initial_value: "Your Cluster",
 });
+MasterGroup.define({
+	name: "mods_directory",
+	title: "Mods Directory",
+	description: "Path to directory where mods shared with the cluster are stored.",
+	type: "string",
+	initial_value: "mods",
+});
 MasterGroup.define({
 	name: "database_directory",
 	title: "Database directory",
@@ -108,6 +115,13 @@ MasterGroup.define({
 	type: "number",
 	initial_value: 15,
 });
+MasterGroup.define({
+	name: "default_mod_pack_id",
+	title: "Default Mod Pack",
+	description: "Mod pack used by default for instances.",
+	type: "number",
+	optional: true,
+});
 MasterGroup.define({
 	name: "default_role_id",
 	title: "Default role",
@@ -153,6 +167,20 @@ SlaveGroup.define({
 	type: "string",
 	initial_value: "factorio",
 });
+SlaveGroup.define({
+	name: "mods_directory",
+	title: "Mods Directory",
+	description: "Path to directory where mods for instances are cached.",
+	type: "string",
+	initial_value: "mods",
+});
+SlaveGroup.define({
+	name: "mods_directory_is_shared",
+	title: "Mods Directory is Shared",
+	description: "True if the mods directory is shared with the master server.",
+	type: "boolean",
+	initial_value: true,
+});
 SlaveGroup.define({
 	name: "instances_directory",
 	description: "Path to directory to store instances in.",
@@ -273,6 +301,12 @@ FactorioGroup.define({
 	type: "number",
 	initial_value: 5,
 });
+FactorioGroup.define({
+	name: "mod_pack",
+	description: "Mod pack to use on this server",
+	type: "number",
+	optional: true,
+});
 FactorioGroup.define({
 	name: "enable_save_patching",
 	description: "Patch saves with Lua code. Required for Clusterio integrations, lua modules, and most plugins.",

packages/lib/data/ExportManifest.js

@@ -0,0 +1,46 @@
+"use strict";
+
+/**
+ * Export manifest for a mod pack
+ *
+ * Tracks files stored on disk for data that extracted from Factorio using a
+ * given mod pack.
+ * @alias module:lib/data.ExportManifest
+ */
+class ExportManifest {
+
+	/**
+	 * Mapping between known assets and their file names.
+	 *
+	 * Currently exported assets are
+	 * - settings: JSON of mod settings prototypes.
+	 * - prototypes: JSON of all game prototypes except settings.
+	 * - item-spritesheet: PNG containing icons from the game.
+	 * - item-metadata: JSON of Map entries describing icons in item-spritesheet
+	 * - locale: JSON of flattened Map entries of en locale strings from the game.
+	 * @type {Object<string, string>}
+	 */
+	assets = {};
+
+	static jsonSchema = {
+		type: "object",
+		additionalProperties: false,
+		required: ["assets"],
+		properties: {
+			"assets": {
+				type: "object",
+				additionalProperties: { type: "string" },
+			},
+		},
+	};
+
+	constructor(json = {}) {
+		if (json.assets) { this.assets = json.assets; }
+	}
+
+	toJSON() {
+		return { assets: this.assets };
+	}
+}
+
+module.exports = ExportManifest;

packages/lib/data/ModInfo.js

@@ -0,0 +1,231 @@
+"use strict";
+const fs = require("fs-extra");
+const JSZip = require("jszip");
+const path = require("path");
+
+const libHash = require("../hash");
+const libSchema = require("../schema");
+const { findRoot } = require("../zip_ops");
+
+const { integerModVersion, integerFactorioVersion, modVersionRegExp } = require("./version");
+
+
+/**
+ * Info about a mod available on the master server.
+ *
+ * See https://wiki.factorio.com/Tutorial:Mod_structure#info.json for
+ * details of the invidiual fields sourced from info.json.
+ * @alias module:lib/data.ModInfo
+ */
+class ModInfo {
+	/**
+	 * Internal name of this mod.
+	 * This is the name of the zip file as well as the name that appears in
+	 * info.json.
+	 * @type {string}
+	 */
+	name = "";
+
+	/**
+	 * Version of the mod.
+	 * Sourced from info.json.
+	 * @type {string}
+	 */
+	version = "";
+
+	/**
+	 * Integer representation of the version
+	 * @type {number}
+	 */
+	get integerVersion() {
+		return integerModVersion(this.version);
+	}
+
+	/**
+	 * Display name of the mod.
+	 * Sourced from info.json.
+	 * @type {string}
+	 */
+	title = "";
+
+	/**
+	 * Author of the mod.
+	 * Sourced from info.json.
+	 * @type {string}
+	 */
+	author = "";
+
+	/**
+	 * Contact field for mod.
+	 * Sourced from info.json.
+	 * @type {string}
+	 */
+	contact = "";
+
+	/**
+	 * Homepage of the mod.
+	 * Sourced from info.json.
+	 * @type {string}
+	 */
+	homepage = "";
+
+	/**
+	 * Description of the mod.
+	 * Sourced from info.json.
+	 * @type {string}
+	 */
+	description = "";
+
+	/**
+	 * Major version of Factorio this mod supports.
+	 * Sourced from info.json.
+	 * @type {string}
+	 */
+	factorioVersion = "0.12";
+
+	/**
+	 * Integer representation of the factorioVersion
+	 * @type {number}
+	 */
+	get integerFactorioVersion() {
+		return integerFactorioVersion(this.factorioVersion);
+	}
+
+	/**
+	 * Dependiences for this mod.
+	 * Sourced from info.json.
+	 * @type {Array<string>}
+	 */
+	dependencies = ["base"];
+
+	/**
+	 * Expected name of zip file containing this mod.
+	 * @type {number}
+	 */
+	get filename() {
+		return `${this.name}_${this.version}.zip`;
+	}
+
+	/**
+	 * Size of the mod in bytes
+	 * @type {number}
+	 */
+	size = 0;
+
+	/**
+	 * SHA1 hash of this mod
+	 * @type {string=}
+	 */
+	sha1;
+
+	/**
+	 * True if this mod has been deleted
+	 * @type {boolean}
+	 */
+	isDeleted = false;
+
+	// Content of info.json found in mod files
+	static infoJsonSchema = {
+		type: "object",
+		required: ["name", "version", "title", "author"],
+		properties: {
+			"name": { type: "string" },
+			"version": { type: "string" },
+			"title": { type: "string" },
+			"author": { type: "string" },
+			"contact": { type: "string" },
+			"homepage": { type: "string" },
+			"description": { type: "string" },
+			"factorio_version": { type: "string" },
+			"dependencies": { type: "array", items: { type: "string" } },
+		},
+	};
+
+	static validateInfo = libSchema.compile(this.infoJsonSchema);
+
+	static jsonSchema = {
+		type: "object",
+		properties: {
+			...this.infoJsonSchema.properties,
+			"size": { type: "integer" },
+			"sha1": { type: "string" },
+			"is_deleted": { type: "boolean" },
+		},
+	};
+
+	static validate = libSchema.compile(this.jsonSchema);
+
+	constructor(json = {}) {
+		// info.json fields
+		if (json.name) { this.name = json.name; }
+		if (json.version) { this.version = json.version; }
+		if (json.title) { this.title = json.title; }
+		if (json.author) { this.author = json.author; }
+		if (json.contact) { this.contact = json.contact; }
+		if (json.homepage) { this.homepage = json.homepage; }
+		if (json.description) { this.description = json.description; }
+		if (json.factorio_version) { this.factorioVersion = json.factorio_version; }
+		if (json.dependencies) { this.dependencies = json.dependencies; }
+
+		// Additional data
+		if (json.size) { this.size = json.size; }
+		if (json.sha1) { this.sha1 = json.sha1; }
+		if (json.is_deleted) { this.isDeleted = json.is_deleted; }
+	}
+
+	toJSON() {
+		let json = {
+			name: this.name,
+			version: this.version,
+			title: this.title,
+			author: this.author,
+		};
+		if (this.contact) { json.contact = this.contact; }
+		if (this.homepage) { json.homepage = this.homepage; }
+		if (this.description) { json.description = this.description; }
+		if (this.factorio_version !== "0.12") { json.factorio_version = this.factorioVersion; }
+		if (this.dependencies.length !== 1 || this.dependencies[0] !== "base") {
+			json.dependencies = this.dependencies;
+		}
+		if (this.size) { json.size = this.size; }
+		if (this.sha1) { json.sha1 = this.sha1; }
+		if (this.isDeleted) { json.is_deleted = this.isDeleted; }
+		return json;
+	}
+
+	static async fromModFile(modPath) {
+		let modInfo;
+		{
+			// XXX: JSZip needs the whole archive loaded in memory to work.
+			// This is clearly untenable and will be replaced later.
+			let zip = await JSZip.loadAsync(await fs.readFile(modPath));
+			let root = zip.folder(findRoot(zip));
+
+			let infoFile = root.file("info.json");
+			if (!infoFile) {
+				throw new Error("Mod contains no info.json file");
+			}
+
+			modInfo = JSON.parse(await infoFile.async("string"));
+		}
+
+		let valid = this.validateInfo(modInfo);
+		if (!valid) {
+			throw new Error("Mod's info.json is not valid");
+		}
+
+		if (!modVersionRegExp.test(modInfo.version)) {
+			throw new Error(`Mod's version (${modInfo.version}) is invalid`);
+		}
+
+		return new this({
+			...modInfo,
+
+			size: (await fs.stat(modPath)).size,
+			sha1: await libHash.hashFile(modPath),
+			is_deleted: false,
+		});
+	}
+}
+
+module.exports = ModInfo;

packages/lib/data/ModPack.js

@@ -0,0 +1,355 @@
+"use strict";
+const zlib = require("zlib");
+
+const libSchema = require("../schema");
+
+const ExportManifest = require("./ExportManifest");
+const { integerFactorioVersion } = require("./version");
+
+
+/**
+ * A setting for a mod.
+ * @typedef {object} module:lib/data.ModPack~ModSetting
+ * @property {boolean|number|string} value - Value of the given mod setting.
+ */
+const ModSettingJsonSchema = {
+	type: "object",
+	additionalProperties: false,
+	required: ["value"],
+	properties: {
+		"value": { type: ["boolean", "number", "string"] },
+	},
+};
+
+const ModSettingsJsonSchema = {
+	type: "object",
+	additionalProperties: ModSettingJsonSchema,
+};
+
+/**
+ * A mod in a mod pack
+ * @typedef {object} module:lib/data.ModPack~ModRecord
+ * @property {string} name - name of the mod.
+ * @property {boolean} enabled - if mod is to be loaded.
+ * @property {string} version - version of the mod.
+ * @property {string=} sha1 - SHA1 hash of the zip file.
+ */
+const ModRecordJsonSchema = {
+	type: "object",
+	required: ["name", "enabled", "version"],
+	properties: {
+		"name": { type: "string" },
+		"enabled": { type: "boolean" },
+		"version": { type: "string" },
+		"sha1": { type: "string" },
+	},
+};
+
+/**
+ * Factorio Mod Pack
+ *
+ * Tracks mods and settings for a collection of Factorio mods.
+ * @alias module:lib/data.ModPack
+ */
+class ModPack {
+	/**
+	 * Id of this mod pack
+	 * @type {number}
+	 */
+	id;
+
+	/**
+	 * Name of this mod pack.
+	 * @type {string}
+	 */
+	name = "";
+
+	/**
+	 * Description for this modpcak.
+	 * @type {string}
+	 */
+	description = "";
+
+	/**
+	 * Version of Factorio this mod pack is for
+	 * @type {string}
+	 */
+	factorioVersion = "1.1.0";
+
+	/**
+	 * Integer representation of the factorioVersion
+	 * @type {number}
+	 */
+	get integerFactorioVersion() {
+		return integerFactorioVersion(this.factorioVersion);
+	}
+
+	/**
+	 * Mods included in this mod pack
+	 * @type {Map<string, module:lib/data.ModPack~ModRecord>}
+	 */
+	mods = new Map();
+
+	/**
+	 * Mod settings for this mod pack
+	 * @type {Object<string, Map<string, module:lib/data.ModPack~ModSetting>>=}
+	 */
+	settings = {
+		"startup": new Map(),
+		"runtime-global": new Map(),
+		"runtime-per-user": new Map(),
+	};
+
+	/**
+	 * Mapping to files containing exported data for this modpack
+	 * @type {module:lib/data.ExportManifest|undefined}
+	 */
+	exportManifest;
+
+	/**
+	 * True if this mod pack has been deleted from the list of mod packs.
+	 * @type {boolean}
+	 */
+	isDeleted = false;
+
+	static jsonSchema = {
+		type: "object",
+		additionalProperties: false,
+		required: ["name", "description", "factorio_version", "mods", "settings"],
+		properties: {
+			"id": { type: "integer" },
+			"name": { type: "string" },
+			"description": { type: "string" },
+			"factorio_version": { type: "string" },
+			"mods": {
+				type: "array",
+				items: ModRecordJsonSchema,
+			},
+			"settings": {
+				type: "object",
+				additionalProperties: false,
+				required: ["startup", "runtime-global", "runtime-per-user"],
+				properties: {
+					"startup": ModSettingsJsonSchema,
+					"runtime-global": ModSettingsJsonSchema,
+					"runtime-per-user": ModSettingsJsonSchema,
+				},
+			},
+			"export_manifest": ExportManifest.jsonSchema,
+			"is_deleted": { type: "boolean" },
+		},
+	};
+
+	static validate = libSchema.compile(this.jsonSchema);
+
+	constructor(json = {}) {
+		if (json.id) {
+			this.id = json.id;
+		} else {
+			this.id = Math.random() * 2**31 | 0;
+		}
+		if (json.name) { this.name = json.name; }
+		if (json.description) { this.description = json.description; }
+		if (json.factorio_version) { this.factorioVersion = json.factorio_version; }
+		if (json.mods) { this.mods = new Map(json.mods.map(m => [m.name, m])); }
+		if (json.settings) {
+			this.settings = {
+				"startup": new Map(Object.entries(json.settings["startup"])),
+				"runtime-global": new Map(Object.entries(json.settings["runtime-global"])),
+				"runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])),
+			};
+		}
+		if (json.export_manifest) { this.exportManifest = new ExportManifest(json.export_manifest); }
+		if (json.is_deleted) { this.isDeleted = json.is_deleted; }
+
+		if (!this.mods.has("base")) {
+			this.mods.set("base", { name: "base", enabled: true, version: this.factorioVersion });
+		}
+	}
+
+	toJSON() {
+		let json = {
+			id: this.id,
+			name: this.name,
+			description: this.description,
+			factorio_version: this.factorioVersion,
+			mods: [...this.mods.values()],
+			settings: {
+				"startup": Object.fromEntries(this.settings["startup"]),
+				"runtime-global": Object.fromEntries(this.settings["runtime-global"]),
+				"runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]),
+			},
+		};
+		if (this.exportManifest) { json.export_manifest = this.exportManifest.toJSON(); }
+		if (this.isDeleted) { json.is_deleted = this.isDeleted; }
+		return json;
+	}
+
+	toModPackString() {
+		const json = this.toJSON();
+		delete json.id;
+		delete json.export_manifest;
+
+		// eslint-disable-next-line node/no-sync
+		let buf = zlib.deflateSync(JSON.stringify(json));
+		return buf.toString("base64");
+	}
+
+	static fromModPackString(modPackString) {
+		let buf = Buffer.from(modPackString, "base64");
+		try {
+			// eslint-disable-next-line node/no-sync
+			buf = zlib.inflateSync(buf);
+		} catch (err) {
+			if (err.code.startsWith("Z_")) {
+				throw new Error("Malformed mod pack string: zlib inflate failed");
+			}
+		}
+
+		let json;
+		try {
+			json = JSON.parse(buf);
+		} catch (err) {
+			throw new Error(`Malformed mod pack string: ${err.message}`);
+		}
+
+		if (!this.validate(json)) {
+			throw new Error("Malformed mod pack string: Schema validation failed");
+		}
+
+		return new this(json);
+	}
+
+	toModSettingsDat() {
+		function uint8Byte(int) {
+			return Buffer.from(Uint8Array.from([int]).buffer);
+		}
+		function int16Bytes(int) {
+			return Buffer.from(Int16Array.from([int]).buffer);
+		}
+		function uint32Bytes(int) {
+			return Buffer.from(Uint32Array.from([int]).buffer);
+		}
+		function uint32SpaceOptimizedBytes(int) {
+			if (0 <= int && int < 0xff) {
+				return uint8Byte(int);
+			}
+			return Buffer.concat([
+				uint8Byte(0xff),
+				uint32Bytes(int),
+			]);
+		}
+		function doubleBytes(double) {
+			return Buffer.from(Float64Array.from([double]).buffer);
+		}
+		function stringBytes(str) {
+			const buf = Buffer.from(str, "utf8");
+			return Buffer.concat([
+				uint32SpaceOptimizedBytes(buf.length),
+				buf,
+			]);
+		}
+		function immutableStringBytes(str) {
+			if (str === undefined) {
+				return uint8Byte(1); // empty
+			}
+			return Buffer.concat([
+				uint8Byte(0), // empty
+				stringBytes(str),
+			]);
+		}
+		function versionBytes(version) {
+			const [main, major, minor] = version.split(".").map(n => Number.parseInt(n, 10));
+			return Buffer.concat([
+				int16Bytes(main),
+				int16Bytes(major),
+				int16Bytes(minor),
+				Buffer.alloc(2), // developer
+				Buffer.alloc(1), // reserved byte
+			]);
+		}
+		function propertyTreeListBytes(entries) {
+			const sizeBytes = uint32Bytes(entries.length);
+			const itemBytes = entries.flatMap(([key, item]) => [
+				immutableStringBytes(key),
+				// eslint-disable-next-line no-use-before-define
+				propertyTreeBytes(item),
+			]);
+			return Buffer.concat([
+				sizeBytes,
+				...itemBytes,
+			]);
+		}
+		function propertyTreeBytes(element) {
+			let type;
+			let dataBytes;
+			if (typeof element === null) {
+				type = 1;
+				dataBytes = Buffer.alloc(0);
+			} else if (typeof element === "boolean") {
+				type = 1;
+				dataBytes = uint8Byte(element);
+			} else if (typeof element === "number") {
+				type = 2;
+				dataBytes = doubleBytes(element);
+			} else if (typeof element === "string") {
+				type = 3;
+				dataBytes = immutableStringBytes(element);
+			} else if (element instanceof Array) {
+				type = 4;
+				dataBytes = propertyTreeListBytes(element.map(item => [undefined, item]));
+			} else if (typeof element === "object") {
+				type = 5;
+				dataBytes = propertyTreeListBytes(Object.entries(element));
+			} else {
+				throw new Error("Bad element passed to propertyTreeBytes");
+			}
+			return Buffer.concat([
+				uint8Byte(type),
+				Buffer.alloc(1), // anyTypeFlag
+				dataBytes,
+			]);
+		}
+
+		return Buffer.concat([
+			versionBytes(this.factorioVersion),
+			propertyTreeBytes(this.toJSON().settings),
+		]);
+	}
+
+	/**
+	 * Fill in missing settings with their default values
+	 *
+	 * Uses the provided setting prototypes to add any missing mod settings
+	 * in the mod pack with the default value from the prototype.
+	 *
+	 * @param {Object<string, object>} settingPrototypes -
+	 *     Setting prototypes exported from the game.
+	 * @param {Logger} logger - Logger used to report warnings on.
+	 */
+	fillDefaultSettings(settingPrototypes, logger) {
+		const knownTypes = ["bool-setting", "int-setting", "double-setting", "string-setting"];
+		let prototypes = Object.entries(settingPrototypes)
+			.filter(([type, _]) => knownTypes.includes(type))
+			.flatMap(([_, settings]) => Object.values(settings))
+		;
+
+		for (let prototype of prototypes) {
+			if (!["startup", "runtime-global", "runtime-per-user"].includes(prototype.setting_type)) {
+				logger.warn(`Ignoring ${prototype.name} with unknown setting_type ${prototype.setting_type}`);
+				continue;
+			}
+			if (prototype.default_value === undefined) {
+				logger.warn(`Ignoring ${prototype.name} with missing default_value`);
+				continue;
+			}
+			if (this.settings[prototype.setting_type].get(prototype.name)) {
+				continue;
+			}
+
+			this.settings[prototype.setting_type].set(prototype.name, { value: prototype.default_value });
+		}
+	}
+}
+
+module.exports = ModPack;

packages/lib/data/index.js

@@ -0,0 +1,17 @@
+/**
+ * Shared data types used in Clusterio
+ * @module lib/data
+ * @author Hornwitser
+ */
+"use strict";
+const ExportManifest = require("./ExportManifest");
+const ModInfo = require("./ModInfo");
+const ModPack = require("./ModPack");
+const version = require("./version");
+
+module.exports = {
+	ExportManifest,
+	ModInfo,
+	ModPack,
+	...version,
+};

packages/lib/data/version.js

@@ -0,0 +1,44 @@
+// Helper functions for dealing with factorio version numbers
+"use strict";
+
+
+/**
+ * Integer representation of a mod version
+ *
+ * Suitable for sorting and comparing version numbers which may have different
+ * number of leading zeros in them.
+ * @param {string} version - string representation of version.
+ * @returns {number} numeric representation.
+ * @alias module:lib/data.integerModVersion
+ */
+function integerModVersion(version) {
+	const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10));
+	return major * 0x100000000 + minor * 0x10000 + sub; // Can't use bitwise here because this is 48-bits.
+}
+
+/**
+ * Integer representation of a factorio version
+ *
+ * Suitable for sorting and comparing version numbers which may have different
+ * number of leading zeros in them.
+ * @param {string} version - string representation of version.
+ * @returns {number} numeric representation.
+ * @alias module:lib/data.integerFactorioVersion
+ */
+function integerFactorioVersion(version) {
+	const [main, major, minor] = version.split(".").map(n => Number.parseInt(n, 10));
+	return main * 0x100000000 + major * 0x10000 + (minor || 0); // Can't use bitwise here because this is 48-bits.
+}
+
+/**
+ * Matches valid mod versions.
+ * @type {RegExp}
+ * @alias module:lib/data.modVersionRegExp
+ */
+const modVersionRegExp = /^\d+\.\d+\.\d+$/;
+
+module.exports = {
+	integerModVersion,
+	integerFactorioVersion,
+	modVersionRegExp,
+};

packages/lib/factorio/export.js

@@ -7,6 +7,7 @@ const JSZip = require("jszip");
 
 const libIni = require("../ini");
 const libBuildMod = require("../build_mod");
+const libZipOps = require("../zip_ops");
 
 /**
  * Generate the export mod needed for exportData
@@ -56,7 +57,7 @@ async function loadZip(server, modVersions, mod) {
 		zipCache.set(zipPath, zip);
 	}
 
-	return zip.folder(`${mod}_${modVersion}`);
+	return zip.folder(libZipOps.findRoot(zip));
 }
 
 /**
@@ -207,6 +208,42 @@ async function loadLayeredIcon(server, modVersions, item, size, iconCache) {
 	return icon;
 }
 
+const itemTypes = new Set([
+	"item",
+	"ammo",
+	"capsule",
+	"gun",
+	"item-with-entity-data",
+	"item-with-label",
+	"item-with-inventory",
+	"blueprint-book",
+	"item-with-tags",
+	"selection-tool",
+	"blueprint",
+	"copy-paste-tool",
+	"deconstruction-item",
+	"upgrade-item",
+	"module",
+	"rail-planner",
+	"spidertron-remote",
+	"tool",
+	"armor",
+	"mining-tool",
+	"repair-tool",
+
+	// XXX Bad hack to get icons for fluids. These should be treated as a
+	// separate namespace from items as they may be named the same as items.
+	"fluid",
+	// TODO consider adding virtual signals and recipes
+]);
+
+function filterItems(prototypes) {
+	return Object.entries(prototypes)
+		.filter(([type, _]) => itemTypes.has(type))
+		.flatMap(([_, typePrototypes]) => Object.values(typePrototypes))
+	;
+}
+
 /**
  * Export item icons and data
  *
@@ -348,11 +385,29 @@ async function exportLocale(server, modVersions, modOrder, languageCode) {
 async function exportData(server) {
 	await generateExportMod(server);
 
-	let items = [];
+	let settings = {};
+	let prototypes = {};
 	let modVersions = new Map();
 	let modOrder = [];
 
-	server.on("ipc-item_export", data => items.push(data));
+	function add(obj, prototype) {
+		if (!Object.prototype.hasOwnProperty.call(obj, prototype.type)) {
+			obj[prototype.type] = {};
+		}
+		obj[prototype.type][prototype.name] = prototype;
+	}
+
+	server.on("ipc-prototype_export", data => add(prototypes, data));
+	server.on("ipc-settings_export", data => add(settings, data));
+	server.on("ipc-mod_setting_mod", ({ name, mod }) => {
+		for (let type of Object.values(settings)) {
+			if (Object.prototype.hasOwnProperty.call(type, name)) {
+				type[name].mod = mod;
+				return;
+			}
+		}
+		server._logger.error(`Unable to find ${name} in settings prototypes`);
+	});
 	server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); });
 	server.on("output", parsed => {
 		if (parsed.format === "seconds" && parsed.type === "generic") {
@@ -370,8 +425,8 @@ async function exportData(server) {
 		await fs.unlink(server.writePath("mods", "export_0.0.0.zip"));
 	}
 
-	if (!items.length) {
-		throw new Error("No items got exported");
+	if (!Object.keys(prototypes).length) {
+		throw new Error("No prototypes got exported");
 	}
 
 	// Some mod authors put leading zeros into the versions of their zip files.
@@ -396,13 +451,15 @@ async function exportData(server) {
 		}
 	}
 
-	let { iconSheet, itemData } = await exportItems(server, modVersions, items);
+	let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes));
 	let locale = await exportLocale(server, modVersions, modOrder, "en");
 
 	// Free up the memory used by zip files loaded during the export.
 	zipCache.clear();
 
 	let zip = new JSZip();
+	zip.file("export/settings.json", JSON.stringify(settings));
+	zip.file("export/prototypes.json", JSON.stringify(prototypes));
 	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
 	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
 	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));

packages/lib/factorio/patch.js

@@ -8,6 +8,7 @@ const path = require("path");
 const semver = require("semver");
 
 const libHash = require("../hash");
+const { findRoot } = require("../zip_ops");
 
 
 const knownScenarios = {
@@ -21,42 +22,6 @@ const knownScenarios = {
 	},
 };
 
-/**
- * Returns the root folder in the zip file
- *
- * Returns the name of the folder that all files in the zip file is
- * contained in.  Throws an error if there are multiple such folders.
- *
- * @param {JSZip} zip - Zip to search through.
- * @returns {string} name of the root folder.
- * @memberof module:lib/factorio
- * @private
- * @inner
- */
-function findRoot(zip) {
-	let root = null;
-	zip.forEach((relativePath, file) => {
-		let index = relativePath.indexOf("/");
-		if (index === -1) {
-			throw new Error(`Zip contains file '${relativePath}' in root dir`);
-		}
-
-		let pathRoot = relativePath.slice(0, index);
-		if (root === null) {
-			root = pathRoot;
-		} else if (root !== pathRoot) {
-			throw new Error("Zip contains multiple root folders");
-		}
-	});
-
-	if (root === null) {
-		throw new Error("Empty zip file");
-	}
-
-	return root;
-}
-
-
 /**
  * Generates control.lua code for loading the Clusterio modules
  *
@@ -295,7 +260,6 @@ module.exports = {
 	patch,
 
 	// For testing only
-	_findRoot: findRoot,
 	_generateLoader: generateLoader,
 	_reorderDependencies: reorderDependencies,
 };

packages/lib/file_ops.js

@@ -108,9 +108,94 @@ async function safeOutputFile(file, data, options={}) {
 	await fs.rename(temporary, file);
 }
 
+
+// Reserved names by allmost all filesystems
+const badNames = [".", ".."];
+
+// Reserved namespaces in Windows
+const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9];
+const badWinNamespaces = [
+	"CON", "PRN", "AUX", "NUL",
+	...oneToNine.map(n => `COM${n}`),
+	...oneToNine.map(n => `LPT${n}`),
+];
+
+/**
+ * Check if a string is a valid file name
+ *
+ * @param {string} name - Name to check
+ * @throws Error if the name is unsuitable.
+ */
+function checkFilename(name) {
+	// All of these are bad in Windows only, except for /, . and ..
+	// See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+	const badChars = /[<>:"\/\\|?*\x00-\x1f]/g;
+	const badEnd = /[. ]$/;
+
+	if (typeof name !== "string") {
+		throw new Error("must be a string");
+	}
+
+	if (name === "") {
+		throw new Error("cannot be empty");
+	}
+
+	if (badChars.test(name)) {
+		throw new Error('cannot contain <>:"\\/|=* or control characters');
+	}
+
+	if (badNames.includes(name)) {
+		throw new Error(
+			`cannot be named ${name}`
+		);
+	}
+
+	if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) {
+		throw new Error(
+			"cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"
+		);
+	}
+
+	if (badEnd.test(name)) {
+		throw new Error("cannot end with . or space");
+	}
+}
+
+/**
+ * Clean up string to be suitable for use as filename
+ *
+ * @param {string} name - Arbitrary name string
+ * @returns {string} Filename suitable to use in the filesystem
+ */
+function cleanFilename(name) {
+	// copied from checkFilename due to RegExp with global flag containing state.
+	const badChars = /[<>:"\/\\|?*\x00-\x1f]/g;
+	const badEnd = /[. ]$/;
+
+	if (typeof name !== "string") {
+		throw new Error("name must be a string");
+	}
+
+	if (name === "" || badNames.includes(name.toUpperCase())) {
+		name += "_";
+	}
+
+	if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) {
+		name = [`${name.split(".")[0]}_`, ...name.split(".").slice(1)].join(".");
+	}
+
+	name = name.replace(badChars, "_");
+	name = name.replace(badEnd, "_");
+
+	return name;
+}
+
+
 module.exports = {
 	getNewestFile,
 	findUnusedName,
 	getTempFile,
 	safeOutputFile,
+	checkFilename,
+	cleanFilename,
 };

packages/lib/hash.js

@@ -4,7 +4,7 @@
  */
 "use strict";
 const crypto = require("crypto");
-const fs = require("fs");
+const fs = require("fs-extra");
 
 
 /**

packages/lib/helpers.js

@@ -12,6 +12,7 @@
  *
  * @param {*} value - value to return the type of.
  * @returns {string} basic type of the value passed.
+ * @static
  */
 function basicType(value) {
 	if (value === null) { return "null"; }
@@ -24,6 +25,7 @@ function basicType(value) {
  * Asynchronously wait for the given duration
  *
  * @param {number} duration - Time to wait for in milliseconds.
+ * @static
  */
 async function wait(duration) {
 	await new Promise(resolve => { setTimeout(resolve, duration); });
@@ -36,6 +38,7 @@ async function wait(duration) {
  * @param {Promise} promise - Promise to wait for.
  * @param {number} time - Maximum time im milliseconds to wait for.
  * @param {*=} timeoutResult - Value to return if the operation timed out.
+ * @static
  */
 async function timeout(promise, time, timeoutResult) {
 	let timer;
@@ -61,6 +64,7 @@ async function timeout(promise, time, timeoutResult) {
  *
  * @param {Readable} stream - byte stream to read to the end.
  * @returns {Buffer} content of the stream.
+ * @static
  */
 async function readStream(stream) {
 	let chunks = [];
@@ -85,11 +89,194 @@ async function readStream(stream) {
  *
  * @param {string} text - Text to escape RegExp meta chars in.
  * @returns {string} escaped text.
+ * @static
  */
 function escapeRegExp(text) {
 	return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
 }
 
+/**
+ * Format byte count for human readable display
+ *
+ * Shortens a large number of bytes using the kB/MB/GB/TB prefixes.
+ * @param {number} bytes - Count of bytes to format.
+ * @returns {string} formatted text.
+ * @static
+ */
+function formatBytes(bytes) {
+	if (bytes === 0) {
+		return "0\u{A0}Bytes"; // No-break space
+	}
+
+	let units = ["\u{A0}Bytes", "\u{A0}kB", "\u{A0}MB", "\u{A0}GB", "\u{A0}TB"];
+	let factor = 1000;
+	let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length);
+	return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power];
+}
+
+function parseSearchIdentifier(pos, input) {
+	// character = ? all characters excluding space : " ?
+	// identifier = 1*character
+	let startPos = pos;
+	while (pos < input.length && ![" ", '"', ":"].includes(input.charAt(pos))) {
+		pos += 1;
+	}
+	return [pos, input.slice(startPos, pos)];
+}
+
+function parseSearchWord(pos, input) {
+	// non quote character = ? all characters excluding " ?
+	// word op = "-"
+	// word = [ word op ], ( identifier | '"', *non quote character, '"' )
+	let word = {
+		type: "word",
+	};
+	if (input.charAt(pos) === "-") {
+		word.exclude = true;
+		pos += 1;
+	}
+	if (input.charAt(pos) === '"') {
+		pos += 1;
+		let startPos = pos;
+		let endPos = pos;
+		while (pos < input.length && input.charAt(pos) !== '"') {
+			pos += 1;
+			endPos += 1;
+		}
+		if (input.charAt(pos) === '"') {
+			pos += 1;
+		}
+		word.value = input.slice(startPos, endPos);
+	} else {
+		([pos, word.value] = parseSearchIdentifier(pos, input));
+	}
+	return [pos, word];
+}
+
+function parseSearchTerm(pos, input, attributes, issues) {
+	// attribute = identifier, ':', word
+	// term = word | attribute
+	let term;
+	if (["-", '"'].includes(input.charAt(pos))) {
+		([pos, term] = parseSearchWord(pos, input));
+	} else {
+		let identifier;
+		([pos, identifier] = parseSearchIdentifier(pos, input));
+		if (input.charAt(pos) === ":") {
+			pos += 1;
+			identifier = identifier.toLowerCase();
+			if (!Object.prototype.hasOwnProperty.call(attributes, identifier)) {
+				issues.push(`Unregonized attribute "${identifier}", use quotes to escape colons`);
+			} else if (attributes[identifier] === "word") {
+				let value;
+				([pos, value] = parseSearchWord(pos, input));
+				term = { type: "attribute", name: identifier, value };
+			} else {
+				throw new Error(`Bad attribute format ${attributes[identifier]} for ${identifier}`);
+			}
+		} else {
+			term = { type: "word", value: identifier };
+		}
+	}
+	return [pos, term];
+}
+
+/**
+ * Match a parsed word term with the given texts.
+ *
+ * Returns true if the passed word term in the search matches at least one
+ * of the passed text snippets. If the word mode is exclude then returns
+ * true if none of the passed text snippets match.
+ *
+ * @param {lib/helpers.ParsedTerm} word - Word to match.
+ * @param {...string} texts - Text to match word in.
+ * @returns {boolean} true if the word matches.
+ */
+function wordMatches(word, ...texts) {
+	if (word.type !== "word") {
+		throw Error("wordMatches: parameter is not a word");
+	}
+	let matches = false;
+	for (let text of texts) {
+		if (text.includes(word.value)) {
+			matches = true;
+			break;
+		}
+	}
+	if (word.exclude) {
+		matches = !matches;
+	}
+	return matches;
+}
+
+/**
+ * @typedef {object} module:lib/helpers~ParsedTerm
+ * @property {string} type - Type of term, either attribute or word.
+ * @property {string=} name - attribute only: Name of attribute.
+ * @property {boolean=} exclude - word only: exclude results with this word.
+ * @property {string | module:lib/helpers.ParsedTerm} value -
+ *     Parsed value of this term. Is a string if type is word, and a
+ *     ParsedTerm if type is attribute.
+ */
+
+/**
+ * @typedef {object} module:lib/helpers~ParsedSearch
+ * @property {Array<module:lib/helpers~ParsedTerm>} terms -
+ *     Parsed result of search terms.
+ * @property {Array<string>} issues -
+ *     Issues detected while parsing the seach string.
+ */
+
+/**
+ * Parse a search string with optional attributes
+ *
+ * Parses the given input as a search expression consisting of space
+ * delimited words and attributes:word pairs.  For aattributes to be
+ * recognized they need to be passed as name: "word" in the attributes
+ * parameter.
+ *
+ * Words in the search expression can optionally be prefixed by a - to
+ * search for results that does not contain that word. E.g. -author contains
+ * all results not matching author.
+ *
+ * If parsing fails an string describing the problem will be added to the
+ * issues array returned and parsing will resume at some arbitrary point.
+ * The issues should be shown to the end user so that they can correct their
+ * search.
+ *
+ * @param {string} input - Search expression to parse.
+ * @param {Object<string, string>} attributes -
+ *     Recognized attributes and their format. Currently only word is
+ *     supported.
+ * @returns {module:lib/helpers~ParsedSearch} parsed terms of the search.
+ * @static
+ */
+function parseSearchString(input, attributes = {}) {
+	// whitespace = 1*" "
+	// search = [ term, *( [ whitespace ], term ) ]
+	input = input.trim();
+	let parsed = {
+		terms: [],
+		issues: [],
+	};
+	let pos = 0;
+	while (pos < input.length) {
+		let term;
+		([pos, term] = parseSearchTerm(pos, input, attributes, parsed.issues));
+		if (term) {
+			parsed.terms.push(term);
+		}
+		while (pos < input.length && input.charAt(pos) === " ") {
+			pos += 1;
+		}
+	}
+	// istanbul ignore if (should not be possible)
+	if (pos !== input.length) {
+		throw new Error(`parse search ended at ${pos} which is not the end of the input (${input.length})`);
+	}
+	return parsed;
+}
+
 
 module.exports = {
 	basicType,
@@ -97,4 +284,7 @@ module.exports = {
 	timeout,
 	readStream,
 	escapeRegExp,
+	formatBytes,
+	wordMatches,
+	parseSearchString,
 };

packages/lib/index.js

@@ -2,6 +2,7 @@
 const libBuildMod = require("./build_mod");
 const libCommand = require("./command");
 const libConfig = require("./config");
+const libData = require("./data");
 const libDatabase = require("./database");
 const libErrors = require("./errors");
 const libFactorio = require("./factorio");
@@ -18,6 +19,7 @@ const libPrometheus = require("./prometheus");
 const libSchema = require("./schema");
 const libSharedCommands = require("./shared_commands");
 const libUsers = require("./users");
+const libZipOps = require("./zip_ops");
 
 const ExponentialBackoff = require("./ExponentialBackoff");
 const RateLimiter = require("./RateLimiter");
@@ -27,6 +29,7 @@ module.exports = {
 	libBuildMod,
 	libCommand,
 	libConfig,
+	libData,
 	libDatabase,
 	libErrors,
 	libFactorio,
@@ -43,6 +46,7 @@ module.exports = {
 	libSchema,
 	libSharedCommands,
 	libUsers,
+	libZipOps,
 
 	ExponentialBackoff,
 	RateLimiter,

packages/lib/link/messages.js

@@ -1,8 +1,9 @@
 // Message definitions for links
 "use strict";
-const libSchema = require("../schema");
+const libData = require("../data");
 const libErrors = require("../errors");
 const { logger } = require("../logging");
+const libSchema = require("../schema");
 const PlayerStats = require("../PlayerStats");
 
 class MissingLinkHandlerError extends Error {
@@ -709,6 +710,178 @@ messages.sendRcon = new Request({
 	},
 });
 
+messages.getModPack = new Request({
+	type: "get_mod_pack",
+	links: ["control-master", "instance-slave", "slave-master"],
+	permission: "core.mod_pack.get",
+	forwardTo: "master",
+	requestProperties: {
+		"id": { type: "integer" },
+	},
+	responseProperties: {
+		"mod_pack": libData.ModPack.jsonSchema,
+	},
+});
+
+messages.getDefaultModPack = new Request({
+	type: "get_default_mod_pack",
+	links: ["control-master", "instance-slave", "slave-master"],
+	permission: "core.mod_pack.get",
+	forwardTo: "master",
+	responseProperties: {
+		"mod_pack": libData.ModPack.jsonSchema,
+	},
+});
+
+messages.listModPacks = new Request({
+	type: "list_mod_packs",
+	links: ["control-master"],
+	permission: "core.mod_pack.list",
+	responseProperties: {
+		"list": {
+			type: "array",
+			items: libData.ModPack.jsonSchema,
+		},
+	},
+});
+
+messages.createModPack = new Request({
+	type: "create_mod_pack",
+	links: ["control-master"],
+	permission: "core.mod_pack.create",
+	requestProperties: {
+		"mod_pack": libData.ModPack.jsonSchema,
+	},
+});
+
+messages.updateModPack = new Request({
+	type: "update_mod_pack",
+	links: ["control-master"],
+	permission: "core.mod_pack.update",
+	requestProperties: {
+		"mod_pack": libData.ModPack.jsonSchema,
+	},
+});
+
+messages.deleteModPack = new Request({
+	type: "delete_mod_pack",
+	links: ["control-master"],
+	permission: "core.mod_pack.delete",
+	requestProperties: {
+		"id": { type: "integer" },
+	},
+});
+
+messages.setModPackSubscriptions = new Request({
+	type: "set_mod_pack_subscriptions",
+	links: ["control-master"],
+	permission: "core.mod_pack.subscribe",
+	requestProperties: {
+		"all": { type: "boolean" },
+		"mod_pack_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+	},
+});
+
+messages.getMod = new Request({
+	type: "get_mod",
+	links: ["control-master"],
+	permission: "core.mod.get",
+	requestProperties: {
+		"name": { type: "string" },
+		"version": { type: "string" },
+	},
+	responseProperties: {
+		"mod": libData.ModInfo.jsonSchema,
+	},
+});
+
+messages.listMods = new Request({
+	type: "list_mods",
+	links: ["control-master"],
+	permission: "core.mod.list",
+	responseProperties: {
+		"list": {
+			type: "array",
+			items: libData.ModInfo.jsonSchema,
+		},
+	},
+});
+
+messages.searchMods = new Request({
+	type: "search_mods",
+	links: ["control-master"],
+	permission: "core.mod.search",
+	requestRequired: ["query", "factorio_version", "page"],
+	requestProperties: {
+		"query": { type: "string" },
+		"factorio_version": { type: "string" },
+		"page_size": { type: "integer" },
+		"page": { type: "integer" },
+		"sort": { type: "string" },
+		"sort_order": { type: "string" },
+	},
+	responseProperties: {
+		"query_issues": {
+			type: "array",
+			items: { type: "string" },
+		},
+		"page_count": { type: "integer" },
+		"result_count": { type: "integer" },
+		"results": {
+			type: "array",
+			items: {
+				type: "object",
+				properties: {
+					"name": { type: "string" },
+					"versions": {
+						type: "array",
+						items: libData.ModInfo.jsonSchema,
+					},
+				},
+			},
+		},
+	},
+});
+
+messages.setModSubscriptions = new Request({
+	type: "set_mod_subscriptions",
+	links: ["control-master"],
+	permission: "core.mod.subscribe",
+	requestProperties: {
+		"all": { type: "boolean" },
+		"mod_names": {
+			type: "array",
+			items: { type: "string" },
+		},
+	},
+});
+
+messages.downloadMod = new Request({
+	type: "download_mod",
+	links: ["control-master"],
+	permission: "core.mod.download",
+	requestProperties: {
+		"name": { type: "string" },
+		"version": { type: "string" },
+	},
+	responseProperties: {
+		"stream_id": { type: "string" },
+	},
+});
+
+messages.deleteMod = new Request({
+	type: "delete_mod",
+	links: ["control-master"],
+	permission: "core.mod.delete",
+	requestProperties: {
+		"name": { type: "string" },
+		"version": { type: "string" },
+	},
+});
+
 messages.listPermissions = new Request({
 	type: "list_permissions",
 	links: ["control-master"],
@@ -1263,6 +1436,22 @@ messages.saveListUpdate = new Event({
 	},
 });
 
+messages.modPackUpdate = new Event({
+	type: "mod_pack_update",
+	links: ["master-control"],
+	eventProperties: {
+		"mod_pack": libData.ModPack.jsonSchema,
+	},
+});
+
+messages.modUpdate = new Event({
+	type: "mod_update",
+	links: ["master-control"],
+	eventProperties: {
+		"mod": libData.ModInfo.jsonSchema,
+	},
+});
+
 messages.userUpdate = new Event({
 	type: "user_update",
 	links: ["master-control"],

packages/lib/plugin.js

@@ -479,7 +479,6 @@ class BaseMasterPlugin {
 	 */
 	onControlConnectionEvent(connection, event) { }
 
-
 	/**
 	 * Called when a slave is preparing to disconnect from the master
 	 *
@@ -494,6 +493,31 @@ class BaseMasterPlugin {
 	 */
 	async onPrepareSlaveDisconnect(connection) { }
 
+	/**
+	 * Called when a mod pack is updated
+	 *
+	 * Invoked when a mod pack has been added, updated or deleted in the
+	 * cluster.
+	 *
+	 * If the mod pack has been deleted its `.isDeleted` property will be
+	 * true.
+	 *
+	 * @param {module:lib/data.ModPack} modPack - Mod pack that updated.
+	 */
+	async onModPackUpdated(modPack) { }
+
+	/**
+	 * Called when a mod stored on the master server is updated
+	 *
+	 * Invoked when a mod has been added, updated or deleted from the pool
+	 * of shared mods stored on the cluster.
+	 *
+	 * If a mod has been deleted its `.isDeleted` property will be true.
+	 *
+	 * @param {module:lib/data.ModInfo} mod - Mod that updated.
+	 */
+	async onModUpdated(mod) { }
+
 	/**
 	 * Called when a player joins or leaves an instance
 	 *

packages/lib/users.js

@@ -511,6 +511,80 @@ definePermission({
 	description: "Send arbitrary RCON commands to instances.",
 });
 
+definePermission({
+	name: "core.mod_pack.get",
+	title: "Retrieve mod pack",
+	description: "Get the details of a mod pack in the cluster.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod_pack.list",
+	title: "List mod packs",
+	description: "Get the full list of a mod packs in the cluster.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod_pack.subscribe",
+	title: "Subscribe to mod pack updates",
+	description: "Subscribe to be notified on updates on the details of mod packs.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod_pack.create",
+	title: "Create mod pack",
+	description: "Create new mod pack in the cluster.",
+});
+definePermission({
+	name: "core.mod_pack.update",
+	title: "Update mod pack",
+	description: "Edit existing mod pack definition in the cluster.",
+});
+definePermission({
+	name: "core.mod_pack.delete",
+	title: "Delete mod pack",
+	description: "Delete mod pack definition in the cluster.",
+});
+definePermission({
+	name: "core.mod.get",
+	title: "Get mods",
+	description: "Get the details of a mod stored in the cluster.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod.list",
+	title: "List mods",
+	description: "Get the full list of mods stored on the master.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod.search",
+	title: "Search mods",
+	description: "Search through the list of mods stored on the master.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod.subscribe",
+	title: "Subscribe to mod updates",
+	description: "Subscribe to be notified on updates on the details of mods.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod.upload",
+	title: "Upload mod",
+	description: "Upload mods to the master.",
+});
+definePermission({
+	name: "core.mod.download",
+	title: "Download mod",
+	description: "Download mods stored on the master.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.mod.delete",
+	title: "Delete mod",
+	description: "Delete mods stored on the master.",
+});
+
 definePermission({
 	name: "core.permission.list",
 	title: "List permissions",

packages/lib/zip_ops.js

@@ -0,0 +1,40 @@
+"use strict";
+
+/**
+ * Returns the root folder in the zip file
+ *
+ * Returns the name of the folder that all files in the zip file is
+ * contained in.  Throws an error if there are multiple such folders.
+ *
+ * @param {JSZip} zip - Zip to search through.
+ * @returns {string} name of the root folder.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
+ */
+function findRoot(zip) {
+	let root = null;
+	zip.forEach((relativePath, file) => {
+		let index = relativePath.indexOf("/");
+		if (index === -1) {
+			throw new Error(`Zip contains file '${relativePath}' in root dir`);
+		}
+
+		let pathRoot = relativePath.slice(0, index);
+		if (root === null) {
+			root = pathRoot;
+		} else if (root !== pathRoot) {
+			throw new Error("Zip contains multiple root folders");
+		}
+	});
+
+	if (root === null) {
+		throw new Error("Empty zip file");
+	}
+
+	return root;
+}
+
+module.exports = {
+	findRoot,
+};

packages/master/src/BaseConnection.js

@@ -84,6 +84,27 @@ class BaseConnection extends libLink.Link {
 	get connected() {
 		return !this._disconnecting && this.connector.connected;
 	}
+
+	async getModPackRequestHandler(message) {
+		let { id } = message.data;
+		let modPack = this._master.modPacks.get(id);
+		if (!modPack) {
+			throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`);
+		}
+		return { mod_pack: modPack.toJSON() };
+	}
+
+	async getDefaultModPackRequestHandler(message) {
+		let id = this._master.config.get("master.default_mod_pack_id");
+		if (id === null) {
+			throw new libErrors.RequestError("Default mod pack not set on master");
+		}
+		let modPack = this._master.modPacks.get(id);
+		if (!modPack) {
+			throw new libErrors.RequestError(`Default mod pack configured (${id}) does not exist`);
+		}
+		return { mod_pack: modPack.toJSON() };
+	}
 }
 
 module.exports = BaseConnection;

packages/master/src/ControlConnection.js

@@ -1,9 +1,13 @@
 "use strict";
-const jwt = require("jsonwebtoken");
 const events = require("events");
+const fs = require("fs-extra");
+const jwt = require("jsonwebtoken");
+const path = require("path");
 
 const libConfig = require("@clusterio/lib/config");
+const libData = require("@clusterio/lib/data");
 const libErrors = require("@clusterio/lib/errors");
+const libHelpers = require("@clusterio/lib/helpers");
 const libLink = require("@clusterio/lib/link");
 const { logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
@@ -14,6 +18,8 @@ const libUsers = require("@clusterio/lib/users");
 const BaseConnection = require("./BaseConnection");
 const routes = require("./routes");
 
+const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare;
+
 const lastQueryLogTime = new libPrometheus.Gauge(
 	"clusterio_master_last_query_log_duration_seconds",
 	"Time in seconds the last log query took to execute."
@@ -53,6 +59,16 @@ class ControlConnection extends BaseConnection {
 			instance_ids: [],
 		};
 
+		this.modPackSubscriptions = {
+			all: false,
+			mod_pack_ids: [],
+		};
+
+		this.modSubscriptions = {
+			all: false,
+			mod_names: [],
+		};
+
 		this.userSubscriptions = {
 			all: false,
 			names: [],
@@ -458,11 +474,196 @@ class ControlConnection extends BaseConnection {
 		return { save: result.save };
 	}
 
+	async listModPacksRequestHandler(message) {
+		return { list: [...this._master.modPacks.values()].map(pack => pack.toJSON()) };
+	}
+
+	async setModPackSubscriptionsRequestHandler(message) {
+		this.modPackSubscriptions = message.data;
+	}
+
+	async createModPackRequestHandler(message) {
+		let modPack = new libData.ModPack(message.data.mod_pack);
+		if (this._master.modPacks.has(modPack.id)) {
+			throw new libErrors.RequestError(`Mod pack with ID ${modPack.id} already exist`);
+		}
+		this._master.modPacks.set(modPack.id, modPack);
+		this._master.modPackUpdated(modPack);
+	}
+
+	async updateModPackRequestHandler(message) {
+		let modPack = new libData.ModPack(message.data.mod_pack);
+		if (!this._master.modPacks.has(modPack.id)) {
+			throw new libErrors.RequestError(`Mod pack with ID ${modPack.id} does not exist`);
+		}
+		this._master.modPacks.set(modPack.id, modPack);
+		this._master.modPackUpdated(modPack);
+	}
+
+	async deleteModPackRequestHandler(message) {
+		let { id } = message.data;
+		let modPack = this._master.modPacks.get(id);
+		if (!modPack) {
+			throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`);
+		}
+		modPack.isDeleted = true;
+		this._master.modPacks.delete(id);
+		this._master.modPackUpdated(modPack);
+	}
+
+	modPackUpdated(modPack) {
+		if (
+			this.modPackSubscriptions.all
+			|| this.modPackSubscriptions.mod_pack_ids.includes(modPack.id)
+		) {
+			libLink.messages.modPackUpdate.send(this, { mod_pack: modPack.toJSON() });
+		}
+	}
+
+	async getModRequestHandler(message) {
+		let { name, version } = message.data;
+		let filename = `${name}_${version}.zip`;
+		let mod = this._master.mods.get(filename);
+		if (!mod) {
+			throw new libErrors.RequestError(`Mod ${filename} does not exist`);
+		}
+		return {
+			mod: mod.toJSON(),
+		};
+	}
+
+	async listModsRequestHandler(message) {
+		return { list: [...this._master.mods.values()].map(mod => mod.toJSON()) };
+	}
+
+	static termsMatchesMod(terms, mod) {
+		for (let term of terms) {
+			if (term.type === "word") {
+				if (!libHelpers.wordMatches(term,
+					mod.name, mod.version, mod.title, mod.author, mod.contact,
+					mod.homepage, mod.description, mod.filename
+				)) {
+					return false;
+				}
+			} else if (term.type === "attribute") {
+				if (!libHelpers.wordMatches(term.value, mod[term.name])) {
+					return false;
+				}
+			}
+		}
+		return true;
+	}
+
+	async searchModsRequestHandler(message) {
+		let query = libHelpers.parseSearchString(message.data.query, {
+			name: "word",
+			// version
+			title: "word",
+			author: "word",
+			contact: "word",
+			homepage: "word",
+			description: "word",
+			// factorioVersion
+			// dependencies
+			filename: "word",
+			// size
+			sha1: "word",
+		});
+		let factorioVersion = message.data.factorio_version;
+
+		let results = new Map();
+		for (let mod of this._master.mods.values()) {
+			if (
+				mod.factorioVersion !== factorioVersion
+				|| !ControlConnection.termsMatchesMod(query.terms, mod)
+			) {
+				continue;
+			}
+			let result = results.get(mod.name);
+			if (!result) {
+				result = {
+					name: mod.name,
+					versions: [],
+				};
+				results.set(mod.name, result);
+			}
+			result.versions.push(mod);
+		}
+		for (let result of results.values()) {
+			result.versions.sort((a, b) => b.integerVersion - a.integerVersion);
+			result.versions.map(e => e.toJSON());
+		}
+		let resultList = [...results.values()];
+
+		const sort = message.data.sort;
+		if (sort) {
+			const sorters = {
+				name: (a, b) => strcmp(a.versions[0].name, b.versions[0].name),
+				title: (a, b) => strcmp(a.versions[0].title, b.versions[0].title),
+				author: (a, b) => strcmp(a.versions[0].author, b.versions[0].author),
+			};
+			if (!Object.prototype.hasOwnProperty.call(sorters, sort)) {
+				throw new libErrors.RequestError(`Invalid value for sort: ${sort}`);
+			}
+			resultList.sort(sorters[sort]);
+			let order = message.data.sort_order;
+			if (order === "desc") {
+				resultList.reverse();
+			}
+		}
+
+		const page = message.data.page;
+		const pageSize = message.data.page_size || 10;
+		resultList = resultList.slice((page - 1) * pageSize, page * pageSize);
+
+		return {
+			query_issues: query.issues,
+			page_count: Math.ceil(results.size / pageSize),
+			result_count: results.size,
+			results: resultList,
+		};
+	}
+
+	async setModSubscriptionsRequestHandler(message) {
+		this.modSubscriptions = message.data;
+	}
+
+	async downloadModRequestHandler(message) {
+		let { name, version } = message.data;
+		let filename = `${name}_${version}.zip`;
+		let mod = this._master.mods.get(filename);
+		if (!mod) {
+			throw new libErrors.RequestError(`Mod ${filename} does not exist`);
+		}
+		let modPath = path.join(this._master.config.get("master.mods_directory"), mod.filename);
+
+		let stream = await routes.createProxyStream(this._master.app);
+		stream.filename = mod.filename;
+		stream.source = fs.createReadStream(modPath);
+		stream.mime = "application/zip";
+		stream.size = mod.size;
+
+		return { stream_id: stream.id };
+	}
+
+	async deleteModRequestHandler(message) {
+		await this._master.deleteMod(message.data.name, message.data.version);
+	}
+
 	async setLogSubscriptionsRequestHandler(message) {
 		this.logSubscriptions = message.data;
 		this.updateLogSubscriptions();
 	}
 
+	modUpdated(mod) {
+		if (
+			this.modSubscriptions.all
+			|| this.modSubscriptions.mod_names.includes(mod.name)
+		) {
+			libLink.messages.modUpdate.send(this, { mod: mod.toJSON() });
+		}
+	}
+
 	updateLogSubscriptions() {
 		let { all, master, slave_ids, instance_ids } = this.logSubscriptions;
 		if (all || master || slave_ids.length || instance_ids.length) {

packages/master/src/Master.js

@@ -9,6 +9,7 @@ const path = require("path");
 
 
 const libConfig = require("@clusterio/lib/config");
+const libData = require("@clusterio/lib/data");
 const libErrors = require("@clusterio/lib/errors");
 const libFileOps = require("@clusterio/lib/file_ops");
 const libLink = require("@clusterio/lib/link");
@@ -65,6 +66,18 @@ class Master {
 		 */
 		this.instances = null;
 
+		/**
+		 * Mapping of mod pack id to mod pack
+		 * @type {Map<number, module:lib/data.ModPack>}
+		 */
+		this.modPacks = null;
+
+		/**
+		 * Mapping of mod names to mod infos
+		 * @type {Map<string, module:lib/data.ModInfo>}
+		 */
+		this.mods = null;
+
 		/**
 		 * User and roles manager for the cluster
 		 * @type {module:master/src/UserManager}
@@ -161,9 +174,13 @@ class Master {
 
 		this.slaves = await Master.loadSlaves(path.join(databaseDirectory, "slaves.json"));
 		this.instances = await Master.loadInstances(path.join(databaseDirectory, "instances.json"));
+		this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json"));
 		this.userManager = new UserManager(this.config);
 		await this.userManager.load(path.join(databaseDirectory, "users.json"));
-		this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json"));
+
+		let modsDirectory = this.config.get("master.mods_directory");
+		await fs.ensureDir(modsDirectory);
+		this.mods = await Master.loadModInfos(modsDirectory);
 
 		this.config.on("fieldChanged", (group, field, prev) => {
 			libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev);
@@ -284,6 +301,10 @@ class Master {
 			await Master.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances);
 		}
 
+		if (this.modPacks) {
+			await Master.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks);
+		}
+
 		if (this.userManager) {
 			await this.userManager.save(path.join(databaseDirectory, "users.json"));
 		}
@@ -360,6 +381,57 @@ class Master {
 		await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
 	}
 
+	static async loadModPacks(filePath) {
+		let json;
+		try {
+			json = JSON.parse(await fs.readFile(filePath));
+		} catch (err) {
+			if (err.code !== "ENOENT") {
+				throw err;
+			}
+			return new Map();
+		}
+		return new Map(json.map(e => [e.id, new libData.ModPack(e)]));
+	}
+
+	static async saveModPacks(filePath, modPacks) {
+		await libFileOps.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, 4));
+	}
+
+	static async loadModInfos(modsDirectory) {
+		let mods = new Map();
+		for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
+			if (entry.isDirectory()) {
+				logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
+				continue;
+			}
+
+			if (entry.name.toLowerCase().endsWith("tmp.zip")) {
+				continue;
+			}
+
+			logger.info(`Loading info for Mod ${entry.name}`);
+			let modInfo;
+			try {
+				modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, entry.name));
+			} catch (err) {
+				logger.error(`Error loading mod ${entry.name}: ${err.message}`);
+				continue;
+			}
+
+			if (modInfo.filename !== entry.name) {
+				logger.error(
+					`Mod ${entry.name}'s filename does not match the expected ` +
+					`name ${modInfo.filename}, it will be ignored.`
+				);
+				continue;
+			}
+
+			mods.set(modInfo.filename, modInfo);
+		}
+		return mods;
+	}
+
 	static async loadJsonObject(filePath, throwOnMissing = false) {
 		let manifest = {};
 		try {
@@ -372,12 +444,18 @@ class Master {
 		return manifest;
 	}
 
-	async updateExportManifest(newManifest) {
-		let databaseDirectory = this.config.get("master.database_directory");
-		await libFileOps.safeOutputFile(
-			path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4)
-		);
-		this.exportManifest = newManifest;
+	async deleteMod(name, version) {
+		let filename = `${name}_${version}.zip`;
+		let mod = this.mods.get(filename);
+		if (!mod) {
+			throw new Error(`Mod ${filename} does not exist`);
+		}
+
+		let modsDirectory = this.config.get("master.mods_directory");
+		await fs.unlink(path.join(modsDirectory, filename));
+		this.mods.delete(filename);
+		mod.isDeleted = true;
+		this.modUpdated(mod);
 	}
 
 	/**
@@ -503,6 +581,29 @@ class Master {
 		}
 	}
 
+	modPackUpdated(modPack) {
+		for (let controlConnection of this.wsServer.controlConnections) {
+			if (controlConnection.connector.closing) {
+				continue;
+			}
+			controlConnection.modPackUpdated(modPack);
+		}
+
+		libPlugin.invokeHook(this.plugins, "onModPackUpdated", modPack);
+	}
+
+	modUpdated(mod) {
+		for (let controlConnection of this.wsServer.controlConnections) {
+			if (controlConnection.connector.closing) {
+				continue;
+			}
+
+			controlConnection.modUpdated(mod);
+		}
+
+		libPlugin.invokeHook(this.plugins, "onModUpdated", mod);
+	}
+
 	userUpdated(user) {
 		for (let controlConnection of this.wsServer.controlConnections) {
 			if (controlConnection.connector.closing) {

packages/master/src/routes.js

@@ -2,11 +2,14 @@
 const Busboy = require("busboy");
 const crypto = require("crypto");
 const events = require("events");
+const fs = require("fs-extra");
 const JSZip = require("jszip");
 const jwt = require("jsonwebtoken");
 const path = require("path");
+const nodeStream = require("stream");
 const util = require("util");
 
+const libData = require("@clusterio/lib/data");
 const libErrors = require("@clusterio/lib/errors");
 const libFileOps = require("@clusterio/lib/file_ops");
 const libHash = require("@clusterio/lib/hash");
@@ -18,6 +21,8 @@ const { logger } = require("@clusterio/lib/logging");
 
 const { endpointHitCounter } = require("./metrics");
 
+const finished = util.promisify(nodeStream.finished);
+
 
 // Merges samples from sourceResult to destinationResult
 function mergeSamples(destinationResult, sourceResult) {
@@ -182,6 +187,21 @@ async function uploadExport(req, res) {
 		return;
 	}
 
+	if (typeof req.query.mod_pack_id !== "string") {
+		res.sendStatus(400);
+		return;
+	}
+	let modPackId = Number.parseInt(req.query.mod_pack_id, 10);
+	if (!Number.isInteger(modPackId)) {
+		res.sendStatus(400);
+		return;
+	}
+	let modPack = res.app.locals.master.modPacks.get(modPackId);
+	if (!modPack) {
+		res.sendStatus(400);
+		return;
+	}
+
 	let data = [];
 	for await (let chunk of req) {
 		data.push(chunk);
@@ -192,33 +212,38 @@ async function uploadExport(req, res) {
 
 	// This is hardcoded to prevent path expansion attacks
 	let exportFiles = [
+		"export/settings.json",
+		"export/prototypes.json",
 		"export/item-spritesheet.png",
 		"export/item-metadata.json",
 		"export/locale.json",
 	];
 
-	let manifest = {};
+	let assets = {};
+	let settingPrototypes = {};
 	for (let filePath of exportFiles) {
 		let file = zip.file(filePath);
 		if (!file) {
 			continue;
 		}
 
+		if (filePath === "export/settings.json") {
+			settingPrototypes = JSON.parse(await file.async("text"));
+		}
+
 		let { name, ext } = path.posix.parse(filePath);
 		let hash = await libHash.hashStream(file.nodeStream());
-		manifest[name] = `static/${name}.${hash}${ext}`;
+		assets[name] = `${name}.${hash}${ext}`;
 		await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
 	}
-	await res.app.locals.master.updateExportManifest(manifest);
+
+	modPack.exportManifest = new libData.ExportManifest({ assets });
+	modPack.fillDefaultSettings(settingPrototypes, logger);
+	res.app.locals.master.modPackUpdated(modPack);
 
 	res.sendStatus(200);
 }
 
-async function getExportManifest(req, res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	res.json(res.app.locals.master.exportManifest);
-}
-
 async function createProxyStream(app) {
 	let asyncRandomBytes = util.promisify(crypto.randomBytes);
 	let id = (await asyncRandomBytes(8)).toString("hex");
@@ -306,6 +331,8 @@ const zipMimes = [
 	"application/x-zip-compressed",
 ];
 
+const contentTypeRegExp = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/;
+
 async function uploadSave(req, res) {
 	try {
 		res.locals.user.checkPermission("core.instance.save.upload");
@@ -313,9 +340,10 @@ async function uploadSave(req, res) {
 		res.status(403).json({ request_errors: [err.message] });
 		return;
 	}
+	endpointHitCounter.labels(req.route.path).inc();
 
 	let contentType = req.get("Content-Type");
-	let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType);
+	let match = contentTypeRegExp.exec(contentType);
 	if (!match) {
 		res.status(415).json({ request_errors: ["invalid Content-Type"] });
 		return;
@@ -436,6 +464,154 @@ async function uploadSave(req, res) {
 	res.json({ saves });
 }
 
+function checkModName(name) {
+	try {
+		libFileOps.checkFilename(name);
+	} catch (err) {
+		throw new libErrors.RequestError(`Mod name ${err.message}`);
+	}
+}
+
+async function uploadMod(req, res) {
+	try {
+		res.locals.user.checkPermission("core.mod.upload");
+	} catch (err) {
+		res.status(403).json({ request_errors: [err.message] });
+		return;
+	}
+	endpointHitCounter.labels(req.route.path).inc();
+
+	let contentType = req.get("Content-Type");
+	let match = contentTypeRegExp.exec(contentType);
+	if (!match) {
+		res.status(415).json({ request_errors: ["invalid Content-Type"] });
+		return;
+	}
+	let mimeType = match[1].toLowerCase();
+
+	let tasks = [];
+	let errors = [];
+	let requestErrors = [];
+	let mods = [];
+
+	async function handleFile(stream, filename) {
+		try {
+			checkModName(filename);
+
+		} catch (err) {
+			logger.error(`Error uploading mod: ${err.message}`);
+			errors.push(err.message);
+			stream.resume();
+			return;
+		}
+
+		const modsDirectory = req.app.locals.master.config.get("master.mods_directory");
+		let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
+		try {
+
+			let writeStream;
+			while (true) {
+				try {
+					writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" });
+					await events.once(writeStream, "open");
+					break;
+				} catch (err) {
+					if (err.code === "EEXIST") {
+						tempFilename = await libFileOps.findUnusedName(modsDirectory, tempFilename, ".tmp.zip");
+					} else {
+						throw err;
+					}
+				}
+			}
+			stream.pipe(writeStream);
+			await finished(writeStream);
+
+			const modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
+			await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
+			req.app.locals.master.mods.set(modInfo.filename, modInfo);
+			req.app.locals.master.modUpdated(modInfo);
+			mods.push(modInfo.toJSON());
+
+		} catch (err) {
+			logger.error(`Error uploading mod: ${err.message}`);
+			errors.push(err.message);
+			stream.resume();
+
+			// Attempt to clean up.
+			writeStream.destroy();
+			try {
+				await fs.unlink(path.join(modsDirectory, tempFilename));
+			} catch (unlinkErr) {
+				if (unlinkErr.code !== "ENOENT") {
+					logger.error(`Error removing ${tempFilename}: ${err.message}`);
+				}
+			}
+		}
+	}
+
+	if (mimeType === "multipart/form-data") {
+		await new Promise(resolve => {
+			let busboy = new Busboy({ headers: req.headers });
+			busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => {
+				if (!zipMimes.includes(fileMime)) {
+					requestErrors.push("invalid file Content-Type");
+				}
+
+				if (!filename.endsWith(".zip")) {
+					requestErrors.push("filename must end with .zip");
+				}
+
+				if (errors.length || requestErrors.length) {
+					stream.resume();
+					return;
+				}
+
+				tasks.push(handleFile(stream, filename));
+			});
+			busboy.on("finish", resolve);
+			busboy.on("error", (err) => {
+				logger.error(`Error parsing multipart request in upload-mod:\n${err.stack}`);
+				errors.push(err.message);
+			});
+			req.pipe(busboy);
+		});
+
+	} else if (zipMimes.includes(mimeType)) {
+		let filename = req.query.filename;
+		if (typeof filename !== "string") {
+			requestErrors.push("Missing or invalid filename parameter");
+		} else if (!filename.endsWith(".zip")) {
+			requestErrors.push("filename must end with .zip");
+		}
+
+		if (errors.length || requestErrors.length) {
+			req.resume();
+		} else {
+			tasks.push(handleFile(req, filename));
+		}
+
+	} else {
+		res.status(415).json({ request_errors: ["invalid Content-Type"] });
+		return;
+	}
+
+	await Promise.all(tasks);
+
+	if (errors.length) {
+		res.status(500);
+		res.json({ errors, request_errors: requestErrors });
+		return;
+	}
+	if (requestErrors.length) {
+		res.status(400);
+		res.json({ request_errors: requestErrors });
+		return;
+	}
+
+	res.json({ mods });
+}
+
+
 function addRouteHandlers(app) {
 	app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next));
 	app.get("/api/plugins", getPlugins);
@@ -443,13 +619,16 @@ function addRouteHandlers(app) {
 		validateSlaveToken,
 		(req, res, next) => uploadExport(req, res).catch(next)
 	);
-	app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next));
 	app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next));
 	app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next));
 	app.post("/api/upload-save",
 		validateUserToken,
 		(req, res, next) => uploadSave(req, res).catch(next)
 	);
+	app.post("/api/upload-mod",
+		validateUserToken,
+		(req, res, next) => uploadMod(req, res).catch(next)
+	);
 }
 
 // Routes used in the web interface and served by the master server
@@ -460,6 +639,8 @@ const webRoutes = [
 	"/slaves/:id/view",
 	"/instances",
 	"/instances/:id/view",
+	"/mods",
+	"/mods/mod-packs/:id/view",
 	"/users",
 	"/users/:id/view",
 	"/roles",

packages/slave/lua/export/clusterio_api.lua

@@ -0,0 +1,73 @@
+local function is_array(t)
+	local count = 0
+	for k in pairs(t) do
+		if type(k) ~= "number" then
+			return false
+		end
+		count = count + 1
+	end
+
+	for i = 1, count do
+		if not t[i] then
+			return false
+		end
+	end
+
+	return count ~= 0
+end
+
+-- no game.table_to_json here :(
+local function table_to_json(data)
+	if type(data) == "number" then
+		if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it.
+		if data == -math.huge then return "-1e500" end
+		if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead.
+		return string.format("%.17g", data)
+
+	elseif type(data) == "string" then
+		data = data:gsub('([\x00-\x1f\\"])', function(match)
+			return "\\u00" .. string.format("%02x", match:byte())
+		end)
+		return '"' .. data .. '"'
+
+	elseif data == nil then
+		return "null"
+
+	elseif data == true then
+		return "true"
+	elseif data == false then
+		return "false"
+
+	elseif type(data) == "table" then
+		local r
+		if is_array(data) then
+			r = {"["}
+			for i=1, #data do
+				r[#r+1] = table_to_json(data[i])
+				r[#r+1] = ","
+			end
+
+			-- overwrite last comma, empty table is an object
+			r[#r] = "]"
+
+		else
+			r = {"{"}
+			for k, v in pairs(data) do
+				r[#r+1] = table_to_json(tostring(k))
+				r[#r+1] = ":"
+				r[#r+1] = table_to_json(v)
+				r[#r+1] = ","
+			end
+			r[#r + (r[#r] == "," and 0 or 1)] = "}"
+		end
+
+		return table.concat(r)
+	end
+end
+
+local clusterio_api = {}
+function clusterio_api.send_json(channel, data)
+	data = table_to_json(data)
+	print("\f$ipc:" .. channel .. "?j" .. data)
+end
+return clusterio_api

packages/slave/lua/export/control.lua

@@ -5,4 +5,7 @@ end
 
 script.on_init(function()
 	send_json("mod_list", game.active_mods)
+	for name, mod_setting in pairs(game.mod_setting_prototypes) do
+		send_json("mod_setting_mod", { name = name, mod = mod_setting.mod })
+	end
 end)

packages/slave/lua/export/data-final-fixes.lua

@@ -1,104 +1,8 @@
-local item_types = {
-	-- Item categories
-	"item",
-	"ammo",
-	"capsule",
-	"gun",
-	"item-with-entity-data",
-	"item-with-label",
-	"item-with-inventory",
-	"blueprint-book",
-	"item-with-tags",
-	"selection-tool",
-	"blueprint",
-	"deconstruction-item",
-	"copy-paste-tool",
-	"upgrade-item",
-	"module",
-	"rail-planner",
-	"tool",
-	"armor",
-	"mining-tool",
-	"repair-tool",
+local clusterio_api = require("clusterio_api")
 
-	-- Fluids
-	"fluid",
-}
-
-local function is_array(t)
-	local count = 0
-	for k in pairs(t) do
-		if type(k) ~= "number" then
-			return false
-		end
-		count = count + 1
-	end
-
-	for i = 1, count do
-		if not t[i] then
-			return false
-		end
-	end
-
-	return count ~= 0
-end
-
--- no game.table_to_json here :(
-local function table_to_json(data)
-	if type(data) == "number" then
-		if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it.
-		if data == -math.huge then return "-1e500" end
-		if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead.
-		return string.format("%.17g", data)
-
-	elseif type(data) == "string" then
-		data = data:gsub('([\x00-\x1f\\"])', function(match)
-			return "\\u00" .. string.format("%02x", match:byte())
-		end)
-		return '"' .. data .. '"'
-
-	elseif data == nil then
-		return "null"
-
-	elseif data == true then
-		return "true"
-	elseif data == false then
-		return "false"
-
-	elseif type(data) == "table" then
-		local r
-		if is_array(data) then
-			r = {"["}
-			for i=1, #data do
-				r[#r+1] = table_to_json(data[i])
-				r[#r+1] = ","
-			end
-
-			-- overwrite last comma, empty table is an object
-			r[#r] = "]"
-
-		else
-			r = {"{"}
-			for k, v in pairs(data) do
-				r[#r+1] = table_to_json(tostring(k))
-				r[#r+1] = ":"
-				r[#r+1] = table_to_json(v)
-				r[#r+1] = ","
-			end
-			r[#r + (r[#r] == "," and 0 or 1)] = "}"
-		end
-
-		return table.concat(r)
-	end
-end
-
-local function send_json(channel, data)
-	data = table_to_json(data)
-	print("\f$ipc:" .. channel .. "?j" .. data)
-end
-
-for _, group in ipairs(item_types) do
-	for _, item in pairs(data.raw[group]) do
-		send_json("item_export", item)
+log("Exporting prototypes")
+for group, prototypes in pairs(data.raw) do
+	for name, prototype in pairs(prototypes) do
+		clusterio_api.send_json("prototype_export", prototype)
 	end
 end

packages/slave/lua/export/settings-final-fixes.lua

@@ -0,0 +1,8 @@
+local clusterio_api = require("clusterio_api")
+
+log("Exporting settings")
+for group, prototypes in pairs(data.raw) do
+	for name, prototype in pairs(prototypes) do
+		clusterio_api.send_json("settings_export", prototype)
+	end
+end

packages/slave/slave.js

@@ -177,7 +177,7 @@ async function startSlave() {
 	// If we get here the command was run
 
 	await fs.ensureDir(slaveConfig.get("slave.instances_directory"));
-	await fs.ensureDir("sharedMods");
+	await fs.ensureDir(slaveConfig.get("slave.mods_directory"));
 	await fs.ensureDir("modules");
 
 	// Set the process title, shows up as the title of the CMD window on windows

packages/slave/src/Instance.js

@@ -6,6 +6,7 @@ const phin = require("phin");
 const util = require("util");
 
 // internal libraries
+const libData = require("@clusterio/lib/data");
 const libFileOps = require("@clusterio/lib/file_ops");
 const libFactorio = require("@clusterio/lib/factorio");
 const libLink = require("@clusterio/lib/link");
@@ -94,71 +95,6 @@ const serverSettingsActions = {
 	},
 };
 
-/**
- * Create and update symlinks for shared mods in an instance
- *
- * Creates symlinks for .zip and .dat files that are not present in the
- * instance mods directory but is present in the sharedMods directory,
- * and removes any symlinks that don't point to a file in the instance
- * mods directory.  If the instance mods directory doesn't exist it will
- * be created.
- *
- * Note that on Windows this creates hard links instead of symbolic
- * links as the latter requires elevated privileges.  This unfortunately
- * means the removal of mods from the shared mods dir can't be detected.
- *
- * @param {Instance} instance - Instance to link mods for
- * @param {string} sharedMods - Path to folder to link mods from.
- * @private
- */
-async function symlinkMods(instance, sharedMods) {
-	await fs.ensureDir(instance.path("mods"));
-
-	// Remove broken symlinks in instance mods.
-	for (let entry of await fs.readdir(instance.path("mods"), { withFileTypes: true })) {
-		if (entry.isSymbolicLink()) {
-			if (!await fs.pathExists(instance.path("mods", entry.name))) {
-				instance.logger.verbose(`Removing broken symlink ${entry.name}`);
-				await fs.unlink(instance.path("mods", entry.name));
-			}
-		}
-	}
-
-	// Link entries that are in sharedMods but not in instance mods.
-	let instanceModsEntries = new Set(await fs.readdir(instance.path("mods")));
-	for (let entry of await fs.readdir(sharedMods, { withFileTypes: true })) {
-		if (entry.isFile()) {
-			if ([".zip", ".dat", ".json"].includes(path.extname(entry.name))) {
-				if (!instanceModsEntries.has(entry.name)) {
-					instance.logger.verbose(`linking ${entry.name} from ${sharedMods}`);
-					let target = path.join(sharedMods, entry.name);
-					let link = instance.path("mods", entry.name);
-
-					/* eslint-disable max-depth */
-					if (process.platform !== "win32") {
-						await fs.symlink(path.relative(path.dirname(link), target), link);
-
-					// On Windows symlinks require elevated privileges, which is
-					// not something we want to have.  For this reason the mods
-					// are hard linked instead.  This has the drawback of not
-					// being able to identify when mods are removed from the
-					// sharedMods directory, or which mods are linked.
-					} else {
-						await fs.link(target, link);
-					}
-					/* eslint-enable max-depth */
-				}
-
-			} else {
-				instance.logger.warn(`Warning: ignoring file '${entry.name}' in sharedMods`);
-			}
-
-		} else {
-			instance.logger.warn(`Warning: ignoring non-file '${entry.name}' in sharedMods`);
-		}
-	}
-}
-
 
 /**
  * Keeps track of the runtime parameters of an instance
@@ -255,6 +191,12 @@ class Instance extends libLink.Link {
 			}
 		});
 
+		/**
+		 * Mod pack currently running on this instance
+		 * @type {module:lib/data.ModPack>}
+		 */
+		this.activeModPack = undefined;
+
 		/**
 		 * Per player statistics recorded by this instance.
 		 * @type {Map<string, module:lib/PlayerStats>}
@@ -643,6 +585,77 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 		await fs.ensureDir(path.join(instanceDir, "saves"));
 	}
 
+	/**
+	 * Sync instance mods directory with configured mod pack
+	 *
+	 * Adds, deletes, and updates all files in the instance mods folder to
+	 * match up with the mod pack that's configured. If no mod pack is
+	 * configured then an empty mod pack is used, esentially turning it into
+	 * a vanilla mods folder.  If the instance mods directory does't exist
+	 * then it will be created.
+	 *
+	 * On Linux this creates symlinks to mods in the slave's mods folder, on
+	 * Windows hard links are used instead due to symlinks being privileged.
+	 */
+	async syncMods() {
+		const modPackId = this.config.get("factorio.mod_pack");
+		let response;
+		if (modPackId === null) {
+			response = await libLink.messages.getDefaultModPack.send(this);
+		} else {
+			response = await libLink.messages.getModPack.send(this, { id: modPackId });
+		}
+		this.activeModPack = new libData.ModPack(response.mod_pack);
+
+		// TODO validate factorioVersion
+
+		if (!this._slave.config.get("slave.mods_directory_is_shared")) {
+			throw new Error("Fetching mods is not implemented");
+		}
+
+		await fs.ensureDir(this.path("mods"));
+
+		// Remove all files
+		for (let entry of await fs.readdir(this.path("mods"), { withFileTypes: true })) {
+			if (entry.isDirectory()) {
+				this.logger.warn(
+					`Found unexpected directory ${entry.name} in mods folder, it may break Clusterio's mod syncing`
+				);
+				continue;
+			}
+
+			if (entry.isFile() || entry.isSymbolicLink()) {
+				await fs.unlink(this.path("mods", entry.name));
+			}
+		}
+
+		// Add mods from mod the pack
+		const modsDir = this._slave.config.get("slave.mods_directory");
+		for (let mod of this.activeModPack.mods.values()) {
+			const modFile = `${mod.name}_${mod.version}.zip`;
+			const target = path.join(modsDir, modFile);
+			const link = this.path("mods", modFile);
+
+			if (process.platform !== "win32") {
+				await fs.symlink(path.relative(path.dirname(link), target), link);
+
+			// On Windows symlinks require elevated privileges, which is
+			// not something we want to have.  For this reason the mods
+			// are hard linked instead.
+			} else {
+				await fs.link(target, link);
+			}
+		}
+
+		// Write mod-list.json
+		await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({
+			mods: [{ name: "base" }, ...this.activeModPack.mods.values()].map(m => ({ name: m.name, enabled: true })),
+		}, null, 2));
+
+		// Write mod-settings.dat
+		await fs.outputFile(this.path("mods", "mod-settings.dat"), this.activeModPack.toModSettingsDat());
+	}
+
 	/**
 	 * Prepare instance for starting
 	 *
@@ -678,8 +691,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			);
 		}
 
-		// eslint-disable-next-line no-use-before-define
-		await symlinkMods(this, "sharedMods");
+		await this.syncMods();
 	}
 
 	/**
@@ -1030,8 +1042,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			await this.writeServerSettings();
 
 			this.logger.verbose("Creating save .....");
-			// eslint-disable-next-line no-use-before-define
-			await symlinkMods(this, "sharedMods");
+			await this.syncMods();
 
 		} catch (err) {
 			this.notifyExit();
@@ -1053,13 +1064,13 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			await this.writeServerSettings();
 
 			this.logger.info("Exporting data .....");
-			// eslint-disable-next-line no-use-before-define
-			await symlinkMods(this, "sharedMods");
+			await this.syncMods();
 			let zip = await libFactorio.exportData(this.server);
 
 			let content = await zip.generateAsync({ type: "nodebuffer" });
 			let url = new URL(this._slave.config.get("slave.master_url"));
 			url.pathname += "api/upload-export";
+			url.searchParams.set("mod_pack_id", this.activeModPack.id);
 			let response = await phin({
 				url, method: "PUT",
 				data: content,
@@ -1116,6 +1127,3 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 }
 
 module.exports = Instance;
-
-// For testing only
-module.exports._symlinkMods = symlinkMods;

packages/slave/src/Slave.js

@@ -25,86 +25,9 @@ const InstanceConnection = require("./InstanceConnection");
 const finished = util.promisify(stream.finished);
 
 
-function checkFilename(name) {
-	// All of these are bad in Windows only, except for /, . and ..
-	// See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
-	const badChars = /[<>:"\/\\|?*\x00-\x1f]/g;
-	const badEnd = /[. ]$/;
-
-	const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9];
-	const badNames = [
-		// Relative path components
-		".", "..",
-
-		// Reserved filenames in Windows
-		"CON", "PRN", "AUX", "NUL",
-		...oneToNine.map(n => `COM${n}`),
-		...oneToNine.map(n => `LPT${n}`),
-	];
-
-	if (typeof name !== "string") {
-		throw new Error("must be a string");
-	}
-
-	if (name === "") {
-		throw new Error("cannot be empty");
-	}
-
-	if (badChars.test(name)) {
-		throw new Error('cannot contain <>:"\\/|=* or control characters');
-	}
-
-	if (badNames.includes(name.toUpperCase())) {
-		throw new Error(
-			"cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"
-		);
-	}
-
-	if (badEnd.test(name)) {
-		throw new Error("cannot end with . or space");
-	}
-}
-
-/**
- * Clean up string to be suitable for use as filename
- *
- * @param {string} name - Arbitrary name string
- * @returns {string} Filename suitable to use in the filesystem
- * @private
- */
-function cleanFilename(name) {
-	// copied from checkFilename
-	const badChars = /[<>:"\/\\|?*\x00-\x1f]/g;
-	const badEnd = /[. ]$/;
-
-	const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9];
-	const badNames = [
-		// Relative path components
-		".", "..",
-
-		// Reserved filenames in Windows
-		"CON", "PRN", "AUX", "NUL",
-		...oneToNine.map(n => `COM${n}`),
-		...oneToNine.map(n => `LPT${n}`),
-	];
-
-	if (typeof name !== "string") {
-		throw new Error("name must be a string");
-	}
-
-	if (name === "" || badNames.includes(name.toUpperCase())) {
-		name += "_";
-	}
-
-	name = name.replace(badChars, "_");
-	name = name.replace(badEnd, "_");
-
-	return name;
-}
-
 function checkRequestSaveName(name) {
 	try {
-		checkFilename(name);
+		libFileOps.checkFilename(name);
 	} catch (err) {
 		throw new libErrors.RequestError(`Save name ${err.message}`);
 	}
@@ -243,9 +166,9 @@ class Slave extends libLink.Link {
 	}
 
 	async _createNewInstanceDir(name) {
-		name = cleanFilename(name);
+		name = libFileOps.cleanFilename(name);
 		try {
-			checkFilename(name);
+			libFileOps.checkFilename(name);
 		} catch (err) {
 			throw new Error(`Instance folder was unepectedly invalid: name ${err.message}`);
 		}
@@ -837,8 +760,5 @@ ${err.stack}`
 module.exports = Slave;
 
 // For testing only
-module.exports._checkFilename = checkFilename;
-module.exports._cleanFilename = cleanFilename;
 module.exports._discoverInstances = discoverInstances;
-module.exports._Slave = Slave;
 

packages/web_ui/src/components/ModDetails.jsx

@@ -0,0 +1,53 @@
+import React, { Fragment } from "react";
+import { Descriptions, Grid, Typography } from "antd";
+import ExclamationCircleOutlined from "@ant-design/icons/ExclamationCircleOutlined";
+
+import { libHelpers } from "@clusterio/lib";
+
+const { useBreakpoint } = Grid;
+
+
+export default function ModDetails(props) {
+	let screens = useBreakpoint();
+	const mod = props.mod;
+
+	return <Descriptions className="borderless" bordered size="small" column={{ xs: 1, sm: 1, md: 1, lg: 2 }}>
+		{mod.error && <Descriptions.Item label={<><ExclamationCircleOutlined/> Error</>} span={2}>
+			{mod.error === "missing" && "The given version of this mod is missing from the master storage."}
+			{mod.error === "bad_checksum"
+				&& "Checksum of the mod in the master storage does not match the checksum provided by this mod pack."
+			}
+		</Descriptions.Item>}
+		{props.actions && !screens.lg
+			&& <Descriptions.Item label="Action">{props.actions(mod)}</Descriptions.Item>
+		}
+		{mod.title && <Descriptions.Item label="Title">{mod.title}</Descriptions.Item>}
+		{mod.version && <Descriptions.Item label="Version" span={mod.title ? 1 : 2}>{mod.version}</Descriptions.Item>}
+		{mod.description && <Descriptions.Item label="Description" span={2}>{mod.description}</Descriptions.Item> }
+		{mod.author && <Descriptions.Item label="Author" span={2}>{mod.author}</Descriptions.Item>}
+		<Descriptions.Item label="Mod&nbsp;Portal" span={2}>
+			<Typography.Link href={`https://mods.factorio.com/mod/${mod.name}`}>
+				{`https://mods.factorio.com/mod/${mod.name}`}
+			</Typography.Link>
+		</Descriptions.Item>
+		{mod.contact && <Descriptions.Item label="Contact" span={2}>{mod.contact}</Descriptions.Item>}
+		{mod.homepage && <Descriptions.Item label="Homepage" span={2}>{mod.homepage}</Descriptions.Item>}
+		<Descriptions.Item label="Internal&nbsp;Name" span={mod.factorioVersion ? 1 : 2}>{mod.name}</Descriptions.Item>
+		{mod.factorioVersion
+			&& <Descriptions.Item label="Factorio&nbsp;Version">{mod.factorioVersion}</Descriptions.Item>
+		}
+		{mod.filename && <Descriptions.Item label="Filename">{mod.filename}</Descriptions.Item>}
+		{mod.size !== undefined
+			&& <Descriptions.Item label="Size">{libHelpers.formatBytes(mod.size)}</Descriptions.Item>
+		}
+		{mod.sha1 && <Descriptions.Item label="SHA1" span={2}>{mod.sha1}</Descriptions.Item>}
+		{mod.dependencies && mod.dependencies.length
+			&& <Descriptions.Item label="Dependencies" span={2}>
+				{mod.dependencies
+					.filter(e => !e.startsWith("(?)"))
+					.map((e, i) => <Fragment key={i}>{e}<br/></Fragment>)
+				}
+			</Descriptions.Item>
+		}
+	</Descriptions>;
+}

packages/web_ui/src/components/ModPackViewPage.jsx

@@ -0,0 +1,803 @@
+import React, { Fragment, useEffect, useContext, useRef, useState } from "react";
+import { useHistory, useParams } from "react-router-dom";
+import {
+	Button, Card, Checkbox, Col, ConfigProvider, Descriptions, Form, Input, PageHeader, Pagination,
+	Popconfirm, Row, Table, Tag, Typography, Select, Skeleton, Space, Spin, Switch, Modal, Tooltip,
+} from "antd";
+import InfoCircleOutlined from "@ant-design/icons/InfoCircleOutlined";
+import ExportOutlined from "@ant-design/icons/ExportOutlined";
+import FileUnknownOutlined from "@ant-design/icons/FileUnknownOutlined";
+import FileExclamationOutlined from "@ant-design/icons/FileExclamationOutlined";
+import CloseOutlined from "@ant-design/icons/CloseOutlined";
+import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
+import PlusOutlined from "@ant-design/icons/PlusOutlined";
+import UploadOutlined from "@ant-design/icons/UploadOutlined";
+
+import { libData, libLink, libLogging } from "@clusterio/lib";
+import ControlContext from "./ControlContext";
+import { useAccount } from "../model/account";
+import { useModPack } from "../model/mod_pack";
+import { useModList } from "../model/mods";
+import { useClipboard } from "../util/clipboard";
+import notify, { notifyErrorHandler } from "../util/notify";
+import PageLayout from "./PageLayout";
+import SectionHeader from "./SectionHeader";
+import ModDetails from "./ModDetails";
+
+const { logger } = libLogging;
+const { Paragraph, Text } = Typography;
+const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare;
+
+function SearchModsTable(props) {
+	const control = useContext(ControlContext);
+	const [searchText, setSearchText] = useState("");
+	const [modResultSort, setModResultSort] = useState("title");
+	const [modResultSortOrder, setModResultSortOrder] = useState("asc");
+	const [modResults, setModResults] = useState([]);
+	const [modResultPage, setModResultPage] = useState(1);
+	const [modResultPageSize, setModResultPageSize] = useState(10);
+	const [modResultCount, setModResultCount] = useState(2);
+	const [modResultSelectedVersion, setModResultSelectedVersion] = useState(new Map());
+
+	const factorioVersion = props.modPack.factorioVersion.split(".").slice(0, 2).join(".");
+
+	useEffect(() => {
+		let canceled = false;
+		libLink.messages.searchMods.send(control, {
+			query: searchText,
+			factorio_version: factorioVersion,
+			page_size: modResultPageSize,
+			page: modResultPage,
+			sort: modResultSort,
+			sort_order: modResultSortOrder,
+		}).then(response => {
+			if (canceled) {
+				return;
+			}
+			// In React < v18 this causes 3 renders and a partial state necessitating the use
+			// of || 0 on modResultSelectedVersion.get() calls. Remove when updating React.
+			setModResults(response.results.map(
+				({ name, versions }) => ({ name, versions: versions.map(mod => new libData.ModInfo(mod)) })
+			));
+			setModResultSelectedVersion(new Map(response.results.map(({ name, versions }) => [name, 0])));
+			setModResultCount(response.result_count);
+		});
+
+		return () => {
+			canceled = true;
+		};
+	}, [searchText, factorioVersion, modResultPageSize, modResultPage, modResultSort, modResultSortOrder]);
+
+	function modResultPageChanged(page, pageSize) {
+		setModResultPage(page);
+		setModResultPageSize(pageSize);
+	}
+
+	function modResultsTableChanged(pagination, filters, sorters, extra) {
+		if (extra.action === "sort") {
+			if (sorters.order) {
+				const mapping = {
+					ascend: "asc",
+					descend: "desc",
+				};
+				setModResultSort(sorters.columnKey);
+				setModResultSortOrder(mapping[sorters.order]);
+			} else {
+				setModResultSort(undefined);
+				setModResultSortOrder(undefined);
+			}
+		}
+	}
+
+	function actions(mod) {
+		return <Space>
+			{(!props.modPack.mods.has(mod.name) || props.modPack.mods.get(mod.name).version !== mod.version)
+				&& <Typography.Link
+					onClick={() => {
+						props.onChange({
+							type: "mods.set",
+							name: mod.name,
+							value: { name: mod.name, enabled: true, version: mod.version, sha1: mod.sha1 },
+						});
+					}}
+				>{props.modPack.mods.has(mod.name) ? "change" : "add"}</Typography.Link>
+			}
+			{props.modPack.mods.has(mod.name)
+				&& <Typography.Link
+					type="danger"
+					onClick={() => {
+						props.onChange({
+							type: "mods.delete",
+							name: mod.name,
+							value: { name: mod.name, version: mod.version, sha1: mod.sha1 },
+						});
+					}}
+				>remove</Typography.Link>
+			}
+		</Space>;
+	}
+
+	return <>
+		<Input
+			placeholder="Search Mods"
+			style={{ marginBottom: 8 }}
+			onChange={(event) => {
+				let search = event.target.value.trim();
+				setSearchText(search);
+				setModResultPage(1);
+			}}
+		/>
+		<SectionHeader
+			title="Stored Mods"
+			extra={<Pagination
+				current={modResultPage}
+				onChange={modResultPageChanged}
+				total={modResultCount}
+				showTotal={total => `${total} results`}
+				showSizeChanger={true}
+				size="small"
+			/>}
+		/>
+		<ConfigProvider renderEmpty={() => "No results"}>
+			<Table
+				size="small"
+				columns={[
+					{
+						title: "Title",
+						key: "title",
+						render: results => results.versions[0].title,
+						defaultSortOrder: "ascend",
+						sorter: true,
+					},
+					{
+						title: "Author",
+						key: "author",
+						render: results => results.versions[0].author,
+						responsive: ["lg"],
+						sorter: true,
+					},
+					{
+						title: "Version",
+						key: "version",
+						align: "right",
+						render: result => <Select
+							size="small"
+							bordered={false}
+							value={modResultSelectedVersion.get(result.name) || 0}
+							onChange={index => {
+								const newVersions = new Map(modResultSelectedVersion);
+								newVersions.set(result.name, index);
+								setModResultSelectedVersion(newVersions);
+							}}
+							options={result.versions.map((mod, index) => ({ label: mod.version, value: index }))}
+						/>,
+					},
+					{
+						title: "Action",
+						key: "action",
+						responsive: ["lg"],
+						render: result => actions(result.versions[modResultSelectedVersion.get(result.name) || 0]),
+					},
+				]}
+				expandable={{
+					expandedRowRender: result => <ModDetails
+						mod={result.versions[modResultSelectedVersion.get(result.name) || 0]}
+						actions={actions}
+					/>,
+					expandedRowClassName: () => "no-expanded-padding",
+				}}
+				onChange={modResultsTableChanged}
+				dataSource={modResults}
+				pagination={false}
+				rowKey={result => result.name}
+			/>
+		</ConfigProvider>
+	</>;
+}
+
+function ModsTable(props) {
+	const [showAddMods, setShowAddMods] = useState(false);
+	const [modList] = useModList();
+
+	const deletedMods = new Map();
+	const changedMods = new Map();
+	for (let change of props.changes) {
+		if (props.modPack.mods.has(change.name)) {
+			if (change.type === "mods.set") {
+				changedMods.set(change.name, change.value);
+			}
+		} else if (change.type === "mods.delete") {
+			deletedMods.set(change.name, change.value);
+		}
+	}
+
+	const modListMap = new Map(modList.map(mod => [`${mod.name}_${mod.version}`, mod]));
+	const mods = [...props.modPack.mods.values(), ...deletedMods.values()].map(
+		mod => {
+			const candidate = modListMap.get(`${mod.name}_${mod.version}`);
+			if (!candidate) {
+				return {
+					...mod,
+					error: "missing",
+				};
+			} else if (mod.sha1 && candidate.sha1 !== mod.sha1) {
+				return {
+					...mod,
+					error: "bad_checksum",
+				};
+			}
+			return candidate;
+		}
+	);
+
+	function actions(mod) {
+		return <Space>
+			{!deletedMods.has(mod.name) && <Typography.Link
+				type="danger"
+				onClick={() => {
+					props.onChange({
+						type: "mods.delete",
+						name: mod.name,
+						value: mod,
+					});
+				}}
+			>remove</Typography.Link>}
+			{(deletedMods.has(mod.name) || changedMods.has(mod.name)) && <Typography.Link
+				onClick={() => {
+					props.onRevert({
+						type: deletedMods.has(mod.name) ? "mods.delete" : "mods.set",
+						name: mod.name,
+						value: mod,
+					});
+				}}
+			>revert</Typography.Link>}
+		</Space>;
+	}
+
+	return <>
+		<SectionHeader
+			title="Mods"
+			extra={<Space>
+				<Button icon={<PlusOutlined />} onClick={() => setShowAddMods(true)}>Add</Button>
+			</Space>}
+		/>
+		{showAddMods && <Card
+			title="Add mods"
+			size="small"
+			extra={<Button type="text" icon={<CloseOutlined />} onClick={() => setShowAddMods(false)} />}
+			style={{ marginBottom: 8 }}
+		>
+			<SearchModsTable modPack={props.modPack} changes={props.changes} onChange={props.onChange} />
+		</Card>}
+		<Table
+			size="small"
+			rowSelection={{
+				type: "checkbox",
+				selectedRowKeys: [...props.modPack.mods.values()].filter(mod => mod.enabled).map(mod => mod.name),
+				getCheckboxProps(mod) {
+					return { disabled: deletedMods.has(mod.name) };
+				},
+				onChange(selectedRowKeys, selectedRows) {
+					function setEnabled(modNames, enabled) {
+						for (let name of modNames) {
+							props.onChange({
+								type: "mods.set",
+								name,
+								value: { ...props.modPack.mods.get(name), enabled },
+							});
+						}
+					}
+					const added = selectedRowKeys.filter(key => !props.modPack.mods.get(key).enabled);
+					const removed = [...props.modPack.mods.values()].filter(
+						mod => mod.enabled && selectedRowKeys.indexOf(mod.name) === -1
+					).map(mod => mod.name);
+					setEnabled(added, true);
+					setEnabled(removed, false);
+				},
+			}}
+			columns={[
+				{
+					title: "Name",
+					key: "name",
+					render: mod => <>
+						{mod.error === "missing" && <Tooltip title="Mod is missing from storage.">
+							<FileUnknownOutlined style={{ color: "#a61d24" }} />{" "}
+						</Tooltip>}
+						{mod.error === "bad_checksum" && <Tooltip title="Mod checksum missmatch.">
+							<FileExclamationOutlined style={{ color: "#a61d24" }} />{" "}
+						</Tooltip>}
+						{mod.title || mod.name}
+					</>,
+					defaultSortOrder: "ascend",
+					sorter: (a, b) => strcmp(a.name, b.name),
+				},
+				{
+					title: "Version",
+					dataIndex: "version",
+					sorter: (a, b) => strcmp(a.version, b.version),
+				},
+				{
+					title: "Action",
+					key: "action",
+					responsive: ["lg"],
+					render: mod => actions(mod),
+				},
+			]}
+			expandable={{
+				expandedRowRender: mod => <ModDetails mod={mod} actions={actions} />,
+				expandedRowClassName: () => "no-expanded-padding",
+			}}
+			dataSource={mods}
+			pagination={false}
+			rowKey={mod => mod.name}
+			rowClassName={mod => {
+				if (deletedMods.has(mod.name)) {
+					return "deleted";
+				}
+				if (changedMods.has(mod.name)) {
+					return "changed";
+				}
+				return "";
+			}}
+		/>
+	</>;
+}
+
+// Returns true if a change record with the given fields exist
+function hasChange(changes, field) {
+	return changes.some(change => Object.entries(field).every(([key, value]) => change[key] === value));
+}
+
+// picks out the value property from settings.
+function pickValue(map) {
+	return [...map].map(([name, value]) => [name, value.value]);
+}
+
+function groupToMap(array, fn) {
+	const map = new Map();
+	for (let index = 0; index < array.length; index++) {
+		let key = fn(array[index], index, array);
+		if (!map.has(key)) {
+			map.set(key, []);
+		}
+		map.get(key).push(array[index]);
+	}
+	return map;
+}
+
+function SettingsTable(props) {
+	const [modList] = useModList();
+	const modsInPack = new Set([...props.modPack.mods.values()].map(mod => `${mod.name}_${mod.version}`));
+	const modTitles = new Map(modList
+		.filter(mod => modsInPack.has(`${mod.name}_${mod.version}`))
+		.map(mod => [`${mod.name}`, mod.title])
+	);
+
+	const types = ["bool-setting", "int-setting", "double-setting", "string-setting"];
+	let prototypes = Object.entries(props.prototypes)
+		.filter(([type, _]) => types.includes(type))
+		.flatMap(([_, settingPrototypes]) => Object.values(settingPrototypes))
+	;
+
+	function controls(scope, storedFields) {
+		let fields = new Map(prototypes
+			.filter(p => p.setting_type === scope)
+			.sort((a, b) => strcmp(a.mod, b.mod) || strcmp(a.order || "", b.order || "") || strcmp(a.name, b.name))
+			.map(p => [p.name, p])
+		);
+		for (let [name, value] of storedFields) {
+			if (!fields.has(name)) {
+				let type;
+				if (typeof value === "boolean") {
+					type = "bool-setting";
+				} else if (typeof value === "number") {
+					type = "double-setting";
+				} else if (typeof value === "string") {
+					type = "string-setting";
+				} else {
+					throw Error(`Unhandled setting type ${typeof value}`);
+				}
+				fields.set(name, { mod: "Unknown Fields", name, setting_type: scope, type });
+			}
+		}
+		let fieldsByMod = groupToMap([...fields.values()], prototype => prototype.mod);
+		return [...fieldsByMod].map(([mod, fieldsForMod]) => <Fragment key={mod}>
+			<Typography.Title level={5}>
+				{props.locale.get(`mod-name.${mod}`) || modTitles.get(mod) || mod}
+			</Typography.Title>
+			{[...fieldsForMod].map(prototype => {
+				const name = prototype.name;
+				const isBoolean = prototype.type === "bool-setting";
+				const isNumber = ["int-setting", "double-setting"].includes(prototype.type);
+				const isSelect = Boolean(prototype.allowed_values);
+				let input;
+				if (isBoolean) {
+					input = <Checkbox />;
+				} else if (isSelect) {
+					const options = Object.values(prototype.allowed_values).map(value => ({
+						value,
+						label: props.locale.get(`string-mod-setting.${name}-${value}`) || value,
+					}));
+					input = <Select options={options} />;
+				} else {
+					input = <Input />;
+				}
+
+				return <Form.Item
+					name={[scope, name]}
+					key={`${scope} ${name}`}
+					label={props.locale.get(`mod-setting-name.${name}`) || name}
+					tooltip={props.locale.get(`mod-setting-description.${name}`)}
+					labelCol={{ span: 8 }}
+					wrapperCol={{ span: 16 }}
+					valuePropName={isBoolean ? "checked" : "value"}
+					rules={[...isNumber ? [{
+						validator(rule, value) {
+							if (Number.isNaN(Number(value))) {
+								return Promise.reject(new Error("Must be a number"));
+							}
+							return Promise.resolve();
+						},
+					}] : []]}
+					className={
+						hasChange(props.changes, { type: "settings.set", scope, name }) && "changed"
+					}
+				>
+					{input}
+				</Form.Item>;
+			})}
+		</Fragment>);
+	}
+
+	return <>
+		<SectionHeader
+			title="Mod Settings"
+		/>
+		<Typography.Paragraph>Run export data on an instance to update available fields.</Typography.Paragraph>
+		<Typography.Title level={4}>Startup</Typography.Title>
+		{controls("startup", pickValue(props.modPack.settings["startup"]))}
+
+		<Typography.Title level={4}>Map</Typography.Title>
+		<Typography.Paragraph>These only apply for new saves.</Typography.Paragraph>
+		{controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))}
+
+		<Typography.Title level={4}>Per Player</Typography.Title>
+		<Typography.Paragraph>Default settings for new players, these only apply for new saves.</Typography.Paragraph>
+		{controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))}
+	</>;
+}
+
+function CopyButton(props) {
+	let clipboard = useClipboard();
+	let [copiedVisible, setCopiedVisible] = useState(false);
+	let copiedTimeout = useRef();
+
+	useEffect(() => () => {
+		clearTimeout(copiedTimeout.current);
+		setCopiedVisible(false);
+	}, []);
+
+	let denied = clipboard.readPermissionState === "denied";
+	return <Tooltip title={denied ? clipboard.deniedReason : "Copied!"} visible={denied ? undefined : copiedVisible}>
+		<Button disabled={denied} onClick={() => {
+			try {
+				clipboard.writeText(props.content);
+				clearTimeout(copiedTimeout.current);
+				setCopiedVisible(undefined);
+				copiedTimeout.current = setTimeout(() => {
+					setCopiedVisible(false);
+				}, 3000);
+			} catch (err) {
+				logger.error(`Writing to clipboard failed:\n${err.stack}`);
+				notify("Unable to copy to clipboard", "error", "Use ordinary select and copy instead.");
+			}
+		}}>Copy</Button>
+	</Tooltip>;
+}
+
+function ExportButton(props) {
+	let [visible, setVisible] = useState(false);
+	function close() {
+		setVisible(false);
+	}
+
+	let exportString = props.modPack.toModPackString();
+
+	return <>
+		<Button icon={<ExportOutlined />} onClick={() => { setVisible(true); }}>Export to string</Button>
+		<Modal
+			title="Mod Pack String"
+			visible={visible}
+			onOk={close}
+			onCancel={close}
+			destroyOnClose
+			footer={<Space>
+				<CopyButton content={exportString} />
+				<Button onClick={close}>Close</Button>
+			</Space>}
+		>
+			<Text code className="code">
+				{exportString}
+			</Text>
+		</Modal>
+	</>;
+}
+
+function useExportedAsset(modPack, asset) {
+	let [assetData, setAssetData] = useState(asset === "locale" ? new Map() : {});
+	let assetFilename;
+	if (modPack instanceof libData.ModPack && modPack.exportManifest && modPack.exportManifest.assets[asset]) {
+		assetFilename = modPack.exportManifest.assets[asset];
+	}
+	useEffect(() => {
+		async function load() {
+			if (!assetFilename) {
+				return;
+			}
+
+			let response = await fetch(`${staticRoot}static/${assetFilename}`);
+			if (response.ok) {
+				let data = await response.json();
+				setAssetData(asset === "locale" ? new Map(data) : data);
+			} else {
+				logger.error(`Error loading mod pack asset "${asset}" (${response.status}): ${await response.text()}`);
+			}
+		}
+
+		load().catch(notifyErrorHandler(`Loading mod pack asset "${asset}" failed`));
+	}, [assetFilename]);
+
+	return assetData;
+}
+
+export default function ModPackViewPage() {
+	let account = useAccount();
+	let history = useHistory();
+	let control = useContext(ControlContext);
+	let params = useParams();
+	let modPackId = Number(params.id);
+
+	const [form] = Form.useForm();
+	let [modPack] = useModPack(modPackId);
+	let prototypes = useExportedAsset(modPack, "settings");
+	let locale = useExportedAsset(modPack, "locale");
+	let [changes, setChanges] = useState([]);
+	let [modifiedModPack, setModifiedModPack] = useState(modPack);
+	let syncTimeout = useRef();
+
+	useEffect(() => () => { clearTimeout(syncTimeout.current); });
+
+	function pushChange(change) {
+		setChanges(oldChanges => {
+			const newChanges = [...oldChanges];
+			if (
+				newChanges.length
+				&& newChanges.slice(-1)[0].type === change.type
+				&& newChanges.slice(-1)[0].name === change.name
+				&& newChanges.slice(-1)[0].scope === change.scope
+			) {
+				newChanges.splice(-1, 1, change);
+			} else {
+				newChanges.push(change);
+			}
+			return newChanges;
+		});
+	}
+
+	function revertChange(change) {
+		const newChanges = [...changes];
+		const index = newChanges.findLastIndex(reference => (
+			change.type === reference.type && change.name === reference.name
+		));
+		if (index === -1) {
+			logger.error(`Unable to revert ${change.type} ${change.name}: change not found`);
+			return;
+		}
+		newChanges.splice(index, 1);
+		setChanges(newChanges);
+	}
+
+	function syncPack(fields) {
+		for (let field of fields) {
+			if (!field.touched || field.validating || field.errors.length) {
+				continue;
+			}
+			if (["name", "description", "factorioVersion"].includes(field.name[0])) {
+				if (modifiedModPack[field.name[0]] !== field.value) {
+					pushChange({ type: field.name[0], value: field.value });
+				}
+
+			} else if (["startup", "runtime-global", "runtime-per-user"].includes(field.name[0])) {
+				let [scope, settingName] = field.name;
+				let value = field.value;
+				if (typeof modPack.settings[scope].get(settingName).value === "number") {
+					value = Number(value);
+					if (Number.isNaN(value)) {
+						continue;
+					}
+				}
+				if (modifiedModPack.settings[scope].get(settingName).value !== value) {
+					pushChange({ type: "settings.set", scope, name: settingName, value: { value }});
+				}
+			}
+		}
+	}
+
+	useEffect(() => {
+		// Loading or invalid id
+		if (!(modPack instanceof libData.ModPack)) {
+			setModifiedModPack(modPack);
+			return;
+		}
+
+		const modified = new libData.ModPack(modPack.toJSON());
+		for (let change of changes) {
+			if (change.type === "name") {
+				modified.name = change.value;
+			} else if (change.type === "description") {
+				modified.description = change.value;
+			} else if (change.type === "factorioVersion") {
+				modified.factorioVersion = change.value;
+			} else if (change.type === "mods.set") {
+				modified.mods.set(change.name, change.value);
+			} else if (change.type === "mods.delete") {
+				modified.mods.delete(change.name);
+			} else if (change.type === "settings.set") {
+				modified.settings[change.scope].set(change.name, change.value);
+			} else if (change.type === "settings.delete") {
+				modified.settings[change.scope].delete(change.name);
+			} else {
+				throw new Error(`Unknown change type ${change.type}`);
+			}
+		}
+		setModifiedModPack(modified);
+	}, [modPack, changes]);
+
+	let nav = [{ name: "Mods", path: "/mods" }, { name: "Mod Packs" }, { name: modPack.name || modPackId }];
+	if (modifiedModPack.loading) {
+		return <PageLayout nav={nav}>
+			<PageHeader
+				className="site-page-header"
+				title={modPackId}
+			/>
+			<Spin size="large" />
+		</PageLayout>;
+	}
+
+	if (modifiedModPack.missing) {
+		return <PageLayout nav={nav}>
+			<PageHeader
+				className="site-page-header"
+				title="Mod Pack Not Found"
+			/>
+			<p>Mod pack with id {modPackId} was not found on the master server.</p>
+		</PageLayout>;
+	}
+
+	return <PageLayout nav={nav}>
+		<PageHeader
+			className="site-page-header"
+			title={modPack.name}
+			extra={<Space>
+				<ExportButton modPack={modifiedModPack}/>
+				{account.hasPermission("core.mod-pack.delete") && <Popconfirm
+					title="Delete mod pack and all of its settings?"
+					placement="bottomRight"
+					okText="Delete"
+					okButtonProps={{ danger: true }}
+					onConfirm={() => {
+						libLink.messages.deleteModPack.send(
+							control, { id: modPack.id }
+						).then(() => {
+							history.push("/mods");
+						}).catch(notifyErrorHandler("Error deleting mod pack"));
+					}}
+				>
+					<Button danger icon={<DeleteOutlined />}>Delete</Button>
+				</Popconfirm>}
+			</Space>}
+		/>
+		<Form
+			form={form}
+			onFieldsChange={(changedFields, allFields) => {
+				clearTimeout(syncTimeout.current);
+				if (typeof changedFields[0].value === "boolean") {
+					syncPack(allFields);
+				} else {
+					// Defer sync if this is not a checkbox to make the UI responsive.
+					syncTimeout.current = setTimeout(syncPack.bind(undefined, allFields), 500);
+				}
+			}}
+			initialValues={{
+				name: modPack.name,
+				description: modPack.description,
+				factorioVersion: modPack.factorioVersion,
+				"startup": Object.fromEntries(pickValue(modPack.settings["startup"])),
+				"runtime-global": Object.fromEntries(pickValue(modPack.settings["runtime-global"])),
+				"runtime-per-user": Object.fromEntries(pickValue(modPack.settings["runtime-per-user"])),
+			}}
+		>
+			<Descriptions size="small" bordered column={{ xs: 1, sm: 2, lg: 3 }}>
+				<Descriptions.Item span={3} label="Name">
+					<Form.Item noStyle name="name">
+						<Input
+							className={hasChange(changes, { type: "name" }) && "changed"}
+						/>
+					</Form.Item>
+				</Descriptions.Item>
+				<Descriptions.Item span={3} label="Description">
+					<Form.Item noStyle name="description">
+						<Input.TextArea
+							autoSize={{ minRows: 2 }}
+							className={hasChange(changes, { type: "description" }) && "changed"}
+						/>
+					</Form.Item>
+				</Descriptions.Item>
+				<Descriptions.Item label="Mods">
+					{modifiedModPack.mods.size}
+				</Descriptions.Item>
+				<Descriptions.Item label="Factorio Version">
+					<Form.Item
+						name="factorioVersion"
+						style={{ marginBottom: 0 }}
+						rules={[{
+							pattern: /^\d+\.\d+\.\d+?$/,
+							message: "Must be an a.b.c version number.",
+						}]}
+					>
+						<Input
+							className={hasChange(changes, { type: "factorioVersion" }) && "changed"}
+						/>
+					</Form.Item>
+				</Descriptions.Item>
+			</Descriptions>
+			<ModsTable modPack={modifiedModPack} changes={changes} onChange={pushChange} onRevert={revertChange} />
+			<SettingsTable
+				modPack={modifiedModPack}
+				changes={changes}
+				onChange={pushChange}
+				onRevert={revertChange}
+				prototypes={prototypes}
+				locale={locale}
+			/>
+		</Form>
+
+		<div
+			className="ant-notification ant-notification-bottom"
+			style={{
+				visibility: changes.length ? "visible" : "hidden",
+				inset: "auto auto 24px auto",
+				position: "sticky",
+				marginTop: 16,
+			}}
+		>
+			<div className="ant-notification-notice" style={{ width: "auto", maxWidth: 384, marginBottom: 0 }}>
+				<Row style={{ alignItems: "center", rowGap: 12 }}>
+					<Col flex="auto">You have unsaved changes</Col>
+					<Col flex="0 0 auto" style={{ marginLeft: "auto" }}>
+						<Space>
+							<Button
+								onClick={() => {
+									form.resetFields();
+									setChanges([]);
+								}}
+							>Revert</Button>
+							<Button
+								type="primary"
+								onClick={() => {
+									libLink.messages.updateModPack.send(
+										control, { mod_pack: modifiedModPack.toJSON() }
+									).then(() => {
+										form.resetFields();
+										setChanges([]);
+									}).catch(notifyErrorHandler("Error deleting mod pack"));
+								}}
+							>Apply</Button>
+						</Space>
+					</Col>
+				</Row>
+			</div>
+		</div>
+	</PageLayout>;
+
+}

packages/web_ui/src/components/ModsPage.jsx

@@ -0,0 +1,277 @@
+import React, { Fragment, useState, useContext } from "react";
+import { useHistory } from "react-router-dom";
+import { Button, Form, Input, Modal, PageHeader, Popconfirm, Space, Table, Typography, Upload } from "antd";
+import ImportOutlined from "@ant-design/icons/ImportOutlined";
+import PlusOutlined from "@ant-design/icons/PlusOutlined";
+import UploadOutlined from "@ant-design/icons/UploadOutlined";
+
+import { libData, libHelpers, libLink, libLogging } from "@clusterio/lib";
+
+import { useAccount } from "../model/account";
+import { useModList } from "../model/mods";
+import { useModPackList } from "../model/mod_pack";
+import { notifyErrorHandler } from "../util/notify";
+import ControlContext from "./ControlContext";
+import PageLayout from "./PageLayout";
+import PluginExtra from "./PluginExtra";
+import SectionHeader from "./SectionHeader";
+import ModDetails from "./ModDetails";
+
+const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare;
+
+const { logger } = libLogging;
+
+
+function ImportModPackButton() {
+	let control = useContext(ControlContext);
+	let history = useHistory();
+	let [visible, setVisible] = useState(false);
+	let [form] = Form.useForm();
+	function close() {
+		setVisible(false);
+	}
+	return <>
+		<Button icon={<ImportOutlined />} onClick={() => { setVisible(true); }}>Import string</Button>
+		<Modal
+			title="Import Mod Pack String"
+			visible={visible}
+			okText="Import"
+			cancelText="Cancel"
+			onCancel={() => { setVisible(false); }}
+			onOk={() => {
+				(async () => {
+					let values;
+					try {
+						values = await form.validateFields();
+					} catch {
+						return; // Validation failed
+					}
+					const modPack = libData.ModPack.fromModPackString(values.string);
+					await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() });
+					history.push(`/mods/mod-packs/${modPack.id}/view`);
+				})().catch(notifyErrorHandler("Error creating mod pack"));
+			}}
+		>
+			<Form form={form} layout="vertical">
+				<Form.Item
+					name="string"
+					rules={[
+						{ required: true },
+						{ async validator(rule, value) {
+							if (value) {
+								libData.ModPack.fromModPackString(value);
+							}
+						}},
+					]}
+				>
+					<Input.TextArea autoSize={{ minRows: 6, maxRows: 6 }} />
+				</Form.Item>
+			</Form>
+		</Modal>
+	</>;
+}
+
+function CreateModPackButton() {
+	let control = useContext(ControlContext);
+	let history = useHistory();
+	let [visible, setVisible] = useState(false);
+	let [form] = Form.useForm();
+	function close() {
+		setVisible(false);
+	}
+	return <>
+		<Button type="primary" icon={<PlusOutlined />} onClick={() => { setVisible(true); }}>Create</Button>
+		<Modal
+			title="Create Mod Pack"
+			visible={visible}
+			okText="Create"
+			cancelText="Cancel"
+			onCancel={() => { setVisible(false); }}
+			onOk={() => {
+				(async () => {
+					let values;
+					try {
+						values = await form.validateFields();
+					} catch {
+						return; // Validation failed
+					}
+					const modPack = new libData.ModPack();
+					modPack.name = values.name;
+					modPack.factorioVersion = values.factorioVersion;
+					if (values.description) { modPack.description = values.description; }
+					await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() });
+					history.push(`/mods/mod-packs/${modPack.id}/view`);
+				})().catch(notifyErrorHandler("Error creating mod pack"));
+			}}
+		>
+			<Form form={form} layout="vertical" requiredMark="optional">
+				<Form.Item name="name" label="Name" rules={[{ required: true }]}>
+					<Input />
+				</Form.Item>
+				<Form.Item name="description" label="Description">
+					<Input.TextArea autoSize={{ minRows: 2, maxRows: 6 }} />
+				</Form.Item>
+				<Form.Item
+					name="factorioVersion"
+					label="Factorio Version"
+					rules={[{
+						required: true,
+						pattern: /^\d+\.\d+(\.\d+)?$/,
+						message: "Must be an a.b or a.b.c version number.",
+					}]}
+				>
+					<Input />
+				</Form.Item>
+			</Form>
+		</Modal>
+	</>;
+}
+
+export default function ModsPage() {
+	let account = useAccount();
+	let control = useContext(ControlContext);
+	let history = useHistory();
+	let [modList] = useModList();
+	let [modPackList] = useModPackList();
+
+	function actions(mod) {
+		return <Space>
+			{account.hasPermission("core.mod.download")
+				&& <Typography.Link
+					onClick={() => {
+						libLink.messages.downloadMod.send(
+							control, { name: mod.name, version: mod.version }
+						).then(response => {
+							let url = new URL(webRoot, document.location);
+							url.pathname += `api/stream/${response.stream_id}`;
+							document.location = url;
+						}).catch(
+							notifyErrorHandler("Error downloading save")
+						);
+					}}
+				>download</Typography.Link>
+			}
+			{account.hasPermission("core.mod.delete")
+				&& <Popconfirm
+					title="Are you sure you want to delete this mod?"
+					onConfirm={event => {
+						libLink.messages.deleteMod.send(
+							control, { name: mod.name, version: mod.version }
+						).catch(notifyErrorHandler("Error deleting mod"));
+					}}
+					okText="Delete"
+					okButtonProps={{ danger: true }}
+				>
+					<Typography.Link type="danger">delete</Typography.Link>
+				</Popconfirm>
+			}
+		</Space>;
+	}
+
+	let uploadButton;
+	if (account.hasPermission("core.mod.upload")) {
+		uploadButton = <Upload
+			name="file"
+			accept=".zip"
+			multiple
+			headers={{
+				"X-Access-Token": control.connector.token,
+			}}
+			action={`${webRoot}api/upload-mod`}
+		>
+			<Button icon={<UploadOutlined/>}>Upload</Button>
+		</Upload>;
+	}
+
+	return <PageLayout nav={[{ name: "Mods" }]}>
+		<PageHeader
+			className="site-page-header"
+			title="Mods"
+		/>
+		<SectionHeader
+			title="Mod Packs"
+			extra={<Space>
+				{account.hasPermission("core.mod-pack.create") && <ImportModPackButton />}
+				{account.hasPermission("core.mod-pack.create") && <CreateModPackButton />}
+			</Space>}
+		/>
+		<Table
+			columns={[
+				{
+					title: "Name",
+					dataIndex: "name",
+					defaultSortOrder: "ascend",
+					sorter: (a, b) => strcmp(a.name, b.name),
+				},
+				{
+					title: "Factorio Version",
+					dataIndex: "factorioVersion",
+					sorter: (a, b) => a.integerFactorioVersion - b.integerFactorioVersion,
+				},
+				{
+					title: "Mods",
+					key: "roles",
+					render: modPack => modPack.mods.size,
+				},
+			]}
+			dataSource={modPackList}
+			pagination={false}
+			rowKey={modPack => modPack.id}
+			onRow={(modPack, rowIndex) => ({
+				onClick: event => {
+					history.push(`/mods/mod-packs/${modPack.id}/view`);
+				},
+			})}
+		/>
+		<SectionHeader title="Stored Mods" extra={uploadButton} />
+		<Table
+			columns={[
+				{
+					title: "Name",
+					dataIndex: "title",
+					defaultSortOrder: "ascend",
+					sorter: (a, b) => (
+						strcmp(a.name, b.name) || a.integerVersion - b.integerVersion
+					),
+				},
+				{
+					title: "Version",
+					key: "version",
+					align: "right",
+					render: mod => <>
+						{`${mod.version} `}
+						<Typography.Text type="secondary">{`/ ${mod.factorioVersion}`}</Typography.Text>
+					</>,
+				},
+				{
+					title: "Filename",
+					dataIndex: "filename",
+					responsive: ["xl"],
+					sorter: (a, b) => strcmp(a.filename, b.filename),
+				},
+				{
+					title: "Size",
+					key: "size",
+					responsive: ["lg"],
+					render: mod => libHelpers.formatBytes(mod.size),
+					align: "right",
+					sorter: (a, b) => a.size - b.size,
+				},
+				{
+					title: "Action",
+					key: "action",
+					responsive: ["lg"],
+					render: mod => actions(mod),
+				},
+			]}
+			expandable={{
+				expandedRowRender: mod => <ModDetails mod={mod} actions={actions} />,
+				expandedRowClassName: () => "no-expanded-padding",
+			}}
+			dataSource={modList}
+			pagination={false}
+			rowKey={mod => `${mod.name}_${mod.version}`}
+		/>
+		<PluginExtra component="ModsPage" />
+	</PageLayout>;
+}

packages/web_ui/src/components/SavesList.jsx

@@ -6,7 +6,7 @@ import {
 import CaretLeftOutlined from "@ant-design/icons/CaretLeftOutlined";
 import LeftOutlined from "@ant-design/icons/LeftOutlined";
 
-import { libLink } from "@clusterio/lib";
+import { libLink, libHelpers } from "@clusterio/lib";
 
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
@@ -179,17 +179,6 @@ function TransferModal(props) {
 	</>;
 }
 
-function formatBytes(bytes) {
-	if (bytes === 0) {
-		return "0 Bytes";
-	}
-
-	let units = [" Bytes", " kB", " MB", " GB", " TB"];
-	let factor = 1000;
-	let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length);
-	return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power];
-}
-
 export default function SavesList(props) {
 	let account = useAccount();
 	let control = useContext(ControlContext);
@@ -214,7 +203,7 @@ export default function SavesList(props) {
 				title: "Size",
 				key: "size",
 				responsive: ["sm"],
-				render: save => formatBytes(save.size),
+				render: save => libHelpers.formatBytes(save.size),
 				align: "right",
 				sorter: (a, b) => a.size - b.size,
 			},

packages/web_ui/src/components/SiteLayout.jsx

@@ -49,7 +49,13 @@ export default function SiteLayout(props) {
 	let menuItems = [];
 	let menuGroups = new Map();
 	for (let { sidebarName, sidebarGroup, permission, path } of combinedPages) {
-		if (!sidebarName || permission && !account.hasPermission(permission)) {
+		if (
+			!sidebarName || permission && (
+				typeof permission === "function"
+					? !permission(account)
+					: !account.hasPermission(permission)
+			)
+		) {
 			continue;
 		}
 		if (sidebarGroup) {

packages/web_ui/src/index.css

@@ -27,7 +27,7 @@ body {
 }
 
 .site-layout {
-	overflow-x: hidden;
+	overflow-x: clip;
 }
 
 .site-layout-content-container {
@@ -73,6 +73,7 @@ body {
 
 .section-header {
 	display: flex;
+	flex-wrap: wrap;
 	align-items: center;
 	margin-top: 16px;
 	margin-bottom: 8px;
@@ -295,6 +296,10 @@ body {
 	width: 32px;
 }
 
+.code > code {
+	display: block;
+}
+
 /*
  * Render codeblock with options for syntax highlighting
  * https://user-images.githubusercontent.com/6792749/152066701-21420461-710b-4fae-a80a-89ef90a0b18f.png
@@ -314,3 +319,38 @@ body {
 	top: 8px;
 	right: 8px;
 }
+
+
+.ant-descriptions-bordered.borderless > .ant-descriptions-view {
+	border: none;
+}
+
+tr.no-expanded-padding > td,
+.ant-table.ant-table-small .ant-table-tbody > tr.no-expanded-padding > td {
+	padding: 0;
+}
+
+.ant-form-item.changed,
+input.ant-input.changed,
+textarea.ant-input.changed,
+tr.ant-table-row.changed > td {
+	background: #2a1912;
+}
+
+.ant-table-tbody > tr.ant-table-row.changed:hover > td,
+.ant-table-tbody > tr.changed > td.ant-table-cell-row-hover
+tr.ant-table-row.deleted > td {
+	background: #3a2922;
+}
+
+tr.ant-table-row.deleted {
+	color: rgba(255, 255, 255, 0.45);
+	background: #340c08;
+	text-decoration: line-through;
+}
+
+.ant-table-tbody > tr.ant-table-row.deleted:hover > td,
+.ant-table-tbody > tr.deleted > td.ant-table-cell-row-hover
+tr.ant-table-row.deleted {
+	background: #441c18;
+}

packages/web_ui/src/model/export_manifest.jsx

@@ -1,15 +1,20 @@
-import { useEffect, useState } from "react";
+import { useEffect, useContext, useState } from "react";
+
+import { libData, libLink } from "@clusterio/lib";
+import ControlContext from "../components/ControlContext";
 
 
 let exportManifestCache = null;
 let emptyCache = {};
 export function useExportManifest() {
+	const control = useContext(ControlContext);
 	let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache);
 	useEffect(() => {
 		async function load() {
-			let response = await fetch(`${webRoot}api/export-manifest`);
-			if (response.ok) {
-				exportManifestCache = await response.json();
+			let result = await libLink.messages.getDefaultModPack.send(control);
+			let modPack = new libData.ModPack(result.mod_pack);
+			if (modPack.exportManifest) {
+				exportManifestCache = modPack.exportManifest;
 				setExportManifest(exportManifestCache);
 			}
 		}

packages/web_ui/src/model/item_metadata.jsx

@@ -9,10 +9,14 @@ export function useItemMetadata() {
 	let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map());
 	useEffect(() => {
 		async function load() {
-			if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) {
+			if (
+				!exportManifest
+				|| !exportManifest.assets["item-metadata"]
+				|| !exportManifest.assets["item-spritesheet"]
+			) {
 				return;
 			}
-			let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`);
+			let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				itemMetadataCache = new Map(data);
@@ -22,7 +26,7 @@ export function useItemMetadata() {
 				for (let [name, meta] of itemMetadataCache) {
 					style.sheet.insertRule(
 						`.item-${CSS.escape(name)} {
-	background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}");
+	background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}");
 	background-repeat: no-repeat;
 	background-position: -${meta.x}px -${meta.y}px;
 	height: ${meta.size}px;

packages/web_ui/src/model/locale.jsx

@@ -9,10 +9,10 @@ export function useLocale() {
 	let [locale, setLocale] = useState(localeCache || new Map());
 	useEffect(() => {
 		async function load() {
-			if (!exportManifest["locale"]) {
+			if (!exportManifest || !exportManifest.assets["locale"]) {
 				return;
 			}
-			let response = await fetch(`${staticRoot}${exportManifest["locale"]}`);
+			let response = await fetch(`${staticRoot}static/${exportManifest.assets["locale"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				localeCache = new Map(data);

packages/web_ui/src/model/mod_pack.jsx

@@ -0,0 +1,78 @@
+import { useContext, useEffect, useState } from "react";
+
+import { libData, libLink, libLogging } from "@clusterio/lib";
+
+import ControlContext from "../components/ControlContext";
+
+const { logger } = libLogging;
+
+export function useModPack(id) {
+	let control = useContext(ControlContext);
+	let [modPack, setModPack] = useState({ loading: true });
+
+	function updateModPack() {
+		libLink.messages.getModPack.send(control, { id }).then(result => {
+			setModPack(new libData.ModPack(result.mod_pack));
+		}).catch(err => {
+			logger.error(`Failed to get mod pack: ${err}`);
+			setModPack({ missing: true });
+		});
+	}
+
+	useEffect(() => {
+		if (typeof id !== "number") {
+			setModPack({ missing: true });
+			return undefined;
+		}
+		updateModPack();
+
+		control.onModPackUpdate(id, setModPack);
+		return () => {
+			control.offModPackUpdate(id, setModPack);
+		};
+	}, [id]);
+
+	return [modPack];
+}
+
+export function useModPackList() {
+	let control = useContext(ControlContext);
+	let [modPackList, setModPackList] = useState([]);
+
+	function updateModPackList() {
+		libLink.messages.listModPacks.send(control).then(result => {
+			setModPackList(result.list.map(pack => new libData.ModPack(pack)));
+		}).catch(err => {
+			logger.error(`Failed to list mod packs:\n${err}`);
+		});
+	}
+
+	useEffect(() => {
+		updateModPackList();
+
+		function updateHandler(newModPack) {
+			setModPackList(oldList => {
+				let newList = oldList.concat();
+				let index = newList.findIndex(u => u.id === newModPack.id);
+				if (!newModPack.isDeleted) {
+					if (index !== -1) {
+						newList[index] = newModPack;
+					} else {
+						newList.push(newModPack);
+					}
+				} else if (index !== -1) {
+					newList.splice(index, 1);
+				}
+				return newList;
+			});
+		}
+
+		control.onModPackUpdate(null, updateHandler);
+		return () => {
+			control.offModPackUpdate(null, updateHandler);
+		};
+	}, []);
+
+	return [modPackList];
+
+}

packages/web_ui/src/model/mods.jsx

@@ -0,0 +1,50 @@
+import { useContext, useEffect, useState } from "react";
+
+import { libData, libLink, libLogging } from "@clusterio/lib";
+
+import ControlContext from "../components/ControlContext";
+
+const { logger } = libLogging;
+
+
+export function useModList() {
+	let control = useContext(ControlContext);
+	let [modList, setModList] = useState([]);
+
+	function updateModList() {
+		libLink.messages.listMods.send(control).then(result => {
+			setModList(result.list.map(mod => new libData.ModInfo(mod)));
+		}).catch(err => {
+			logger.error(`Failed to list mods:\n${err}`);
+		});
+	}
+
+	useEffect(() => {
+		updateModList();
+
+		function updateHandler(newMod) {
+			setModList(oldList => {
+				let newList = oldList.concat();
+				let index = newList.findIndex(mod => mod.name === newMod.name && mod.version === newMod.version);
+				if (!newMod.isDeleted) {
+					if (index !== -1) {
+						newList[index] = newMod;
+					} else {
+						newList.push(newMod);
+					}
+				} else if (index !== -1) {
+					newList.splice(index, 1);
+				}
+				return newList;
+			});
+		}
+
+		control.onModUpdate(null, updateHandler);
+		return () => {
+			control.offModUpdate(null, updateHandler);
+		};
+	}, []);
+
+	return [modList];
+}
+

packages/web_ui/src/pages.jsx

@@ -5,6 +5,8 @@ import SlavesPage from "./components/SlavesPage";
 import SlaveViewPage from "./components/SlaveViewPage";
 import InstancesPage from "./components/InstancesPage";
 import InstanceViewPage from "./components/InstanceViewPage";
+import ModPackViewPage from "./components/ModPackViewPage";
+import ModsPage from "./components/ModsPage";
 import UsersPage from "./components/UsersPage";
 import UserViewPage from "./components/UserViewPage";
 import RolesPage from "./components/RolesPage";
@@ -41,6 +43,17 @@ export const pages = [
 		sidebarPath: "/instances",
 		content: <InstanceViewPage />,
 	},
+	{
+		path: "/mods",
+		sidebarName: "Mods",
+		permission: account => account.hasAnyPermission("core.mod.list", "core.mod-pack.list"),
+		content: <ModsPage />,
+	},
+	{
+		path: "/mods/mod-packs/:id/view",
+		sidebarPath: "/mods",
+		content: <ModPackViewPage />,
+	},
 	{
 		path: "/users",
 		sidebarName: "Users",

packages/web_ui/src/util/clipboard.jsx

@@ -0,0 +1,69 @@
+import { useEffect, useState } from "react";
+import { EventEmitter } from "events";
+
+class Permission extends EventEmitter {
+	status;
+
+	constructor(name) {
+		super();
+		navigator.permissions.query({ name }).then(
+			status => {
+				this.status = status;
+				this.emit("change", this.status.state);
+
+				if (status.addEventListener) {
+					status.addEventListener("change", () => {
+						this.emit("change", this.status.state);
+					});
+				}
+			},
+			// Pretend the clipboard will work if this query fails.
+			() => {
+				this.status = { state: "prompt" };
+				this.emit("change", "prompt");
+			}
+		);
+	}
+
+	get state() {
+		return this.status.state || "prompt";
+	}
+}
+
+let readPermission = new Permission("clipboard-read");
+let writePermission = new Permission("clipboard-write");
+
+// Wrapper for navigator.clipboard suitable for use in React
+export function useClipboard() {
+	let [readPermissionState, setReadPermissionState] = useState(readPermission.state);
+	let [writePermissionState, setWritePermissionState] = useState(writePermission.state);
+
+	useEffect(() => {
+		readPermission.on("change", setReadPermissionState);
+		writePermission.on("change", setWritePermissionState);
+		return () => {
+			readPermission.off("change", setReadPermissionState);
+			writePermission.off("change", setWritePermissionState);
+		};
+	}, []);
+
+	return {
+		readPermissionState: readPermissionState,
+		read: async () => await navigator.clipboard.read(),
+		readText: async () => await navigator.clipboard.readText(),
+		writePermissionState: writePermissionState,
+		write: async (data) => await navigator.clipboard.write(data),
+		writeText: async (text) => await navigator.clipboard.writeText(text),
+
+		// Give a possible reason for why the clipboard is not available
+		deniedReason: () => {
+			if (window.isSecureContext === false) {
+				if (window.location.protocol !== "https:") {
+					return "Clipboard interaction is blocked due this page not being served over https.";
+				}
+				return "Clipboard interaction is blocked due this page not being a secure context.";
+			}
+			return "Clipboard interaction is blocked by the browser.";
+		},
+	};
+}

packages/web_ui/src/util/websocket.jsx

@@ -1,4 +1,4 @@
-import { libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib";
+import { libData, libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib";
 const { logger } = libLogging;
 import packageJson from "../../package.json";
 
@@ -86,6 +86,8 @@ export class Control extends libLink.Link {
 		this.slaveUpdateHandlers = new Map();
 		this.instanceUpdateHandlers = new Map();
 		this.saveListUpdateHandlers = new Map();
+		this.modPackUpdateHandlers = new Map();
+		this.modUpdateHandlers = new Map();
 		this.userUpdateHandlers = new Map();
 		this.logHandlers = new Map();
 
@@ -102,6 +104,12 @@ export class Control extends libLink.Link {
 			this.updateSaveListSubscriptions().catch(err => logger.error(
 				`Unexpected error updating save list subscriptions:\n${err.stack}`
 			));
+			this.updateModPackSubscriptions().catch(err => logger.error(
+				`Unexpected error updating mod pack subscriptions:\n${err.stack}`
+			));
+			this.updateModSubscriptions().catch(err => logger.error(
+				`Unexpected error updating mod subscriptions:\n${err.stack}`
+			));
 			this.updateUserSubscriptions().catch(err => logger.error(
 				`Unexpected error updating user subscriptions:\n${err.stack}`
 			));
@@ -319,6 +327,122 @@ export class Control extends libLink.Link {
 		});
 	}
 
+	async modPackUpdateEventHandler(message) {
+		let modPack = new libData.ModPack(message.data.mod_pack);
+		let handlers = [].concat(
+			this.modPackUpdateHandlers.get(null) || [],
+			this.modPackUpdateHandlers.get(modPack.id) || [],
+		);
+		for (let handler of handlers) {
+			handler(modPack);
+		}
+	}
+
+	async onModPackUpdate(id, handler) {
+		if (id !== null && typeof id !== "number") {
+			throw new Error("Invalid mod pack id");
+		}
+
+		let handlers = this.modPackUpdateHandlers.get(id);
+		if (!handlers) {
+			handlers = [];
+			this.modPackUpdateHandlers.set(id, handlers);
+		}
+
+		handlers.push(handler);
+
+		if (handlers.length === 1) {
+			await this.updateModPackSubscriptions();
+		}
+	}
+
+	async offModPackUpdate(id, handler) {
+		let handlers = this.modPackUpdateHandlers.get(id);
+		if (!handlers || !handlers.length) {
+			throw new Error(`No handlers for mod pack ${id} exist`);
+		}
+
+		let index = handlers.lastIndexOf(handler);
+		if (index === -1) {
+			throw new Error(`Given handler is not registered for mod pack ${id}`);
+		}
+
+		handlers.splice(index, 1);
+		if (!handlers.length) {
+			this.modPackUpdateHandlers.delete(id);
+			await this.updateModPackSubscriptions();
+		}
+	}
+
+	async updateModPackSubscriptions() {
+		if (!this.connector.connected) {
+			return;
+		}
+
+		await libLink.messages.setModPackSubscriptions.send(this, {
+			all: this.modPackUpdateHandlers.has(null),
+			mod_pack_ids: [...this.modPackUpdateHandlers.keys()].filter(k => k !== null),
+		});
+	}
+
+	async modUpdateEventHandler(message) {
+		let mod = new libData.ModInfo(message.data.mod);
+		let handlers = [].concat(
+			this.modUpdateHandlers.get(null) || [],
+			this.modUpdateHandlers.get(mod.name) || []
+		);
+		for (let handler of handlers) {
+			handler(mod);
+		}
+	}
+
+	async onModUpdate(name, handler) {
+		if (name !== null && typeof name !== "string") {
+			throw new Error("Invalid mod name");
+		}
+
+		let handlers = this.modUpdateHandlers.get(name);
+		if (!handlers) {
+			handlers = [];
+			this.modUpdateHandlers.set(name, handlers);
+		}
+
+		handlers.push(handler);
+
+		if (handlers.length === 1) {
+			await this.updateModSubscriptions();
+		}
+	}
+
+	async offModUpdate(name, handler) {
+		let handlers = this.modUpdateHandlers.get(name);
+		if (!handlers || !handlers.length) {
+			throw new Error(`No handlers for mod ${name} exist`);
+		}
+
+		let index = handlers.lastIndexOf(handler);
+		if (index === -1) {
+			throw new Error(`Given handler is not registered for mod ${name}`);
+		}
+
+		handlers.splice(index, 1);
+		if (!handlers.length) {
+			this.modUpdateHandlers.delete(name);
+			await this.updateModSubscriptions();
+		}
+	}
+
+	async updateModSubscriptions() {
+		if (!this.connector.connected) {
+			return;
+		}
+
+		await libLink.messages.setModSubscriptions.send(this, {
+			all: this.modUpdateHandlers.has(null),
+			mod_names: [...this.modUpdateHandlers.keys()].filter(k => k !== null),
+		});
+	}
+
 	async userUpdateEventHandler(message) {
 		let handlers = [].concat(
 			this.userUpdateHandlers.get(null) || [],

packages/web_ui/webpack.common.js

@@ -95,6 +95,8 @@ module.exports = (env = {}) => ({
 	resolve: {
 		extensions: [".js", ".json", ".jsx"],
 		fallback: {
+			"crypto": false,
+
 			// Required for winston
 			"util": require.resolve("util/"),
 			"os": require.resolve("os-browserify/browser"),

test/file/empty_mod/info.json

@@ -0,0 +1,9 @@
+{
+    "name": "empty_mod",
+    "version": "1.0.0",
+    "title": "An Empty Mod",
+    "author": "Me",
+    "description": "An empty mod for testing",
+    "factorio_version": "1.1",
+    "dependencies": []
+}

test/integration/clusterio.js

@@ -5,6 +5,9 @@ const path = require("path");
 const events = require("events");
 const phin = require("phin");
 
+const libBuildMod = require("@clusterio/lib/build_mod");
+const libData = require("@clusterio/lib/data");
+const libHash = require("@clusterio/lib/hash");
 const libLink = require("@clusterio/lib/link");
 const libUsers = require("@clusterio/lib/users");
 const { wait } = require("@clusterio/lib/helpers");
@@ -303,13 +306,19 @@ describe("Integration of Clusterio", function() {
 				let exportPath = path.join("temp", "test", "static");
 				await fs.remove(exportPath);
 				await execCtl("instance export-data test");
-				let response = await get("/api/export-manifest");
-				let manifest = JSON.parse(response.body);
-				assert(Object.keys(manifest).length > 1, "Export manifest is empty");
-				assert(
-					await fs.exists(path.join(exportPath, "..", manifest["item-metadata"])),
-					"Export was not written to filesystem"
-				);
+				let result = await libLink.messages.getDefaultModPack.send(getControl());
+				let modPack = new libData.ModPack(result.mod_pack);
+				let assets = modPack.exportManifest.assets;
+				assert(Object.keys(assets).length > 1, "Export assets is empty");
+				for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) {
+					assert(assets[key], `Missing ${key} from assets`);
+					assert(
+						await fs.exists(path.join(exportPath, assets[key])),
+						`Manifest entry for ${key} was not written to filesystem`
+					);
+				}
+				let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, assets["prototypes"])));
+				assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types");
 				await checkInstanceStatus(44, "stopped");
 			});
 		});
@@ -818,6 +827,180 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("mod-pack create", function() {
+			it("should create a mod-pack", async function() {
+				await execCtl("mod-pack create empty-pack 1.1.0");
+				let response = await libLink.messages.listModPacks.send(getControl());
+				assert(response.list.some(modPack => modPack.name === "empty-pack"), "created pack is not in the list");
+			});
+			it("should allow setting all fields", async function() {
+				await execCtl(
+					"mod-pack create full-pack 0.17.59 " +
+					"--description Description " +
+					"--mods empty_mod:1.0.0 " +
+					"--bool-setting startup MyBool true " +
+					"--int-setting runtime-global MyInt 1235 " +
+					"--double-setting runtime-global MyDouble 12.25 " +
+					"--string-setting runtime-per-user MyString a-string"
+				);
+				let response = await libLink.messages.listModPacks.send(getControl());
+				let modPack = response.list.find(entry => entry.name === "full-pack");
+				assert(modPack, "created mod pack not found");
+				let reference = new libData.ModPack();
+				reference.id = modPack.id;
+				reference.name = "full-pack";
+				reference.description = "Description";
+				reference.factorioVersion = "0.17.59";
+				reference.mods.set("empty_mod", { name: "empty_mod", enabled: true, version: "1.0.0" });
+				reference.settings["startup"].set("MyBool", { value: true });
+				reference.settings["runtime-global"].set("MyInt", { value: 1235 });
+				reference.settings["runtime-global"].set("MyDouble", { value: 12.25 });
+				reference.settings["runtime-per-user"].set("MyString", { value: "a-string" });
+				assert.deepEqual(new libData.ModPack(modPack), reference);
+			});
+		});
+
+		describe("mod-pack list", function() {
+			it("runs", async function() {
+				await execCtl("mod-pack list");
+			});
+		});
+
+		describe("mod-pack show", function() {
+			it("runs", async function() {
+				await execCtl("mod-pack show empty-pack");
+			});
+		});
+
+		describe("mod-pack import/export", function() {
+			it("should should roundtrip a mod-pack", async function() {
+				let reference = new libData.ModPack();
+				reference.name = "imported-pack";
+				reference.description = "Description";
+				reference.factorioVersion = "0.17.59";
+				reference.mods.set("empty_mod", { name: "empty_mod", enabled: true, version: "1.0.0" });
+				reference.settings["startup"].set("MyBool", { value: true });
+				reference.settings["runtime-global"].set("MyInt", { value: 1235 });
+				reference.settings["runtime-global"].set("MyDouble", { value: 12.25 });
+				reference.settings["runtime-per-user"].set("MyString", { value: "a-string" });
+				await execCtl(`mod-pack import ${reference.toModPackString()}`);
+				const result = await execCtl("mod-pack export imported-pack");
+				const roundtrip = libData.ModPack.fromModPackString(result.stdout.trim());
+				roundtrip.id = reference.id;
+				assert.deepEqual(roundtrip, reference);
+			});
+		});
+
+		describe("mod-pack edit", function() {
+			it("runs", async function() {
+				await execCtl("mod-pack edit full-pack --factorio-version 1.2.0");
+				let response = await libLink.messages.listModPacks.send(getControl());
+				let modPack = response.list.find(entry => entry.name === "full-pack");
+				assert(modPack, "created mod pack not found");
+				assert.equal(modPack.factorio_version, "1.2.0");
+			});
+		});
+
+		describe("mod-pack delete", function() {
+			it("deletes the pack", async function() {
+				await execCtl("mod-pack delete full-pack");
+				let response = await libLink.messages.listModPacks.send(getControl());
+				let modPack = response.list.find(entry => entry.name === "full-pack");
+				assert(!modPack, "mod pack not deleted");
+			});
+		});
+
+		describe("mod upload", function() {
+			it("uploads a mod", async function() {
+				await libBuildMod.build({
+					build: true,
+					pack: true,
+					sourceDir: path.join("test", "file", "empty_mod"),
+					outputDir: path.join("temp", "test"),
+				});
+				await execCtl("mod upload empty_mod_1.0.0.zip");
+				assert(
+					await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")),
+					"mod not present in mods directory"
+				);
+			});
+		});
+
+		describe("mod show", function() {
+			it("gives details of a mod", async function() {
+				let result = await execCtl("mod show empty_mod 1.0.0");
+				let hash = await libHash.hashFile(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip"));
+				let stat = await fs.stat(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip"));
+				assert.equal(
+					result.stdout,
+					"name: empty_mod\n" +
+					"version: 1.0.0\n" +
+					"title: An Empty Mod\n" +
+					"author: Me\n" +
+					"description: An empty mod for testing\n" +
+					"factorio_version: 1.1\n" +
+					"dependencies:\n" +
+					`size: ${stat.size}\n` +
+					`sha1: ${hash}\n`,
+				);
+			});
+		});
+
+		describe("mod list", function() {
+			it("shows the list of mods", async function() {
+				let result = await execCtl("mod list");
+				assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the list");
+			});
+		});
+
+		describe("mod search", function() {
+			it("searches the list of mods", async function() {
+				let result = await execCtl("mod search 1.1 name:empty_mod");
+				assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the result");
+			});
+		});
+
+		describe("mod download", function() {
+			it("downloads a mod", async function() {
+				await fs.unlink(path.join("temp", "test", "empty_mod_1.0.0.zip"));
+				await execCtl("mod download empty_mod 1.0.0");
+				assert(
+					await fs.pathExists(path.join("temp", "test", "empty_mod_1.0.0.zip")),
+					"mod not downloaded to cwd"
+				);
+			});
+		});
+
+		describe("mod delete", function() {
+			it("deletes a mod", async function() {
+				await execCtl("mod delete empty_mod 1.0.0");
+				assert(
+					!await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")),
+					"mod still present in mods dir"
+				);
+			});
+		});
+
+		describe("modUpdateEventHandler()", function() {
+			it("should have triggered for the previous mod updates", function() {
+				let eventsToCheck = new Set(["updated", "deleted"]);
+				let eventsNotSeen = new Set(eventsToCheck);
+
+				for (let modUpdate of getControl().modUpdates) {
+					if (modUpdate.name !== "empty_mod" || modUpdate.version !== "1.0.0") {
+						continue;
+					}
+					if (modUpdate.is_deleted) {
+						eventsNotSeen.delete("deleted");
+					} else {
+						eventsNotSeen.delete("updated");
+					}
+				}
+
+				assert(eventsNotSeen.size === 0, `Did not see the events ${[...eventsNotSeen]}`);
+			});
+		});
+
 		describe("permission list", function() {
 			it("runs", async function() {
 				await execCtl("permission list");

test/integration/index.js

@@ -8,6 +8,7 @@ const phin = require("phin");
 const util = require("util");
 const events = require("events");
 
+const libData = require("@clusterio/lib/data");
 const libLink = require("@clusterio/lib/link");
 const { LineSplitter } = require("@clusterio/lib/stream");
 const { ConsoleTransport, logger } = require("@clusterio/lib/logging");
@@ -28,6 +29,8 @@ class TestControl extends libLink.Link {
 		this.slaveUpdates = [];
 		this.instanceUpdates = [];
 		this.saveListUpdates = [];
+		this.modUpdates = [];
+		this.modPackUpdates = [];
 		this.userUpdates = [];
 
 		this.connector.on("connect", () => {
@@ -40,6 +43,9 @@ class TestControl extends libLink.Link {
 			libLink.messages.setSaveListSubscriptions.send(
 				this, { all: true, instance_ids: [] }
 			).catch(err => logger.error(`Error setting save list subscriptions:\n${err.stack}`));
+			libLink.messages.setModSubscriptions.send(
+				this, { all: true, mod_names: [] }
+			).catch(err => logger.error(`Error setting mod subscriptions:\n${err.stack}`));
 			libLink.messages.setUserSubscriptions.send(
 				this, { all: true, names: [] }
 			).catch(err => logger.error(`Error setting user subscriptions:\n${err.stack}`));
@@ -67,6 +73,14 @@ class TestControl extends libLink.Link {
 		this.saveListUpdates.push(message.data);
 	}
 
+	async modUpdateEventHandler(message) {
+		this.modUpdates.push(message.data.mod);
+	}
+
+	async modPackUpdateEventHandler(message) {
+		this.modPackUpdates.push(message.data.mod_pack);
+	}
+
 	async userUpdateEventHandler(message) {
 		this.userUpdates.push(message.data);
 	}
@@ -207,6 +221,15 @@ before(async function() {
 	controlConnector.token = controlToken;
 	control = new TestControl(controlConnector);
 	await controlConnector.connect();
+
+	const testPack = new libData.ModPack();
+	testPack.id = 12;
+	testPack.name = "subspace_storage-pack";
+	testPack.factorioVersion = "1.1.0";
+	testPack.mods.set("clusterio_lib", { name: "clusterio_lib", enabled: true, version: "0.1.2" });
+	testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" });
+	await libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() });
+	await libLink.messages.setMasterConfigField.send(control, { field: "master.default_mod_pack_id", value: "12" });
 });
 
 after(async function() {

test/lib/data/ModInfo.js

@@ -0,0 +1,101 @@
+"use strict";
+const assert = require("assert").strict;
+const fs = require("fs-extra");
+const path = require("path");
+
+const libSchema = require("@clusterio/lib/schema");
+const libHash = require("@clusterio/lib/hash");
+const libBuildMod = require("@clusterio/lib/build_mod");
+const ModInfo = require("@clusterio/lib/data/ModInfo");
+
+
+describe("lib/data/ModInfo", function() {
+	describe("class ModInfo", function() {
+		it("should round trip serialize", function() {
+			const validate = libSchema.compile(ModInfo.jsonSchema);
+			function check(pack) {
+				const json = JSON.parse(JSON.stringify(pack));
+				if (!validate(json)) {
+					throw validate.errors;
+				}
+				assert.deepEqual(new ModInfo(json), pack);
+			}
+
+			check(new ModInfo());
+			check(new ModInfo({ name: "MyMod" }));
+			check(new ModInfo({ version: "1.0.0" }));
+			check(new ModInfo({ title: "My Extravagent Mod" }));
+			check(new ModInfo({ author: "Me" }));
+			check(new ModInfo({ contact: "Use my email" }));
+			check(new ModInfo({ homepage: "It's over there" }));
+			check(new ModInfo({ description: "My Description" }));
+			check(new ModInfo({ factorio_version: "2.0" }));
+			check(new ModInfo({ dependencies: [] }));
+			check(new ModInfo({ dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"] }));
+			check(new ModInfo({ filename: "MyMod_1.0.0.zip" }));
+			check(new ModInfo({ size: 1024 }));
+			check(new ModInfo({ sha1: "verified-as-MyMod" }));
+			check(new ModInfo({ is_deleted: true }));
+
+			// All at once
+			check(new ModInfo({
+				name: "MyMod",
+				version: "1.0.0",
+				title: "My Extravagent Mod",
+				author: "Me",
+				contact: "Use my email",
+				homepage: "It's over there",
+				description: "My Description",
+				factorio_version: "2.0",
+				dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"],
+				filename: "MyMod_1.0.0.zip",
+				size: 1024,
+				sha1: "verified-as-MyMod",
+				is_deleted: true,
+			}));
+		});
+
+		it("should sort integer mod versions lexicographically", function() {
+			let unsortedVersions = ["1.0.0", "1.1.0", "0.1.0", "3.0.0", "1.2.0", "0.3.1", "0.3.3", "2.1.1", "0.0.1"];
+			let sortedVersions = ["0.0.1", "0.1.0", "0.3.1", "0.3.3", "1.0.0", "1.1.0", "1.2.0", "2.1.1", "3.0.0"];
+			let mods = unsortedVersions.map(v => new ModInfo({ version: v }));
+			mods.sort((a, b) => a.integerVersion - b.integerVersion);
+			assert.deepEqual(mods.map(mod => mod.version), sortedVersions);
+		});
+		it("should sort integer factorio versions lexicographically", function() {
+			let unsortedVersions = ["1.0", "1.1", "0.1", "3.0", "1.2", "0.3", "2.1"];
+			let sortedVersions = ["0.1", "0.3", "1.0", "1.1", "1.2", "2.1", "3.0"];
+			let factorioMods = unsortedVersions.map(v => new ModInfo({ factorio_version: v }));
+			factorioMods.sort((a, b) => a.integerFactorioVersion - b.integerFactorioVersion);
+			assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions);
+		});
+
+		it("should load from a mod zip file", async function() {
+			await libBuildMod.build({
+				build: true,
+				pack: true,
+				sourceDir: path.join("test", "file", "empty_mod"),
+				outputDir: path.join("temp", "test"),
+			});
+			const filePath = path.join("temp", "test", "empty_mod_1.0.0.zip");
+			const hash = await libHash.hashFile(filePath);
+			const stat = await fs.stat(filePath);
+			const mod = await ModInfo.fromModFile(filePath);
+			assert.deepEqual(
+				mod,
+				new ModInfo({
+					name: "empty_mod",
+					version: "1.0.0",
+					title: "An Empty Mod",
+					author: "Me",
+					description: "An empty mod for testing",
+					factorio_version: "1.1",
+					dependencies: [],
+					filename: "empty_mod_1.0.0.zip",
+					size: stat.size,
+					sha1: hash,
+				})
+			);
+		});
+	});
+});

test/lib/data/ModPack.js

@@ -0,0 +1,267 @@
+"use strict";
+const assert = require("assert").strict;
+const zlib = require("zlib");
+
+const libSchema = require("@clusterio/lib/schema");
+const ExportManifest = require("@clusterio/lib/data/ExportManifest");
+const ModPack = require("@clusterio/lib/data/ModPack");
+
+
+describe("lib/data/ModPack", function() {
+	describe("class ModPack", function() {
+		it("should round trip serialize", function() {
+			const validate = libSchema.compile(ModPack.jsonSchema);
+			function check(pack) {
+				const json = JSON.parse(JSON.stringify(pack));
+				if (!validate(json)) {
+					throw validate.errors;
+				}
+				assert.deepEqual(new ModPack(json), pack);
+				const packStringed = ModPack.fromModPackString(pack.toModPackString());
+				packStringed.id = pack.id;
+				packStringed.exportManifest = pack.exportManifest;
+				assert.deepEqual(packStringed, pack);
+			}
+
+			check(new ModPack());
+			check(new ModPack({ name: "MyPack" }));
+			check(new ModPack({ description: "My Description" }));
+			check(new ModPack({ factorio_version: "2.0" }));
+			check(new ModPack({ mods: [
+				{ name: "subspace_storage", enabled: true, version: "1.99.8" },
+				{ name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" },
+			]}));
+			check(new ModPack({ settings: {
+				"startup": {
+					"bool-setting": { "value": true },
+				},
+				"runtime-global": {
+					"number-setting": { "value": 123 },
+				},
+				"runtime-per-user": {
+					"string-setting": { "value": "a string" },
+				},
+			}}));
+			check(new ModPack({ export_manifest: { assets: { setting: "settings.json" }}}));
+			check(new ModPack({ deleted: true }));
+			check(new ModPack({
+				name: "Super pack",
+				description: "Every option at once.",
+				factorio_version: "2.0",
+				mods: [
+					{ name: "subspace_storage", enabled: true, version: "1.99.8" },
+					{ name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" },
+				],
+				settings: {
+					"startup": {
+						"bool-setting": { "value": true },
+					},
+					"runtime-global": {
+						"number-setting": { "value": 123 },
+					},
+					"runtime-per-user": {
+						"string-setting": { "value": "a string" },
+					},
+				},
+				export_manifest: { assets: { setting: "settings.json" }},
+				deleted: true,
+			}));
+		});
+
+		it("should sort integer factorio versions lexicographically", function() {
+			let unsortedVersions = ["1.0", "1.1.0", "0.1", "3.0.0", "1.2", "0.3.1", "0.3.3", "2.1.1", "0.0.1"];
+			let sortedVersions = ["0.0.1", "0.1", "0.3.1", "0.3.3", "1.0", "1.1.0", "1.2", "2.1.1", "3.0.0"];
+			let factorioMods = unsortedVersions.map(v => new ModPack({ factorio_version: v }));
+			factorioMods.sort((a, b) => a.integerFactorioVersion - b.integerFactorioVersion);
+			assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions);
+		});
+
+		describe(".fillDefaultSettings()", function() {
+			const prototypes = {
+				"bool-setting": {
+					"bool": {
+						name: "bool",
+						type: "bool-setting",
+						setting_type: "startup",
+						default_value: true,
+					},
+				},
+				"double-setting": {
+					"number": {
+						name: "number",
+						type: "double-setting",
+						setting_type: "runtime-global",
+						default_value: 123,
+					},
+				},
+				"string-setting": {
+					"string": {
+						name: "string",
+						type: "string-setting",
+						setting_type: "runtime-per-user",
+						default_value: "a string",
+					},
+				},
+			};
+			const mockLogger = { warn: () => {} };
+			it("should fill in defaults for settings", function() {
+				const pack = new ModPack();
+				pack.fillDefaultSettings(prototypes, mockLogger);
+				assert.deepEqual(pack.toJSON().settings, {
+					"startup": {
+						"bool": { "value": true },
+					},
+					"runtime-global": {
+						"number": { "value": 123 },
+					},
+					"runtime-per-user": {
+						"string": { "value": "a string" },
+					},
+				});
+			});
+			it("should not overwrite existing values", function() {
+				const pack = new ModPack({ settings: {
+					"startup": {
+						"bool": { "value": false },
+					},
+					"runtime-global": {
+						"number": { "value": 2 },
+					},
+					"runtime-per-user": {
+						"string": { "value": "spam" },
+					},
+				}});
+				pack.fillDefaultSettings(prototypes, mockLogger);
+				assert.deepEqual(pack.toJSON().settings, {
+					"startup": {
+						"bool": { "value": false },
+					},
+					"runtime-global": {
+						"number": { "value": 2 },
+					},
+					"runtime-per-user": {
+						"string": { "value": "spam" },
+					},
+				});
+			});
+			it("should ignore unknown setting_type and missing default_value", function() {
+				const pack = new ModPack();
+				pack.fillDefaultSettings({
+					"string-setting": {
+						"foo": {
+							name: "foo",
+							type: "string-setting",
+							setting_type: "magic-that-does-not-exist",
+							default_value: "a string",
+						},
+						"bar": {
+							name: "bar",
+							type: "string-setting",
+							setting_type: "startup",
+						},
+					},
+				}, mockLogger);
+				assert.deepEqual(pack.toJSON().settings, {
+					"startup": {},
+					"runtime-global": {},
+					"runtime-per-user": {},
+				});
+			});
+		});
+
+		describe(".fromModPackString()", function() {
+			it("should handle malformed strings", function() {
+				assert.throws(
+					() => ModPack.fromModPackString("AMalformedString"),
+					new Error("Malformed mod pack string: zlib inflate failed")
+				);
+				let badJsonMsg;
+				try {
+					JSON.parse("Not Json");
+				} catch (err) {
+					badJsonMsg = err.message;
+				}
+				assert.throws(
+					// eslint-disable-next-line node/no-sync
+					() => ModPack.fromModPackString(Buffer.from(zlib.deflateSync("Not Json")).toString("base64")),
+					new Error(`Malformed mod pack string: ${badJsonMsg}`)
+				);
+				assert.throws(
+					// eslint-disable-next-line node/no-sync
+					() => ModPack.fromModPackString(Buffer.from(zlib.deflateSync('{"i":1}')).toString("base64")),
+					new Error("Malformed mod pack string: Schema validation failed")
+				);
+			});
+		});
+
+		describe("toModSettingsDat()", function() {
+			it("should properly serialise settings", function() {
+				const pack = new ModPack({ settings: {
+					"startup": {
+						"bool-setting": { "value": true },
+					},
+					"runtime-global": {
+						"number-setting": { "value": 123 },
+					},
+					"runtime-per-user": {
+						"string-setting": { "value": "a string" },
+					},
+				}});
+
+				function istr(str) {
+					return [
+						Uint8Array.from([0, str.length]),
+						Buffer.from(str),
+					];
+				}
+				/* eslint-disable indent */
+				assert.deepEqual(
+					pack.toModSettingsDat(),
+					Buffer.concat([
+						new Uint8Array(Uint16Array.from([1, 1, 0, 0]).buffer), // version
+						Uint8Array.from([0]), // reserved
+
+						Uint8Array.from([5, 0]), // dictionary
+						new Uint8Array(Uint32Array.from([3]).buffer), // items
+
+							...istr("startup"), // name
+							Uint8Array.from([5, 0]), // dictionary
+							new Uint8Array(Uint32Array.from([1]).buffer), // items
+
+								...istr("bool-setting"), // name
+								Uint8Array.from([5, 0]), // dictinary
+								new Uint8Array(Uint32Array.from([1]).buffer), // items
+
+									...istr("value"), // name
+									Uint8Array.from([1, 0, 1]), // boolean
+
+							...istr("runtime-global"), // name
+							Uint8Array.from([5, 0]), // dictionary
+							new Uint8Array(Uint32Array.from([1]).buffer), // items
+
+								...istr("number-setting"), // name
+								Uint8Array.from([5, 0]), // dictinary
+								new Uint8Array(Uint32Array.from([1]).buffer), // items
+
+									...istr("value"), // name
+									Uint8Array.from([2, 0]), // number
+									new Uint8Array(Float64Array.from([123]).buffer),
+
+							...istr("runtime-per-user"), // name
+							Uint8Array.from([5, 0]), // dictionary
+							new Uint8Array(Uint32Array.from([1]).buffer), // items
+
+								...istr("string-setting"), // name
+								Uint8Array.from([5, 0]), // dictinary
+								new Uint8Array(Uint32Array.from([1]).buffer), // items
+
+									...istr("value"), // name
+									Uint8Array.from([3, 0, 0, "a string".length]), // string
+									Buffer.from("a string"),
+					])
+				);
+				/* eslint-enable indent */
+			});
+		});
+	});
+});

test/lib/factorio/patch.js

@@ -8,47 +8,6 @@ const libFactorio = require("@clusterio/lib/factorio");
 
 
 describe("lib/factorio", function() {
-	describe("findRoot()", function() {
-		it("should find the root of a zip", function() {
-			let zip = new JSZip();
-			zip.file("root/foo.txt", "");
-			zip.file("root/bar.txt", "");
-
-			assert.equal(libFactorio._findRoot(zip), "root");
-		});
-
-		it("should throw if files are at the root of the zip", function() {
-			let zip = new JSZip();
-			zip.file("root/foo.txt", "");
-			zip.file("bar.txt", "");
-
-			assert.throws(
-				() => libFactorio._findRoot(zip),
-				new Error("Zip contains file 'bar.txt' in root dir")
-			);
-		});
-
-		it("should throw if there are multiple root dirs", function() {
-			let zip = new JSZip();
-			zip.file("root-1/foo.txt", "");
-			zip.file("root-2/bar.txt", "");
-
-			assert.throws(
-				() => libFactorio._findRoot(zip),
-				new Error("Zip contains multiple root folders")
-			);
-		});
-
-		it("should throw if given an empty zip file", function() {
-			let zip = new JSZip();
-
-			assert.throws(
-				() => libFactorio._findRoot(zip),
-				new Error("Empty zip file")
-			);
-		});
-	});
-
 	describe("generateLoader()", function() {
 		let reference = [
 			"-- Auto generated scenario module loader created by Clusterio",

test/lib/file_ops.js

@@ -84,4 +84,109 @@ describe("lib/file_ops", function() {
 			assert.equal(await fs.readFile(target, "utf8"), "current");
 		});
 	});
+
+	describe("checkFilename()", function() {
+		it("should allow a basic name", function() {
+			libFileOps.checkFilename("file");
+		});
+
+		function check(item, msg) {
+			assert.throws(() => libFileOps.checkFilename(item), new Error(msg));
+		}
+
+		it("should throw on non-string", function() {
+			check(undefined, "must be a string");
+			check(null, "must be a string");
+			check({}, "must be a string");
+			check([], "must be a string");
+			check(0, "must be a string");
+			check(false, "must be a string");
+		});
+
+		it("should throw on empty name", function() {
+			check("", "cannot be empty");
+		});
+
+		it("should throw on <>:\"\\/|?* \\x00\\r\\n\\t", function() {
+			for (let char of '<>:"\\/|?*\x00\r\n\t') {
+				check(char, 'cannot contain <>:"\\/|=* or control characters');
+			}
+		});
+
+		it("should throw on CON, PRN, AUX, NUL, COM1, LPT1", function() {
+			for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) {
+				check(bad, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9");
+				check(`${bad}.zip`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9");
+				check(`${bad}.anything.txt`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9");
+				check(`${bad}....a`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9");
+				check(`${bad}.`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9");
+			}
+		});
+
+		it("should throw on . and ..", function() {
+			for (let bad of [".", ".."]) {
+				check(bad, `cannot be named ${bad}`);
+			}
+		});
+
+		it("should throw on names ending with . or space", function() {
+			check("a ", "cannot end with . or space");
+			check("a.", "cannot end with . or space");
+		});
+	});
+
+	describe("cleanFilename()", function() {
+		function clean(item, expected) {
+			assert.equal(libFileOps.cleanFilename(item), expected);
+		}
+
+		it("should allow a basic name", function() {
+			clean("file", "file");
+		});
+
+		function check(item, msg) {
+			assert.throws(() => libFileOps.cleanFilename(item), new Error(msg));
+		}
+
+		it("should throw on non-string", function() {
+			check(undefined, "name must be a string");
+			check(null, "name must be a string");
+			check({}, "name must be a string");
+			check([], "name must be a string");
+			check(0, "name must be a string");
+			check(false, "name must be a string");
+		});
+
+		it("should clean empty name", function() {
+			clean("", "_");
+		});
+
+		it("should clean <>:\"\\/|?* \\x00\\r\\n\\t", function() {
+			for (let char of '<>:"\\/|?*\x00\r\n\t') {
+				clean(char, "_");
+			}
+		});
+
+		it("should clean CON, PRN, AUX, NUL, COM1, LPT1", function() {
+			for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) {
+				clean(bad, `${bad}_`);
+				clean(`${bad}.zip`, `${bad}_.zip`);
+				clean(`${bad}.anything.txt`, `${bad}_.anything.txt`);
+				clean(`${bad}....a`, `${bad}_....a`);
+				clean(`${bad}.`, `${bad}__`);
+			}
+		});
+
+		it("should clean . and ..", function() {
+			for (let bad of [".", ".."]) {
+				clean(bad, `${bad}_`);
+			}
+		});
+
+		it("should clean names ending with . or space", function() {
+			clean("a ", "a_");
+			clean("a.", "a_");
+		});
+	});
+
 });

test/lib/helpers.js

@@ -43,4 +43,113 @@ describe("lib/helpers", function() {
 			), "timeout");
 		});
 	});
+
+	function parse(input, attributes) {
+		const result = libHelpers.parseSearchString(input, attributes);
+		if (!result.issues.length) {
+			return result.terms;
+		}
+		return result;
+	}
+
+	describe("parseSearchString()", function() {
+		function word(value, opts = {}) {
+			return { type: "word", value, ...opts };
+		}
+		function attr(name, value) {
+			return { type: "attribute", name, value: typeof value === "string" ? word(value) : value };
+		}
+		it("should split words", function() {
+			assert.deepEqual(parse(""), [].map(word));
+			assert.deepEqual(parse("word"), ["word"].map(word));
+			assert.deepEqual(parse("some words here"), ["some", "words", "here"].map(word));
+		});
+		it("should trim whitespace", function() {
+			assert.deepEqual(parse("   "), [].map(word));
+			assert.deepEqual(parse("word  "), ["word"].map(word));
+			assert.deepEqual(parse("   some  words    here"), ["some", "words", "here"].map(word));
+		});
+		it("should parse negated words", function() {
+			assert.deepEqual(parse("this -that"), [word("this"), word("that", { exclude: true })]);
+			assert.deepEqual(parse("not-negated"), [word("not-negated")]);
+		});
+		it("should not split quoted words", function() {
+			assert.deepEqual(parse('"some quoted words" here'), ["some quoted words", "here"].map(word));
+			assert.deepEqual(parse('"a quoted string"'), ["a quoted string"].map(word));
+			assert.deepEqual(parse('words and "a quoted string"'), ["words", "and", "a quoted string"].map(word));
+		});
+		it("should split on quotation mark", function() {
+			assert.deepEqual(parse('"quoted"unquoted'), ["quoted", "unquoted"].map(word));
+			assert.deepEqual(parse('"quoted""double"'), ["quoted", "double"].map(word));
+			assert.deepEqual(parse('word"with"quote'), ["word", "with", "quote"].map(word));
+			assert.deepEqual(
+				parse('word-"with"-negation'),
+				[word("word-"), word("with"), word("negation", { exclude: true })]
+			);
+		});
+		it("should allow colons in quotes", function() {
+			assert.deepEqual(parse('"word:value"'), ["word:value"].map(word));
+		});
+		it("should parse attribute:value", function() {
+			assert.deepEqual(parse("author:Me", { author: "word" }), [attr("author", "Me")]);
+			assert.deepEqual(parse("word author:Me", { author: "word" }), [word("word"), attr("author", "Me")]);
+			assert.deepEqual(
+				parse("author:Me and author:You", { author: "word" }),
+				[attr("author", "Me"), word("and"), attr("author", "You")]
+			);
+			assert.deepEqual(
+				parse("author:Me"),
+				{
+					terms: [word("Me")],
+					issues: ['Unregonized attribute "author", use quotes to escape colons'],
+				}
+			);
+			assert.deepEqual(
+				parse("constructor:value"),
+				{
+					terms: [word("value")],
+					issues: ['Unregonized attribute "constructor", use quotes to escape colons'],
+				}
+			);
+		});
+		it("should throw on bad attribute format", function() {
+			assert.throws(
+				() => parse("author:Me", { author: "bad" }),
+				new Error("Bad attribute format bad for author")
+			);
+		});
+		it("should ignore attribute case", function() {
+			assert.deepEqual(parse("Author:Me", { author: "word" }), [attr("author", "Me")]);
+			assert.deepEqual(parse("AUTHOR:Me", { author: "word" }), [attr("author", "Me")]);
+			assert.deepEqual(parse("auThOR:Me", { author: "word" }), [attr("author", "Me")]);
+		});
+		it("should parse partial inputs", function() {
+			assert.deepEqual(parse("-"), [word("", { exclude: true })]);
+			assert.deepEqual(parse('"'), [""].map(word));
+			assert.deepEqual(parse('"word'), ["word"].map(word));
+			assert.deepEqual(parse("author:", { author: "word" }), [attr("author", "")]);
+			assert.deepEqual(parse("author:-", { author: "word" }), [attr("author", word("", { exclude: true }))]);
+		});
+	});
+	describe("wordMatches()", function() {
+		const match = libHelpers.wordMatches;
+		it("should match any inputs", function() {
+			assert(match(parse("word")[0], "a word in a string"), "one text");
+			assert(!match(parse("word")[0], "a string without"), "one text no match");
+			assert(match(parse("word")[0], "no match", "no things", "a word", "bad text", ""), "five text");
+			assert(!match(parse("unword")[0], "no match", "no things", "a word", "bad text", ""), "five text no match");
+		});
+		it("should invert match if using exclude", function() {
+			assert(!match(parse("-word")[0], "a word in a string"), "one text");
+			assert(match(parse("-word")[0], "a string without"), "one text no match");
+			assert(!match(parse("-word")[0], "no match", "no things", "a word", "bad text", ""), "five text");
+			assert(match(parse("-unword")[0], "no match", "no things", "a word", "bad text", ""), "five text no match");
+		});
+		it("should throw on bad input", function() {
+			assert.throws(
+				() => match(parse("attr:value", { attr: "word" })[0], "a string"),
+				new Error("wordMatches: parameter is not a word"),
+			);
+		});
+	});
 });

test/lib/plugin_loader.js

@@ -45,9 +45,16 @@ describe("lib/plugin_loader", function() {
 			);
 		});
 		it("should reject on broken plugin", async function() {
+			let brokenMessage;
+			try {
+				// eslint-disable-next-line node/global-require
+				require(path.resolve(brokenPlugin, "info.js"));
+			} catch (err) {
+				brokenMessage = err.message;
+			}
 			await assert.rejects(
 				libPluginLoader.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])),
-				{ message: "PluginError: Unexpected identifier" }
+				{ message: `PluginError: ${brokenMessage}` }
 			);
 		});
 		it("should reject on invalid plugin", async function() {

test/lib/zip_ops.js

@@ -0,0 +1,49 @@
+"use strict";
+const assert = require("assert").strict;
+const JSZip = require("jszip");
+
+const { findRoot } = require("@clusterio/lib/zip_ops");
+
+
+describe("lib/factorio", function() {
+	describe("findRoot()", function() {
+		it("should find the root of a zip", function() {
+			let zip = new JSZip();
+			zip.file("root/foo.txt", "");
+			zip.file("root/bar.txt", "");
+
+			assert.equal(findRoot(zip), "root");
+		});
+
+		it("should throw if files are at the root of the zip", function() {
+			let zip = new JSZip();
+			zip.file("root/foo.txt", "");
+			zip.file("bar.txt", "");
+
+			assert.throws(
+				() => findRoot(zip),
+				new Error("Zip contains file 'bar.txt' in root dir")
+			);
+		});
+
+		it("should throw if there are multiple root dirs", function() {
+			let zip = new JSZip();
+			zip.file("root-1/foo.txt", "");
+			zip.file("root-2/bar.txt", "");
+
+			assert.throws(
+				() => findRoot(zip),
+				new Error("Zip contains multiple root folders")
+			);
+		});
+
+		it("should throw if given an empty zip file", function() {
+			let zip = new JSZip();
+
+			assert.throws(
+				() => findRoot(zip),
+				new Error("Empty zip file")
+			);
+		});
+	});
+});

test/slave.js

@@ -1,11 +1,8 @@
 "use strict";
 const assert = require("assert").strict;
 const path = require("path");
-const fs = require("fs-extra");
 
-const libLink = require("@clusterio/lib/link");
 const libConfig = require("@clusterio/lib/config");
-const Instance = require("@clusterio/slave/src/Instance");
 const Slave = require("@clusterio/slave/src/Slave");
 
 describe("Slave testing", function() {
@@ -13,166 +10,6 @@ describe("Slave testing", function() {
 		libConfig.InstanceConfig.finalize();
 	});
 
-	describe("checkFilename()", function() {
-		it("should allow a basic name", function() {
-			Slave._checkFilename("file");
-		});
-
-		function check(item, msg) {
-			assert.throws(() => Slave._checkFilename(item), new Error(msg));
-		}
-
-		it("should throw on non-string", function() {
-			check(undefined, "must be a string");
-			check(null, "must be a string");
-			check({}, "must be a string");
-			check([], "must be a string");
-			check(0, "must be a string");
-			check(false, "must be a string");
-		});
-
-		it("should throw on empty name", function() {
-			check("", "cannot be empty");
-		});
-
-		it("should throw on <>:\"\\/|?* \\x00\\r\\n\\t", function() {
-			for (let char of '<>:"\\/|?*\x00\r\n\t') {
-				check(char, 'cannot contain <>:"\\/|=* or control characters');
-			}
-		});
-
-		it("should throw on CON, PRN, AUX, NUL, COM1, LPT1", function() {
-			for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) {
-				check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9");
-			}
-		});
-
-		it("should throw on . and ..", function() {
-			for (let bad of [".", ".."]) {
-				check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9");
-			}
-		});
-
-		it("should throw on names ending with . or space", function() {
-			check("a ", "cannot end with . or space");
-			check("a.", "cannot end with . or space");
-		});
-	});
-
-	describe("cleanFilename()", function() {
-		function clean(item, expected) {
-			assert.equal(Slave._cleanFilename(item), expected);
-		}
-
-		it("should allow a basic name", function() {
-			clean("file", "file");
-		});
-
-		function check(item, msg) {
-			assert.throws(() => Slave._cleanFilename(item), new Error(msg));
-		}
-
-		it("should throw on non-string", function() {
-			check(undefined, "name must be a string");
-			check(null, "name must be a string");
-			check({}, "name must be a string");
-			check([], "name must be a string");
-			check(0, "name must be a string");
-			check(false, "name must be a string");
-		});
-
-		it("should clean empty name", function() {
-			clean("", "_");
-		});
-
-		it("should clean <>:\"\\/|?* \\x00\\r\\n\\t", function() {
-			for (let char of '<>:"\\/|?*\x00\r\n\t') {
-				clean(char, "_");
-			}
-		});
-
-		it("should clean CON, PRN, AUX, NUL, COM1, LPT1", function() {
-			for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) {
-				clean(bad, `${bad}_`);
-			}
-		});
-
-		it("should clean . and ..", function() {
-			for (let bad of [".", ".."]) {
-				clean(bad, `${bad}_`);
-			}
-		});
-
-		it("should clean names ending with . or space", function() {
-			clean("a ", "a_");
-			clean("a.", "a_");
-		});
-	});
-
-	describe("symlinkMods()", function() {
-		let testDir = path.join("temp", "test", "symlink");
-
-		let instance;
-		before(async function() {
-			// Remove previous test output
-			await fs.remove(testDir);
-
-			// Add some test mods
-			await fs.outputFile(path.join(testDir, "shared", "mod_a.zip"), "a");
-			await fs.outputFile(path.join(testDir, "shared", "mod_b.zip"), "b");
-			await fs.outputFile(path.join(testDir, "shared", "mod.dat"), "c");
-			await fs.outputFile(path.join(testDir, "shared", "mod-list.json"), "d");
-
-			let instanceConfig = new libConfig.InstanceConfig("slave");
-			await instanceConfig.init();
-			instanceConfig.set("instance.name", "test");
-			instance = new Instance(
-				{}, new libLink.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig
-			);
-			await fs.outputFile(instance.path("mods", "mod_i.zip"), "i");
-		});
-
-		it("should link mods and data files", async function() {
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			assert.equal(await fs.readFile(instance.path("mods", "mod_a.zip"), "utf-8"), "a");
-			assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b");
-			assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c");
-			assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d");
-			assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i");
-		});
-
-		it("should ignore directories", async function() {
-			await fs.ensureDir(path.join(testDir, "shared", "dir"));
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			assert(!await fs.exists(instance.path("mods", "dir"), "utf-8"), "dir was unxpectedly linked");
-		});
-
-		it("should ignore files", async function() {
-			await fs.outputFile(path.join(testDir, "shared", "file"), "f");
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			assert(!await fs.exists(instance.path("mods", "file"), "utf-8"), "dir was unxpectedly linked");
-		});
-
-		it("should unlink removed mods", async function() {
-			// This does not work on Windows
-			if (process.platform === "win32") {
-				this.skip();
-			}
-
-			await fs.unlink(path.join(testDir, "shared", "mod_a.zip"));
-			await Instance._symlinkMods(instance, path.join(testDir, "shared"));
-
-			await assert.rejects(fs.lstat(instance.path("mods", "mod_a.zip")), { code: "ENOENT" });
-			assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b");
-			assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c");
-			assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d");
-			assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i");
-		});
-	});
-
 	describe("discoverInstances()", function() {
 		it("should discover test instance", async function() {
 			let instancePath = path.join("test", "file", "instances");