Add Factorio version management and download functionality

Co-authored-by: danielv <danielv@danielv.no>

.devcontainer/devcontainer.json

@@ -1,4 +1,4 @@
 {
 	"image":"mcr.microsoft.com/devcontainers/universal:2",
-	"postCreateCommand": "wget https://factorio.com/get-download/1.1.107/headless/linux64 && tar -xf linux64 && pnpm install",
+	"postCreateCommand": "mkdir factorio && cd factorio && wget https://factorio.com/get-download/1.1.107/headless/linux64 && tar -xf linux64 && mv factorio factorio_1.1.107 && rm linux64 && wget https://factorio.com/get-download/2.0.47/headless/linux64 && tar -xf linux64 && mv factorio factorio_2.0.47 && rm linux64 && pnpm install",
 }

.eslintrc.js

@@ -191,7 +191,7 @@ module.exports = {
 		"multiline-comment-style": ["error", "separate-lines"],
 		"multiline-ternary": "off",
 		"new-cap": ["error", {
-			"capIsNewExceptions": ["StringEnum"],
+			"capIsNewExceptions": ["StringEnum", "StringKey"],
 			"capIsNewExceptionPattern": "^Type\\.",
 		}],
 		"new-parens": "error",

.github/workflows/node.js.yml

@@ -12,10 +12,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x, 20.x, 22.x]
-        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
-        factorio-version: [1.1.110]
-        # Supported versions are latest stable/experimental and one minor version before stable (even if from a previous major) 
+        # Supported versions are active lts and maintenance see https://nodejs.org/en/about/releases/
+        node-version: [20.x, 22.x]
+        # Supported versions are latest stable and one minor version before stable (even if from a previous major) 
+        factorio-version: [1.1.110, 2.0.47]
 
     steps:
     - uses: actions/checkout@v4
@@ -31,13 +31,13 @@ jobs:
     - name: Use Factorio ${{ matrix.factorio-version }}
       run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/${{ matrix.factorio-version }}/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
     - name: Run tests
-      if: ${{ matrix.node-version != '18.x' || matrix.factorio-version != '1.1.110' }}
+      if: ${{ matrix.node-version != '20.x' || matrix.factorio-version != '1.1.110' }}
       run: pnpm test
     - name: Run coverage
-      if: ${{ matrix.node-version == '18.x' && matrix.factorio-version == '1.1.110' }}
+      if: ${{ matrix.node-version == '20.x' && matrix.factorio-version == '1.1.110' }}
       run: pnpm run ci-cover
     - name : Upload coverage to Codecov
-      if: ${{ matrix.node-version == '18.x' && matrix.factorio-version == '1.1.110' }}
+      if: ${{ matrix.node-version == '20.x' && matrix.factorio-version == '1.1.110' }}
       uses: codecov/codecov-action@v4
       with:
         files: ./coverage/lcov.info

.gitignore

@@ -32,6 +32,8 @@ node_modules
 Factorio*
 /factorio
 !/test/file/factorio
+!/test/lib/factorio
+!/packages/lib/src/factorio
 /instances
 /mods
 /database

CHANGELOG.md

@@ -31,6 +31,52 @@ Many thanks to the following for contributing to this release:
 [@username](https://github.com/username)
 -->
 
+## Version 2.0.0-alpha.21
+
+### Major Features
+
+- Added import, export, and restore of user lists (bans / admins / whitelist / combined). [#717](https://github.com/clusterio/clusterio/issues/717)
+- Added remote update for core packages and third party plugins. [#761](https://github.com/clusterio/clusterio/issues/761)
+- Added config cloning on instance creation. [#720](https://github.com/clusterio/clusterio/issues/720)
+
+### Features
+
+- Added social links and instructions to error page. [#742](https://github.com/clusterio/clusterio/issues/742)
+- Added button to download control config. [#740](https://github.com/clusterio/clusterio/issues/740)
+- Added server action filter (aka "chat filter") to instance log in web ui. [#744](https://github.com/clusterio/clusterio/issues/744)
+- Added `factorio.console_logging` to enable dedicated console log file for servers. [#748](https://github.com/clusterio/clusterio/issues/748)
+- Added a recovery mode to host and controller accessible as an option on the run command. [#752](https://github.com/clusterio/clusterio/issues/752)
+- Added username search to the users list. [#751](https://github.com/clusterio/clusterio/issues/751)
+- Added log level filtering to the web ui. [#745](https://github.com/clusterio/clusterio/issues/745)
+- Added pending update detection for host and controller. [#758](https://github.com/clusterio/clusterio/issues/758)
+- Added restart required indicator for host and controller. [#758](https://github.com/clusterio/clusterio/issues/758)
+- Added version mismatch indicator in host list. [#758](https://github.com/clusterio/clusterio/issues/758)
+- Added host plugin list command to crl where previously only controller plugin list existed. [#760](https://github.com/clusterio/clusterio/issues/760)
+- Added optional feature which allows new plugins to be installed remotely.
+- Added bidirectional whitelist syncing. [#716](https://github.com/clusterio/clusterio/issues/716)
+- Added support for v2 map exchange strings. [#700](https://github.com/clusterio/clusterio/issues/700)
+
+### Fixes
+
+- Fixed handling of zip files with multiple top-level folders. [#739](https://github.com/clusterio/clusterio/issues/739)
+- Fixed looping between instances of ban and admin sync events. [#736](https://github.com/clusterio/clusterio/issues/736)
+- Fixed quality being stripped from power armour equipment when moving between servers [#753](https://github.com/clusterio/clusterio/pull/753)
+- Resolved webpack type export warnings [#750](https://github.com/clusterio/clusterio/pull/750)
+- Reduce page load time on users list by enabling pagination. [#718](https://github.com/clusterio/clusterio/issues/718)
+- Corrected stack trace and enabled source mapping during bootstrap. [#764](https://github.com/clusterio/clusterio/pull/764)
+- Fixed map gen settings using the incorrect name for cliff settings following exchange string decode.
+
+### Meta
+
+- Silent testing is now truly silent. [#764](https://github.com/clusterio/clusterio/pull/764)
+- Node 18 has reached end of life and is no longer maintained, we no longer guarantee support for it.
+- CI testing now runs on node 20 & 22 as well as factorio 1.1.110 & 2.0.47.
+
+Many thanks to the following for contributing to this release:  
+[@Cooldude2606](https://github.com/Cooldude2606)
+[@Danielv123](https://github.com/Danielv123)
+[@Hornwitser](https://github.com/Hornwitser)
+
 ## Version 2.0.0-alpha.20
 
 ### Features

docs/managing-a-cluster.md

@@ -28,6 +28,21 @@ Generate a token for a host. `--id` is the host id.
 
 Create a host config. `--id` id the host id. `--name` is the host name. `--generate-token` decides whether to also generate a token, defaulting to `0`. `--output` is the filename for the output of the config; Default is "config-host.json", "-" for stdout.
 
+### Update Clusterio
+
+    ctl> host update [--id <host id>] [--name <host name>] [--restart]
+
+If remote updates are enabled on the host, this will attempt to update `@clusterio/host` on the machine. Changes will not be applied until the host is restarted. If the `--restart` flag is provided, the host will automatically restart once the update is complete.
+
+Remote updates are enabled by default but can be disabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the following command locally while the host is offline:
+
+    host> config set host.allow_remote_updates true/false
+
+The controller also supports remote updates, following the same logic. The corresponding commands are:
+
+    ctl> controller update [--restart]
+    controller> config set controller.allow_remote_updates true/false
+
 ## Instances
 
 ### List Instances
@@ -199,3 +214,50 @@ Note: If the player joins the cluster again a new account will be made for them
 ## Roles
 
 To be written.
+
+
+## Plugins (Local)
+
+To be written.
+
+
+## Plugins (Remote)
+
+By default, remote updates are enabled unless you selected "no" during installation.
+Remote updates allow ctl and the web UI to update plugins on machines within the cluster.
+However, remote installations are disabled by default to prevent the risk of arbitrary code execution through the installation of malicious npm packages onto the machine. See below if you wish to enable this feature.
+
+### List
+
+    ctl> controller plugin list
+    ctl> host plugin list <host>
+
+This command lists all plugins currently known to the remote. Newly installed plugins will not be displayed until the remote is restarted. Plugins that are distributed as an npm package can be updated remotely.
+
+### Update
+
+    ctl> controller plugin update <name> [--restart]
+    ctl> host plugin update <host> <name> [--restart]
+
+If remote plugin updates are enabled on the remote, this will execute `npm update --save <name>`. Changes will not be applied until the remote is restarted. If the `--restart` flag is provided, the remote will automatically restart once the update is complete.
+
+### Install
+
+    ctl> controller plugin install <name> [--restart]
+    ctl> host plugin install <host> <name> [--restart]
+
+If remote plugin installs are enabled on the remote, this will execute `npm install --save <name>`. Changes will not be applied until the remote is restarted. If the `--restart` flag is provided, the remote will automatically restart once the update is complete.
+
+### Enable/Disable Updates
+
+    host> config set host.allow_plugin_updates true/false
+    controller> config set controller.allow_plugin_updates true/false
+
+Remote updates are enabled by default but can be disabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the above commands locally while the remote is offline.
+
+### Enable/Disable Installs
+
+    host> config set host.allow_plugin_install true/false
+    controller> config set controller.allow_plugin_install true/false
+
+Remote installs are disabled by default but can be enabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the above commands locally while the remote is offline.

docs/migrating-from-2.0.0-alpha.13.md

@@ -10,6 +10,10 @@ You can do this by running the follow RCON command on an instance that is curren
 
     /sc local m={name="pack",description="",factorio_version=game.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}} for n,v in pairs(game.active_mods) do table.insert(m.mods,{name=n,enabled=true,version=v}) end local function set(s,c) for n,v in pairs(s) do m.settings[c][n]=v end end set(settings.startup,"startup") set(settings.global,"runtime-global") if game.player then set(game.player.mod_settings,"runtime-per-user") else set(settings.player,"runtime-per-user") end local e=game.encode_string(game.table_to_json(m)) log(e) rcon.print(e)
 
+If you for some reason want to run this function in 2.0+, use the following command:
+
+    /sc local a={name="pack",description="",factorio_version=script.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}}for b,c in pairs(script.active_mods)do table.insert(a.mods,{name=b,enabled=true,version=c})end;local function d(e,f)for b,c in pairs(e)do a.settings[f][b]=c end end;d(settings.startup,"startup")d(settings.global,"runtime-global")if game.player then d(game.player.mod_settings,"runtime-per-user")else d(settings.player,"runtime-per-user")end;local g=helpers.encode_string(helpers.table_to_json(a))log(g)rcon.print(g)
+
 This should produce a [Mod Pack String](https://forums.factorio.com/viewtopic.php?f=96&t=103578) which you need to copy and paste somewhere to keep it for later.
 Mod Pack strings looks something like this:
 

packages/controller/controller.ts

@@ -130,8 +130,16 @@ async function handleBootstrapCommand(
 	controllerConfig: lib.ControllerConfig
 ): Promise<void> {
 	let subCommand = args._[1];
-	let userManager = new UserManager(controllerConfig);
-	await userManager.load(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
+
+	const databaseDirectory = controllerConfig.get("controller.database_directory");
+	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		path.join(databaseDirectory, "roles.json"),
+		lib.Role.fromJSON.bind(lib.Role),
+	).bootstrap());
+
+	const userManager = new UserManager(controllerConfig, roles);
+	await userManager.load(path.join(databaseDirectory, "users.json"));
+
 	if (subCommand === "create-admin") {
 		if (!args.name) {
 			logger.error("name cannot be blank");
@@ -144,7 +152,7 @@ async function handleBootstrapCommand(
 			admin = userManager.createUser(args.name);
 		}
 
-		let adminRole = lib.ensureDefaultAdminRole(userManager.roles);
+		let adminRole = lib.ensureDefaultAdminRole(roles);
 		admin.roleIds.add(adminRole.id);
 		admin.isAdmin = true;
 		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
@@ -210,7 +218,6 @@ interface InitializeParameters {
 	shouldRun: boolean;
 	clusterLogger: winston.Logger;
 	pluginInfos: lib.PluginNodeEnvInfo[] | null;
-	controllerConfigPath: string | null;
 	controllerConfig: lib.ControllerConfig | null;
 }
 
@@ -270,6 +277,10 @@ async function initialize(): Promise<InitializeParameters> {
 				type: "boolean", nargs: 0, default: false,
 				describe: "Indicate that a process monitor will restart the controller on failure",
 			});
+			yargs.option("recovery", {
+				type: "boolean", nargs: 0, default: false,
+				describe: "Start the controller in recovery mode with all plugins disabled and hosts disconnected",
+			});
 			yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 });
 			yargs.option("dev-plugin", { hidden: true, type: "array" });
 		})
@@ -313,6 +324,9 @@ async function initialize(): Promise<InitializeParameters> {
 	let shouldRun = false;
 	if (command === "run") {
 		logger.info(`Starting Clusterio controller ${version}`);
+		if (args.recovery) {
+			logger.warn("Controller recovery mode enabled. Some features will be disabled.");
+		}
 		shouldRun = true;
 	}
 
@@ -327,7 +341,6 @@ async function initialize(): Promise<InitializeParameters> {
 			shouldRun,
 			clusterLogger,
 			pluginInfos: null,
-			controllerConfigPath: null,
 			controllerConfig: null,
 		};
 	}
@@ -337,37 +350,34 @@ async function initialize(): Promise<InitializeParameters> {
 	lib.registerPluginMessages(pluginInfos);
 	lib.addPluginConfigFields(pluginInfos);
 
+	let controllerConfig;
 	const controllerConfigPath = args.config;
 	logger.info(`Loading config from ${controllerConfigPath}`);
-	let controllerConfig = new lib.ControllerConfig("controller");
 	try {
-		let jsonConfig = JSON.parse(await fs.readFile(controllerConfigPath, { encoding: "utf8" }));
-		controllerConfig = lib.ControllerConfig.fromJSON(jsonConfig, "controller");
+		controllerConfig = await lib.ControllerConfig.fromFile("controller", controllerConfigPath);
 
 	} catch (err: any) {
 		if (err.code === "ENOENT") {
 			logger.info("Config not found, initializing new config");
-			controllerConfig = new lib.ControllerConfig("controller");
+			controllerConfig = new lib.ControllerConfig("controller", undefined, controllerConfigPath);
 
 		} else {
-			throw new lib.StartupError(`Failed to load ${controllerConfigPath}: ${err.message}`);
+			throw new lib.StartupError(`Failed to load ${controllerConfigPath}: ${err.stack ?? err.message ?? err}`);
 		}
 	}
 
+	controllerConfig.set("controller.version", version); // Allows tracking last loaded version
+
 	if (!controllerConfig.get("controller.auth_secret")) {
 		logger.info("Generating new controller authentication secret");
 		let asyncRandomBytes = util.promisify(crypto.randomBytes);
 		let bytes = await asyncRandomBytes(256);
 		controllerConfig.set("controller.auth_secret", bytes.toString("base64"));
-		await lib.safeOutputFile(
-			controllerConfigPath, JSON.stringify(controllerConfig, null, "\t")
-		);
+		await controllerConfig.save();
 	}
 
 	if (command === "config") {
-		await lib.handleConfigCommand(
-			args, controllerConfig, controllerConfigPath
-		);
+		await lib.handleConfigCommand(args, controllerConfig);
 
 	} else if (command === "bootstrap") {
 		await handleBootstrapCommand(args, controllerConfig);
@@ -377,7 +387,6 @@ async function initialize(): Promise<InitializeParameters> {
 		args,
 		clusterLogger,
 		pluginInfos,
-		controllerConfigPath,
 		controllerConfig,
 		shouldRun,
 	};
@@ -393,19 +402,18 @@ async function startup() {
 		shouldRun,
 		clusterLogger,
 		pluginInfos,
-		controllerConfigPath,
 		controllerConfig,
 	} = await initialize();
-	if (!shouldRun || !pluginInfos || !controllerConfigPath || !controllerConfig) {
+	if (!shouldRun || !pluginInfos || !controllerConfig) {
 		return;
 	}
 
 	controller = new Controller(
 		clusterLogger,
 		pluginInfos,
-		controllerConfigPath,
 		controllerConfig,
 		Boolean(args.canRestart),
+		Boolean(args.recovery),
 		...await Controller.bootstrap(controllerConfig)
 	);
 

packages/controller/index.ts

@@ -5,6 +5,7 @@ export { default as ControllerRouter } from "./src/ControllerRouter";
 export { default as ControlConnection } from "./src/ControlConnection";
 export { default as HostConnection } from "./src/HostConnection";
 export { default as InstanceInfo } from "./src/InstanceInfo";
+export { default as HostInfo } from "./src/HostInfo";
 export { default as BaseControllerPlugin } from "./src/BaseControllerPlugin";
 export { default as UserManager } from "./src/UserManager";
 export { default as WsServer } from "./src/WsServer";

packages/controller/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio controller",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",

packages/controller/src/ControlConnection.ts

@@ -70,6 +70,7 @@ export default class ControlConnection extends BaseConnection {
 
 		this.handle(lib.ControllerStopRequest, this.handleControllerStopRequest.bind(this));
 		this.handle(lib.ControllerRestartRequest, this.handleControllerRestartRequest.bind(this));
+		this.handle(lib.ControllerUpdateRequest, this.handleControllerUpdateRequest.bind(this));
 		this.handle(lib.ControllerConfigGetRequest, this.handleControllerConfigGetRequest.bind(this));
 		this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this));
 		this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this));
@@ -98,6 +99,7 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this));
 		this.handle(lib.ModListRequest, this.handleModListRequest.bind(this));
 		this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this));
+		this.handle(lib.ModPortalGetAllRequest, this.handleModPortalGetAllRequest.bind(this));
 		this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this));
 		this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this));
 		this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this));
@@ -116,7 +118,13 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.UserSetBannedRequest, this.handleUserSetBannedRequest.bind(this));
 		this.handle(lib.UserSetWhitelistedRequest, this.handleUserSetWhitelistedRequest.bind(this));
 		this.handle(lib.UserDeleteRequest, this.handleUserDeleteRequest.bind(this));
+		this.handle(lib.UserBulkImportRequest, this.handleUserBulkImportRequest.bind(this));
+		this.handle(lib.UserBulkExportRequest, this.handleUserBulkExportRequest.bind(this));
+		this.handle(lib.PluginListRequest, this.handlePluginListRequest.bind(this));
+		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
+		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
 		this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this));
+		this.handle(lib.ModPortalDownloadRequest, this.handleModPortalDownloadRequest.bind(this));
 	}
 
 	validateIngress(message: lib.MessageRequest | lib.MessageEvent) {
@@ -249,6 +257,14 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleInstanceCreateRequest(request: lib.InstanceCreateRequest) {
 		const instanceConfig = new lib.InstanceConfig("controller");
+		if (request.cloneFromId) {
+			const baseInstance = this._controller.instances.get(request.cloneFromId);
+			if (!baseInstance) {
+				throw new lib.RequestError(`Instance with ID ${request.cloneFromId} does not exist`);
+			}
+			instanceConfig.update(baseInstance.config.toJSON(), false);
+			instanceConfig.set("instance.assigned_host", null); // New instances are unassigned
+		}
 		instanceConfig.update(request.config, false, "control");
 		await this._controller.instanceCreate(instanceConfig);
 	}
@@ -297,7 +313,7 @@ export default class ControlConnection extends BaseConnection {
 		let stream = await routes.createProxyStream(this._controller.app);
 		stream.filename = name;
 
-		let ready = new Promise((resolve, reject) => {
+		let ready = new Promise<void>((resolve, reject) => {
 			stream.events.on("source", resolve);
 			stream.events.on("timeout", () => reject(
 				new lib.RequestError("Timed out establishing stream from host")
@@ -514,6 +530,88 @@ export default class ControlConnection extends BaseConnection {
 		};
 	}
 
+	/**
+	 * Handle request to fetch all mods from the Factorio Mod Portal for a given version.
+	 * @param request - The request object containing factorioVersion.
+	 */
+	async handleModPortalGetAllRequest(request: lib.ModPortalGetAllRequest) {
+		const cacheKey = `${request.factorioVersion}-${request.hide_deprecated}`;
+		const cachedData = this._controller.modPortalCache.get(cacheKey);
+
+		const cacheDuration = this._controller.config.get("controller.mod_portal_cache_duration_minutes") * 60 * 1000;
+		if (cachedData && Date.now() - cachedData.timestamp < cacheDuration) {
+			return new lib.ModPortalGetAllRequest.Response(cachedData.data);
+		}
+
+		try {
+			logger.info(`Fetching mod portal data for ${cacheKey}`);
+			const mods = await lib.ModStore.fetchAllModsFromPortal(
+				request.factorioVersion,
+				this._controller.config.get("controller.mod_portal_page_size"),
+				request.hide_deprecated
+			);
+			this._controller.modPortalCache.set(cacheKey, { timestamp: Date.now(), data: mods });
+			// The Response class is defined inline within the Request class
+			return new lib.ModPortalGetAllRequest.Response(mods);
+		} catch (error: any) {
+			logger.error(`Error fetching all mods from portal (${request.factorioVersion}): ${error}`);
+			// Propagate a user-friendly error back to the client
+			throw new lib.RequestError(`Portal mod fetch failed: ${error}`);
+		}
+	}
+
+	/**
+	 * Handle request to download a mod from the Factorio Mod Portal to the controller.
+	 * @param request - Request object with mod name, version, and factorioVersion.
+	 */
+	async handleModPortalDownloadRequest(request: lib.ModPortalDownloadRequest): Promise<lib.ModInfo> {
+		const { name, version, factorioVersion } = request;
+		const filename = lib.ModInfo.filename(name, version);
+
+		// Check if mod already exists
+		if (this._controller.modStore.files.has(filename)) {
+			logger.verbose(`Mod ${filename} already exists, skipping download.`);
+			return this._controller.modStore.files.get(filename)!;
+		}
+
+		// Get Factorio credentials from config
+		const username = this._controller.config.get("controller.factorio_username");
+		const token = this._controller.config.get("controller.factorio_token");
+
+		if (!username || !token) {
+			throw new lib.RequestError("Factorio credentials (username, token) not configured on the controller.");
+		}
+
+		try {
+			logger.info(`Downloading mod ${filename} for Factorio ${factorioVersion} from portal.`);
+			const modMap = new Map([[name, version]]);
+			await this._controller.modStore.downloadMods(modMap, username || "", token || "", factorioVersion);
+
+			// Verify download and return ModInfo
+			const downloadedMod = this._controller.modStore.files.get(filename);
+			if (!downloadedMod) {
+				throw new lib.RequestError(`Failed to find mod ${filename} in store after download attempt.`);
+			}
+			logger.info(`Successfully downloaded mod ${filename}.`);
+			return downloadedMod;
+
+		} catch (error: any) {
+			logger.error(`Error downloading mod ${filename} from portal: ${error.message}`);
+			// Improve error message clarity
+			let errorMessage = `Mod portal download failed for ${name} v${version}`;
+			if (error instanceof lib.RequestError) {
+				errorMessage += `: ${error.message}`;
+			} else if (error.message.includes("401") || error.message.includes("Unauthorized")) {
+				errorMessage += ": Authentication failed. Check Factorio username/token in controller config.";
+			} else if (error.message.includes("404") || error.message.includes("Not Found")) {
+				errorMessage += ": Mod version not found on the portal.";
+			} else {
+				errorMessage += `: ${error.message}`;
+			}
+			throw new lib.RequestError(errorMessage);
+		}
+	}
+
 	async handleModDeleteRequest(request: lib.ModDeleteRequest) {
 		await this._controller.modStore.deleteMod(request.name, request.version);
 	}
@@ -573,22 +671,21 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleRoleListRequest() {
-		return [...this._controller.userManager.roles.values()];
+		return [...this._controller.roles.values()];
 	}
 
 	async handleRoleCreateRequest(request: lib.RoleCreateRequest) {
-		let lastId = Math.max.apply(null, [...this._controller.userManager.roles.keys()]);
+		let lastId = Math.max.apply(null, [...this._controller.roles.keys()]);
 
 		// Start at 5 to leave space for future default roles
 		let id = Math.max(5, lastId+1);
-		this._controller.userManager.roles.set(id, lib.Role.fromJSON({ id, ...request }));
-		this._controller.userManager.dirty = true;
+		this._controller.roles.set(lib.Role.fromJSON({ id, ...request }));
 		return id;
 	}
 
 	async handleRoleUpdateRequest(request: lib.RoleUpdateRequest) {
 		let { id, name, description, permissions } = request;
-		let role = this._controller.userManager.roles.get(id);
+		let role = this._controller.roles.getMutable(id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
@@ -596,27 +693,29 @@ export default class ControlConnection extends BaseConnection {
 		role.name = name;
 		role.description = description;
 		role.permissions = new Set(permissions);
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}
 
 	async handleRoleGrantDefaultPermissionsRequest(request: lib.RoleGrantDefaultPermissionsRequest) {
-		let role = this._controller.userManager.roles.get(request.id);
+		let role = this._controller.roles.get(request.id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${request.id} does not exist`);
 		}
 
 		role.grantDefaultPermissions();
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}
 
 	async handleRoleDeleteRequest(request: lib.RoleDeleteRequest) {
 		let id = request.id;
 
-		if (!this._controller.userManager.roles.delete(id)) {
+		const role = this._controller.roles.get(id);
+		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
-		this._controller.userManager.dirty = true;
+		this._controller.roles.delete(role);
 
+		this._controller.userManager.dirty = true;
 		for (let user of this._controller.userManager.users.values()) {
 			if (user.roleIds.delete(id)) {
 				this._controller.userPermissionsUpdated(user);
@@ -668,7 +767,7 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		for (let roleId of request.roles) {
-			if (!this._controller.userManager.roles.has(roleId)) {
+			if (!this._controller.roles.has(roleId)) {
 				throw new lib.RequestError(`Role with ID ${roleId} does not exist`);
 			}
 		}
@@ -752,6 +851,240 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
+	async handleUserBulkRestoreRequest(request: lib.UserBulkImportRequest, updated: Map<string, ControllerUser>) {
+		if (request.importType === "users") {
+			// Update all fields for all users
+			const users = request.users as Extract<typeof request.users, { username: string }[]>;
+			const admin = new Set(users.filter(u => u.is_admin).map(u => u.username.toLowerCase()));
+			const banned = new Set(users.filter(u => u.is_banned).map(u => u.username.toLowerCase()));
+			const whitelist = new Set(users.filter(u => u.is_whitelisted).map(u => u.username.toLowerCase()));
+			for (const user of this._controller.userManager.users.values()) {
+				if (user.isAdmin && !admin.has(user.name)) {
+					user.isAdmin = false;
+					updated.set(user.id, user);
+				}
+				if (user.isBanned && !banned.has(user.name)) {
+					user.isBanned = false;
+					user.banReason = "";
+					updated.set(user.id, user);
+				}
+				if (user.isWhitelisted && !whitelist.has(user.name)) {
+					user.isWhitelisted = false;
+					updated.set(user.id, user);
+				}
+			}
+		} else if (request.importType === "bans") {
+			// Bans have extra logic to handle ban reason
+			const users = request.users as (string | { username: string, reason: string })[];
+			const expected = new Set(
+				users.map(u => (typeof u === "string" ? u.toLowerCase() : u.username.toLowerCase()))
+			);
+			for (const user of this._controller.userManager.users.values()) {
+				if (user.isBanned && !expected.has(user.name)) {
+					user.isBanned = false;
+					user.banReason = "";
+					updated.set(user.id, user);
+				}
+			}
+		} else {
+			// Whitelist and admin have the same logic
+			const users = request.users as string[];
+			const expected = new Set(users.map(u => u.toLowerCase()));
+			const prop = request.importType === "admins" ? "isAdmin" : "isWhitelisted";
+			for (const user of this._controller.userManager.users.values()) {
+				if (user[prop] && !expected.has(user.name)) {
+					user[prop] = false;
+					updated.set(user.id, user);
+				}
+			}
+		}
+	}
+
+	// A large number of small if statements are unavoidable when reducing the number of updates sent
+	// eslint-disable-next-line complexity
+	async handleUserBulkImportRequest(request: lib.UserBulkImportRequest) {
+		let backup: undefined | Awaited<ReturnType<ControlConnection["handleUserBulkExportRequest"]>>;
+		const updated = new Map<ControllerUser["id"], ControllerUser>();
+		if (request.restore) {
+			// Unban / Demote / Unwhitelist players not on the list
+			backup = await this.handleUserBulkExportRequest(new lib.UserBulkExportRequest(request.importType));
+			await this.handleUserBulkRestoreRequest(request, updated);
+		}
+
+		// Will get a user or attempt to create one
+		const getUserOrCreate = (username: string) => {
+			const user = this._controller.userManager.getByName(username);
+			if (user) { return user; }
+			this.user.checkPermission("core.user.create");
+			return this._controller.userManager.createUser(username);
+		};
+
+		// Merge the imported data with existing data, this is strictly additive
+		if (request.importType === "users") {
+			// Update all fields
+			const users = request.users as Extract<typeof request.users, { username: string }[]>;
+			for (const user of users) {
+				const cUser = getUserOrCreate(user.username);
+				if (user.is_admin && !cUser.isAdmin) {
+					cUser.isAdmin = true;
+					updated.set(cUser.id, cUser);
+				}
+				if (user.is_banned && !cUser.isBanned) {
+					cUser.isBanned = true;
+					cUser.banReason = user.ban_reason ?? "";
+					updated.set(cUser.id, cUser);
+				}
+				if (user.is_whitelisted && !cUser.isWhitelisted) {
+					cUser.isWhitelisted = true;
+					updated.set(cUser.id, cUser);
+				}
+			}
+		} else if (request.importType === "bans") {
+			// Bans have extra logic to handle ban reason
+			const users = request.users as (string | { username: string, reason: string })[];
+			for (const user of users) {
+				if (typeof user === "string") {
+					const cUser = getUserOrCreate(user);
+					if (!cUser.isBanned) {
+						cUser.isBanned = true;
+						cUser.banReason = "";
+						updated.set(cUser.id, cUser);
+					}
+				} else {
+					const cUser = getUserOrCreate(user.username);
+					if (!cUser.isBanned) {
+						cUser.isBanned = true;
+						cUser.banReason = user.reason;
+						updated.set(cUser.id, cUser);
+					}
+				}
+			}
+		} else {
+			// Whitelist and admin have the same logic
+			const users = request.users as string[];
+			const prop = request.importType === "admins" ? "isAdmin" : "isWhitelisted";
+			for (const user of users) {
+				const cUser = getUserOrCreate(user);
+				if (!cUser[prop]) {
+					cUser[prop] = true;
+					updated.set(cUser.id, cUser);
+				}
+			}
+		}
+
+		// Send the necessary update events
+		const updatedUsers = [...updated.values()];
+		this._controller.usersUpdated(updatedUsers);
+
+		// Resync the host user lists
+		const adminlist: Set<string> = new Set();
+		const banlist: Map<string, string> = new Map();
+		const whitelist: Set<string> = new Set();
+
+		for (let user of this._controller.userManager.users.values()) {
+			if (user.isAdmin) {
+				adminlist.add(user.id);
+			}
+			if (user.isBanned) {
+				banlist.set(user.id, user.banReason);
+			}
+			if (user.isWhitelisted) {
+				whitelist.add(user.id);
+			}
+		}
+
+		this._controller.sendEvent(
+			new lib.SyncUserListsEvent(adminlist, banlist, whitelist),
+			lib.Address.fromShorthand("allHosts")
+		);
+
+		return backup ?? [];
+	}
+
+	async handleUserBulkExportRequest(request: lib.UserBulkExportRequest) {
+		if (request.exportType === "users") {
+			// Send a full user export
+			const usersToSend = [] as lib.ClusterioUserExport["users"];
+			for (const user of this._controller.userManager.users.values()) {
+				let send = false;
+				const userToSend = { username: user.name } as typeof usersToSend[0];
+				if (user.isBanned) {
+					send = true;
+					userToSend.is_banned = true;
+					if (user.banReason !== "") {
+						userToSend.ban_reason = user.banReason;
+					}
+				}
+				if (user.isAdmin) {
+					send = true;
+					userToSend.is_admin = true;
+				}
+				if (user.isWhitelisted) {
+					send = true;
+					userToSend.is_whitelisted = true;
+				}
+				if (send) {
+					usersToSend.push(userToSend);
+				}
+			}
+			return new lib.ClusterioUserExport(usersToSend);
+		} else if (request.exportType === "bans") {
+			// Bans have extra logic to handle ban reason
+			const usersToSend = [] as Array<string | { username: string, reason: string }>;
+			for (const user of this._controller.userManager.users.values()) {
+				if (user.isBanned) {
+					if (user.banReason && user.banReason !== "") {
+						usersToSend.push({ username: user.name, reason: user.banReason });
+					} else {
+						usersToSend.push(user.name);
+					}
+				}
+			}
+			return usersToSend;
+		}
+
+		// This is a whitelist or admins export
+		const usersToSend = [] as string[];
+		const prop = request.exportType === "admins" ? "isAdmin" : "isWhitelisted";
+		for (const user of this._controller.userManager.users.values()) {
+			if (user[prop]) {
+				usersToSend.push(user.name);
+			}
+		}
+		return usersToSend;
+	}
+
+	async handleControllerUpdateRequest(request: lib.ControllerUpdateRequest) {
+		if (!this._controller.config.get("controller.allow_remote_updates")) {
+			throw new lib.RequestError("Remote updates are disabled on this machine");
+		}
+		return lib.updatePackage("@clusterio/controller");
+	}
+
+	async handlePluginUpdateRequest(request: lib.PluginUpdateRequest) {
+		if (!this._controller.config.get("controller.allow_plugin_updates")) {
+			throw new lib.RequestError("Plugin updates are disabled on this machine");
+		}
+		return await lib.handlePluginUpdate(request.pluginPackage, this._controller.pluginInfos);
+	}
+
+	async handlePluginInstallRequest(request: lib.PluginInstallRequest) {
+		if (!this._controller.config.get("controller.allow_plugin_install")) {
+			throw new lib.RequestError("Plugin installs are disabled on this machine");
+		}
+		return await lib.handlePluginInstall(request.pluginPackage);
+	}
+
+	async handlePluginListRequest(request: lib.PluginListRequest) {
+		return this._controller.pluginInfos.map(pluginInfo => lib.PluginDetails.fromNodeEnvInfo(
+			pluginInfo,
+			this._controller.plugins.has(pluginInfo.name),
+			this._controller.config.get(
+				`${pluginInfo.name}.load_plugin` as keyof lib.ControllerConfigFields
+			) as boolean,
+		));
+	}
+
 	async handleDebugDumpWsRequest(request: lib.DebugDumpWsRequest) {
 		this.ws_dumper = data => {
 			if (this.connector.connected) {

packages/controller/src/Controller.ts

@@ -6,7 +6,7 @@ import type { Static } from "@sinclair/typebox";
 import finalhandler from "finalhandler";
 
 import compression from "compression";
-import events, { EventEmitter } from "events";
+import events from "events";
 import fs from "fs-extra";
 import http from "http";
 import https from "https";
@@ -42,6 +42,14 @@ const logSizeGauge = new Gauge(
 
 type InstanceId = { instanceId: number };
 
+type ControllerDebugEvents = {
+	"message": [ { direction: "in" | "out", content: string } ],
+};
+
+type ControllerEvents = {
+	"stop": [],
+}
+
 /**
  * Manages all controller related operations
  * @alias module:controller/src/Controller
@@ -50,7 +58,6 @@ export default class Controller {
 	clusterLogger: winston.Logger;
 	/** Array of plugin info objects for known plugins */
 	pluginInfos: lib.PluginNodeEnvInfo[];
-	configPath: string;
 	/** Controller config. */
 	config: lib.ControllerConfig;
 	app: Application;
@@ -67,8 +74,8 @@ export default class Controller {
 	wsServer: WsServer;
 	router = new ControllerRouter(this);
 	trustedProxies: BlockList;
-	debugEvents: events.EventEmitter = new events.EventEmitter();
-	private _events: events.EventEmitter = new events.EventEmitter();
+	debugEvents = new events.EventEmitter<ControllerDebugEvents>();
+	private _events = new events.EventEmitter<ControllerEvents>();
 
 	/** Event subscription controller */
 	subscriptions = new lib.SubscriptionController();
@@ -96,13 +103,17 @@ export default class Controller {
 	clusterLogIndex: lib.LogIndex | null = null;
 	clusterLogBuildInterval: ReturnType<typeof setInterval> | null = null;
 
+	// Cache for mod portal requests
+	modPortalCache = new Map<string, { timestamp: number, data: any[] }>();
+
 	static async bootstrap(config: lib.ControllerConfig) {
 		let databaseDirectory = config.get("controller.database_directory");
 		await fs.ensureDir(databaseDirectory);
 
 		const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "systems.json"),
-			lib.SystemInfo.fromJSON.bind(lib.SystemInfo)
+			lib.SystemInfo.fromJSON.bind(lib.SystemInfo),
+			this.migrateSystems,
 		).bootstrap());
 
 		const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
@@ -127,17 +138,28 @@ export default class Controller {
 			lib.ModPack.fromJSON.bind(lib.ModPack)
 		).bootstrap());
 
-		if (modPacks.size === 0) {
-			modPacks.setMany(lib.ModPack.defaultModPacks);
-		}
+		const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "roles.json"),
+			lib.Role.fromJSON.bind(lib.Role),
+			this.migrateRoles,
+		).bootstrap());
 
-		const userManager = new UserManager(config);
+		const userManager = new UserManager(config, roles);
 		await userManager.load(path.join(databaseDirectory, "users.json"));
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
+		// Add default mod packs
+		if (modPacks.size === 0) {
+			modPacks.setMany(lib.ModPack.defaultModPacks);
+		}
+
+		// Add default roles
+		lib.ensureDefaultAdminRole(roles);
+		lib.ensureDefaultPlayerRole(roles);
+
 		return [
 			systems,
 			hosts,
@@ -145,6 +167,7 @@ export default class Controller {
 			saves,
 			modPacks,
 			modStore,
+			roles,
 			userManager,
 		] as const;
 	}
@@ -152,7 +175,6 @@ export default class Controller {
 	constructor(
 		clusterLogger: winston.Logger,
 		pluginInfos: lib.PluginNodeEnvInfo[],
-		configPath: string,
 		config: lib.ControllerConfig,
 
 		/**
@@ -160,6 +182,11 @@ export default class Controller {
 		 * will restart the controller on non-zero exit codes.
 		 */
 		public canRestart: boolean = false,
+		/**
+		 * If true indicates the controller is in recovery mode and should
+		 * disable certain actions such as loading plugins or connecting to hosts
+		 */
+		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
 		/** Mapping of host id to host info */
 		public hosts = new lib.SubscribableDatastore<HostInfo>(),
@@ -171,12 +198,13 @@ export default class Controller {
 		public modPacks = new lib.SubscribableDatastore<lib.ModPack>(),
 		/** Mods stored on the controller */
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
+		/** Mapping of mod pack id to mod pack */
+		public roles = new lib.SubscribableDatastore<lib.Role>(),
 		/** User and roles manager for the cluster */
-		public userManager = new UserManager(config),
+		public userManager = new UserManager(config, roles),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
-		this.configPath = configPath;
 		this.config = config;
 
 		this.app = express();
@@ -200,6 +228,7 @@ export default class Controller {
 		this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this));
 		this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this));
 		this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this));
+		this.subscriptions.handle(lib.RoleUpdatesEvent, this.handleRoleSubscription.bind(this));
 
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
@@ -207,6 +236,7 @@ export default class Controller {
 		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
+		this.roles.on("update", this.rolesUpdated.bind(this));
 	}
 
 	async start(args: ControllerArgs) {
@@ -288,7 +318,7 @@ export default class Controller {
 		let tls_cert = this.config.get("controller.tls_certificate");
 		let tls_key = this.config.get("controller.tls_private_key");
 
-		if (httpsPort && (!tls_cert || !tls_key)) {
+		if (httpsPort && (!tls_cert || !tls_key)) {
 			throw new lib.StartupError(
 				"tls_certificate and tls_private_key must be configure in order to use https_port"
 			);
@@ -520,7 +550,19 @@ export default class Controller {
 				}
 				requests.push(hostConnection.send(new lib.SystemInfoRequest()));
 			}
-			requests.push(lib.gatherSystemInfo("controller", this.canRestart));
+			if (!this.config.restartRequired) {
+				// If a restart isn't already required, then test if a new version is installed
+				try {
+					const runningVersion = this.config.get("controller.version");
+					const packageJson = await fs.readJSON(path.join(__dirname, "..", "package.json"));
+					if (runningVersion !== packageJson.version) {
+						this.config.restartRequired = true;
+					}
+				} catch (err: any) {
+					logger.warn(`Failed to read package json:\n${err.stack ?? err.message}`);
+				}
+			}
+			requests.push(lib.gatherSystemInfo("controller", this.canRestart, this.config.restartRequired));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
 				this.systems.set(metric);
@@ -535,6 +577,10 @@ export default class Controller {
 			clearInterval(this.autosaveInterval);
 			this.autosaveInterval = undefined;
 		}
+		if (this.recoveryMode) {
+			logger.warn("Recovery | autosaving disabled");
+			return;
+		}
 		const autosaveIntervalSeconds = this.config.get("controller.autosave_interval");
 		if (autosaveIntervalSeconds > 0) {
 			this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000);
@@ -560,17 +606,14 @@ export default class Controller {
 	});
 
 	private async _saveDataInternal() {
-		if (this.config.dirty) {
-			this.config.dirty = false;
-			await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t"));
-		}
-
 		await Promise.all([
+			this.config.save(),
 			this.systems.save(),
 			this.hosts.save(),
 			this.instances.save(),
 			this.saves.save(),
 			this.modPacks.save(),
+			this.roles.save(),
 		]);
 
 		if (this.userManager.dirty) {
@@ -580,10 +623,23 @@ export default class Controller {
 		await lib.invokeHook(this.plugins, "onSaveData");
 	}
 
+	static migrateSystems(rawJson: unknown[]): Static<typeof lib.SystemInfo.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof lib.SystemInfo.jsonSchema>[];
+		return serialized.map(json => {
+			if (!json.canRestart) { // Added in 2.0.0.alpha.17
+				json.canRestart = false;
+			}
+			if (!json.restartRequired) { // Added in 2.0.0.alpha.21
+				json.restartRequired = false;
+			}
+			return json;
+		});
+	}
+
 	static migrateHosts(rawJson: unknown[]): Static<typeof HostInfo.jsonSchema>[] {
 		let serialized = rawJson as any;
 
-		// migrate from pre alpha.19 format
+		// New format 2.0.0.alpha.19
 		if (serialized.length && serialized[0] instanceof Array) {
 			serialized = serialized.map((e: any) => e[1]);
 		}
@@ -602,7 +658,7 @@ export default class Controller {
 	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceInfo.jsonSchema>[] {
 		const serialized = rawJson as Static<typeof InstanceInfo.jsonSchema>[];
 		return serialized.map(json => {
-			if (!json.config) { // migrate: from pre Alpha 14 format.
+			if (!json.config) { // New format 2.0.0.alpha.14
 				return { config: json as any, status: "running" }; // Use running to force updatedAtMs
 			}
 			return json;
@@ -618,6 +674,20 @@ export default class Controller {
 		return instance;
 	}
 
+	static migrateRoles(rawJson: unknown[]): Static<typeof lib.Role.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof lib.Role.jsonSchema>[];
+		return serialized.map(json => {
+			json.permissions = json.permissions.map(permission => {
+				// migrate: core.instance.save.list.subscribe was renamed in alpha 17
+				if (permission === "core.instance.save.list.subscribe") {
+					return "core.instance.save.subscribe";
+				}
+				return permission;
+			});
+			return json;
+		});
+	}
+
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -1021,27 +1091,34 @@ export default class Controller {
 		}
 	}
 
-	/**
-	 * Notify connected control clients with the given role that the
-	 * permissions may have changed.
-	 * @param role - Role permisions updated for.
-	 */
-	rolePermissionsUpdated(role: lib.Role) {
-		for (let controlConnection of this.wsServer.controlConnections.values()) {
-			if (controlConnection.user.roles.has(role)) {
-				controlConnection.send(
-					new lib.AccountUpdateEvent(
-						[...controlConnection.user.roles].map(r => ({
-							name: r.name,
-							id: r.id,
-							permissions: [...r.permissions],
-						}))
-					)
-				);
+	rolesUpdated(roles: lib.Role[]) {
+		this.subscriptions.broadcast(new lib.RoleUpdatesEvent(roles));
+		// lib.invokeHook(this.plugins, "onRolesUpdated", roles); // This doesn't exist at the moment
+		// Notify connected control clients with the given role that the permissions may have changed.
+		for (const role of roles) {
+			for (let controlConnection of this.wsServer.controlConnections.values()) {
+				if (controlConnection.user.roles.has(role)) {
+					controlConnection.send(
+						new lib.AccountUpdateEvent(
+							[...controlConnection.user.roles].map(r => ({
+								name: r.name,
+								id: r.id,
+								permissions: [...r.permissions],
+							}))
+						)
+					);
+				}
 			}
 		}
 	}
 
+	async handleRoleSubscription(request: lib.SubscriptionRequest) {
+		const roles = [...this.roles.values()].filter(
+			role => role.updatedAtMs > request.lastRequestTimeMs,
+		);
+		return roles.length ? new lib.RoleUpdatesEvent(roles) : null;
+	}
+
 	async loadPlugins() {
 		for (let pluginInfo of this.pluginInfos) {
 			try {
@@ -1055,6 +1132,11 @@ export default class Controller {
 				continue;
 			}
 
+			if (this.recoveryMode) {
+				logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
+				continue;
+			}
+
 			let ControllerPluginClass = BaseControllerPlugin;
 			try {
 				if (pluginInfo.controllerEntrypoint) {

packages/controller/src/ControllerUser.ts

@@ -1,10 +1,13 @@
 import { Static, Type } from "@sinclair/typebox";
-import { IControllerUser, PermissionError, PlayerStats, Role, User, permissions } from "@clusterio/lib";
-import type UserManager from "./UserManager";
+
+import {
+	IControllerUser, PermissionError, PlayerStats,
+	Role, SubscribableDatastore, User, permissions,
+} from "@clusterio/lib";
 
 export default class ControllerUser extends User implements IControllerUser {
 	constructor(
-		public userManager: UserManager,
+		private _controllerRoles: SubscribableDatastore<Role>,
 		/** Unix time in seconds the user token must be issued after to be valid.  */
 		public tokenValidAfter = 0,
 		...args: ConstructorParameters<typeof User>
@@ -17,7 +20,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		token_valid_after: Type.Optional(Type.Number()),
 	});
 
-	static fromJSON(json: Static<typeof this.jsonSchema>, userManager: UserManager) {
+	static fromJSON(json: Static<typeof this.jsonSchema>, controllerRoles: SubscribableDatastore<Role>) {
 		const roleIds = new Set<number>(json.roles ?? []);
 		const instanceStats = new Map(
 			(json.instance_stats ?? []).map(
@@ -26,7 +29,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		);
 		const playerStats = User._calculatePlayerStats(instanceStats);
 		return new this(
-			userManager,
+			controllerRoles,
 			json.token_valid_after,
 			json.name,
 			roleIds,
@@ -57,7 +60,7 @@ export default class ControllerUser extends User implements IControllerUser {
 
 	get roles(): ReadonlySet<Readonly<Role>> {
 		return new Set([...this.roleIds].map(
-			id => this.userManager.roles.get(id)
+			id => this._controllerRoles.get(id)
 		).filter(
 			(r): r is Role => Boolean(r)
 		));
@@ -79,7 +82,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		}
 
 		for (let roleId of this.roleIds) {
-			let role = this.userManager.roles.get(roleId);
+			let role = this._controllerRoles.get(roleId);
 			if (!role) {
 				continue;
 			}

packages/controller/src/UserManager.ts

@@ -10,7 +10,6 @@ import ControllerUser from "./ControllerUser";
  * @alias module:controller/src/UserManager
  */
 export default class UserManager {
-	roles: Map<lib.Role["id"], lib.Role> = new Map();
 	users: Map<ControllerUser["id"], ControllerUser> = new Map();
 	dirty = false;
 
@@ -20,7 +19,8 @@ export default class UserManager {
 	onlineUsers = new Set<ControllerUser>();
 
 	constructor(
-		private _config: lib.ControllerConfig
+		private _config: lib.ControllerConfig,
+		private _roles: lib.SubscribableDatastore<lib.Role>,
 	) {
 	}
 
@@ -31,14 +31,15 @@ export default class UserManager {
 	async load(filePath: string): Promise<void> {
 		try {
 			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-			for (let serializedRole of content.roles) {
-				let role = lib.Role.fromJSON(serializedRole);
-				this.roles.set(role.id, role);
+			if ("roles" in content) { // Split file format after 2.0.0-alpha.22
+				this._roles.setMany(content.roles.map((serializedRole: any) => lib.Role.fromJSON(serializedRole)));
+				await this._roles.save(); // Force a save to prevent data loss
+				content = content.users;
 			}
 
 			let duplicates = 0;
-			for (let serializedUser of content.users) {
-				let user = ControllerUser.fromJSON(serializedUser, this);
+			for (const serializedUser of content) {
+				const user = ControllerUser.fromJSON(serializedUser, this._roles);
 				const existingUser = this.users.get(user.id);
 				if (existingUser) {
 					// Required migration to all lowercase ids in alpha 19
@@ -65,30 +66,14 @@ export default class UserManager {
 			if (err.code !== "ENOENT") {
 				throw err;
 			}
-
-			// Create default roles if loading failed
-			lib.ensureDefaultAdminRole(this.roles);
-			lib.ensureDefaultPlayerRole(this.roles);
 		}
 	}
 
 	async save(filePath: string): Promise<void> {
-		let serializedRoles = [];
-		for (let role of this.roles.values()) {
-			serializedRoles.push(role.toJSON());
-		}
-
-		let serializedUsers = [];
-		for (let user of this.users.values()) {
-			serializedUsers.push(user.toJSON(true));
-		}
-
-		let serialized = {
-			users: serializedUsers,
-			roles: serializedRoles,
-		};
 		this.dirty = false;
-		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
+		await lib.safeOutputFile(filePath,
+			JSON.stringify([...this.users.values()].map(user => user.toJSON(true)), null, "\t")
+		);
 	}
 
 	/**
@@ -103,11 +88,11 @@ export default class UserManager {
 
 		let roles = new Set<number>();
 		let defaultRoleId = this._config.get("controller.default_role_id");
-		if (defaultRoleId !== null && this.roles.has(defaultRoleId)) {
+		if (defaultRoleId !== null && this._roles.has(defaultRoleId)) {
 			roles.add(defaultRoleId);
 		}
 
-		let user = new ControllerUser(this, 0, name, roles);
+		let user = new ControllerUser(this._roles, 0, name, roles);
 		this.users.set(user.id, user);
 		this.dirty = true;
 		return user;

packages/controller/src/WsServer.ts

@@ -239,6 +239,13 @@ ${err.stack}`
 	}
 
 	async registerHost(data: lib.RegisterHostData, socket: WebSocket, req: IncomingMessage) {
+		if (this.controller.recoveryMode) {
+			logger.warn(`Recovery | rejected host connection attempt for ${this.remoteAddr(req)}`);
+			wsRejectedConnectionsCounter.inc();
+			socket.close(lib.ConnectionClosed.RecoveryMode, "Connection Refused: Controller in recovery mode");
+			return;
+		}
+
 		try {
 			let tokenPayload = jwt.verify(
 				data.token,

packages/controller/src/WsServerConnector.ts

@@ -1,6 +1,7 @@
 import type WebSocket from "ws";
 
 import events from "events";
+import assert from "assert/strict";
 
 import * as lib from "@clusterio/lib";
 const { logger } = lib;
@@ -113,8 +114,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 
 	_timedOut() {
 		logger.verbose("Connector | Connection timed out");
-		this._reset();
-		this.emit("close");
+		this._close();
 	}
 
 	_attachSocketHandlers() {
@@ -130,8 +130,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 			this._socket = null;
 
 			if (this._closing) {
-				this._reset();
-				this.emit("close");
+				this._close();
 
 			} else {
 				this._state = "resuming";
@@ -187,8 +186,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 		}
 
 		if (this._state === "resuming") {
-			this._reset();
-			this.emit("close");
+			this._close();
 			return;
 		}
 
@@ -223,8 +221,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 	 */
 	setClosing() {
 		if (this._state === "resuming") {
-			this._reset();
-			this.emit("close");
+			this._close();
 			return;
 		}
 

packages/controller/src/routes.ts

@@ -266,6 +266,12 @@ async function uploadExport(req: Request, res: Response) {
 	res.sendStatus(200);
 }
 
+type ProxyStreamEvents = {
+	"error": [ err: any ],
+	"close": [],
+	"timeout": [],
+	"source": [],
+}
 
 export interface ProxyStream {
 	id: string;
@@ -274,7 +280,7 @@ export interface ProxyStream {
 	size?: string;
 	mime?: string;
 	filename: string | null;
-	events: events.EventEmitter;
+	events: events.EventEmitter<ProxyStreamEvents>;
 	timeout: ReturnType<typeof setTimeout>;
 }
 
@@ -690,4 +696,5 @@ export const webRoutes = [
 	"/roles/:id/view",
 	"/plugins",
 	"/plugins/:name/view",
+	"/error",
 ];

packages/create/create.js

@@ -6,7 +6,6 @@ const fs = require("fs-extra");
 const inquirer = require("inquirer");
 const os = require("os");
 const path = require("path");
-const phin = require("phin");
 const stream = require("stream");
 const util = require("util");
 const yargs = require("yargs");
@@ -624,8 +623,8 @@ async function inquirerMissingArgs(args) {
 			answers.publicAddress = args.publicAddress;
 		} else {
 			try {
-				let result = await phin("https://api.ipify.org/");
-				myIp = result.body.toString();
+				let result = await fetch("https://api.ipify.org/");
+				myIp = await result.text();
 			} catch (err) { /* ignore */ }
 		}
 		answers = await inquirer.prompt([
@@ -689,6 +688,17 @@ async function inquirerMissingArgs(args) {
 		}
 	}
 
+	if (["standalone", "controller", "host"].includes(answers.mode)) {
+		answers = await inquirer.prompt([
+			{
+				type: "confirm",
+				name: "remoteNpm",
+				message: "Allow remote updates via npm?",
+				default: true,
+			},
+		], answers);
+	}
+
 	if (answers.mode === "plugin-template") {
 		if (args["plugin-template"] && args["plugin-template"].length > 0) {
 			answers.pluginTemplate = args["plugin-template"];
@@ -741,7 +751,7 @@ async function inquirerMissingArgs(args) {
 }
 
 async function downloadLinuxServer() {
-	let res = await phin("https://factorio.com/get-download/stable/headless/linux64");
+	let res = await fetch("https://factorio.com/get-download/stable/headless/linux64");
 
 	const url = new URL(res.headers.location);
 	// get the filename of the latest factorio archive from redirected url
@@ -764,14 +774,11 @@ async function downloadLinuxServer() {
 		await fs.ensureDir(tmpDir);
 
 		// follow the redirect
-		res = await phin({
-			url: url.href,
-			stream: true,
-		});
+		res = await fetch(url.href);
 
 		logger.info("Downloading latest Factorio server release. This may take a while.");
 		const writeStream = fs.createWriteStream(tmpArchivePath);
-		res.pipe(writeStream);
+		stream.Readable.fromWeb(res.body).pipe(writeStream);
 
 		await finished(writeStream);
 
@@ -828,6 +835,9 @@ async function main() {
 		.option("factorio-dir", {
 			nargs: 1, describe: "Path to Factorio installation [standalone/host]", type: "string",
 		})
+		.option("remote-npm", {
+			nargs: 0, description: "Allow remote updates via npm [standalone/controller/host]", type: "boolean",
+		})
 		.option("plugins", {
 			array: true, describe: "Plugins to install", type: "string",
 		})
@@ -893,6 +903,8 @@ async function main() {
 		logger.info("Setting up controller");
 		await execController(["bootstrap", "create-admin", answers.admin]);
 		await execController(["config", "set", "controller.http_port", answers.httpPort]);
+		await execController(["config", "set", "controller.allow_remote_updates", answers.remoteNpm]);
+		await execController(["config", "set", "controller.allow_plugin_updates", answers.remoteNpm]);
 		let result = await execController(["bootstrap", "generate-user-token", answers.admin]);
 		adminToken = result.stdout.split("\n").slice(-2)[0];
 	}
@@ -912,6 +924,8 @@ async function main() {
 		await execHost(["config", "set", "host.controller_token", hostToken]);
 		await execHost(["config", "set", "host.public_address", answers.publicAddress]);
 		await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]);
+		await execHost(["config", "set", "host.allow_remote_updates", answers.remoteNpm]);
+		await execHost(["config", "set", "host.allow_plugin_updates", answers.remoteNpm]);
 	}
 
 	if (answers.mode === "host") {
@@ -923,6 +937,8 @@ async function main() {
 		await execHost(["config", "set", "host.controller_token", answers.controllerToken]);
 		await execHost(["config", "set", "host.public_address", answers.publicAddress]);
 		await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]);
+		await execHost(["config", "set", "host.allow_remote_updates", answers.remoteNpm]);
+		await execHost(["config", "set", "host.allow_plugin_updates", answers.remoteNpm]);
 	}
 
 	if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) {

packages/create/package.json

@@ -3,7 +3,7 @@
 	"description": "Installer for clusterio",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"main": "create.js",
 	"bin": {
 		"clusteriocreate": "./create.js"
@@ -19,7 +19,6 @@
 		"chalk": "^4.1.2",
 		"fs-extra": "^11.3.0",
 		"inquirer": "^8.2.4",
-		"phin": "^3.7.0",
 		"yargs": "^17.7.2"
 	},
 	"publishConfig": {

packages/ctl/ctl.ts

@@ -143,7 +143,19 @@ interface CtlArguments {
 	pluginList: string,
 }
 
-async function startControl() {
+interface InitializeParameters {
+	args: CtlArguments;
+	shouldRun: boolean;
+	ctlPlugins?: Map<string, BaseCtlPlugin>;
+	rootCommands?: lib.CommandTree;
+	controlConfig?: lib.ControlConfig;
+}
+
+export async function initialize(
+	argv: string | string[],
+	ctlPlugins?: Map<string, BaseCtlPlugin>,
+	noLoggerTransport?: boolean,
+): Promise<InitializeParameters> {
 	yargs
 		.scriptName("clusterioctl")
 		.usage("$0 <command> [options]")
@@ -175,53 +187,56 @@ async function startControl() {
 
 	// Parse the args first to get the configured plugin list.
 	// eslint-disable-next-line node/no-sync
-	let args = yargs.parseSync() as CtlArguments;
+	let args = yargs.parseSync(argv) as CtlArguments;
 
-	// Log stream for the ctl session.
-	logger.add(
-		new ConsoleTransport({
-			errorLevels: Object.keys(levels),
-			level: args.logLevel,
-			format: new lib.TerminalFormat(),
-		})
-	);
-	lib.handleUnhandledErrors();
-
-	logger.verbose(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = await lib.loadPluginList(args.pluginList);
-
-	// If the command is plugin management we don't try to load plugins
-	if (args._[0] === "plugin") {
-		await lib.handlePluginCommand(args, pluginList, args.pluginList);
-		return;
+	// Log stream for the ctl session. Skipped in testing.
+	if (!noLoggerTransport) {
+		logger.add(
+			new ConsoleTransport({
+				errorLevels: Object.keys(levels),
+				level: args.logLevel,
+				format: new lib.TerminalFormat(),
+			})
+		);
+		lib.handleUnhandledErrors();
 	}
 
-	logger.verbose("Loading Plugins");
-	let ctlPlugins = await loadPlugins(pluginList);
+	// Discover and load plugins. This check exists to allow tests to inject plugins.
+	if (!ctlPlugins || args._[0] === "plugin") {
+		logger.verbose(`Loading available plugins from ${args.pluginList}`);
+		const pluginList = await lib.loadPluginList(args.pluginList);
 
-	// Add all cluster management commands including ones from plugins
-	let rootCommands = await commands.registerCommands(ctlPlugins, yargs);
+		// If the command is plugin management we don't try to load plugins
+		if (args._[0] === "plugin") {
+			await lib.handlePluginCommand(args, pluginList, args.pluginList);
+			return { args, shouldRun: false };
+		}
 
-	// Reparse after commands have been added with help and strict checking.
+		logger.verbose("Loading Plugins");
+		ctlPlugins = await loadPlugins(pluginList);
+	}
+
+	// Add all commands including from plugins and reparse with help and strict checking.
+	const rootCommands = await commands.registerCommands(ctlPlugins, yargs);
 	args = yargs
 		.help()
 		.strict()
-		.parse() as CtlArguments
+		.parse(argv) as CtlArguments
 	;
 
-	logger.verbose(`Loading config from ${args.config}`);
 	let controlConfig;
+	const controlConfigPath = args.config;
+	logger.verbose(`Loading config from ${controlConfigPath}`);
 	try {
-		const jsonConfig = JSON.parse(await fs.readFile(args.config, "utf8"));
-		controlConfig = lib.ControlConfig.fromJSON(jsonConfig, "control");
+		controlConfig = await lib.ControlConfig.fromFile("control", controlConfigPath);
 
 	} catch (err: any) {
 		if (err.code === "ENOENT") {
 			logger.verbose("Config not found, initializing new config");
-			controlConfig = new lib.ControlConfig("control");
+			controlConfig = new lib.ControlConfig("control", undefined, controlConfigPath);
 
 		} else {
-			throw new lib.StartupError(`Failed to load ${args.config}: ${err.message}`);
+			throw new lib.StartupError(`Failed to load ${args.config}: ${err.stack ?? err.message ?? err}`);
 		}
 	}
 
@@ -232,19 +247,35 @@ async function startControl() {
 
 	// Handle the control-config command before trying to connect.
 	if (args._[0] === "control-config") {
-		await lib.handleConfigCommand(args, controlConfig, args.config);
-		return;
+		await lib.handleConfigCommand(args, controlConfig);
+		return { args, controlConfig, ctlPlugins, rootCommands, shouldRun: false };
 	}
 
-	// Determine which command is being executed.
+	return { args, controlConfig, ctlPlugins, rootCommands, shouldRun: true };
+}
+
+export function selectTargetCommand(args: CtlArguments, rootCommands: lib.CommandTree): lib.Command {
 	let commandPath = [...args._] as string[];
 	let targetCommand: lib.CommandTree | lib.Command = rootCommands;
 	while (commandPath.length && targetCommand instanceof lib.CommandTree) {
 		targetCommand = targetCommand.get(commandPath.shift()!)!;
 	}
 	assert(targetCommand instanceof lib.Command);
+	return targetCommand;
+}
+
+async function startControl() {
+	const {
+		args,
+		shouldRun,
+		ctlPlugins,
+		rootCommands,
+		controlConfig,
+	} = await initialize(process.argv.slice(2));
+	if (!shouldRun || !ctlPlugins || !rootCommands || !controlConfig) {
+		return;
+	}
 
-	// The remaining commands require connecting to the controller.
 	if (!controlConfig.get("control.controller_url") || !controlConfig.get("control.controller_token")) {
 		logger.error("Missing URL and/or token to connect with.  See README.md for setting up access.");
 		process.exitCode = 1;
@@ -263,6 +294,7 @@ async function startControl() {
 		tlsCa,
 		controlConfig.get("control.controller_token")!,
 	);
+
 	let control = new Control(controlConnector, controlConfig, tlsCa, ctlPlugins);
 	try {
 		await controlConnector.connect();
@@ -284,6 +316,7 @@ async function startControl() {
 	});
 
 	try {
+		const targetCommand = selectTargetCommand(args, rootCommands);
 		await targetCommand.run(args, control);
 
 	} catch (err) {

packages/ctl/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio command line management tool",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",

packages/ctl/src/commands.ts

@@ -95,6 +95,19 @@ controllerCommands.add(new lib.Command({
 		await control.send(new lib.ControllerRestartRequest());
 	},
 }));
+controllerCommands.add(new lib.Command({
+	definition: ["update", "Update the controller", (yargs) => {
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { restart: boolean }, control: Control) {
+		await control.send(new lib.ControllerUpdateRequest());
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Controller updated; a restart is required to apply the changes.");
+		}
+	},
+}));
 
 const controllerConfigCommands = new lib.CommandTree({
 	name: "config", alias: ["c"], description: "controller config management",
@@ -184,28 +197,24 @@ controllerConfigCommands.add(new lib.Command({
 		editorSpawn.on("data", (data) => {
 			process.stdout.pipe(data);
 		});
-		let doneEmitter = new events.EventEmitter();
-		editorSpawn.on("exit", async (exit) => {
-			const data = await fs.readFile(tmpFile, "utf8");
-			const final = await configToKeyVal(data);
-			for (let index in final) {
-				if (index in final) {
-					try {
-						await control.send(new lib.ControllerConfigSetFieldRequest(
-							index,
-							final[index],
-						));
-					} catch (err) {
-						// eslint-disable-next-line
-						print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-						print(err);
-						// If the string is empty, it's better to just print "" instead of nothing
-					}
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.ControllerConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					print(err);
+					// If the string is empty, it's better to just print "" instead of nothing
 				}
 			}
-			doneEmitter.emit("dot_on_done");
-		});
-		await events.once(doneEmitter, "dot_on_done");
+		}
 		try {
 			await fs.unlink(tmpFile);
 		} catch (err) {
@@ -234,6 +243,35 @@ controllerPluginCommands.add(new lib.Command({
 		print(asTable(response.body));
 	},
 }));
+controllerPluginCommands.add(new lib.Command({
+	definition: ["update <plugin>", "Update a plugin on the controller", (yargs) => {
+		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
+		await control.sendTo("controller", new lib.PluginUpdateRequest(args.plugin));
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Plugin updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+controllerPluginCommands.add(new lib.Command({
+	definition: ["install <plugin>", "Install a plugin on the controller", (yargs) => {
+		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
+		await control.sendTo("controller", new lib.PluginInstallRequest(args.plugin));
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Plugin installed; a restart is required to apply the changes.");
+		}
+	},
+}));
+
 controllerCommands.add(controllerPluginCommands);
 
 
@@ -322,6 +360,22 @@ hostCommands.add(new lib.Command({
 		}
 	},
 }));
+hostCommands.add(new lib.Command({
+	definition: ["update <host>", "Update the host", (yargs) => {
+		yargs.positional("host", { describe: "Host to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostUpdateRequest());
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Host updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+
 
 const hostConfigCommands = new lib.CommandTree({
 	name: "config", alias: ["c"], description: "Host config management",
@@ -429,33 +483,29 @@ hostConfigCommands.add(new lib.Command({
 		editorSpawn.on("data", (data) => {
 			process.stdout.pipe(data);
 		});
-		let doneEmitter = new events.EventEmitter();
-		editorSpawn.on("exit", async (exit) => {
-			const data = await fs.readFile(tmpFile, "utf8");
-			const final = await configToKeyVal(data);
-			for (let index in final) {
-				if (index in final) {
-					try {
-						await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(
-							index,
-							final[index],
-						));
-					} catch (err) {
-						// eslint-disable-next-line
-						print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-						// If the string is empty, it's better to just print "" instead of nothing
-						print("This message shouldn't normally appear; if the below message does not indicate it");
-						print("was a user mistake, please report it to the clustorio devs.");
-						// added this because it could be a missed entry in disallowedList
-						// i've added all the vanilla clustorio configs, but there may be
-						// some modded ones that need to be added.
-						print(err);
-					}
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
 				}
 			}
-			doneEmitter.emit("dot_on_done");
-		});
-		await events.once(doneEmitter, "dot_on_done");
+		}
 		try {
 			await fs.unlink(tmpFile);
 		} catch (err) {
@@ -464,9 +514,60 @@ hostConfigCommands.add(new lib.Command({
 		}
 	},
 }));
+
 hostCommands.add(hostConfigCommands);
 
 
+const hostPluginCommands = new lib.CommandTree({
+	name: "plugin", alias: ["p"], description: "host plugin inspection",
+});
+hostPluginCommands.add(new lib.Command({
+	definition: ["list <host>", "List plugins on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to list for", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		const plugins = await control.sendTo({ hostId }, new lib.PluginListRequest());
+		print(asTable(plugins.map(p => ({
+			title: p.title, version: p.version, loaded: p.loaded, enabled: p.enabled, npmPackage: p.npmPackage,
+		}))));
+	},
+}));
+hostPluginCommands.add(new lib.Command({
+	definition: ["update <host> <plugin>", "Update a plugin on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to update on", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.PluginUpdateRequest(args.plugin));
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Plugin updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+hostPluginCommands.add(new lib.Command({
+	definition: ["install <host> <plugin>", "Install a plugin on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to install on", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.PluginInstallRequest(args.plugin));
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Plugin installed; a restart is required to apply the changes.");
+		}
+	},
+}));
+hostCommands.add(hostPluginCommands);
+
+
 const instanceCommands = new lib.CommandTree({
 	name: "instance", alias: ["i"], description: "Instance management",
 });
@@ -484,16 +585,21 @@ instanceCommands.add(new lib.Command({
 		yargs.positional("name", { describe: "Instance name", type: "string" });
 		yargs.options({
 			"id": { type: "number", nargs: 1, describe: "Instance id" },
+			"from": { type: "number", nargs: 1, describe: "Clone config from instance id" },
 		});
 	}],
-	handler: async function(args: { name: string, id?: number }, control: Control) {
+	handler: async function(args: { name: string, id?: number, from?: number, }, control: Control) {
 		let instanceConfig = new lib.InstanceConfig("control");
 		if (args.id !== undefined) {
 			instanceConfig.set("instance.id", args.id);
 		}
 		instanceConfig.set("instance.name", args.name);
-		const serializedConfig = instanceConfig.toRemote("controller");
-		await control.send(new lib.InstanceCreateRequest(serializedConfig));
+		await control.send(new lib.InstanceCreateRequest(
+			instanceConfig.toRemote("controller", [
+				"instance.id", "instance.name",
+			]),
+			args.from
+		));
 	},
 }));
 
@@ -618,34 +724,30 @@ instanceConfigCommands.add(new lib.Command({
 		editorSpawn.on("data", (data) => {
 			process.stdout.pipe(data);
 		});
-		let doneEmitter = new events.EventEmitter();
-		editorSpawn.on("exit", async (exit) => {
-			const data = await fs.readFile(tmpFile, "utf8");
-			const final = await configToKeyVal(data);
-			for (let index in final) {
-				if (index in final) {
-					try {
-						await control.send(new lib.InstanceConfigSetFieldRequest(
-							instanceId,
-							index,
-							final[index],
-						));
-					} catch (err) {
-						// eslint-disable-next-line
-						print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-						// If the string is empty, it's better to just print "" instead of nothing
-						print("This message shouldn't normally appear; if the below message does not indicate it");
-						print("was a user mistake, please report it to the clustorio devs.");
-						// added this because it could be a missed entry in disallowedList
-						// i've added all the vanilla clustorio configs, but there may be
-						// some modded ones that need to be added.
-						print(err);
-					}
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.InstanceConfigSetFieldRequest(
+						instanceId,
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
 				}
 			}
-			doneEmitter.emit("dot_on_done");
-		});
-		await events.once(doneEmitter, "dot_on_done");
+		}
 		try {
 			await fs.unlink(tmpFile);
 		} catch (err) {
@@ -1836,6 +1938,122 @@ userCommands.add(new lib.Command({
 	},
 }));
 
+async function handleImportOrRestore(args: {
+	filepath: string,
+	users: boolean,
+	bans: boolean,
+	admins: boolean,
+	whitelist: boolean,
+	noBackup?: boolean,
+}, control: Control, restore: boolean) {
+	let importType: ConstructorParameters<typeof lib.UserBulkImportRequest>[0];
+	const data = JSON.parse((await fs.readFile(args.filepath)).toString());
+	const optionCount = [args.users, args.bans, args.admins, args.whitelist]
+		.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
+
+	if (optionCount > 1) {
+		throw new lib.CommandError("Can not specify multiple options");
+	}
+
+	// Assign based on options or attempt to guess based on filename
+	const filename = path.basename(args.filepath);
+	if (args.users || (optionCount === 0 && data.export_version)) {
+		importType = "users";
+	} else if (args.bans || (optionCount === 0 && filename.includes("ban"))) {
+		importType = "bans";
+	} else if (args.admins || (optionCount === 0 && filename.includes("admin"))) {
+		importType = "admins";
+	} else if (args.whitelist || (optionCount === 0 && filename.includes("whitelist"))) {
+		importType = "whitelist";
+	} else {
+		throw new lib.CommandError("Unknown json file, please specify an option");
+	}
+
+	const backup = await control.send(
+		new lib.UserBulkImportRequest(importType, importType === "users" ? data.users : data, restore)
+	);
+
+	print(`${restore ? "Restored" : "Imported"} ${importType} from ${args.filepath}`);
+	if (restore && backup && !args.noBackup) {
+		await fs.writeJSON(`${importType}-backup.json`, backup, { spaces: 2 });
+		print(`Wrote backup to ${importType}-backup.json`);
+	}
+}
+
+userCommands.add(new lib.Command({
+	definition: ["import <filepath>", "Import user data from a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
+		yargs.options({
+			"users": { describe: "Import users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Import banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Import adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Import whitelist json", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: any, control: any) {
+		await handleImportOrRestore(args, control, false);
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["restore <filepath>", "Restore user data from a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
+		yargs.options({
+			"users": { describe: "Restore users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Restore banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Restore adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Restore whitelist json", nargs: 0, type: "boolean", default: false },
+			"no-backup": { describe: "Don't save a backup to the cwd", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: any, control: any) {
+		await handleImportOrRestore(args, control, true);
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["export <filepath>", "Export user data to a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to save to", type: "string" });
+		yargs.options({
+			"users": { describe: "Export users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Export banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Export adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Export whitelist json", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: {
+		filepath: string,
+		users: boolean,
+		bans: boolean,
+		admins: boolean,
+		whitelist: boolean,
+	}, control: Control) {
+		let exportType = "users" as ConstructorParameters<typeof lib.UserBulkExportRequest>[0];
+		const optionCount = [args.bans, args.admins, args.whitelist]
+			.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
+
+		if (optionCount > 1) {
+			throw new lib.CommandError("Can not specify multiple options");
+		}
+
+		// Assign based on options or attempt to guess based on filename
+		if (args.users) {
+			exportType = "users";
+		} else if (args.bans) {
+			exportType = "bans";
+		} else if (args.admins) {
+			exportType = "admins";
+		} else if (args.whitelist) {
+			exportType = "whitelist";
+		}
+
+		fs.writeJSON(args.filepath, await control.send(
+			new lib.UserBulkExportRequest(exportType)
+		), { spaces: 2 });
+		print(`Exported ${exportType} to ${args.filepath}`);
+	},
+}));
+
 const logCommands = new lib.CommandTree({ name: "log", description: "Log inspection" });
 logCommands.add(new lib.Command({
 	definition: ["follow", "follow cluster log", (yargs) => {

packages/host/host.ts

@@ -120,6 +120,10 @@ async function startHost() {
 				type: "boolean", nargs: 0, default: false,
 				describe: "Indicate that a process monitor will restart this host on failure",
 			});
+			yargs.option("recovery", {
+				type: "boolean", nargs: 0, default: false,
+				describe: "Start the host in recovery mode with all plugins disabled and controller disconnected",
+			});
 		})
 		.demandCommand(1, "You need to specify a command to run")
 		.strict()
@@ -144,6 +148,9 @@ async function startHost() {
 	let command = args._[0];
 	if (command === "run") {
 		logger.info(`Starting Clusterio host ${version}`);
+		if (args.recovery) {
+			logger.warn("Host recovery mode enabled. Some features will be disabled.");
+		}
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
@@ -160,29 +167,30 @@ async function startHost() {
 	lib.registerPluginMessages(pluginInfos);
 	lib.addPluginConfigFields(pluginInfos);
 
-	logger.info(`Loading config from ${args.config}`);
 	let hostConfig;
+	const hostConfigPath = args.config;
+	logger.info(`Loading config from ${hostConfigPath}`);
 	try {
-		const jsonConfig = JSON.parse(await fs.readFile(args.config, "utf8"));
-		hostConfig = lib.HostConfig.fromJSON(jsonConfig, "host");
+		hostConfig = await lib.HostConfig.fromFile("host", hostConfigPath);
 
 	} catch (err: any) {
 		if (err.code === "ENOENT") {
 			logger.info("Config not found, initializing new config");
-			hostConfig = new lib.HostConfig("host");
+			hostConfig = new lib.HostConfig("host", undefined, hostConfigPath);
 
 		} else {
-			throw new lib.StartupError(`Failed to load ${args.config}: ${err.message}`);
+			throw new lib.StartupError(`Failed to load ${args.config}: ${err.stack ?? err.message ?? err}`);
 		}
 	}
 
+	hostConfig.set("host.version", version); // Allows tracking last loaded version
+
 	if (command === "config") {
-		await lib.handleConfigCommand(args, hostConfig, args.config);
+		await lib.handleConfigCommand(args, hostConfig);
 		return;
 	}
 
 	// If we get here the command was run
-
 	await fs.ensureDir(hostConfig.get("host.instances_directory"));
 	await fs.ensureDir(hostConfig.get("host.mods_directory"));
 	await fs.ensureDir("modules");
@@ -221,11 +229,11 @@ async function startHost() {
 	let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos);
 	host = new Host(
 		hostConnector,
-		args.config,
 		hostConfig,
 		tlsCa,
 		pluginInfos,
 		Boolean(args.canRestart),
+		Boolean(args.recovery),
 		...await Host.bootstrap(hostConfig)
 	);
 

packages/host/modules/clusterio/serialize.lua

@@ -21,6 +21,8 @@ local has_quality = compat.version_ge("2.0.0")
 -- Equipment Grids are serialized into an array of equipment entries
 -- where ench entry is a table with the following fields:
 --	 n: name
+--	 g: ghost name (optional)
+--	 t: to_be_removed (optional)
 --	 p: position (array of 2 numbers corresponding to x and y)
 --	 s: shield (optional)
 --	 e: energy (optional)
@@ -29,6 +31,8 @@ local has_quality = compat.version_ge("2.0.0")
 --	 r: result inventory
 --	 b: currently burning (optional)
 --	 f: remaining_burning_fuel (optional)
+-- If the equipment has a quality the following is also present:
+--	 q: quality (optional)
 function serialize.serialize_equipment_grid(grid)
 	local serialized = {}
 	local processed = {}
@@ -44,6 +48,11 @@ function serialize.serialize_equipment_grid(grid)
 						n = equipment.name,
 						p = {pos.x, pos.y},
 					}
+					if equipment.type == "equipment-ghost" then
+						if equipment.ghost_name then
+							entry.g = equipment.ghost_name
+						end
+					end
 					if equipment.shield > 0 then entry.s = equipment.shield end
 					if equipment.energy > 0 then entry.e = equipment.energy end
 					if equipment.burner then
@@ -55,6 +64,13 @@ function serialize.serialize_equipment_grid(grid)
 							entry.f = burner.remaining_burning_fuel
 						end
 					end
+					if equipment.quality then
+						entry.q = equipment.quality.name
+					end
+					if equipment.to_be_removed then
+						entry.t = equipment.to_be_removed
+					end
+					log(entry)
 					table.insert(serialized, entry)
 				end
 			end
@@ -67,8 +83,10 @@ function serialize.deserialize_equipment_grid(grid, serialized)
 	grid.clear()
 	for _, entry in ipairs(serialized) do
 		local equipment = grid.put({
-			name = entry.n,
+			name = entry.g or entry.n,
+			quality = has_quality and entry.q or nil,
 			position = entry.p,
+			ghost = entry.g and true or nil,
 		})
 		if equipment then
 			if entry.s then equipment.shield = entry.s end

packages/host/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio host server",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",

packages/host/src/Host.ts

@@ -6,6 +6,7 @@ import setBlocking from "set-blocking";
 import phin from "phin";
 import stream from "stream";
 import util from "util";
+import child_process from "child_process";
 
 // internal libraries
 import * as lib from "@clusterio/lib";
@@ -43,19 +44,20 @@ async function discoverInstances(instancesDir: string) {
 	let instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
 	for (let entry of await fs.readdir(instancesDir, { withFileTypes: true })) {
 		if (entry.isDirectory()) {
-			let instanceConfig = new lib.InstanceConfig("host");
-			let configPath = path.join(instancesDir, entry.name, "instance.json");
+			let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
+			const instancePath = path.join(instancesDir, entry.name);
+			const configPath = path.join(instancePath, "instance.json");
 
 			try {
-				const jsonConfig = JSON.parse(await fs.readFile(configPath, "utf8"));
-				instanceConfig = lib.InstanceConfig.fromJSON(jsonConfig, "host");
+				instanceConfig = await lib.InstanceConfig.fromFile("host", configPath);
+				instanceConfig.update(Host.instanceConfigWarning, false); // File might be missing the warning
 
 			} catch (err: any) {
 				if (err.code === "ENOENT") {
 					continue; // Ignore folders without config.json
 				}
 
-				logger.error(`Error occured while parsing ${configPath}: ${err.message}`);
+				logger.error(`Error occurred while parsing ${configPath}: ${err.message}`);
 				continue;
 			}
 
@@ -64,7 +66,6 @@ async function discoverInstances(instancesDir: string) {
 				continue;
 			}
 
-			let instancePath = path.join(instancesDir, entry.name);
 			logger.verbose(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`);
 			instanceInfos.set(instanceConfig.get("instance.id"), {
 				path: instancePath,
@@ -258,7 +259,6 @@ export default class Host extends lib.Link {
 	 */
 	tlsCa?: string;
 	pluginInfos: lib.PluginNodeEnvInfo[];
-	configPath: string;
 	config: lib.HostConfig;
 
 	/** Mapping of plugin name to loaded plugin */
@@ -279,6 +279,10 @@ export default class Host extends lib.Link {
 	_disconnecting = false;
 	_shuttingDown = false;
 
+	static instanceConfigWarning = {
+		"_warning": "Changes to this file will be overwritten by the controller's copy.",
+	};
+
 	static async bootstrap(hostConfig: lib.HostConfig) {
 		const modsDirectory = hostConfig.get("host.mods_directory");
 		await fs.ensureDir(modsDirectory);
@@ -289,7 +293,6 @@ export default class Host extends lib.Link {
 
 	constructor(
 		connector: HostConnector,
-		hostConfigPath: string,
 		hostConfig: lib.HostConfig,
 		tlsCa: string | undefined,
 		pluginInfos: lib.PluginNodeEnvInfo[],
@@ -298,13 +301,17 @@ export default class Host extends lib.Link {
 		 * will restart this host on a non-zero exit codes.
 		 */
 		public canRestart = false,
+		/**
+		 * If true indicates the host is in recovery mode and should
+		 * disable certain actions such as loading plugins or instance autostart
+		 */
+		public recoveryMode = false,
 		public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
 	) {
 		super(connector);
 		this.tlsCa = tlsCa;
 
 		this.pluginInfos = pluginInfos;
-		this.configPath = hostConfigPath;
 		this.config = hostConfig;
 
 		this.config.on("fieldChanged", (name, curr, prev) => {
@@ -371,17 +378,25 @@ export default class Host extends lib.Link {
 
 		this.handle(lib.HostStopRequest, this.handleHostStopRequest.bind(this));
 		this.handle(lib.HostRestartRequest, this.handleHostRestartRequest.bind(this));
+		this.handle(lib.HostUpdateRequest, this.handleHostUpdateRequest.bind(this));
 		this.handle(lib.HostConfigGetRequest, this.handleHostConfigGetRequest.bind(this));
 		this.handle(lib.HostConfigSetFieldRequest, this.handleHostConfigSetFieldRequest.bind(this));
 		this.handle(lib.HostConfigSetPropRequest, this.handleHostConfigSetPropRequest.bind(this));
+		this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this));
 		this.handle(lib.SyncUserListsEvent, this.handleSyncUserListsEvent.bind(this));
+		this.handle(lib.SystemInfoRequest, this.handleSystemInfoRequest.bind(this));
+		this.handle(lib.HostFactorioVersionsRequest, this.handleHostFactorioVersionsRequest.bind(this));
+		this.handle(lib.HostDownloadFactorioRequest, this.handleHostDownloadFactorioRequest.bind(this));
+		this.handle(lib.PluginListRequest, this.handlePluginListRequest.bind(this));
+		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
+		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
+
 		this.snoopEvent(lib.InstanceAdminlistUpdateEvent, this.handleAdminlistUpdateEvent.bind(this));
 		this.snoopEvent(lib.InstanceBanlistUpdateEvent, this.handleBanlistUpdateEvent.bind(this));
 		this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.handleWhitelistUpdateEvent.bind(this));
+
 		this.handle(lib.InstanceAssignInternalRequest, this.handleInstanceAssignInternalRequest.bind(this));
 		this.handle(lib.InstanceUnassignInternalRequest, this.handleInstanceUnassignInternalRequest.bind(this));
-		this.handle(lib.SystemInfoRequest, this.handleSystemInfoRequest.bind(this));
-		this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this));
 		this.fallbackRequest(
 			lib.InstanceSaveDetailsListRequest, this.fallbackInstanceSaveDetailsListRequest.bind(this),
 		);
@@ -403,6 +418,11 @@ export default class Host extends lib.Link {
 				continue;
 			}
 
+			if (this.recoveryMode) {
+				logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
+				continue;
+			}
+
 			let HostPluginClass = BaseHostPlugin;
 			try {
 				if (pluginInfo.hostEntrypoint) {
@@ -645,10 +665,11 @@ export default class Host extends lib.Link {
 				instanceInfo.config.update(config, true, "controller");
 
 			} else {
-				let instanceConfig = new lib.InstanceConfig("host");
+				let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
 				instanceConfig.update(config, false, "controller");
 
 				let instanceDir = await this._createNewInstanceDir(instanceConfig.get("instance.name"));
+				instanceConfig.filepath = path.join(instanceDir, "instance.json");
 
 				logger.info(`Creating ${instanceDir}`);
 				await Instance.populate_folders(instanceDir);
@@ -683,18 +704,7 @@ export default class Host extends lib.Link {
 		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
 
 		// save a copy of the instance config
-		await this.saveInstanceConfig(instanceInfo);
-	}
-
-	async saveInstanceConfig(instanceInfo: { path: string, config: lib.InstanceConfig }) {
-		let warnedOutput = {
-			_warning: "Changes to this file will be overwritten by the controller's copy.",
-			...instanceInfo.config.toJSON(),
-		};
-		await lib.safeOutputFile(
-			path.join(instanceInfo.path, "instance.json"),
-			JSON.stringify(warnedOutput, null, "\t")
-		);
+		await instanceInfo.config.save();
 	}
 
 	async handleInstanceUnassignInternalRequest(request: lib.InstanceUnassignInternalRequest) {
@@ -798,7 +808,19 @@ export default class Host extends lib.Link {
 	}
 
 	async handleSystemInfoRequest() {
-		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart);
+		if (!this.config.restartRequired) {
+			// If a restart isn't already required, then test if a new version is installed
+			try {
+				const runningVersion = this.config.get("host.version");
+				const packageJson = await fs.readJSON(path.join(__dirname, "..", "package.json"));
+				if (runningVersion !== packageJson.version) {
+					this.config.restartRequired = true;
+				}
+			} catch (err: any) {
+				logger.warn(`Failed to read package json:\n${err.stack ?? err.message}`);
+			}
+		}
+		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, this.config.restartRequired);
 	}
 
 	async handleHostMetricsRequest() {
@@ -840,6 +862,160 @@ export default class Host extends lib.Link {
 		return { results };
 	}
 
+	async handleHostFactorioVersionsRequest() {
+		const factorioDir = this.config.get("host.factorio_directory");
+		const versions: string[] = [];
+
+		try {
+			// Check if factorioDir is a simple installation
+			const simpleVersion = await this.getFactorioVersion(path.join(factorioDir, "data", "changelog.txt"));
+			if (simpleVersion !== null) {
+				versions.push(simpleVersion);
+			} else {
+				// Check for multiple versions in subdirectories
+				const entries = await fs.readdir(factorioDir, { withFileTypes: true });
+				for (const entry of entries) {
+					if (!entry.isDirectory()) {
+						continue;
+					}
+
+					const version = await this.getFactorioVersion(path.join(factorioDir, entry.name, "data", "changelog.txt"));
+					if (version !== null) {
+						versions.push(version);
+					}
+				}
+			}
+		} catch (error: any) {
+			if (error.code !== "ENOENT") {
+				logger.warn(`Error reading Factorio directory: ${error.message}`);
+			}
+		}
+
+		// Sort versions in descending order (newest first)
+		versions.sort((a, b) => {
+			const aParts = a.split(".").map(s => parseInt(s, 10));
+			const bParts = b.split(".").map(s => parseInt(s, 10));
+
+			for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) {
+				const aPart = aParts[i] || 0;
+				const bPart = bParts[i] || 0;
+				if (aPart !== bPart) {
+					return bPart - aPart;
+				}
+			}
+			return 0;
+		});
+
+		return new lib.HostFactorioVersionsRequest.Response(versions);
+	}
+
+	async handleHostDownloadFactorioRequest() {
+		try {
+			const factorioDir = this.config.get("host.factorio_directory");
+			
+			// Fetch the latest version info
+			const res = await fetch("https://factorio.com/get-download/stable/headless/linux64");
+			const location = res.headers.get("location");
+			if (!location) {
+				throw new Error("No redirect location found in response headers");
+			}
+			const url = new URL(location);
+			const filename = path.posix.basename(url.pathname);
+			const match = filename.match(/(?<=factorio_headless_x64_|factorio-headless_linux_)\d+\.\d+\.\d+(?=\.tar\.xz)/);
+			
+			if (!match || !match.length) {
+				throw new Error(`Unable to extract version from filename: ${filename}`);
+			}
+			
+			const version = match[0];
+			const tmpDir = "temp/create-temp/";
+			const archivePath = path.join(tmpDir, filename);
+			const tmpArchivePath = `${archivePath}.tmp`;
+			const versionDir = path.join(factorioDir, version);
+			const tmpFactorioDir = path.join(tmpDir, version);
+
+			// Check if version already exists
+			if (await fs.pathExists(versionDir)) {
+				return new lib.HostDownloadFactorioRequest.Response(true, version);
+			}
+
+			await fs.ensureDir(tmpDir);
+
+			// Download the factorio archive
+			const downloadRes = await fetch(url.href);
+			if (!downloadRes.ok) {
+				throw new Error(`Failed to download Factorio: ${downloadRes.statusText}`);
+			}
+
+			logger.info(`Downloading Factorio ${version}. This may take a while.`);
+			const writeStream = fs.createWriteStream(tmpArchivePath);
+			if (downloadRes.body) {
+				const readableStream = stream.Readable.fromWeb(downloadRes.body);
+				readableStream.pipe(writeStream);
+				await finished(writeStream);
+			} else {
+				throw new Error("No response body received from download");
+			}
+
+			await fs.rename(tmpArchivePath, archivePath);
+
+			// Extract the archive
+			await fs.ensureDir(tmpFactorioDir);
+			await this.execFile("tar", [
+				"xf", archivePath, "-C", tmpFactorioDir, "--strip-components", "1",
+			]);
+
+			// Clean up archive and move to final location
+			await fs.unlink(archivePath);
+			await fs.rename(tmpFactorioDir, versionDir);
+
+			logger.info(`Successfully downloaded and installed Factorio ${version}`);
+			return new lib.HostDownloadFactorioRequest.Response(true, version);
+
+		} catch (error: any) {
+			logger.error(`Failed to download Factorio: ${error.message}`);
+			return new lib.HostDownloadFactorioRequest.Response(false, undefined, error.message);
+		}
+	}
+
+	private async getFactorioVersion(changelogPath: string): Promise<string | null> {
+		try {
+			const changelog = await fs.readFile(changelogPath, "utf-8");
+			for (const line of changelog.split(/[\r\n]+/)) {
+				const index = line.indexOf(":");
+				if (index !== -1) {
+					const name = line.slice(0, index).trim();
+					if (name.toLowerCase() === "version") {
+						return line.slice(index + 1).trim();
+					}
+				}
+			}
+			return null;
+		} catch (error: any) {
+			if (error.code === "ENOENT") {
+				return null;
+			}
+			throw error;
+		}
+	}
+
+	private async execFile(cmd: string, args: string[]): Promise<{ stdout: string; stderr: string }> {
+		return new Promise((resolve, reject) => {
+			child_process.execFile(
+				cmd,
+				args,
+				{ shell: true },
+				(err, stdout, stderr) => {
+					if (err) {
+						reject(err);
+					} else {
+						resolve({ stdout, stderr });
+					}
+				}
+			);
+		});
+	}
+
 	async fallbackInstanceSaveDetailsListRequest(
 		request: lib.InstanceSaveDetailsListRequest,
 		src: lib.Address,
@@ -1043,6 +1219,35 @@ export default class Host extends lib.Link {
 		await fs.remove(instanceInfo.path);
 	}
 
+	async handleHostUpdateRequest(request: lib.HostUpdateRequest) {
+		if (!this.config.get("host.allow_remote_updates")) {
+			throw new lib.RequestError("Remote updates are disabled on this machine");
+		}
+		return lib.updatePackage("@clusterio/host");
+	}
+
+	async handlePluginUpdateRequest(request: lib.PluginUpdateRequest) {
+		if (!this.config.get("host.allow_plugin_updates")) {
+			throw new lib.RequestError("Plugin updates are disabled on this machine");
+		}
+		return await lib.handlePluginUpdate(request.pluginPackage, this.pluginInfos);
+	}
+
+	async handlePluginInstallRequest(request: lib.PluginInstallRequest) {
+		if (!this.config.get("host.allow_plugin_install")) {
+			throw new lib.RequestError("Plugin installs are disabled on this machine");
+		}
+		return await lib.handlePluginInstall(request.pluginPackage);
+	}
+
+	async handlePluginListRequest(request: lib.PluginListRequest) {
+		return this.pluginInfos.map(pluginInfo => lib.PluginDetails.fromNodeEnvInfo(
+			pluginInfo,
+			this.plugins.has(pluginInfo.name),
+			this.config.get(`${pluginInfo.name}.load_plugin` as keyof lib.HostConfigFields) as boolean,
+		));
+	}
+
 	sendHostUpdate() {
 		this.send(
 			new lib.HostInfoUpdateEvent(
@@ -1080,6 +1285,10 @@ export default class Host extends lib.Link {
 
 			for (let [instanceId, instanceInfo] of this.instanceInfos) {
 				if (instanceInfo.config.get("instance.auto_start")) {
+					if (this.recoveryMode) {
+						logger.warn(`Recovery | skipping auto startup for ${instanceInfo.config.get("instance.name")}`);
+						continue;
+					}
 					try {
 						let instanceConnection = await this._connectInstance(instanceId);
 						await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
@@ -1131,11 +1340,11 @@ export default class Host extends lib.Link {
 		}
 
 		logger.info("Saving config");
-		await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t"));
-		for (const instanceInfo of this.instanceInfos.values()) {
-			// Save in case host_assigned_game_port changed in the config.
-			await this.saveInstanceConfig(instanceInfo);
-		}
+		await this.config.save();
+		// Save host side in case host_assigned_game_port changed in the config.
+		await Promise.all(
+			[...this.instanceInfos.values()].map(instanceInfo => instanceInfo.config.save())
+		);
 
 		try {
 			// Clear silly interval in pidfile library.

packages/host/src/Instance.ts

@@ -218,6 +218,7 @@ export default class Instance extends lib.Link {
 			enableWhitelist: this.config.get("factorio.enable_whitelist"),
 			enableAuthserverBans: this.config.get("factorio.enable_authserver_bans"),
 			verboseLogging: this.config.get("factorio.verbose_logging"),
+			consoleLogging: this.config.get("factorio.console_logging"),
 			stripPaths: this.config.get("factorio.strip_paths"),
 			maxConcurrentCommands: this.config.get("factorio.max_concurrent_commands"),
 			shutdownTimeoutMs: this.config.get("factorio.shutdown_timeout") * 1000,
@@ -265,6 +266,15 @@ export default class Instance extends lib.Link {
 			}
 		});
 
+		this.server.on("whitelist-change", (added: string[], removed: string[]) => {
+			for (const player of added) {
+				this._recordUserUpdate("WHITELISTED", player);
+			}
+			for (const player of removed) {
+				this._recordUserUpdate("UNWHITELISTED", player);
+			}
+		});
+
 		this.handle(lib.InstanceExtractPlayersRequest, this.handleInstanceExtractPlayersRequest.bind(this));
 		this.handle(lib.InstanceAdminlistUpdateEvent, this.handleInstanceAdminlistUpdateEvent.bind(this));
 		this.handle(lib.InstanceBanlistUpdateEvent, this.handleInstanceBanlistUpdateEvent.bind(this));
@@ -645,6 +655,11 @@ end`.replace(/\r?\n/g, " ");
 				continue;
 			}
 
+			if (this._host.recoveryMode) {
+				this.logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
+				continue;
+			}
+
 			try {
 				await this._loadPlugin(pluginInfo, this._host);
 			} catch (err) {
@@ -1030,13 +1045,12 @@ end`.replace(/\r?\n/g, " ");
 		name: string,
 		reason: string = ""
 	) {
-		// TODO: Implement bidirectional whitelist sync, likely by watching the json file
 		const addr = lib.Address.fromShorthand("allInstances");
 		const expectedIndex = this._expectedUserUpdates.findIndex(
 			expected => expected.name === name && expected.action === action && expected.reason === reason
 		);
 		if (expectedIndex >= 0) {
-			this._expectedUserUpdates = this._expectedUserUpdates.splice(expectedIndex, 1);
+			this._expectedUserUpdates.splice(expectedIndex, 1);
 		} else if (action === "BAN") {
 			if (this.config.get("factorio.sync_banlist") !== "bidirectional") { return; }
 			this.sendTo(addr, new lib.InstanceBanlistUpdateEvent(name, true, reason ?? ""));

packages/host/src/server.ts

@@ -438,6 +438,8 @@ export interface FactorioServerOptions {
 	enableAuthserverBans?: boolean,
 	/** Enable verbose logging. */
 	verboseLogging?: boolean,
+	/** Enable console logging. */
+	consoleLogging?: boolean,
 	/** Strip paths in the console. */
 	stripPaths?: boolean,
 	/**
@@ -452,6 +454,32 @@ export interface FactorioServerOptions {
 	shutdownTimeoutMs?: number,
 }
 
+type FactorioServerEvents = {
+	// Public Events
+	"stdout": [ rawLine: Buffer ],
+	"stderr": [ rawLine: Buffer ],
+	"output": [ parsed: lib.ParsedFactorioOutput, line: string ]
+
+	"error": [ err: any ],
+	"rcon-ready": [],
+	"game-ready": [],
+	"exit": [],
+
+	"autosave-start": [ name: string ],
+	"autosave-finished": [ name: string ],
+	"save-finished": [],
+
+	"whitelist-change": [ added: string[], removed: string[] ],
+
+	// Private Events
+	"_autosave": [ name: string ]
+	"_saving": [],
+	"_saved": [],
+
+	// IPS events
+	[ipcEvent: `ipc-${string}`]: [ event: any ],
+};
+
 /**
  * Factorio Server interface
  *
@@ -471,7 +499,7 @@ export interface FactorioServerOptions {
  * - exit - invoked when the sterver has exited
  * @extends events.EventEmitter
  */
-export class FactorioServer extends events.EventEmitter {
+export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	/** Path to executable to invoke when starting the server */
 	executablePath?: string;
 	/** UDP port used for hosting the Factorio game server on */
@@ -486,6 +514,8 @@ export class FactorioServer extends events.EventEmitter {
 	enableAuthserverBans: boolean;
 	/** Enable verbose logging */
 	verboseLogging: boolean;
+	/** Enable console logging */
+	consoleLogging: boolean;
 	_shutdownTimeoutMs = 0;
 	_stopTimeoutId?: ReturnType<typeof setTimeout>;
 
@@ -496,6 +526,10 @@ export class FactorioServer extends events.EventEmitter {
 	_version: string | null = null;
 	_dataDir: string | null = null;
 
+	// Due to inconsistencies in the factorio api, we must manually watch the whitelist
+	// https://forums.factorio.com/viewtopic.php?t=123673
+	_whitelistWatcher: fs.FSWatcher | null = null;
+	_whitelist = new Set<string>();
 
 	_logger: lib.Logger;
 	_targetVersion: string;
@@ -541,6 +575,8 @@ export class FactorioServer extends events.EventEmitter {
 		this.enableAuthserverBans = options.enableAuthserverBans || false;
 		/** Enable verbose logging */
 		this.verboseLogging = options.verboseLogging || false;
+		/** Enable console logging */
+		this.consoleLogging = options.consoleLogging || false;
 		/** Maximum number of RCON commands transmitted in parallel on the RCON connection  */
 		this.maxConcurrentCommands = options.maxConcurrentCommands || 5;
 		if (options.shutdownTimeoutMs !== undefined) {
@@ -735,6 +771,62 @@ export class FactorioServer extends events.EventEmitter {
 		}
 	}
 
+	_watchWhitelist() {
+		if (this._whitelistWatcher !== null) {
+			throw new Error("Whitelist watcher already started");
+		}
+
+		if (!this.enableWhitelist) {
+			return; // The file is not updated if the whitelist is disabled
+		}
+
+		const filePath = this.writePath("server-whitelist.json");
+		try {
+			this._whitelistWatcher = fs.watch(filePath);
+		} catch (err: any) {
+			this._logger.error(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`);
+			return;
+		}
+
+		this._whitelistWatcher.unref();
+		this._whitelistWatcher.on("error", (err) => {
+			this._whitelistWatcher = null;
+			this._logger.error(`Whitelist Watcher has errored:\n${err?.stack ?? err?.message ?? err}`);
+		});
+		this._whitelistWatcher.on("close", () => {
+			this._whitelistWatcher = null;
+		});
+		this._whitelistWatcher.on("change", async (eventType) => {
+			if (eventType !== "change") {
+				this._logger.warn(`Unexpected file watcher event: ${eventType}`);
+				return;
+			}
+
+			let newWhitelistJson;
+			try {
+				newWhitelistJson = await fs.readJSON(filePath);
+			} catch (err: any) {
+				this._logger.error(`Unable to read whitelist, bidirectional sync will not be available:\n${err}`);
+				return;
+			}
+
+			if (!(newWhitelistJson instanceof Array) || !newWhitelistJson.every(e => typeof e === "string")) {
+				// The whitelist must be an array of strings
+				this._logger.error("Unable to read whitelist, bidirectional sync will not be available:" +
+					"Unexpected whitelist format");
+				return;
+			}
+
+			const newWhitelist = new Set(newWhitelistJson);
+			// Set.difference added in node v22 and so we can not use this due to v20 support
+			const added = newWhitelistJson.filter(e => !this._whitelist.has(e));
+			const removed = [...this._whitelist.values()].filter(e => !newWhitelist.has(e));
+			this._whitelist = newWhitelist;
+
+			this.emit("whitelist-change", added, removed);
+		});
+	}
+
 	/** Maximum number of RCON commands transmitted in parallel on the RCON connection  */
 	get maxConcurrentCommands() {
 		return this._maxConcurrentCommands;
@@ -848,6 +940,10 @@ export class FactorioServer extends events.EventEmitter {
 				this._rconClient.end().catch(() => {});
 			}
 
+			if (this._whitelistWatcher) {
+				this._whitelistWatcher.close();
+			}
+
 			this._resetState();
 			this.emit("exit");
 		});
@@ -862,6 +958,10 @@ export class FactorioServer extends events.EventEmitter {
 				this._rconClient.end().catch(() => {});
 			}
 
+			if (this._whitelistWatcher) {
+				this._whitelistWatcher.close();
+			}
+
 			this._resetState();
 			this.emit("exit");
 		});
@@ -973,12 +1073,14 @@ export class FactorioServer extends events.EventEmitter {
 					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
 					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
 					...(this.verboseLogging ? ["--verbose"] : []),
+					...(this.consoleLogging ? ["--console-log", this.writePath("console.log")] : []),
 				],
 				{
 					detached: true,
 					stdio: "pipe",
 				}
 			);
+			this._watchWhitelist();
 			this._attachStdio();
 			this._watchExit();
 			/* eslint-enable node/no-sync */
@@ -1029,12 +1131,14 @@ export class FactorioServer extends events.EventEmitter {
 					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
 					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
 					...(this.verboseLogging ? ["--verbose"] : []),
+					...(this.consoleLogging ? ["--console-log", this.writePath("console.log")] : []),
 				],
 				{
 					detached: true,
 					stdio: "pipe",
 				}
 			);
+			this._watchWhitelist();
 			this._attachStdio();
 			this._watchExit();
 			/* eslint-enable node/no-sync */
@@ -1110,6 +1214,9 @@ export class FactorioServer extends events.EventEmitter {
 		if (this._rconClient) {
 			this._rconClient.end().catch(() => {});
 		}
+		if (this._whitelistWatcher) {
+			this._whitelistWatcher.close();
+		}
 		this._server!.kill("SIGKILL");
 	}
 
@@ -1131,7 +1238,7 @@ export class FactorioServer extends events.EventEmitter {
 		// complete before the RCON connection can be used
 		if (!this._rconReady) {
 			// Not using events.once here to avoid throwing on error events.
-			await new Promise(resolve => this.once("rcon-ready", resolve));
+			await new Promise<void>(resolve => this.once("rcon-ready", resolve));
 		}
 
 		// The Factorio server may have decided to get ahead of us and

packages/lib/browser.ts

@@ -15,4 +15,3 @@ export * from "./src/subscriptions";
 
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as RateLimiter } from "./src/RateLimiter";
-export { default as TypedEventEmitter } from "./src/TypedEventEmitter";

packages/lib/index.ts

@@ -30,8 +30,8 @@ export * from "./src/zip_ops";
 export * from "./src/subscriptions";
 export * from "./src/datastore";
 export * from "./src/load_plugin_list";
+export * from "./src/rce_ops";
 
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";
-export { default as TypedEventEmitter } from "./src/TypedEventEmitter";

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@clusterio/lib",
 	"description": "Shared library for Clusterio",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"repository": "https://github.com/clusterio/clusterio",
 	"license": "MIT",
 	"scripts": {

packages/lib/src/ModStore.ts

@@ -1,17 +1,43 @@
 import fs from "fs-extra";
 import path from "path";
+import events from "events";
 import { Static } from "@sinclair/typebox";
 import { logger } from "./logging";
-import { ModInfo } from "./data";
-import TypedEventEmitter from "./TypedEventEmitter";
+import { ModInfo, ModPack } from "./data";
 import { safeOutputFile } from "./file_ops";
+import { Writable } from "stream";
+import { ModPortalReleaseSchema, ModPortalDetailsSchema } from "./data/messages_mod";
 
 export interface ModStoreEvents {
 	/** A stored mod was created, updated or deleted */
-	change: (mod: ModInfo) => void;
+	"change": [ mod: ModInfo ];
 }
 
-export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, ModStoreEvents> {
+interface ModsInfoResponse {
+	pagination: any,
+	results: Array<Static<typeof ModPortalDetailsSchema>>,
+}
+
+/**
+ * Interface describing the structure of the pagination info from the portal API
+ */
+interface ModPortalPagination {
+	count: number;
+	links: { next?: string; prev?: string; };
+	page: number;
+	page_count: number;
+	page_size: number;
+}
+
+/**
+ * Interface describing the structure of the portal API response for a single page
+ */
+interface ModPortalResponse {
+	pagination: ModPortalPagination;
+	results: Static<typeof ModPortalDetailsSchema>[];
+}
+
+export default class ModStore extends events.EventEmitter<ModStoreEvents> {
 	constructor(
 		public modsDirectory: string,
 		public files: Map<string, ModInfo>,
@@ -27,11 +53,6 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
 		return mod;
 	}
 
-	hasMod(name: string, version: string, sha1?: string): boolean {
-		const mod = this.files.get(ModInfo.filename(name, version));
-		return Boolean(mod && (!sha1 || sha1 !== mod.sha1));
-	}
-
 	mods() { return this.files.values(); }
 
 	static async loadFile(modsDirectory: string, file: string) {
@@ -44,8 +65,7 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
 
 	async loadFile(file: string) {
 		let modInfo = await ModStore.loadFile(this.modsDirectory, file);
-		this.files.set(modInfo.filename, modInfo);
-		this.emit("change", modInfo);
+		this.addMod(modInfo);
 		return modInfo;
 	}
 
@@ -134,4 +154,182 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
 
 		return new this(modsDirectory, files);
 	}
+
+	private async downloadModFromReleases(releases: Array<Static<typeof ModPortalReleaseSchema>>, version: string,
+		username: string, token: string
+	) {
+		const selectedRelease = releases.find(release => release.version === version);
+
+		if (selectedRelease === undefined) {
+			logger.warn(`Mod release matching version ${version} not found.`);
+			return;
+		}
+
+		try {
+			const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`);
+			url.searchParams.set("username", username);
+			url.searchParams.set("token", token);
+			const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
+			const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`;
+
+			await ModStore.downloadFile(url, tempFileName);
+			await fs.rename(tempFileName, finalFileName);
+			await this.loadFile(selectedRelease.file_name);
+
+		} catch (err: any) {
+			logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
+			// Attempt to clean up temp file if it exists
+			try {
+				const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
+				await fs.unlink(tempFileName);
+			} catch (unlinkErr: any) {
+				if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist
+					logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`);
+				}
+			}
+		}
+	}
+
+	private async downloadModsChunk(
+		modNames: string[], modVersions: string[], username: string, token: string, factorioVersion: string
+	) {
+		const url = new URL("https://mods.factorio.com/api/mods");
+		url.searchParams.set("page_size", "max");
+		url.searchParams.set("version", factorioVersion);
+		const response = await fetch(url, {
+			method: "POST",
+			body: new URLSearchParams({ namelist: modNames.join(",") }),
+		});
+		if (response.status !== 200) {
+			throw Error(`Fetch: ${url} returned ${response.status} ${response.statusText}`);
+		}
+		const mods = (await response.json() as ModsInfoResponse).results;
+
+		await Promise.all(mods.map((mod) => {
+			if (mod.releases) {
+				return this.downloadModFromReleases(mod.releases,
+					modVersions[modNames.indexOf(mod.name)], username, token);
+			} else if (mod.latest_release) {
+				return this.downloadModFromReleases([mod.latest_release],
+					modVersions[modNames.indexOf(mod.name)], username, token);
+			}
+			throw new Error(`Mod ${mod.name} has no releases or latest_release`);
+		}));
+	}
+
+
+	/**
+	 *
+	 * downloads mods to the mod dir, will not download mods for which are already in the mod directory
+	 *
+	 * @param modMap a map of what mods to download and what version to download
+	 * @param username factorio username used for auth to download mods
+	 * @param token factorio token used for auth to download mods
+	 * @param factorioVersion factorio version to filter for (mods are not guaranteed to work between Middle versions)
+	 *
+	 */
+	async downloadMods(modMap: Map<string, string>, username: string, token: string, factorioVersion: string = "1.1") {
+		const chunkSize = 500;
+		let futures: Promise<void>[] = [];
+		modMap = new Map(modMap);
+
+		// Truncate factorioVersion to major.minor (e.g., "1.1.100" -> "1.1") for API/builtin checks
+		const versionParts = factorioVersion.split(".");
+		if (versionParts.length > 2) {
+			factorioVersion = versionParts.slice(0, 2).join(".");
+		}
+
+		const builtInMods = new Set(ModPack.getBuiltinModNames(factorioVersion));
+
+		// get rid of mods already downloaded, built-in, or core
+		modMap.forEach((modVersion, modName, map) => {
+			const filename = `${modName}_${modVersion}.zip`;
+			// Filter existing, built-in from getBuiltinModNames, and explicitly filter "core"
+			if (this.files.has(filename) || builtInMods.has(modName) || modName === "core") {
+				map.delete(modName);
+			}
+		});
+
+		const modNames = Array.from(modMap.keys());
+		const modVersions = Array.from(modMap.values());
+		for (let i = 0; i < modMap.size; i += chunkSize) {
+			futures.push(
+				this.downloadModsChunk(modNames.slice(i, i + chunkSize), modVersions.slice(i, i + chunkSize),
+					username, token, factorioVersion)
+			);
+		}
+		await Promise.all(futures);
+	}
+
+	static async downloadFile(url: string | URL, filePath: string) {
+		try {
+			const response = await fetch(url);
+
+			if (!response.ok) {
+				throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+			}
+
+			if (!response.body) {
+				throw new Error("Response body is missing.");
+			}
+
+			const fileStream = fs.createWriteStream(filePath);
+			const writer = Writable.toWeb(fileStream);
+			await response.body.pipeTo(writer);
+		} catch (error) {
+			logger.error("Error downloading file:", error);
+			throw error;
+		}
+	}
+
+	/**
+	 * Fetch all mods from the Factorio Mod Portal for a specific Factorio version.
+	 *
+	 * Handles pagination automatically to retrieve the complete list.
+	 *
+	 * @param factorioVersion - The Factorio version to fetch mods for (e.g., "1.1").
+	 * @param pageSize - The page size for fetching mods.
+	 * @param hide_deprecated - Whether to exclude deprecated mods.
+	 * @returns An array of mod details.
+	 */
+	static async fetchAllModsFromPortal(
+		factorioVersion: string,
+		pageSize: number,
+		hide_deprecated: boolean = false
+	): Promise<Static<typeof ModPortalDetailsSchema>[]> {
+		logger.info(`Fetching all mods for Factorio version ${factorioVersion}...`);
+		let currentPage = 1;
+		let allMods: Static<typeof ModPortalDetailsSchema>[] = [];
+		let hasMorePages = true;
+
+		while (hasMorePages) {
+			const url = new URL("https://mods.factorio.com/api/mods");
+			url.searchParams.set("page_size", String(pageSize));
+			url.searchParams.set("version", factorioVersion);
+			url.searchParams.set("page", String(currentPage));
+			url.searchParams.set("hide_deprecated", String(hide_deprecated));
+
+			logger.verbose(`Fetching mod portal page ${currentPage}: ${url.toString()}`);
+			const response = await fetch(url.toString());
+
+			if (!response.ok) {
+				let errorMessage = `Mod portal fetch page ${currentPage} failed: `;
+				errorMessage += `${response.status} ${response.statusText}`;
+				try {
+					errorMessage += ` - ${await response.text()}`;
+				} catch (bodyError) {
+					logger.warn("Failed to read response body for failed portal request");
+				}
+				throw new Error(errorMessage);
+			}
+
+			const data = await response.json() as ModPortalResponse;
+			allMods = allMods.concat(data.results);
+
+			hasMorePages = currentPage < data.pagination.page_count;
+			currentPage += 1;
+		}
+		logger.info(`Successfully fetched ${allMods.length} mods for Factorio version ${factorioVersion}.`);
+		return allMods;
+	}
 }

packages/lib/src/TypedEventEmitter.ts

@@ -1,27 +0,0 @@
-import EventEmitter from "events";
-
-/**
- * Type checked version of EventEmitter
- *
- * @example
- * ```ts
- * interface Events {
- *     "foo": (a: number, b: number) => void,
- *     "bar": (spam: string) => void,
- * }
- *
- * class Emitter extends TypedEventEmitter<keyof Events, Events> {}
- *
- * const e = new Emitter();
- * e.emit("foo", 1, 2) // type checked arguments
- * ```
- */
-export default abstract class TypedEventEmitter<
-	Keys extends string,
-	Events extends Record<Keys, (...args: [...any]) => void>,
-> extends EventEmitter {
-	declare ["on"]: <E extends Keys>(event: E, listener: Events[E]) => this;
-	declare ["off"]: <E extends Keys>(event: E, listener: Events[E]) => this;
-	declare ["once"]: <E extends Keys>(event: E, listener: Events[E]) => this;
-	declare ["emit"]: <E extends Keys>(event: E, ...args: Parameters<Events[E]>) => boolean;
-}

packages/lib/src/config/classes.ts

@@ -1,11 +1,13 @@
 // Configuration classes
 import { Type, Static } from "@sinclair/typebox";
-import TypedEventEmitter from "../TypedEventEmitter";
+import * as fs from "fs-extra";
+import events from "events";
 
 import isDeepStrictEqual from "../is_deep_strict_equal";
 import { basicType } from "../helpers";
 import * as libSchema from "../schema";
 import { StringEnum } from "../data/composites";
+import { safeOutputFile } from "../file_ops";
 
 const ConfigLocation = StringEnum(["controller", "host", "control"]);
 export type ConfigLocation = Static<typeof ConfigLocation>;
@@ -150,7 +152,7 @@ function defaultValue(def: FieldDefinition) {
 }
 
 export type ConfigEvents = {
-	fieldChanged: (name: string, curr: FieldValue, prev: FieldValue) => void;
+	fieldChanged: [ name: string, curr: FieldValue, prev: FieldValue ],
 }
 
 export enum ConfigAccess {
@@ -176,10 +178,7 @@ export enum ConfigAccess {
  */
 export class Config<
 	Fields extends { [Field in keyof Fields]: FieldValue },
-> extends TypedEventEmitter<
-	keyof ConfigEvents,
-	ConfigEvents
-> {
+> extends events.EventEmitter<ConfigEvents> {
 	/**
 	 * Mapping of config name to field meta data
 	 * Note that mutating this object will lead to unexpected behaviour.
@@ -198,24 +197,32 @@ export class Config<
 		return config;
 	}
 
-	fields: Fields;
-	_unknownFields: Record<string, FieldValue> = {};
+	/** Defines the default access locations for the config class */
+	static defaultAccess: ConfigLocation[] = ["controller", "host", "control"];
+
+	/** Contains the fields and their values */
+	private fields: Fields;
+	private _unknownFields: Record<string, FieldValue> = {};
 
 	/** Set to true when a field in the config is changed. */
-	dirty = false;
+	private dirty = false;
+	/** Set to true when a 'restart required' field in the config is changed. */
+	restartRequired = false;
 
 	/**
 	 * Create a new instance of the given config
 	 *
 	 * @param location -
-	 *     Location to evaluate access for this instance from
+	 *     Location to evaluate access for this instance from.
 	 * @param fields -
 	 *     Serialized representation of fields to load.
+	 * @param filepath -
+	 *     Filepath for the config to save to.
 	 */
 	constructor(
 		public location: ConfigLocation,
 		fields?: Static<typeof Config.jsonSchema>,
-		public defaultAccess: ConfigLocation[] = ["controller", "host", "control"],
+		public filepath?: string,
 	) {
 		if (typeof location !== "string") {
 			throw new Error("location must be a string");
@@ -260,7 +267,7 @@ export class Config<
 			|| (
 				(!def.credential || mode & ConfigAccess.write)
 				&& (!def.readonly || mode & ConfigAccess.read)
-				&& !(def.access ?? this.defaultAccess).includes(location)
+				&& !(def.access ?? this.constructor.defaultAccess).includes(location)
 			)
 		) {
 			if (error) {
@@ -272,8 +279,52 @@ export class Config<
 		return true;
 	}
 
-	static jsonSchema = ConfigSchema;
+	/**
+	 * Set the value of a field, while also settings the required flags and events.
+	 * This method does not perform any validation on the value being set
+	 *
+	 * @param name - Name of the field to set
+	 * @param value - The new value of the field
+	 * @param notify - False to skip setting dirty flags and emitting events
+	 */
+	_set(name: keyof Fields & string, value: FieldValue, notify: boolean = true) {
+		const prev = this.fields[name];
+		this.fields[name] = value as any;
+		if (notify && !isDeepStrictEqual(value, prev)) {
+			const def = this.constructor.fieldDefinitions[name];
+			if (def.restartRequired) {
+				this.restartRequired = true;
+			}
+			this.dirty = true;
+			this.emit("fieldChanged", name, value, prev);
+		}
+	}
 
+	/**
+	 * Create config from file
+	 *
+	 * @param location - Location used for access control.
+	 * @param filepath - Filepath for the config to save to.
+	 * @returns Instance of this config
+	 */
+	static async fromFile(location: ConfigLocation, filepath: string) {
+		const fields = await fs.readJson(filepath, { encoding: "utf8" });
+		return this.fromJSON(fields, location, filepath);
+	}
+
+	/** Saves this config to file if there are unsaved changes */
+	async save() {
+		if (!this.dirty) {
+			return;
+		}
+		if (!this.filepath) {
+			throw new Error("Cannot save config which has no filepath");
+		}
+		await safeOutputFile(this.filepath, JSON.stringify(this, null, "\t"));
+		this.dirty = false;
+	}
+
+	static jsonSchema = ConfigSchema;
 	static validate = libSchema.compile(this.jsonSchema as any);
 
 	/**
@@ -281,11 +332,13 @@ export class Config<
 	 *
 	 * @param json - Serialized config to load.
 	 * @param location - Location used for access control.
+	 * @param filepath - Filepath for the config to save to.
 	 * @returns Instance of this config
 	 */
 	static fromJSON(
 		json: Static<typeof this.jsonSchema>,
 		location: ConfigLocation,
+		filepath?: string,
 	) {
 		// migrate: Pre alpha 14 config format
 		if (typeof json === "object" && json !== null && json.groups instanceof Array) {
@@ -301,7 +354,7 @@ export class Config<
 		if (!valid) {
 			throw new Error("Invalid config");
 		}
-		return new this(location, json);
+		return new this(location, json, filepath);
 	}
 
 	toJSON() {
@@ -312,11 +365,16 @@ export class Config<
 	 * Serialize the config to a plain JavaScript object
 	 *
 	 * @param remote - Location this serialised representation is for
+	 * @param filter - When provided, only the given fields are serialized
 	 * @returns JSON serializable representation of the config.
 	 */
-	toRemote(remote: ConfigLocation): Static<typeof Config.jsonSchema> {
+	toRemote(remote: ConfigLocation, filter?: (keyof Fields)[]): Static<typeof Config.jsonSchema> {
 		let fields: Record<string, FieldValue> = {};
 		for (let [name, value] of Object.entries(this.fields)) {
+			if (filter && !filter.includes(name as keyof Fields)) {
+				continue;
+			}
+
 			let def = this.constructor.fieldDefinitions[name];
 			if (
 				this.location !== "control" && !this._checkAccess(name, def, this.location, ConfigAccess.write, false)
@@ -386,12 +444,7 @@ export class Config<
 				continue;
 			}
 
-			let prev = this.fields[name];
-			this.fields[name] = value as any;
-			if (notify && !isDeepStrictEqual(value, prev)) {
-				this.dirty = true;
-				this.emit("fieldChanged", name, value, prev);
-			}
+			this._set(name, value, notify);
 		}
 	}
 
@@ -506,12 +559,7 @@ export class Config<
 			throw new InvalidValue(`Expected type of ${name} to be ${def.type}, not ${basicType(value)}`);
 		}
 
-		let prev = this.fields[name];
-		this.fields[name] = value as Fields[Field];
-		if (!isDeepStrictEqual(value, prev)) {
-			this.dirty = true;
-			this.emit("fieldChanged", name, value, prev);
-		}
+		this._set(name, value);
 	}
 
 	/**
@@ -554,10 +602,7 @@ export class Config<
 		} else {
 			delete updated[prop];
 		}
-		this.fields[name] = updated as Fields[Field];
-		if (!isDeepStrictEqual(updated, prev)) {
-			this.dirty = true;
-			this.emit("fieldChanged", name, updated, prev);
-		}
+
+		this._set(name, updated);
 	}
 }

packages/lib/src/config/definitions.ts

@@ -3,7 +3,11 @@ import * as classes from "./classes";
 import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin";
 import { Static } from "@sinclair/typebox";
 
+type configFromJSON<T> = (...args: Parameters<typeof classes.Config.fromJSON>) => T;
+type configFromFile<T> = (...args: Parameters<typeof classes.Config.fromFile>) => Promise<T>;
+
 export interface ControllerConfigFields {
+	"controller.version": string;
 	"controller.name": string;
 	"controller.mods_directory": string;
 	"controller.database_directory": string;
@@ -26,6 +30,11 @@ export interface ControllerConfigFields {
 	"controller.default_mod_pack_id": number | null;
 	"controller.default_role_id": number | null;
 	"controller.autosave_interval": number;
+	"controller.mod_portal_cache_duration_minutes": number;
+	"controller.mod_portal_page_size": number;
+	"controller.allow_remote_updates": boolean;
+	"controller.allow_plugin_updates": boolean;
+	"controller.allow_plugin_install": boolean;
 }
 
 /**
@@ -33,7 +42,8 @@ export interface ControllerConfigFields {
  * @extends classes.Config
  */
 export class ControllerConfig extends classes.Config<ControllerConfigFields> {
-	declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => ControllerConfig;
+	declare static fromJSON: configFromJSON<ControllerConfig>;
+	declare static fromFile: configFromFile<ControllerConfig>;
 	static migrations(config: Static<typeof this.jsonSchema>) {
 		if (config.hasOwnProperty("controller.external_address")) {
 			config["controller.public_url"] = config["controller.external_address"];
@@ -44,6 +54,12 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 	}
 
 	static fieldDefinitions: classes.ConfigDefs<ControllerConfigFields> = {
+		"controller.version": {
+			type: "string",
+			initialValue: "", // Set on start
+			readonly: ["controller"],
+			hidden: true,
+		},
 		"controller.name": {
 			title: "Name",
 			description: "Name of the cluster.",
@@ -197,10 +213,44 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			optional: true,
 			initialValue: 1,
 		},
+		"controller.mod_portal_cache_duration_minutes": {
+			title: "Mod Portal Cache Duration",
+			description: "Duration in minutes to cache mod portal API responses.",
+			type: "number",
+			initialValue: 30,
+		},
+		"controller.mod_portal_page_size": {
+			title: "Mod Portal Page Size",
+			description: "Maximum number of results per page when querying the Factorio mod portal API.",
+			type: "number",
+			initialValue: 1000,
+		},
+		"controller.allow_remote_updates": {
+			description: "When true, allows a remote event to trigger a clusterio update via npm",
+			type: "boolean",
+			initialValue: true,
+			readonly: ["controller"],
+			hidden: true,
+		},
+		"controller.allow_plugin_updates": {
+			description: "When true, allows a remote event to trigger a plugin update via npm",
+			type: "boolean",
+			initialValue: true,
+			readonly: ["controller"],
+			hidden: true,
+		},
+		"controller.allow_plugin_install": {
+			description: "When true, allows a remote event to trigger a plugin install via npm",
+			type: "boolean",
+			initialValue: false,
+			readonly: ["controller"],
+			hidden: true,
+		},
 	};
 }
 
 export interface HostConfigFields {
+	"host.version": string;
 	"host.name": string;
 	"host.id": number;
 	"host.factorio_directory": string;
@@ -214,6 +264,9 @@ export interface HostConfigFields {
 	"host.factorio_username": string | null,
 	"host.factorio_token": string | null,
 	"host.max_reconnect_delay": number;
+	"host.allow_remote_updates": boolean;
+	"host.allow_plugin_updates": boolean;
+	"host.allow_plugin_install": boolean;
 }
 
 /**
@@ -221,8 +274,15 @@ export interface HostConfigFields {
  * @extends classes.Config
  */
 export class HostConfig extends classes.Config<HostConfigFields> {
-	declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => HostConfig;
+	declare static fromJSON: configFromJSON<HostConfig>;
+	declare static fromFile: configFromFile<HostConfig>;
 	static fieldDefinitions: classes.ConfigDefs<HostConfigFields> = {
+		"host.version": {
+			type: "string",
+			initialValue: "", // Set on start
+			readonly: ["host"],
+			hidden: true,
+		},
 		"host.name": {
 			description: "Name of the host",
 			type: "string",
@@ -309,6 +369,27 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			type: "number",
 			initialValue: 60,
 		},
+		"host.allow_remote_updates": {
+			description: "When true, allows a remote event to trigger a clusterio update via npm",
+			type: "boolean",
+			initialValue: true,
+			readonly: ["host"],
+			hidden: true,
+		},
+		"host.allow_plugin_updates": {
+			description: "When true, allows a remote event to trigger a plugin update via npm",
+			type: "boolean",
+			initialValue: true,
+			readonly: ["host"],
+			hidden: true,
+		},
+		"host.allow_plugin_install": {
+			description: "When true, allows a remote event to trigger a plugin install via npm",
+			type: "boolean",
+			initialValue: false,
+			readonly: ["host"],
+			hidden: true,
+		},
 	};
 }
 
@@ -333,6 +414,7 @@ export interface InstanceConfigFields {
 	"factorio.enable_authserver_bans": boolean;
 	"factorio.settings": Record<string, unknown>;
 	"factorio.verbose_logging": boolean;
+	"factorio.console_logging": boolean;
 	"factorio.strip_paths": boolean;
 	"factorio.sync_adminlist": "enabled" | "disabled" | "bidirectional";
 	"factorio.sync_whitelist": "enabled" | "disabled" | "bidirectional";
@@ -346,7 +428,8 @@ export interface InstanceConfigFields {
  * @extends classes.Config
  */
 export class InstanceConfig extends classes.Config<InstanceConfigFields> {
-	declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => InstanceConfig;
+	declare static fromJSON: configFromJSON<InstanceConfig>;
+	declare static fromFile: configFromFile<InstanceConfig>;
 	static migrations(config: Static<typeof this.jsonSchema>) {
 		function boolToEnableDisable(name: string) {
 			if (config.hasOwnProperty(name) && typeof config[name] === "boolean") {
@@ -461,12 +544,13 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 		},
 		"factorio.enable_whitelist": {
 			description: "Turn on whitelist for joining the server.",
+			restartRequired: true, // Because of the cli option "--use-server-whitelist"
 			type: "boolean",
 			initialValue: false,
 		},
 		"factorio.enable_authserver_bans": {
 			description: "Turn on Factorio.com based multiplayer bans.",
-			restartRequired: true,
+			restartRequired: true, // Because of the cli option "--use-authserver-bans"
 			type: "boolean",
 			initialValue: false,
 		},
@@ -484,7 +568,13 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 		},
 		"factorio.verbose_logging": {
 			description: "Enable verbose logging on the Factorio server",
-			restartRequired: true,
+			restartRequired: true, // Because of the cli option "--verbose"
+			type: "boolean",
+			initialValue: false,
+		},
+		"factorio.console_logging": {
+			description: "Enable console logging to a separate file, useful for 3rd party integrations",
+			restartRequired: true, // Because of the cli option "--console-log"
 			type: "boolean",
 			initialValue: false,
 		},
@@ -501,10 +591,10 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			initialValue: "bidirectional",
 		},
 		"factorio.sync_whitelist": {
-			description: "Synchronize whitelist with the controller",
+			description: "Synchronize whitelist with the controller (whitelist must be enabled)",
 			type: "string",
-			enum: ["disabled", "enabled"], // TODO: Implement bidirectional
-			initialValue: "enabled",
+			enum: ["disabled", "enabled", "bidirectional"],
+			initialValue: "bidirectional",
 		},
 		"factorio.sync_banlist": {
 			description: "Synchronize banlist with the controller",
@@ -533,7 +623,8 @@ export interface ControlConfigFields {
  * @extends classes.Config
  */
 export class ControlConfig extends classes.Config<ControlConfigFields> {
-	declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => ControlConfig;
+	declare static fromJSON: configFromJSON<ControlConfig>;
+	declare static fromFile: configFromFile<ControlConfig>;
 	static fieldDefinitions: classes.ConfigDefs<ControlConfigFields> = {
 		"control.controller_url": {
 			description: "URL to connect to the controller at",

packages/lib/src/data/Role.ts

@@ -10,6 +10,8 @@ export default class Role {
 		public name: string,
 		public description: string,
 		public permissions: Set<string>,
+		public updatedAtMs = 0,
+		public isDeleted = false,
 	) { }
 
 	static jsonSchema = Type.Object({
@@ -17,22 +19,18 @@ export default class Role {
 		name: Type.String(),
 		description: Type.String(),
 		permissions: Type.Array(Type.String()),
+		updated_at_ms: Type.Optional(Type.Number()),
+		is_deleted: Type.Optional(Type.Boolean()),
 	});
 
-	static remapPermission(permission: string) {
-		// migrate: core.instance.save.list.subscribe was renamed in alpha 17
-		if (permission === "core.instance.save.list.subscribe") {
-			return "core.instance.save.subscribe";
-		}
-		return permission;
-	}
-
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
 		return new this(
 			json.id,
 			json.name,
 			json.description,
-			new Set(json.permissions.map(this.remapPermission)),
+			new Set(json.permissions),
+			json.updated_at_ms,
+			json.is_deleted,
 		);
 	}
 
@@ -42,6 +40,8 @@ export default class Role {
 			name: this.name,
 			description: this.description,
 			permissions: [...this.permissions],
+			updated_at_ms: this.updatedAtMs,
+			is_deleted: this.isDeleted,
 		};
 	}
 

packages/lib/src/data/composites.ts

@@ -4,6 +4,10 @@ export function StringEnum<T extends string[]>(values: [...T]) {
 	return Type.Unsafe<T[number]>({ type: "string", enum: values });
 }
 
+export function StringKey<T extends string>(object: Record<T, any>) {
+	return Type.Unsafe<keyof typeof object>({ type: "string", enum: Object.keys(object) });
+}
+
 export interface JSONDeserialisable<T> {
 	fromJSON(json: unknown): T;
 	jsonSchema: object;

packages/lib/src/data/index.ts

@@ -5,12 +5,15 @@
  */
 export { default as ExportManifest } from "./ExportManifest";
 export { default as ModInfo } from "./ModInfo";
-export { default as ModPack, ModSetting, ModSettingColor, ModRecord } from "./ModPack";
+export { ModSettingColor } from "./ModPack";
+export type { ModSetting, ModRecord } from "./ModPack";
+export { default as ModPack } from "./ModPack";
 export { default as ModuleInfo } from "./ModuleInfo";
 export { default as Permission } from "./Permission";
 export { default as PlayerStats } from "./PlayerStats";
 export { default as Role } from "./Role";
-export { default as User, IControllerUser } from "./User";
+export { default as User } from "./User";
+export type { IControllerUser } from "./User";
 export * from "./composites";
 export * from "./messages_core";
 export * from "./messages_controller";
@@ -18,4 +21,6 @@ export * from "./messages_host";
 export * from "./messages_instance";
 export * from "./messages_mod";
 export * from "./messages_user";
+export * from "./messages_role";
+export * from "./messages_plugin";
 export * from "./version";

packages/lib/src/data/messages_controller.ts

@@ -1,5 +1,5 @@
 import { Type, Static } from "@sinclair/typebox";
-import { JsonString, StringEnum, plainJson } from "./composites";
+import { JsonString, StringEnum, StringKey, plainJson } from "./composites";
 import { levels } from "../logging";
 import { ControllerConfig, HostConfig } from "../config";
 
@@ -19,6 +19,14 @@ export class ControllerRestartRequest {
 	static permission = "core.controller.restart" as const;
 }
 
+export class ControllerUpdateRequest {
+	declare ["constructor"]: typeof ControllerUpdateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.controller.update" as const;
+}
+
 export class ControllerConfigGetRequest {
 	declare ["constructor"]: typeof ControllerConfigGetRequest;
 	static type = "request" as const;
@@ -217,12 +225,12 @@ export class LogMessageEvent {
 	static dst = ["controller", "control"] as const;
 
 	constructor(
-		public info: { level: string, message: string },
+		public info: { level: keyof typeof levels, message: string },
 	) { }
 
 	static jsonSchema = Type.Object({
 		"info": Type.Object({
-			"level": Type.String(),
+			"level": StringKey(levels),
 			"message": Type.String(),
 		}),
 	});
@@ -250,6 +258,7 @@ export class SystemInfo {
 		public diskCapacity: number,
 		public diskAvailable: number,
 		public canRestart: boolean,
+		public restartRequired: boolean,
 		/** Millisecond Unix timestamp this entry was last updated at */
 		public updatedAtMs: number,
 		public isDeleted: boolean,
@@ -268,6 +277,7 @@ export class SystemInfo {
 		"diskCapacity": Type.Number(),
 		"diskAvailable": Type.Number(),
 		"canRestart": Type.Boolean(),
+		"restartRequired": Type.Boolean(),
 		"updatedAtMs": Type.Number(),
 		"isDeleted": Type.Boolean(),
 	});
@@ -286,6 +296,7 @@ export class SystemInfo {
 			json.diskCapacity,
 			json.diskAvailable,
 			json.canRestart,
+			json.restartRequired,
 			json.updatedAtMs,
 			json.isDeleted,
 		);

packages/lib/src/data/messages_host.ts

@@ -19,6 +19,14 @@ export class HostRestartRequest {
 	static permission = "core.host.restart" as const;
 }
 
+export class HostUpdateRequest {
+	declare ["constructor"]: typeof HostUpdateRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.update" as const;
+}
+
 export class HostConfigGetRequest {
 	declare ["constructor"]: typeof HostConfigGetRequest;
 	static type = "request" as const;
@@ -280,3 +288,51 @@ export class HostRevokeTokensRequest {
 		return new this(json.hostId);
 	}
 }
+
+export class HostFactorioVersionsRequest {
+	declare ["constructor"]: typeof HostFactorioVersionsRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.list" as const;
+
+	static Response = class Response {
+		constructor(
+			public versions: string[],
+		) { }
+
+		static jsonSchema = Type.Object({
+			"versions": Type.Array(Type.String()),
+		});
+
+		static fromJSON(json: Static<typeof this.jsonSchema>) {
+			return new this(json.versions);
+		}
+	};
+}
+
+export class HostDownloadFactorioRequest {
+	declare ["constructor"]: typeof HostDownloadFactorioRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.update" as const;
+
+	static Response = class Response {
+		constructor(
+			public success: boolean,
+			public version?: string,
+			public error?: string,
+		) { }
+
+		static jsonSchema = Type.Object({
+			"success": Type.Boolean(),
+			"version": Type.Optional(Type.String()),
+			"error": Type.Optional(Type.String()),
+		});
+
+		static fromJSON(json: Static<typeof this.jsonSchema>) {
+			return new this(json.success, json.version, json.error);
+		}
+	};
+}

packages/lib/src/data/messages_instance.ts

@@ -137,14 +137,16 @@ export class InstanceCreateRequest {
 
 	constructor(
 		public config: Static<typeof InstanceConfig.jsonSchema>,
+		public cloneFromId?: number,
 	) { }
 
 	static jsonSchema = Type.Object({
 		"config": InstanceConfig.jsonSchema,
+		"cloneFromId": Type.Optional(Type.Number()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.config);
+		return new this(json.config, json.cloneFromId);
 	}
 }
 

packages/lib/src/data/messages_mod.ts

@@ -197,6 +197,77 @@ export class ModSearchRequest {
 	};
 }
 
+// Define the structure for the latest release info from the portal
+export const ModPortalReleaseSchema = Type.Object({
+	version: Type.String(),
+	// Match the structure from ModStore's ModRelease/ModDetails
+	info_json: Type.Object({ factorio_version: Type.String() }),
+	released_at: Type.String(), // ISO 8601 date string
+	download_url: Type.String(),
+	file_name: Type.String(),
+	sha1: Type.String(),
+});
+
+// Define the structure for mod details returned by the portal API
+// This should align with the ModDetails interface in ModStore.ts
+export const ModPortalDetailsSchema = Type.Object({
+	name: Type.String(),
+	title: Type.String(),
+	summary: Type.String(),
+	owner: Type.String(),
+	downloads_count: Type.Integer(),
+	category: Type.Optional(Type.String()),
+	score: Type.Optional(Type.Number()),
+	latest_release: Type.Optional(ModPortalReleaseSchema),
+	releases: Type.Optional(Type.Array(ModPortalReleaseSchema)),
+});
+
+export class ModPortalGetAllRequest {
+	declare ["constructor"]: typeof ModPortalGetAllRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.mod.search_portal" as const;
+
+	// Define allowed Factorio versions
+	static allowedVersions = [
+		Type.Literal("0.13"), Type.Literal("0.14"), Type.Literal("0.15"),
+		Type.Literal("0.16"), Type.Literal("0.17"), Type.Literal("0.18"),
+		Type.Literal("1.0"), Type.Literal("1.1"), Type.Literal("2.0"),
+	] as const;
+
+	constructor(
+		public factorioVersion: Static<typeof ModPortalGetAllRequest.allowedVersions[number]>,
+		public hide_deprecated?: boolean,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"factorioVersion": Type.Union([...this.allowedVersions]), // Use Union for validation
+		"hide_deprecated": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.factorioVersion, json.hide_deprecated);
+	}
+
+	// Define the Response class inline
+	static Response = class ModPortalGetAllResponse {
+		declare ["constructor"]: typeof ModPortalGetAllResponse;
+		constructor(
+			public mods: Static<typeof ModPortalDetailsSchema>[],
+		) { }
+
+		static jsonSchema = Type.Object({
+			mods: Type.Array(ModPortalDetailsSchema),
+		});
+
+		static fromJSON(json: Static<typeof this.jsonSchema>) {
+			// No transformation needed if schema matches API structure
+			return new this(json.mods);
+		}
+	};
+}
+
 export class ModDownloadRequest {
 	declare ["constructor"]: typeof ModDownloadRequest;
 	static type = "request" as const;
@@ -284,3 +355,39 @@ export class ModUpdatesEvent {
 		return new this(json.updates.map(update => ModInfo.fromJSON(update)));
 	}
 }
+
+/**
+ * Request mods to be downloaded from the Factorio mod portal to the controller.
+ *
+ * The controller handles the actual download process asynchronously.
+ * Requires Factorio credentials to be configured on the controller if the portal
+ * requires authentication for downloads.
+ *
+ * @param name - Name of the mod to download.
+ * @param version - Version of the mod to download.
+ * @param factorioVersion - Factorio version context for the download.
+ */
+export class ModPortalDownloadRequest {
+	declare ["constructor"]: typeof ModPortalDownloadRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.mod.download_from_portal" as const;
+	static Response = ModInfo;
+
+	constructor(
+		public name: string,
+		public version: string,
+		public factorioVersion: string,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"name": Type.String(),
+		"version": Type.String(),
+		"factorioVersion": Type.String(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.name, json.version, json.factorioVersion);
+	}
+}

packages/lib/src/data/messages_plugin.ts

@@ -0,0 +1,93 @@
+import { Type, Static } from "@sinclair/typebox";
+import { jsonArray } from "./composites";
+import { PluginNodeEnvInfo } from "../plugin";
+
+/* This is similar to other plugin definitions but with lots removed */
+export class PluginDetails {
+	constructor(
+		public name: string,
+		public title: string,
+		public version: string,
+		public loaded: boolean,
+		public enabled: boolean,
+		public description?: string,
+		public npmPackage?: string,
+	) {}
+
+	static jsonSchema = Type.Object({
+		name: Type.String(),
+		title: Type.String(),
+		version: Type.String(),
+		loaded: Type.Boolean(),
+		enabled: Type.Boolean(),
+		description: Type.Optional(Type.String()),
+		npmPackage: Type.Optional(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.name, json.title, json.version, json.loaded, json.enabled,
+			json.description, json.npmPackage
+		);
+	}
+
+	static fromNodeEnvInfo(pluginInfo: PluginNodeEnvInfo, loaded: boolean, enabled: boolean) {
+		return new this(
+			pluginInfo.name, pluginInfo.title, pluginInfo.version, loaded, enabled,
+			pluginInfo.description, pluginInfo.npmPackage,
+		);
+	}
+}
+
+export class PluginListRequest {
+	declare ["constructor"]: typeof PluginListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = ["controller", "host"] as const;
+	static permission = "core.plugin.list";
+	static Response = jsonArray(PluginDetails);
+}
+
+export class PluginUpdateRequest {
+	declare ["constructor"]: typeof PluginUpdateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = ["controller", "host"] as const;
+	static permission = "core.plugin.update";
+
+	constructor(
+		public pluginPackage: string,
+	) {}
+
+	static jsonSchema = Type.String();
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json);
+	}
+
+	toJSON() {
+		return this.pluginPackage;
+	}
+}
+
+export class PluginInstallRequest {
+	declare ["constructor"]: typeof PluginInstallRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = ["controller", "host"] as const;
+	static permission = "core.plugin.install";
+
+	constructor(
+		public pluginPackage: string,
+	) {}
+
+	static jsonSchema = Type.String();
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json);
+	}
+
+	toJSON() {
+		return this.pluginPackage;
+	}
+}

packages/lib/src/data/messages_role.ts

@@ -0,0 +1,134 @@
+import { Type, Static } from "@sinclair/typebox";
+import { JsonNumber, jsonArray } from "./composites";
+import Permission from "./Permission";
+import Role from "./Role";
+
+export class PermissionListRequest {
+	declare ["constructor"]: typeof PermissionListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.permission.list" as const;
+	static Response = jsonArray(Permission);
+}
+
+export class RoleListRequest {
+	declare ["constructor"]: typeof RoleListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.list" as const;
+	static Response = jsonArray(Role);
+}
+
+export class RoleUpdatesEvent {
+	declare ["constructor"]: typeof RoleUpdatesEvent;
+	static type = "event" as const;
+	static src = "controller" as const;
+	static dst = "control" as const;
+	static permission = "core.role.subscribe" as const;
+
+	constructor(
+		public updates: Role[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		"updates": Type.Array(Role.jsonSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.updates.map(update => Role.fromJSON(update)));
+	}
+}
+
+export class RoleCreateRequest {
+	declare ["constructor"]: typeof RoleCreateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.create" as const;
+
+	constructor(
+		public name: string,
+		public description: string,
+		public permissions: string[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		name: Type.String(),
+		description: Type.String(),
+		permissions: Type.Array(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.name, json.description, json.permissions);
+	}
+
+	static Response = JsonNumber;
+}
+
+export class RoleUpdateRequest {
+	declare ["constructor"]: typeof RoleUpdateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.update" as const;
+
+	constructor(
+		public id: number,
+		public name: string,
+		public description: string,
+		public permissions: string[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+		name: Type.String(),
+		description: Type.String(),
+		permissions: Type.Array(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id, json.name, json.description, json.permissions);
+	}
+}
+
+export class RoleGrantDefaultPermissionsRequest {
+	declare ["constructor"]: typeof RoleGrantDefaultPermissionsRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.update" as const;
+
+	constructor(
+		public id: number,
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id);
+	}
+}
+
+export class RoleDeleteRequest {
+	declare ["constructor"]: typeof RoleDeleteRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.delete" as const;
+
+	constructor(
+		public id: number,
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id);
+	}
+}

packages/lib/src/data/messages_user.ts

@@ -1,118 +1,7 @@
 import { Type, Static } from "@sinclair/typebox";
-import Permission from "./Permission";
-import Role from "./Role";
-import User from "./User";
-import { JsonNumber, jsonArray } from "./composites";
-
-export class PermissionListRequest {
-	declare ["constructor"]: typeof PermissionListRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.permission.list" as const;
-	static Response = jsonArray(Permission);
-}
-
-export class RoleListRequest {
-	declare ["constructor"]: typeof RoleListRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.list" as const;
-	static Response = jsonArray(Role);
-}
-
-export class RoleCreateRequest {
-	declare ["constructor"]: typeof RoleCreateRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.create" as const;
-
-	constructor(
-		public name: string,
-		public description: string,
-		public permissions: string[],
-	) { }
-
-	static jsonSchema = Type.Object({
-		name: Type.String(),
-		description: Type.String(),
-		permissions: Type.Array(Type.String()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.name, json.description, json.permissions);
-	}
-
-	static Response = JsonNumber;
-}
-
-export class RoleUpdateRequest {
-	declare ["constructor"]: typeof RoleUpdateRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.update" as const;
-
-	constructor(
-		public id: number,
-		public name: string,
-		public description: string,
-		public permissions: string[],
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-		name: Type.String(),
-		description: Type.String(),
-		permissions: Type.Array(Type.String()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id, json.name, json.description, json.permissions);
-	}
-}
-
-export class RoleGrantDefaultPermissionsRequest {
-	declare ["constructor"]: typeof RoleGrantDefaultPermissionsRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.update" as const;
-
-	constructor(
-		public id: number,
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id);
-	}
-}
-
-export class RoleDeleteRequest {
-	declare ["constructor"]: typeof RoleDeleteRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.delete" as const;
-
-	constructor(
-		public id: number,
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id);
-	}
-}
+import User, { IControllerUser } from "./User";
+import { StringEnum, jsonArray, plainJson } from "./composites";
+import { MessageRequest } from "./messages_core";
 
 export class UserGetRequest {
 	declare ["constructor"]: typeof UserGetRequest;
@@ -320,3 +209,129 @@ export class UserUpdatesEvent {
 		return new this(json.updates.map(update => User.fromJSON(update)));
 	}
 }
+
+export class ClusterioUserExport {
+	public export_version = "2.0.0-alpha.20"; // Only increment when jsonSchema changes
+
+	static clusterioUserSchema = Type.Object({
+		"username": Type.String(),
+		"is_admin": Type.Optional(Type.Boolean()),
+		"is_whitelisted": Type.Optional(Type.Boolean()),
+		"is_banned": Type.Optional(Type.Boolean()),
+		"ban_reason": Type.Optional(Type.String()),
+	});
+
+	static factorioUserSchema = Type.Union([
+		Type.String(),
+		Type.Object({
+			"username": Type.String(),
+			"reason": Type.String(),
+		}),
+	]);
+
+	static jsonSchema = Type.Object({
+		"export_version": Type.String(),
+		"users": Type.Array(this.clusterioUserSchema),
+	});
+
+	constructor(
+		public users: Static<typeof ClusterioUserExport.jsonSchema.properties.users>,
+	) { }
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		const obj = new this(json.users);
+		obj.export_version = json.export_version;
+		return obj;
+	}
+}
+
+export class UserBulkImportRequest {
+	declare ["constructor"]: typeof UserBulkImportRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission(user: IControllerUser, message: MessageRequest) {
+		if (typeof message.data === "object" && message.data !== null) {
+			const data = message.data as Static<typeof UserBulkImportRequest.jsonSchema>;
+			// Check if this is a restore or import request
+			if (data.restore) {
+				user.checkPermission("core.user.bulk_restore");
+			} else {
+				user.checkPermission("core.user.bulk_import");
+			}
+			// Check if they have the permission for that type
+			switch (data.importType) {
+				case "users":
+					user.checkPermission("core.user.set_admin");
+					user.checkPermission("core.user.set_banned");
+					user.checkPermission("core.user.set_whitelisted");
+					break;
+				case "admins":
+					user.checkPermission("core.user.set_admin");
+					break;
+				case "bans":
+					user.checkPermission("core.user.set_banned");
+					break;
+				case "whitelist":
+					user.checkPermission("core.user.set_whitelisted");
+					break;
+				default:
+					// @ts-expect-error Unreachable
+					throw new Error(`Unknown import / restore type: ${data.importType}`);
+			}
+		}
+	}
+
+	static Response = plainJson(Type.Union([
+		Type.Array(ClusterioUserExport.factorioUserSchema),
+		ClusterioUserExport.jsonSchema,
+	]));
+
+	constructor(
+		public importType: "users" | "bans" | "admins" | "whitelist",
+		public users: Static<typeof ClusterioUserExport.clusterioUserSchema>[]
+			| Static<typeof ClusterioUserExport.factorioUserSchema>[],
+		public restore?: boolean
+	) { }
+
+	static jsonSchema = Type.Union([
+		Type.Object({
+			"importType": StringEnum(["bans", "admins", "whitelist"]),
+			"users": Type.Array(ClusterioUserExport.factorioUserSchema),
+			"restore": Type.Optional(Type.Boolean()),
+		}),
+		Type.Object({
+			"importType": Type.Literal("users"),
+			"users": Type.Array(ClusterioUserExport.clusterioUserSchema),
+			"restore": Type.Optional(Type.Boolean()),
+		}),
+	]);
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.importType, json.users, json.restore);
+	}
+}
+
+export class UserBulkExportRequest {
+	declare ["constructor"]: typeof UserBulkExportRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.user.bulk_export" as const;
+	static Response = plainJson(Type.Union([
+		Type.Array(ClusterioUserExport.factorioUserSchema),
+		ClusterioUserExport.jsonSchema,
+	]));
+
+	constructor(
+		public exportType: "users" | "bans" | "admins" | "whitelist",
+	) { }
+
+	static jsonSchema = Type.Object({
+		"exportType": StringEnum(["users", "bans", "admins", "whitelist"]),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.exportType);
+	}
+}

packages/lib/src/datastore.ts

@@ -1,11 +1,11 @@
+import path from "path";
 import fs from "fs-extra";
+import { EventEmitter } from "events";
 
 import { safeOutputFile } from "./file_ops";
 import { SubscribableValue } from "./subscriptions";
 import { logger } from "./logging";
-import { EventEmitter } from "stream";
 import { ControllerConfig } from "./config";
-import path from "path";
 
 type DatastoreKey = string | number;
 type DatastoreValue = string | number | boolean | object;
@@ -30,7 +30,7 @@ export class MemoryDatastoreProvider<
 	V extends DatastoreValue,
 > extends DatastoreProvider<K, V> {
 	constructor(
-		private value: Map<K, V> = new Map()
+		private value = new Map<K, V>()
 	) {
 		super();
 	}
@@ -39,34 +39,33 @@ export class MemoryDatastoreProvider<
 		this.value = new Map(data);
 	}
 
-	async load() {
+	async load(): Promise<Map<K, V>> {
 		return new Map(this.value);
 	}
 }
 
 // Store all data within a json object file and validate the result against a schema
 export class JsonDatastoreProvider<
-	K extends DatastoreKey,
 	V extends DatastoreValue,
 	J, // Intermediate type returned from migrations
-> extends DatastoreProvider<K, V> {
+> extends DatastoreProvider<string, V> {
 	constructor(
 		private filePath: string,
-		private fromJson: (json: J) => V = v => v as any,
-		private migrations: (rawJson: Record<DatastoreKey, unknown>) => Record<DatastoreKey, J> = v => v as any,
+		private fromJson: (json: J) => V = v => v as any as V,
+		private migrations: (rawJson: Record<string, unknown>) => Record<string, J> = v => v as Record<string, J>,
 		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}
 
 	// Save the data to the json file
-	async save(data: Map<K, V>) {
+	async save(data: Map<string, V>) {
 		logger.verbose(`Saving ${this.filePath}`);
 		await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t"));
 	}
 
 	// Load the data from the json file
-	async load() {
+	async load(): Promise<Map<string, V>> {
 		// Read the raw json from the file
 		let rawJson;
 		try {
@@ -89,27 +88,26 @@ export class JsonDatastoreProvider<
 
 // Store all data within a json array file where each element has a unique id
 export class JsonIdDatastoreProvider<
-	K extends DatastoreKey,
-	V extends DatastoreValue & { id: K },
+	V extends DatastoreValue & { id: DatastoreKey },
 	J, // Intermediate type returned from migrations
-> extends DatastoreProvider<K, V> {
+> extends DatastoreProvider<V["id"], V> {
 	constructor(
 		private filePath: string,
-		private fromJson: (json: J) => V = v => v as any,
-		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as any,
+		private fromJson: (json: J) => V = v => v as any as V,
+		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as Array<J>,
 		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}
 
 	// Save the data to the json file
-	async save(data: Map<K, V>) {
+	async save(data: Map<V["id"], V>) {
 		logger.verbose(`Saving ${this.filePath}`);
 		await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t"));
 	}
 
 	// Load the data from the json file
-	async load() {
+	async load(): Promise<Map<V["id"], V>> {
 		// Read the raw json from the file
 		let rawJson;
 		try {
@@ -137,7 +135,10 @@ export class JsonIdDatastoreProvider<
 export abstract class Datastore<
 	K extends DatastoreKey,
 	V extends DatastoreValue,
-> extends EventEmitter {
+	U = V,
+> extends EventEmitter<{
+	"update": [ updates: U[] ],
+}> {
 	private dirty = false;
 
 	constructor(
@@ -167,7 +168,7 @@ export abstract class Datastore<
 	}
 
 	// Set the dirty flag and call update handlers
-	protected touch(updates: any[]) {
+	protected touch(updates: U[]) {
 		if (updates.length) {
 			this.dirty = true;
 			this.emit("update", updates);
@@ -208,13 +209,17 @@ export abstract class Datastore<
 	valuesMutable() {
 		return this.data.values();
 	}
+
+	keys() {
+		return this.data.keys();
+	}
 }
 
 // General key value mapping which can be saved and loaded with the file system
 export class KeyValueDatastore<
 	K extends DatastoreKey,
 	V extends DatastoreValue,
-> extends Datastore<K, V> {
+> extends Datastore<K, V, [key: K, value: V, isDeleted?: boolean]> {
 	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
 	set(key: K, value: V) {
 		this.data.set(key, value);
@@ -232,8 +237,10 @@ export class KeyValueDatastore<
 	// Delete a value in the datastore
 	delete(key: K) {
 		const value = this.data.get(key);
-		this.data.delete(key);
-		this.touch([[key, value!, true]]);
+		if (value !== undefined) {
+			this.data.delete(key);
+			this.touch([[key, value, true]]);
+		}
 	}
 
 	// Delete many values at once from an array of keys

packages/lib/src/factorio/exchange_string.ts

@@ -3,6 +3,8 @@ import zlib from "zlib";
 class MapReaderState {
 	pos = 0;
 	last_position = { x: 0, y: 0 };
+	/** True when a version greater than 2.0.0 is detected */
+	v2 = false;
 	constructor(
 		public buf: Buffer
 	) { }
@@ -159,18 +161,49 @@ function readBoundingBox(state: MapReaderState) {
 }
 
 function readCliffSettings(state: MapReaderState) {
-	return {
+	return state.v2 ? {
 		name: readString(state),
-		elevation_0: readFloat(state),
-		elevation_interval: readFloat(state),
+		control: readString(state), // v2
+		cliff_elevation_0: readFloat(state),
+		cliff_elevation_interval: readFloat(state),
+		richness: readFloat(state),
+		cliff_smoothing: readFloat(state), // v2
+	} : {
+		name: readString(state),
+		cliff_elevation_0: readFloat(state),
+		cliff_elevation_interval: readFloat(state),
 		richness: readFloat(state),
 	};
 }
 
-function readMapGenSettings(state: MapReaderState) {
+function readTerritorySettings(state: MapReaderState) {
 	return {
-		terrain_segmentation: readFloat(state),
-		water: readFloat(state),
+		units: readArray(state, readString),
+		territory_index_expression: readString(state),
+		territory_variation_expression: readString(state),
+		minimum_territory_size: readUInt32(state),
+	};
+}
+
+function readMapGenSettings(state: MapReaderState) {
+	return state.v2 ? {
+		autoplace_controls: Object.fromEntries(readDict(state, readString, readFrequencySizeRichness)),
+		autoplace_settings: Object.fromEntries(readDict(state, readString, readAutoplaceSetting)),
+		default_enable_all_autoplace_controls: readBool(state),
+		seed: readUInt32(state),
+		width: readUInt32(state),
+		height: readUInt32(state),
+		area_to_generate_at_start: readBoundingBox(state),
+		starting_area: readFloat(state),
+		peaceful_mode: readBool(state),
+		no_enemies_mode: readBool(state), // v2
+		starting_points: readArray(state, readMapPosition),
+		property_expression_names: Object.fromEntries(readDict(state, readString, readString)),
+		cliff_settings: readCliffSettings(state),
+		territory_settings: readOptional(state, readTerritorySettings), // v2
+	} : {
+		terrain_segmentation: readFloat(state), // v1
+		water: readFloat(state), // v1
 		autoplace_controls: Object.fromEntries(readDict(state, readString, readFrequencySizeRichness)),
 		autoplace_settings: Object.fromEntries(readDict(state, readString, readAutoplaceSetting)),
 		default_enable_all_autoplace_controls: readBool(state),
@@ -303,16 +336,36 @@ function readPathFinder(state: MapReaderState) {
 }
 
 function readDifficultySettings(state: MapReaderState) {
-	return {
-		recipe_difficulty: readUInt8(state),
-		technology_difficulty: readUInt8(state),
+	return state.v2 ? {
 		technology_price_multiplier: readDouble(state),
-		research_queue_setting: ["always", "after-victory", "never"][readUInt8(state)],
+		spoil_time_modifier: readDouble(state), // v2
+	} : {
+		recipe_difficulty: readUInt8(state), // v1
+		technology_difficulty: readUInt8(state), // v1
+		technology_price_multiplier: readDouble(state),
+		research_queue_setting: ["always", "after-victory", "never"][readUInt8(state)], // v1
+	};
+}
+
+function readAsteroids(state: MapReaderState) {
+	return {
+		spawning_rate: readOptional(state, readDouble) ?? 1,
+		max_ray_portals_expanded_per_tick: readOptional(state, readUInt32) ?? 100,
 	};
 }
 
 function readMapSettings(state: MapReaderState) {
-	return {
+	return state.v2 ? {
+		pollution: readPollution(state),
+		steering: readSteering(state),
+		enemy_evolution: readEnemyEvolution(state),
+		enemy_expansion: readEnemyExpansion(state),
+		unit_group: readUnitGroup(state),
+		path_finder: readPathFinder(state),
+		max_failed_behavior_count: readUInt32(state),
+		difficulty_settings: readDifficultySettings(state),
+		asteroids: readAsteroids(state), // v2
+	} : {
 		pollution: readPollution(state),
 		steering: readSteering(state),
 		enemy_evolution: readEnemyEvolution(state),
@@ -372,8 +425,10 @@ export function readMapExchangeString(exchangeString: string) {
 	let data: MapExchangeData;
 
 	try {
+		const version = readVersion(state);
+		state.v2 = version[0] >= 2;
 		data = {
-			version: readVersion(state),
+			version: version,
 			unknown: readUInt8(state),
 			map_gen_settings: readMapGenSettings(state),
 			map_settings: readMapSettings(state),

packages/lib/src/link/connectors.ts

@@ -1,8 +1,8 @@
 // Connection adapters for Links
 import { strict as assert } from "assert";
 import events from "events";
-import WebSocket from "./WebSocket";
 
+import WebSocket from "./WebSocket";
 import * as libData from "../data";
 import * as libErrors from "../errors";
 import { logger } from "../logging";
@@ -23,6 +23,7 @@ export enum ConnectionClosed {
 	ProtocolError = 1002, // Protocol not followed
 	PolicyViolation = 1008, // Generic code for any endpoint policy
 	InternalError = 1011, // Endpoint failed to fulfil request
+	TryAgainLater = 1013, // Temporary server condition blocking requests
 
 	// Codes after 3000 are available for frameworks
 	// However they should be registered to IANA
@@ -32,6 +33,14 @@ export enum ConnectionClosed {
 
 	// Codes after 4000 are available for applications
 	MalformedMessage = 4000,
+	RecoveryMode = 4001,
+};
+
+type BaseConnectorEvents = {
+	"message": [ message: libData.Message ],
+	"error": [ err: any ],
+	"invalidate": [],
+	"reset": [],
 };
 
 /**
@@ -39,7 +48,9 @@ export enum ConnectionClosed {
  *
  * @extends events.EventEmitter
  */
-export abstract class BaseConnector extends events.EventEmitter {
+export abstract class BaseConnector<
+	E extends Record<string, any[]> | [never] = [never], // Should be "extends EventMap = DefaultEventMap"
+> extends events.EventEmitter<E | BaseConnectorEvents> {
 	protected _seq = 1;
 
 	constructor(
@@ -51,10 +62,12 @@ export abstract class BaseConnector extends events.EventEmitter {
 
 	_reset() {
 		this._seq = 1;
+		this.emit("reset");
 	}
 
 	_invalidate() {
 		this._seq = 1;
+		this.emit("invalidate");
 	}
 
 	protected abstract send(message: libData.MessageRoutable): void;
@@ -129,6 +142,15 @@ export abstract class BaseConnector extends events.EventEmitter {
 
 type ConnectorState = "closed" | "connecting" | "connected" | "resuming";
 
+type WebSocketBaseConnectorEvents = {
+	"disconnectReady": [],
+	"disconnectPrepare": [],
+	"connect": [ data: any ],
+	"resume": [],
+	"close": [],
+	"drop": [],
+};
+
 export type WebSocketClusterio = WebSocket.WebSocket & {
 	clusterio_ignore_dump?: boolean;
 };
@@ -138,7 +160,9 @@ export type WebSocketClusterio = WebSocket.WebSocket & {
  *
  * @extends module:lib.BaseConnector
  */
-export abstract class WebSocketBaseConnector extends BaseConnector {
+export abstract class WebSocketBaseConnector<
+	E extends Record<string, any[]> | [never] = [never], // Should be "extends EventMap = DefaultEventMap"
+> extends BaseConnector<E | WebSocketBaseConnectorEvents> {
 	// One of closed, connecting (client only), connected and resuming.
 	_state: ConnectorState = "closed";
 	_closing = false;
@@ -164,7 +188,7 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 	_invalidate() {
 		this._state = "connecting";
 		assert(this._closing === false);
-		assert(this._socket);
+		// We do not care about the socket state
 		this._lastHeartbeatMs = null;
 		assert(this._heartbeatId === null);
 		this._heartbeatInterval = null;
@@ -173,6 +197,11 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 		super._invalidate();
 	}
 
+	_close() {
+		this._reset();
+		this.emit("close");
+	}
+
 	_check(...expectedStates: ConnectorState[]) {
 		if (!expectedStates.includes(this._state)) {
 			throw new Error(
@@ -383,12 +412,16 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 	}
 }
 
+type WebSocketClientConnectorEvents = {
+	"hello": [ data: any ],
+};
+
 /**
  * Connector for controller clients
  *
  * @extends module:lib.WebSocketBaseConnector
  */
-export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
+export abstract class WebSocketClientConnector extends WebSocketBaseConnector<WebSocketClientConnectorEvents> {
 	_reconnectId?: ReturnType<typeof setTimeout>;
 	_sessionToken: string | null = null;
 	_sessionTimeout: number | null = null;
@@ -416,8 +449,8 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		this._sessionToken = null;
 		this._sessionTimeout = null;
 		this._startedResumingMs = null;
-		super._reset();
 		this.src = undefined as any;
+		super._reset();
 	}
 
 	_invalidate() {
@@ -425,8 +458,8 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		this._sessionToken = null;
 		this._sessionTimeout = null;
 		this._startedResumingMs = null;
-		super._invalidate();
 		this.src = undefined as any;
+		super._invalidate();
 	}
 
 	/**
@@ -461,8 +494,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			await events.once(this, "close");
 
 		} else {
-			this._reset();
-			this.emit("close");
+			this._close();
 		}
 	}
 
@@ -487,8 +519,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			this._socket!.close(ConnectionClosed.NormalClosure, "Connector closing");
 
 		} else {
-			this._reset();
-			this.emit("close");
+			this._close();
 		}
 	}
 
@@ -544,13 +575,10 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			&& this._startedResumingMs! + this._sessionTimeout! * 1000 < Date.now() + delayMs
 		) {
 			logger.error("Connector | Session timed out trying to resume");
-			this._reset();
 			if (this._closing) {
-				this._reset();
-				this.emit("close");
+				this._close();
 			} else {
-				this._state = "connecting";
-				this.emit("invalidate");
+				this._invalidate();
 			}
 		}
 		logger.verbose(
@@ -602,8 +630,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			} else {
 				// eslint-disable-next-line no-lonely-if
 				if (this._closing) {
-					this._reset();
-					this.emit("close");
+					this._close();
 
 				} else {
 					this.reconnect();
@@ -709,11 +736,9 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		} else if (type === "invalidate") {
 			logger.warn("Connector | session invalidated by controller");
 			if (this._closing) {
-				this._reset();
-				this.emit("close");
+				this._close();
 			} else {
 				this._invalidate();
-				this.emit("invalidate");
 				this.register();
 			}
 		}

packages/lib/src/link/link.ts

@@ -113,7 +113,7 @@ export class Link {
 		// Process messages received by the connector
 		connector.on("message", payload => {
 			try {
-				this._processMessage(payload);
+				this._processMessage(payload as MessageRoutable);
 			} catch (err) {
 				if (err instanceof libErrors.InvalidMessage) {
 					logger.error(err.message);
@@ -128,10 +128,10 @@ export class Link {
 			}
 		});
 
+		// The events below exist only on websocket connectors, we can't early return because of test mocking
+		const webSocketConnector = this.connector as WebSocketBaseConnector;
 		connector.on("disconnectPrepare", () => {
 			this.prepareDisconnect().finally(() => {
-				// Only WebSocket connectors issue disconnection events.
-				const webSocketConnector = this.connector as WebSocketBaseConnector;
 				if (webSocketConnector.hasSession) {
 					webSocketConnector.sendDisconnectReady();
 				}

packages/lib/src/link/messages.ts

@@ -4,6 +4,8 @@ import * as host from "../data/messages_host";
 import * as instance from "../data/messages_instance";
 import * as mod from "../data/messages_mod";
 import * as user from "../data/messages_user";
+import * as role from "../data/messages_role";
+import * as plugin from "../data/messages_plugin";
 import * as subscriptions from "../subscriptions";
 import type { RequestClass, EventClass } from "./link";
 
@@ -19,6 +21,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 
 	controller.ControllerStopRequest,
 	controller.ControllerRestartRequest,
+	controller.ControllerUpdateRequest,
 	controller.ControllerConfigGetRequest,
 	controller.ControllerConfigSetFieldRequest,
 	controller.ControllerConfigSetPropRequest,
@@ -37,6 +40,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 
 	host.HostStopRequest,
 	host.HostRestartRequest,
+	host.HostUpdateRequest,
 	host.HostConfigGetRequest,
 	host.HostConfigSetFieldRequest,
 	host.HostConfigSetPropRequest,
@@ -48,6 +52,8 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	host.PrepareControllerDisconnectRequest,
 	host.SyncUserListsEvent,
 	host.HostRevokeTokensRequest,
+	host.HostFactorioVersionsRequest,
+	host.HostDownloadFactorioRequest,
 
 	instance.InstanceDetailsGetRequest,
 	instance.InstanceDetailsListRequest,
@@ -97,17 +103,21 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	mod.ModGetRequest,
 	mod.ModListRequest,
 	mod.ModSearchRequest,
+	mod.ModPortalGetAllRequest,
 	mod.ModDownloadRequest,
 	mod.ModDeleteRequest,
 	mod.ModPackUpdatesEvent,
 	mod.ModUpdatesEvent,
+	mod.ModPortalDownloadRequest,
+
+	role.PermissionListRequest,
+	role.RoleListRequest,
+	role.RoleUpdatesEvent,
+	role.RoleCreateRequest,
+	role.RoleUpdateRequest,
+	role.RoleGrantDefaultPermissionsRequest,
+	role.RoleDeleteRequest,
 
-	user.PermissionListRequest,
-	user.RoleListRequest,
-	user.RoleCreateRequest,
-	user.RoleUpdateRequest,
-	user.RoleGrantDefaultPermissionsRequest,
-	user.RoleDeleteRequest,
 	user.UserGetRequest,
 	user.UserListRequest,
 	user.UserCreateRequest,
@@ -118,4 +128,10 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	user.UserSetBannedRequest,
 	user.UserDeleteRequest,
 	user.UserUpdatesEvent,
+	user.UserBulkImportRequest,
+	user.UserBulkExportRequest,
+
+	plugin.PluginListRequest,
+	plugin.PluginUpdateRequest,
+	plugin.PluginInstallRequest,
 ];

packages/lib/src/permissions.ts

@@ -7,6 +7,7 @@
  * @module lib/users
  */
 import { Permission, Role } from "./data";
+import { SubscribableDatastore } from "./datastore";
 
 
 export const permissions = new Map<string, Permission>();
@@ -66,7 +67,7 @@ export function definePermission({
  * @param roles - role storage to modify.
  * @returns the default admin role
  */
-export function ensureDefaultAdminRole(roles: Map<number, Role>) {
+export function ensureDefaultAdminRole(roles: SubscribableDatastore<Role>) {
 	let admin = roles.get(0);
 	if (!admin) {
 		admin = Role.fromJSON({
@@ -75,9 +76,9 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
 			description: "Cluster wide administrator.",
 			permissions: [],
 		});
-		roles.set(0, admin);
 	}
 	admin.permissions.add("core.admin");
+	roles.set(admin);
 	return admin;
 }
 
@@ -90,7 +91,7 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
  *
  * @param roles - role storage to modify.
  */
-export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
+export function ensureDefaultPlayerRole(roles: SubscribableDatastore<Role>) {
 	let player = roles.get(1);
 	if (!player) {
 		player = Role.fromJSON({
@@ -99,9 +100,9 @@ export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
 			description: "Default player role.",
 			permissions: [],
 		});
-		roles.set(1, player);
 	}
 	player.grantDefaultPermissions();
+	roles.set(player);
 }
 
 
@@ -131,6 +132,11 @@ definePermission({
 	title: "Restart controller",
 	description: "Restart the Node.js controller process if the system is set up for restarting.",
 });
+definePermission({
+	name: "core.controller.update",
+	title: "Update controller",
+	description: "Remotely update the controller if the controller allows for remote updates.",
+});
 definePermission({
 	name: "core.controller.get_config",
 	title: "Get controller config",
@@ -160,6 +166,11 @@ definePermission({
 	title: "Restart Hosts",
 	description: "Restart Node.js host processes if the system they run on are set up for restarting.",
 });
+definePermission({
+	name: "core.host.update",
+	title: "Update Hosts",
+	description: "Remotely update a host if the host allows for remote updates.",
+});
 definePermission({
 	name: "core.host.get_config",
 	title: "Get host configs",
@@ -373,6 +384,12 @@ definePermission({
 	description: "Search through the list of mods stored on the controller.",
 	grantByDefault: true,
 });
+definePermission({
+	name: "core.mod.search_portal",
+	title: "Search mod portal",
+	description: "Search for mods on the Factorio mod portal.",
+	grantByDefault: true,
+});
 definePermission({
 	name: "core.mod.subscribe",
 	title: "Subscribe to mod updates",
@@ -381,7 +398,7 @@ definePermission({
 });
 definePermission({
 	name: "core.mod.upload",
-	title: "Upload mod",
+	title: "Upload mods",
 	description: "Upload mods to the controller.",
 });
 definePermission({
@@ -395,6 +412,12 @@ definePermission({
 	title: "Delete mod",
 	description: "Delete mods stored on the controller.",
 });
+definePermission({
+	name: "core.mod.download_from_portal",
+	title: "Download mods from portal",
+	description: "Allow downloading mods directly from the Factorio mod portal to the controller.",
+	grantByDefault: true,
+});
 
 definePermission({
 	name: "core.permission.list",
@@ -409,6 +432,12 @@ definePermission({
 	description: "Get the full list of roles and their permissions in the cluster.",
 	grantByDefault: true,
 });
+definePermission({
+	name: "core.role.subscribe",
+	title: "Subscribe to role updates",
+	description: "Subscribe to be notified on updates on the details of roles.",
+	grantByDefault: true,
+});
 definePermission({
 	name: "core.role.create",
 	title: "Create role",
@@ -484,6 +513,24 @@ definePermission({
 	title: "Delete user",
 	description: "Delete users and all data stored for them.",
 });
+definePermission({
+	name: "core.user.bulk_import",
+	title: "Bulk user import",
+	description: "Bulk import users including admin, whitelist, and bans." +
+	" (Imports types are restricted by other permissions, e.g. core.user.set_admin)",
+});
+definePermission({
+	name: "core.user.bulk_export",
+	title: "Bulk user export",
+	description: "Bulk export users including admin, whitelist, and bans.",
+});
+definePermission({
+	name: "core.user.bulk_restore",
+	title: "Bulk user restore",
+	description: "Bulk restore users including admin, whitelist, and bans." +
+	" (Restore types are restricted by other permissions, e.g. core.user.set_admin)",
+});
+
 definePermission({
 	name: "core.log.follow",
 	title: "Follow cluster log",
@@ -495,6 +542,22 @@ definePermission({
 	description: "Query past entries in the cluster log.  Required to see past entries in instance console.",
 });
 
+definePermission({
+	name: "core.plugin.list",
+	title: "List plugins",
+	description: "List all installed plugins on a machine.",
+});
+definePermission({
+	name: "core.plugin.update",
+	title: "Update plugin",
+	description: "Remotely update a plugin if the target allows remote updates of plugins.",
+});
+definePermission({
+	name: "core.plugin.install",
+	title: "Install plugin",
+	description: "Remotely install a plugin if the target allows remote installs of plugins.",
+});
+
 definePermission({
 	name: "core.debug.dump_ws",
 	title: "Dump WebSocket",

packages/lib/src/rce_ops.ts

@@ -0,0 +1,54 @@
+import util from "util";
+import path from "path";
+import { exec } from "child_process";
+import { logger } from "./logging";
+import { RequestError } from "./errors";
+import { PluginNodeEnvInfo } from "./plugin";
+const execAsync = util.promisify(exec);
+
+async function isDev() {
+	//  dev:                 <devRoot>/packages/lib/dist/src/rce_ops.js
+	// prod: <prodRoot>/node_modules/@clusterio/lib/dist/src/rce_ops.js
+	return __dirname.split(path.sep).at(-5) === "packages"; // opposed to "@clusterio"
+}
+
+async function logExec(cmd: string) {
+	logger.audit(`RCE | ${cmd}`);
+	if (!isDev()) {
+		await execAsync(cmd);
+	}
+}
+
+export async function updatePackage(name: string) {
+	return logExec(`npm update --save ${name}`);
+}
+
+export async function installPackage(name: string) {
+	return logExec(`npm install --save ${name}`);
+}
+
+export async function handlePluginUpdate(pluginName: string, pluginInfos: PluginNodeEnvInfo[]) {
+	if (!pluginInfos.some(plugin => plugin.npmPackage === pluginName)) {
+		throw new RequestError(`Plugin ${pluginName} is not installed on this machine`);
+	}
+
+	return await updatePackage(pluginName);
+}
+
+export async function handlePluginInstall(pluginName: string) {
+	if (pluginName.length > 214 || /[^a-zA-Z0-9\-_.+@\/]/.test(pluginName)) {
+		// https://docs.npmjs.com/cli/v11/configuring-npm/package-json#name
+		// https://www.npmjs.com/package/validate-npm-package-name
+		throw new RequestError(`Invalid plugin name: ${pluginName}`);
+	}
+
+	const packageName = encodeURI(pluginName);
+	const npmRequest = await fetch(`https://registry.npmjs.com/${packageName}`, {
+		method: "HEAD",
+	});
+	if (!npmRequest.ok) {
+		throw new RequestError(`Unknown plugin: ${packageName}`);
+	}
+
+	return await installPackage(packageName);
+}

packages/lib/src/shared_commands.ts

@@ -135,7 +135,6 @@ export function configCommand(yargs: any) {
 export async function handleConfigCommand(
 	args: Record<string, unknown>,
 	instance: libConfig.Config<any>,
-	configPath: string
 ) {
 	let command = (args._ as string[])[1];
 
@@ -165,7 +164,7 @@ export async function handleConfigCommand(
 
 		try {
 			instance.set(args.field as string, args.value);
-			await libFileOps.safeOutputFile(configPath, JSON.stringify(instance, null, "\t"));
+			await instance.save();
 		} catch (err) {
 			if (err instanceof libConfig.InvalidField || err instanceof libConfig.InvalidValue) {
 				logger.error(err.message);

packages/lib/src/subscriptions.ts

@@ -222,10 +222,12 @@ export class EventSubscriber<
 		public control: Link,
 	) {
 		control.handle(Event, this._handleEvent.bind(this));
-		control.connector.on("connect", () => {
+		// The events below exist only on websocket connectors, we can't early return because of test mocking
+		const webSocketConnector = control.connector as WebSocketBaseConnector;
+		webSocketConnector.on("connect", () => {
 			this.handleConnectionEvent("connect");
 		});
-		control.connector.on("close", () => {
+		webSocketConnector.on("close", () => {
 			this.handleConnectionEvent("close");
 		});
 	}

packages/lib/src/system_collectors.ts

@@ -134,7 +134,7 @@ function minZip<T>(a: T[], b: T[]): [T, T][] {
 
 let previousTotalCpuMs: number[] = [];
 let previousIdleCpuMs: number[] = [];
-export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean) {
+export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean, restartRequired: boolean) {
 	const cpus = os.cpus();
 	const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
 	const currentIdleCpuMs = cpus.map(({ times }) => times.idle);
@@ -165,6 +165,7 @@ export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean
 		diskCapacity,
 		diskAvailable,
 		canRestart,
+		restartRequired,
 		Date.now(),
 		false,
 	);

packages/lib/src/zip_ops.ts

@@ -3,31 +3,25 @@ import type JSZip from "jszip";
 /**
  * Returns the root folder in the zip file
  *
- * Returns the name of the folder that all files in the zip file is
- * contained in.  Throws an error if there are multiple such folders.
+ * Returns the name of the folder that the first entry in the zip file is
+ * contained in.  Throws an error if this is not in a folder.
+ *
+ * This matches the logic Factorio uses when determining the folder to look
+ * for content in a zip file.
  *
  * @param zip - Zip to search through.
  * @returns name of the root folder.
  */
 export function findRoot(zip: JSZip) {
-	let root: undefined | string;
-	zip.forEach((relativePath, file) => {
-		let index = relativePath.indexOf("/");
-		if (index === -1) {
-			throw new Error(`Zip contains file '${relativePath}' in root dir`);
-		}
-
-		let pathRoot = relativePath.slice(0, index);
-		if (root === undefined) {
-			root = pathRoot;
-		} else if (root !== pathRoot) {
-			throw new Error("Zip contains multiple root folders");
-		}
-	});
-
-	if (root === undefined) {
+	const relativePath = Object.keys(zip.files)[0];
+	if (relativePath === undefined) {
 		throw new Error("Empty zip file");
 	}
 
-	return root;
+	let index = relativePath.indexOf("/");
+	if (index === -1) {
+		throw new Error(`Zip contains file '${relativePath}' in root dir`);
+	}
+
+	return relativePath.slice(0, index);
 }

packages/lib/tsconfig.browser.json

@@ -6,6 +6,7 @@
 		"./src/api.ts",
 		"./src/config/*.ts",
 		"./src/data/*.ts",
+		"./src/datastore.ts",
 		"./src/errors.ts",
 		"./src/factorio/exchange_string.ts",
 		"./src/helpers.ts",
@@ -20,6 +21,7 @@
 		"./src/schema.ts",
 		"./src/subscriptions.ts",
 		"./src/zip_ops.ts",
+		"./src/file_ops.ts",
 		"./src/ExponentialBackoff.ts",
 		"./src/RateLimiter.ts",
 		"./src/TypedEventEmitter.ts",

packages/web_ui/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio web interface implementation",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.19.b",
+	"version": "2.0.0-alpha.21",
 	"browser": "src/index.ts",
 	"main": "dist/browser/src/index.js",
 	"scripts": {

packages/web_ui/src/components/ControllerPage.tsx

@@ -1,10 +1,11 @@
-import React, {useContext} from "react";
-import { Button, Descriptions, Popconfirm, Space, Typography } from "antd";
+import React, {useContext, useState} from "react";
+import { Button, Descriptions, Flex, Popconfirm, Space, Tooltip, Typography } from "antd";
+import { ExclamationCircleOutlined } from "@ant-design/icons";
 
 import * as lib from "@clusterio/lib";
 
 import PluginExtra from "./PluginExtra";
-import LogConsole from "./LogConsole";
+import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
 	MetricDiskUsed, MetricDiskRatio,
@@ -17,6 +18,7 @@ import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import { notifyErrorHandler } from "../util/notify";
 import webUiPackage from "../../package.json";
+import { hasNpmButtonPermission, NpmButton } from "./NpmButton";
 
 const { Title } = Typography;
 
@@ -26,6 +28,7 @@ export default function ControllerPage() {
 	const control = useContext(ControlContext);
 	const [systems] = useSystems();
 	const system = systems.get("controller");
+	const [maxLevel, setMaxLevel] = useState<keyof typeof lib.levels>("info");
 
 	return <PageLayout nav={[{ name: "Controller" }]}>
 		<PageHeader
@@ -53,16 +56,24 @@ export default function ControllerPage() {
 				}
 				{
 					account.hasPermission("core.controller.restart")
-					&& <Button
-						disabled={system?.canRestart === false}
-						onClick={() => {
-							control.send(
-								new lib.ControllerRestartRequest()
-							).catch(notifyErrorHandler("Error restarting controller"));
-						}}
-					>
-						Restart
-					</Button>
+					&& <Tooltip title={system?.restartRequired ? "Restart Required" : null}>
+						<Button
+							disabled={system?.canRestart === false}
+							onClick={() => {
+								control.send(
+									new lib.ControllerRestartRequest()
+								).catch(notifyErrorHandler("Error restarting controller"));
+							}}
+						>
+							Restart
+							{!system?.restartRequired ? undefined
+								: <ExclamationCircleOutlined style={{ color: "yellow" }}/>}
+						</Button>
+					</Tooltip>
+				}
+				{
+					hasNpmButtonPermission(true)
+					&& <NpmButton target="controller" canRestart={system?.canRestart}/>
 				}
 			</Space>}
 		/>
@@ -82,8 +93,15 @@ export default function ControllerPage() {
 			<Descriptions.Item label="Disk"><MetricDiskUsed system={system} /></Descriptions.Item>
 		</Descriptions>
 		{account.hasPermission("core.log.follow") && <>
-			<Title level={5} style={{ marginTop: 16 }}>Console</Title>
-			<LogConsole controller={true} />
+			<Flex justify="space-between" align="baseline">
+				<Title level={5} style={{ marginTop: 16 }}>Console</Title>
+				<SelectMaxLogLevel
+					value={maxLevel}
+					onChange={setMaxLevel}
+					hidden={["server"]}
+				/>
+			</Flex>
+			<LogConsole controller={true} maxLevel={maxLevel}/>
 		</>}
 		{account.hasPermission("core.controller.get_config") && <ControllerConfigTree />}
 		<PluginExtra component="ControllerPage" />

packages/web_ui/src/components/ErrorPage.tsx

@@ -1,15 +1,52 @@
 import React from "react";
-import { Typography } from "antd";
+import { Button, Card, Flex, Image, Typography } from "antd";
 
+import example from "../images/console-error-edge.png";
 import PageLayout from "./PageLayout";
 import type { ErrorProps } from "./App";
 
 const { Paragraph } = Typography;
 
+function SupportLinkCard(props:{
+	name: string,
+	href: string,
+	image: string,
+}) {
+	return <Card style={{ alignContent: "center" }} styles={{ body: { padding: 10 } }}>
+		<Flex align="center" gap="middle">
+			<img src={props.image} style={{ width: 50 }}/>
+			<Button type="primary" href={props.href}>{props.name}</Button>
+		</Flex>
+	</Card>;
+}
+
+export default function ErrorPage(props: ErrorProps & { throw?: boolean }) {
+	if (props.throw) {
+		throw props.error;
+	}
 
-export default function ErrorPage(props: ErrorProps) {
 	return <PageLayout nav={[{ name: "Error" }]}>
-		<h2>An unexpected error occured</h2>
-		<Paragraph code className="error-traceback">{props.error.stack}</Paragraph>
+		<h2>An Unexpected Error Occurred</h2>
+		<Paragraph code className="error-traceback">{props.error.message}</Paragraph>
+		<Paragraph>
+			Further details can found by saving the console log in dev tools
+			(F12, "Console", "Save as", see example below).
+			If this is an issue with a plugin then please report it to the plugin creator;
+			otherwise, please reach out to the Clusterio team via Github or Discord.
+		</Paragraph>
+		<Flex gap="middle">
+			<SupportLinkCard
+				name="Github Issues"
+				href="https://github.com/clusterio/clusterio/issues"
+				image="https://github.githubassets.com/favicons/favicon-dark.svg"
+			/>
+			<SupportLinkCard
+				name="Support Server"
+				href="https://discord.gg/mzAsgnm"
+				image="https://cdn.prod.website-files.com/6257adef93867e50d84d30e2/66e3d7f4ef6498ac018f2c55_Symbol.svg"
+			/>
+		</Flex>
+		<br/>
+		<Image height={250} src={example}/>
 	</PageLayout>;
 };

packages/web_ui/src/components/HostViewPage.tsx

@@ -1,5 +1,6 @@
-import React, { useContext } from "react";
-import { Descriptions, Spin, Tag, Typography, Button, Space, Modal, Popconfirm } from "antd";
+import React, { useContext, useState } from "react";
+import { Descriptions, Spin, Tag, Typography, Button, Space, Modal, Popconfirm, Flex, Tooltip } from "antd";
+import { ExclamationCircleOutlined } from "@ant-design/icons";
 import { useParams } from "react-router-dom";
 
 import * as lib from "@clusterio/lib";
@@ -7,19 +8,21 @@ import notify, { notifyErrorHandler } from "../util/notify";
 import ControlContext from "./ControlContext";
 import HostConfigTree from "./HostConfigTree";
 import InstanceList from "./InstanceList";
-import LogConsole from "./LogConsole";
+import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import { useAccount } from "../model/account";
 import { useInstances } from "../model/instance";
 import { useHost } from "../model/host";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import FactorioVersionManager from "./FactorioVersionManager";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
 	MetricDiskUsed, MetricDiskRatio,
 } from "./system_metrics";
 import { formatTimestamp } from "../util/time_format";
 import { useSystems } from "../model/system";
+import { hasNpmButtonPermission, NpmButton } from "./NpmButton";
 
 const { Title } = Typography;
 
@@ -32,6 +35,7 @@ export default function HostViewPage() {
 	const [systems] = useSystems();
 	const system = systems.get(hostId);
 	const [host, synced] = useHost(hostId);
+	const [maxLevel, setMaxLevel] = useState<keyof typeof lib.levels>("info");
 	const hostInstances = new Map([...instances].filter(([_id, instance]) => instance.assignedHost === hostId));
 
 	let nav = [{ name: "Hosts", path: "/hosts" }, { name: host?.name ?? String(hostId) }];
@@ -88,17 +92,25 @@ export default function HostViewPage() {
 		}
 		{
 			account.hasPermission("core.host.restart")
-			&& <Button
-				disabled={system?.canRestart === false}
-				onClick={() => {
-					control.sendTo(
-						{ hostId },
-						new lib.HostRestartRequest()
-					).catch(notifyErrorHandler("Error restarting host"));
-				}}
-			>
-				Restart
-			</Button>
+			&& <Tooltip title={system?.restartRequired ? "Restart Required" : null}>
+				<Button
+					disabled={system?.canRestart === false}
+					onClick={() => {
+						control.sendTo(
+							{ hostId },
+							new lib.HostRestartRequest()
+						).catch(notifyErrorHandler("Error restarting host"));
+					}}
+				>
+					Restart
+					{!system?.restartRequired ? undefined
+						: <ExclamationCircleOutlined style={{ color: "yellow" }} />}
+				</Button>
+			</Tooltip>
+		}
+		{
+			hasNpmButtonPermission(false)
+			&& <NpmButton target={{ hostId }} canRestart={system?.canRestart} disabled={!host["connected"]}/>
 		}
 	</Space>;
 
@@ -140,9 +152,17 @@ export default function HostViewPage() {
 			<Title level={5} style={{ marginTop: 16 }}>Instances</Title>
 			<InstanceList instances={hostInstances} size="small" hideAssignedHost />
 		</>}
+		<FactorioVersionManager hostId={hostId} system={system} connected={host.connected} />
 		{account.hasPermission("core.log.follow") && <>
-			<Title level={5} style={{ marginTop: 16 }}>Console</Title>
-			<LogConsole hosts={[hostId]} />
+			<Flex justify="space-between" align="baseline">
+				<Title level={5} style={{ marginTop: 16 }}>Console</Title>
+				<SelectMaxLogLevel
+					value={maxLevel}
+					onChange={setMaxLevel}
+					hidden={["server", "http"]}
+				/>
+			</Flex>
+			<LogConsole hosts={[hostId]} maxLevel={maxLevel}/>
 		</>}
 		{account.hasPermission("core.host.get_config") && <HostConfigTree id={hostId} available={host.connected} />}
 		<PluginExtra component="HostViewPage" host={host} />

packages/web_ui/src/components/HostsPage.tsx

@@ -1,10 +1,11 @@
 import React, { useContext, useRef, useState, useEffect } from "react";
 import { useNavigate } from "react-router-dom";
-import { Button, Flex, Form, InputNumber, Modal, Progress, Switch, Table, Tag, Typography } from "antd";
-import CopyOutlined from "@ant-design/icons/lib/icons/CopyOutlined";
+import { Button, Flex, Form, InputNumber, Modal, Progress, Space, Switch, Table, Tag, Tooltip, Typography } from "antd";
+import { ExclamationCircleOutlined, CopyOutlined } from "@ant-design/icons";
 
 import * as lib from "@clusterio/lib";
 
+import webUiPackage from "../../package.json";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
@@ -13,6 +14,7 @@ import PluginExtra from "./PluginExtra";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
 	MetricDiskUsed, MetricDiskRatio,
+	RestartRequired,
 } from "./system_metrics";
 import { useHosts } from "../model/host";
 import { useSystems } from "../model/system";
@@ -268,6 +270,12 @@ export default function HostsPage() {
 				{
 					title: "Version",
 					dataIndex: "version",
+					render: (_, host) => <Space>
+						{host.version}
+						{webUiPackage.version === host.version ? undefined : <Tooltip title="Version Mismatch">
+							<ExclamationCircleOutlined style={{ color: "yellow" }}/>
+						</Tooltip>}
+					</Space>,
 					sorter: (a, b) => strcmp(a.version, b.version),
 				},
 				{
@@ -278,11 +286,14 @@ export default function HostsPage() {
 				{
 					title: "Connected",
 					key: "connected",
-					render: (_, host) => <Tag
-						color={host.connected ? "#389e0d" : "#cf1322"}
-					>
-						{host.connected ? "Connected" : "Disconnected"}
-					</Tag>,
+					render: (_, host) => <>
+						<Tag
+							color={host.connected ? "#389e0d" : "#cf1322"}
+						>
+							{host.connected ? "Connected" : "Disconnected"}
+						</Tag>
+						<RestartRequired system={systems.get(host.id)}/>
+					</>,
 					sorter: (a, b) => Number(a.connected) - Number(b.connected),
 				},
 			]}

packages/web_ui/src/components/InputRole.tsx

@@ -1,21 +1,10 @@
-import React, { useContext, useEffect, useState } from "react";
 import { Select } from "antd";
 
-import * as lib from "@clusterio/lib";
-
-import ControlContext from "./ControlContext";
-import { notifyErrorHandler } from "../util/notify";
 import { InputComponentProps } from "../BaseWebPlugin";
+import { useRoles } from "../model/roles";
 
 export default function InputRole(props: InputComponentProps) {
-	let control = useContext(ControlContext);
-	let [roles, setRoles] = useState<lib.Role[]>([]);
-
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(newRoles);
-		}).catch(notifyErrorHandler("Error fetching role list"));
-	}, []);
+	const [roles] = useRoles();
 
 	return <Select
 		showSearch

packages/web_ui/src/components/InstanceViewPage.tsx

@@ -1,6 +1,6 @@
 import React, { useContext, useState } from "react";
 import { useNavigate, useParams } from "react-router-dom";
-import { Alert, Button, Descriptions, Dropdown, MenuProps, Modal, Space, Spin, Typography } from "antd";
+import { Alert, Button, Descriptions, Dropdown, Flex, MenuProps, Modal, Space, Spin, Switch, Typography } from "antd";
 import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 import DownOutlined from "@ant-design/icons/DownOutlined";
 
@@ -12,7 +12,7 @@ import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
 import InstanceConfigTree from "./InstanceConfigTree";
-import LogConsole from "./LogConsole";
+import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import InstanceRcon from "./InstanceRcon";
 import AssignInstanceModal from "./AssignInstanceModal";
 import StartStopInstanceButton from "./StartStopInstanceButton";
@@ -185,6 +185,8 @@ function InstanceButtons(props: { instance: lib.InstanceDetails }) {
 export default function InstanceViewPage() {
 	let params = useParams();
 	let instanceId = Number(params.id);
+	const [maxLevel, setMaxLevel] = useState<keyof typeof lib.levels>("server");
+	const [actionsOnly, setActionsOnly] = useState<boolean>(true);
 
 	let navigate = useNavigate();
 
@@ -229,9 +231,30 @@ export default function InstanceViewPage() {
 		}
 		{
 			account.hasAnyPermission("core.log.follow", "core.instance.send_rcon")
-			&& <Title level={5} style={{ marginTop: 16 }}>Console</Title>
+			&& <Flex justify="space-between" align="baseline">
+				<Title level={5} style={{ marginTop: 16 }}>Console</Title>
+				{
+					account.hasPermission("core.log.follow")
+					&& <Flex align="center" gap="middle">
+						<Switch
+							checkedChildren="Chat"
+							unCheckedChildren="Log"
+							checked={actionsOnly}
+							onChange={setActionsOnly}
+						/>
+						<SelectMaxLogLevel
+							value={maxLevel}
+							onChange={setMaxLevel}
+							hidden={["http"]}
+						/>
+					</Flex>
+				}
+			</Flex>
+		}
+		{
+			account.hasPermission("core.log.follow")
+			&& <LogConsole instances={[instanceId]} maxLevel={maxLevel} actionsOnly={actionsOnly}/>
 		}
-		{account.hasPermission("core.log.follow") && <LogConsole instances={[instanceId]} />}
 		{
 			account.hasPermission("core.instance.send_rcon")
 			&& <InstanceRcon id={instanceId} disabled={instance.status !== "running"} />

packages/web_ui/src/components/InstancesPage.tsx

@@ -1,6 +1,6 @@
 import React, { useContext, useState } from "react";
 import { useNavigate } from "react-router-dom";
-import { Button, Form, Input, Modal } from "antd";
+import { Button, Form, Input, Modal, Select, Tooltip } from "antd";
 
 import * as lib from "@clusterio/lib";
 
@@ -13,7 +13,7 @@ import { useInstances } from "../model/instance";
 import InstanceList from "./InstanceList";
 import { notifyErrorHandler } from "../util/notify";
 
-function CreateInstanceButton() {
+function CreateInstanceButton(props: { instances: ReturnType<typeof useInstances>[0] }) {
 	let control = useContext(ControlContext);
 	let navigate = useNavigate();
 	let [open, setOpen] = useState(false);
@@ -28,8 +28,14 @@ function CreateInstanceButton() {
 
 		let instanceConfig = new lib.InstanceConfig("control");
 		instanceConfig.set("instance.name", values.instanceName);
-		const serializedConfig = instanceConfig.toRemote("controller");
-		await control.send(new lib.InstanceCreateRequest(serializedConfig));
+
+		await control.send(new lib.InstanceCreateRequest(
+			instanceConfig.toRemote("controller", [
+				"instance.id", "instance.name",
+			]),
+			values.instanceClone >= 0 ? values.instanceClone : undefined,
+		));
+
 		setOpen(false);
 		navigate(`/instances/${instanceConfig.get("instance.id")}/view`);
 	}
@@ -53,6 +59,16 @@ function CreateInstanceButton() {
 				<Form.Item name="instanceName" label="Name">
 					<Input />
 				</Form.Item>
+				<Tooltip title="Perform a one time copy of an existing config (assigned host is not copied)">
+					<Form.Item name="instanceClone" label="Copy Config">
+						<Select
+							defaultValue={-1}
+							options={[{ id: -1, name: "Default Config" }, ...props.instances.values()]
+								.map(i => ({ value: i.id, label: i.name }))
+							}
+						/>
+					</Form.Item>
+				</Tooltip>
 			</Form>
 		</Modal>
 	</>;
@@ -67,7 +83,7 @@ export default function InstancesPage() {
 		<PageHeader
 			title="Instances"
 			extra={<>
-				{account.hasPermission("core.instance.create") && <CreateInstanceButton />}
+				{account.hasPermission("core.instance.create") && <CreateInstanceButton instances={instances}/>}
 				{account.hasPermission("core.instance.start")
 					&& <Button onClick={e => instances.forEach(instance => {
 						if (instance.status === "stopped") {

packages/web_ui/src/components/LogConsole.tsx

@@ -1,5 +1,5 @@
 import React, { useEffect, useContext, useRef, useState, ReactElement } from "react";
-import { Typography } from "antd";
+import { Select, Typography } from "antd";
 
 import * as lib from "@clusterio/lib";
 
@@ -51,10 +51,11 @@ function formatParsedOutput(parsed: Parsed, key: number) {
 }
 
 type Info = {
-	level: string;
+	level: keyof typeof lib.levels;
 	message: string;
 	parsed?: Parsed;
 };
+
 function formatLog(info: Info, key: number): ReactElement {
 	if (info.level === "server" && info.parsed) {
 		return formatParsedOutput(info.parsed, key);
@@ -68,7 +69,30 @@ type LogConsoleProps = {
 	controller?: boolean;
 	hosts?: number[];
 	instances?: number[];
+	maxLevel?: keyof typeof lib.levels;
+	actionsOnly?: boolean;
 };
+
+export function SelectMaxLogLevel(props: {
+	value: keyof typeof lib.levels,
+	hidden?: (keyof typeof lib.levels)[],
+	onChange: (option: keyof typeof lib.levels) => void
+}) {
+	return <Select
+		showSearch
+		style={{ width: 100 }}
+		defaultValue={props.value}
+		onChange={props.onChange}
+		options={(Object.entries(lib.levels) as [keyof typeof lib.levels, number][])
+			.filter(([level]) => !props.hidden || !props.hidden.includes(level))
+			.map(([level, index]) => ({
+				value: level, label: level.charAt(0).toUpperCase() + level.slice(1), index: index,
+			}))
+			.sort((a, b) => a.index - b.index)
+		}
+	/>;
+}
+
 export default function LogConsole(props: LogConsoleProps) {
 	let account = useAccount();
 	let control = useContext(ControlContext);
@@ -82,6 +106,7 @@ export default function LogConsole(props: LogConsoleProps) {
 			controller: props.controller || false,
 			hostIds: props.hosts || [],
 			instanceIds: props.instances || [],
+			maxLevel: props.maxLevel || undefined,
 		};
 
 		// Scroll view to the anchor so it sticks to the bottom
@@ -95,11 +120,19 @@ export default function LogConsole(props: LogConsoleProps) {
 				logFilter.controller,
 				logFilter.hostIds,
 				logFilter.instanceIds,
-				undefined,
+				logFilter.maxLevel,
 				400,
 				"desc",
 			)).then(result => {
-				setPastLines(result.log.map((info, index) => formatLog(info as Info, -index - 1)).reverse());
+				setPastLines(result.log
+					.filter(info => (
+						!props.actionsOnly
+						|| (info as Info).level !== "server"
+						|| (info as Info).parsed?.type === "action"
+					))
+					.map((info, index) => formatLog(info as Info, -index - 1))
+					.reverse()
+				);
 			}).catch(err => {
 				setPastLines([<span key={0}>{`Error loading log: ${err.message}`}<br/></span>]);
 			});
@@ -108,16 +141,26 @@ export default function LogConsole(props: LogConsoleProps) {
 		}
 
 		function logHandler(info: Info) {
-			setLines(currentLines => currentLines.concat(
-				[formatLog(info, currentLines.length)]
-			));
+			if (!props.actionsOnly || info.level !== "server" || info.parsed?.type === "action") {
+				setLines(currentLines => currentLines.concat(
+					[formatLog(info, currentLines.length)]
+				));
+			}
 		}
 
+		setLines([]);
 		control.onLog(logFilter, logHandler);
 		return () => {
 			control.offLog(logFilter, logHandler);
 		};
-	}, [props.all, props.controller, (props.hosts || []).join(), (props.instances || []).join()]);
+	}, [
+		props.all,
+		props.controller,
+		(props.hosts || []).join(),
+		(props.instances || []).join(),
+		props.maxLevel,
+		props.actionsOnly,
+	]);
 
 	return <>
 		<Paragraph code className="instance-console">

packages/web_ui/src/components/ModsPage.tsx

@@ -1,8 +1,12 @@
-import React, { Fragment, useState, useContext } from "react";
+import React, { Fragment, useState, useContext, useEffect, useMemo } from "react";
 import { useNavigate } from "react-router-dom";
-import { Button, Form, Input, Modal, Popconfirm, Space, Table, Typography, Upload } from "antd";
-import ImportOutlined from "@ant-design/icons/ImportOutlined";
-import PlusOutlined from "@ant-design/icons/PlusOutlined";
+import {
+	Button, Form, Input, Modal, Popconfirm, Space, Table, Typography, Upload, Alert, Select, Tooltip, notification,
+} from "antd";
+import {
+	ImportOutlined, PlusOutlined, SearchOutlined, DownloadOutlined,
+} from "@ant-design/icons";
+import { Static } from "@sinclair/typebox";
 
 import * as lib from "@clusterio/lib";
 
@@ -21,6 +25,12 @@ import UploadButton from "./UploadButton";
 
 const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare;
 
+// Define the allowed Factorio versions based on the lib schema
+type FactorioVersion = Static<(typeof lib.ModPortalGetAllRequest)["allowedVersions"][number]>;
+
+// Type alias for the mod structure derived from the library response
+type ModPortalModType = InstanceType<typeof lib.ModPortalGetAllRequest.Response>["mods"][number];
+
 function ImportModPackButton() {
 	let control = useContext(ControlContext);
 	let navigate = useNavigate();
@@ -129,6 +139,375 @@ function CreateModPackButton() {
 	</>;
 }
 
+function SearchModsButton() {
+	const control = useContext(ControlContext);
+	const account = useAccount();
+	const [open, setOpen] = useState(false);
+	const [form] = Form.useForm();
+	const [searchText, setSearchText] = useState("");
+	const [factorioVersion, setFactorioVersion] = useState<FactorioVersion>("2.0");
+
+	// State for all mods fetched from backend
+	const [allMods, setAllMods] = useState<ModPortalModType[]>([]);
+	const [loading, setLoading] = useState<boolean>(false);
+	const [error, setError] = useState<Error | null>(null);
+
+	// State for client-side pagination and sorting
+	const [page, setPage] = useState(1);
+	const [pageSize, setPageSize] = useState(10);
+	const [sort, setSort] = useState<string>("name");
+	const [sortOrder, setSortOrder] = useState<"asc" | "desc">("asc");
+
+	// Function to handle download from Mod Portal to Controller
+	const handleControllerDownload = (
+		modName: string,
+		modTitle: string | undefined,
+		modVersion: string | undefined,
+		portalFactorioVersion: FactorioVersion,
+	) => {
+		if (modVersion) {
+			control.send(
+				new lib.ModPortalDownloadRequest(
+					modName,
+					modVersion,
+					portalFactorioVersion
+				)
+			).then(() => {
+				notification.success({
+					message: "Download started",
+					description: `${modTitle || modName} v${
+						modVersion
+					} is being downloaded to the controller.`,
+				});
+			}).catch(
+				notifyErrorHandler("Error starting mod download")
+			);
+		}
+	};
+
+	// Fetch all mods from backend when modal opens or version changes
+	useEffect(() => {
+		if (!open || !factorioVersion) {
+			setAllMods([]); // Clear mods if modal closed or no version
+			return;
+		}
+
+		setLoading(true);
+		setError(null);
+		let canceled = false;
+
+		control.send(
+			new lib.ModPortalGetAllRequest(factorioVersion)
+		).then((response: any) => {
+			if (canceled) { return; }
+			setAllMods(response.mods || []);
+			setLoading(false);
+		}).catch(err => {
+			if (canceled) { return; }
+			notifyErrorHandler("Error fetching mods from portal")(err);
+			setError(err);
+			setAllMods([]);
+			setLoading(false);
+		});
+
+		// eslint-disable-next-line consistent-return
+		return () => {
+			canceled = true;
+		};
+		// Re-fetch when modal opens or factorio version changes
+	}, [open, factorioVersion, control]);
+
+
+	// Memoize the filtered, sorted, and paginated mods (logic remains the same)
+	const displayedMods = useMemo(() => {
+		let filtered = allMods;
+
+		// Apply search filter (case-insensitive)
+		if (searchText) {
+			const lowerSearchText = searchText.toLowerCase();
+			filtered = filtered.filter(mod => mod.name.toLowerCase().includes(lowerSearchText)
+				|| mod.title.toLowerCase().includes(lowerSearchText)
+			);
+		}
+
+		// Apply sorting
+		if (sort && sortOrder) {
+			filtered = [...filtered].sort((a, b) => {
+				let aValue: any = a[sort as keyof ModPortalModType] ?? "";
+				let bValue: any = b[sort as keyof ModPortalModType] ?? "";
+
+				// Handle specific types if necessary (e.g., numbers, dates)
+				if (sort === "downloads_count") {
+					aValue = a.downloads_count ?? 0;
+					bValue = b.downloads_count ?? 0;
+					return sortOrder === "asc" ? aValue - bValue : bValue - aValue;
+				}
+
+				// Default string comparison
+				const comparison = strcmp(String(aValue), String(bValue));
+				return sortOrder === "asc" ? comparison : -comparison;
+			});
+		}
+
+		// Apply pagination
+		const startIndex = (page - 1) * pageSize;
+		return filtered.slice(startIndex, startIndex + pageSize);
+
+	}, [allMods, searchText, sort, sortOrder, page, pageSize]);
+
+	// Calculate total results based on the filtered list *before* pagination (logic remains the same)
+	const totalResults = useMemo(() => {
+		let filtered = allMods;
+		if (searchText) {
+			const lowerSearchText = searchText.toLowerCase();
+			filtered = filtered.filter(mod => mod.name.toLowerCase().includes(lowerSearchText)
+				|| mod.title.toLowerCase().includes(lowerSearchText)
+			);
+		}
+		return filtered.length;
+	}, [allMods, searchText]);
+
+	// Update search text and factorio version from form
+	const handleSearch = (changedValues: any, allValues: any) => {
+		const nameValue = allValues.name;
+		const versionValue = allValues.factorioVersion as FactorioVersion | undefined;
+
+		// Only trigger state updates if values actually changed
+		if (nameValue !== searchText) {
+			setSearchText(nameValue || "");
+		}
+		if (versionValue && versionValue !== factorioVersion) {
+			setFactorioVersion(versionValue);
+			setAllMods([]); // Clear mods when version changes, useEffect will fetch new ones
+			setLoading(true); // Show loading indicator immediately
+		}
+		// Reset page only if search text or version changed
+		if (nameValue !== searchText || (versionValue && versionValue !== factorioVersion)) {
+			setPage(1);
+		}
+	};
+
+	// Update pagination and sorting state when table changes (logic remains the same)
+	const handleTableChange = (
+		pagination: any,
+		filters: any,
+		sorter: any
+	) => {
+		if (pagination.current !== page) {
+			setPage(pagination.current);
+		}
+		if (pagination.pageSize !== pageSize) {
+			setPageSize(pagination.pageSize);
+			setPage(1); // Reset to first page if page size changes
+		}
+
+		// Update sort state if sorter exists and has changed
+		if (sorter.field && (sorter.field !== sort || sorter.order !== (sortOrder === "asc" ? "ascend" : "descend"))) {
+			setSort(sorter.field);
+			setSortOrder(sorter.order === "ascend" ? "asc" : "desc");
+			setPage(1); // Reset to first page on sort change
+		} else if (!sorter.field && sort !== "name") { // Reset only if not already default
+			// Clear sort if column header clicked without order - default to name ascending
+			setSort("name");
+			setSortOrder("asc");
+			setPage(1);
+		}
+	};
+
+	// Helper to get antd sort order format (logic remains the same)
+	const getColumnSortOrder = (columnKey: string): "ascend" | "descend" | undefined => {
+		if (sort === columnKey) {
+			return sortOrder === "asc" ? "ascend" : "descend";
+		}
+		return undefined;
+	};
+
+	return <>
+		<Button icon={<SearchOutlined />} onClick={() => { setOpen(true); }}>Search</Button>
+		<Modal
+			title="Search Mod Portal"
+			open={open}
+			onCancel={() => { setOpen(false); }}
+			width={1000}
+			footer={[
+				<Button key="close" onClick={() => { setOpen(false); }}>
+					Close
+				</Button>,
+			]}
+		>
+			<Form
+				form={form}
+				layout="horizontal"
+				labelCol={{ span: 6 }}
+				wrapperCol={{ span: 18 }}
+				onValuesChange={handleSearch}
+				initialValues={{ factorioVersion: factorioVersion, name: searchText }}
+			>
+				<Form.Item name="name" label="Filter by Name or Title">
+					<Input placeholder="Start typing to filter..." />
+				</Form.Item>
+				<Form.Item
+					name="factorioVersion"
+					label="Factorio Version"
+				>
+					<Select>
+						{lib.ModPortalGetAllRequest.allowedVersions.map((literalSchema: any) => (
+							<Select.Option
+								key={literalSchema.const}
+								value={literalSchema.const}
+							>
+								{literalSchema.const}
+							</Select.Option>
+						)).reverse()}
+					</Select>
+				</Form.Item>
+			</Form>
+
+			{/* Display error message if fetching failed */}
+			{error && <Alert
+				message="Error Fetching Mods"
+				description={error.message}
+				type="error"
+				showIcon
+				style={{ marginBottom: 16 }}
+			/>}
+
+			<Table
+				dataSource={displayedMods}
+				rowKey={record => record.name}
+				loading={loading}
+				onChange={handleTableChange}
+				pagination={{
+					current: page,
+					pageSize: pageSize,
+					total: totalResults,
+					showSizeChanger: true,
+					pageSizeOptions: ["10", "20", "50", "100"],
+				}}
+				expandable={{
+					expandedRowRender: record => (
+						<div>
+							<p><strong>Summary:</strong> {record.summary ?? "N/A"}</p>
+							<p><strong>Downloads:</strong> {record.downloads_count?.toLocaleString() ?? "N/A"}</p>
+							<p><strong>Latest Release:</strong></p>
+							{record.latest_release ? (
+								<ul>
+									<li>Version: {record.latest_release.version ?? "N/A"}</li>
+									<li>
+										Factorio Version: {
+											record.latest_release.info_json?.factorio_version ?? "N/A"
+										}
+									</li>
+									<li>
+										Released: {
+											record.latest_release.released_at
+												? new Date(record.latest_release.released_at).toLocaleString()
+												: "N/A"
+										}
+									</li>
+								</ul>
+							) : <p>No release information available.</p>}
+							{account.hasPermission("core.mod.download_from_portal") && record.latest_release && (
+								<Button
+									onClick={() => {
+										handleControllerDownload(
+											record.name,
+											record.title,
+											record.latest_release!.version,
+											factorioVersion
+										);
+									}}
+									disabled={!record.latest_release?.version}
+								>
+									Download Latest Version
+								</Button>
+							)}
+						</div>
+					),
+				}}
+				columns={[
+					// Columns remain largely the same
+					{
+						title: "Name",
+						dataIndex: "name",
+						key: "name",
+						sorter: true,
+						sortOrder: getColumnSortOrder("name"),
+					},
+					{
+						title: "Title",
+						dataIndex: "title",
+						key: "title",
+						sorter: true,
+						sortOrder: getColumnSortOrder("title"),
+					},
+					{
+						title: "Author",
+						dataIndex: "owner",
+						key: "owner",
+						sorter: true,
+						sortOrder: getColumnSortOrder("owner"),
+					},
+					{
+						title: "Downloads",
+						dataIndex: "downloads_count",
+						key: "downloads_count",
+						sorter: true,
+						sortOrder: getColumnSortOrder("downloads_count"),
+						align: "right",
+						render: (count) => count?.toLocaleString() ?? "N/A",
+					},
+					{
+						title: "Latest Version",
+						key: "version",
+						render: (_, record) => record.latest_release?.version ?? "N/A",
+					},
+					{
+						title: "Actions",
+						key: "actions",
+						align: "center",
+						render: (_, record) => (
+							<Space>
+								<Tooltip title="Open in Factorio Mod Portal">
+									<a
+										href={`https://mods.factorio.com/mod/${record.name}`}
+										target="_blank"
+										rel="noopener noreferrer"
+									>
+										<Button type="text">
+											<img
+												src="https://mods.factorio.com/static/favicon.ico"
+												alt="Factorio Mod Portal"
+											/>
+										</Button>
+									</a>
+								</Tooltip>
+								{account.hasPermission("core.mod.download_from_portal") && record.latest_release && (
+									<Tooltip title="Download to Controller">
+										<Button
+											type="text"
+											icon={<DownloadOutlined />}
+											disabled={!record.latest_release.version}
+											onClick={() => {
+												handleControllerDownload(
+													record.name,
+													record.title,
+													record.latest_release!.version,
+													factorioVersion
+												);
+											}}
+										/>
+									</Tooltip>
+								)}
+							</Space>
+						),
+					},
+				]}
+			/>
+		</Modal>
+	</>;
+}
+
 export default function ModsPage() {
 	let account = useAccount();
 	let control = useContext(ControlContext);
@@ -223,7 +602,10 @@ export default function ModsPage() {
 				},
 			})}
 		/>
-		<SectionHeader title="Stored Mods" extra={uploadButton} />
+		<SectionHeader title="Stored Mods" extra={<Space>
+			<SearchModsButton />
+			{uploadButton}
+		</Space>} />
 
 		<Upload.Dragger
 			className="save-list-dragger"

packages/web_ui/src/components/NpmButton.tsx

@@ -0,0 +1,211 @@
+import { useContext, useEffect, useState } from "react";
+import { Button, Checkbox, Form, FormInstance, Input, Modal, Radio, Select } from "antd";
+
+import * as lib from "@clusterio/lib";
+
+import { useAccount } from "../model/account";
+import { notifyErrorHandler } from "../util/notify";
+import ControlContext from "./ControlContext";
+
+type NpmActionProps = {
+	setApplyAction(func: () => Promise<void>): void,
+	form: FormInstance,
+	target: lib.AddressShorthand,
+}
+
+function NpmRemoteUpdate({ setApplyAction, form, target }: NpmActionProps) {
+	const control = useContext(ControlContext);
+
+	setApplyAction(async () => {
+		if (target === "controller") {
+			return await control.send(new lib.ControllerUpdateRequest());
+		}
+		return await control.sendTo(target, new lib.HostUpdateRequest());
+	});
+
+	return <>
+	</>;
+}
+
+function NpmPluginUpdate({ setApplyAction, form, target }: NpmActionProps) {
+	const control = useContext(ControlContext);
+	const [plugins, setPlugins] = useState<lib.PluginDetails[]>([]);
+
+	setApplyAction(async () => {
+		const values = form.getFieldsValue();
+		if (!values.plugin || values.plugin === "") {
+			throw new Error("Plugin not provided");
+		}
+		await control.sendTo(target, new lib.PluginUpdateRequest(values.plugin));
+	});
+
+	useEffect(() => {
+		control.sendTo(target, new lib.PluginListRequest()).then(newPlugins => {
+			setPlugins(newPlugins);
+		}).catch(notifyErrorHandler("Error fetching role list"));
+	}, [target]);
+
+	return <>
+		<Form.Item label="Plugin" name="plugin">
+			<Select
+				showSearch
+				placeholder="Select a plugin"
+				optionLabelProp="label"
+				options={plugins
+					.filter(p => p.npmPackage !== undefined)
+					.map(plugin => ({
+						value: plugin.npmPackage,
+						label: `${plugin.title} (v${plugin.version})`,
+					}))
+				}
+			/>
+		</Form.Item>
+	</>;
+}
+
+function NpmPluginInstall({ setApplyAction, form, target }: NpmActionProps) {
+	const control = useContext(ControlContext);
+
+	setApplyAction(async () => {
+		const values = form.getFieldsValue();
+		if (!values.newPlugin || values.newPlugin === "") {
+			throw new Error("Plugin not provided");
+		}
+		await control.sendTo(target, new lib.PluginInstallRequest(values.newPlugin));
+	});
+
+	// TODO custom npm search based on tags
+	return <>
+		<Form.Item label="Plugin" name="newPlugin">
+			<Input />
+		</Form.Item>
+	</>;
+}
+
+export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: boolean, disabled?: boolean }) {
+	const control = useContext(ControlContext);
+	const account = useAccount();
+	const [open, setOpen] = useState(false);
+	const [formAction, setFormAction] = useState<string | undefined>(undefined);
+	const [form] = Form.useForm();
+
+	const [allows, setAllows] = useState({
+		remoteUpdates: true, pluginUpdates: true, pluginInstall: true,
+	});
+
+	useEffect(() => {
+		if (props.target === "controller") {
+			if (!account.hasPermission("core.controller.get_config")) {
+				setAllows({ remoteUpdates: true, pluginUpdates: true, pluginInstall: true });
+				return; // If a user doesn't have config perms, then assume true to allow the sending the request
+			}
+			control.send(new lib.ControllerConfigGetRequest()).then(async (serializedConfig) => {
+				const config = lib.ControllerConfig.fromJSON(serializedConfig, "control");
+				setAllows({
+					remoteUpdates: config.get("controller.allow_remote_updates"),
+					pluginUpdates: config.get("controller.allow_plugin_updates"),
+					pluginInstall: config.get("controller.allow_plugin_install"),
+				});
+			}).catch(notifyErrorHandler("Failed to fetch controller config for remote updates"));
+		} else {
+			if (!account.hasPermission("core.host.get_config")) {
+				setAllows({ remoteUpdates: true, pluginUpdates: true, pluginInstall: true });
+				return; // If a user doesn't have config perms, then assume true to allow the sending the request
+			}
+			control.sendTo(props.target, new lib.HostConfigGetRequest()).then(async (serializedConfig) => {
+				const config = lib.HostConfig.fromJSON(serializedConfig, "control");
+				setAllows({
+					remoteUpdates: config.get("host.allow_remote_updates"),
+					pluginUpdates: config.get("host.allow_plugin_updates"),
+					pluginInstall: config.get("host.allow_plugin_install"),
+				});
+			}).catch(notifyErrorHandler("Failed to fetch host config for remote updates"));
+		}
+	}, [props.target]);
+
+	function onValuesChange({ action } : { action?: string }) {
+		if (action) {
+			setFormAction(action);
+		}
+	}
+
+	let applyAction: () => Promise<void>;
+	const setApplyAction = (func: () => Promise<void>) => { applyAction = func; };
+	async function onOk() {
+		if (!applyAction) {
+			form.setFields([{ name: "action", errors: ["Action is required"] }]);
+			return;
+		}
+		await applyAction();
+
+		const values = form.getFieldsValue();
+		if (values.restart) {
+			// Ternary not used because TS disliked it
+			if (props.target === "controller") {
+				await control.sendTo(props.target, new lib.ControllerRestartRequest());
+			} else {
+				await control.sendTo(props.target, new lib.HostRestartRequest());
+			}
+		}
+
+		setFormAction(undefined);
+		setOpen(false);
+	}
+
+	return <>
+		<Button
+			type="default"
+			disabled={props.disabled}
+			onClick={() => { setOpen(true); }}
+		>Updates</Button>
+		<Modal
+			title="Updates"
+			okText="Apply"
+			open={open}
+			okButtonProps={{disabled: formAction === undefined}}
+			onOk={() => { onOk().catch(notifyErrorHandler(`Error running ${formAction}`)); }}
+			onCancel={() => { setOpen(false); }}
+			destroyOnClose
+		>
+			<Form form={form} onValuesChange={onValuesChange} clearOnDestroy>
+				<Form.Item label="Action" name="action">
+					<Radio.Group value={formAction}>
+						{account.hasPermission(`core.${props.target === "controller" ? "controller" : "host"}.update`)
+							? <Radio.Button value="remote_update" disabled={!allows.remoteUpdates}>
+								Update Clusterio
+							</Radio.Button>: undefined}
+						{account.hasPermission("core.plugin.update")
+							? <Radio.Button value="plugin_update" disabled={!allows.pluginUpdates}>
+								Update Plugin
+							</Radio.Button> : undefined}
+						{account.hasPermission("core.plugin.install")
+							? <Radio.Button value="plugin_install" disabled={!allows.pluginInstall}>
+								Install Plugin
+							</Radio.Button> : undefined}
+					</Radio.Group>
+				</Form.Item>
+				{
+					formAction !== undefined
+					&& account.hasPermission(`core.${props.target === "controller" ? "controller" : "host"}.restart`)
+					&& <Form.Item label="Restart on completion" name="restart">
+						<Checkbox disabled={props.canRestart === false}/>
+					</Form.Item>
+				}
+				{formAction === "remote_update"
+					? <NpmRemoteUpdate {...{setApplyAction, form, target: props.target}}/> : undefined}
+				{formAction === "plugin_update"
+					? <NpmPluginUpdate {...{setApplyAction, form, target: props.target}}/> : undefined}
+				{formAction === "plugin_install"
+					? <NpmPluginInstall {...{setApplyAction, form, target: props.target}}/> : undefined}
+			</Form>
+		</Modal>
+	</>;
+}
+
+export function hasNpmButtonPermission(controller: boolean) {
+	const account = useAccount();
+	return account.hasAnyPermission(
+		controller ? "core.controller.update" : "core.host.update",
+		"core.plugin.update", "core.plugin.install"
+	);
+}

packages/web_ui/src/components/RoleViewPage.tsx

@@ -5,6 +5,7 @@ import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 
 import * as lib from "@clusterio/lib";
 
+import { useRole } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
@@ -12,30 +13,6 @@ import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
 import { notifyErrorHandler } from "../util/notify";
 
-
-function useRole(id: number): [Readonly<lib.Role> | undefined, boolean, () => void] {
-	let control = useContext(ControlContext);
-	let [role, setRole] = useState<lib.Role>();
-	let [synced, setSynced] = useState(false);
-
-	function updateRole() {
-		// XXX optimize by requesting only the role in question
-		control.send(new lib.RoleListRequest()).then(roles => {
-			let match = roles.find(u => u.id === id);
-			if (match) {
-				setRole(match);
-			}
-			setSynced(true);
-		});
-	}
-
-	useEffect(() => {
-		updateRole();
-	}, [id]);
-
-	return [role, synced, updateRole];
-}
-
 export default function RoleViewPage() {
 	let params = useParams();
 	let roleId = Number(params.id);
@@ -44,7 +21,7 @@ export default function RoleViewPage() {
 
 	let account = useAccount();
 	let control = useContext(ControlContext);
-	const [role, synced, updateRole] = useRole(roleId);
+	const [role, synced] = useRole(roleId);
 	let [edited, setEdited] = useState(false);
 
 
@@ -63,6 +40,7 @@ export default function RoleViewPage() {
 		</PageLayout>;
 	}
 
+	// TODO: Update displayed perms when role perms change and have not been edited locally
 	let initialValues = {
 		name: role["name"],
 		description: role["description"],
@@ -88,7 +66,6 @@ export default function RoleViewPage() {
 					new lib.RoleUpdateRequest(roleId, values.name || "", values.description || "", newPermissions)
 				).then(() => {
 					setEdited(false);
-					updateRole();
 				}).catch(notifyErrorHandler("Error applying changes"));
 			}}
 		>

packages/web_ui/src/components/RolesPage.tsx

@@ -4,6 +4,7 @@ import { Button, Form, Input, Modal, Table } from "antd";
 
 import * as lib from "@clusterio/lib";
 
+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import { notifyErrorHandler } from "../util/notify";
@@ -60,16 +61,8 @@ function CreateRoleButton() {
 
 export default function RolesPage() {
 	let account = useAccount();
-	let control = useContext(ControlContext);
 	let navigate = useNavigate();
-
-	let [roles, setRoles] = useState<lib.Role[]>([]);
-
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(newRoles);
-		}).catch(notifyErrorHandler("Error fetching role list"));
-	}, []);
+	const [roles] = useRoles();
 
 	return <PageLayout nav={[{ name: "Roles" }]}>
 		<PageHeader
@@ -88,7 +81,7 @@ export default function RolesPage() {
 					dataIndex: "description",
 				},
 			]}
-			dataSource={roles}
+			dataSource={[...roles.values()]}
 			pagination={false}
 			rowKey={role => role.id}
 			onRow={(role, rowIndex) => ({

packages/web_ui/src/components/SiteLayout.tsx

@@ -1,6 +1,6 @@
 import React, { Fragment, useContext, useEffect, useState } from "react";
 import { Route, Routes, useNavigate } from "react-router-dom";
-import { Dropdown, Layout, Menu, MenuProps } from "antd";
+import { Button, Dropdown, Layout, Menu, MenuProps, Tooltip } from "antd";
 import UserOutlined from "@ant-design/icons/UserOutlined";
 import webUiPackage from "../../package.json";
 
@@ -10,6 +10,9 @@ import ErrorPage from "./ErrorPage";
 import ControlContext from "./ControlContext";
 import { pages } from "../pages";
 import { DraggingContext } from "../model/is_dragging";
+import { saveJson } from "../util/save_file";
+import { ControlConfig } from "@clusterio/lib";
+import { DownloadOutlined } from "@ant-design/icons";
 
 type MenuItem = Required<MenuProps>["items"][number];
 
@@ -24,6 +27,8 @@ function isActiveDropzone(element: HTMLElement | null): boolean {
 	while (element && depth < checkDepth) {
 		if (element.classList.contains("dropzone")) {
 			return element.classList.contains("enabled");
+		} else if (element.classList.contains("ant-upload-drag")) {
+			return !element.classList.contains("disabled");
 		}
 		element = element.parentElement;
 		depth += 1;
@@ -36,6 +41,7 @@ export default function SiteLayout() {
 	let [currentSidebarPath, setCurrentSidebarPath] = useState<string | null>(null);
 	let account = useAccount();
 	let plugins = useContext(ControlContext).plugins;
+	const control = useContext(ControlContext);
 	const [dragging, setDragging] = useState(0);
 
 	function SetSidebar(props: { path: string | null }) {
@@ -49,12 +55,24 @@ export default function SiteLayout() {
 		onClick: ({ key }: { key: string }) => {
 			if (key === "user") {
 				navigate(`/users/${account.name}/view`);
+			} else if (key === "ctlConfig") {
+				const config = new ControlConfig("control", {
+					"control.controller_token": control.connector.token,
+					"control.controller_url": (new URL(webRoot, document.location.href)).href,
+				});
+				saveJson("config-control.json", config.toJSON());
 			} else if (key === "logOut") {
 				account.logOut();
 			}
 		},
 		items: [
 			{ label: account.name, key: "user" },
+			{
+				label: <Tooltip title="Download credentials and configuration file for cli interface">
+					Ctl Config <Button size="small" icon={<DownloadOutlined />} />
+				</Tooltip>,
+				key: "ctlConfig",
+			},
 			{ type: "divider" },
 			{ label: "Log out", danger: true, key: "logOut" },
 		],

packages/web_ui/src/components/UserViewPage.tsx

@@ -8,6 +8,7 @@ import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 
 import * as lib from "@clusterio/lib";
 
+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import { useInstances } from "../model/instance";
 import ControlContext from "./ControlContext";
@@ -35,21 +36,13 @@ export default function UserViewPage() {
 	let control = useContext(ControlContext);
 	let [instances] = useInstances();
 	const [user, synced] = useUser(userName);
-	let [roles, setRoles] = useState<Map<number, lib.Role>>(new Map());
+	let [roles] = useRoles();
 	let [form] = Form.useForm();
 	let [rolesDirty, setRolesDirty] = useState<boolean>(false);
 	let [banReasonDirty, setBanReasonDirty] = useState<boolean>(false);
 	let [applyingRoles, setApplyingRoles] = useState<boolean>(false);
 	let [rolesError, setRolesError] = useState<string|undefined>();
 
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(new Map(newRoles.map(role => [role.id, role])));
-		}).catch(() => {
-			setRoles(new Map());
-		});
-	}, []);
-
 	useEffect(() => {
 		if (
 			account.hasPermission("core.user.set_banned")
@@ -76,6 +69,7 @@ export default function UserViewPage() {
 		</PageLayout>;
 	}
 
+	// TODO: Update displayed roles when user roles change and have not been edited locally
 	let roleSelector = <Form.Item
 		noStyle
 		name="roles"

packages/web_ui/src/components/UsersPage.tsx

@@ -1,13 +1,24 @@
-import React, { useEffect, useContext, useState } from "react";
+import React, { useEffect, useContext, useState, useRef } from "react";
 import { useNavigate } from "react-router-dom";
-import { Button, Form, Input, Modal, Space, Table, Tag } from "antd";
+import { Static } from "@sinclair/typebox";
+
+import {
+	Button, Form, FormInstance, Input, InputRef, Modal,
+	Radio, Space, Table, Tag, Upload, UploadProps,
+} from "antd";
+
+import {
+	InboxOutlined, SearchOutlined,
+} from "@ant-design/icons";
 
 import * as lib from "@clusterio/lib";
 
+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import { notifyErrorHandler } from "../util/notify";
 import { formatDuration } from "../util/time_format";
+import { saveJson } from "../util/save_file";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
@@ -16,7 +27,6 @@ import Link from "./Link";
 
 const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare;
 
-
 function CreateUserButton() {
 	let control = useContext(ControlContext);
 	let navigate = useNavigate();
@@ -57,26 +67,326 @@ function CreateUserButton() {
 	</>;
 }
 
+type UserBulkActionProps = {
+	setApplyAction(func: () => Promise<void>): void,
+	form: FormInstance,
+	restore?: boolean,
+}
+
+function UserBulkActionImport({ setApplyAction, form, restore }: UserBulkActionProps) {
+	const control = useContext(ControlContext);
+	const account = useAccount();
+
+	// Combines an array of users into the main array to be sent to the controller
+	function combineMixed(
+		prop: "is_admin" | "is_banned" | "is_whitelisted",
+		users: Array<Static<typeof lib.ClusterioUserExport.factorioUserSchema>>,
+		usersToSend: Map<string, Static<typeof lib.ClusterioUserExport.clusterioUserSchema>>
+	) {
+		for (const user of users) {
+			if (typeof user !== "string") {
+				// It can only be a ban in this case
+				const existingUser = usersToSend.get(user.username);
+				if (existingUser) {
+					existingUser.is_banned = true;
+					existingUser.ban_reason = existingUser.ban_reason ? existingUser.ban_reason : user.reason;
+				} else {
+					usersToSend.set(user.username, {
+						is_banned: true,
+						...user,
+					});
+				}
+			} else {
+				// We only have the username
+				const existingUser = usersToSend.get(user);
+				if (existingUser) {
+					existingUser[prop] = true;
+				} else {
+					usersToSend.set(user, {
+						username: user,
+						[prop]: true,
+					});
+				}
+			}
+		}
+	}
+
+	// Parse a user json and combine into the format to be sent to the controller
+	function parseUsers(
+		users: Array<Static<typeof lib.ClusterioUserExport.clusterioUserSchema>>,
+		usersToSend: Map<string, Static<typeof lib.ClusterioUserExport.clusterioUserSchema>>
+	) {
+		for (const user of users) {
+			const existingUser = usersToSend.get(user.username);
+			// Combine all fields with an existing user
+			if (existingUser) {
+				existingUser.is_admin ||= user.is_admin;
+				existingUser.is_banned ||= user.is_banned;
+				existingUser.is_whitelisted ||= user.is_whitelisted;
+				existingUser.ban_reason = existingUser.ban_reason ? existingUser.ban_reason : user.ban_reason;
+			}
+			// We do not send false values
+			if (user.is_admin === false) {
+				delete user.is_admin;
+			}
+			if (user.is_banned === false) {
+				delete user.is_banned;
+				delete user.ban_reason;
+			}
+			if (user.is_whitelisted === false) {
+				delete user.is_whitelisted;
+			}
+			usersToSend.set(user.username, user);
+		}
+	}
+
+	// Parse a ban json and combine into the format to be sent to the controller
+	function parseBans(
+		users: Array<Static<typeof lib.ClusterioUserExport.factorioUserSchema>>,
+		usersToSend: Map<string, string | { username: string, reason: string }>
+	) {
+		for (const user of users) {
+			if (typeof user === "string") {
+				if (!usersToSend.has(user)) {
+					usersToSend.set(user, user);
+				}
+			} else if (!user.reason) {
+				if (!usersToSend.has(user.username)) {
+					usersToSend.set(user.username, user.username);
+				}
+			} else {
+				usersToSend.set(user.username, user);
+			}
+		}
+	}
+
+	// Parse an admin or whitelist json and combine into the format to be sent to the controller
+	function parseAdminsWhitelist(
+		users: Array<string>,
+		usersToSend: Set<string>,
+	) {
+		for (const user of users) {
+			usersToSend.add(user);
+		}
+	}
+
+	setApplyAction(async () => {
+		let backup: object | undefined;
+		const values = form.getFieldsValue();
+		const importType = restore ? values.restoreType : values.importType;
+		switch (importType) {
+			case "mixed": {
+				// Parse and combine multiple json types then send
+				const usersToSend = new Map<string, Static<typeof lib.ClusterioUserExport.clusterioUserSchema>>();
+				for (const file of values.fileList as File[]) {
+					const json = JSON.parse(await file.text());
+					if (json.export_version) {
+						parseUsers(json.users, usersToSend);
+					} else if (file.name.includes("ban")) {
+						const users = new Map();
+						parseBans(json, users);
+						combineMixed("is_banned", [...users.values()], usersToSend);
+					} else if (file.name.includes("admin")) {
+						const users = new Set<string>();
+						parseAdminsWhitelist(json, users);
+						combineMixed("is_admin", [...users.values()], usersToSend);
+					} else if (file.name.includes("whitelist")) {
+						const users = new Set<string>();
+						parseAdminsWhitelist(json, users);
+						combineMixed("is_whitelisted", [...users.values()], usersToSend);
+					} else {
+						throw new Error(`Unknown json (could not guess by file name): ${file.name}`);
+					}
+				}
+				backup = await control.send(new lib.UserBulkImportRequest("users", [...usersToSend.values()], restore));
+				break;
+			}
+
+			case "users": {
+				// Parse and user jsons then send
+				const usersToSend = new Map<string, Static<typeof lib.ClusterioUserExport.clusterioUserSchema>>();
+				for (const file of values.fileList as File[]) {
+					parseUsers(JSON.parse(await file.text()).users, usersToSend);
+				}
+				backup = await control.send(new lib.UserBulkImportRequest("users", [...usersToSend.values()], restore));
+				break;
+			}
+
+			case "bans": {
+				// Parse and combine ban jsons then send
+				const usersToSend = new Map<string, Static<typeof lib.ClusterioUserExport.factorioUserSchema>>();
+				for (const file of values.fileList as File[]) {
+					parseBans(JSON.parse(await file.text()), usersToSend);
+				}
+				backup = await control.send(new lib.UserBulkImportRequest("bans", [...usersToSend.values()], restore));
+				break;
+			}
+
+			case "admins":
+			case "whitelist": {
+				// Parse and combine admin or whitelist jsons then send
+				const usersToSend = new Set<string>();
+				for (const file of values.fileList as File[]) {
+					parseAdminsWhitelist(JSON.parse(await file.text()), usersToSend);
+				}
+				backup = await control.send(
+					new lib.UserBulkImportRequest(importType, [...usersToSend.keys()], restore)
+				);
+				break;
+			}
+
+			default: {
+				// Should be unreachable
+				throw new Error(`Unknown importType: ${importType}`);
+			}
+		}
+		if (restore && backup) {
+			saveJson(`${importType}-backup.json`, backup);
+		}
+	});
+
+	const normaliseFiles = (e: any) => (Array.isArray(e) ? e : e.fileList).map((f: any) => f.originFileObj);
+	const uploadProps: UploadProps = {
+		accept: ".json",
+		multiple: true,
+		beforeUpload: (file) => false,
+	};
+
+	return <>
+		<Form.Item
+			label="Type"
+			name={restore ? "restoreType" : "importType"}
+			initialValue={restore ? "users" : "mixed"}
+		>
+			<Radio.Group>
+				{restore ? undefined : <Radio.Button value="mixed" disabled={!account.hasAnyPermission(
+					"core.user.set_admin", "core.user.set_banned", "core.user.set_whitelisted"
+				)}>Mixed</Radio.Button>}
+				<Radio.Button value="users" disabled={!account.hasAllPermission(
+					"core.user.set_admin", "core.user.set_banned", "core.user.set_whitelisted"
+				)}>Users</Radio.Button>
+				<Radio.Button value="admins" disabled={!account.hasPermission(
+					"core.user.set_admin"
+				)}>Admins</Radio.Button>
+				<Radio.Button value="bans" disabled={!account.hasPermission(
+					"core.user.set_banned"
+				)}>Bans</Radio.Button>
+				<Radio.Button value="whitelist" disabled={!account.hasPermission(
+					"core.user.set_whitelisted"
+				)}>Whitelist</Radio.Button>
+			</Radio.Group>
+		</Form.Item>
+		<Form.Item name="fileList" valuePropName="fileList" getValueFromEvent={normaliseFiles} noStyle>
+			<Upload.Dragger {...uploadProps}>
+				<p className="ant-upload-drag-icon">
+					<InboxOutlined />
+				</p>
+				<p className="ant-upload-text">Click or drag file to this area to import</p>
+				<p className="ant-upload-hint">Support for a single or bulk import.</p>
+			</Upload.Dragger>
+		</Form.Item>
+	</>;
+}
+
+function UserBulkActionExport({ setApplyAction, form }: UserBulkActionProps) {
+	const control = useContext(ControlContext);
+
+	setApplyAction(async () => {
+		const values = form.getFieldsValue();
+		const response = await control.send(new lib.UserBulkExportRequest(values.exportType));
+		switch (values.exportType) {
+			case "users": return saveJson("clusterio-userlist.json", response);
+			case "admins": return saveJson("server-adminlist.json", response);
+			case "bans": return saveJson("server-banlist.json", response);
+			case "whitelist": return saveJson("server-whitelist.json", response);
+			default: throw new Error(`Unknown exportType: ${values.importType}`);
+		}
+	});
+
+	return <>
+		<Form.Item label="Type" name="exportType" initialValue="users">
+			<Radio.Group>
+				<Radio.Button value="users">Users</Radio.Button>
+				<Radio.Button value="admins">Admins</Radio.Button>
+				<Radio.Button value="bans">Bans</Radio.Button>
+				<Radio.Button value="whitelist">Whitelist</Radio.Button>
+			</Radio.Group>
+		</Form.Item>
+	</>;
+}
+
+function BulkUserActionButton() {
+	const account = useAccount();
+	const [open, setOpen] = useState(false);
+	const [formAction, setFormAction] = useState<string | undefined>(undefined);
+	const [form] = Form.useForm();
+
+	function onValuesChange({ action } : { action?: string }) {
+		if (action) {
+			setFormAction(action);
+		}
+	}
+
+	let applyAction: () => Promise<void>;
+	const setApplyAction = (func: () => Promise<void>) => { applyAction = func; };
+	async function onOk() {
+		if (!applyAction) {
+			form.setFields([{ name: "action", errors: ["Action is required"] }]);
+			return;
+		}
+		await applyAction();
+		setFormAction(undefined);
+		setOpen(false);
+	}
+
+	return <>
+		<Button
+			type="default"
+			onClick={() => { setOpen(true); }}
+		>Bulk Actions</Button>
+		<Modal
+			title="Bulk Actions"
+			okText="Apply"
+			open={open}
+			okButtonProps={{disabled: formAction === undefined}}
+			onOk={() => { onOk().catch(notifyErrorHandler(`Error running ${formAction}`)); }}
+			onCancel={() => { setOpen(false); }}
+			destroyOnClose
+		>
+			<Form form={form} onValuesChange={onValuesChange} clearOnDestroy>
+				<Form.Item label="Action" name="action">
+					<Radio.Group value={formAction}>
+						{account.hasPermission("core.user.bulk_import")
+							? <Radio.Button value="import">Import</Radio.Button> : undefined}
+						{account.hasPermission("core.user.bulk_export")
+							? <Radio.Button value="export">Export</Radio.Button> : undefined}
+						{account.hasPermission("core.user.bulk_restore")
+							? <Radio.Button value="restore">Restore</Radio.Button> : undefined}
+					</Radio.Group>
+				</Form.Item>
+				{formAction === "import" ? <UserBulkActionImport {...{setApplyAction, form}}/> : undefined}
+				{formAction === "export" ? <UserBulkActionExport {...{setApplyAction, form}}/> : undefined}
+				{formAction === "restore" ? <UserBulkActionImport restore {...{setApplyAction, form}}/> : undefined}
+			</Form>
+		</Modal>
+	</>;
+}
+
 export default function UsersPage() {
 	let account = useAccount();
-	let control = useContext(ControlContext);
 	let navigate = useNavigate();
 	let [users] = useUsers();
-
-	let [roles, setRoles] = useState(new Map());
-
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(new Map(newRoles.map(role => [role.id, role])));
-		}).catch(() => {
-			// ignore
-		});
-	}, []);
+	const [roles] = useRoles();
+	const searchInput = useRef<InputRef>(null);
 
 	return <PageLayout nav={[{ name: "Users" }]}>
 		<PageHeader
 			title="Users"
-			extra={account.hasPermission("core.user.create") ? <CreateUserButton /> : undefined}
+			extra={<>
+				{account.hasPermission("core.user.create") ? <CreateUserButton /> : undefined}
+				{account.hasAnyPermission("core.user.bulk_import", "core.user.bulk_export")
+					? <BulkUserActionButton /> : undefined}
+			</>}
 		/>
 		<Table
 			columns={[
@@ -93,14 +403,33 @@ export default function UsersPage() {
 					</Space>,
 					defaultSortOrder: "ascend",
 					sorter: (a, b) => strcmp(a.name, b.name),
+					filterIcon: (filtered) => <SearchOutlined style={{ color: filtered ? "#1677ff" : undefined }} />,
+					onFilter: (value, record) => record.name.toLowerCase().includes((value as string).toLowerCase()),
+					filterDropdownProps: {
+						onOpenChange: open => open && setTimeout(() => searchInput.current?.select(), 100),
+					},
+					filterDropdown: ({ selectedKeys, setSelectedKeys, confirm, clearFilters }) => (
+						<div style={{ padding: 4 }} onKeyDown={(e) => e.stopPropagation()}>
+							<Input.Search
+								allowClear
+								ref={searchInput}
+								placeholder={"Search username"}
+								value={selectedKeys[0]}
+								onChange={(e) => setSelectedKeys([e.target.value])}
+								onClear={() => clearFilters && clearFilters({ confirm: true, closeDropdown: true })}
+								onSearch={() => confirm({ closeDropdown: true })}
+							/>
+						</div>
+					),
 				},
 				{
 					title: "Roles",
 					key: "roles",
 					render: (_, user) => (
-						[...user.roleIds].map(id => <Link key={id} to={`/roles/${id}/view`} onClick={e => e.stopPropagation()}>
-							<Tag>{(roles.get(id) || { name: id }).name}</Tag>
-						</Link>)
+						[...user.roleIds]
+							.map(id => <Link key={id} to={`/roles/${id}/view`} onClick={e => e.stopPropagation()}>
+								<Tag>{(roles.get(id) || { name: id }).name}</Tag>
+							</Link>)
 					),
 				},
 				{
@@ -127,7 +456,12 @@ export default function UsersPage() {
 				},
 			]}
 			dataSource={[...users.values()]}
-			pagination={false}
+			pagination={{
+				defaultPageSize: 50,
+				showSizeChanger: true,
+				pageSizeOptions: ["10", "20", "50", "100", "200"],
+				showTotal: total => `${total} Users`,
+			}}
 			rowKey={user => user.name}
 			onRow={(user, rowIndex) => ({
 				onClick: event => {

packages/web_ui/src/components/system_metrics.tsx

@@ -1,7 +1,8 @@
 import React from "react";
 
 import * as lib from "@clusterio/lib";
-import { Progress } from "antd";
+import { Progress, Tooltip } from "antd";
+import { ReloadOutlined } from "@ant-design/icons";
 
 export function MetricCpuRatio(props: { system?: lib.SystemInfo }) {
 	if (!props.system) {
@@ -53,3 +54,12 @@ export function MetricDiskUsed(props: { system?: lib.SystemInfo }) {
 	const capacity = lib.formatBytes(props.system.diskCapacity);
 	return `${used} / ${capacity}`;
 }
+
+export function RestartRequired(props: { system?: lib.SystemInfo }) {
+	if (!props.system || !props.system.restartRequired) {
+		return "";
+	}
+	return <Tooltip title="Restart Required">
+		<ReloadOutlined style={{ color: "yellow" }}/>
+	</Tooltip>;
+}

packages/web_ui/src/model/roles.ts

@@ -0,0 +1,23 @@
+import { useCallback, useContext, useSyncExternalStore } from "react";
+import ControlContext from "../components/ControlContext";
+
+import * as lib from "@clusterio/lib";
+
+export function useRolesOfUser(user?: lib.User) {
+	const [roles, synced] = useRoles();
+	return [
+		user !== undefined ? new Map([...user.roleIds].map(roleId => [roleId, roles.get(roleId)])) : new Map(),
+		synced,
+	] as const;
+}
+
+export function useRole(id?: number) {
+	const [roles, synced] = useRoles();
+	return [id !== undefined ? roles.get(id) : undefined, synced] as const;
+}
+
+export function useRoles() {
+	const control = useContext(ControlContext);
+	const subscribe = useCallback((callback: () => void) => control.roles.subscribe(callback), [control]);
+	return useSyncExternalStore(subscribe, () => control.roles.getSnapshot());
+}

packages/web_ui/src/pages.tsx

@@ -15,6 +15,7 @@ import RolesPage from "./components/RolesPage";
 import RoleViewPage from "./components/RoleViewPage";
 import PluginsPage from "./components/PluginsPage";
 import PluginViewPage from "./components/PluginViewPage";
+import ErrorPage from "./components/ErrorPage";
 
 export const pages: PluginPage[] = [
 	{
@@ -87,6 +88,10 @@ export const pages: PluginPage[] = [
 		sidebarPath: "/plugins",
 		content: <PluginViewPage />,
 	},
+	{
+		path: "/error",
+		content: <ErrorPage throw error={new Error("Example Error Message")} />,
+	},
 ];
 
 export const sidebarPages = [

packages/web_ui/src/util/clipboard.ts

@@ -1,7 +1,11 @@
 import { useEffect, useState } from "react";
 import { EventEmitter } from "events";
 
-class Permission extends EventEmitter {
+type PermissionEvents = {
+	"change": [ status: string ]
+};
+
+class Permission extends EventEmitter<PermissionEvents> {
 	status!: { state: string };
 
 	constructor(name: PermissionName) {

packages/web_ui/src/util/save_file.ts

@@ -0,0 +1,25 @@
+
+/**
+ * Downloads a file to the users PC, should only be used following direct user input.
+ * @param name Name of the file to save
+ * @param blob Content to save
+ */
+export function saveFile(name: string, blob: Blob) {
+	// This is the most common and best supported method, for full support we should consider using npm:file-saver
+	const a = document.createElement("a");
+	a.href = URL.createObjectURL(blob);
+	a.download = name;
+	a.addEventListener("click", (e) => {
+		setTimeout(() => URL.revokeObjectURL(a.href), 30 * 1000);
+	});
+	a.click();
+};
+
+/**
+ * Downloads a json file to the users PC
+ * @param name Name of the file to save, should include .json
+ * @param json The object to save as a formated json string
+ */
+export function saveJson(name: string, json: object) {
+	saveFile(name, new Blob([JSON.stringify(json, null, 2)], { type: "application/json" }));
+}

packages/web_ui/src/util/websocket.ts

@@ -11,7 +11,7 @@ type saveListHandler = (saveListEvent: lib.SaveDetails) => void;
 type modPackHandler = (modPack: lib.ModPack) => void;
 type modInfoHandler = (modInfo: lib.ModInfo) => void;
 type userHandler = (rawUser: lib.User) => void;
-type logHandler = (info: { level:string, message:string }) => void;
+type logHandler = (info: { level: keyof typeof lib.levels, message: string }) => void;
 
 /**
  * Connector for control connection to controller
@@ -67,6 +67,7 @@ export class Control extends lib.Link {
 	modPacks = new lib.EventSubscriber(lib.ModPackUpdatesEvent, this);
 	mods = new lib.EventSubscriber(lib.ModUpdatesEvent, this);
 	users = new lib.EventSubscriber(lib.UserUpdatesEvent, this);
+	roles = new lib.EventSubscriber(lib.RoleUpdatesEvent, this);
 
 	declare connector: ControlConnector;
 

plugins/global_chat/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-global_chat",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"description": "Clusterio plugin forwarding between Factorio servers",
 	"author": "Hornwitser <github@hornwitser.no>",
 	"homepage": "https://github.com/clusterio/clusterio#readme",

plugins/global_chat/test/plugin.js

@@ -26,7 +26,6 @@ describe("global_chat plugin", function() {
 		let instancePlugin;
 
 		before(async function() {
-			lib.Link.register(ChatEvent);
 			instancePlugin = new instance.InstancePlugin(info, new mock.MockInstance(), new mock.MockHost());
 			await instancePlugin.init();
 		});

plugins/inventory_sync/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-inventory_sync",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"description": "Clusterio plugin forwarding between Factorio servers",
 	"author": "Danielv <danielv@danielv.no>",
 	"homepage": "https://github.com/clusterio/factorioClusterio#readme",

plugins/player_auth/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-player_auth",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"description": "Clusterio plugin authenticating logged in players to the web interface",
 	"author": "Hornwitser <github@hornwitser.no>",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
@@ -38,7 +38,6 @@
 		"@types/node": "^20.17.19",
 		"@types/react": "^18.3.18",
 		"antd": "^5.24.2",
-		"phin": "^3.7.0",
 		"react": "^18.2.0",
 		"react-dom": "^18.2.0",
 		"typescript": "^5.7.3",

plugins/player_auth/test/plugin.js

@@ -1,6 +1,5 @@
 "use strict";
 const assert = require("assert").strict;
-const phin = require("phin");
 const jwt = require("jsonwebtoken");
 
 const mock = require("../../../test/mock");
@@ -11,6 +10,13 @@ const info = require("../dist/node/index").plugin;
 const { FetchPlayerCodeRequest, SetVerifyCodeRequest } = require("../dist/node/messages");
 const lib = require("@clusterio/lib");
 
+function postJSON(url, body) {
+	return fetch(url, {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(body),
+	});
+}
 
 describe("player_auth", function() {
 	describe("controller.js", function() {
@@ -39,6 +45,7 @@ describe("player_auth", function() {
 		describe("ControllerPlugin", function() {
 			let controllerPlugin;
 			let controllerUrl;
+			let endpoint;
 			before(async function() {
 				controllerPlugin = await mock.createControllerPlugin(controller.ControllerPlugin, info);
 				controllerPlugin.controller.mockConfigEntries.set("player_auth.code_length", 10);
@@ -75,11 +82,8 @@ describe("player_auth", function() {
 					addInstance(3, "running", false, "running unloaded");
 					addInstance(4, "stopped", false, "stopped unloaded");
 					addInstance(5, "running", true, undefined);
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/servers`,
-						parse: "json",
-					});
-					assert.deepEqual(result.body, ["running loaded", "unnamed server"]);
+					const result = await fetch(`${controllerUrl}/api/player_auth/servers`);
+					assert.deepEqual(await result.json(), ["running loaded", "unnamed server"]);
 					for (let id of [1, 2, 3, 4, 5]) {
 						controllerPlugin.controller.instances.delete(id);
 					}
@@ -87,128 +91,81 @@ describe("player_auth", function() {
 			});
 
 			describe("/api/player_auth/player_code", function() {
+				before(function() {
+					endpoint = `${controllerUrl}/api/player_auth/player_code`;
+				});
 				it("should return 400 on invalid json", async function() {
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/player_code`,
-						method: "POST",
-						headers: {
-							"Content-Type": "application/json",
-						},
-						data: "invalid",
-					});
-					assert.equal(result.statusCode, 400);
+					const result = await postJSON(endpoint, "invalid");
+					assert.equal(result.status, 400);
 				});
 				it("should return 400 on valid json which is not an object", async function() {
 					for (let obj of [true, false, null, "string", 123, ["an", "array"]]) {
-						let result = await phin({
-							url: `${controllerUrl}/api/player_auth/player_code`,
-							method: "POST",
-							headers: {
-								"Content-Type": "application/json",
-							},
-							data: JSON.stringify(obj),
-						});
-						assert.equal(result.statusCode, 400);
+						const result = await postJSON(endpoint, obj);
+						assert.equal(result.status, 400);
 					}
 				});
 				it("should return 400 if player_code is not a string", async function() {
 					for (let obj of [true, false, null, 123, [], {}]) {
-						let result = await phin({
-							url: `${controllerUrl}/api/player_auth/player_code`,
-							method: "POST",
-							headers: {
-								"Content-Type": "application/json",
-							},
-							data: JSON.stringify({ player_code: obj }),
-						});
-						assert.equal(result.statusCode, 400);
+						const result = await postJSON(endpoint, { player_code: obj });
+						assert.equal(result.status, 400);
 					}
 				});
 				it("should return invalid player_code if the code is expired", async function() {
 					let expiresMs = Date.now() - 1000;
 					controllerPlugin.players.set("expired", { playerCode: "expired", verifyCode: null, expiresMs });
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/player_code`,
-						method: "POST",
-						data: { player_code: "expired" },
-						parse: "json",
-					});
-					assert.deepEqual(result.body, { error: true, message: "invalid player_code" });
+					const result = await postJSON(endpoint, { player_code: "expired" });
+					assert.deepEqual(await result.json(), { error: true, message: "invalid player_code" });
 				});
 				it("should return a verify code and token if code is valid", async function() {
 					let expiresMs = Date.now() + 1000;
 					controllerPlugin.players.set("valid", { playerCode: "valid", verifyCode: null, expiresMs });
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/player_code`,
-						method: "POST",
-						data: { player_code: "valid" },
-						parse: "json",
-					});
-					assert.equal(typeof result.body.verify_code, "string");
-					assert.equal(typeof result.body.verify_token, "string");
+					const result = await postJSON(endpoint, { player_code: "valid" });
+					const body = await result.json();
+					assert.equal(typeof body.verify_code, "string");
+					assert.equal(typeof body.verify_token, "string");
 				});
 			});
 
 			describe("/api/player_auth/verify", function() {
+				before(function() {
+					endpoint = `${controllerUrl}/api/player_auth/verify`;
+				});
 				it("should return 400 on invalid json", async function() {
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/verify`,
-						method: "POST",
-						headers: {
-							"Content-Type": "application/json",
-						},
-						data: "invalid",
-					});
-					assert.equal(result.statusCode, 400);
+					const result = await postJSON(endpoint, "invalid");
+					assert.equal(result.status, 400);
 				});
 				it("should return 400 on valid json which is not an object", async function() {
 					for (let obj of [true, false, null, "string", 123, ["an", "array"]]) {
-						let result = await phin({
-							url: `${controllerUrl}/api/player_auth/verify`,
-							method: "POST",
-							headers: {
-								"Content-Type": "application/json",
-							},
-							data: JSON.stringify(obj),
-						});
-						assert.equal(result.statusCode, 400);
+						const result = await postJSON(endpoint, obj);
+						assert.equal(result.status, 400);
 					}
 				});
 				it("should return 400 if player_code, verify_code or verify_token are not strings", async function() {
 					for (let obj of [true, false, null, 123, [], {}]) {
 						for (let field of ["player_code", "verify_code", "verify_token"]) {
-							let result = await phin({
-								url: `${controllerUrl}/api/player_auth/verify`,
-								method: "POST",
-								data: {
-									player_code: "str",
-									verify_code: "str",
-									verify_token: "str",
-									[field]: obj,
-								},
+							const result = await postJSON(endpoint, {
+								player_code: "str",
+								verify_code: "str",
+								verify_token: "str",
+								[field]: obj,
 							});
-							assert.equal(result.statusCode, 400);
+							assert.equal(result.status, 400);
 						}
 					}
 				});
 				it("should return error if verify_token is not valid", async function() {
 					async function verify(tokenParams, error) {
-						let result = await phin({
-							url: `${controllerUrl}/api/player_auth/verify`,
-							method: "POST",
-							data: {
+						const result = await postJSON(endpoint, {
+							player_code: "player",
+							verify_code: "verify",
+							verify_token: jwt.sign({
+								aud: "player_auth.verify_code",
 								player_code: "player",
 								verify_code: "verify",
-								verify_token: jwt.sign({
-									aud: "player_auth.verify_code",
-									player_code: "player",
-									verify_code: "verify",
-									...tokenParams,
-								}, Buffer.from(tokenParams.secret || "TestSecretDoNotUse", "base64")),
-							},
-							parse: "json",
+								...tokenParams,
+							}, Buffer.from(tokenParams.secret || "TestSecretDoNotUse", "base64")),
 						});
-						assert.deepEqual(result.body.error, true);
+						assert.deepEqual((await result.json()).error, true);
 					}
 					await verify({ secret: "InvalidSecret" });
 					await verify({ verify_code: "bad" });
@@ -218,21 +175,16 @@ describe("player_auth", function() {
 				it("should return invalid player_code if the code is expired", async function() {
 					let expiresMs = Date.now() - 1000;
 					controllerPlugin.players.set("expired", { playerCode: "expired", verifyCode: "verify", expiresMs });
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/verify`,
-						method: "POST",
-						data: {
+					const result = await postJSON(endpoint, {
+						player_code: "expired",
+						verify_code: "verify",
+						verify_token: jwt.sign({
+							aud: "player_auth.verify_code",
 							player_code: "expired",
 							verify_code: "verify",
-							verify_token: jwt.sign({
-								aud: "player_auth.verify_code",
-								player_code: "expired",
-								verify_code: "verify",
-							}, Buffer.from("TestSecretDoNotUse", "base64")),
-						},
-						parse: "json",
+						}, Buffer.from("TestSecretDoNotUse", "base64")),
 					});
-					assert.deepEqual(result.body, { error: true, message: "invalid player_code" });
+					assert.deepEqual(await result.json(), { error: true, message: "invalid player_code" });
 				});
 				it("should return verified false if verify code has not yet been set", async function() {
 					let expiresMs = Date.now() + 1000;
@@ -240,60 +192,46 @@ describe("player_auth", function() {
 						"unverified",
 						{ playerCode: "unverified", verifyCode: null, expiresMs },
 					);
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/verify`,
-						method: "POST",
-						data: {
+					const result = await postJSON(endpoint, {
+						player_code: "unverified",
+						verify_code: "verify",
+						verify_token: jwt.sign({
+							aud: "player_auth.verify_code",
 							player_code: "unverified",
 							verify_code: "verify",
-							verify_token: jwt.sign({
-								aud: "player_auth.verify_code",
-								player_code: "unverified",
-								verify_code: "verify",
-							}, Buffer.from("TestSecretDoNotUse", "base64")),
-						},
-						parse: "json",
+						}, Buffer.from("TestSecretDoNotUse", "base64")),
 					});
-					assert.deepEqual(result.body, { verified: false });
+					assert.deepEqual(await result.json(), { verified: false });
 				});
 				it("should return error if user is missing", async function() {
 					let expiresMs = Date.now() + 1000;
 					controllerPlugin.players.set("missing", { playerCode: "missing", verifyCode: "verify", expiresMs });
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/verify`,
-						method: "POST",
-						data: {
+					const result = await postJSON(endpoint, {
+						player_code: "missing",
+						verify_code: "verify",
+						verify_token: jwt.sign({
+							aud: "player_auth.verify_code",
 							player_code: "missing",
 							verify_code: "verify",
-							verify_token: jwt.sign({
-								aud: "player_auth.verify_code",
-								player_code: "missing",
-								verify_code: "verify",
-							}, Buffer.from("TestSecretDoNotUse", "base64")),
-						},
-						parse: "json",
+						}, Buffer.from("TestSecretDoNotUse", "base64")),
 					});
-					assert.deepEqual(result.body, { error: true, message: "invalid user" });
+					assert.deepEqual(await result.json(), { error: true, message: "invalid user" });
 				});
 				it("should return verified true with token if valid verification", async function() {
 					let expiresMs = Date.now() + 1000;
 					controllerPlugin.players.set("player", { playerCode: "player", verifyCode: "verify", expiresMs });
-					let result = await phin({
-						url: `${controllerUrl}/api/player_auth/verify`,
-						method: "POST",
-						data: {
+					const result = await postJSON(endpoint, {
+						player_code: "player",
+						verify_code: "verify",
+						verify_token: jwt.sign({
+							aud: "player_auth.verify_code",
 							player_code: "player",
 							verify_code: "verify",
-							verify_token: jwt.sign({
-								aud: "player_auth.verify_code",
-								player_code: "player",
-								verify_code: "verify",
-							}, Buffer.from("TestSecretDoNotUse", "base64")),
-						},
-						parse: "json",
+						}, Buffer.from("TestSecretDoNotUse", "base64")),
 					});
-					assert.equal(result.body.verified, true);
-					assert.deepEqual(typeof result.body.token, "string", "missing token");
+					const body = await result.json();
+					assert.equal(body.verified, true);
+					assert.deepEqual(typeof body.token, "string", "missing token");
 				});
 			});
 
@@ -338,25 +276,24 @@ describe("player_auth", function() {
 						new FetchPlayerCodeRequest("test")
 					);
 
-					let playerCodeResult = await phin({
-						url: `${controllerUrl}/api/player_auth/player_code`,
-						method: "POST",
-						data: { player_code: playerCode },
-						parse: "json",
-					});
+					const playerCodeResult = await postJSON(
+						`${controllerUrl}/api/player_auth/player_code`,
+						{ player_code: playerCode }
+					);
 
-					let { verify_code, verify_token } = playerCodeResult.body;
+					let { verify_code, verify_token } = await playerCodeResult.json();
 					await controllerPlugin.handleSetVerifyCodeRequest(
 						new SetVerifyCodeRequest("test", verify_code)
 					);
 
-					let verifyResult = await phin({
-						url: `${controllerUrl}/api/player_auth/verify`,
-						method: "POST",
-						data: { player_code: playerCode, verify_code, verify_token },
-						parse: "json",
-					});
-					assert.equal(verifyResult.body.verified, true);
+					const user = controllerPlugin.controller.userManager.getByName("test");
+					const token = controllerPlugin.controller.userManager.signUserToken(user);
+					const verifyResult = await postJSON(
+						`${controllerUrl}/api/player_auth/verify`,
+						{ player_code: playerCode, verify_code, verify_token }
+					);
+					assert.equal(verifyResult.status, 200);
+					assert.deepEqual(await verifyResult.json(), { verified: true, token: token });
 				});
 			});
 		});

plugins/research_sync/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-research_sync",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"description": "Clusterio plugin for synchronizing research between Factorio servers",
 	"author": "Hornwitser <github@hornwitser.no>",
 	"homepage": "https://github.com/clusterio/clusterio#readme",

plugins/statistics_exporter/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-statistics_exporter",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"description": "Clusterio plugin for gathering statistics from Factorio servers",
 	"author": "Hornwitser <github@hornwitser.no>",
 	"homepage": "https://github.com/clusterio/clusterio#readme",

plugins/subspace_storage/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-subspace_storage",
-	"version": "2.0.0-alpha.20",
+	"version": "2.0.0-alpha.21",
 	"description": "Clusterio plugin for sharing storage between Factorio servers",
 	"author": "Hornwitser <github@hornwitser.no>",
 	"homepage": "https://github.com/clusterio/clusterio#readme",

test/controller/Controller.js

@@ -1,7 +1,7 @@
 "use strict";
 const assert = require("assert").strict;
-const { Controller } = require("@clusterio/controller");
-const { ControllerConfig, Address, RequestError } = require("@clusterio/lib");
+const { Controller, HostInfo, InstanceInfo } = require("@clusterio/controller");
+const { ControllerConfig, Address, RequestError, InstanceConfig, SystemInfo, Role } = require("@clusterio/lib");
 const { EventEmitter } = require("stream");
 
 class MockEvent {}
@@ -28,7 +28,7 @@ describe("controller/src/Controller", function() {
 		/** @type {Controller} */
 		let controller, mockInstanceConfig;
 		before(async function() {
-			controller = new Controller({}, [], "", new ControllerConfig("controller"));
+			controller = new Controller({}, [], new ControllerConfig("controller"));
 			mockInstanceConfig = new MockInstanceConfig(new Map([
 				["instance.id", 100], ["instance.name", "test"], ["factorio.settings", []],
 			]));
@@ -189,5 +189,158 @@ describe("controller/src/Controller", function() {
 				assert(controller.instances.dirty === true, "dirty flag was not set");
 			});
 		});
+		describe(".finaliseHosts()", function() {
+			it("resets the connected state to false", function() {
+				const hostInfo = new HostInfo(1, "", "", new Map(), true);
+				Controller.finaliseHosts(hostInfo);
+				assert.equal(hostInfo.connected, false);
+				assert(hostInfo.updatedAtMs > 0, "updatedAtMs not incremented");
+			});
+			it("does not increment updatedAtMs when there are no changes", function() {
+				const hostInfo = new HostInfo(1, "", "", new Map(), false);
+				Controller.finaliseHosts(hostInfo);
+				assert.equal(hostInfo.connected, false);
+				assert(hostInfo.updatedAtMs === 0, "updatedAtMs incremented");
+			});
+		});
+		describe(".finaliseInstances()", function() {
+			it("resets the stats for assigned instances", function() {
+				const instanceConfig = new InstanceConfig("controller", { "instance.assigned_host": 1 });
+				const instanceInfo = new InstanceInfo(instanceConfig, "running");
+				Controller.finaliseInstances(instanceInfo);
+				assert.equal(instanceInfo.status, "unknown");
+				assert(instanceInfo.updatedAtMs > 0, "updatedAtMs not incremented");
+			});
+			it("resets the status for unassigned instances", function() {
+				const instanceConfig = new InstanceConfig("controller", { "instance.assigned_host": null });
+				const instanceInfo = new InstanceInfo(instanceConfig, "running");
+				Controller.finaliseInstances(instanceInfo);
+				assert.equal(instanceInfo.status, "unassigned");
+				assert(instanceInfo.updatedAtMs > 0, "updatedAtMs not incremented");
+			});
+			it("does not update updatedAtMs when there are no changes (assigned)", function() {
+				const instanceConfig = new InstanceConfig("controller", { "instance.assigned_host": 1 });
+				const instanceInfo = new InstanceInfo(instanceConfig, "unknown");
+				Controller.finaliseInstances(instanceInfo);
+				assert.equal(instanceInfo.status, "unknown");
+				assert(instanceInfo.updatedAtMs === 0, "updatedAtMs incremented");
+			});
+			it("does not update updatedAtMs when there are no changes (unassigned)", function() {
+				const instanceConfig = new InstanceConfig("controller", { "instance.assigned_host": null });
+				const instanceInfo = new InstanceInfo(instanceConfig, "unassigned");
+				Controller.finaliseInstances(instanceInfo);
+				assert.equal(instanceInfo.status, "unassigned");
+				assert(instanceInfo.updatedAtMs === 0, "updatedAtMs incremented");
+			});
+		});
+	});
+	describe("migrations", function() {
+		describe("SystemInfo", function() {
+			it("migrates 'canRestart' from undefined to false", function() { // Alpha 17
+				const result = Controller.migrateSystems([
+					{}, { canRestart: true }, { canRestart: false },
+				]);
+				assert.equal(result[0].canRestart, false);
+				assert.equal(result[1].canRestart, true);
+				assert.equal(result[2].canRestart, false);
+			});
+			it("migrates 'restartRequired' from undefined to false", function() { // Alpha 21
+				const result = Controller.migrateSystems([
+					{}, { restartRequired: true }, { restartRequired: false },
+				]);
+				assert.equal(result[0].restartRequired, false);
+				assert.equal(result[1].restartRequired, true);
+				assert.equal(result[2].restartRequired, false);
+			});
+			it("does nothing for upto date data", function() {
+				const systemInfo = new SystemInfo(
+					1, "name", "n", "k", "m", "cm", "cr", "mc", "ma", "dc", "da", false, false, 0, false
+				);
+				const jsonString = JSON.stringify(systemInfo);
+				const systemInfoJson = JSON.parse(jsonString);
+				const systemInfoJsonCopy = JSON.parse(jsonString);
+				const result = Controller.migrateSystems([
+					systemInfoJson, systemInfoJson, systemInfoJson,
+				]);
+				assert.deepEqual(result, [
+					systemInfoJsonCopy, systemInfoJsonCopy, systemInfoJsonCopy,
+				]);
+			});
+		});
+		describe("HostInfo", function() {
+			it("migrates to the new format", function() { // Alpha 19
+				const result = Controller.migrateHosts([
+					["", "foo"], ["", "bar"], ["", "baz"],
+				]);
+				assert.deepEqual(result, [
+					"foo", "bar", "baz",
+				]);
+			});
+			it("does nothing for upto date data", function() {
+				const hostInfo = new HostInfo(1, "", "", new Map());
+				const jsonString = JSON.stringify(hostInfo);
+				const hostInfoJson = JSON.parse(jsonString);
+				const hostInfoJsonCopy = JSON.parse(jsonString);
+				const result = Controller.migrateHosts([
+					hostInfoJson, hostInfoJson, hostInfoJson,
+				]);
+				assert.deepEqual(result, [
+					hostInfoJsonCopy, hostInfoJsonCopy, hostInfoJsonCopy,
+				]);
+			});
+		});
+		describe("InstanceInfo", function() {
+			it("migrates to the new format", function() { // Alpha 14
+				const result = Controller.migrateInstances([
+					{ name: "foo" }, { name: "bar" }, { name: "baz" },
+				]);
+				assert.deepEqual(result, [
+					{ config: { name: "foo" }, status: "running" },
+					{ config: { name: "bar" }, status: "running" },
+					{ config: { name: "baz" }, status: "running" },
+				]);
+			});
+			it("does nothing for upto date data", function() {
+				const instanceConfig = new InstanceConfig("controller");
+				const instanceInfo = new InstanceInfo(instanceConfig, "unknown");
+				const jsonString = JSON.stringify(instanceInfo);
+				const instanceInfoJson = JSON.parse(jsonString);
+				const instanceInfoJsonCopy = JSON.parse(jsonString);
+				const result = Controller.migrateInstances([
+					instanceInfoJson, instanceInfoJson, instanceInfoJson,
+				]);
+				assert.deepEqual(result, [
+					instanceInfoJsonCopy, instanceInfoJsonCopy, instanceInfoJsonCopy,
+				]);
+			});
+		});
+		describe("Role", function() {
+			it("migrates renamed permissions", function() { // Alpha 17
+				const result = Controller.migrateRoles([
+					{ permissions: ["foo", "bar", "core.instance.save.list.subscribe"] },
+					{ permissions: ["bar", "core.instance.save.list.subscribe"] },
+					{ permissions: ["core.instance.save.subscribe"] },
+				]);
+				assert.deepEqual(result, [
+					{ permissions: ["foo", "bar", "core.instance.save.subscribe"] },
+					{ permissions: ["bar", "core.instance.save.subscribe"] },
+					{ permissions: ["core.instance.save.subscribe"] },
+				]);
+			});
+			it("does nothing for upto date data", function() {
+				const role = new Role(
+					0, "n", "d", new Set("p1", "p2"), 0, false
+				);
+				const jsonString = JSON.stringify(role);
+				const roleJson = JSON.parse(jsonString);
+				const roleJsonCopy = JSON.parse(jsonString);
+				const result = Controller.migrateRoles([
+					roleJson, roleJson, roleJson,
+				]);
+				assert.deepEqual(result, [
+					roleJsonCopy, roleJsonCopy, roleJsonCopy,
+				]);
+			});
+		});
 	});
 });

test/controller/ControllerUser.js

@@ -5,20 +5,17 @@ const lib = require("@clusterio/lib");
 
 describe("controller/src/ControllerUser", function() {
 	describe("class ControllerUser", function() {
-		class MockUserManager {
-			roles = new Map([
-				[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
-				[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
-			]);
-		}
-		const userManager = new MockUserManager();
+		const roles = new Map([
+			[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
+			[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
+		]);
 
 		it("should round trip serialize", function() {
 			function test_roundtrip(serialized) {
-				let user = ControllerUser.fromJSON(serialized, userManager);
+				let user = ControllerUser.fromJSON(serialized, roles);
 				let user_serialized = user.toJSON(true);
 				assert.deepEqual(user_serialized, serialized);
-				let user_deserialized = ControllerUser.fromJSON(user_serialized, userManager);
+				let user_deserialized = ControllerUser.fromJSON(user_serialized, roles);
 				assert.deepEqual(user_deserialized, user);
 			}
 
@@ -31,9 +28,9 @@ describe("controller/src/ControllerUser", function() {
 		describe(".checkPermission()", function() {
 			it("should correctly resolve permissions", function() {
 				lib.definePermission({ name: "user-test", title: "Test", description: "User Test" });
-				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, userManager);
-				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, userManager);
-				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, userManager);
+				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, roles);
+				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, roles);
+				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, roles);
 
 				a.checkPermission("core.control.connect");
 				assert.throws(() => b.checkPermission("core.control.connect"), new Error("Permission denied"));

test/controller/WsServer.js

@@ -10,7 +10,7 @@ describe("controller/src/WsServer", function() {
 		before(function() {
 			let config = new ControllerConfig("controller");
 			config.set("controller.trusted_proxies", "127.0.0.0/8, ::1");
-			let controller = new Controller({}, [], "", config);
+			let controller = new Controller({}, [], config);
 			wsServer = controller.wsServer;
 		});
 		describe(".remoteAddr()", function() {

test/controller/routes.js

@@ -4,13 +4,11 @@ const events = require("events");
 const FormData = require("form-data");
 const jwt = require("jsonwebtoken");
 const http = require("http");
-const phin = require("phin");
 
 const { wait } = require("@clusterio/lib");
 const routes = require("@clusterio/controller/dist/node/src/routes");
 const mock = require("../mock");
 
-
 describe("controller/src/routes", function() {
 	let controller;
 	let server;
@@ -27,20 +25,23 @@ describe("controller/src/routes", function() {
 		server.close();
 	});
 	describe("/api/stream/:id", function() {
+		let endpoint;
+		beforeEach(function() {
+			endpoint = `http://localhost:${port}/api/stream`;
+		});
 		it("should respond with 404 if stream doesn't exist", async function() {
 			let response;
-			response = await phin(`http://localhost:${port}/api/stream/01020304`);
-			assert.equal(response.statusCode, 404);
-			response = await phin({
-				url: `http://localhost:${port}/api/stream/01020304`, method: "PUT",
-				data: "test stream",
+			response = await fetch(`${endpoint}/01020304`);
+			assert.equal(response.status, 404);
+			response = await fetch(`${endpoint}/01020304`, {
+				method: "PUT", body: "test stream",
 			});
-			assert.equal(response.statusCode, 404);
+			assert.equal(response.status, 404);
 		});
 		it("should respond with 500 if stream times out", async function() {
 			let stream = await routes.createProxyStream(controller.app);
-			let response = await phin(`http://localhost:${port}/api/stream/${stream.id}`);
-			assert.equal(response.statusCode, 500);
+			let response = await fetch(`${endpoint}/${stream.id}`);
+			assert.equal(response.status, 500);
 		});
 		it("should passthrough a stream", async function() {
 			let stream;
@@ -49,121 +50,123 @@ describe("controller/src/routes", function() {
 			responses = await Promise.all([
 				(async function() {
 					await wait(100);
-					return await phin({
-						url: `http://localhost:${port}/api/stream/${stream.id}`, method: "PUT",
-						data: "test content",
+					return await fetch(`${endpoint}/${stream.id}`, {
+						method: "PUT", body: "test content",
 					});
 				}()),
-				phin(`http://localhost:${port}/api/stream/${stream.id}`),
+				fetch(`${endpoint}/${stream.id}`),
 			]);
-			assert.equal(responses[0].statusCode, 200);
-			assert.equal(responses[1].statusCode, 200);
-			assert.equal(responses[1].body.toString(), "test content");
+			assert.equal(responses[0].status, 200);
+			assert.equal(responses[1].status, 200);
+			assert.equal(await responses[1].text(), "test content");
 			stream = await routes.createProxyStream(controller.app);
 			responses = await Promise.all([
-				phin({
-					url: `http://localhost:${port}/api/stream/${stream.id}`, method: "PUT",
-					data: "test content",
+				fetch(`${endpoint}/${stream.id}`, {
+					method: "PUT", body: "test content",
 				}),
 				(async function() {
 					await wait(100);
-					return await phin(`http://localhost:${port}/api/stream/${stream.id}`);
+					return await fetch(`${endpoint}/${stream.id}`);
 				}()),
 			]);
-			assert.equal(responses[0].statusCode, 200);
-			assert.equal(responses[1].statusCode, 200);
-			assert.equal(responses[1].body.toString(), "test content");
+			assert.equal(responses[0].status, 200);
+			assert.equal(responses[1].status, 200);
+			assert.equal(await responses[1].text(), "test content");
 		});
 	});
 	describe("/api/upload-save", function() {
+		let endpoint;
+		beforeEach(function() {
+			endpoint = `http://localhost:${port}/api/upload-save`;
+		});
 		it("should respond with 401 with invalid token", async function() {
 			let response;
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: { "Content-Type": "application/zip" },
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 401);
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			assert.equal(response.status, 401);
+			response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: {
 					"Content-Type": "application/zip",
 					"X-Access-Token": "invalid",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 401);
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			assert.equal(response.status, 401);
+			response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: {
 					"Content-Type": "application/zip",
 					"X-Access-Token": jwt.sign(
 						{ aud: "user", user: "invalid" }, Buffer.from("TestSecretDoNotUse", "base64")
 					),
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 401);
+			assert.equal(response.status, 401);
 			controller.userManager.getByName("test").tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000);
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: {
 					"Content-Type": "application/zip",
 					"X-Access-Token": jwt.sign(
 						{ aud: "user", user: "test" }, Buffer.from("TestSecretDoNotUse", "base64")
 					),
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 401);
+			assert.equal(response.status, 401);
 		});
 
 		it("should respond with 403 when user has insufficient permission", async function() {
-			let response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			let response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: {
 					"Content-Type": "application/zip",
 					"X-Access-Token": jwt.sign(
 						{ aud: "user", user: "player" }, Buffer.from("TestSecretDoNotUse", "base64")
 					),
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 403);
+			assert.equal(response.status, 403);
 		});
 
 		let testToken = jwt.sign({ aud: "user", user: "test" }, Buffer.from("TestSecretDoNotUse", "base64"));
 		it("should respond with 415 if content type is missing or invalid", async function() {
 			let response;
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 415);
-			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			assert.equal(response.status, 415);
+			assert.deepEqual(await response.json(), { request_errors: ["invalid Content-Type"] });
+			response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "invalid",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 415);
-			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+			assert.equal(response.status, 415);
+			assert.deepEqual(await response.json(), { request_errors: ["invalid Content-Type"] });
+			response = await fetch(`${endpoint}?instance_id=123&filename=file.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "text/plain",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 415);
-			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
+			assert.equal(response.status, 415);
+			assert.deepEqual(await response.json(), { request_errors: ["invalid Content-Type"] });
 		});
 
 		it("should respond with 400 if invalid multipart request", async function() {
@@ -172,96 +175,96 @@ describe("controller/src/routes", function() {
 			form = new FormData();
 			form.append("instance_id", "123");
 			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "text/plain" });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+			response = await fetch(endpoint, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					...form.getHeaders(),
 				},
-				data: form.getBuffer(),
+				body: form.getBuffer(),
 			});
-			assert.equal(response.statusCode, 400);
-			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid file Content-Type"] });
+			assert.equal(response.status, 400);
+			assert.deepEqual(await response.json(), { request_errors: ["invalid file Content-Type"] });
 			form = new FormData();
 			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
 			form.append("instance_id", "123");
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+			response = await fetch(endpoint, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					...form.getHeaders(),
 				},
-				data: form.getBuffer(),
+				body: form.getBuffer(),
 			});
-			assert.equal(response.statusCode, 400);
+			assert.equal(response.status, 400);
 			assert.deepEqual(
-				JSON.parse(response.body), { request_errors: ["instance_id must come before files uploaded"] }
+				await response.json(), { request_errors: ["instance_id must come before files uploaded"] }
 			);
 			form = new FormData();
 			form.append("instance_id", "123");
 			form.append("file", "totally a zip file", { filename: "test.txt", contentType: "application/zip" });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+			response = await fetch(endpoint, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					...form.getHeaders(),
 				},
-				data: form.getBuffer(),
+				body: form.getBuffer(),
 			});
-			assert.equal(response.statusCode, 400);
+			assert.equal(response.status, 400);
 			assert.deepEqual(
-				JSON.parse(response.body), { request_errors: ["filename must end with .zip"] }
+				await response.json(), { request_errors: ["filename must end with .zip"] }
 			);
 			form = new FormData();
 			form.append("instance_id", "invalid");
 			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+			response = await fetch(endpoint, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					...form.getHeaders(),
 				},
-				data: form.getBuffer(),
+				body: form.getBuffer(),
 			});
-			assert.equal(response.statusCode, 400);
+			assert.equal(response.status, 400);
 			assert.deepEqual(
-				JSON.parse(response.body), { request_errors: ["invalid instance_id"] }
+				await response.json(), { request_errors: ["invalid instance_id"] }
 			);
 		});
 
 		it("should respond with 400 if invalid file request", async function() {
 			let response;
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123`, method: "POST",
+			response = await fetch(`${endpoint}?instance_id=123`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "application/zip",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 400);
-			assert.deepEqual(JSON.parse(response.body), { request_errors: ["Missing or invalid filename parameter"] });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.txt`, method: "POST",
+			assert.equal(response.status, 400);
+			assert.deepEqual(await response.json(), { request_errors: ["Missing or invalid filename parameter"] });
+			response = await fetch(`${endpoint}?instance_id=123&filename=test.txt`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "application/zip",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 400);
-			assert.deepEqual(JSON.parse(response.body), { request_errors: ["filename must end with .zip"] });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?filename=test.zip`, method: "POST",
+			assert.equal(response.status, 400);
+			assert.deepEqual(await response.json(), { request_errors: ["filename must end with .zip"] });
+			response = await fetch(`${endpoint}?filename=test.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "application/zip",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 400);
+			assert.equal(response.status, 400);
 			assert.deepEqual(
-				JSON.parse(response.body), { request_errors: ["Missing or invalid instance_id parameter"] }
+				await response.json(), { request_errors: ["Missing or invalid instance_id parameter"] }
 			);
 		});
 
@@ -274,82 +277,78 @@ describe("controller/src/routes", function() {
 			form = new FormData();
 			form.append("instance_id", "123");
 			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+			response = await fetch(endpoint, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					...form.getHeaders(),
 				},
-				data: form.getBuffer(),
+				body: form.getBuffer(),
 			});
-			assert.equal(response.statusCode, 500);
-			assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] });
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+			assert.equal(response.status, 500);
+			assert.deepEqual(await response.json(), { errors: ["Something went wrong"], request_errors: [] });
+			response = await fetch(`${endpoint}?instance_id=123&filename=test.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "application/zip",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 500);
-			assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] });
+			assert.equal(response.status, 500);
+			assert.deepEqual(await response.json(), { errors: ["Something went wrong"], request_errors: [] });
 			controller.sendToHostByInstanceId = async (request) => {
 				await new Promise(() => {});
 			};
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+			response = await fetch(`${endpoint}?instance_id=123&filename=test.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "application/zip",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.equal(response.statusCode, 500);
+			assert.equal(response.status, 500);
 			assert.deepEqual(
-				JSON.parse(response.body), { errors: ["Timed out establishing stream to host"], request_errors: [] }
+				await response.json(), { errors: ["Timed out establishing stream to host"], request_errors: [] }
 			);
 		});
 
 		it("should complete a valid transfer", async function() {
 			controller.sendToHostByInstanceId = async (request) => {
-				let stream = await phin({
-					url: `http://localhost:${port}/api/stream/${request.streamId}`,
-				});
-				assert.equal(stream.body.toString(), "totally a zip file");
+				const stream = await fetch(`http://localhost:${port}/api/stream/${request.streamId}`);
+				assert.equal(await stream.text(), "totally a zip file");
 				return request.name;
 			};
 			let response;
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+			response = await fetch(`${endpoint}?instance_id=123&filename=test.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "application/zip",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.deepEqual(JSON.parse(response.body), { saves: ["test.zip"] });
-			assert.equal(response.statusCode, 200);
+			assert.deepEqual(await response.json(), { saves: ["test.zip"] });
+			assert.equal(response.status, 200);
 		});
 		it("should complete a valid transfer with non-standerd mime type", async function() {
 			controller.sendToHostByInstanceId = async (request) => {
-				let stream = await phin({
-					url: `http://localhost:${port}/api/stream/${request.streamId}`,
-				});
-				assert.equal(stream.body.toString(), "totally a zip file");
+				const stream = await fetch(`http://localhost:${port}/api/stream/${request.streamId}`);
+				assert.equal(await stream.text(), "totally a zip file");
 				return request.name;
 			};
 			let response;
-			response = await phin({
-				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+			response = await fetch(`${endpoint}?instance_id=123&filename=test.zip`, {
+				method: "POST",
 				headers: {
 					"X-Access-Token": testToken,
 					"Content-Type": "application/x-zip-compressed",
 				},
-				data: "totally a zip file",
+				body: "totally a zip file",
 			});
-			assert.deepEqual(JSON.parse(response.body), { saves: ["test.zip"] });
-			assert.equal(response.statusCode, 200);
+			assert.deepEqual(await response.json(), { saves: ["test.zip"] });
+			assert.equal(response.status, 200);
 		});
 	});
 });

test/host.js

@@ -10,13 +10,15 @@ const { _discoverInstances } = require("@clusterio/host/dist/node/src/Host");
 describe("Host testing", function() {
 	describe("discoverInstances()", function() {
 		it("should discover test instance", async function() {
-			let instancePath = path.join("test", "file", "instances");
-			let instanceInfos = await _discoverInstances(instancePath);
+			const instancePath = path.join("test", "file", "instances");
+			const instanceInfos = await _discoverInstances(instancePath);
 
-			let referenceConfig = new lib.InstanceConfig("host");
-			referenceConfig.set("instance.id", 1);
-			referenceConfig.set("instance.name", "test");
-			referenceConfig.dirty = false;
+			const configPath = path.join(instancePath, "test", "instance.json");
+			const referenceConfig = new lib.InstanceConfig("host", {
+				...Host.instanceConfigWarning,
+				"instance.id": 1,
+				"instance.name": "test",
+			}, configPath);
 
 			assert.deepEqual(instanceInfos, new Map([
 				[1, {

test/host/Instance.js

@@ -200,7 +200,7 @@ describe("class Instance", function() {
 		beforeEach(function() {
 			instance.config.set("factorio.sync_banlist", "bidirectional");
 			instance.config.set("factorio.sync_adminlist", "bidirectional");
-			// instance.config.set("factorio.sync_whitelist", "bidirectional"); // Not Implemented
+			instance.config.set("factorio.sync_whitelist", "bidirectional");
 		});
 		it("should send InstanceBanlistUpdateEvent for bans", function() {
 			instance._recordUserUpdate("BAN", "player", "reason");
@@ -257,7 +257,6 @@ describe("class Instance", function() {
 			);
 		});
 		it("should send InstanceWhitelistUpdateEvent for whitelist add", function() {
-			this.skip(); // Bidirectional not implemented
 			instance._recordUserUpdate("WHITELISTED", "player");
 			assert.deepEqual(
 				connector.sentMessages[0],
@@ -271,7 +270,6 @@ describe("class Instance", function() {
 			);
 		});
 		it("should send InstanceWhitelistUpdateEvent for whitelist remove", function() {
-			this.skip(); // Bidirectional not implemented
 			instance._recordUserUpdate("UNWHITELISTED", "player");
 			assert.deepEqual(
 				connector.sentMessages[0],

test/integration/clusterio.js

@@ -13,7 +13,8 @@ const { wait } = lib;
 const testStrings = require("../lib/factorio/test_strings");
 const {
 	TestControl, TestControlConnector, url, controlToken, slowTest,
-	exec, execCtl, sendRcon, getControl, spawnNode, instancesDir, factorioDir,
+	exec, execCtl, execCtlProcess, execHost, sendRcon, getControl,
+	spawnNode, instancesDir, factorioDir,
 } = require("./index");
 
 
@@ -39,11 +40,11 @@ async function startAltHost() {
 	await fs.remove(path.join("temp", "test", "alt-instances"));
 	await fs.remove(path.join("temp", "test", "alt-mods"));
 	await execCtl(`host create-config --id 5 --name alt-host --generate-token --output ${config}`);
-	await exec(`node ../../packages/host --config ${config} config set host.tls_ca ../../test/file/tls/cert.pem`);
-	await exec(`node ../../packages/host --config ${config} config set host.instances_directory alt-instances`);
+	await execHost(`--config ${config} config set host.tls_ca ../../test/file/tls/cert.pem`);
+	await execHost(`--config ${config} config set host.instances_directory alt-instances`);
 	const dir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir);
-	await exec(`node ../../packages/host --config ${config} config set host.factorio_directory ${dir}`);
-	await exec(`node ../../packages/host --config ${config} config set host.mods_directory alt-mods`);
+	await execHost(`--config ${config} config set host.factorio_directory ${dir}`);
+	await execHost(`--config ${config} config set host.mods_directory alt-mods`);
 	return await spawnAltHost(config);
 }
 
@@ -256,11 +257,12 @@ describe("Integration of Clusterio", function() {
 				assert.equal(Object.prototype.hasOwnProperty.call(result, "controller.auth_secret"), false);
 			});
 		});
+
 		describe("controller config set", function() {
 			it("sets given config option", async function() {
-				await execCtl('controller config set controller.name "Test Cluster"');
+				await execCtl("controller config set controller.name Test-Cluster");
 				let result = await getControl().send(new lib.ControllerConfigGetRequest());
-				assert.equal(result["controller.name"], "Test Cluster");
+				assert.equal(result["controller.name"], "Test-Cluster");
 			});
 			it("should not allow setting auth_secret", async function() {
 				await assert.rejects(execCtl("controller config set controller.auth_secret root"));
@@ -273,15 +275,62 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("controller plugin update", function() {
+			it("runs", async function() {
+				// In dev plugins have no npm package, so best we can do is get an error from the controller
+				await assert.rejects(
+					execCtl("controller plugin update foo"),
+					/Plugin foo is not installed on this machine/
+				);
+			});
+			// Update always fails, we can not test restart option
+		});
+
+		describe("controller plugin install", function() {
+			it("runs", async function() {
+				// Default is to disallow updates, changing this value would require a restart
+				// Additionally, it can not be changed via ctl, so best we can do is get an error from the controller
+				await assert.rejects(
+					execCtl("controller plugin install foo"),
+					/Plugin installs are disabled on this machine/
+				);
+			});
+			// Install always fails, we can not test restart option
+		});
+
+		describe("controller update", function() {
+			it("runs", async function() {
+				await execCtl("controller update");
+			});
+			it("accepts --restart", async function() {
+				// We cannot restart the controller, so we check for controller error instead
+				await assert.rejects(
+					execCtl("controller update --restart"),
+					/Cannot restart, controller does not have a process monitor to restart it./
+				);
+			});
+		});
+
+		describe("controller restart", function() {
+			it("runs", async function() {
+				// We cannot restart the controller, so we check for controller error instead
+				await assert.rejects(
+					execCtl("controller restart"),
+					/Cannot restart, controller does not have a process monitor to restart it./
+				);
+			});
+		});
+
 		describe("host list", function() {
 			it("runs", async function() {
 				await execCtl("host list");
 			});
 		});
+
 		describe("host config", function() {
 			it("changes host config", async function() {
 				await execCtl("host config set 4 host.name My-Host");
-				const result = await execCtl("host config list 4");
+				const result = await execCtlProcess("host config list 4");
 				assert(/My-Host/.test(result.stdout), "New host name not in config output");
 			});
 			it("should not allow setting host.controller_token", async function() {
@@ -301,6 +350,59 @@ describe("Integration of Clusterio", function() {
 				assert.equal(Object.prototype.hasOwnProperty.call(result, "host.controller_token"), false);
 			});
 		});
+
+		describe("host plugin list", function() {
+			it("runs", async function() {
+				await execCtl("host plugin list 4");
+			});
+		});
+
+		describe("host plugin update", function() {
+			it("runs", async function() {
+				// In dev plugins have no npm package, so best we can do is get an error from the host
+				await assert.rejects(
+					execCtl("host plugin update 4 foo"),
+					/Plugin foo is not installed on this machine/
+				);
+			});
+			// Update always fails, we can not test restart option
+		});
+
+		describe("host plugin install", function() {
+			it("runs", async function() {
+				// Default is to disallow updates, changing this value would require a restart
+				// Additionally, it can not be changed via ctl, so best we can do is get an error from the host
+				await assert.rejects(
+					execCtl("host plugin install 4 foo"),
+					/Plugin installs are disabled on this machine/
+				);
+			});
+			// Install always fails, we can not test restart option
+		});
+
+		describe("host update", function() {
+			it("runs", async function() {
+				await execCtl("host update 4");
+			});
+			it("accepts --restart", async function() {
+				// We cannot restart the host, so we check for host error instead
+				await assert.rejects(
+					execCtl("host update 4 --restart"),
+					/Cannot restart, host does not have a process monitor to restart it./
+				);
+			});
+		});
+
+		describe("host restart", function() {
+			it("runs", async function() {
+				// We cannot restart the host, so we check for host error instead
+				await assert.rejects(
+					execCtl("host restart 4"),
+					/Cannot restart, host does not have a process monitor to restart it./
+				);
+			});
+		});
+
 		describe("host generate-token", function() {
 			it("runs", async function() {
 				await execCtl("host generate-token --id 42");
@@ -309,14 +411,18 @@ describe("Integration of Clusterio", function() {
 				await execCtl("host generate-token");
 			});
 		});
+
 		describe("host revoke-token", async function() {
 			it("should disconnect existing host", async function() {
 				slowTest(this);
 				let sawDisconnected;
 				await runWithAltHost(async () => {
-					getControl().hostUpdates = [];
+					const control = getControl();
+					control.hostUpdates = [];
+					await wait(50); // Allow for host to update instances
 					await execCtl("host revoke-token 5");
-					for (let update of getControl().hostUpdates) {
+					await wait(50); // Allow controller to notify control
+					for (let update of control.hostUpdates) {
 						if (update.name !== "alt-host") {
 							continue;
 						}
@@ -330,6 +436,7 @@ describe("Integration of Clusterio", function() {
 				assert(sawDisconnected, "No host update with status disconnected was sent after revoking token");
 			});
 		});
+
 		describe("instance list", function() {
 			it("runs", async function() {
 				await execCtl("instance list");
@@ -345,12 +452,27 @@ describe("Integration of Clusterio", function() {
 				assert.equal(instances.get(44).status, "unassigned", "incorrect instance status");
 
 				// Make sure the following tests does not fail due to not having internet
-				let value = JSON.stringify({ lan: true, public: false }).replace(
-					/"/g, process.platform === "win32" ? '""' : '\\"'
-				);
-				await execCtl(`instance config set-prop test factorio.settings visibility "${value}"`);
+				const value = jsonArg({ lan: true, public: false });
+				await execCtlProcess(`instance config set-prop test factorio.settings visibility ${value}`);
 				await execCtl("instance config set-prop test factorio.settings require_user_verification false");
 			});
+			it("can clone an instance", async function() {
+				slowTest(this);
+				await execCtl("instance create testClone --id 440 --from 44");
+				const instances = await getInstances();
+				assert(instances.has(440), "instance was not created");
+				assert.equal(instances.get(440).status, "unassigned", "incorrect instance status");
+				const config = await getControl().send(new lib.InstanceConfigGetRequest(44));
+				assert.deepEqual(config["factorio.settings"]["visibility"], { lan: true, public: false });
+				assert.equal(config["factorio.settings"]["require_user_verification"], false);
+			});
+			it("errors when cloning an invalid instance", async function() {
+				slowTest(this);
+				await assert.rejects(
+					execCtl("instance create testClone --id 440 --from 441"),
+					new lib.RequestError("Instance with ID 441 does not exist")
+				);
+			});
 		});
 
 		describe("instance assign", function() {
@@ -380,7 +502,7 @@ describe("Integration of Clusterio", function() {
 		describe("instance save list", function() {
 			it("lists the created save", async function() {
 				slowTest(this);
-				let result = await execCtl("instance save list test");
+				let result = await execCtlProcess("instance save list test");
 				assert(/world\.zip/.test(result.stdout), "world.zip not present in list save output");
 			});
 		});
@@ -434,6 +556,18 @@ describe("Integration of Clusterio", function() {
 				await execCtl("instance start test --save world.zip");
 				await checkInstanceStatus(44, "running");
 			});
+			it("allows having a separate console log", async function() {
+				slowTest(this);
+				await execCtl("instance stop 44");
+				await execCtl("instance config set 44 factorio.console_logging true");
+				await execCtl("instance start test");
+				await checkInstanceStatus(44, "running");
+				const checkMessage = `check${Date.now()}`;
+				await sendRcon(44, checkMessage);
+				const consoleLog = await fs.readFile(path.join("temp", "test", "instances", "test", "console.log"));
+				assert(consoleLog.includes(checkMessage));
+				await execCtl("instance config set 44 factorio.console_logging false");
+			});
 			it("copies the save if an autosave is the target", async function() {
 				slowTest(this);
 				await execCtl("instance stop 44");
@@ -483,7 +617,7 @@ describe("Integration of Clusterio", function() {
 				assert(r1.log.some(info => /\[COMMAND\]/.test(info.message)), "Command was not sent");
 				await execCtl("instance config set test factorio.enable_script_commands false");
 
-				assert.rejects(
+				await assert.rejects(
 					execCtl("instance send-rcon test /c"),
 					new Error(
 						"Attempted to use script command while disabled. " +
@@ -566,8 +700,8 @@ describe("Integration of Clusterio", function() {
 				];
 
 				for (let [prop, value] of testConfigs) {
-					let args = `test factorio.settings ${prop} ${jsonArg(value)}`;
-					await execCtl(`instance config set-prop ${args}`);
+					const args = `test factorio.settings ${prop} ${jsonArg(value)}`;
+					await execCtlProcess(`instance config set-prop ${args}`);
 				}
 
 				// Do this afterwards to leave enough to time for the
@@ -575,9 +709,8 @@ describe("Integration of Clusterio", function() {
 				for (let [, , configName, expectedResult] of testConfigs) {
 					assert.equal(await sendRcon(44, `/config get ${configName}`), `${expectedResult}\n`);
 				}
-			});
-			it("should not change the instance status", async function() {
-				slowTest(this);
+
+				// should not change instance status
 				await checkInstanceStatus(44, "running");
 			});
 			it("should allow creating and removing props", async function() {
@@ -778,18 +911,30 @@ describe("Integration of Clusterio", function() {
 				await execCtl("instance config set 44 research_sync.load_plugin false");
 				await execCtl("instance config set 44 statistics_exporter.load_plugin false");
 				await execCtl("instance config set 44 subspace_storage.load_plugin false");
+				await execCtl("instance config set 44 inventory_sync.load_plugin false");
+
+				await execCtl("instance start 44");
+				const instance = (await getInstances()).get(44);
+				const isV2 = lib.integerFactorioVersion(instance.factorioVersion) > lib.integerFactorioVersion("2.0.0");
+				const exchangeString = (isV2 ? testStrings.modified_v2 : testStrings.modified).replace(/[\n\r]+/g, "");
+				const args = `base/freeplay --seed 1234 --map-exchange-string "${exchangeString}"`;
+				await execCtl("instance stop 44");
 
-				let exchangeString = testStrings.modified.replace(/[\n\r]+/g, "");
-				let args = `base/freeplay --seed 1234 --map-exchange-string "${exchangeString}"`;
 				await execCtl(`instance load-scenario test ${args}`);
 				await checkInstanceStatus(44, "running");
 				await sendRcon(44, '/c game.print("disable achievements")');
 				await sendRcon(44, '/c game.print("disable achievements")');
 				assert.equal(await sendRcon(44, "/c rcon.print(game.default_map_gen_settings.seed)"), "1234\n");
 				assert.equal(await sendRcon(44, "/c rcon.print(game.map_settings.pollution.ageing)"), "1.5\n");
-				assert.equal(
-					await sendRcon(44, "/c rcon.print(game.difficulty_settings.research_queue_setting)"), "never\n"
-				);
+				if (isV2) {
+					assert.equal(
+						await sendRcon(44, "/c rcon.print(game.difficulty_settings.spoil_time_modifier)"), "1\n"
+					);
+				} else {
+					assert.equal(
+						await sendRcon(44, "/c rcon.print(game.difficulty_settings.research_queue_setting)"), "never\n"
+					);
+				}
 			});
 		});
 
@@ -1046,7 +1191,7 @@ describe("Integration of Clusterio", function() {
 			});
 			it("should allow setting all fields", async function() {
 				const color = { r: 1, g: 0, b: 1, a: 0 };
-				await execCtl(
+				await execCtlProcess(
 					"mod-pack create full-pack 0.17.59 " +
 					"--description Description " +
 					"--mods empty_mod:1.0.0 " +
@@ -1078,7 +1223,7 @@ describe("Integration of Clusterio", function() {
 		describe("mod-pack list", function() {
 			let result = null;
 			it("runs", async function() {
-				result = await execCtl("mod-pack list");
+				result = await execCtlProcess("mod-pack list");
 			});
 			it("contains defaults", function() {
 				assert(result !== null, "Failed to return a value");
@@ -1107,7 +1252,7 @@ describe("Integration of Clusterio", function() {
 				reference.settings["runtime-global"].set("MyDouble", { value: 12.25 });
 				reference.settings["runtime-per-user"].set("MyString", { value: "a-string" });
 				await execCtl(`mod-pack import ${reference.toModPackString()}`);
-				const result = await execCtl("mod-pack export imported-pack");
+				const result = await execCtlProcess("mod-pack export imported-pack");
 				const roundtrip = lib.ModPack.fromModPackString(result.stdout.trim());
 				roundtrip.id = reference.id;
 				assert.deepEqual(roundtrip, reference);
@@ -1166,7 +1311,7 @@ describe("Integration of Clusterio", function() {
 
 		describe("mod show", function() {
 			it("gives details of a mod", async function() {
-				let result = await execCtl("mod show empty_mod 1.0.0");
+				let result = await execCtlProcess("mod show empty_mod 1.0.0");
 				let hash = await lib.hashFile(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip"));
 				let stat = await fs.stat(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip"));
 				assert.equal(
@@ -1191,14 +1336,14 @@ describe("Integration of Clusterio", function() {
 
 		describe("mod list", function() {
 			it("shows the list of mods", async function() {
-				let result = await execCtl("mod list");
+				let result = await execCtlProcess("mod list");
 				assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the list");
 			});
 		});
 
 		describe("mod search", function() {
 			it("searches the list of mods", async function() {
-				let result = await execCtl("mod search 1.1 name:empty_mod");
+				let result = await execCtlProcess("mod search 1.1 name:empty_mod");
 				assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the result");
 			});
 		});
@@ -1262,6 +1407,7 @@ describe("Integration of Clusterio", function() {
 				await execCtl(`role create temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let tempRole = roles.find(role => role.name === "temp");
+				if (tempRole) { tempRole.updatedAtMs = 0; }
 				assert.deepEqual(
 					tempRole,
 					new lib.Role(5, "temp", "A temp role", new Set(["core.control.connect"]))
@@ -1272,9 +1418,10 @@ describe("Integration of Clusterio", function() {
 		describe("role edit", function() {
 			it("should modify the given role", async function() {
 				let args = "--name new --description \"A new role\" --set-perms";
-				await execCtl(`role edit temp ${args}`);
+				await execCtlProcess(`role edit temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let newRole = roles.find(role => role.name === "new");
+				if (newRole) { newRole.updatedAtMs = 0; }
 				assert.deepEqual(newRole, new lib.Role(5, "new", "A new role", new Set()));
 			});
 			it("should add permissions with --add-perms", async function() {
@@ -1353,7 +1500,7 @@ describe("Integration of Clusterio", function() {
 		describe("user set-roles", function() {
 			it("should set the roles on the user", async function() {
 				getControl().userUpdates = [];
-				await execCtl('user set-roles temp "Cluster Admin"');
+				await execCtlProcess('user set-roles temp "Cluster Admin"');
 				let users = await getControl().send(new lib.UserListRequest());
 				let tempUser = users.find(user => user.id === "temp");
 				assert.deepEqual(tempUser.roleIds, new Set([0]));
@@ -1361,7 +1508,7 @@ describe("Integration of Clusterio", function() {
 			});
 
 			it("should restrict actions based on roles", async function() {
-				await execCtl('user set-roles temp "Player"');
+				await execCtl("user set-roles temp Player");
 				let tempControl;
 				try {
 					let tlsCa = await fs.readFile("test/file/tls/cert.pem");
@@ -1397,5 +1544,246 @@ describe("Integration of Clusterio", function() {
 				assert.equal(getControl().userUpdates[0].isDeleted, true);
 			});
 		});
+
+		describe("user import", function() {
+			it("should import users", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-users.json"), {
+					export_version: "2.0.0.alpha20",
+					users: [
+						{ username: "import_user_admin", is_admin: true },
+						{ username: "import_user_banned", is_banned: true },
+						{ username: "import_user_ban_reason", is_banned: true, ban_reason: "banned" },
+						{ username: "import_user_whitelist", is_whitelisted: true },
+						{ username: "import_user_none" },
+					],
+				});
+
+				await execCtl("user import --users import-users.json");
+
+				const users = await getUsers();
+				assert(users.get("import_user_admin").isAdmin, "import_user_admin was not admin");
+				assert(users.get("import_user_banned").isBanned, "import_user_banned was not banned");
+				assert(
+					users.get("import_user_whitelist").isWhitelisted,
+					"import_user_whitelist was not whitelisted"
+				);
+
+				const userNone = users.get("import_user_none");
+				assert(!userNone.isAdmin, "import_user_none was admin");
+				assert(!userNone.isBanned, "import_user_none was banned");
+				assert(!userNone.isWhitelisted, "import_user_none was whitelisted");
+
+				const userBanReason = users.get("import_user_ban_reason");
+				assert(userBanReason.isBanned, "import_user_ban_reason was not banned");
+				assert.equal(userBanReason.banReason, "banned", "import_user_ban_reason had incorrect ban reason");
+			});
+			it("should import users by json format", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-users.json"), {
+					export_version: "2.0.0.alpha20",
+					users: [{ username: "import_user", is_admin: true, is_whitelisted: true }],
+				});
+
+				await execCtl("user import import-users.json");
+				const userFoo = (await getUsers()).get("import_user");
+				assert(userFoo.isAdmin, "import_user was not admin");
+				assert(userFoo.isWhitelisted, "import_user was not whitelisted");
+			});
+			it("should import bans", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-bans.json"), [
+					{ username: "import_ban_reason", reason: "banned" },
+					"import_ban",
+				]);
+
+				await execCtl("user import --bans import-bans.json");
+
+				const users = await getUsers();
+
+				const userFoo = users.get("import_ban_reason");
+				assert(userFoo.isBanned, "import_ban_reason was not banned");
+				assert.equal(userFoo.banReason, "banned", "import_ban_reason had incorrect ban reason");
+
+				const userBar = users.get("import_ban");
+				assert(userBar.isBanned, "import_ban was not banned");
+				assert.equal(userBar.banReason, "", "import_ban had incorrect ban reason");
+			});
+			it("should import bans by filename", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-bans.json"), [
+					"import_ban_filename",
+				]);
+
+				await execCtl("user import import-bans.json");
+				assert((await getUsers()).get("import_ban_filename").isBanned, "import_ban_filename was not banned");
+			});
+			it("should import admins", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-admins.json"), [
+					"import_admin",
+				]);
+
+				await execCtl("user import --admins import-admins.json");
+				assert((await getUsers()).get("import_admin").isAdmin, "import_admin was not admin");
+			});
+			it("should import admins by filename", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-admins.json"), [
+					"import_admin_filename",
+				]);
+
+				await execCtl("user import import-admins.json");
+				assert((await getUsers()).get("import_admin_filename").isAdmin, "import_admin_filename was not admin");
+			});
+			it("should import whitelist", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-whitelist.json"), [
+					"import_whitelist",
+				]);
+
+				await execCtl("user import --whitelist import-whitelist.json");
+				assert(
+					(await getUsers()).get("import_whitelist").isWhitelisted,
+					"import_whitelist was not whitelisted"
+				);
+			});
+			it("should import whitelist by filename", async function() {
+				await fs.writeJSON(path.join("temp", "test", "import-whitelist.json"), [
+					"import_whitelist_filename",
+				]);
+
+				await execCtl("user import import-whitelist.json");
+				assert(
+					(await getUsers()).get("import_whitelist_filename").isWhitelisted,
+					"import_whitelist_filename was not whitelisted"
+				);
+			});
+			it("should reject multiple provided options", async function() {
+				await assert.rejects(execCtl("user import --bans --admins import-admins.json"));
+			});
+		});
+
+		describe("user export", function() {
+			before(async function() {
+				await execCtl("user create export_control");
+			});
+			it("should export users", async function() {
+				await execCtl("user set-admin export_user --create");
+				await execCtl("user set-whitelisted export_user");
+				await execCtl("user export --users user-export.json");
+				const data = await fs.readJson(path.join("temp", "test", "user-export.json"));
+				assert(data.users, "Users array does not exist");
+				assert(!data.users.find(u => u.username === "export_control"), "Control user is present");
+				assert.deepEqual(data.users.find(u => u.username === "export_user"), {
+					username: "export_user", is_admin: true, is_whitelisted: true,
+				});
+			});
+			it("should export bans", async function() {
+				await execCtl("user set-banned export_ban --create");
+				await execCtl("user set-banned export_ban_reason --create --reason banned");
+				await execCtl("user export --bans ban-export.json");
+				const data = await fs.readJson(path.join("temp", "test", "ban-export.json"));
+				assert.equal(data.indexOf("export_control"), -1, "Control user is present");
+				assert(data.indexOf("export_ban") >= 0, "export_ban is missing");
+				assert.deepEqual(data.find(u => typeof u === "object" && u.username === "export_ban_reason"), {
+					username: "export_ban_reason", reason: "banned",
+				});
+			});
+			it("should export admins", async function() {
+				await execCtl("user set-admin export_admin --create");
+				await execCtl("user export --admins admin-export.json");
+				const data = await fs.readJson(path.join("temp", "test", "admin-export.json"));
+				assert.equal(data.indexOf("export_control"), -1, "Control user is present");
+				assert(data.indexOf("export_admin") >= 0, "export_admin is missing");
+			});
+			it("should export whitelist", async function() {
+				await execCtl("user set-whitelisted export_whitelist --create");
+				await execCtl("user export --whitelist whitelist-export.json");
+				const data = await fs.readJson(path.join("temp", "test", "whitelist-export.json"));
+				assert.equal(data.indexOf("export_control"), -1, "Control user is present");
+				assert(data.indexOf("export_whitelist") >= 0, "export_whitelist is missing");
+			});
+		});
+
+		describe("user restore", function() {
+			beforeEach(async function() {
+				slowTest(this);
+				await execCtl("user set-admin restore_control --create");
+				await execCtl("user set-banned restore_control");
+				await execCtl("user set-whitelisted restore_control");
+			});
+			it("should restore users", async function() {
+				await fs.writeJSON(path.join("temp", "test", "restore-users.json"), {
+					export_version: "2.0.0.alpha20",
+					users: [{ username: "restore_user", is_admin: true, is_whitelisted: true }],
+				});
+
+				await execCtl("user restore restore-users.json");
+				const users = await getUsers();
+
+				const user = users.get("restore_user");
+				assert(user.isAdmin, "restore_user was not admin");
+				assert(user.isWhitelisted, "restore_user was not whitelisted");
+
+				const control = users.get("restore_control");
+				assert(!control.isAdmin, "restore_user was admin");
+				assert(!control.isBanned, "restore_user was banned");
+				assert(!control.isWhitelisted, "restore_user was whitelisted");
+
+				const data = await fs.readJson(path.join("temp", "test", "users-backup.json"));
+				assert(data.users, "Users array does not exist");
+				assert.deepEqual(data.users.find(u => u.username === "restore_control"), {
+					username: "restore_control", is_admin: true, is_banned: true, is_whitelisted: true,
+				});
+			});
+			it("should restore bans", async function() {
+				await fs.writeJSON(path.join("temp", "test", "restore-bans.json"), [
+					{ username: "restore_ban_reason", reason: "banned" },
+					"restore_ban",
+				]);
+
+				await execCtl("user restore --bans restore-bans.json");
+
+				const users = await getUsers();
+
+				const userFoo = users.get("restore_ban_reason");
+				assert(userFoo.isBanned, "restore_ban_reason was not banned");
+				assert.equal(userFoo.banReason, "banned", "restore_ban_reason had incorrect ban reason");
+
+				const userBar = users.get("restore_ban");
+				assert(userBar.isBanned, "restore_ban was not banned");
+				assert.equal(userBar.banReason, "", "restore_ban had incorrect ban reason");
+
+				assert(!users.get("restore_control").isBanned, "restore_user was banned");
+
+				const data = await fs.readJson(path.join("temp", "test", "bans-backup.json"));
+				assert(data.indexOf("restore_control") >= 0, "restore_control is missing");
+			});
+			it("should restore admins", async function() {
+				await fs.writeJSON(path.join("temp", "test", "restore-admins.json"), [
+					"restore_admin",
+				]);
+
+				await execCtl("user restore --admins restore-admins.json");
+				const users = await getUsers();
+
+				assert(users.get("restore_admin").isAdmin, "restore_admin was not admin");
+				assert(!users.get("restore_control").isAdmin, "restore_user was admin");
+
+				const data = await fs.readJson(path.join("temp", "test", "admins-backup.json"));
+				assert(data.indexOf("restore_control") >= 0, "restore_control is missing");
+			});
+			it("should restore whitelist", async function() {
+				await fs.writeJSON(path.join("temp", "test", "restore-whitelist.json"), [
+					"restore_whitelist",
+				]);
+
+				await execCtl("user restore --whitelist restore-whitelist.json");
+				const users = await getUsers();
+
+				assert(users.get("restore_whitelist").isWhitelisted, "restore_whitelist was not whitelisted");
+				assert(!users.get("restore_control").isWhitelisted, "restore_user was whitelisted");
+
+				const data = await fs.readJson(path.join("temp", "test", "whitelist-backup.json"));
+				assert(data.indexOf("restore_control") >= 0, "restore_control is missing");
+			});
+			it("should reject multiple provided options", async function() {
+				await assert.rejects(execCtl("user restore --bans --admins restore-admins.json"));
+			});
+		});
 	});
 });

test/integration/create.js

@@ -4,7 +4,6 @@ const fs = require("fs-extra");
 const jwt = require("jsonwebtoken");
 const path = require("path");
 const events = require("events");
-const phin = require("phin");
 const child_process = require("child_process");
 const util = require("util");
 

test/integration/index.js

@@ -11,6 +11,8 @@ const events = require("events");
 const lib = require("@clusterio/lib");
 const { LineSplitter, ConsoleTransport, logger } = lib;
 
+const { selectTargetCommand, initialize: initializeCtl } = require("@clusterio/ctl");
+
 // Make sure permissions from plugins are loaded
 require("../../plugins/global_chat/dist/node/index");
 require("../../plugins/player_auth/dist/node/index");
@@ -84,6 +86,10 @@ class TestControl extends lib.Link {
 	async handleUserUpdatesEvent(event) {
 		this.userUpdates.push(...event.updates);
 	}
+
+	async setLogSubscriptions(args) {
+		// Do nothing
+	}
 }
 
 class TestControlConnector extends lib.WebSocketClientConnector {
@@ -175,11 +181,37 @@ async function exec(command, options = {}) {
 	return await util.promisify(child_process.exec)(command, options);
 }
 
-async function execCtl(...args) {
+async function execController(...args) {
+	args[0] = `node --enable-source-maps ../../packages/controller ${args[0]}`;
+	return await exec(...args);
+}
+
+async function execHost(...args) {
+	args[0] = `node --enable-source-maps ../../packages/host ${args[0]}`;
+	return await exec(...args);
+}
+
+async function execCtlProcess(...args) {
 	args[0] = `node --enable-source-maps ../../packages/ctl ${args[0]}`;
 	return await exec(...args);
 }
 
+async function execCtl(command) {
+	process.chdir("temp/test");
+	try {
+		const initArgs = await initializeCtl(command, control.plugins, true);
+		if (!initArgs.shouldRun) {
+			return;
+		}
+
+		control.config = initArgs.controlConfig;
+		const targetCommand = selectTargetCommand(initArgs.args, initArgs.rootCommands);
+		await targetCommand.run(initArgs.args, control);
+	} finally {
+		process.chdir("../..");
+	}
+}
+
 async function sendRcon(instanceId, command) {
 	return await control.sendTo({ instanceId }, new lib.InstanceSendRconRequest(command));
 }
@@ -191,8 +223,15 @@ function getControl() {
 function spawn(name, cmd, waitFor) {
 	// eslint-disable-next-line node/no-process-env
 	const silent = process.env.SILENT_TEST;
+	const bootstrap = !controllerProcess || !hostProcess;
+	function log(...args) {
+		if (!silent || bootstrap) {
+			console.log(...args);
+		}
+	}
+
 	return new Promise((resolve, reject) => {
-		console.log(cmd);
+		log(cmd);
 		let parts = cmd.split(" ");
 		let process = child_process.spawn(parts[0], parts.slice(1), { cwd: path.join("temp", "test") });
 		let stdout = new LineSplitter({ readableObjectMode: true });
@@ -206,10 +245,10 @@ function spawn(name, cmd, waitFor) {
 				}
 				resolve(process);
 			}
-			console.log(name, line);
+			log(name, line);
 		};
 		let onDataErr = line => {
-			console.log(name, line.toString("utf8"));
+			log(name, line.toString("utf8"));
 		};
 		stdout.on("data", onDataOut);
 		stderr.on("data", onDataErr);
@@ -265,42 +304,47 @@ before(async function() {
 	});
 
 	console.log("Setting Controller Config");
-	await exec("node ../../packages/controller config set controller.auth_secret TestSecretDoNotUse");
-	await exec("node ../../packages/controller config set controller.http_port 8880");
-	await exec("node ../../packages/controller config set controller.https_port 4443");
-	await exec("node ../../packages/controller config set controller.heartbeat_interval 0.25");
-	await exec("node ../../packages/controller config set controller.session_timeout 2");
-	await exec("node ../../packages/controller config set controller.tls_certificate ../../test/file/tls/cert.pem");
-	await exec("node ../../packages/controller config set controller.tls_private_key ../../test/file/tls/key.pem");
+	await execController("config set controller.auth_secret TestSecretDoNotUse");
+	await execController("config set controller.http_port 8880");
+	await execController("config set controller.https_port 4443");
+	await execController("config set controller.heartbeat_interval 0.25");
+	await execController("config set controller.session_timeout 2");
+	await execController("config set controller.tls_certificate ../../test/file/tls/cert.pem");
+	await execController("config set controller.tls_private_key ../../test/file/tls/key.pem");
 
 	console.log("Setting Controller Plugins");
-	await exec("node ../../packages/ctl plugin add ../../plugins/global_chat");
-	await exec("node ../../packages/ctl plugin add ../../plugins/research_sync");
-	await exec("node ../../packages/ctl plugin add ../../plugins/statistics_exporter");
-	await exec("node ../../packages/ctl plugin add ../../plugins/subspace_storage");
-	await exec("node ../../packages/ctl plugin add ../../plugins/player_auth");
+	await execCtlProcess("plugin add ../../plugins/global_chat");
+	await execCtlProcess("plugin add ../../plugins/research_sync");
+	await execCtlProcess("plugin add ../../plugins/statistics_exporter");
+	await execCtlProcess("plugin add ../../plugins/subspace_storage");
+	await execCtlProcess("plugin add ../../plugins/inventory_sync");
+	await execCtlProcess("plugin add ../../plugins/player_auth");
 
 	console.log("Bootstrapping");
-	await exec("node ../../packages/controller bootstrap create-admin test");
-	await exec("node ../../packages/controller bootstrap create-ctl-config test");
-	await exec("node ../../packages/ctl control-config set control.tls_ca ../../test/file/tls/cert.pem");
+	await execController("bootstrap create-admin test");
+	await execController("bootstrap create-ctl-config test");
+	await execCtlProcess("control-config set control.tls_ca ../../test/file/tls/cert.pem");
 
 	controllerProcess = await spawnNode("controller:", "../../packages/controller run", /Started controller/);
 
-	await execCtl("host create-config --id 4 --name host --generate-token");
-
 	const relativeFactorioDir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir);
-	await exec(`node ../../packages/host config set host.factorio_directory ${relativeFactorioDir}`);
-	await exec("node ../../packages/host config set host.tls_ca ../../test/file/tls/cert.pem");
+	await execCtlProcess("host create-config --id 4 --name host --generate-token");
+	await execHost(`config set host.factorio_directory ${relativeFactorioDir}`);
+	await execHost("config set host.tls_ca ../../test/file/tls/cert.pem");
 
 	hostProcess = await spawnNode("host:", "../../packages/host run", /Started host/);
 
-	let tlsCa = await fs.readFile("test/file/tls/cert.pem");
-	let controlConnector = new TestControlConnector(url, 2, tlsCa);
+	const tlsCa = await fs.readFile("test/file/tls/cert.pem");
+	const controlConnector = new TestControlConnector(url, 2, tlsCa);
 	controlConnector.token = controlToken;
 	control = new TestControl(controlConnector);
 	await controlConnector.connect();
 
+	const initArgs = await initializeCtl(`--plugin-list ${pluginListPath} control-config list`, undefined, true);
+	control.config = initArgs.controlConfig;
+	control.plugins = initArgs.ctlPlugins;
+	control.tlsCa = tlsCa;
+
 	const testPack = lib.ModPack.fromJSON({});
 	testPack.id = 12;
 	testPack.name = "subspace_storage-pack";
@@ -341,10 +385,12 @@ module.exports = {
 	TestControl,
 	TestControlConnector,
 	TestHostConnector,
+	execCtlProcess,
 	slowTest,
 	get,
 	exec,
 	execCtl,
+	execHost,
 	sendRcon,
 	getControl,
 	spawn,

test/integration/instance.js

@@ -3,7 +3,7 @@ const assert = require("assert").strict;
 const lib = require("@clusterio/lib");
 
 const {
-	slowTest, exec, execCtl, sendRcon, getControl,
+	slowTest, exec, execCtl, execCtlProcess, sendRcon, getControl,
 } = require("./index");
 
 const instId = 48;
@@ -30,19 +30,21 @@ describe("Clusterio Instance", function() {
 
 		// Create a new instance
 		await execCtl(`instance create ${instName} --id ${instId}`);
-		await execCtl(`instance config set-prop ${instName} factorio.settings visibility "${visibility}"`);
+		await execCtl(`instance config set ${instName} factorio.enable_whitelist true`);
+		await execCtlProcess(`instance config set-prop ${instName} factorio.settings visibility "${visibility}"`);
 		await execCtl(`instance config set-prop ${instName} factorio.settings require_user_verification false`);
 		await execCtl(`instance assign ${instName} 4`);
 
 		for (const plugin of [
-			"global_chat", "research_sync", "statistics_exporter", "subspace_storage", "player_auth",
+			"global_chat", "research_sync", "statistics_exporter", "subspace_storage", "player_auth", "inventory_sync",
 		]) {
 			await execCtl(`instance config set ${instName} ${plugin}.load_plugin false`);
 		}
 
 		// Create a new alt instance
 		await execCtl(`instance create ${instAltName} --id ${instAltId}`);
-		await execCtl(`instance config set-prop ${instAltName} factorio.settings visibility "${visibility}"`);
+		await execCtl(`instance config set ${instAltName} factorio.enable_whitelist true`);
+		await execCtlProcess(`instance config set-prop ${instAltName} factorio.settings visibility "${visibility}"`);
 		await execCtl(`instance config set-prop ${instAltName} factorio.settings require_user_verification false`);
 		await execCtl(`instance assign ${instAltName} 4`);
 		await execCtl(`instance start ${instAltName}`);
@@ -73,12 +75,12 @@ describe("Clusterio Instance", function() {
 				if (savePatchingEnabled) {
 					// IPC expects save patching to be enabled
 					it("should do nothing for an unknown type", async function() {
-						assert.rejects(getUser("DoesNotExist"), "User should not exist");
+						await assert.rejects(getUser("DoesNotExist"), "User should not exist");
 						await sendRcon(instId,
 							`/sc ${requireApi} api.send_json("player_event",` +
 							"{ type='invalid type', name='DoesNotExist' })"
 						);
-						assert.rejects(getUser("DoesNotExist"), "User was created");
+						await assert.rejects(getUser("DoesNotExist"), "User was created");
 					});
 					it("should respond to a player join and leave event", async function() {
 						await sendRcon(instId,
@@ -98,12 +100,14 @@ describe("Clusterio Instance", function() {
 				it("should respond to a player ban and unban event", async function() {
 					await execCtl(`instance config set ${instName} factorio.sync_banlist bidirectional`);
 					await sendRcon(instId, "/ban BannedPlayer");
+					await lib.wait(15); // Propagation time, 10ms worked for me, so i made it 15ms
 					const userBanned = await getUser("BannedPlayer");
 					assert(userBanned.isBanned, "Player is not banned");
 					const isAltBanned = await sendRcon(instAltId, "/banlist get BannedPlayer");
 					assert.equal(isAltBanned, "BannedPlayer is banned.\n", "User is not banned on alt");
 
 					await sendRcon(instId, "/unban BannedPlayer");
+					await lib.wait(15); // Propagation time, 10ms worked for me, so i made it 15ms
 					const userUnbanned = await getUser("BannedPlayer");
 					assert(!userUnbanned.isBanned, "Player is not unbanned");
 					const isAltUnbanned = await sendRcon(instAltId, "/banlist get BannedPlayer");
@@ -113,6 +117,7 @@ describe("Clusterio Instance", function() {
 					// Because there is no admin list command this test will be different to the other two
 					await execCtl(`instance config set ${instName} factorio.sync_adminlist bidirectional`);
 					await sendRcon(instId, "/promote AdminPlayer");
+					await lib.wait(15); // Propagation time, 10ms worked for me, so i made it 15ms
 					const userPromote = await getUser("AdminPlayer");
 					assert(userPromote.isAdmin, "Player is not promoted");
 					const isAltPromoted = await sendRcon(instAltId, "/promote AdminPlayer");
@@ -122,6 +127,7 @@ describe("Clusterio Instance", function() {
 					);
 
 					await sendRcon(instId, "/demote AdminPlayer");
+					await lib.wait(15); // Propagation time, 10ms worked for me, so i made it 15ms
 					const userDemote = await getUser("AdminPlayer");
 					assert(!userDemote.isAdmin, "Player is not demoted");
 					const isAltDemoted = await sendRcon(instAltId, "/demote AdminPlayer");
@@ -131,21 +137,22 @@ describe("Clusterio Instance", function() {
 					);
 				});
 				it("should respond to a player whitelist add and remove event", async function() {
-					this.skip(); // Not implemented
 					await execCtl(`instance config set ${instName} factorio.sync_whitelist bidirectional`);
 					await sendRcon(instId, "/whitelist add WhitelistPlayer");
+					await lib.wait(15); // Propagation time, 10ms worked for me, so i made it 15ms
 					const userAdded = await getUser("WhitelistPlayer");
 					assert(userAdded.isWhitelisted, "Player is not whitelisted");
-					const isAltAdded = await sendRcon(instAltId, "/whitelist get whitelisted");
+					const isAltAdded = await sendRcon(instAltId, "/whitelist get WhitelistPlayer");
 					assert.equal(isAltAdded,
 						"WhitelistPlayer is whitelisted.\n",
 						"User is not whitelisted on alt"
 					);
 
 					await sendRcon(instId, "/whitelist remove WhitelistPlayer");
+					await lib.wait(15); // Propagation time, 10ms worked for me, so i made it 15ms
 					const userRemoved = await getUser("WhitelistPlayer");
 					assert(!userRemoved.isWhitelisted, "Player is not unwhitelisted");
-					const isAltRemoved = await sendRcon(instAltId, "/whitelist get whitelisted");
+					const isAltRemoved = await sendRcon(instAltId, "/whitelist get WhitelistPlayer");
 					assert.equal(isAltRemoved,
 						"WhitelistPlayer is not whitelisted.\n",
 						"User is not unwhitelisted on alt"

test/integration/routing.js

@@ -23,7 +23,7 @@ function connectHost(controller, hostId, plugins) {
 	controller.wsServer.hostConnections.set(hostId, hostConnection);
 	const hostConfig = new lib.HostConfig("host");
 	hostConfig.set("host.id", hostId);
-	return new Host(hostSide, "invalid", hostConfig, undefined, []);
+	return new Host(hostSide, hostConfig, undefined, []);
 }
 
 /**
@@ -107,12 +107,7 @@ describe("Integration of link routing", function() {
 	let controlB;
 
 	beforeEach(function() {
-		controller = new Controller(
-			{},
-			[],
-			"invalid",
-			new lib.ControllerConfig("controller")
-		);
+		controller = new Controller({}, [], new lib.ControllerConfig("controller"));
 		hostA = connectHost(controller, 10, []);
 		hostB = connectHost(controller, 20, []);
 		instanceA1 = connectInstance(controller, hostA, 11);