2.0.0-alpha.23

Small things 16

.github/workflows/node.js.yml

@@ -40,6 +40,12 @@ jobs:
       with:
         files: ./coverage/lcov.info
         token: ${{ secrets.CODECOV_TOKEN }}
+    - name: Upload test logs for debugging
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: logs-${{ matrix.node-version }}-${{ matrix.factorio-version }}
+        path: temp/test/logs
 
   lint:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -31,6 +31,66 @@ Many thanks to the following for contributing to this release:
 [@username](https://github.com/username)
 -->
 
+## Version 2.0.0-alpha.23
+
+### Major Features
+
+- Added automatic headless download and update on linux. [#330](https://github.com/clusterio/clusterio/issues/330)
+- Added recipe notification sync to inventory sync. [#506](https://github.com/clusterio/clusterio/issues/506)
+
+### Features
+
+- Added config lock files to prevent process duplication. [#815](https://github.com/clusterio/clusterio/issues/815)
+- Added process and system start times to system metrics. [#826](https://github.com/clusterio/clusterio/issues/826)
+- Added icons to status tags for host and instances. [#827](https://github.com/clusterio/clusterio/issues/827)
+- Extended export-data to produce separate spritesheets for recipes, signals, technologies, planets, qualities, entities, and static UI icons. [#838](https://github.com/clusterio/clusterio/pull/838)
+- Added Icon Reference page to web interface at `/icon-reference` for browsing all exported icons. [#838](https://github.com/clusterio/clusterio/pull/838)
+- Added visual flash to RCON result box to confirm command execution, even when the output is identical to the previous result. [#793](https://github.com/clusterio/clusterio/issues/793)
+
+### Changes
+
+- Login screen now displays the cluster name. [#818](https://github.com/clusterio/clusterio/issues/818)
+- Instance versions can now be set using Major.Minor without specifing a patch version. [#820](https://github.com/clusterio/clusterio/issues/820)
+- Versions for factorio and modpacks can now be selected using a drop down menu. [#820](https://github.com/clusterio/clusterio/issues/820)
+- The controller will now refuse to start if no users were loaded from file. [#814](https://github.com/clusterio/clusterio/issues/814)
+- Improved wording and display of the generate host token modal. [#828](https://github.com/clusterio/clusterio/issues/828)
+- Installations where muliple conflicting clusterio/lib packages have been installed will now fail with an error [#863](https://github.com/clusterio/clusterio/issues/863).
+- Export completion now logs per-category icon counts. [#838](https://github.com/clusterio/clusterio/pull/838)
+
+### Fixes
+
+- Removed debug log statement from equipment grid serialization. [#XXX](https://github.com/clusterio/clusterio/pull/XXX)
+- Fix that the documentation had a dead link to details of the message passing protocol (#812)
+- Fixed case where achievements were not disabled. [#817](https://github.com/clusterio/clusterio/pull/817)
+- Fix headless download on linux install. [#815](https://github.com/clusterio/clusterio/issues/816)
+- Fix option `--no-download-headless` being ignored. [#815](https://github.com/clusterio/clusterio/issues/816)
+- Fixed action buttons overflowing on narrow screens across the web interface. [#810](https://github.com/clusterio/clusterio/issues/810)
+- Fixed player logistics failing to load on servers without unlocked logistics networks. [#1234](https://github.com/clusterio/clusterio/issues/1234)
+- Fixed mod dependency parser breaking with inconsistent spaces. [#834](https://github.com/clusterio/clusterio/issues/834)
+- No longer attempts to sync inventory in single player. [#835](https://github.com/clusterio/clusterio/issues/835)
+- Fixed tests failing when the configured factorio_directory contains stuff other than Factorio installs [#540](https://github.com/clusterio/clusterio/issues/540).
+- Always create a whitelist file so it can be watched for changes. [#865](https://github.com/clusterio/clusterio/issues/865)
+- Added missing await statements which could cause an instance to start with an outdated admin/ban/whitelist. [#869](https://github.com/clusterio/clusterio/issues/869)
+- Fixed metrics breaking when labels contain comma or equal signs [#861](https://github.com/clusterio/clusterio/issues/861)
+- Fixed Host page show repeated errors when viewing an offline local host [#848](https://github.com/clusterio/clusterio/issues/848)
+- Fixed bad mod paths in icon specifications crashing the export. [#841](https://github.com/clusterio/clusterio/issues/841)
+- Fixed prototypes with no icon data causing errors during export. [#841](https://github.com/clusterio/clusterio/issues/841)
+
+### Breaking Changes
+
+- Custom permission methods should use `IUser` rather than `IControllerUser` [#837](https://github.com/clusterio/clusterio/pull/837)
+- `controller.userManager` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837)
+- `controller.instances` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837)
+- `User/Instance/HostInfo` renamed to `User/Instance/HostRecord` [#837](https://github.com/clusterio/clusterio/pull/837)
+- Removed `host.tls_ca` and `control.tls_ca` config options, see [section on self signed certificates](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate) for the replacement for this option [#857](https://github.com/clusterio/clusterio/issues/857)
+
+Many thanks to the following for contributing to this release:  
+[@Cooldude2606](https://github.com/Cooldude2606)
+[@Daniel](https://github.com/Danielv123)
+[@Double](https://github.com/DoubleThoughtTheProgrammer)
+[@Hornwitser](https://github.com/Hornwitser)
+[@Ryan Walker](https://github.com/solarcloud7)
+
 ## Version 2.0.0-alpha.22
 
 ### Major Features

clean.js

@@ -104,7 +104,7 @@ async function main() {
 			removeBuildArtifacts();
 			removeTestArtifacts();
 		})
-		.command("fast", "Remove all build artifacts", () => {}, argv => {
+		.command("$0", "Remove all build artifacts", () => {}, argv => {
 			DRY = argv.dry;
 			removeBuildArtifacts();
 		})

docs/configuration.md

@@ -182,14 +182,6 @@ You can generate an access token with `clusterioctl host generate-token --id <ho
 Defaults to "enter token here".
 
 
-### host.tls_ca
-
-Path to certificate in PEM format to use for validating a TLS connection to the controller.
-If you have a self signed certificate on the controller you will need to copy the certificate to your hosts and set it as the certificate authority with this option.
-
-Defaults to null meaning use Node.js's default set of trusted certificate authorities.
-
-
 ### host.public_address
 
 External address instances hosted on this host can be accessed on.
@@ -413,14 +405,6 @@ You can generate an access token with `clusteriocontroller bootstrap generate-us
 Defaults to null meaning complain about it not being set and exit.
 
 
-### control.tls_ca
-
-Path to certificate in PEM format to use for validating a TLS connection to the controller.
-If you have a self signed certificate on the controller you will need to copy the certificate to the computer you run clusterctl from and set it as the certificate authority with this option.
-
-Defaults to null meaning use Node.js's default set of trusted certificate authorities.
-
-
 ### control.max_reconnect_delay
 
 Max duration in seconds to wait before attempting to reconnect with the controller after the connection is dropped.

docs/setting-up-tls.md

@@ -63,10 +63,25 @@ This creates a private key file named `key.pem` and a public certificate file na
     npx clusteriocontroller config set controller.tls_certificate cert.pem
     npx clusteriocontroller config set controller.tls_private_key key.pem
 
-Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure it as the `host.tls_ca` option:
+Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure node to trust it by setting the `NODE_EXTRA_CA_CERTS` environment variable to a path to this file before the host is started. You can do this by passing `--self-signed-cert path/to/cert` to the installer when setting up a host, or manually by editing run-host.cmd, run-host.sh and/or clusteriohost.service by adding the following line next to the one setting `NODE_OPTIONS`: 
 
-    npx clusteriohost config set host.tls_ca cert.pem
+**For run-host.cmd**
 
-You will also need to copy the `cert.pem` file to all of the computers you want to remotely manage the cluster via clusterctl and configure it as `control.tls_ca` option:
+    set "NODE_EXTRA_CA_CERTS=path\to\cert.pem"
 
-    npx clusterioctl control-config set control.tls_ca cert.pem
+**For run-host.sh**
+
+    export "NODE_EXTRA_CA_CERTS=path/to/cert.pem"
+
+**For clusteriohost.service**
+
+    Environment=NODE_EXTRA_CA_CERTS=path/to/cert.pem
+
+You will also need to copy the `cert.pem` file to all of the computers you want to remotely manage the cluster via clusterctl and set the `NODE_EXTRA_CA_CERTS` environment variable as a path to that file before running clusterioctl. For example:
+
+    # On Windows
+    set "NODE_EXTRA_CA_CERTS=path\to\cert.pem"
+    # On Linux
+    export NODE_EXTRA_CA_CERTS="path/to/cert.pem"
+    # You can now use clusterioctl
+    npx clusterioctl instance list

package.json

@@ -7,7 +7,7 @@
 		"build": "tsc --build",
 		"watch": "tsc --build --watch",
 		"build-mod": "pnpm run -r build-mod --output-dir ../../dist",
-		"test": "pnpm run build && mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
+		"test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
 		"silent-test": "SILENT_TEST=y pnpm test",
 		"fast-test": "FAST_TEST=y pnpm test",
 		"cover": "nyc pnpm test",
@@ -15,9 +15,7 @@
 		"ci-cover": "nyc --reporter=lcovonly pnpm run-script test",
 		"lint": "eslint packages plugins test",
 		"lint-fix": "eslint --fix packages plugins test",
-		"clean": "node ./clean.js fast",
-		"clean-tests": "node ./clean.js tests",
-		"clean-all": "node ./clean.js all",
+		"clean": "node ./clean.js",
 		"docs": "typedoc",
 		"bundle-dependencies": "bundle-dependencies"
 	},
@@ -58,7 +56,6 @@
 		"jszip": "^3.10.1",
 		"mocha": "^11.7.1",
 		"nyc": "^17.1.0",
-		"phin": "^3.7.1",
 		"typedoc": "^0.28.9",
 		"typescript": "^5.9.2",
 		"yargs": "^18.0.0"

packages/controller/controller.ts

@@ -32,6 +32,7 @@ const { ConsoleTransport, levels, logger } = lib;
 
 import Controller from "./src/Controller";
 import UserManager from "./src/UserManager";
+import UserRecord from "./src/UserRecord";
 import { version } from "./package.json";
 
 // globals
@@ -133,13 +134,22 @@ async function handleBootstrapCommand(
 	let subCommand = args._[1];
 
 	const databaseDirectory = controllerConfig.get("controller.database_directory");
+	await UserManager.attemptMigrateLegacyUsersFile(
+		path.join(databaseDirectory, "users.json"),
+		path.join(databaseDirectory, "roles.json"),
+	);
+
 	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 		path.join(databaseDirectory, "roles.json"),
 		lib.Role.fromJSON.bind(lib.Role),
 	).bootstrap());
 
-	const userManager = new UserManager(controllerConfig, roles);
-	await userManager.load(path.join(databaseDirectory, "users.json"));
+	const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		path.join(databaseDirectory, "users.json"),
+		UserRecord.fromJSON.bind(UserRecord),
+	).bootstrap());
+
+	const userManager = new UserManager(users, roles, controllerConfig);
 
 	if (subCommand === "create-admin") {
 		await controllerConfigLock.acquire(); // Also needed to write to the database files
@@ -150,15 +160,12 @@ async function handleBootstrapCommand(
 			return;
 		}
 
-		let admin = userManager.getByName(args.name);
-		if (!admin) {
-			admin = userManager.createUser(args.name);
-		}
+		const user = userManager.getOrCreateUser(args.name);
+		user.roleIds.add(lib.Role.DefaultAdminRoleId);
+		user.isAdmin = true;
+		user.saveRecord();
 
-		let adminRole = lib.ensureDefaultAdminRole(roles);
-		admin.roleIds.add(adminRole.id);
-		admin.isAdmin = true;
-		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
+		await userManager.records.save();
 		await controllerConfigLock.release();
 
 	} else if (subCommand === "generate-user-token") {
@@ -431,7 +438,7 @@ async function startup() {
 	);
 
 	// Refuse to start if there are no users loaded
-	if (args.checkUserCount && controller.userManager.users.size === 0) {
+	if (args.checkUserCount && controller.users.records.size === 0) {
 		logger.fatal(
 			"Cannot start controller, no users loaded.\n" +
 			"Try `npx clusteriocontroller bootstrap create-admin <username>`");
@@ -483,7 +490,7 @@ I           version of clusterio.  Expect things to break. I
 `
 	);
 	startup().catch(err => {
-		if (err instanceof lib.StartupError) {
+		if (err instanceof lib.StartupError || err.code === "InstallationError") {
 			logger.fatal(`
 +-------------------------------+
 | Unable to to start controller |

packages/controller/index.ts

@@ -1,13 +1,15 @@
 import { bootstrap } from "./controller";
 export { default as Controller } from "./src/Controller";
-export { default as ControllerUser } from "./src/ControllerUser";
 export { default as ControllerRouter } from "./src/ControllerRouter";
 export { default as ControlConnection } from "./src/ControlConnection";
 export { default as HostConnection } from "./src/HostConnection";
-export { default as InstanceInfo } from "./src/InstanceInfo";
-export { default as HostInfo } from "./src/HostInfo";
+export { default as InstanceRecord } from "./src/InstanceRecord";
+export { default as InstanceManager } from "./src/InstanceManager";
+export { default as HostRecord } from "./src/HostRecord";
 export { default as BaseControllerPlugin } from "./src/BaseControllerPlugin";
 export { default as UserManager } from "./src/UserManager";
+export { default as UserRecord } from "./src/UserRecord";
+export { default as User } from "./src/User";
 export { default as WsServer } from "./src/WsServer";
 
 if (module === require.main) {

packages/controller/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio controller",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.23",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --mode production",

packages/controller/src/BaseConnection.ts

@@ -32,6 +32,7 @@ export default class BaseConnection extends lib.Link {
 		this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this));
 		this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this));
 		this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
+		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 
 		this.handle(lib.SubscriptionRequest, this.handleSubscriptionRequest.bind(this));
 		this.connector.on("close", () => {
@@ -106,4 +107,8 @@ export default class BaseConnection extends lib.Link {
 	async handleSubscriptionRequest(request: lib.SubscriptionRequest, src: lib.Address, dst: lib.Address) {
 		return await this._controller.subscriptions.handleRequest(this, request, src, dst);
 	}
+
+	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
+		return await this._controller.factorioVersions.get(request.maxAgeMs);
+	}
 }

packages/controller/src/BaseControllerPlugin.ts

@@ -3,7 +3,7 @@ import type {
 	ModPack, ModInfo, PlayerEvent, PluginNodeEnvInfo,
 } from "@clusterio/lib";
 import type Controller from "./Controller";
-import type InstanceInfo from "./InstanceInfo";
+import type InstanceRecord from "./InstanceRecord";
 import type ControlConnection from "./ControlConnection";
 import type HostConnection from "./HostConnection";
 
@@ -91,7 +91,7 @@ export default class BaseControllerPlugin {
 	 *     The instance that changed.
 	 * @param prev - the previous status of the instance.
 	 */
-	async onInstanceStatusChanged(instance: InstanceInfo, prev?: InstanceStatus) { }
+	async onInstanceStatusChanged(instance: InstanceRecord, prev?: InstanceStatus) { }
 
 	/**
 	 * Called when the value of a controller config field changed.
@@ -117,7 +117,7 @@ export default class BaseControllerPlugin {
 	 * @param curr - The current value of the field.
 	 * @param prev - The previous value of the field.
 	 */
-	async onInstanceConfigFieldChanged(instance: InstanceInfo, field: string, curr: unknown, prev: unknown) { }
+	async onInstanceConfigFieldChanged(instance: InstanceRecord, field: string, curr: unknown, prev: unknown) { }
 
 	/**
 	 * Called before collecting Prometheus metrics
@@ -261,7 +261,7 @@ export default class BaseControllerPlugin {
 	 *     The instance it occured on.
 	 * @param event - Information about the event.
 	 */
-	async onPlayerEvent(instance: InstanceInfo, event: PlayerEvent) { }
+	async onPlayerEvent(instance: InstanceRecord, event: PlayerEvent) { }
 
 	/**
 	 * Broadcast event to all connected hosts

packages/controller/src/ControlConnection.ts

@@ -6,7 +6,8 @@ import * as lib from "@clusterio/lib";
 const { logFilter, logger } = lib;
 
 import BaseConnection from "./BaseConnection";
-import ControllerUser from "./ControllerUser";
+import User from "./User";
+import UserRecord from "./UserRecord";
 import * as routes from "./routes";
 import Controller from "./Controller";
 
@@ -40,7 +41,7 @@ export default class ControlConnection extends BaseConnection {
 		registerData: { version: string },
 		connector: WsServerConnector,
 		controller: Controller,
-		public user: ControllerUser, // The user making this connection.
+		public user: User, // The user making this connection.
 		public id: number
 	) {
 		super(connector, controller);
@@ -126,7 +127,6 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
 		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
 		this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this));
-		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 	}
 
 	validateIngress(message: lib.MessageRequest | lib.MessageEvent) {
@@ -250,7 +250,7 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleInstanceDetailsGetRequest(request: lib.InstanceDetailsGetRequest) {
-		return this._controller.getRequestInstance(request.instanceId).toInstanceDetails();
+		return this._controller.instances.getForRequest(request.instanceId).toInstanceDetails();
 	}
 
 	async handleInstanceDetailsListRequest() {
@@ -268,20 +268,20 @@ export default class ControlConnection extends BaseConnection {
 			instanceConfig.set("instance.assigned_host", null); // New instances are unassigned
 		}
 		instanceConfig.update(request.config, false, "control");
-		await this._controller.instanceCreate(instanceConfig);
+		await this._controller.instances.createInstance(instanceConfig);
 	}
 
 	async handleInstanceDeleteRequest(request: lib.InstanceDeleteRequest) {
-		await this._controller.instanceDelete(request.instanceId);
+		await this._controller.instances.deleteInstance(request.instanceId);
 	}
 
 	async handleInstanceConfigGetRequest(request: lib.InstanceConfigGetRequest) {
-		let instance = this._controller.getRequestInstance(request.instanceId);
+		let instance = this._controller.instances.getForRequest(request.instanceId);
 		return instance.config.toRemote("control");
 	}
 
 	async handleInstanceConfigSetFieldRequest(request: lib.InstanceConfigSetFieldRequest) {
-		let instance = this._controller.getRequestInstance(request.instanceId);
+		let instance = this._controller.instances.getForRequest(request.instanceId);
 		if (request.field === "instance.assigned_host") {
 			throw new lib.RequestError("instance.assigned_host must be set through the assign-host interface");
 		}
@@ -296,14 +296,14 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleInstanceConfigSetPropRequest(request: lib.InstanceConfigSetPropRequest) {
-		let instance = this._controller.getRequestInstance(request.instanceId);
+		let instance = this._controller.instances.getForRequest(request.instanceId);
 		let { field, prop, value } = request;
 		instance.config.setProp(field as keyof lib.InstanceConfigFields, prop, value, "control");
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
 	async handleInstanceAssignRequest(request: lib.InstanceAssignRequest) {
-		await this._controller.instanceAssign(request.instanceId, request.hostId);
+		await this._controller.instances.assignInstance(request.instanceId, request.hostId ?? null);
 	}
 
 	async handleInstanceSaveDetailsListRequest() {
@@ -337,8 +337,8 @@ export default class ControlConnection extends BaseConnection {
 		if (request.sourceInstanceId === request.targetInstanceId) {
 			throw new lib.RequestError("Source and target instance may not be the same");
 		}
-		let sourceInstance = this._controller.getRequestInstance(request.sourceInstanceId);
-		let targetInstance = this._controller.getRequestInstance(request.targetInstanceId);
+		let sourceInstance = this._controller.instances.getForRequest(request.sourceInstanceId);
+		let targetInstance = this._controller.instances.getForRequest(request.targetInstanceId);
 		let sourceHostId = sourceInstance.config.get("instance.assigned_host");
 		let targetHostId = targetInstance.config.get("instance.assigned_host");
 		if (sourceHostId === null) {
@@ -614,22 +614,21 @@ export default class ControlConnection extends BaseConnection {
 		return modInfos;
 	}
 
+	// eslint-disable-next-line complexity
 	async handleModDependencyResolveRequest(request: lib.ModDependencyResolveRequest) {
 		const {factorioVersion, checkForUpdates} = request;
 		const searchQueue = [...request.mods];
 		const localInfos = [...this._controller.modStore.mods()];
 
-		const _builtInModNames = lib.ModPack.getBuiltinModNames(factorioVersion);
-		const builtInModNames = new Set(_builtInModNames);
-		const skipSearch = new Set(_builtInModNames);
+		const builtInModNames = lib.ModPack.getBuiltinModNames(factorioVersion);
+		const skipSearch = new Set(builtInModNames);
 
 		type ModPortalModType = InstanceType<typeof lib.ModPortalGetAllRequest.Response>["mods"][number];
 		type ModPortalReleaseType = NonNullable<ModPortalModType["releases"]>[number];
 		const dependencyRequirements = new Map<string, lib.ModVersionRange>();
 		const optionalRequirements = new Map<string, lib.ModVersionRange>();
 		const candidateReleases = new Map<string, lib.ModInfo>();
-		const incompatible = new Set<string>();
-		const invalid = new Set<string>();
+		const errors = new Map<string, lib.ModDependencyResolveErrors>();
 
 		const _modPortalReleases = new Map<string, ModPortalModType | null>();
 		async function fetchModReleases(modName: string) {
@@ -650,10 +649,44 @@ export default class ControlConnection extends BaseConnection {
 			return releases;
 		}
 
+		function selectModRelease(releases: ModPortalModType | null, versionRange: lib.ModVersionRange) {
+			if (!releases || !releases.releases) {
+				return null;
+			}
+
+			// Select the latest matching version
+			const release = releases.releases
+				.filter(info => versionRange.testVersion(info.version))
+				.reduce<ModPortalReleaseType | undefined>((max, cur) => (
+					max && lib.integerFullVersion(max.version) > lib.integerFullVersion(cur.version) ? max : cur
+				), undefined);
+
+			if (!release) {
+				return null;
+			}
+
+			// Create the mod info class from the selected
+			try {
+				return lib.ModInfo.fromJSON({
+					...release.info_json, // includes dependencies
+					sha1: release.sha1,
+					version: release.version,
+					name: releases.name,
+					author: releases.owner,
+					title: releases.title,
+				});
+			} catch {
+				// Can error if there are invalid dependencies
+				errors.set(releases.name, "invalidDependency");
+				return null;
+			}
+		}
+
 		while (searchQueue.length > 0) {
+			// Dequeue the next mod, skip if incompatible
 			const mod = searchQueue.shift()!;
 			if (mod.incompatible) {
-				incompatible.add(mod.name);
+				errors.set(mod.name, "incompatible");
 				continue;
 			}
 
@@ -674,13 +707,13 @@ export default class ControlConnection extends BaseConnection {
 				versionRange.combineVersion(mod.version);
 				if (!versionRange.valid) {
 					candidateReleases.delete(mod.name);
-					invalid.add(mod.name);
+					errors.set(mod.name, "unsatisfiable");
 				}
 			}
 
-			// Do not add dependencies for invalids, optionals, skips, or valid candidates
+			// Do not add dependencies for errors, skips, optionals, or valid candidates
 			const existingCandidate = candidateReleases.get(mod.name);
-			if (invalid.has(mod.name) || optionalRequirements.has(mod.name) || skipSearch.has(mod.name) || (
+			if (errors.has(mod.name) || skipSearch.has(mod.name) || optionalRequirements.has(mod.name) || (
 				existingCandidate && versionRange.testVersion(existingCandidate.version)
 			)) {
 				continue;
@@ -695,28 +728,11 @@ export default class ControlConnection extends BaseConnection {
 
 			// Check if a mod poral version fits the requirements
 			if (!candidate || checkForUpdates) {
-				const modReleases = await fetchModReleases(mod.name);
-				if (modReleases && modReleases.releases) {
-					const release = modReleases.releases
-						.filter(info => (
-							info.info_json.factorio_version === factorioVersion
-							&& versionRange.testVersion(info.version)
-						))
-						.reduce<ModPortalReleaseType | undefined>((max, cur) => (
-							max && lib.integerFullVersion(max.version) > lib.integerFullVersion(cur.version) ? max : cur
-						), undefined);
-
-					// We only select the mod portal release if it is more recent than the local candidate
-					if (release && (!candidate || lib.integerFullVersion(release.version) > candidate.integerVersion)) {
-						candidate = lib.ModInfo.fromJSON({
-							...release.info_json, // Also includes dependences
-							sha1: release.sha1,
-							version: release.version,
-							name: modReleases.name,
-							author: modReleases.owner,
-							title: modReleases.title,
-						});
-					}
+				const releases = await fetchModReleases(mod.name);
+				const release = selectModRelease(releases, versionRange);
+				// We only select the mod portal release if it is more recent than the local candidate
+				if (release && (!candidate || release.integerVersion > candidate.integerVersion)) {
+					candidate = release;
 				}
 			}
 
@@ -732,14 +748,22 @@ export default class ControlConnection extends BaseConnection {
 			}
 		}
 
+		// Add the not found dependencies as errors
+		for (const modName of dependencyRequirements.keys()) {
+			if (!candidateReleases.has(modName) && !errors.has(modName)) {
+				errors.set(modName, "notFound");
+			}
+		}
+
+		// Remove incompatible dependencies if they aren't required by others
+		for (const [modName, error] of errors) {
+			if (error === "incompatible" && !dependencyRequirements.has(modName)) {
+				errors.delete(modName);
+			}
+		}
+
 		return new lib.ModDependencyResolveRequest.Response(
-			[...candidateReleases.values()],
-			[...invalid.values(), ...[...incompatible.values()].filter(mod => (
-				candidateReleases.has(mod) || builtInModNames.has(mod)
-			))],
-			[...dependencyRequirements.keys()].filter(mod => (
-				!candidateReleases.has(mod) && !incompatible.has(mod) && !invalid.has(mod)
-			)),
+			[...candidateReleases.values()], errors,
 		);
 	}
 
@@ -846,35 +870,33 @@ export default class ControlConnection extends BaseConnection {
 		}
 		this._controller.roles.delete(role);
 
-		this._controller.userManager.dirty = true;
-		for (let user of this._controller.userManager.users.values()) {
-			if (user.roleIds.delete(id)) {
+		for (const user of this._controller.users.values()) {
+			if (user.removeRole(id)) {
 				this._controller.userPermissionsUpdated(user);
 			}
 		}
 	}
 
-	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.User> {
+	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.UserDetails> {
 		let name = request.name;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User ${name} does not exist`);
 		}
 
-		return user;
+		return user.toUserDetails();
 	}
 
-	async handleUserListRequest(): Promise<lib.User[]> {
-		return [...this._controller.userManager.users.values()];
+	async handleUserListRequest(): Promise<lib.UserDetails[]> {
+		return [...this._controller.users.records.values()].map(user => user.toUserDetails());
 	}
 
 	async handleUserCreateRequest(request: lib.UserCreateRequest) {
-		let user = this._controller.userManager.createUser(request.name);
-		this._controller.usersUpdated([user]);
+		this._controller.users.createUser(request.name);
 	}
 
 	async handleUserRevokeTokenRequest(request: lib.UserRevokeTokenRequest) {
-		let user = this._controller.userManager.getByName(request.name);
+		let user = this._controller.users.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
@@ -888,11 +910,10 @@ export default class ControlConnection extends BaseConnection {
 				controlConnection.connector.terminate();
 			}
 		}
-		this._controller.usersUpdated([user]);
 	}
 
 	async handleUserUpdateRolesRequest(request: lib.UserUpdateRolesRequest) {
-		let user = this._controller.userManager.getByName(request.name);
+		let user = this._controller.users.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
@@ -903,35 +924,33 @@ export default class ControlConnection extends BaseConnection {
 			}
 		}
 
-		user.roleIds = new Set(request.roles);
+		user.set("roleIds", new Set(request.roles));
 		this._controller.userPermissionsUpdated(user);
-		this._controller.usersUpdated([user]);
 	}
 
 	async handleUserSetAdminRequest(request: lib.UserSetAdminRequest) {
 		let { name, create, admin } = request;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.userManager.createUser(name);
+				user = this._controller.users.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
 		}
 
-		user.isAdmin = admin;
-		this._controller.usersUpdated([user]);
+		user.set("isAdmin", admin);
 		this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, admin));
 	}
 
 	async handleUserSetBannedRequest(request: lib.UserSetBannedRequest) {
 		let { name, create, banned, reason } = request;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByNameMutable(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.userManager.createUser(name);
+				user = this._controller.users.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
@@ -939,37 +958,34 @@ export default class ControlConnection extends BaseConnection {
 
 		user.isBanned = banned;
 		user.banReason = reason;
-		this._controller.usersUpdated([user]);
+		user.saveRecord();
 		this._controller.sendTo("allInstances", new lib.InstanceBanlistUpdateEvent(name, banned, reason));
 	}
 
 	async handleUserSetWhitelistedRequest(request: lib.UserSetWhitelistedRequest) {
 		let { name, create, whitelisted } = request;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.userManager.createUser(name);
+				user = this._controller.users.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
 		}
 
-		user.isWhitelisted = whitelisted;
-		this._controller.usersUpdated([user]);
+		user.set("isWhitelisted", whitelisted);
 		this._controller.sendTo("allInstances", new lib.InstanceWhitelistUpdateEvent(name, whitelisted));
 	}
 
 	async handleUserDeleteRequest(request: lib.UserDeleteRequest) {
 		let name = request.name;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User '${name}' does not exist`);
 		}
 
-		user.isDeleted = true;
-		this._controller.userManager.users.delete(name);
-		this._controller.usersUpdated([user]);
+		this._controller.users.deleteUser(user);
 
 		if (user.isAdmin) {
 			this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, false));
@@ -982,14 +998,15 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
-	async handleUserBulkRestoreRequest(request: lib.UserBulkImportRequest, updated: Map<string, ControllerUser>) {
+	/** This does not do a full restore, rather it demotes / unbanns / unwhitelistes users */
+	async handleUserBulkRestoreRequest(request: lib.UserBulkImportRequest, updated: Map<string, UserRecord>) {
 		if (request.importType === "users") {
 			// Update all fields for all users
 			const users = request.users as Extract<typeof request.users, { username: string }[]>;
 			const admin = new Set(users.filter(u => u.is_admin).map(u => u.username.toLowerCase()));
 			const banned = new Set(users.filter(u => u.is_banned).map(u => u.username.toLowerCase()));
 			const whitelist = new Set(users.filter(u => u.is_whitelisted).map(u => u.username.toLowerCase()));
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.valuesMutable()) {
 				if (user.isAdmin && !admin.has(user.name)) {
 					user.isAdmin = false;
 					updated.set(user.id, user);
@@ -1010,7 +1027,7 @@ export default class ControlConnection extends BaseConnection {
 			const expected = new Set(
 				users.map(u => (typeof u === "string" ? u.toLowerCase() : u.username.toLowerCase()))
 			);
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.valuesMutable()) {
 				if (user.isBanned && !expected.has(user.name)) {
 					user.isBanned = false;
 					user.banReason = "";
@@ -1022,7 +1039,7 @@ export default class ControlConnection extends BaseConnection {
 			const users = request.users as string[];
 			const expected = new Set(users.map(u => u.toLowerCase()));
 			const prop = request.importType === "admins" ? "isAdmin" : "isWhitelisted";
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.valuesMutable()) {
 				if (user[prop] && !expected.has(user.name)) {
 					user[prop] = false;
 					updated.set(user.id, user);
@@ -1031,11 +1048,9 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
-	// A large number of small if statements are unavoidable when reducing the number of updates sent
-
 	async handleUserBulkImportRequest(request: lib.UserBulkImportRequest) {
 		let backup: undefined | Awaited<ReturnType<ControlConnection["handleUserBulkExportRequest"]>>;
-		const updated = new Map<ControllerUser["id"], ControllerUser>();
+		const updated = new Map<User["id"], User>();
 		if (request.restore) {
 			// Unban / Demote / Unwhitelist players not on the list
 			backup = await this.handleUserBulkExportRequest(new lib.UserBulkExportRequest(request.importType));
@@ -1043,11 +1058,12 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		// Will get a user or attempt to create one
+		// UserManager.getOrCreateUser not used due to inclusion of checkPermission
 		const getUserOrCreate = (username: string) => {
-			const user = this._controller.userManager.getByName(username);
-			if (user) { return user; }
+			const user = this._controller.users.getByNameMutable(username);
+			if (user) { return updated.get(user.id) ?? user; }
 			this.user.checkPermission("core.user.create");
-			return this._controller.userManager.createUser(username);
+			return this._controller.users.createUser(username);
 		};
 
 		// Merge the imported data with existing data, this is strictly additive
@@ -1105,14 +1121,14 @@ export default class ControlConnection extends BaseConnection {
 
 		// Send the necessary update events
 		const updatedUsers = [...updated.values()];
-		this._controller.usersUpdated(updatedUsers);
+		this._controller.users.records.setMany(updatedUsers);
 
 		// Resync the host user lists
 		const adminlist: Set<string> = new Set();
 		const banlist: Map<string, string> = new Map();
 		const whitelist: Set<string> = new Set();
 
-		for (let user of this._controller.userManager.users.values()) {
+		for (let user of this._controller.users.values()) {
 			if (user.isAdmin) {
 				adminlist.add(user.id);
 			}
@@ -1136,7 +1152,7 @@ export default class ControlConnection extends BaseConnection {
 		if (request.exportType === "users") {
 			// Send a full user export
 			const usersToSend = [] as lib.ClusterioUserExport["users"];
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.values()) {
 				let send = false;
 				const userToSend = { username: user.name } as typeof usersToSend[0];
 				if (user.isBanned) {
@@ -1162,7 +1178,7 @@ export default class ControlConnection extends BaseConnection {
 		} else if (request.exportType === "bans") {
 			// Bans have extra logic to handle ban reason
 			const usersToSend = [] as Array<string | { username: string, reason: string }>;
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.values()) {
 				if (user.isBanned) {
 					if (user.banReason && user.banReason !== "") {
 						usersToSend.push({ username: user.name, reason: user.banReason });
@@ -1177,7 +1193,7 @@ export default class ControlConnection extends BaseConnection {
 		// This is a whitelist or admins export
 		const usersToSend = [] as string[];
 		const prop = request.exportType === "admins" ? "isAdmin" : "isWhitelisted";
-		for (const user of this._controller.userManager.users.values()) {
+		for (const user of this._controller.users.values()) {
 			if (user[prop]) {
 				usersToSend.push(user.name);
 			}
@@ -1223,10 +1239,6 @@ export default class ControlConnection extends BaseConnection {
 		this.connector._socket!.clusterio_ignore_dump = true;
 		this._controller.debugEvents.on("message", this.ws_dumper);
 	}
-
-	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
-		return await this._controller.factorioVersions.get(request.maxAgeMs);
-	}
 }
 
 module.exports = ControlConnection;

packages/controller/src/Controller.ts

@@ -18,16 +18,17 @@ import * as lib from "@clusterio/lib";
 const { logger, Summary, Gauge } = lib;
 
 import HttpCloser from "./HttpCloser";
-import InstanceInfo from "./InstanceInfo";
+import InstanceRecord from "./InstanceRecord";
 import * as metrics from "./metrics";
 import * as routes from "./routes";
-import ControllerUser from "./ControllerUser";
+import User from "./User";
+import UserRecord from "./UserRecord";
 import UserManager from "./UserManager";
 import WsServer from "./WsServer";
-import HostConnection from "./HostConnection";
-import HostInfo from "./HostInfo";
+import HostRecord from "./HostRecord";
 import BaseControllerPlugin from "./BaseControllerPlugin";
 import ControllerRouter from "./ControllerRouter";
+import InstanceManager from "./InstanceManager";
 
 const endpointDurationSummary = new Summary(
 	"clusterio_controller_http_endpoint_duration_seconds",
@@ -79,6 +80,10 @@ export default class Controller {
 
 	/** Event subscription controller */
 	subscriptions = new lib.SubscriptionController();
+	/** User manager for the cluster, also contains access to the user records */
+	users: UserManager;
+	/** Instance manager for the cluster, also contains access to the user records */
+	instances: InstanceManager;
 
 	// Possible states are new, starting, running, stopping, stopped
 	private _state: string = "new";
@@ -96,6 +101,7 @@ export default class Controller {
 
 	devMiddleware: any | null = null;
 
+	startedAtMs = Date.now();
 	autosaveInterval?: ReturnType<typeof setInterval>;
 	systemMetricsInterval?: ReturnType<typeof setInterval>;
 
@@ -106,12 +112,25 @@ export default class Controller {
 	// Cache for mod portal requests
 	modPortalCache = new Map<string, { timestamp: number, data: any[] }>();
 	// Cache for factorio versions
-	factorioVersions = new lib.ValueCache(lib.fetchFactorioVersions);
+	factorioVersions = new lib.ValueCache(async () => {
+		try {
+			return await lib.fetchFactorioVersions();
+		} catch (err: any) {
+			logger.warn(`Failed to retrieve Factorio versions ${err.message}`);
+			return [];
+		}
+	});
 
 	static async bootstrap(config: lib.ControllerConfig) {
 		let databaseDirectory = config.get("controller.database_directory");
 		await fs.ensureDir(databaseDirectory);
 
+		// Migrate combined user and role file from 2.0.0-alpha.22
+		await UserManager.attemptMigrateLegacyUsersFile(
+			path.join(databaseDirectory, "users.json"),
+			path.join(databaseDirectory, "roles.json"),
+		);
+
 		const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "systems.json"),
 			lib.SystemInfo.fromJSON.bind(lib.SystemInfo),
@@ -120,13 +139,13 @@ export default class Controller {
 
 		const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "hosts.json"),
-			HostInfo.fromJSON.bind(HostInfo),
+			HostRecord.fromJSON.bind(HostRecord),
 			this.migrateHosts, this.finaliseHosts,
 		).bootstrap());
 
 		const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "instances.json"),
-			json => InstanceInfo.fromJSON(json, "controller"),
+			InstanceRecord.fromJSON.bind(InstanceRecord),
 			this.migrateInstances, this.finaliseInstances,
 		).bootstrap());
 
@@ -147,8 +166,11 @@ export default class Controller {
 			this.migrateRoles,
 		).bootstrap());
 
-		const userManager = new UserManager(config, roles);
-		await userManager.load(path.join(databaseDirectory, "users.json"));
+		const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "users.json"),
+			UserRecord.fromJSON.bind(UserRecord),
+			undefined, this.finaliseUsers,
+		).bootstrap());
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
@@ -160,8 +182,8 @@ export default class Controller {
 		}
 
 		// Add default roles
-		lib.ensureDefaultAdminRole(roles);
-		lib.ensureDefaultPlayerRole(roles);
+		lib.Role.ensureDefaultAdminRole(roles);
+		lib.Role.ensureDefaultPlayerRole(roles);
 
 		return [
 			systems,
@@ -171,7 +193,7 @@ export default class Controller {
 			modPacks,
 			modStore,
 			roles,
-			userManager,
+			users,
 		] as const;
 	}
 
@@ -191,10 +213,10 @@ export default class Controller {
 		 */
 		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
-		/** Mapping of host id to host info */
-		public hosts = new lib.SubscribableDatastore<HostInfo>(),
-		/** Mapping of instance id to instance info */
-		public instances = new lib.SubscribableDatastore<InstanceInfo>(),
+		/** Mapping of host id to host record */
+		public hosts = new lib.SubscribableDatastore<HostRecord>(),
+		/** Mapping of instance id to instance record */
+		instances = new lib.SubscribableDatastore<InstanceRecord>(),
 		/** Mapping of save id to save details */
 		public saves = new lib.SubscribableDatastore<lib.SaveDetails>(),
 		/** Mapping of mod pack id to mod pack */
@@ -203,8 +225,8 @@ export default class Controller {
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
 		/** Mapping of mod pack id to mod pack */
 		public roles = new lib.SubscribableDatastore<lib.Role>(),
-		/** User and roles manager for the cluster */
-		public userManager = new UserManager(config, roles),
+		/** Mapping of user id to user record */
+		users = new lib.SubscribableDatastore<UserRecord>(),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -217,6 +239,9 @@ export default class Controller {
 		this.trustedProxies = this.parseTrustedProxies();
 		this.wsServer = new WsServer(this);
 
+		this.users = new UserManager(users, roles, config);
+		this.instances = new InstanceManager(instances, this);
+
 		this.modStore.on("change", mod => {
 			this.modsUpdated([mod]);
 		});
@@ -236,10 +261,11 @@ export default class Controller {
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
 		this.hosts.on("update", this.hostsUpdated.bind(this));
-		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
+		this.instances.records.on("update", this.instanceUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
 		this.roles.on("update", this.rolesUpdated.bind(this));
+		this.users.records.on("update", this.usersUpdated.bind(this));
 	}
 
 	async start(args: ControllerArgs) {
@@ -304,9 +330,6 @@ export default class Controller {
 			}
 			lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev);
 		});
-		for (let instance of this.instances.values()) {
-			this.addInstanceHooks(instance);
-		}
 
 		// Make sure we're actually going to listen on a port
 		let httpPort = this.config.get("controller.http_port");
@@ -397,6 +420,7 @@ export default class Controller {
 		this.onSystemMetricsIntervalChanged();
 
 		logger.info("Started controller");
+		this.startedAtMs = Date.now();
 		this._state = "running";
 	}
 
@@ -553,7 +577,7 @@ export default class Controller {
 				requests.push(hostConnection.send(new lib.SystemInfoRequest()));
 			}
 			const restartRequired = await this.checkRestartRequired();
-			requests.push(lib.gatherSystemInfo("controller", this.canRestart, restartRequired));
+			requests.push(lib.gatherSystemInfo("controller", this.canRestart, restartRequired, this.startedAtMs));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
 				this.systems.set(metric);
@@ -641,16 +665,13 @@ export default class Controller {
 			this.config.save(),
 			this.systems.save(),
 			this.hosts.save(),
-			this.instances.save(),
+			this.instances.records.save(),
 			this.saves.save(),
 			this.modPacks.save(),
 			this.roles.save(),
+			this.users.records.save(),
 		]);
 
-		if (this.userManager.dirty) {
-			await this.userManager.save(path.join(this.config.get("controller.database_directory"), "users.json"));
-		}
-
 		await lib.invokeHook(this.plugins, "onSaveData");
 	}
 
@@ -663,11 +684,17 @@ export default class Controller {
 			if (!json.restartRequired) { // Added in 2.0.0.alpha.21
 				json.restartRequired = false;
 			}
+			if (json.systemStartedAtMs === undefined) { // Added in 2.0.0.alpha.23
+				json.systemStartedAtMs = Date.now();
+			}
+			if (json.processStartedAtMs === undefined) { // Added in 2.0.0.alpha.23
+				json.processStartedAtMs = Date.now();
+			}
 			return json;
 		});
 	}
 
-	static migrateHosts(rawJson: unknown[]): Static<typeof HostInfo.jsonSchema>[] {
+	static migrateHosts(rawJson: unknown[]): Static<typeof HostRecord.jsonSchema>[] {
 		let serialized = rawJson as any;
 
 		// New format 2.0.0.alpha.19
@@ -678,7 +705,7 @@ export default class Controller {
 		return serialized;
 	}
 
-	static finaliseHosts(host: HostInfo): HostInfo {
+	static finaliseHosts(host: HostRecord): HostRecord {
 		if (host.connected) {
 			host.connected = false;
 			host.updatedAtMs = Date.now();
@@ -686,8 +713,8 @@ export default class Controller {
 		return host;
 	}
 
-	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceInfo.jsonSchema>[] {
-		const serialized = rawJson as Static<typeof InstanceInfo.jsonSchema>[];
+	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceRecord.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof InstanceRecord.jsonSchema>[];
 		return serialized.map(json => {
 			if (!json.config) { // New format 2.0.0.alpha.14
 				return { config: json as any, status: "running" }; // Use running to force updatedAtMs
@@ -696,7 +723,7 @@ export default class Controller {
 		});
 	}
 
-	static finaliseInstances(instance: InstanceInfo): InstanceInfo {
+	static finaliseInstances(instance: InstanceRecord): InstanceRecord {
 		const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown";
 		if (instance.status !== status) {
 			instance.status = status;
@@ -730,6 +757,20 @@ export default class Controller {
 		});
 	}
 
+	static finaliseUsers(user: UserRecord, users: Map<UserRecord["id"], UserRecord>): UserRecord {
+		const existingUser = users.get(user.id);
+		if (existingUser) {
+			// All lowercase ids in 2.0.0.alpha.19
+			if (user.playerStats.onlineTimeMs <= existingUser.playerStats.onlineTimeMs) {
+				// We assume the user with the lower online time is the duplicate
+				existingUser.merge(user);
+				return existingUser;
+			}
+			user.merge(existingUser);
+		}
+		return user;
+	}
+
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -848,7 +889,7 @@ export default class Controller {
 		);
 	}
 
-	hostsUpdated(hosts: HostInfo[]) {
+	hostsUpdated(hosts: HostRecord[]) {
 		let updates = hosts.map(host => host.toHostDetails());
 		this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates));
 	}
@@ -860,75 +901,6 @@ export default class Controller {
 		return hosts.length ? new lib.HostUpdatesEvent(hosts) : null;
 	}
 
-	/**
-	 * Get instance by ID for a request
-	 *
-	 * @param instanceId - ID of instance to get.
-	 * @returns Info for the given instance if it exists
-	 * @throws {module:lib.RequestError} if the instance does not exist.
-	 */
-	getRequestInstance(instanceId:number): InstanceInfo {
-		let instance = this.instances.get(instanceId);
-		if (!instance) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
-		}
-		return instance;
-	}
-
-	/**
-	 * Create a new instance
-	 *
-	 * Adds common Factorio settings to the provided instance config then
-	 * creates an instance using that config in the cluster.
-	 *
-	 * @example
-	 * let instanceConfig = new lib.InstanceConfig("controller");
-	 * instanceConfig.set("instance.name", "My instance");
-	 * let instance = await controller.instanceAssign(instanceConfig);
-	 * await controller.instanceAssign(instance.id, hostId);
-	 *
-	 * @param instanceConfig -
-	 *     Config to base newly created instance on.
-	 * @returns The created instance
-	 */
-	async instanceCreate(instanceConfig: lib.InstanceConfig): Promise<InstanceInfo> {
-		let instanceId = instanceConfig.get("instance.id");
-		if (this.instances.has(instanceId)) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} already exists`);
-		}
-
-		// Add common settings for the Factorio server
-		let settings = {
-			"name": `${this.config.get("controller.name")} - ${instanceConfig.get("instance.name")}`,
-			"description": `Clusterio instance for ${this.config.get("controller.name")}`,
-			"tags": ["clusterio"],
-			"max_players": 0,
-			"visibility": { "public": true, "lan": true },
-			"game_password": "",
-			"require_user_verification": true,
-			"max_upload_in_kilobytes_per_second": 0,
-			"max_upload_slots": 5,
-			"ignore_player_limit_for_returning_players": false,
-			"allow_commands": "admins-only",
-			"autosave_interval": 10,
-			"autosave_slots": 5,
-			"afk_autokick_interval": 0,
-			"auto_pause": false,
-			"only_admins_can_pause_the_game": true,
-			"autosave_only_on_server": true,
-			"non_blocking_saving": false,
-
-			...instanceConfig.get("factorio.settings"),
-		};
-		instanceConfig.set("factorio.settings", settings);
-
-		let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, undefined, Date.now());
-		this.instances.set(instance);
-		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance);
-		this.addInstanceHooks(instance);
-		return instance;
-	}
-
 	/**
 	 * Removes all saves currently stored for the given instance, if any
 	 * @param instanceId - Id of Instance to clear saves for.
@@ -938,84 +910,6 @@ export default class Controller {
 		this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId));
 	}
 
-	/**
-	 * Change assigned host of an instance
-	 *
-	 * Unassigns instance from existing host if already assigned to one and
-	 * then assigns it to the given host.  This is the only supported way of
-	 * changing the instance.assigned_host config entry.
-	 *
-	 * Note: this will not transfer any files or saves stored on the host
-	 * the instance was previously assgined to the new one.
-	 *
-	 * @param instanceId - ID of Instance to assign.
-	 * @param hostId - ID of host to assign instance to.
-	 */
-	async instanceAssign(instanceId: number, hostId?: number) {
-		let instance = this.getRequestInstance(instanceId);
-
-		// Check if target host is connected
-		let newHostConnection: HostConnection | undefined;
-		if (hostId !== undefined) {
-			newHostConnection = this.wsServer.hostConnections.get(hostId);
-			if (!newHostConnection) {
-				// The case of the host not getting the assign instance message
-				// still have to be handled, so it's not a requirement that the
-				// target host be connected to the controller while doing the
-				// assignment, but it is IMHO a better user experience if this
-				// is the case.
-				throw new lib.RequestError("Target host is not connected to the controller");
-			}
-		}
-
-		// Unassign from currently assigned host if it is connected.
-		let currentAssignedHost = instance.config.get("instance.assigned_host");
-		if (currentAssignedHost !== null && hostId !== currentAssignedHost) {
-			let oldHostConnection = this.wsServer.hostConnections.get(currentAssignedHost);
-			if (oldHostConnection && !oldHostConnection.connector.closing) {
-				await oldHostConnection.send(new lib.InstanceUnassignInternalRequest(instanceId));
-			}
-		}
-
-		// Remove saves recorded from currently assigned host if any
-		this.clearSavesOfInstance(instanceId);
-
-		// Assign to target
-		instance.config.set("instance.assigned_host", hostId ?? null);
-		if (hostId !== undefined && newHostConnection) {
-			await newHostConnection.send(
-				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host"))
-			);
-		} else {
-			instance.status = "unassigned";
-		}
-		this.instances.set(instance);
-	}
-
-	/**
-	 * Delete an instance
-	 *
-	 * Permanently deletes the instance from its assigned host (if assigned)
-	 * and the controller.  This action cannot be undone.
-	 *
-	 * @param instanceId - ID of instance to delete.
-	 */
-	async instanceDelete(instanceId: number) {
-		let instance = this.getRequestInstance(instanceId);
-		let hostId = instance.config.get("instance.assigned_host");
-		if (hostId !== null) {
-			await this.sendTo({ hostId }, new lib.InstanceDeleteInternalRequest(instanceId));
-		}
-
-		let prev = instance.status;
-		this.instances.delete(instance);
-
-		this.clearSavesOfInstance(instanceId);
-		this.userManager.clearStatsOfInstance(instanceId);
-
-		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance, prev);
-	}
-
 	/**
 	 * Notify the config of an instance updated
 	 *
@@ -1025,7 +919,7 @@ export default class Controller {
 	 * @param instance -
 	 *     Instance to nodify the config updated.
 	 */
-	async instanceConfigUpdated(instance: InstanceInfo) {
+	async instanceConfigUpdated(instance: InstanceRecord) {
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId !== null) {
 			let connection = this.wsServer.hostConnections.get(hostId);
@@ -1037,16 +931,7 @@ export default class Controller {
 		}
 	}
 
-	addInstanceHooks(instance: InstanceInfo) {
-		instance.config.on("fieldChanged", (field: string, curr: any, prev: any) => {
-			instance.updatedAtMs = Date.now();
-			this.instances.set(instance); // Trigger update logic
-
-			lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, field, curr, prev);
-		});
-	}
-
-	instanceDetailsUpdated(instances: InstanceInfo[]) {
+	instanceUpdated(instances: InstanceRecord[]) {
 		for (const instance of instances) {
 			if (instance.status === "stopped") {
 				this.subscriptions.unsubscribeAddress(lib.Address.fromShorthand({instanceId: instance.id}));
@@ -1098,17 +983,12 @@ export default class Controller {
 		return mods.length ? new lib.ModUpdatesEvent(mods) : null;
 	}
 
-	usersUpdated(users: ControllerUser[]) {
-		const now = Date.now();
-		for (const user of users) {
-			user.updatedAtMs = now;
-		}
-		this.userManager.dirty = true;
+	usersUpdated(users: UserRecord[]) {
 		this.subscriptions.broadcast(new lib.UserUpdatesEvent(users));
 	}
 
 	async handleUserSubscription(request: lib.SubscriptionRequest) {
-		const users = [...this.userManager.users.values()].filter(
+		const users = [...this.users.records.values()].filter(
 			user => user.updatedAtMs > request.lastRequestTimeMs,
 		);
 		return users.length ? new lib.UserUpdatesEvent(users) : null;
@@ -1119,7 +999,7 @@ export default class Controller {
 	 * permissions for this user may have changed.
 	 * @param user - User permisions updated for.
 	 */
-	userPermissionsUpdated(user: ControllerUser) {
+	userPermissionsUpdated(user: Readonly<User>) {
 		for (let controlConnection of this.wsServer.controlConnections.values()) {
 			if (controlConnection.user === user) {
 				controlConnection.send(
@@ -1195,6 +1075,9 @@ export default class Controller {
 				this.plugins.set(pluginInfo.name, controllerPlugin);
 
 			} catch (err: any) {
+				if (err.code === "InstallationError") {
+					throw err;
+				}
 				throw new lib.PluginError(pluginInfo.name, err);
 			}
 
@@ -1378,7 +1261,7 @@ export default class Controller {
 			}
 
 		} else if (dst.type === lib.Address.instance) {
-			let instance = this.getRequestInstance(dst.id);
+			const instance = this.instances.getForRequest(dst.id);
 			let hostId = instance.config.get("instance.assigned_host");
 			if (hostId === null) {
 				throw new lib.RequestError("Instance is not assigned to a host");
@@ -1413,7 +1296,7 @@ export default class Controller {
 			}
 
 		} else if (dst.type === lib.Address.instance) {
-			let instance = this.instances.get(dst.id);
+			const instance = this.instances.get(dst.id);
 			if (!instance) {
 				throw new Error(`Instance with ID ${dst.id} does not exist`);
 			}
@@ -1472,7 +1355,7 @@ export default class Controller {
 	}
 
 	async sendRequestToHostByInstanceId<Req extends InstanceId, Res>(request: Req & lib.Request<Req, Res>) {
-		let instance = this.getRequestInstance(request.instanceId);
+		let instance = this.instances.getForRequest(request.instanceId);
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId === null) {
 			throw new lib.RequestError("Instance is not assigned to a host");
@@ -1485,7 +1368,7 @@ export default class Controller {
 	}
 
 	sendEventToHostByInstanceId<T extends { instanceId: number }>(event: T & lib.Event<T>) {
-		let instance = this.getRequestInstance(event.instanceId);
+		let instance = this.instances.getForRequest(event.instanceId);
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId === null) {
 			return;

packages/controller/src/ControllerUser.ts

@@ -1,96 +0,0 @@
-import { Static, Type } from "@sinclair/typebox";
-
-import {
-	IControllerUser, PermissionError, PlayerStats,
-	Role, SubscribableDatastore, User, permissions,
-} from "@clusterio/lib";
-
-export default class ControllerUser extends User implements IControllerUser {
-	constructor(
-		private _controllerRoles: SubscribableDatastore<Role>,
-		/** Unix time in seconds the user token must be issued after to be valid.  */
-		public tokenValidAfter = 0,
-		...args: ConstructorParameters<typeof User>
-	) {
-		super(...args);
-	}
-
-	static jsonSchema = Type.Object({
-		...User.jsonSchema.properties,
-		token_valid_after: Type.Optional(Type.Number()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>, controllerRoles: SubscribableDatastore<Role>) {
-		const roleIds = new Set<number>(json.roles ?? []);
-		const instanceStats = new Map(
-			(json.instance_stats ?? []).map(
-				([id, stats]) => [id, PlayerStats.fromJSON(stats)]
-			)
-		);
-		const playerStats = User._calculatePlayerStats(instanceStats);
-		return new this(
-			controllerRoles,
-			json.token_valid_after,
-			json.name,
-			roleIds,
-			new Set(json.instances),
-			json.is_admin,
-			json.is_banned,
-			json.is_whitelisted,
-			json.ban_reason,
-			json.updated_at_ms,
-			json.is_deleted,
-			playerStats,
-			instanceStats,
-		);
-	}
-
-	toJSON(controller?: boolean): Static<typeof ControllerUser.jsonSchema> {
-		if (!controller) {
-			return super.toJSON();
-		}
-
-		const json: Static<typeof ControllerUser.jsonSchema> = super.toJSON();
-		if (this.tokenValidAfter) {
-			json.token_valid_after = this.tokenValidAfter;
-		}
-
-		return json;
-	}
-
-	get roles(): ReadonlySet<Readonly<Role>> {
-		return new Set([...this.roleIds].map(
-			id => this._controllerRoles.get(id)
-		).filter(
-			(r): r is Role => Boolean(r)
-		));
-	}
-
-	/**
-	 * Invalidate current tokens for the user
-	 *
-	 * Sets the tokenValidAfter property to the current time, which causes
-	 * all currently issued tokens for the user to become invalid.
-	 */
-	invalidateToken() {
-		this.tokenValidAfter = Math.floor(Date.now() / 1000);
-	}
-
-	checkPermission(permission: string) {
-		if (!permissions.has(permission)) {
-			throw new Error(`permission ${permission} does not exist`);
-		}
-
-		for (let roleId of this.roleIds) {
-			let role = this._controllerRoles.get(roleId);
-			if (!role) {
-				continue;
-			}
-			if (role.permissions.has("core.admin") || role.permissions.has(permission)) {
-				return;
-			}
-		}
-
-		throw new PermissionError("Permission denied");
-	}
-}

packages/controller/src/HostConnection.ts

@@ -5,8 +5,8 @@ import * as lib from "@clusterio/lib";
 const { logger } = lib;
 
 import BaseConnection from "./BaseConnection";
-import InstanceInfo from "./InstanceInfo";
-import HostInfo from "./HostInfo";
+import InstanceRecord from "./InstanceRecord";
+import HostRecord from "./HostRecord";
 
 
 /**
@@ -16,7 +16,7 @@ import HostInfo from "./HostInfo";
  * @alias module:controller/src/HostConnection
  */
 export default class HostConnection extends BaseConnection {
-	private info: HostInfo;
+	private info: HostRecord;
 	plugins: Map<string, string>;
 
 	constructor(
@@ -30,7 +30,7 @@ export default class HostConnection extends BaseConnection {
 		this.plugins = new Map(Object.entries(registerData.plugins));
 
 		const previousHostInfo = this._controller.hosts.get(registerData.id);
-		this.info = new HostInfo(
+		this.info = new HostRecord(
 			registerData.id,
 			previousHostInfo?.name ?? "",
 			registerData.version,
@@ -57,7 +57,7 @@ export default class HostConnection extends BaseConnection {
 
 		this.connector.on("close", () => {
 			// Update status to unknown for instances on this host.
-			const instances: InstanceInfo[] = [];
+			const instances: InstanceRecord[] = [];
 			for (let instance of this._controller.instances.valuesMutable()) {
 				if (instance.config.get("instance.assigned_host") !== this.id) {
 					continue;
@@ -68,7 +68,7 @@ export default class HostConnection extends BaseConnection {
 				instance.status = "unknown";
 				lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 			}
-			this._controller.instances.setMany(instances);
+			this._controller.instances.records.setMany(instances);
 
 			this.info.connected = false;
 			this._controller.hosts.set(this.info);
@@ -207,9 +207,14 @@ export default class HostConnection extends BaseConnection {
 
 		let prev = instance.status;
 		instance.status = request.status;
+		// If the instance has transitioned to running then updated the started at time
+		if (request.status === "running" && prev === "starting") {
+			instance.startedAtMs = Date.now();
+		}
+
 		instance.gamePort = request.gamePort;
 		instance.factorioVersion = request.factorioVersion;
-		this._controller.instances.set(instance);
+		this._controller.instances.records.set(instance);
 		logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
 		await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 	}
@@ -225,7 +230,7 @@ export default class HostConnection extends BaseConnection {
 		}
 
 		// Assign instances the host has but controller does not
-		const instanceUpdates: InstanceInfo[] = [];
+		const instanceUpdates: InstanceRecord[] = [];
 		for (let instanceData of request.instances) {
 			const instanceConfig = new lib.InstanceConfig("controller");
 			instanceConfig.update(instanceData.config, false, "host");
@@ -246,6 +251,9 @@ export default class HostConnection extends BaseConnection {
 					controllerInstance.status = instanceData.status;
 					controllerInstance.gamePort = instanceData.gamePort;
 					controllerInstance.factorioVersion = instanceData.factorioVersion;
+					if (instanceData.status === "running" && prev === "starting") {
+						controllerInstance.startedAtMs = Date.now();
+					}
 					instanceUpdates.push(controllerInstance);
 					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
 					await lib.invokeHook(
@@ -256,31 +264,30 @@ export default class HostConnection extends BaseConnection {
 			}
 
 			instanceConfig.set("instance.assigned_host", this.id);
-			let newInstance = new InstanceInfo(
-				instanceConfig,
-				instanceData.status,
-				instanceData.gamePort,
-				instanceData.factorioVersion,
-				Date.now(),
-			);
+			const newInstance = await this._controller.instances.createInstance(instanceConfig, true);
+			newInstance.factorioVersion = instanceData.factorioVersion;
+			newInstance.gamePort = instanceData.gamePort;
+			newInstance.status = instanceData.status;
 			instanceUpdates.push(newInstance);
-			this._controller.addInstanceHooks(newInstance);
-			await this.send(
-				new lib.InstanceAssignInternalRequest(
-					instanceConfig.get("instance.id"), instanceConfig.toRemote("host")
-				)
-			);
-			await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance);
 		}
 
-		this._controller.instances.setMany(instanceUpdates);
+		// We suppress the changes made by createInstance so we must manually make them here
+		this._controller.instances.records.setMany(instanceUpdates);
+		await Promise.all(instanceUpdates.flatMap(newInstance => [
+			lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance),
+			this.send(
+				new lib.InstanceAssignInternalRequest(
+					newInstance.config.get("instance.id"), newInstance.config.toRemote("host")
+				)
+			),
+		]));
 
 		// Push lists to make sure they are in sync.
 		let adminlist: Set<string> = new Set();
 		let banlist: Map<string, string> = new Map();
 		let whitelist: Set<string> = new Set();
 
-		for (let user of this._controller.userManager.users.values()) {
+		for (let user of this._controller.users.values()) {
 			if (user.isAdmin) {
 				adminlist.add(user.id);
 			}
@@ -329,21 +336,18 @@ export default class HostConnection extends BaseConnection {
 
 	async handleInstancePlayerUpdateEvent(event: lib.InstancePlayerUpdateEvent, src: lib.Address) {
 		let instanceId = src.id;
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
+		const user = this._controller.users.getOrCreateUser(event.name);
 
 		if (event.type === "join") {
-			this._controller.userManager.notifyJoin(user, instanceId);
+			user.notifyJoin(instanceId);
 		} else if (event.type === "leave") {
-			this._controller.userManager.notifyLeave(user, instanceId);
+			user.notifyLeave(instanceId);
 		}
 
 		user.instanceStats.set(instanceId, event.stats);
 
 		user.recalculatePlayerStats();
-		this._controller.usersUpdated([user]);
+		user.saveRecord();
 
 		let instance = this._controller.instances.get(instanceId)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
@@ -359,13 +363,9 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
+		const user = this._controller.users.getOrCreateUser(event.name);
+		user.set("isAdmin", event.admin);
 
-		user.isAdmin = event.admin;
-		this._controller.usersUpdated([user]);
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.admin ? "promote" : "demote" as lib.PlayerEvent["type"],
@@ -379,14 +379,11 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
-
+		const user = this._controller.users.getOrCreateUser(event.name);
 		user.isBanned = event.banned;
 		user.banReason = event.reason;
-		this._controller.usersUpdated([user]);
+		user.saveRecord();
+
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.banned ? "ban" : "unban" as lib.PlayerEvent["type"],
@@ -401,13 +398,9 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
+		const user = this._controller.users.getOrCreateUser(event.name);
+		user.set("isWhitelisted", event.whitelisted);
 
-		user.isWhitelisted = event.whitelisted;
-		this._controller.usersUpdated([user]);
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.whitelisted ? "whitelisted" : "unwhitelisted" as lib.PlayerEvent["type"],

packages/controller/src/HostRecord.ts

@@ -1,10 +1,8 @@
 import { Type, Static } from "@sinclair/typebox";
 import { HostDetails } from "@clusterio/lib";
 
-/**
- * Info about a host known to the controller.
- */
-export default class HostInfo {
+/** Underlying data class for hosts on the controller */
+export default class HostRecord {
 	constructor(
 		/** Id of this host */
 		public id: number,
@@ -33,12 +31,12 @@ export default class HostInfo {
 		"name": Type.String(),
 		"version": Type.String(),
 		"plugins": Type.Record(Type.String(), Type.String()),
-		"connected": Type.Boolean(),
-		"remote_address": Type.String(),
-		"public_address": Type.String(),
-		"token_valid_after": Type.Number(),
-		"updated_at_ms": Type.Number(),
-		"is_deleted": Type.Boolean(),
+		"connected": Type.Optional(Type.Boolean()),
+		"remote_address": Type.Optional(Type.String()),
+		"public_address": Type.Optional(Type.String()),
+		"token_valid_after": Type.Optional(Type.Number()),
+		"updated_at_ms": Type.Optional(Type.Number()),
+		"is_deleted": Type.Optional(Type.Boolean()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
@@ -56,19 +54,50 @@ export default class HostInfo {
 		);
 	}
 
-	toJSON(): Static<typeof HostInfo.jsonSchema> {
-		return {
+	toJSON() {
+		const json = {
 			id: this.id,
 			name: this.name,
 			version: this.version,
 			plugins: Object.fromEntries(this.plugins),
-			connected: this.connected,
-			remote_address: this.remoteAddress,
-			public_address: this.publicAddress,
-			token_valid_after: this.tokenValidAfter,
-			updated_at_ms: this.updatedAtMs,
-			is_deleted: this.isDeleted,
-		};
+		} as Static<typeof HostRecord.jsonSchema>;
+
+		if (this.connected !== false) {
+			json.connected = this.connected;
+		}
+		if (this.remoteAddress !== "") {
+			json.remote_address = this.remoteAddress;
+		}
+		if (this.publicAddress !== "") {
+			json.public_address = this.publicAddress;
+		}
+		if (this.tokenValidAfter !== 0) {
+			json.token_valid_after = this.tokenValidAfter;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updated_at_ms = this.updatedAtMs;
+		}
+		if (this.isDeleted !== false) {
+			json.is_deleted = this.isDeleted;
+		}
+
+		return json;
+	}
+
+
+	static fromHostDetails(details: HostDetails, plugins: Map<string, string>) {
+		return new HostRecord(
+			details.id,
+			details.name,
+			details.version,
+			plugins,
+			details.connected,
+			details.remoteAddress,
+			details.publicAddress,
+			details.tokenValidAfter,
+			details.updatedAtMs,
+			details.isDeleted,
+		);
 	}
 
 	toHostDetails() {
@@ -85,4 +114,3 @@ export default class HostInfo {
 		);
 	}
 }
-

packages/controller/src/InstanceManager.ts

@@ -0,0 +1,274 @@
+import * as lib from "@clusterio/lib";
+
+import type Controller from "./Controller";
+import type HostConnection from "./HostConnection";
+import InstanceRecord from "./InstanceRecord";
+
+export default class InstanceManager {
+	private static readonly DefaultFactorioSettings = {
+		tags: ["clusterio"],
+		max_players: 0,
+		visibility: { public: true, lan: true },
+		game_password: "",
+		require_user_verification: true,
+		max_upload_in_kilobytes_per_second: 0,
+		max_upload_slots: 5,
+		ignore_player_limit_for_returning_players: false,
+		allow_commands: "admins-only",
+		autosave_interval: 10,
+		autosave_slots: 5,
+		afk_autokick_interval: 0,
+		auto_pause: false,
+		only_admins_can_pause_the_game: true,
+		autosave_only_on_server: true,
+		non_blocking_saving: false,
+	} as const;
+
+	constructor(
+		public readonly records: lib.SubscribableDatastore<InstanceRecord>,
+		private readonly _controller: Controller,
+	) {
+		for (const instance of records.values()) {
+			this.addInstanceHooks(instance);
+		}
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Basic Accessors                                                           */
+	/* ------------------------------------------------------------------------- */
+
+	has(id: InstanceRecord["id"]): boolean {
+		return this.records.has(id);
+	}
+
+	getMutable(id: InstanceRecord["id"]): InstanceRecord | undefined {
+		return this.records.get(id);
+	}
+
+	get(id: InstanceRecord["id"]): Readonly<InstanceRecord> | undefined {
+		return this.getMutable(id);
+	}
+
+	valuesMutable(): IterableIterator<InstanceRecord> {
+		return this.records.values();
+	}
+
+	values(): IterableIterator<Readonly<InstanceRecord>> {
+		return this.valuesMutable();
+	}
+
+	/**
+	 * Get instance by ID for a request.
+	 *
+	 * Convenience wrapper around {@link getMutable} that throws
+	 * if the instance does not exist.
+	 *
+	 * @param instanceId - ID of instance to get.
+	 * @returns The instance record.
+	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 */
+	getForRequest(instanceId: number): InstanceRecord {
+		const instance = this.records.get(instanceId);
+		if (!instance) {
+			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
+		}
+		return instance;
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Internal Helpers                                                          */
+	/* ------------------------------------------------------------------------- */
+
+	private async _notifyStatusChanged(
+		instance: InstanceRecord,
+		prev?: lib.InstanceStatus,
+	): Promise<void> {
+		await lib.invokeHook(
+			this._controller.plugins,
+			"onInstanceStatusChanged",
+			instance,
+			prev,
+		);
+	}
+
+	private async _notifyConfigFieldChanged(
+		instance: InstanceRecord,
+		field: string,
+		curr: unknown,
+		prev: unknown,
+	): Promise<void> {
+		await lib.invokeHook(
+			this._controller.plugins,
+			"onInstanceConfigFieldChanged",
+			instance,
+			field,
+			curr,
+			prev,
+		);
+	}
+
+	/**
+	 * Attach change listeners to an instance record.
+	 *
+	 * Hooks into configuration field changes and ensures the datastore
+	 * is updated and plugins are notified when a config field changes.
+	 *
+	 * @param instance - The instance record to attach hooks to.
+	 * @internal
+	 */
+	addInstanceHooks(instance: InstanceRecord): void {
+		instance.config.on("fieldChanged", (field: string, curr: unknown, prev: unknown) => {
+			this.records.set(instance);
+			this._notifyConfigFieldChanged(instance, field, curr, prev);
+		});
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Lifecycle                                                                 */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Create a new instance.
+	 *
+	 * @example
+	 * const config = new lib.InstanceConfig("controller", { "instance.name": "My instance" });
+	 * const instance = await controller.instances.createInstance(config);
+	 *
+	 * @param instanceConfig - Config to base the newly created instance on.
+	 * @param _suppressChanges - When true, side effects are suppressed, caller takes ownership.
+	 * @returns The created instance record.
+	 * @throws {module:lib.RequestError} if an instance with the same ID already exists.
+	 */
+	async createInstance(
+		instanceConfig: lib.InstanceConfig,
+		_suppressChanges?: boolean,
+	): Promise<InstanceRecord> {
+		const instanceId = instanceConfig.get("instance.id");
+		if (this.records.has(instanceId)) {
+			throw new lib.RequestError(`Instance with ID ${instanceId} already exists`);
+		}
+
+		const controllerName = this._controller.config.get("controller.name");
+		instanceConfig.set("factorio.settings", {
+			name: `${controllerName} - ${instanceConfig.get("instance.name")}`,
+			description: `Clusterio instance for ${controllerName}`,
+			...InstanceManager.DefaultFactorioSettings,
+			...instanceConfig.get("factorio.settings"),
+		});
+
+		const instance = new InstanceRecord(instanceConfig, "unassigned");
+		this.addInstanceHooks(instance);
+
+		if (!_suppressChanges) {
+			this.records.set(instance);
+			await this._notifyStatusChanged(instance);
+		}
+
+		return instance;
+	}
+
+	/**
+	 * Change the assigned host of an instance.
+	 *
+	 * Unassigns the instance from the currently assigned host (if connected)
+	 * and assigns it to the provided host.
+	 *
+	 * This is the only supported way of modifying the
+	 * `instance.assigned_host` configuration entry.
+	 *
+	 * Note:
+	 * - This does NOT transfer any files or saves between hosts.
+	 * - If the instance is already assigned to the specified host, no action is taken.
+	 *
+	 * @param instanceId - ID of instance to assign.
+	 * @param hostId - ID of host to assign the instance to. If undefined,
+	 * the instance will be unassigned.
+	 *
+	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 * @throws {module:lib.RequestError} if the target host is not connected.
+	 */
+	async assignInstance(
+		instanceId: number,
+		hostId: number | null,
+	): Promise<void> {
+		const instance = this.getForRequest(instanceId);
+		const hostConnections = this._controller.wsServer.hostConnections;
+
+		const currentAssignedHost = instance.config.get("instance.assigned_host");
+		if (currentAssignedHost === hostId) {
+			return;
+		}
+
+		let newHostConnection: HostConnection | undefined;
+		if (hostId !== null) {
+			newHostConnection = hostConnections.get(hostId);
+			if (!newHostConnection) {
+				throw new lib.RequestError("Target host is not connected to the controller");
+			}
+		}
+
+		if (currentAssignedHost !== null) {
+			const oldHostConnection = hostConnections.get(currentAssignedHost);
+			if (oldHostConnection && !oldHostConnection.connector.closing) {
+				await oldHostConnection.send(
+					new lib.InstanceUnassignInternalRequest(instanceId),
+				);
+			}
+		}
+
+		this._controller.clearSavesOfInstance(instanceId);
+		instance.config.set("instance.assigned_host", hostId);
+
+		if (newHostConnection) {
+			await newHostConnection.send(
+				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host")),
+			);
+		} else {
+			instance.status = "unassigned";
+		}
+
+		this.records.set(instance);
+	}
+
+	/**
+	 * Unassign an instance from its current host.
+	 *
+	 * Convenience wrapper around {@link assignInstance} that removes
+	 * the assigned host.
+	 *
+	 * @param instanceId - ID of instance to unassign.
+	 */
+	async unassignInstance(instanceId: number): Promise<void> {
+		return this.assignInstance(instanceId, null);
+	}
+
+	/**
+	 * Delete an instance permanently.
+	 *
+	 * Sends a delete request to the assigned host (if any),
+	 * removes the instance from the datastore,
+	 * clears associated saves and user statistics.
+	 *
+	 * @param instanceId - ID of instance to delete.
+	 *
+	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 */
+	async deleteInstance(instanceId: number): Promise<void> {
+		const instance = this.getForRequest(instanceId);
+		const hostId = instance.config.get("instance.assigned_host");
+
+		if (hostId !== null) {
+			await this._controller.sendTo(
+				{ hostId },
+				new lib.InstanceDeleteInternalRequest(instanceId),
+			);
+		}
+
+		const prevStatus = instance.status;
+		this.records.delete(instance);
+		this._controller.clearSavesOfInstance(instanceId);
+		this._controller.users.clearStatsOfInstance(instanceId);
+
+		await this._notifyStatusChanged(instance, prevStatus);
+	}
+}

packages/controller/src/InstanceRecord.ts

@@ -1,52 +1,22 @@
 import * as lib from "@clusterio/lib";
 import { Static, Type } from "@sinclair/typebox";
 
-
-/**
- * Runtime status of an instance on the controller
- * @alias module:controller/src/InstanceInfo
- */
-export default class InstanceInfo {
+/** Underlying data class for instances on the controller */
+export default class InstanceRecord {
 	constructor(
 		public config: lib.InstanceConfig,
 		public status: lib.InstanceStatus,
 		public gamePort?: number,
 		public factorioVersion?: lib.TargetVersion,
+		public startedAtMs = 0,
 		public updatedAtMs = 0,
 	) {
 		this.config = config;
 		this.status = status;
 	}
 
-	static jsonSchema = Type.Object({
-		"config": lib.InstanceConfig.jsonSchema,
-		"status": lib.InstanceStatus,
-		"gamePort": Type.Optional(Type.Number()),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"factorioVersion": Type.Optional(lib.TargetVersionSchema),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>, location: lib.ConfigLocation) {
-		return new this(
-			lib.InstanceConfig.fromJSON(json.config, location),
-			json.status,
-			json.gamePort,
-			json.factorioVersion,
-			json.updatedAtMs,
-		);
-	}
-
-	toInstanceDetails() {
-		return new lib.InstanceDetails(
-			this.config.get("instance.name"),
-			this.id,
-			this.config.get("instance.assigned_host") ?? undefined,
-			this.gamePort,
-			this.status,
-			this.factorioVersion,
-			this.updatedAtMs,
-			this.config.get("instance.exclude_from_start_all"),
-		);
+	get id(): number {
+		return this.config.get("instance.id");
 	}
 
 	get isDeleted() {
@@ -60,8 +30,70 @@ export default class InstanceInfo {
 		this.status = "deleted";
 	}
 
-	/** Shorthand for `instance.config.get("instance.id")` */
-	get id():number {
-		return this.config.get("instance.id");
+	static jsonSchema = Type.Object({
+		"config": lib.InstanceConfig.jsonSchema,
+		"status": lib.InstanceStatus,
+		"gamePort": Type.Optional(Type.Number()),
+		"startedAtMs": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"factorioVersion": Type.Optional(lib.TargetVersionSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			lib.InstanceConfig.fromJSON(json.config, "controller"),
+			json.status,
+			json.gamePort,
+			json.factorioVersion,
+			json.startedAtMs,
+			json.updatedAtMs,
+		);
+	}
+
+	toJSON(): Static<typeof InstanceRecord.jsonSchema> {
+		const json: any = {
+			config: this.config.toJSON(),
+			status: this.status,
+		};
+
+		if (this.gamePort !== undefined) {
+			json.gamePort = this.gamePort;
+		}
+		if (this.factorioVersion !== undefined) {
+			json.factorioVersion = this.factorioVersion;
+		}
+		if (this.startedAtMs !== 0) {
+			json.startedAtMs = this.startedAtMs;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updatedAtMs = this.updatedAtMs;
+		}
+
+		return json;
+	}
+
+	toInstanceDetails() {
+		return new lib.InstanceDetails(
+			this.config.get("instance.name"),
+			this.id,
+			this.config.get("instance.assigned_host") ?? undefined,
+			this.gamePort,
+			this.status,
+			this.factorioVersion,
+			this.startedAtMs,
+			this.updatedAtMs,
+			this.config.get("instance.exclude_from_start_all"),
+		);
+	}
+
+	static fromInstanceDetails(details: lib.InstanceDetails, config: lib.InstanceConfig) {
+		return new InstanceRecord(
+			config,
+			details.status,
+			details.gamePort,
+			details.factorioVersion,
+			details.startedAtMs,
+			details.updatedAtMs,
+		);
 	}
 }

packages/controller/src/User.ts

@@ -0,0 +1,146 @@
+
+import {
+	IUser, Role, SubscribableDatastore,
+	PermissionError, permissions,
+} from "@clusterio/lib";
+
+import UserRecord from "./UserRecord";
+import { Static } from "@sinclair/typebox";
+
+/**
+ * Represents a user as viewed from the controller.
+ *
+ * All methods are safe to use when readonly.
+ * Only use mutable when needing to change multiple values.
+ */
+export default class User extends UserRecord implements IUser {
+	constructor(
+		private _controllerUsers: SubscribableDatastore<UserRecord>,
+		private _controllerRoles: SubscribableDatastore<Role>,
+		...args: ConstructorParameters<typeof UserRecord>
+	) {
+		super(...args);
+	}
+
+	static fromJSON(
+		json: Static<typeof this.jsonSchema>,
+		_controllerUsers: SubscribableDatastore<UserRecord>,
+		_controllerRoles: SubscribableDatastore<Role>,
+	) {
+		return this.fromUserRecord(
+			UserRecord.fromJSON(json),
+			_controllerUsers,
+			_controllerRoles,
+		);
+	}
+
+	static fromUserRecord(
+		userRecord: UserRecord,
+		_controllerUsers: SubscribableDatastore<UserRecord>,
+		_controllerRoles: SubscribableDatastore<Role>,
+	): User {
+		return new this(
+			_controllerUsers,
+			_controllerRoles,
+			userRecord.tokenValidAfter,
+			userRecord.name,
+			userRecord.roleIds,
+			userRecord.instances,
+			userRecord.isAdmin,
+			userRecord.isBanned,
+			userRecord.isWhitelisted,
+			userRecord.banReason,
+			userRecord.updatedAtMs,
+			userRecord.isDeleted,
+			userRecord.instanceStats,
+		);
+	}
+
+	/** Roles this user has */
+	get roles(): ReadonlySet<Readonly<Role>> {
+		return new Set([...this.roleIds].map(
+			id => this._controllerRoles.get(id)
+		).filter(
+			(r): r is Role => Boolean(r)
+		));
+	}
+
+	/** Save the underlying user record */
+	saveRecord() {
+		this._controllerUsers.set(this);
+	}
+
+	/** Set a property on the underlying record, then save it. */
+	set<K extends keyof this>(key: K, value: this[K]) {
+		this[key] = value;
+		this.saveRecord();
+	}
+
+	/**
+	 * Invalidate current tokens for the user
+	 *
+	 * Sets the tokenValidAfter property to the current time, which causes
+	 * all currently issued tokens for the user to become invalid.
+	 */
+	invalidateToken() {
+		this.tokenValidAfter = Math.floor(Date.now() / 1000);
+		this.saveRecord();
+	}
+
+	/** Add a role to this user */
+	addRole(roleId: number) {
+		const role = this._controllerRoles.get(roleId);
+		if (!role) {
+			throw new Error(`role ${roleId} does not exist`);
+		}
+		this.roleIds.add(roleId);
+		this.saveRecord();
+	}
+
+	/** Remove a role from user */
+	removeRole(roleId: number) {
+		if (!this.roleIds.delete(roleId)) {
+			return false;
+		}
+		this.saveRecord();
+		return true;
+	}
+
+	/** Set a user as being online on a specific instance */
+	notifyJoin(instanceId: number) {
+		this.instances.add(instanceId);
+		this.saveRecord();
+	}
+
+	/** Set a user as being offline on a specific instance */
+	notifyLeave(instanceId: number) {
+		this.instances.delete(instanceId);
+		this.saveRecord();
+	}
+
+	/** Clear the stats of a player on a specific instance */
+	clearInstanceStats(instanceId: number) {
+		this.instances.delete(instanceId);
+		this.instanceStats.delete(instanceId);
+		this.recalculatePlayerStats();
+		this.saveRecord();
+	}
+
+	checkPermission(permission: string) {
+		if (!permissions.has(permission)) {
+			throw new Error(`permission ${permission} does not exist`);
+		}
+
+		for (const roleId of this.roleIds) {
+			const role = this._controllerRoles.get(roleId);
+			if (!role) {
+				continue;
+			}
+			if (role.permissions.has("core.admin") || role.permissions.has(permission)) {
+				return;
+			}
+		}
+
+		throw new PermissionError("Permission denied");
+	}
+}

packages/controller/src/UserManager.ts

@@ -3,65 +3,45 @@ import jwt from "jsonwebtoken";
 
 import * as lib from "@clusterio/lib";
 
-import ControllerUser from "./ControllerUser";
+import User from "./User";
+import UserRecord from "./UserRecord";
 
 /**
- * Manages users and roles
+ * Access layer for users on the controller.
+ *
+ * In performance critical paths the raw records can be accessed directly.
+ * Otherwise this class will wrap data in a view class with a set and save method.
+ *
  * @alias module:controller/src/UserManager
  */
 export default class UserManager {
-	users: Map<ControllerUser["id"], ControllerUser> = new Map();
-	dirty = false;
-
-	/**
-	 * Set of users currently online in the cluster.
-	 */
-	onlineUsers = new Set<ControllerUser>();
-
 	constructor(
-		private _config: lib.ControllerConfig,
-		private _roles: lib.SubscribableDatastore<lib.Role>,
+		public records: lib.SubscribableDatastore<UserRecord>,
+		private _controllerRoles: lib.SubscribableDatastore<lib.Role>,
+		private _controllerConfig: lib.ControllerConfig,
 	) {
 	}
 
-	getByName(name: string) {
-		return this.users.get(name.toLowerCase());
-	}
-
-	async load(filePath: string): Promise<void> {
+	/**
+	 * Prior to 2.0.0-alpha.22 users and roles were stored in a single file.
+	 * This migration separates them so they can be loaded by datastore.
+	 *
+	 * @param usersFilePath The src and dst file path for the users json file
+	 * @param rolesFilePath The dst for the roles json file
+	 */
+	static async attemptMigrateLegacyUsersFile(usersFilePath: string, rolesFilePath: string) {
 		try {
-			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-			if ("roles" in content) { // Split file format after 2.0.0-alpha.22
-				this._roles.setMany(content.roles.map((serializedRole: any) => lib.Role.fromJSON(serializedRole)));
-				await this._roles.save(); // Force a save to prevent data loss
-				content = content.users;
+			const combinedJson = JSON.parse(await fs.readFile(usersFilePath, { encoding: "utf8" }));
+			if (!("roles" in combinedJson)) {
+				return; // Files are already split, no action required
 			}
-
-			let duplicates = 0;
-			for (const serializedUser of content) {
-				const user = ControllerUser.fromJSON(serializedUser, this._roles);
-				const existingUser = this.users.get(user.id);
-				if (existingUser) {
-					// Required migration to all lowercase ids in alpha 19
-					duplicates += 1;
-					// We assume the user with the lower online time is the duplicate
-					if (user.playerStats.onlineTimeMs <= existingUser.playerStats.onlineTimeMs) {
-						existingUser.merge(user);
-						continue; // Skip users.set
-					}
-					user.merge(existingUser);
-				}
-				this.users.set(user.id, user);
-			}
-
-			if (duplicates) {
-				const backupPath = `${filePath}.${Date.now()}.bak`;
-				lib.logger.warn(
-					`A total of ${duplicates} users were merged, a backup was written to: ${backupPath}`
-				);
-				await lib.safeOutputFile(backupPath, JSON.stringify(content, null, "\t"));
+			if (!("users" in combinedJson)) {
+				throw new Error("Legacy users json does not contain users property");
 			}
 
+			// We intentionally write the roles first because if it fails then the old user file will still contain them
+			await lib.safeOutputFile(rolesFilePath, JSON.stringify(combinedJson.roles));
+			await lib.safeOutputFile(usersFilePath, JSON.stringify(combinedJson.users));
 		} catch (err: any) {
 			if (err.code !== "ENOENT") {
 				throw err;
@@ -69,66 +49,116 @@ export default class UserManager {
 		}
 	}
 
-	async save(filePath: string): Promise<void> {
-		this.dirty = false;
-		await lib.safeOutputFile(filePath,
-			JSON.stringify([...this.users.values()].map(user => user.toJSON(true)), null, "\t")
+	/* ------------------------------------------------------------------------- */
+	/* Basic Accessors                                                           */
+	/* ------------------------------------------------------------------------- */
+
+	has(id: UserRecord["id"]) {
+		return this.records.has(id);
+	}
+
+	getMutable(id: UserRecord["id"]) {
+		const userRecord = this.records.getMutable(id);
+		if (!userRecord) {
+			return undefined;
+		}
+
+		return User.fromUserRecord(
+			userRecord,
+			this.records,
+			this._controllerRoles,
 		);
 	}
 
+	get(id: UserRecord["id"]): Readonly<User> | undefined {
+		return this.getMutable(id);
+	}
+
+	hasName(name: UserRecord["name"]) {
+		return this.has(name.toLowerCase());
+	}
+
+	getByNameMutable(name: UserRecord["name"]) {
+		return this.getMutable(name.toLowerCase());
+	}
+
+	getByName(name: UserRecord["name"]) {
+		return this.get(name.toLowerCase());
+	}
+
+	* valuesMutable(): IterableIterator<User> {
+		const roles = this._controllerRoles;
+		const users = this.records;
+
+		for (const record of users.values()) {
+			yield User.fromUserRecord(record, users, roles);
+		}
+	}
+
+	values(): IterableIterator<Readonly<User>> {
+		return this.valuesMutable();
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Lifecycle                                                                 */
+	/* ------------------------------------------------------------------------- */
+
 	/**
 	 * Creates a new user and add it to the user database.
 	 * @param name - Name of the user to create.
 	 * @returns The created user.
 	 */
-	createUser(name: string): ControllerUser {
+	createUser(name: string): User {
 		if (this.getByName(name)) {
 			throw new Error(`User '${name}' already exists`);
 		}
 
 		let roles = new Set<number>();
-		let defaultRoleId = this._config.get("controller.default_role_id");
-		if (defaultRoleId !== null && this._roles.has(defaultRoleId)) {
+		let defaultRoleId = this._controllerConfig.get("controller.default_role_id");
+		if (defaultRoleId !== null && this._controllerRoles.has(defaultRoleId)) {
 			roles.add(defaultRoleId);
 		}
 
-		let user = new ControllerUser(this._roles, 0, name, roles);
-		this.users.set(user.id, user);
-		this.dirty = true;
+		const user = new User(this.records, this._controllerRoles, 0, name, roles);
+		this.records.set(user);
 		return user;
 	}
 
+	/** Get or create a user with a given name. */
+	getOrCreateUser(name: string): User {
+		return this.getByNameMutable(name) ?? this.createUser(name);
+	}
+
+	/**
+	 * Deletes an existing user
+	 *
+	 * @param user - user to delete
+	 */
+	deleteUser(user: UserRecord) {
+		return this.records.delete(user);
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Helpers                                                                   */
+	/* ------------------------------------------------------------------------- */
+
 	/**
 	 * Sign access token for the given user
 	 *
 	 * @param user - user to sign token for
 	 * @returns JWT access token for the user.
 	 */
-	signUserToken(user: ControllerUser): string {
+	signUserToken(user: UserRecord): string {
 		return jwt.sign(
 			{ aud: "user", user: user.id },
-			Buffer.from(this._config.get("controller.auth_secret"), "base64")
+			Buffer.from(this._controllerConfig.get("controller.auth_secret"), "base64")
 		);
 	}
 
-	notifyJoin(user: ControllerUser, instance_id: number) {
-		user.instances.add(instance_id);
-		this.onlineUsers.add(user);
-	}
-
-	notifyLeave(user: ControllerUser, instance_id: number) {
-		user.instances.delete(instance_id);
-		if (!user.instances.size) {
-			this.onlineUsers.delete(user);
-		}
-	}
-
+	/** Clear all player stats for specific instance */
 	clearStatsOfInstance(instanceId: number) {
-		for (const user of this.users.values()) {
-			this.notifyLeave(user, instanceId);
-			user.instanceStats.delete(instanceId);
-			user.recalculatePlayerStats();
+		for (const user of this.values()) {
+			user.clearInstanceStats(instanceId);
 		}
-		this.dirty = true;
 	}
 }

packages/controller/src/UserRecord.ts

@@ -0,0 +1,102 @@
+import { UserDetails } from "@clusterio/lib";
+import { Type, Static } from "@sinclair/typebox";
+
+/** Underlying data class for users on the controller */
+export default class UserRecord extends UserDetails {
+	constructor(
+		/** Unix time in seconds the user token must be issued after to be valid.  */
+		public tokenValidAfter = 0,
+		...args: ConstructorParameters<typeof UserDetails>
+	) {
+		super(...args);
+	}
+
+	static jsonSchema = Type.Object({
+		token_valid_after: Type.Optional(Type.Number()),
+		...UserDetails.jsonSchema.properties,
+	});
+
+	// _a and _b are a ts workaround for allowing User to have extra arguments
+	static fromJSON(json: Static<typeof this.jsonSchema>, _a?: any, _b?: any) {
+		return this.fromUserDetails(
+			UserDetails.fromJSON(json),
+			json.token_valid_after,
+		);
+	}
+
+	toJSON() {
+		const json = super.toJSON() as Static<typeof UserRecord.jsonSchema>;
+
+		if (this.tokenValidAfter > 0) {
+			json.token_valid_after = this.tokenValidAfter;
+		}
+
+		return json;
+	}
+
+	static fromUserDetails(
+		userDetails: UserDetails,
+		tokenValidAfter = 0,
+	) {
+		return new this(
+			tokenValidAfter,
+			userDetails.name,
+			userDetails.roleIds,
+			userDetails.instances,
+			userDetails.isAdmin,
+			userDetails.isBanned,
+			userDetails.isWhitelisted,
+			userDetails.banReason,
+			userDetails.updatedAtMs,
+			userDetails.isDeleted,
+			userDetails.instanceStats,
+		);
+	}
+
+	toUserDetails() {
+		return new UserDetails(
+			this.name,
+			this.roleIds,
+			this.instances,
+			this.isAdmin,
+			this.isBanned,
+			this.isWhitelisted,
+			this.banReason,
+			this.updatedAtMs,
+			this.isDeleted,
+			this.instanceStats,
+		);
+	}
+
+	/**
+	 * Merge another user into this one, must have matching ids, user is not deleted
+	 * @param otherUser - User who's details are merged from
+	*/
+	merge(otherUser: UserDetails) {
+		if (this.id !== otherUser.id) {
+			throw new Error("Cannot merge users with different ids");
+		}
+
+		// Merge properties
+		this.roleIds = new Set([...this.roleIds, ...otherUser.roleIds]);
+		this.instances = new Set([...this.instances, ...otherUser.instances]);
+		this.isAdmin = this.isAdmin && otherUser.isAdmin; // More secure to use && rather ||
+		this.isBanned = this.isBanned || otherUser.isBanned;
+		this.isWhitelisted = this.isWhitelisted || otherUser.isWhitelisted;
+		this.banReason = this.updatedAtMs > otherUser.updatedAtMs ? this.banReason : otherUser.banReason;
+		this.updatedAtMs = Date.now();
+
+		// Merge instance stats
+		const thisInstanceStats = this.instanceStats;
+		for (const [instanceId, instanceStats] of otherUser.instanceStats.entries()) {
+			if (thisInstanceStats.has(instanceId)) {
+				thisInstanceStats.get(instanceId)!.merge(instanceStats);
+			} else {
+				thisInstanceStats.set(instanceId, instanceStats);
+			}
+		}
+
+		// Player stats must be recalculated after modifying instance stats
+		this.recalculatePlayerStats();
+	}
+}

packages/controller/src/WsServer.ts

@@ -334,7 +334,7 @@ ${err.stack}`
 				throw new Error("unexpected JsonWebToken type");
 			}
 
-			user = this.controller.userManager.getByName(tokenPayload.user);
+			user = this.controller.users.getByNameMutable(tokenPayload.user);
 			if (!user) {
 				throw new Error("invalid user");
 			}

packages/controller/src/routes.ts

@@ -189,7 +189,7 @@ function validateUserToken(req: Request, res: Response, next: any) {
 		if (typeof tokenPayload === "string") {
 			throw new Error("unexpected JsonWebToken type");
 		}
-		let user = req.app.locals.controller.userManager.getByName(tokenPayload.user);
+		let user = req.app.locals.controller.users.getByName(tokenPayload.user);
 		if (!user) {
 			throw new Error("invalid user");
 		}
@@ -240,9 +240,9 @@ async function uploadExport(req: Request, res: Response) {
 	let exportFiles = [
 		"export/settings.json",
 		"export/prototypes.json",
-		"export/item-spritesheet.png",
-		"export/item-metadata.json",
 		"export/locale.json",
+		"export/spritesheet.png",
+		"export/metadata.json",
 	];
 
 	let assets: any = {};
@@ -263,7 +263,7 @@ async function uploadExport(req: Request, res: Response) {
 		await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
 	}
 
-	modPack.exportManifest = new lib.ExportManifest(assets);
+	modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString());
 	modPack.fillDefaultSettings(settingPrototypes, logger);
 	res.app.locals.controller.modPacks.set(modPack);
 
@@ -701,5 +701,6 @@ export const webRoutes = [
 	"/roles/:id/view",
 	"/plugins",
 	"/plugins/:name/view",
+	"/icon-reference",
 	"/error",
 ];

packages/create/create.js

@@ -335,7 +335,7 @@ async function migrateRename(args) {
 		let mode = "standalone";
 		if (!hasRunSlave) { mode = "controller"; }
 		if (!hasRunMaster) { mode = "host"; }
-		await writeScripts(mode);
+		await writeScripts(mode, "");
 	}
 
 	logger.info(
@@ -420,7 +420,7 @@ async function groupIdToName(gid) {
 	}
 }
 
-async function writeScripts(mode) {
+async function writeScripts(mode, selfSignedCert) {
 	if (["standalone", "controller"].includes(mode)) {
 		if (process.platform === "win32") {
 			await safeOutputFile(
@@ -474,12 +474,13 @@ WantedBy=multi-user.target
 
 	if (["standalone", "host"].includes(mode)) {
 		if (process.platform === "win32") {
+			let extraCaCertsLine = selfSignedCert ? `set "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
 			await safeOutputFile(
 				"run-host.cmd",
 				`\
 @echo off
 set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%"
-:restart
+${extraCaCertsLine}:restart
 call .\\node_modules\\.bin\\clusteriohost.cmd run --can-restart
 if %errorlevel% equ 0 exit /b
 if %errorlevel% equ 8 exit /b
@@ -487,18 +488,20 @@ goto restart
 `
 			);
 		} else {
+			let extraCaCertsLine = selfSignedCert ? `export "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
 			await safeOutputFile(
 				"run-host.sh",
 				`\
 #!/bin/bash
 export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS"
-while true; do
+${extraCaCertsLine}while true; do
 	./node_modules/.bin/clusteriohost run --can-restart
 	if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi
 done
 `,
 				{ mode: 0o755 },
 			);
+			extraCaCertsLine = selfSignedCert ? `Environment=NODE_EXTRA_CA_CERTS=${selfSignedCert}\n` : "";
 			await safeOutputFile(
 				"systemd/clusteriohost.service",
 				`\
@@ -512,7 +515,7 @@ WorkingDirectory=${process.cwd()}
 KillMode=mixed
 KillSignal=SIGINT
 Environment=NODE_OPTIONS=--enable-source-maps
-ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart
+${extraCaCertsLine}ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart
 Restart=on-failure
 RestartPreventExitStatus=8
 
@@ -839,6 +842,9 @@ async function main() {
 		.option("factorio-dir", {
 			nargs: 1, describe: "Path to Factorio installation [standalone/host]", type: "string",
 		})
+		.option("self-signed-cert", {
+			nargs: 1, describe: "Path to self-signed certificate of the controller [standalone/host]", type: "string",
+		})
 		.option("remote-npm", {
 			nargs: 0, description: "Allow remote updates via npm [standalone/controller/host]", type: "boolean",
 		})
@@ -947,7 +953,7 @@ async function main() {
 
 	if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) {
 		logger.info("Writing run scripts");
-		await writeScripts(answers.mode);
+		await writeScripts(answers.mode, args.selfSignedCert);
 	}
 
 	if (answers.mode === "ctl") {

packages/create/package.json

@@ -3,7 +3,7 @@
 	"description": "Installer for clusterio",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.23",
 	"main": "create.js",
 	"bin": {
 		"clusteriocreate": "./create.js"

packages/create/templates/plugin-ts/controller.ts

@@ -1,5 +1,5 @@
 import * as lib from "@clusterio/lib";
-import { BaseControllerPlugin, InstanceInfo } from "@clusterio/controller";
+import { BaseControllerPlugin, InstanceRecord } from "@clusterio/controller";
 //%if multi_context // Messages requires multi context
 
 import {
@@ -37,7 +37,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	}
 //%if instance
 
-	async onInstanceConfigFieldChanged(instance: InstanceInfo, field: string, curr: unknown, prev: unknown) {
+	async onInstanceConfigFieldChanged(instance: InstanceRecord, field: string, curr: unknown, prev: unknown) {
 		this.logger.info(`controller::onInstanceConfigFieldChanged ${instance.id} ${field}`);
 	}
 //%endif
@@ -50,7 +50,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.logger.info("controller::onShutdown");
 	}
 
-	async onPlayerEvent(instance: InstanceInfo, event: lib.PlayerEvent) {
+	async onPlayerEvent(instance: InstanceRecord, event: lib.PlayerEvent) {
 		this.logger.info(`controller::onPlayerEvent ${instance.id} ${JSON.stringify(event)}`);
 	}
 //%if multi_context

packages/ctl/ctl.ts

@@ -27,8 +27,8 @@ import BaseCtlPlugin from "./src/BaseCtlPlugin";
 class ControlConnector extends lib.WebSocketClientConnector {
 	private _token: string;
 
-	constructor(url: string, maxReconnectDelay: number, tlsCa: string | undefined, token: string) {
-		super(url, maxReconnectDelay, tlsCa);
+	constructor(url: string, maxReconnectDelay: number, token: string) {
+		super(url, maxReconnectDelay);
 		this._token = token;
 	}
 
@@ -54,8 +54,6 @@ class ControlConnector extends lib.WebSocketClientConnector {
 export class Control extends lib.Link {
 	/** Control config used for connecting to the controller. */
 	config: lib.ControlConfig;
-	/** Certificate authority used to validate TLS connections to the controller. */
-	tlsCa?: string;
 	/** Mapping of plugin names to their instance for loaded plugins. */
 	plugins: Map<string, BaseCtlPlugin>;
 	/** Keep the control connection alive after the command completes. */
@@ -64,12 +62,10 @@ export class Control extends lib.Link {
 	constructor(
 		connector: ControlConnector,
 		controlConfig: lib.ControlConfig,
-		tlsCa: string | undefined,
 		ctlPlugins: Map<string, BaseCtlPlugin>
 	) {
 		super(connector);
 		this.config = controlConfig;
-		this.tlsCa = tlsCa;
 		this.plugins = ctlPlugins;
 
 		this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this));
@@ -299,20 +295,13 @@ async function startControl() {
 		return;
 	}
 
-	let tlsCa: string | undefined;
-	let tlsCaPath = controlConfig.get("control.tls_ca");
-	if (tlsCaPath) {
-		tlsCa = await fs.readFile(tlsCaPath, "utf8");
-	}
-
 	let controlConnector = new ControlConnector(
 		controlConfig.get("control.controller_url")!,
 		controlConfig.get("control.max_reconnect_delay"),
-		tlsCa,
 		controlConfig.get("control.controller_token")!,
 	);
 
-	let control = new Control(controlConnector, controlConfig, tlsCa, ctlPlugins);
+	let control = new Control(controlConnector, controlConfig, ctlPlugins);
 	try {
 		await controlConnector.connect();
 	} catch (err) {

packages/ctl/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio command line management tool",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22",
+	"version": "2.0.0-alpha.23",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -24,7 +24,6 @@
 		"set-blocking": "catalog:",
 		"fs-extra": "catalog:",
 		"winston": "catalog:",
-		"phin": "catalog:",
 		"as-table": "^1.0.55",
 		"yargs": "^17.7.2"
 	},

packages/ctl/src/command_ops.ts

@@ -0,0 +1,4 @@
+export function print(...content: any) {
+	// eslint-disable-next-line no-console
+	console.log(...content);
+}

packages/ctl/src/commands_controller.ts

@@ -0,0 +1,201 @@
+import fs from "fs-extra";
+import asTableModule from "as-table";
+import events from "events";
+import os from "os";
+import child_process from "child_process";
+
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const controllerCommands = new lib.CommandTree({ name: "controller", description: "Controller management" });
+controllerCommands.add(new lib.Command({
+	definition: ["stop", "Stop controller"],
+	handler: async function(_args: object, control: Control) {
+		await control.send(new lib.ControllerStopRequest());
+	},
+}));
+
+controllerCommands.add(new lib.Command({
+	definition: ["restart", "Restart controller"],
+	handler: async function(_args: object, control: Control) {
+		await control.send(new lib.ControllerRestartRequest());
+	},
+}));
+controllerCommands.add(new lib.Command({
+	definition: ["update", "Update the controller", (yargs) => {
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { restart: boolean }, control: Control) {
+		await control.send(new lib.ControllerUpdateRequest());
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Controller updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+
+const controllerConfigCommands = new lib.CommandTree({
+	name: "config", alias: ["c"], description: "controller config management",
+});
+
+controllerConfigCommands.add(new lib.Command({
+	definition: ["list", "List controller configuration"],
+	handler: async function(args: object, control: Control) {
+		let config = await control.send(new lib.ControllerConfigGetRequest());
+
+		for (let [name, value] of Object.entries(config)) {
+			print(`${name} ${JSON.stringify(value)}`);
+		}
+	},
+}));
+
+controllerConfigCommands.add(new lib.Command({
+	definition: ["set <field> [value]", "Set field in controller config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args: { field: string, value?: string, stdin?: boolean }, control: Control) {
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await control.send(new lib.ControllerConfigSetFieldRequest(args.field, args.value));
+	},
+}));
+
+controllerConfigCommands.add(new lib.Command({
+	definition: ["set-prop <field> <prop> [value]", "Set property of field in controller config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args: { field: string, prop: string, value?: string, stdin?: boolean }, control: Control) {
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let value;
+		try {
+			if (args.value !== undefined) {
+				value = JSON.parse(args.value);
+			}
+		} catch (err: any) {
+			// See note for the instance version of set-prop
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
+				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			value = args.value;
+		}
+		await control.send(new lib.ControllerConfigSetPropRequest(args.field, args.prop, value));
+	},
+}));
+controllerConfigCommands.add(new lib.Command({
+	definition: ["edit [editor]", "Edit controller configuration", (yargs) => {
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args: { editor: string }, control: Control) {
+		let config = await control.send(new lib.ControllerConfigGetRequest());
+		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === undefined) {
+			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl controller config edit <editor of choice>"`);
+		}
+		let allConfigElements = serializedConfigToString(
+			config, lib.ControllerConfig, {}
+		);
+		await fs.writeFile(tmpFile, allConfigElements);
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+			process.stdout.pipe(data);
+		});
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.ControllerConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					print(err);
+					// If the string is empty, it's better to just print "" instead of nothing
+				}
+			}
+		}
+		try {
+			await fs.unlink(tmpFile);
+		} catch (err) {
+			print("err: temporary file", tmpFile, "could not be deleted.");
+			print("This is not fatal, but they may build up over time if the issue persists.");
+		}
+	},
+}));
+
+controllerCommands.add(controllerConfigCommands);
+
+
+const controllerPluginCommands = new lib.CommandTree({
+	name: "plugin", alias: ["p"], description: "controller plugin inspection",
+});
+controllerPluginCommands.add(new lib.Command({
+	definition: ["list", "List plugins on controller"],
+	handler: async function(args: object, control: Control) {
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += "api/plugins";
+		let response = await fetch(url);
+		print(asTable(await response.json() as []));
+	},
+}));
+controllerPluginCommands.add(new lib.Command({
+	definition: ["update <plugin>", "Update a plugin on the controller", (yargs) => {
+		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
+		await control.sendTo("controller", new lib.PluginUpdateRequest(args.plugin));
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Plugin updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+controllerPluginCommands.add(new lib.Command({
+	definition: ["install <plugin>", "Install a plugin on the controller", (yargs) => {
+		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
+		await control.sendTo("controller", new lib.PluginInstallRequest(args.plugin));
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Plugin installed; a restart is required to apply the changes.");
+		}
+	},
+}));
+
+controllerCommands.add(controllerPluginCommands);

packages/ctl/src/commands_debug.ts

@@ -0,0 +1,11 @@
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+
+export const debugCommands = new lib.CommandTree({ name: "debug", description: "Debugging utilities" });
+debugCommands.add(new lib.Command({
+	definition: ["dump-ws", "Dump WebSocket messages sent and received by controller"],
+	handler: async function(args: object, control: Control) {
+		await control.send(new lib.DebugDumpWsRequest());
+		control.keepOpen = true;
+	},
+}));

packages/ctl/src/commands_host.ts

@@ -0,0 +1,305 @@
+import fs from "fs-extra";
+import asTableModule from "as-table";
+import events from "events";
+import os from "os";
+import child_process from "child_process";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const hostCommands = new lib.CommandTree({ name: "host", description: "Host management" });
+hostCommands.add(new lib.Command({
+	definition: ["stop <host>", "Stop the given host", (yargs) => {
+		yargs.positional("host", { describe: "Host to stop", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostStopRequest());
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["restart <host>", "Restart the given host", (yargs) => {
+		yargs.positional("host", { describe: "Host to restart", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostRestartRequest());
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: [["list", "l"], "List hosts connected to the controller"],
+	handler: async function(args: object, control: Control) {
+		let hosts = await control.send(new lib.HostListRequest());
+		print(asTable(hosts));
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["generate-token", "Generate token for a host", (yargs) => {
+		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
+	}],
+	handler: async function(args: { id?: number }, control: Control) {
+		let hostId = typeof args.id === "number" ? args.id : undefined;
+		let response = await control.send(new lib.HostGenerateTokenRequest(hostId));
+		print(response);
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["revoke-token <host>", "Revoke all tokens for a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to revoke tokens for", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.send(new lib.HostRevokeTokensRequest(hostId));
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["create-config", "Create host config", (yargs) => {
+		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
+		yargs.option("name", { type: "string", nargs: 1, describe: "Host name" });
+		yargs.option("generate-token", {
+			type: "boolean", nargs: 0, describe: "Generate authentication token", default: false,
+		});
+		yargs.option("output", {
+			type: "string", nargs: 1, describe: "Path to output config (- for stdout)", default: "config-host.json",
+		});
+	}],
+	handler: async function(
+		args: { id?: number, name?: string, generateToken: boolean, output: string },
+		control: Control
+	) {
+		let config = await control.send(
+			new lib.HostConfigCreateRequest(args.id, args.name, args.generateToken)
+		);
+
+		let content = JSON.stringify(config, null, "\t");
+		if (args.output === "-") {
+			print(content);
+		} else {
+			logger.info(`Writing ${args.output}`);
+			try {
+				await fs.outputFile(args.output, content, { flag: "wx" });
+			} catch (err: any) {
+				if (err.code === "EEXIST") {
+					throw new lib.CommandError(`File ${args.output} already exists`);
+				}
+				throw err;
+			}
+		}
+	},
+}));
+hostCommands.add(new lib.Command({
+	definition: ["update <host>", "Update the host", (yargs) => {
+		yargs.positional("host", { describe: "Host to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostUpdateRequest());
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Host updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+
+
+const hostConfigCommands = new lib.CommandTree({
+	name: "config", alias: ["c"], description: "Host config management",
+});
+hostConfigCommands.add(new lib.Command({
+	definition: ["list <host>", "List configuration for a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to list config for", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		let config = await control.sendTo({ hostId }, new lib.HostConfigGetRequest());
+
+		for (let [name, value] of Object.entries(config)) {
+			print(`${name} ${JSON.stringify(value)}`);
+		}
+	},
+}));
+
+hostConfigCommands.add(new lib.Command({
+	definition: ["set <host> <field> [value]", "Set field in host config", (yargs) => {
+		yargs.positional("host", { describe: "Host to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { host: string, field: string, value?: string, stdin?: boolean },
+		control: Control,
+	) {
+		let hostId = await lib.resolveHost(control, args.host);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(args.field, args.value));
+	},
+}));
+
+hostConfigCommands.add(new lib.Command({
+	definition: ["set-prop <host> <field> <prop> [value]", "Set property of field in host config", (yargs) => {
+		yargs.positional("host", { describe: "Host to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { host: string, field: string, prop: string, value?: string, stdin?: boolean },
+		control: Control
+	) {
+		let hostId = await lib.resolveHost(control, args.host);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let value;
+		try {
+			if (args.value !== undefined) {
+				value = JSON.parse(args.value);
+			}
+		} catch (err: any) {
+			// See note for the instance version of set-prop
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
+				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			value = args.value;
+		}
+		await control.sendTo({ hostId }, new lib.HostConfigSetPropRequest(args.field, args.prop, value));
+	},
+}));
+
+hostConfigCommands.add(new lib.Command({
+	definition: ["edit <host> [editor]", "Edit host configuration", (yargs) => {
+		yargs.positional("host", { describe: "Host to set config on", type: "string" });
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args: { host: string, editor: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		let config = await control.sendTo({ hostId }, new lib.HostConfigGetRequest());
+		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === undefined) {
+			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl controller config edit <editor of choice>"`);
+		}
+		let disallowedList = {"host.id": 0};
+		let allConfigElements = serializedConfigToString(
+			config,
+			lib.HostConfig,
+			disallowedList
+		);
+		await fs.writeFile(tmpFile, allConfigElements);
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+			process.stdout.pipe(data);
+		});
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
+				}
+			}
+		}
+		try {
+			await fs.unlink(tmpFile);
+		} catch (err) {
+			print("err: temporary file", tmpFile, "could not be deleted.");
+			print("This is not fatal, but they may build up over time if the issue persists.");
+		}
+	},
+}));
+
+hostCommands.add(hostConfigCommands);
+
+
+const hostPluginCommands = new lib.CommandTree({
+	name: "plugin", alias: ["p"], description: "host plugin inspection",
+});
+hostPluginCommands.add(new lib.Command({
+	definition: ["list <host>", "List plugins on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to list for", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		const plugins = await control.sendTo({ hostId }, new lib.PluginListRequest());
+		print(asTable(plugins.map(p => ({
+			title: p.title, version: p.version, loaded: p.loaded, enabled: p.enabled, npmPackage: p.npmPackage,
+		}))));
+	},
+}));
+hostPluginCommands.add(new lib.Command({
+	definition: ["update <host> <plugin>", "Update a plugin on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to update on", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.PluginUpdateRequest(args.plugin));
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Plugin updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+hostPluginCommands.add(new lib.Command({
+	definition: ["install <host> <plugin>", "Install a plugin on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to install on", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.PluginInstallRequest(args.plugin));
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Plugin installed; a restart is required to apply the changes.");
+		}
+	},
+}));
+hostCommands.add(hostPluginCommands);

packages/ctl/src/commands_instance.ts

@@ -0,0 +1,685 @@
+import fs from "fs-extra";
+import asTableModule from "as-table";
+import events from "events";
+import path from "path";
+import os from "os";
+import child_process from "child_process";
+import stream from "stream";
+import util from "util";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+const finished = util.promisify(stream.finished);
+
+export const instanceCommands = new lib.CommandTree({
+	name: "instance", alias: ["i"], description: "Instance management",
+});
+instanceCommands.add(new lib.Command({
+	definition: [["list", "l"], "List instances known to the controller"],
+	handler: async function(args: object, control: Control) {
+		let list = await control.send(new lib.InstanceDetailsListRequest());
+		print(asTable(list));
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["create <name>", "Create an instance", (yargs) => {
+		// XXX TODO: set any specific options?
+		yargs.positional("name", { describe: "Instance name", type: "string" });
+		yargs.options({
+			"id": { type: "number", nargs: 1, describe: "Instance id" },
+			"from": { type: "number", nargs: 1, describe: "Clone config from instance id" },
+		});
+	}],
+	handler: async function(args: { name: string, id?: number, from?: number, }, control: Control) {
+		let instanceConfig = new lib.InstanceConfig("control");
+		if (args.id !== undefined) {
+			instanceConfig.set("instance.id", args.id);
+		}
+		instanceConfig.set("instance.name", args.name);
+		await control.send(new lib.InstanceCreateRequest(
+			instanceConfig.toRemote("controller", [
+				"instance.id", "instance.name",
+			]),
+			args.from
+		));
+	},
+}));
+
+const instanceConfigCommands = new lib.CommandTree({
+	name: "config", alias: ["c"], description: "Instance config management",
+});
+instanceConfigCommands.add(new lib.Command({
+	definition: ["list <instance>", "List configuration for an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to list config for", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let config = await control.send(new lib.InstanceConfigGetRequest(instanceId));
+
+		for (let [name, value] of Object.entries(config)) {
+			print(`${name} ${JSON.stringify(value)}`);
+		}
+	},
+}));
+
+instanceConfigCommands.add(new lib.Command({
+	definition: ["set <instance> <field> [value]", "Set field in instance config", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { instance: string, field: string, value?: string, stdin?: boolean },
+		control: Control,
+	) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await control.send(new lib.InstanceConfigSetFieldRequest(instanceId, args.field, args.value));
+	},
+}));
+
+instanceConfigCommands.add(new lib.Command({
+	definition: ["set-prop <instance> <field> <prop> [value]", "Set property of field in instance config", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { instance: string, field: string, prop: string, value?: string, stdin?: boolean },
+		control: Control
+	) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let value;
+		try {
+			if (args.value !== undefined) {
+				value = JSON.parse(args.value);
+			}
+		} catch (err: any) {
+			// If this is from stdin or looks like an array, object or string
+			// literal throw the parse error, otherwise assume this is a string.
+			// The resoning behind this is that correctly quoting the string
+			// with the all the layers of quote removal at play is difficult.
+			// See the following table for how to pass "That's a \" quote" in
+			// different environments:
+			// ----------------------- WARNING -------------------------------
+			// This table is most likely invalid due a change in yargs parsing
+			// ---------------------------------------------------------------
+			// cmd              : """""That's a \\"" quote"""""
+			// cmd + npx        : """""""""""That's a \\\\"""" quote"""""""""""
+			// PowerShell       : '"""""That''s a \\"" quote"""""'
+			// PowerShell + npx : '"""""""""""That''s a \\\\"""" quote"""""""""""'
+			// bash             : '""That'\''s a \" quote""'
+			// bash + npx       : '""That'\''s a \" quote""'
+			// bash + npx -s sh : "'\"\"That'\\''s a \\\" quote\"\"'"
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
+				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			value = args.value;
+		}
+		await control.send(new lib.InstanceConfigSetPropRequest(instanceId, args.field, args.prop, value));
+	},
+}));
+
+instanceConfigCommands.add(new lib.Command({
+	definition: ["edit <instance> [editor]", "Edit instance configuration", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args: { instance: string, editor: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let config = await control.send(new lib.InstanceConfigGetRequest(instanceId));
+		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === undefined) {
+			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl controller config edit <editor of choice>"`);
+		}
+		let disallowedList = {"instance.id": 0, "instance.assigned_host": 0, "factorio.settings": 0};
+		let allConfigElements = serializedConfigToString(
+			config,
+			lib.InstanceConfig,
+			disallowedList
+		);
+		await fs.writeFile(tmpFile, allConfigElements);
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+			process.stdout.pipe(data);
+		});
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.InstanceConfigSetFieldRequest(
+						instanceId,
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
+				}
+			}
+		}
+		try {
+			await fs.unlink(tmpFile);
+		} catch (err) {
+			print("err: temporary file", tmpFile, "could not be deleted.");
+			print("This is not fatal, but they may build up over time if the issue persists.");
+		}
+	},
+}));
+instanceCommands.add(instanceConfigCommands);
+
+instanceCommands.add(new lib.Command({
+	definition: ["assign <instance> [host]", "Assign instance to a host", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to assign", type: "string" });
+		yargs.positional("host", { describe: "Host to assign to or unassign if none", type: "string" });
+	}],
+	handler: async function(args: { instance: string, host?: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let hostId = args.host ? await lib.resolveHost(control, args.host) : undefined;
+		await control.send(new lib.InstanceAssignRequest(instanceId, hostId));
+	},
+}));
+
+async function loadMapSettings(
+	args: { seed?: number, mapExchangeString?: string, mapGenSettings?: string, mapSettings?: string},
+) {
+	let seed = args.seed;
+	let mapGenSettings;
+	let mapSettings;
+	if (args.mapExchangeString) {
+		let parsed = lib.readMapExchangeString(args.mapExchangeString);
+		mapGenSettings = parsed.map_gen_settings;
+		mapSettings = parsed.map_settings;
+	}
+	if (args.mapGenSettings) {
+		mapGenSettings = JSON.parse((await fs.readFile(args.mapGenSettings)).toString());
+	}
+	if (args.mapSettings) {
+		mapSettings = JSON.parse((await fs.readFile(args.mapSettings)).toString());
+	}
+
+	return {
+		seed,
+		mapGenSettings,
+		mapSettings,
+	};
+}
+
+const instanceSaveCommands = new lib.CommandTree({
+	name: "save", alias: ["s"], description: "Instance save management",
+});
+instanceSaveCommands.add(new lib.Command({
+	definition: ["list <instance>", "list saves on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to list saves on", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let saves = await control.sendTo("controller", new lib.InstanceSaveDetailsListRequest());
+		saves = saves.filter(save => save.instanceId === instanceId);
+		print(asTable(saves.map(
+			({ mtimeMs, ...rest }) => ({ ...rest, mtime: new Date(mtimeMs).toLocaleString() })
+		)));
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["create <instance> [name]", "Create a new save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to create on", type: "string" });
+		yargs.positional("name", { describe: "Name of save to create.", type: "string", default: "world.zip" });
+		yargs.options({
+			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
+			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
+			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
+			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(
+		args: {
+			instance: string,
+			name: string,
+			seed?: number,
+			mapExchangeString?: string,
+			mapGenSettings?: string,
+			mapSettings?: string,
+		},
+		control: Control
+	) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo(
+			{ instanceId },
+			new lib.InstanceCreateSaveRequest(args.name, seed, mapGenSettings, mapSettings),
+		);
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["rename <instance> <old-name> <new-name>", "Rename a save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to rename save on", type: "string" });
+		yargs.positional("old-name", { describe: "Old name of save.", type: "string" });
+		yargs.positional("new-name", { describe: "New name of save.", type: "string" });
+	}],
+	handler: async function(args: { instance: string, oldName: string, newName: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceRenameSaveRequest(instanceId, args.oldName, args.newName));
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["copy <instance> <source> <destination>", "Copy a save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to copy save on", type: "string" });
+		yargs.positional("source", { describe: "Save to copy.", type: "string" });
+		yargs.positional("destination", { describe: "Name of copy.", type: "string" });
+	}],
+	handler: async function(args: { instance: string, source: string, destination: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceCopySaveRequest(instanceId, args.source, args.destination));
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["upload <instance> <filepath>", "Upload a save to an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to upload to", type: "string" });
+		yargs.positional("filepath", { describe: "Path to save to upload", type: "string" });
+		yargs.options({
+			"name": { describe: "Name to give save on server", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(args: { instance: string, filepath: string, name?: string }, control: Control) {
+		let filename = args.name || path.basename(args.filepath);
+		if (!filename.endsWith(".zip")) {
+			throw new lib.CommandError("Save name must end with .zip");
+		}
+
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += "api/upload-save";
+		url.searchParams.append("instance_id", String(instanceId));
+		url.searchParams.append("filename", filename);
+
+		const response = await fetch(url, {
+			method: "POST",
+			body: await fs.createReadStream(args.filepath),
+			duplex: "half",
+			headers: {
+				"X-Access-Token": control.config.get("control.controller_token")!,
+				"Content-Type": "application/zip",
+			},
+		});
+
+		type ResponseType = { errors?: string[], request_errors?: string[], saves?: string[] };
+		const result = await response.json() as ResponseType;
+
+		for (let error of result.errors || []) {
+			logger.error(error);
+		}
+
+		for (let requestError of result.request_errors || []) {
+			logger.error(requestError);
+		}
+
+		if (result.saves && result.saves.length) {
+			logger.info(`Successfully uploaded as ${result.saves[0]}`);
+		}
+
+		if ((result.errors || []).length || (result.request_errors || []).length) {
+			throw new lib.CommandError("Uploading save failed");
+		}
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: [
+		"transfer <source-instance> <source-save> <target-instance> [target-save]",
+		"Transfer a save between instances",
+		(yargs) => {
+			yargs.positional("source-instance", { describe: "Instance to transfer save from", type: "string" });
+			yargs.positional("source-save", { describe: "Save to transfer.", type: "string" });
+			yargs.positional("target-instance", { describe: "Instance to transfer to", type: "string" });
+			yargs.positional("target-save", { describe: "Name to give transferred save.", type: "string" });
+			yargs.options({
+				"copy": { describe: "Copy instead of moving the save", type: "boolean", default: false },
+			});
+		},
+	],
+	handler: async function(
+		args: {
+			sourceInstance: string,
+			sourceSave: string,
+			targetInstance: string,
+			targetSave: string,
+			copy: boolean,
+		},
+		control: Control
+	) {
+		let sourceInstanceId = await lib.resolveInstance(control, args.sourceInstance);
+		let targetInstanceId = await lib.resolveInstance(control, args.targetInstance);
+		let storedName = await control.send(
+			new lib.InstanceTransferSaveRequest(
+				sourceInstanceId,
+				args.sourceSave,
+				targetInstanceId,
+				args.targetSave || args.sourceSave,
+				args.copy
+			)
+		);
+		print(`Transferred as ${storedName} to ${args.targetInstance}.`);
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["download <instance> <save>", "Download a save from an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to download save from", type: "string" });
+		yargs.positional("save", { describe: "Save to download", type: "string" });
+	}],
+	handler: async function(args: { instance: string, save: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let streamId = await control.send(new lib.InstanceDownloadSaveRequest(instanceId, args.save));
+
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += `api/stream/${streamId}`;
+		let response = await fetch(url);
+		if (!response.body) {
+			throw new lib.CommandError("Got empty response from server.");
+		}
+
+		let writeStream;
+		let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip");
+		while (true) {
+			try {
+				writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
+				await events.once(writeStream, "open");
+				break;
+			} catch (err: any) {
+				if (err.code === "EEXIST") {
+					tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
+				} else {
+					throw err;
+				}
+			}
+		}
+		stream.Readable.fromWeb(response.body).pipe(writeStream);
+		await finished(writeStream);
+
+		let filename = await lib.findUnusedName(".", args.save, ".zip");
+		await fs.rename(tempFilename, filename);
+
+		logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`);
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["delete <instance> <save>", "Delete a save from an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to delete save from", type: "string" });
+		yargs.positional("save", { describe: "Save to delete", type: "string" });
+	}],
+	handler: async function(args: { instance: string, save: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceDeleteSaveRequest(instanceId, args.save));
+	},
+}));
+instanceCommands.add(instanceSaveCommands);
+
+instanceCommands.add(new lib.Command({
+	definition: ["export-data <instance>", "Export item icons and locale from instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to export from", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceExportDataRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["extract-players <instance>", "Extract players from running save into the cluster.", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to extract players and online time from", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceExtractPlayersRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["start <instance>", "Start instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to start", type: "string" });
+		yargs.options({
+			"save": { describe: "Save to load, defaults to latest", nargs: 1, type: "string" },
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { instance: string, save?: string, keepOpen: boolean }, control: Control) {
+		const instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceStartRequest(args.save));
+		control.keepOpen = args.keepOpen;
+	},
+}));
+instanceCommands.add(new lib.Command({
+	definition: ["restart <instance>", "Restart instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to restart", type: "string" });
+		yargs.options({
+			"save": { describe: "Save to load, defaults to latest", nargs: 1, type: "string" },
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { instance: string, save?: string, keepOpen: boolean }, control: Control) {
+		const instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceRestartRequest(args.save));
+		control.keepOpen = args.keepOpen;
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["start-all", "Start all stopped instances (respects exclude_from_start_all setting)", (yargs) => {
+		yargs.options({
+			"force": {
+				describe: "Start instances even if they have exclude_from_start_all set",
+				nargs: 0,
+				type: "boolean",
+				default: false,
+			},
+		});
+	}],
+	handler: async function(args: { force: boolean }, control: Control) {
+		// Get all instances
+		const instances = await control.send(new lib.InstanceDetailsListRequest());
+
+		// Filter instances that should be started
+		const instancesToStart = instances.filter((instance: lib.InstanceDetails) => {
+			// Only start instances that are currently stopped
+			if (instance.status !== "stopped") {
+				return false;
+			}
+
+			// Respect exclude_from_start_all setting unless --force is used
+			if (!args.force && instance.excludeFromStartAll) {
+				return false;
+			}
+
+			return true;
+		});
+
+		if (instancesToStart.length === 0) {
+			print("No instances to start.");
+			return;
+		}
+
+		print(`Starting ${instancesToStart.length} instance(s):`);
+		for (const instance of instancesToStart) {
+			print(`  - ${instance.name} (ID: ${instance.id})`);
+		}
+
+		// Start all filtered instances
+		const startPromises = instancesToStart.map(async (instance: lib.InstanceDetails) => {
+			try {
+				await control.sendTo({ instanceId: instance.id }, new lib.InstanceStartRequest());
+				print(`✓ Started ${instance.name}`);
+			} catch (error) {
+				print(`✗ Failed to start ${instance.name}: ${error}`);
+			}
+		});
+
+		await Promise.all(startPromises);
+		print("Start all operation completed.");
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["load-scenario <instance> <scenario>", "Start instance by loading a scenario", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to start", type: "string" });
+		yargs.positional("scenario", { describe: "Scenario to load", type: "string" });
+		yargs.options({
+			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
+			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
+			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
+			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(
+		args: {
+			instance: string,
+			scenario: string,
+			seed?: number,
+			mapExchangeString?: string,
+			mapGenSettings?: string,
+			mapSettings?: string,
+			keepOpen: boolean,
+		},
+		control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
+		await control.sendTo(
+			{ instanceId },
+			new lib.InstanceLoadScenarioRequest(args.scenario, seed, mapGenSettings, mapSettings),
+		);
+		control.keepOpen = args.keepOpen;
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["stop <instance>", "Stop instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to stop", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceStopRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["stop-all", "Stop all running instances", () => {}],
+	handler: async function(_args, control: Control) {
+		// Get all instances
+		const instances = await control.send(new lib.InstanceDetailsListRequest());
+
+		// Filter instances that should be stopped
+		const instancesToStop = instances
+			.filter((instance: lib.InstanceDetails) => ["starting", "running"].includes(instance.status));
+
+		if (instancesToStop.length === 0) {
+			print("No instances to stop.");
+			return;
+		}
+
+		print(`Stopping ${instancesToStop.length} instance(s):`);
+		for (const instance of instancesToStop) {
+			print(`  - ${instance.name} (ID: ${instance.id})`);
+		}
+
+		// Stop all filtered instances
+		const stopPromises = instancesToStop.map(async (instance: lib.InstanceDetails) => {
+			try {
+				await control.sendTo({ instanceId: instance.id }, new lib.InstanceStopRequest());
+				print(`✓ Stopped ${instance.name}`);
+			} catch (error) {
+				print(`✗ Failed to stop ${instance.name}: ${error}`);
+			}
+		});
+
+		await Promise.all(stopPromises);
+		print("Stop all operation completed.");
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["kill <instance>", "Kill instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to kill", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceKillRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["delete <instance>", "Delete instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to delete", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceDeleteRequest(instanceId));
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["send-rcon <instance> <command>", "Send RCON command", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to send to", type: "string" });
+		yargs.positional("command", { describe: "command to send", type: "string" });
+	}],
+	handler: async function(args: { instance: string, command: string }, control: Control) {
+		let result = await control.sendTo(
+			{ instanceId: await lib.resolveInstance(control, args.instance) },
+			new lib.InstanceSendRconRequest(args.command),
+		);
+
+		// Factorio includes a newline in its response output.
+		process.stdout.write(result);
+	},
+}));

packages/ctl/src/commands_log.ts

@@ -0,0 +1,90 @@
+import winston from "winston";
+
+import * as lib from "@clusterio/lib";
+import { ConsoleTransport, levels, logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+
+export const logCommands = new lib.CommandTree({ name: "log", description: "Log inspection" });
+logCommands.add(new lib.Command({
+	definition: ["follow", "follow cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Follow the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"controller": { describe: "Follow log of the controller", nargs: 0, type: "boolean", default: false },
+			"host": { describe: "Follow log of given host", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Follow log of given instance", nargs: 1, type: "string", default: null },
+		});
+	}],
+	handler: async function(
+		args: { all: boolean, controller: boolean, host: string | null, instance: string | null },
+		control: Control
+	) {
+		if (!args.all && !args.controller && !args.host && !args.instance) {
+			logger.error("At least one of --all, --controller, --host and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instanceIds = args.instance ? [await lib.resolveInstance(control, args.instance)] : [];
+		let hostIds = args.host ? [await lib.resolveHost(control, args.host)] : [];
+		await control.setLogSubscriptions({ all: args.all, controller: args.controller, hostIds, instanceIds });
+		control.keepOpen = true;
+	},
+}));
+
+logCommands.add(new lib.Command({
+	definition: ["query", "Query cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Query the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"controller": { describe: "Query log of the controller", nargs: 0, type: "boolean", default: false },
+			"host": { describe: "Query log of given host", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Query log of given instance", nargs: 1, type: "string", default: null },
+			"max-level": { describe: "Maximum log level to return", nargs: 1, type: "string", default: undefined },
+			"limit": { describe: "Max number of entries to return", nargs: 1, type: "number", default: 1000 },
+			"start": { describe: "Limit from the start instead of the end", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(
+		args: {
+			all: boolean,
+			controller: boolean,
+			host: string | null,
+			instance: string | null,
+			maxLevel?: string,
+			limit: number,
+			start: boolean,
+		},
+		control: Control
+	) {
+		if (!args.all && !args.controller && !args.host && !args.instance) {
+			logger.error("At least one of --all, --controller, --host and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instanceIds = args.instance ? [await lib.resolveInstance(control, args.instance)] : [];
+		let hostIds = args.host ? [await lib.resolveHost(control, args.host)] : [];
+		let result = await control.send(new lib.LogQueryRequest(
+			args.all,
+			args.controller,
+			hostIds,
+			instanceIds,
+			args.maxLevel as keyof typeof levels,
+			args.limit,
+			args.start ? "asc" : "desc",
+		));
+
+		if (!args.start) {
+			result.log.reverse();
+		}
+
+		let stdoutLogger = winston.createLogger({
+			level: "verbose",
+			levels,
+			format: new lib.TerminalFormat({ showTimestamp: true }),
+			transports: [
+				new ConsoleTransport({ errorLevels: [], warnLevels: [] }),
+			],
+		});
+		for (let info of result.log) {
+			stdoutLogger.log(info as any);
+		}
+	},
+}));

packages/ctl/src/commands_mod.ts

@@ -0,0 +1,237 @@
+import fs from "fs-extra";
+import asTableModule from "as-table";
+import events from "events";
+import path from "path";
+import stream from "stream";
+import util from "util";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+const finished = util.promisify(stream.finished);
+
+export const modCommands = new lib.CommandTree({ name: "mod", description: "Manage uploaded mods" });
+modCommands.add(new lib.Command({
+	definition: ["show <name> <mod-version>", "Show details for a mod stored in the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Mod name to show details for", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of the mod", type: "string" });
+	}],
+	handler: async function(args: { name: string, modVersion: string }, control: Control) {
+		if (!lib.isFullVersion(args.modVersion)) {
+			throw new lib.CommandError("mod-version must match format digit.digit.digit");
+		}
+		let modInfo = await control.send(new lib.ModGetRequest(args.name, args.modVersion));
+		for (let [field, value] of Object.entries(modInfo)) {
+			if (value instanceof Array) {
+				print(`${field}:`);
+				for (let entry of value) {
+					print(`  ${entry}`);
+				}
+			} else {
+				print(`${field}: ${value}`);
+			}
+		}
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: [["list", "l"], "List mods stored in the cluster", (yargs) => {
+		yargs.options({
+			"fields": {
+				describe: "Fields to show, supports 'all'.",
+				array: true,
+				type: "string",
+				default: ["name", "version", "title", "factorio_version"],
+			},
+		});
+	}],
+	handler: async function(args: { fields: string[] }, control: Control) {
+		let mods = await control.send(new lib.ModListRequest());
+		if (!args.fields.includes("all")) {
+			for (let entry of mods) {
+				for (let field of Object.keys(entry)) {
+					if (!args.fields.includes(field)) {
+						// @ts-expect-error terrible hack
+						delete entry[field];
+					}
+				}
+			}
+		}
+		print(asTable(mods));
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["search <factorio-version> [query]", "Search mods stored in the cluster", (yargs) => {
+		yargs.positional("factorio-version", { describe: "Major version of Factorio to search for", type: "string" });
+		yargs.positional("query", { describe: "Search query", type: "string", default: "" });
+		yargs.options({
+			"page": {
+				describe: "Result page to show",
+				type: "number",
+				default: 1,
+			},
+			"page-size": {
+				describe: "Results per page to show",
+				type: "number",
+				default: 10,
+			},
+			"sort": {
+				describe: "sort results by given field",
+				type: "string",
+			},
+			"sort-order": {
+				describe: "order to sort results in (asc/desc)",
+				type: "string",
+				default: "asc",
+			},
+			"fields": {
+				describe: "Fields to show, supports 'all'.",
+				array: true,
+				type: "string",
+				default: ["name", "version", "title", "factorio_version"],
+			},
+		});
+	}],
+	handler: async function(
+		args: {
+			factorioVersion: string,
+			query: string,
+			page: number,
+			pageSize: number,
+			sort?: string,
+			sortOrder: "asc" | "desc",
+			fields: string[],
+		},
+		control: Control
+	) {
+		let response = await control.send(new lib.ModSearchRequest(
+			args.query,
+			lib.normaliseApiVersion(args.factorioVersion as any),
+			args.page,
+			args.pageSize,
+			args.sort,
+			args.sortOrder
+		));
+		let results = response.results.flatMap(result => result.versions);
+		if (!args.fields.includes("all")) {
+			for (let entry of results) {
+				for (let field of Object.keys(entry)) {
+					if (!args.fields.includes(field)) {
+						// @ts-expect-error terrible hack
+						delete entry[field];
+					}
+				}
+			}
+		}
+		for (let issue of response.queryIssues) {
+			print(issue);
+		}
+		print(`page ${args.page} of ${response.pageCount} (${response.resultCount} results)`);
+		print(asTable(results));
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["upload <file>", "Upload mod to the cluster", (yargs) => {
+		yargs.positional("file", { describe: "File to upload", type: "string" });
+	}],
+	handler: async function(args: { file: string }, control: Control) {
+		let filename = path.basename(args.file);
+		if (!filename.endsWith(".zip")) {
+			throw new lib.CommandError("Mod filename must end with .zip");
+		}
+
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += "api/upload-mod";
+		url.searchParams.append("filename", filename);
+
+		let response = await fetch(url, {
+			method: "POST",
+			body: fs.createReadStream(args.file),
+			duplex: "half",
+			headers: {
+				"X-Access-Token": control.config.get("control.controller_token")!,
+				"Content-Type": "application/zip",
+			},
+		});
+
+		type ResponseType = { errors?: [], request_errors?: [], mods: Parameters<typeof lib.ModInfo.fromJSON>[0][]};
+		const result = await response.json() as ResponseType;
+
+		for (let error of result.errors || []) {
+			logger.error(error);
+		}
+
+		for (let requestError of result.request_errors || []) {
+			logger.error(requestError);
+		}
+
+		if (result.mods && result.mods.length) {
+			const mod = lib.ModInfo.fromJSON(result.mods[0]);
+			logger.info(`Successfully uploaded ${mod.filename}`);
+		}
+
+		if ((result.errors || []).length || (result.request_errors || []).length) {
+			throw new lib.CommandError("Uploading mod failed");
+		}
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["download <name> <mod-version>", "Download a mod from the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Internal name of mod to download", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of mod to download", type: "string" });
+	}],
+	handler: async function(args: { name: string, modVersion: string }, control: Control) {
+		if (!lib.isFullVersion(args.modVersion)) {
+			throw new lib.CommandError("mod-version must match format digit.digit.digit");
+		}
+		let streamId = await control.send(new lib.ModDownloadRequest(args.name, args.modVersion));
+
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += `api/stream/${streamId}`;
+		let response = await fetch(url);
+		if (!response.body) {
+			throw new lib.CommandError("Got empty response from server");
+		}
+
+		let writeStream;
+		let filename = `${args.name}_${args.modVersion}.zip`;
+		let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
+		while (true) {
+			try {
+				writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
+				await events.once(writeStream, "open");
+				break;
+			} catch (err: any) {
+				if (err.code === "EEXIST") {
+					tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
+				} else {
+					throw err;
+				}
+			}
+		}
+		stream.Readable.fromWeb(response.body).pipe(writeStream);
+		await finished(writeStream);
+		await fs.rename(tempFilename, filename);
+
+		logger.info(`Downloaded ${filename}`);
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["delete <name> <mod-version>", "Delete a mod stored in the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Name of mod to delete", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of mod to delete", type: "string" });
+	}],
+	handler: async function(args: { name: string, modVersion: string }, control: Control) {
+		if (!lib.isFullVersion(args.modVersion)) {
+			throw new lib.CommandError("mod-version must match format digit.digit.digit");
+		}
+		await control.send(new lib.ModDeleteRequest(args.name, args.modVersion));
+	},
+}));

packages/ctl/src/commands_mod_pack.ts

@@ -0,0 +1,315 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const modPackCommands = new lib.CommandTree({ name: "mod-pack", description: "Mod Pack" });
+modPackCommands.add(new lib.Command({
+	definition: ["show <mod-pack>", "Show details of mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to show", type: "string" });
+	}],
+	handler: async function(args: { modPack: string }, control: Control) {
+		let mods = new Map(
+			(await control.send(new lib.ModListRequest())).map(m => [m.id, m])
+		);
+		let modPack = await control.send(
+			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
+		);
+
+		for (let [field, value] of Object.entries(modPack)) {
+			if (field === "mods") {
+				print(`${field}:`);
+				for (let [name, entry] of value) {
+					const mod = mods.get(`${name}_${entry.version}`);
+					const missing = !mod && "? ";
+					const badChecksum = mod && entry.sha1 && entry.sha1 !== mod.sha1 && "! ";
+					const warn = missing || badChecksum || "";
+					const enabled = entry.enabled ? "" : "(disabled) ";
+					print(`  ${warn}${enabled}${name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`);
+				}
+			} else if (field === "settings") {
+				print(`${field}:`);
+				for (let [scope, settings] of Object.entries(value)) {
+					print(`  ${scope}:`);
+					for (let [setting, settingValue] of settings as any) {
+						print(`    ${setting}: ${JSON.stringify(settingValue.value)}`);
+					}
+				}
+			} else if (field === "exportManifest") {
+				print(`${field}:`);
+				if (value && value.assets) {
+					print("  assets:");
+					for (let [name, fileName] of Object.entries(value.assets)) {
+						print(`    ${name}: ${fileName}`);
+					}
+				}
+
+			} else {
+				print(`${field}: ${value}`);
+			}
+		}
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: [["list", "l"], "List mod packs in the cluster"],
+	handler: async function(args: object, control: Control) {
+		let modPacks = await control.send(new lib.ModPackListRequest());
+		let fields = ["id", "name", "factorioVersion"];
+		for (let entry of modPacks) {
+			for (let field of Object.keys(entry)) {
+				if (!fields.includes(field)) {
+					// @ts-expect-error terrible hack
+					delete entry[field];
+				}
+			}
+		}
+		print(asTable(modPacks));
+	},
+}));
+
+function setModPackSettings(
+	modPack: lib.ModPack,
+	args: {
+		boolSetting?: string[],
+		intSetting?: string[],
+		doubleSetting?: string[],
+		stringSetting?: string[],
+		colorSetting?: string[],
+	},
+) {
+	function doSettings(settings: string[], cast: (value: string) => (string | number | boolean)) {
+		for (let i = 0; i + 2 < settings.length; i += 3) {
+			const [scope, name, value] = settings.slice(i, i + 3);
+			if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
+				throw new lib.CommandError(
+					`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
+				);
+			}
+			const checkedScope = scope as "startup" | "runtime-global" | "runtime-per-user";
+			modPack.settings[checkedScope].set(name, { value: cast(value) });
+		}
+	}
+
+	doSettings(args.boolSetting || [], value => {
+		if (!["true", "false"].includes(value)) {
+			throw new lib.CommandError(`boolean value must be one of true or false, not ${value}`);
+		}
+		return value === "true";
+	});
+	doSettings(args.intSetting || [], value => {
+		// eslint-disable-next-line radix
+		let number = Number.parseInt(value);
+		if (isNaN(number)) {
+			throw new lib.CommandError(`int value must be an integer, not ${value}`);
+		}
+		return number;
+	});
+	doSettings(args.doubleSetting || [], value => {
+		let number = Number.parseFloat(value);
+		if (isNaN(number)) {
+			throw new lib.CommandError(`double value must be a number, not ${value}`);
+		}
+		return number;
+	});
+	doSettings(args.stringSetting || [], value => value);
+	doSettings(args.colorSetting || [], value => {
+		let color = JSON.parse(value);
+		if (typeof color !== "object" || color === null) {
+			throw new lib.CommandError("color value must be a JSON object with r, g, b, a properties");
+		}
+		return color;
+	});
+}
+
+function setModPackMods(modPack: lib.ModPack, mods: string[] | undefined) {
+	for (let mod of mods || []) {
+		const [name, version, sha1] = mod.split(":");
+		if (!version) {
+			throw new lib.CommandError("Added mod must be formatted as name:version or name:version:sha1");
+		}
+		if (!lib.isFullVersion(version)) {
+			throw new lib.CommandError("version must match the format digit.digit.digit");
+		}
+		if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) {
+			throw new lib.CommandError("sha1 must be a 40 digit lower case hex string");
+		}
+		modPack.mods.set(name, { name, enabled: true, version, sha1 });
+	}
+}
+
+function setModPackModsEnabled(modPack: lib.ModPack, mods: string[] | undefined, enabled: boolean) {
+	for (let mod of mods || []) {
+		if (!modPack.mods.has(mod)) {
+			throw new lib.CommandError(`Mod named ${mod} does not exist in the mod pack`);
+		}
+		modPack.mods.get(mod)!.enabled = enabled;
+	}
+}
+
+modPackCommands.add(new lib.Command({
+	definition: ["create <name> <factorio-version>", "Create mod pack", (yargs) => {
+		yargs.positional("name", { describe: "Name of mod pack to create", type: "string" });
+		yargs.positional("factorio-version", { describe: "Version of factorio the mod pack is for", type: "string" });
+		yargs.options({
+			"description": { describe: "Description for mod pack", type: "string" },
+			"mods": { describe: "Mods in the form of name:version[:sha1]", array: true, type: "string" },
+			"disabled-mods": { describe: "Mods that are in the pack but not enabled", array: true, type: "string" },
+			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
+			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
+			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
+			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
+			"color-setting": { describe: "Set color setting", array: true, nargs: 3, type: "string" },
+		});
+	}],
+	handler: async function(
+		args: {
+			name: string,
+			factorioVersion: string,
+			description?: string,
+			mods?: string[],
+			disabledMods?: string[],
+			boolSetting?: string[],
+			intSetting?: string[],
+			doubleSetting?: string[],
+			stringSetting?: string[],
+			colorSetting?: string[],
+		},
+		control: Control,
+	) {
+		const modPack = new lib.ModPack();
+		modPack.name = args.name;
+		if (args.description) { modPack.description = args.description; }
+		if (args.factorioVersion) {
+			if (!lib.isPartialVersion(args.factorioVersion)) {
+				throw new lib.CommandError("factorio-version must match the format digit.digit[.digit]");
+			}
+			modPack.factorioVersion = args.factorioVersion;
+		}
+		setModPackMods(modPack, args.mods);
+		setModPackModsEnabled(modPack, args.disabledMods, false);
+		setModPackSettings(modPack, args);
+		await control.send(new lib.ModPackCreateRequest(modPack));
+		print(`Created mod pack ${modPack.name} (${modPack.id})`);
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["import <string>", "Import mod pack string", (yargs) => {
+		yargs.positional("string", { describe: "Mod pack string to import", type: "string" });
+	}],
+	handler: async function(args: { string: string }, control: Control) {
+		const modPack = lib.ModPack.fromModPackString(args.string);
+		await control.send(new lib.ModPackCreateRequest(modPack));
+		print(`Created mod pack ${modPack.name} (${modPack.id})`);
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["export <mod-pack>", "Export mod pack string", (yargs) => {
+		yargs.positional("string", { describe: "Mod pack to export", type: "string" });
+	}],
+	handler: async function(args: { modPack: string }, control: Control) {
+		const modPack = await control.send(
+			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
+		);
+		print(modPack.toModPackString());
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["edit <mod-pack>", "Edit mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" });
+		yargs.options({
+			"name": { describe: "New name for mod pack", type: "string" },
+			"description": { describe: "New description for mod pack", type: "string" },
+			"factorio-version": { describe: "Set version of factorio the mod pack is for", type: "string" },
+			"add-mods": { describe: "Mods in the form of name:version[:sha1] to add", array: true, type: "string" },
+			"enable-mods": { describe: "Mods to set as enabled", array: true, type: "string" },
+			"disable-mods": { describe: "Mods to set as disabled", array: true, type: "string" },
+			"remove-mods": { describe: "Name of mods to remove", array: true, type: "string" },
+			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
+			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
+			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
+			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
+			"color-setting": { describe: "Set color setting", array: true, nargs: 3, type: "string" },
+			"remove-setting": { describe: "Remove a setting", array: true, nargs: 2, type: "string" },
+		});
+	}],
+	handler: async function(
+		args: {
+			modPack: string,
+			name?: string,
+			description?: string,
+			factorioVersion?: string,
+			addMods?: string[],
+			enableMods?: string[],
+			disableMods?: string[],
+			removeMods?: string[],
+			boolSetting?: string[],
+			intSetting?: string[],
+			doubleSetting?: string[],
+			stringSetting?: string[],
+			colorSetting?: string[],
+			removeSetting?: string[],
+		},
+		control: Control,
+	) {
+		const modPack = await control.send(
+			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
+		);
+
+		if (args.name) { modPack.name = args.name; }
+		if (args.description) { modPack.description = args.description; }
+		if (args.factorioVersion) {
+			if (!lib.isPartialVersion(args.factorioVersion)) {
+				throw new lib.CommandError("factorio-version must match the format digit.digit[.digit]");
+			}
+			modPack.factorioVersion = args.factorioVersion;
+		}
+
+		setModPackMods(modPack, args.addMods);
+		for (let name of args.removeMods || []) {
+			if (modPack.mods.has(name)) {
+				modPack.mods.delete(name);
+			} else {
+				logger.warn(`Mod ${name} did not exist on ${modPack.name}`);
+			}
+		}
+		setModPackModsEnabled(modPack, args.disableMods, false);
+		setModPackModsEnabled(modPack, args.enableMods, true);
+
+		setModPackSettings(modPack, args);
+		if (args.removeSetting) {
+			for (let i = 0; i + 1 < args.removeSetting.length; i += 2) {
+				const [scope, name] = args.removeSetting.slice(i, i + 2);
+				if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
+					throw new lib.CommandError(
+						`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
+					);
+				}
+				const checkedScope = scope as "startup" | "runtime-global" | "runtime-per-user";
+				if (modPack.settings[checkedScope].has(name)) {
+					modPack.settings[checkedScope].delete(name);
+				} else {
+					logger.warn(`Mod setting ${scope} ${name} did not exist on ${modPack.name}`);
+				}
+			}
+		}
+		await control.send(new lib.ModPackUpdateRequest(modPack));
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["delete <mod-pack>", "Delete mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to delete", type: "string" });
+	}],
+	handler: async function(args: { modPack: string }, control: Control) {
+		const id = await lib.resolveModPack(control, args.modPack);
+		await control.send(new lib.ModPackDeleteRequest(id));
+	},
+}));

packages/ctl/src/commands_permission.ts

@@ -0,0 +1,16 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const permissionCommands = new lib.CommandTree({ name: "permission", description: "Permission inspection" });
+permissionCommands.add(new lib.Command({
+	definition: [["list", "l"], "List permissions in the cluster"],
+	handler: async function(args: object, control: Control) {
+		let permissions = await control.send(new lib.PermissionListRequest());
+		print(asTable(permissions));
+	},
+}));

packages/ctl/src/commands_role.ts

@@ -0,0 +1,100 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const roleCommands = new lib.CommandTree({ name: "role", description: "Role management" });
+roleCommands.add(new lib.Command({
+	definition: [["list", "l"], "List roles in the cluster"],
+	handler: async function(args: object, control: Control) {
+		let roles = await control.send(new lib.RoleListRequest());
+		print(asTable(roles));
+	},
+}));
+
+roleCommands.add(new lib.Command({
+	definition: ["create <name>", "Create a new role", (yargs) => {
+		yargs.positional("name", { describe: "Name of role to create", type: "string" });
+		yargs.options({
+			"description": { describe: "Description for role", nargs: 1, type: "string", default: "" },
+			"permissions": { describe: "Permissions role grants", nargs: 1, array: true, type: "string", default: [] },
+		});
+	}],
+	handler: async function(args: { name: string, description: string, permissions: string[] }, control: Control) {
+		let id = await control.send(new lib.RoleCreateRequest(
+			args.name,
+			args.description,
+			args.permissions,
+		));
+		logger.info(`Created role ID ${id}`);
+	},
+}));
+
+roleCommands.add(new lib.Command({
+	definition: ["edit <role>", "Edit existing role", (yargs) => {
+		yargs.positional("role", { describe: "Role to edit", type: "string" });
+		yargs.options({
+			"name": { describe: "New name for role", nargs: 1, type: "string" },
+			"description": { describe: "New description for role", nargs: 1, type: "string" },
+			"set-perms": { describe: "Set permissions for role", array: true, type: "string" },
+			"add-perms": { describe: "Add permissions to role", array: true, type: "string", conflicts: "set-perms" },
+			"remove-perms": {
+				describe: "Remove permissions from role", array: true, type: "string", conflicts: "set-perms",
+			},
+			"grant-default": { describe: "Add default permissions to role", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: {
+			role: string,
+			name?: string,
+			description?: string,
+			setPerms?: string[],
+			addPerms?: string[],
+			removePerms?: string[],
+			grantDefault?: boolean,
+		},
+		control: Control
+	) {
+		let role = await lib.retrieveRole(control, args.role);
+
+		if (args.name !== undefined) {
+			role.name = args.name;
+		}
+		if (args.description !== undefined) {
+			role.description = args.description;
+		}
+		if (args.addPerms) {
+			for (const perm of args.addPerms) {
+				role.permissions.add(perm);
+			}
+		}
+		if (args.removePerms) {
+			for (let perm of args.removePerms) {
+				role.permissions.delete(perm);
+			}
+		}
+		if (args.setPerms !== undefined) {
+			role.permissions = new Set(args.setPerms);
+		}
+		await control.send(new lib.RoleUpdateRequest(role.id, role.name, role.description, [...role.permissions]));
+
+		if (args.grantDefault) {
+			await control.send(new lib.RoleGrantDefaultPermissionsRequest(role.id));
+		}
+	},
+}));
+
+roleCommands.add(new lib.Command({
+	definition: ["delete <role>", "Delete role", (yargs) => {
+		yargs.positional("role", { describe: "Role to delete", type: "string" });
+	}],
+	handler: async function(args: { role: string }, control: Control) {
+		let role = await lib.retrieveRole(control, args.role);
+		await control.send(new lib.RoleDeleteRequest(role.id));
+	},
+}));

packages/ctl/src/commands_user.ts

@@ -0,0 +1,304 @@
+import fs from "fs-extra";
+import asTableModule from "as-table";
+import path from "path";
+
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const userCommands = new lib.CommandTree({ name: "user", alias: ["u"], description: "User management" });
+userCommands.add(new lib.Command({
+	definition: ["show <name>", "Show details for one user", (yargs) => {
+		yargs.positional("name", { decribe: "Name of user to show", type: "string" });
+		yargs.options({
+			"instance-stats": { describe: "include per-instance stats", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { name: string, instanceStats: boolean }, control: Control) {
+		let user = await control.send(new lib.UserGetRequest(args.name));
+		Object.assign(user, user.playerStats);
+		// @ts-expect-error Terrible hack
+		delete user.playerStats;
+		let instanceStats = user.instanceStats;
+		// @ts-expect-error Terrible hack
+		delete user.instanceStats;
+		print(asTable(Object.entries(user).map(([property, value]) => ({ property, value }))));
+
+		if (args.instanceStats) {
+			let instances = await control.send(new lib.InstanceDetailsListRequest());
+			function instanceName(id: number) {
+				let instance = instances.find(i => i.id === id);
+				if (instance) {
+					return instance.name;
+				}
+				return "<deleted>";
+			}
+			for (let [id, playerInstanceStats] of instanceStats || []) {
+				print();
+				print(`Instance ${instanceName(id)} (${id}):`);
+				print(asTable(Object.entries(playerInstanceStats).map(([property, value]) => ({ property, value }))));
+			}
+		}
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: [["list", "l"], "List user in the cluster", (yargs) => {
+		yargs.options({
+			"stats": { describe: "include user stats", nargs: 0, type: "boolean", default: false },
+			"attributes": { describe: "include admin/whitelisted/banned", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { stats: boolean, attributes: boolean }, control: Control) {
+		let users = await control.send(new lib.UserListRequest());
+		for (let user of users) {
+			if (args.stats) {
+				Object.assign(user, user.playerStats);
+			}
+			// @ts-expect-error Terrible hack
+			delete user.playerStats;
+			// @ts-expect-error Terrible hack
+			delete user.isDeleted;
+			// @ts-expect-error Terrible hack
+			delete user.banReason;
+			// @ts-expect-error Terrible hack
+			delete user.instanceStats;
+			if (!args.attributes) {
+				// @ts-expect-error Terrible hack
+				delete user.isAdmin;
+				// @ts-expect-error Terrible hack
+				delete user.isWhitelisted;
+				// @ts-expect-error Terrible hack
+				delete user.isBanned;
+			}
+		}
+		print(asTable(users));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["create <name>", "Create a user", (yargs) => {
+		yargs.positional("name", { describe: "Name of user to create", type: "string" });
+	}],
+	handler: async function(args: { name: string }, control: Control) {
+		await control.send(new lib.UserCreateRequest(args.name));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["revoke-token <name>", "Revoke token for user", (yargs) => {
+		yargs.positional("name", { describe: "Name of user to revoke token for", type: "string" });
+	}],
+	handler: async function(args: { name: string }, control: Control) {
+		await control.send(new lib.UserRevokeTokenRequest(args.name));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-admin <user>", "Promote or demote a user to admin", (yargs) => {
+		yargs.positional("user", { describe: "Name of user set admin status for", type: "string" });
+		yargs.options({
+			"revoke": { describe: "Revoke admin status", nargs: 0, type: "boolean", default: false },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { user: string, revoke: boolean, create: boolean }, control: Control) {
+		await control.send(new lib.UserSetAdminRequest(args.user, args.create, !args.revoke));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-whitelisted <user>", "Add or remove user from the whitelist", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to set whitelist status for", type: "string" });
+		yargs.options({
+			"remove": { describe: "Remove from whitelist", nargs: 0, type: "boolean", default: false },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { user: string, remove: boolean, create: boolean }, control: Control) {
+		await control.send(new lib.UserSetWhitelistedRequest(args.user, args.create, !args.remove));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-banned <user>", "Ban or pardon user from banlist", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to set ban status for", type: "string" });
+		yargs.options({
+			"pardon": { describe: "Remove from banlist", nargs: 0, type: "boolean", default: false },
+			"reason": { describe: "Ban reason", nargs: 1, type: "string", default: "" },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(
+		args: {
+			user: string,
+			pardon: boolean,
+			reason: string,
+			create: boolean
+		},
+		control: Control
+	) {
+		await control.send(new lib.UserSetBannedRequest(args.user, args.create, !args.pardon, args.reason));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-roles <user> [roles...]", "Replace user roles", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to change roles for", type: "string" });
+		yargs.positional("roles", { describe: "roles to assign", type: "string" });
+	}],
+	handler: async function(args: { user: string, roles: string[] }, control: Control) {
+		let roles = await control.send(new lib.RoleListRequest());
+
+		let resolvedRoles = [];
+		for (let roleName of args.roles) {
+			if (/^-?\d+$/.test(roleName)) {
+				let roleId = parseInt(roleName, 10);
+				resolvedRoles.push(roleId);
+
+			} else {
+				let found = false;
+				for (let role of roles) {
+					if (role.name === roleName) {
+						resolvedRoles.push(role.id);
+						found = true;
+						break;
+					}
+				}
+
+				if (!found) {
+					throw new lib.CommandError(`No role named ${roleName}`);
+				}
+			}
+		}
+
+		await control.send(new lib.UserUpdateRolesRequest(args.user, resolvedRoles));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["delete <user>", "Delete user", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to delete", type: "string" });
+	}],
+	handler: async function(args: { user: string }, control: Control) {
+		await control.send(new lib.UserDeleteRequest(args.user));
+	},
+}));
+
+async function handleImportOrRestore(args: {
+	filepath: string,
+	users: boolean,
+	bans: boolean,
+	admins: boolean,
+	whitelist: boolean,
+	noBackup?: boolean,
+}, control: Control, restore: boolean) {
+	let importType: ConstructorParameters<typeof lib.UserBulkImportRequest>[0];
+	const data = JSON.parse((await fs.readFile(args.filepath)).toString());
+	const optionCount = [args.users, args.bans, args.admins, args.whitelist]
+		.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
+
+	if (optionCount > 1) {
+		throw new lib.CommandError("Can not specify multiple options");
+	}
+
+	// Assign based on options or attempt to guess based on filename
+	const filename = path.basename(args.filepath);
+	if (args.users || (optionCount === 0 && data.export_version)) {
+		importType = "users";
+	} else if (args.bans || (optionCount === 0 && filename.includes("ban"))) {
+		importType = "bans";
+	} else if (args.admins || (optionCount === 0 && filename.includes("admin"))) {
+		importType = "admins";
+	} else if (args.whitelist || (optionCount === 0 && filename.includes("whitelist"))) {
+		importType = "whitelist";
+	} else {
+		throw new lib.CommandError("Unknown json file, please specify an option");
+	}
+
+	const backup = await control.send(
+		new lib.UserBulkImportRequest(importType, importType === "users" ? data.users : data, restore)
+	);
+
+	print(`${restore ? "Restored" : "Imported"} ${importType} from ${args.filepath}`);
+	if (restore && backup && !args.noBackup) {
+		await fs.writeJSON(`${importType}-backup.json`, backup, { spaces: 2 });
+		print(`Wrote backup to ${importType}-backup.json`);
+	}
+}
+
+userCommands.add(new lib.Command({
+	definition: ["import <filepath>", "Import user data from a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
+		yargs.options({
+			"users": { describe: "Import users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Import banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Import adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Import whitelist json", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: any, control: any) {
+		await handleImportOrRestore(args, control, false);
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["restore <filepath>", "Restore user data from a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
+		yargs.options({
+			"users": { describe: "Restore users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Restore banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Restore adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Restore whitelist json", nargs: 0, type: "boolean", default: false },
+			"no-backup": { describe: "Don't save a backup to the cwd", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: any, control: any) {
+		await handleImportOrRestore(args, control, true);
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["export <filepath>", "Export user data to a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to save to", type: "string" });
+		yargs.options({
+			"users": { describe: "Export users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Export banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Export adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Export whitelist json", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: {
+		filepath: string,
+		users: boolean,
+		bans: boolean,
+		admins: boolean,
+		whitelist: boolean,
+	}, control: Control) {
+		let exportType = "users" as ConstructorParameters<typeof lib.UserBulkExportRequest>[0];
+		const optionCount = [args.bans, args.admins, args.whitelist]
+			.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
+
+		if (optionCount > 1) {
+			throw new lib.CommandError("Can not specify multiple options");
+		}
+
+		// Assign based on options or attempt to guess based on filename
+		if (args.users) {
+			exportType = "users";
+		} else if (args.bans) {
+			exportType = "bans";
+		} else if (args.admins) {
+			exportType = "admins";
+		} else if (args.whitelist) {
+			exportType = "whitelist";
+		}
+
+		await fs.writeJSON(args.filepath, await control.send(
+			new lib.UserBulkExportRequest(exportType)
+		), { spaces: 2 });
+		print(`Exported ${exportType} to ${args.filepath}`);
+	},
+}));

packages/ctl/src/config_ops.ts

@@ -0,0 +1,58 @@
+import * as lib from "@clusterio/lib";
+
+export async function getEditor(argsEditor: string) {
+	// eslint-disable-next-line
+	return argsEditor || process.env.EDITOR || process.env.VISUAL || undefined
+	// needed for the process.env statements to not be flagged by eslint
+	// priority for editors is CLI argument > env.EDITOR > env.VISUAL
+}
+
+export async function configToKeyVal(data: string) {
+	let final: Record<string, string> = {};
+	let splitData = data.split(/\r?\n/);
+	// split on newlines
+	let filtered = splitData.filter((value) => value[0] !== "#").filter((a) => a);
+	// the last filter removes empty elements left by the first. Not done on one line due to readability.
+	for (let index in filtered) {
+		if (index in filtered) {
+			let split = filtered[index].split("=");
+			let finalIndex = filtered[index][0].trim();
+			// split on the = we added earlier, giving us both value and key
+			let part = "";
+			try {
+				part = split[1].trim();
+				// it's a string if we can read it
+			} catch (err) {
+				// if we can't read it, it's a empty field and therefor null
+				part = "";
+			}
+			final[finalIndex] = part;
+		}
+	}
+	return final;
+}
+
+export function serializedConfigToString(
+	serializedConfig: lib.ConfigSchema,
+	configGroup: typeof lib.ControllerConfig | typeof lib.HostConfig | typeof lib.InstanceConfig,
+	disallowedList: Record<string, unknown>,
+) {
+	let allConfigElements = "";
+	for (let [name, value] of Object.entries(serializedConfig)) {
+		if (name in disallowedList) {
+			continue;
+		}
+		let desc = "";
+		try {
+			desc += (configGroup.fieldDefinitions as any)[name]!.description;
+		} catch (err) {
+			desc += "No description found";
+		}
+		// split onto two lines for readability and es-lint
+		if (String(value) === "null") {
+			value = "";
+		}
+		allConfigElements += `${name} = ${value}\n\n`;
+	}
+	return allConfigElements;
+}

packages/host/host.ts

@@ -51,13 +51,11 @@ void new lib.Gauge(
 export class HostConnector extends lib.WebSocketClientConnector {
 	constructor(
 		public hostConfig: lib.HostConfig,
-		tlsCa: string | undefined,
 		public pluginInfos: lib.PluginNodeEnvInfo[]
 	) {
 		super(
 			hostConfig.get("host.controller_url"),
 			hostConfig.get("host.max_reconnect_delay"),
-			tlsCa
 		);
 	}
 
@@ -230,17 +228,10 @@ async function startHost() {
 		return;
 	}
 
-	let tlsCa: string | undefined;
-	let tlsCaPath = hostConfig.get("host.tls_ca");
-	if (tlsCaPath) {
-		tlsCa = await fs.readFile(tlsCaPath, "utf8");
-	}
-
-	let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos);
+	let hostConnector = new HostConnector(hostConfig, pluginInfos);
 	host = new Host(
 		hostConnector,
 		hostConfig,
-		tlsCa,
 		pluginInfos,
 		Boolean(args.canRestart),
 		Boolean(args.recovery),
@@ -291,6 +282,7 @@ async function startHost() {
 	});
 
 	await hostConnector.connect();
+	host.startedAtMs = Date.now();
 	logger.info("Started host");
 }
 
@@ -307,7 +299,7 @@ I           version of clusterio.  Expect things to break. I
 		if (err instanceof lib.WebSocketError) {
 			// Already handled by connector.on("error") within Host
 
-		} else if (err instanceof lib.StartupError) {
+		} else if (err instanceof lib.StartupError || err.code === "InstallationError") {
 			logger.fatal(`
 +----------------------------------+
 | Unable to to start clusteriohost |

packages/host/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio host server",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.23",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -26,7 +26,6 @@
 		"set-blocking": "catalog:",
 		"fs-extra": "catalog:",
 		"jszip": "catalog:",
-		"phin": "catalog:",
 		"winston": "catalog:",
 		"winston-daily-rotate-file": "catalog:",
 		"jimp": "^0.22.8",

packages/host/src/Host.ts

@@ -3,7 +3,6 @@ import path from "path";
 import events from "events";
 import pidusage from "pidusage";
 import setBlocking from "set-blocking";
-import phin from "phin";
 import stream from "stream";
 import util from "util";
 
@@ -32,7 +31,7 @@ function checkRequestSaveName(name: string) {
  *
  * Looks through all sub-dirs of the provided directory for valid
  * instance definitions and return a mapping of instance id to
- * instanceInfo objects.
+ * instance configs and the root instance paths.
  *
  * @param instancesDir - Directory containing instances
  * @returns
@@ -40,7 +39,7 @@ function checkRequestSaveName(name: string) {
  * @internal
  */
 async function discoverInstances(instancesDir: string) {
-	let instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
+	const instances = new Map<number, UnloadedInstance>();
 	for (let entry of await fs.readdir(instancesDir, { withFileTypes: true })) {
 		if (entry.isDirectory()) {
 			let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
@@ -60,21 +59,25 @@ async function discoverInstances(instancesDir: string) {
 				continue;
 			}
 
-			if (instanceInfos.has(instanceConfig.get("instance.id"))) {
+			if (instances.has(instanceConfig.get("instance.id"))) {
 				logger.warn(`Ignoring instance with duplicate id in folder ${entry.name}`);
 				continue;
 			}
 
 			logger.verbose(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`);
-			instanceInfos.set(instanceConfig.get("instance.id"), {
+			instances.set(instanceConfig.get("instance.id"), {
 				path: instancePath,
 				config: instanceConfig,
 			});
 		}
 	}
-	return instanceInfos;
+	return instances;
 }
 
+// Requests that wake up instances are responsible for making sure the
+// instance status is correctly updated even if they fail with an error. This
+// means they should typically use try catch on anything that can throw
+// and call notifyExit upon any errors.
 const instanceStartingMessages = new Set([
 	lib.InstanceStartRequest.name,
 	lib.InstanceLoadScenarioRequest.name,
@@ -114,10 +117,10 @@ export class HostRouter {
 			dst.type === lib.Address.controller
 			|| dst.type === lib.Address.control
 			|| dst.type === lib.Address.host && dst.id !== this.host.config.get("host.id")
-			|| dst.type === lib.Address.instance && !this.host.instanceInfos.has(dst.id)
+			|| dst.type === lib.Address.instance && !this.host.assignedInstances.has(dst.id)
 		) {
 			nextHop = this.host;
-		} else if (dst.type === lib.Address.instance && this.host.instanceInfos.has(dst.id)) {
+		} else if (dst.type === lib.Address.instance && this.host.assignedInstances.has(dst.id)) {
 			nextHop = this.host.instanceConnections.get(dst.id)!;
 		}
 
@@ -244,6 +247,12 @@ export class HostRouter {
 	}
 }
 
+// Instance information required to location and load an instance
+type UnloadedInstance = {
+	path: string,
+	config: lib.InstanceConfig,
+};
+
 /**
  * Handles running the host
  *
@@ -253,19 +262,18 @@ export default class Host extends lib.Link {
 	declare "connector": HostConnector;
 
 	router = new HostRouter(this);
-	/**
-	 * Certificate authority used to validate TLS connections to the controller.
-	 */
-	tlsCa?: string;
 	pluginInfos: lib.PluginNodeEnvInfo[];
 	config: lib.HostConfig;
 
 	/** Mapping of plugin name to loaded plugin */
 	plugins: Map<string, BaseHostPlugin> = new Map();
 
+	/** A map from instance id to instance connection. Only present when instance is running. */
 	instanceConnections = new Map<number, InstanceConnection>();
-	discoveredInstanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
-	instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
+	/** A map of instances that exist in the instance directory of the host */
+	discoveredInstances = new Map<number, UnloadedInstance>();
+	/** A map of instances that are assigned to this host */
+	assignedInstances = new Map<number, UnloadedInstance>();
 
 	adminlist = new Set<string>();
 	banlist = new Map<string, string>();
@@ -278,6 +286,8 @@ export default class Host extends lib.Link {
 	_disconnecting = false;
 	_shuttingDown = false;
 
+	startedAtMs = Date.now();
+
 	static instanceConfigWarning = {
 		"_warning": "Changes to this file will be overwritten by the controller's copy.",
 	};
@@ -293,7 +303,6 @@ export default class Host extends lib.Link {
 	constructor(
 		connector: HostConnector,
 		hostConfig: lib.HostConfig,
-		tlsCa: string | undefined,
 		pluginInfos: lib.PluginNodeEnvInfo[],
 		/**
 		 * If true indicates that there is a process monitor present that
@@ -308,7 +317,6 @@ export default class Host extends lib.Link {
 		public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
 	) {
 		super(connector);
-		this.tlsCa = tlsCa;
 
 		this.pluginInfos = pluginInfos;
 		this.config = hostConfig;
@@ -436,6 +444,9 @@ export default class Host extends lib.Link {
 				this.plugins.set(pluginInfo.name, hostPlugin);
 
 			} catch (err: any) {
+				if (err.code === "InstallationError") {
+					throw err;
+				}
 				throw new lib.PluginError(pluginInfo.name, err);
 			}
 
@@ -580,18 +591,17 @@ export default class Host extends lib.Link {
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		let response = await phin({
-			url, method: "GET",
-			core: { ca: this.tlsCa } as object,
-			stream: true,
-		});
+		let response = await fetch(url);
+		if (!response.body) {
+			throw new Error(`Empty body downloading stream ${streamId} from controller`);
+		}
 
 		const file = lib.ModInfo.filename(mod.name, mod.version);
 		const filePath = path.join(this.modStore.modsDirectory, file);
 		const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip");
 		const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" });
 		await events.once(writeStream, "open");
-		response.pipe(writeStream);
+		stream.Readable.fromWeb(response.body).pipe(writeStream);
 		await finished(writeStream);
 		await fs.rename(tempFilePath, filePath);
 
@@ -650,16 +660,16 @@ export default class Host extends lib.Link {
 
 	async handleInstanceAssignInternalRequest(request: lib.InstanceAssignInternalRequest) {
 		let { instanceId, config } = request;
-		let instanceInfo = this.instanceInfos.get(instanceId);
-		if (instanceInfo) {
-			instanceInfo.config.update(config, true, "controller");
-			logger.verbose(`Updated config for ${instanceInfo.path}`, this.instanceLogMeta(instanceId, instanceInfo));
+		let instance = this.assignedInstances.get(instanceId);
+		if (instance) {
+			instance.config.update(config, true, "controller");
+			logger.verbose(`Updated config for ${instance.path}`, this.instanceLogMeta(instanceId, instance));
 
 		} else {
-			instanceInfo = this.discoveredInstanceInfos.get(instanceId);
-			if (instanceInfo) {
-				await Instance.populate_folders(instanceInfo.path);
-				instanceInfo.config.update(config, true, "controller");
+			instance = this.discoveredInstances.get(instanceId);
+			if (instance) {
+				await Instance.populate_folders(instance.path);
+				instance.config.update(config, true, "controller");
 
 			} else {
 				let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
@@ -670,16 +680,16 @@ export default class Host extends lib.Link {
 
 				logger.info(`Creating ${instanceDir}`);
 				await Instance.populate_folders(instanceDir);
-				instanceInfo = {
+				instance = {
 					path: instanceDir,
 					config: instanceConfig,
 				};
 
-				this.discoveredInstanceInfos.set(instanceId, instanceInfo);
+				this.discoveredInstances.set(instanceId, instance);
 			}
 
-			this.instanceInfos.set(instanceId, instanceInfo);
-			logger.verbose(`assigned instance ${instanceInfo.config.get("instance.name")}`);
+			this.assignedInstances.set(instanceId, instance);
+			logger.verbose(`assigned instance ${instance.config.get("instance.name")}`);
 		}
 
 		// Somewhat hacky, but in the event of a lost session the status is
@@ -694,45 +704,45 @@ export default class Host extends lib.Link {
 				instanceConnection
 					? instanceConnection.instance.server.version
 					// TODO: factorio.version is not validated
-					: instanceInfo.config.get("factorio.version") as lib.TargetVersion,
+					: instance.config.get("factorio.version") as lib.TargetVersion,
 			)
 		);
 
 		// Send the new list of saves for this assigned instance to the controller.
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 
 		// save a copy of the instance config
-		await instanceInfo.config.save();
+		await instance.config.save();
 	}
 
 	async handleInstanceUnassignInternalRequest(request: lib.InstanceUnassignInternalRequest) {
 		let instanceId = request.instanceId;
-		let instanceInfo = this.instanceInfos.get(instanceId);
-		if (instanceInfo) {
+		let instance = this.assignedInstances.get(instanceId);
+		if (instance) {
 			let instanceConnection = this.instanceConnections.get(instanceId);
 			if (instanceConnection && ["starting", "running"].includes(instanceConnection.instance.status)) {
 				await instanceConnection.send(new lib.InstanceStopRequest());
 			}
 
-			this.instanceInfos.delete(instanceId);
-			logger.verbose(`unassigned instance ${instanceInfo.config.get("instance.name")}`);
+			this.assignedInstances.delete(instanceId);
+			logger.verbose(`unassigned instance ${instance.config.get("instance.name")}`);
 		}
 	}
 
-	instanceLogMeta(instanceId: number, instanceInfo?: { config: lib.InstanceConfig }) {
-		instanceInfo = instanceInfo || this.instanceInfos.get(instanceId);
-		if (!instanceInfo) {
+	instanceLogMeta(instanceId: number, instance?: { config: lib.InstanceConfig }) {
+		instance = instance || this.assignedInstances.get(instanceId);
+		if (!instance) {
 			return { instance_id: instanceId, instance_name: String(instanceId) };
 		}
-		return { instance_id: instanceId, instance_name: instanceInfo.config.get("instance.name") };
+		return { instance_id: instanceId, instance_name: instance.config.get("instance.name") };
 	}
 
-	getRequestInstanceInfo(instanceId: number) {
-		let instanceInfo = this.instanceInfos.get(instanceId);
-		if (!instanceInfo) {
+	getRequestInstance(instanceId: number) {
+		let instance = this.assignedInstances.get(instanceId);
+		if (!instance) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
 		}
-		return instanceInfo;
+		return instance;
 	}
 
 	/**
@@ -741,7 +751,7 @@ export default class Host extends lib.Link {
 	 * @returns Assigned game port or undefined if it does not exist.
 	 */
 	gamePort(instanceId: number) {
-		const instance = this.discoveredInstanceInfos.get(instanceId);
+		const instance = this.discoveredInstances.get(instanceId);
 		if (!instance) {
 			return undefined;
 		}
@@ -754,7 +764,7 @@ export default class Host extends lib.Link {
 
 	assignGamePort(instanceId: number) {
 		const availablePorts = lib.parseRanges(this.config.get("host.factorio_port_range"), 1, 2 ** 16 - 1);
-		for (const [id, instance] of this.instanceInfos) {
+		for (const [id, instance] of this.assignedInstances) {
 			if (id === instanceId) {
 				continue;
 			}
@@ -766,7 +776,7 @@ export default class Host extends lib.Link {
 				availablePorts.delete(port);
 			}
 		}
-		const instanceToAssign = this.instanceInfos.get(instanceId)!;
+		const instanceToAssign = this.assignedInstances.get(instanceId)!;
 		const assignedPort = instanceToAssign.config.get("factorio.host_assigned_game_port");
 		if (assignedPort !== null && availablePorts.has(assignedPort)) {
 			return assignedPort;
@@ -786,7 +796,7 @@ export default class Host extends lib.Link {
 	 * @returns connection to instance.
 	 */
 	async _connectInstance(instanceId: number) {
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let unloadedInstance = this.getRequestInstance(instanceId);
 		if (this.instanceConnections.has(instanceId)) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
 		}
@@ -795,7 +805,9 @@ export default class Host extends lib.Link {
 		let instanceAddress = new lib.Address(lib.Address.instance, instanceId);
 		let [connectionClient, connectionServer] = lib.VirtualConnector.makePair(instanceAddress, hostAddress);
 		let instance = new Instance(
-			this, connectionClient, instanceInfo.path, this.config.get("host.factorio_directory"), instanceInfo.config
+			this, connectionClient,
+			unloadedInstance.path, this.config.get("host.factorio_directory"),
+			unloadedInstance.config
 		);
 		let instanceConnection = new InstanceConnection(connectionServer, this, instance);
 
@@ -807,7 +819,7 @@ export default class Host extends lib.Link {
 
 	async handleSystemInfoRequest() {
 		const restartRequired = await this.checkRestartRequired();
-		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, restartRequired);
+		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, restartRequired, this.startedAtMs);
 	}
 
 	async checkRestartRequired() {
@@ -894,19 +906,19 @@ export default class Host extends lib.Link {
 		src: lib.Address,
 		dst: lib.Address
 	) {
-		let instanceInfo = this.getRequestInstanceInfo(dst.id);
-		return await Instance.listSaves(dst.id, path.join(instanceInfo.path, "saves"), null);
+		let instance = this.getRequestInstance(dst.id);
+		return await Instance.listSaves(dst.id, path.join(instance.path, "saves"), null);
 	}
 
 	async handleInstanceRenameSaveRequest(request: lib.InstanceRenameSaveRequest) {
 		let { instanceId, oldName, newName } = request;
 		checkRequestSaveName(oldName);
 		checkRequestSaveName(newName);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 		try {
 			await fs.move(
-				path.join(instanceInfo.path, "saves", oldName),
-				path.join(instanceInfo.path, "saves", newName),
+				path.join(instance.path, "saves", oldName),
+				path.join(instance.path, "saves", newName),
 				{ overwrite: false },
 			);
 		} catch (err: any) {
@@ -915,18 +927,18 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 	}
 
 	async handleInstanceCopySaveRequest(request: lib.InstanceCopySaveRequest) {
 		let { instanceId, source, destination } = request;
 		checkRequestSaveName(source);
 		checkRequestSaveName(destination);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 		try {
 			await fs.copy(
-				path.join(instanceInfo.path, "saves", source),
-				path.join(instanceInfo.path, "saves", destination),
+				path.join(instance.path, "saves", source),
+				path.join(instance.path, "saves", destination),
 				{ overwrite: false, errorOnExist: true },
 			);
 		} catch (err: any) {
@@ -935,33 +947,33 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 	}
 
 	async handleInstanceTransferSaveRequest(request: lib.InstanceTransferSaveRequest) {
 		let { sourceName, targetName, copy, sourceInstanceId, targetInstanceId } = request;
 		checkRequestSaveName(sourceName);
 		checkRequestSaveName(targetName);
-		let sourceInstanceInfo = this.getRequestInstanceInfo(sourceInstanceId);
-		let targetInstanceInfo = this.getRequestInstanceInfo(targetInstanceId);
+		let sourceInstance = this.getRequestInstance(sourceInstanceId);
+		let targetInstance = this.getRequestInstance(targetInstanceId);
 
 		// For consistency with remote transfer initiated through pullSave the
 		// target is renamed if it already exists.
 		targetName = await lib.findUnusedName(
-			path.join(targetInstanceInfo.path, "saves"), targetName, ".zip"
+			path.join(targetInstance.path, "saves"), targetName, ".zip"
 		);
 
 		try {
 			if (copy) {
 				await fs.copy(
-					path.join(sourceInstanceInfo.path, "saves", sourceName),
-					path.join(targetInstanceInfo.path, "saves", targetName),
+					path.join(sourceInstance.path, "saves", sourceName),
+					path.join(targetInstance.path, "saves", targetName),
 					{ overwrite: true },
 				);
 			} else {
 				await fs.move(
-					path.join(sourceInstanceInfo.path, "saves", sourceName),
-					path.join(targetInstanceInfo.path, "saves", targetName),
+					path.join(sourceInstance.path, "saves", sourceName),
+					path.join(targetInstance.path, "saves", targetName),
 					{ overwrite: true },
 				);
 			}
@@ -971,8 +983,8 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(sourceInstanceId, path.join(sourceInstanceInfo.path, "saves"));
-		await this.sendSaveListUpdate(targetInstanceId, path.join(targetInstanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(sourceInstanceId, path.join(sourceInstance.path, "saves"));
+		await this.sendSaveListUpdate(targetInstanceId, path.join(targetInstance.path, "saves"));
 
 		return targetName;
 	}
@@ -992,38 +1004,34 @@ export default class Host extends lib.Link {
 	async handleInstanceDeleteSaveRequest(request: lib.InstanceDeleteSaveRequest) {
 		let { instanceId, name } = request;
 		checkRequestSaveName(name);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 
 		try {
-			await fs.unlink(path.join(instanceInfo.path, "saves", name));
+			await fs.unlink(path.join(instance.path, "saves", name));
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
 				throw new lib.RequestError(`${name} does not exist`);
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 	}
 
 	async handleInstancePullSaveRequest(request: lib.InstancePullSaveRequest) {
 		let { instanceId, streamId, name } = request;
 		checkRequestSaveName(name);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		let response = await phin({
-			url, method: "GET",
-			core: { ca: this.tlsCa } as object,
-			stream: true,
-		});
+		let response = await fetch(url);
 
-		if (response.statusCode !== 200) {
-			let content = await lib.readStream(response);
-			throw new lib.RequestError(`Stream returned ${response.statusCode}: ${content.toString()}`);
+		if (response.status !== 200 || !response.body) {
+			let body = Buffer.from(await response.arrayBuffer());
+			throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`);
 		}
 
-		let savesDir = path.join(instanceInfo.path, "saves");
+		let savesDir = path.join(instance.path, "saves");
 		let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip");
 		let writeStream: NodeJS.WritableStream;
 		while (true) {
@@ -1039,7 +1047,7 @@ export default class Host extends lib.Link {
 				}
 			}
 		}
-		response.pipe(writeStream);
+		stream.Readable.fromWeb(response.body).pipe(writeStream);
 		await finished(writeStream);
 
 		name = await lib.findUnusedName(savesDir, name, ".zip");
@@ -1052,12 +1060,13 @@ export default class Host extends lib.Link {
 	async handleInstancePushSaveRequest(request: lib.InstancePushSaveRequest) {
 		let { instanceId, streamId, name } = request;
 		checkRequestSaveName(name);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 
-		let content: Buffer;
+		let content = fs.createReadStream(path.join(instance.path, "saves", name));
 		try {
-			// phin doesn't support streaming requests :(
-			content = await fs.readFile(path.join(instanceInfo.path, "saves", name));
+			// TODO: Due to a bug in fs-extra the ready event is not emitted here. Replace
+			// "open" with "ready" when fs-extra is replaced with node's fs module.
+			await events.once(content, "open");
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
 				throw new lib.RequestError(`${name} does not exist`);
@@ -1067,10 +1076,10 @@ export default class Host extends lib.Link {
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		phin({
-			url, method: "PUT",
-			core: { ca: this.tlsCa } as object,
-			data: content,
+		fetch(url, {
+			method: "PUT",
+			body: content,
+			duplex: "half",
 		}).catch(err => {
 			logger.error(`Error pushing save to controller:\n${err.stack}`, this.instanceLogMeta(instanceId));
 		});
@@ -1082,14 +1091,14 @@ export default class Host extends lib.Link {
 			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
 		}
 
-		let instanceInfo = this.discoveredInstanceInfos.get(instanceId);
-		if (!instanceInfo) {
+		let instance = this.discoveredInstances.get(instanceId);
+		if (!instance) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
 		}
 
-		this.discoveredInstanceInfos.delete(instanceId);
-		this.instanceInfos.delete(instanceId);
-		await fs.remove(instanceInfo.path);
+		this.discoveredInstances.delete(instanceId);
+		this.assignedInstances.delete(instanceId);
+		await fs.remove(instance.path);
 	}
 
 	async handleHostUpdateRequest(request: lib.HostUpdateRequest) {
@@ -1139,12 +1148,12 @@ export default class Host extends lib.Link {
 	 * the host and controller with the new list of instances.
 	 */
 	async updateInstances() {
-		this.discoveredInstanceInfos = await discoverInstances(this.config.get("host.instances_directory"));
+		this.discoveredInstances = await discoverInstances(this.config.get("host.instances_directory"));
 		let list = [];
-		for (let [instanceId, instanceInfo] of this.discoveredInstanceInfos) {
+		for (let [instanceId, instance] of this.discoveredInstances) {
 			let instanceConnection = this.instanceConnections.get(instanceId);
 			list.push(new lib.HostInstanceUpdate(
-				instanceInfo.config.toRemote("controller"),
+				instance.config.toRemote("controller"),
 				instanceConnection ? instanceConnection.instance.status : "stopped",
 				instanceConnection ? instanceConnection.instance.server.gamePort : this.gamePort(instanceId),
 				instanceConnection ? instanceConnection.instance.server.version : undefined,
@@ -1156,10 +1165,10 @@ export default class Host extends lib.Link {
 		if (this._startup) {
 			this._startup = false;
 
-			for (let [instanceId, instanceInfo] of this.instanceInfos) {
-				if (instanceInfo.config.get("instance.auto_start")) {
+			for (let [instanceId, instance] of this.assignedInstances) {
+				if (instance.config.get("instance.auto_start")) {
 					if (this.recoveryMode) {
-						logger.warn(`Recovery | skipping auto startup for ${instanceInfo.config.get("instance.name")}`);
+						logger.warn(`Recovery | skipping auto startup for ${instance.config.get("instance.name")}`);
 						continue;
 					}
 					try {
@@ -1167,8 +1176,8 @@ export default class Host extends lib.Link {
 						await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
 					} catch (err: any) {
 						logger.error(
-							`Error during auto startup for ${instanceInfo.config.get("instance.name")}:\n${err.stack}`,
-							this.instanceLogMeta(instanceId, instanceInfo)
+							`Error during auto startup for ${instance.config.get("instance.name")}:\n${err.stack}`,
+							this.instanceLogMeta(instanceId, instance)
 						);
 					}
 				}
@@ -1216,7 +1225,7 @@ export default class Host extends lib.Link {
 		await this.config.save();
 		// Save host side in case host_assigned_game_port changed in the config.
 		await Promise.all(
-			[...this.instanceInfos.values()].map(instanceInfo => instanceInfo.config.save())
+			[...this.assignedInstances.values()].map(instance => instance.config.save())
 		);
 
 		try {

packages/host/src/Instance.ts

@@ -1,7 +1,6 @@
 import fs from "fs-extra";
 import path from "path";
 import pidusage from "pidusage";
-import phin from "phin";
 import util from "util";
 import type { Static } from "@sinclair/typebox";
 import { exec } from "child_process";
@@ -638,9 +637,12 @@ end`.replace(/\r?\n/g, " ");
 	}
 
 	async init(pluginInfos: lib.PluginNodeEnvInfo[]) {
+		// After changing the status this function is responsible for calling notifyExit if it throws.
 		this.notifyStatus("starting");
 		try {
+			const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest());
 			await this._loadStats();
+			await this.server.checkForUpdates(factorioVersions);
 			await this.server.init();
 		} catch (err) {
 			this.notifyExit();
@@ -843,7 +845,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_adminlist") !== "disabled") {
 			this.logger.verbose("Writing server-adminlist.json");
-			lib.safeOutputFile(
+			await lib.safeOutputFile(
 				this.server.writePath("server-adminlist.json"),
 				JSON.stringify([...this._host.adminlist], null, "\t")
 			);
@@ -851,7 +853,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_banlist") !== "disabled") {
 			this.logger.verbose("Writing server-banlist.json");
-			lib.safeOutputFile(
+			await lib.safeOutputFile(
 				this.server.writePath("server-banlist.json"),
 				JSON.stringify([...this._host.banlist].map(
 					([username, reason]) => ({ username, reason })
@@ -861,10 +863,21 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_whitelist") !== "disabled") {
 			this.logger.verbose("Writing server-whitelist.json");
-			lib.safeOutputFile(
+			await lib.safeOutputFile(
 				this.server.writePath("server-whitelist.json"),
 				JSON.stringify([...this._host.whitelist], null, "\t")
 			);
+		} else {
+			// Always create the whitelist even if unused
+			// This allows it to be watched for changes without a server restart
+			try {
+				// wx will throw an error if the file already exists to prevent overwrite
+				await fs.writeFile(this.server.writePath("server-whitelist.json"), "\n[]\n", { flag: "wx" });
+			} catch (err: any) {
+				if (err.code !== "EEXIST") {
+					throw err;
+				}
+			}
 		}
 
 		await this.syncMods();
@@ -1290,17 +1303,18 @@ end`.replace(/\r?\n/g, " ");
 			let url = new URL(this._host.config.get("host.controller_url"));
 			url.pathname += "api/upload-export";
 			url.searchParams.set("mod_pack_id", String(this.activeModPack.id));
-			let response = await phin({
-				url, method: "PUT",
-				data: content,
-				core: { ca: this._host.tlsCa } as object,
+			let response = await fetch(url, {
+				method: "PUT",
+				body: content,
+				duplex: "half",
 				headers: {
 					"Content-Type": "application/zip",
 					"x-access-token": this._host.config.get("host.controller_token"),
 				},
 			});
-			if (response.statusCode !== 200) {
-				throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`);
+			if (response.status !== 200) {
+				const body = Buffer.from(await response.arrayBuffer());
+				throw Error(`Upload failed: ${response.status} ${response.statusText}: ${body.toString()}`);
 			}
 
 		} finally {

packages/host/src/export.ts

@@ -98,7 +98,8 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
 async function loadFile(server: FactorioServer, modVersions: Map<string, string>, modPath: string) {
 	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
 	if (!match) {
-		throw new Error(`Bad mod path ${match}`);
+		server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`);
+		return null;
 	}
 
 	let [, mod, filePath] = match;
@@ -151,7 +152,6 @@ async function loadIcon(
 			iconCache.set(iconPath, icon);
 		} else {
 			icon = null;
-			server._logger.warn(`${iconPath} not found`);
 		}
 		iconCache.set(iconPath, icon);
 	}
@@ -258,6 +258,14 @@ function fixIcons(item: ItemPrototype) {
 	return item;
 }
 
+/** Get the primary icon path from a prototype (first layer for layered, .icon for simple). */
+function getPrimaryIconPath(item: ItemPrototype): string | undefined {
+	if (item.icons) {
+		return (item as LayeredIconSpecification).icons[0]?.icon;
+	}
+	return (item as SimpleIconSpecification).icon;
+}
+
 const itemTypes = new Set([
 	"item",
 	"ammo",
@@ -280,87 +288,153 @@ const itemTypes = new Set([
 	"armor",
 	"mining-tool",
 	"repair-tool",
-
 	// XXX Bad hack to get icons for fluids. These should be treated as a
 	// separate namespace from items as they may be named the same as items.
 	"fluid",
-	// TODO consider adding virtual signals and recipes
 ]);
 
-function filterItems(prototypes: Prototypes): ItemPrototype[] {
+const recipeTypes = new Set(["recipe"]);
+const signalTypes = new Set(["virtual-signal"]);
+const technologyTypes = new Set(["technology"]);
+const planetTypes = new Set(["planet", "space-location"]);
+const qualityTypes = new Set(["quality"]);
+
+// Entity types to export. Entities whose name matches an already-exported
+const entityTypes = new Set([
+	"assembling-machine", "furnace", "inserter", "transport-belt", "underground-belt",
+	"splitter", "loader", "loader-1x1", "mining-drill", "container", "logistic-container",
+	"electric-pole", "pipe", "pipe-to-ground", "pump", "storage-tank", "boiler",
+	"generator", "solar-panel", "accumulator", "reactor", "heat-pipe", "lab", "beacon",
+	"roboport", "lamp", "radar", "wall", "gate", "land-mine", "turret", "ammo-turret",
+	"electric-turret", "fluid-turret", "artillery-turret", "rocket-silo", "car",
+	"spider-vehicle", "locomotive", "cargo-wagon", "fluid-wagon", "artillery-wagon",
+	"train-stop", "rail-signal", "rail-chain-signal", "straight-rail", "curved-rail-a",
+	"curved-rail-b", "half-diagonal-rail", "elevated-straight-rail", "elevated-curved-rail-a",
+	"elevated-curved-rail-b", "elevated-half-diagonal-rail", "rail-ramp", "rail-support",
+	"space-platform-hub", "cargo-bay", "asteroid-collector", "thruster", "cargo-pod",
+	"cargo-landing-pad", "asteroid",
+	"construction-robot", "logistic-robot", "combat-robot", "capture-robot",
+	"unit", "unit-spawner", "spider-unit", "segmented-unit",
+	"tree", "plant", "fish", "resource", "cliff", "simple-entity",
+	"simple-entity-with-owner", "simple-entity-with-force",
+	"arithmetic-combinator", "decider-combinator", "constant-combinator",
+	"selector-combinator", "programmable-speaker", "power-switch", "display-panel",
+	"character",
+	"offshore-pump", "valve", "lightning-attractor",
+	"fusion-generator", "fusion-reactor", "burner-generator",
+	"agricultural-tower", "market", "lane-splitter",
+]);
+
+function filterPrototypes(prototypes: Prototypes, allowedTypes: Set<string>): ItemPrototype[] {
 	return Object.entries(prototypes)
-		.filter(([type, _]) => itemTypes.has(type))
+		.filter(([type]) => allowedTypes.has(type))
 		.flatMap(([_, typePrototypes]) => Object.values(typePrototypes) as ItemPrototype[])
-		.map(fixIcons)
-	;
+		.map(fixIcons);
+}
+
+/** Shared state for building a unified spritesheet across all categories. */
+interface SheetState {
+	sheet: Jimp;
+	metadata: Map<string, { x: number, y: number, size: number, category: string, path?: string }>;
+	iconCache: IconCache;
+	simpleIcons: Map<string, number>;
+	pos: number;
+}
+
+const SHEET_ICON_SIZE = 32;
+const SHEET_WIDTH = 1024;
+
+async function createSheetState(): Promise<SheetState> {
+	return {
+		sheet: await Jimp.create(SHEET_WIDTH, SHEET_ICON_SIZE),
+		metadata: new Map(),
+		iconCache: new Map(),
+		simpleIcons: new Map(),
+		pos: 0,
+	};
+}
+
+/** Grow the spritesheet vertically if needed to fit another row. */
+async function growSheet(state: SheetState) {
+	const row = Math.floor(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
+	const neededHeight = (row + 1) * SHEET_ICON_SIZE;
+	if (neededHeight > state.sheet.bitmap.height) {
+		const extended = await Jimp.create(SHEET_WIDTH, neededHeight);
+		extended.composite(state.sheet, 0, 0);
+		(state.sheet as any).bitmap = (extended as any).bitmap;
+	}
 }
 
 /**
- * Export item icons and data
+ * Export item icons for a single category onto the shared spritesheet.
  *
- * Assembles and packs the icons for the item prototypes given into a single
- * spritesheet and json file with meta data.
- *
- * @param server -
- *     The server to generate the export mod for.
- * @param modVersions -
- *     Mapping of mod name to versions to get icons from.
- * @param items - Array of item prototypes.
- * @return Item spritesheet and metadata.
+ * @param server - The server to load icons from.
+ * @param modVersions - Mapping of mod name to version.
+ * @param items - Array of item prototypes to export.
+ * @param category - Category name for metadata.
+ * @param state - Shared spritesheet state (mutated in place).
+ * @returns Number of icons exported for this category.
  * @internal
  */
-async function exportItems(server: FactorioServer, modVersions: Map<string, string>, items: ItemPrototype[]) {
-
-	// Size to render icons at
-	let size = 32;
-
-	// Width of spritesheet
-	let width = 1024;
-
-	let rows = Math.ceil(items.length / (width / size));
-	let iconSheet = await Jimp.create(width, rows * size);
-	let itemData = new Map();
-	let pos = 0;
-
-	let iconCache: IconCache = new Map();
-	let simpleIcons = new Map();
+async function exportItems(
+	server: FactorioServer,
+	modVersions: Map<string, string>,
+	items: ItemPrototype[],
+	category: string,
+	state: SheetState,
+) {
+	let count = 0;
 	for (let item of items) {
+		// Skip prototypes with no icon data at all
+		if (!item.icons && !item.icon) {
+			continue;
+		}
+
 		let icon: Jimp | null = null;
 		let iconPos: number | undefined;
 		if (item.icons) {
-			icon = await loadLayeredIcon(server, modVersions, item as LayeredIconSpecification, size, iconCache);
-			iconPos = pos;
+			icon = await loadLayeredIcon(
+				server, modVersions, item as LayeredIconSpecification, SHEET_ICON_SIZE, state.iconCache,
+			);
+			iconPos = state.pos;
 
 		} else {
-			iconPos = simpleIcons.get(item.icon);
+			iconPos = state.simpleIcons.get(item.icon as string);
 			if (iconPos === undefined) {
-				icon = await loadSimpleIcon(server, modVersions, item as SimpleIconSpecification, size, iconCache);
+				icon = await loadSimpleIcon(
+					server, modVersions, item as SimpleIconSpecification, SHEET_ICON_SIZE, state.iconCache,
+				);
 				if (icon) {
-					iconPos = pos;
-					simpleIcons.set(item.icon, pos);
+					iconPos = state.pos;
+					state.simpleIcons.set(item.icon as string, state.pos);
 				}
 			}
 		}
 
 		if (iconPos !== undefined) {
-			itemData.set(item.name, {
-				x: iconPos * size % width,
-				y: Math.floor(iconPos / (width / size)) * size,
-				size,
-				localised_name: item.localised_name,
-				localised_description: item.localised_description,
+			const iconPath = getPrimaryIconPath(item);
+			state.metadata.set(`${category}.${item.name}`, {
+				x: iconPos * SHEET_ICON_SIZE % SHEET_WIDTH,
+				y: Math.floor(iconPos / (SHEET_WIDTH / SHEET_ICON_SIZE)) * SHEET_ICON_SIZE,
+				size: SHEET_ICON_SIZE,
+				category,
+				...(iconPath ? { path: iconPath } : {}),
 			});
 		}
 
 		if (icon) {
-			iconSheet.composite(icon, pos * size % width, Math.floor(pos / (width / size)) * size);
-			pos += 1;
+			await growSheet(state);
+			state.sheet.composite(
+				icon,
+				state.pos * SHEET_ICON_SIZE % SHEET_WIDTH,
+				Math.floor(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE)) * SHEET_ICON_SIZE,
+			);
+			state.pos += 1;
+			count += 1;
 		}
 	}
 
-	iconSheet.crop(0, 0, width, Math.ceil(pos / (width / size)) * size);
-
-	return { iconSheet, itemData };
+	return count;
 }
 
 /**
@@ -505,7 +579,16 @@ export async function exportData(server: FactorioServer) {
 		}
 	}
 
-	let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes));
+	const categories: { name: string, types: Set<string> }[] = [
+		{ name: "item", types: itemTypes },
+		{ name: "recipe", types: recipeTypes },
+		{ name: "signal", types: signalTypes },
+		{ name: "technology", types: technologyTypes },
+		{ name: "planet", types: planetTypes },
+		{ name: "quality", types: qualityTypes },
+		{ name: "entity", types: entityTypes },
+	];
+
 	let locale = await exportLocale(server, modVersions, modOrder, "en");
 
 	// Free up the memory used by zip files loaded during the export.
@@ -514,9 +597,28 @@ export async function exportData(server: FactorioServer) {
 	let zip = new JSZip();
 	zip.file("export/settings.json", JSON.stringify(settings));
 	zip.file("export/prototypes.json", JSON.stringify(prototypes));
-	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
-	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
 	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
+
+	// Build a single unified spritesheet across all prototype categories.
+	const state = await createSheetState();
+	for (const category of categories) {
+		const categoryItems = filterPrototypes(prototypes, category.types);
+
+		if (categoryItems.length === 0) {
+			continue;
+		}
+		const count = await exportItems(server, modVersions, categoryItems, category.name, state);
+		server._logger.info(`Exported ${count} ${category.name} icons`);
+	}
+
+	// Crop sheet to actual used height and write single spritesheet + metadata.
+	const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
+	state.sheet.crop(0, 0, SHEET_WIDTH, Math.max(usedRows * SHEET_ICON_SIZE, 1));
+	zip.file("export/spritesheet.png", await state.sheet.getBufferAsync(Jimp.MIME_PNG));
+	zip.file("export/metadata.json", JSON.stringify([...state.metadata.entries()]));
+
+	server._logger.info(`Export complete: ${state.metadata.size} icons on ${usedRows} row(s)`);
+
 	return zip;
 }
 

packages/host/src/server.ts

@@ -2,6 +2,7 @@
 import fs from "fs-extra";
 import child_process from "child_process";
 import path from "path";
+import JSZip from "jszip";
 import events from "events";
 import util from "util";
 import crypto from "crypto";
@@ -9,31 +10,33 @@ import { Rcon } from "rcon-client";
 
 import * as lib from "@clusterio/lib";
 
+const execFileAsync = util.promisify(child_process.execFile);
 
 /**
  * Determines the version of Factorio the datadir is pointing to by
  * reading the changelog.txt in it.
  *
- * @param changelogPath - Path to changelog.txt.
- * @internal
+ * @param factorioDir - Path factorio dir containing data/changelog.txt.
  */
-async function getVersion(changelogPath: string) {
+async function getFactorioVersion(factorioDir: string) {
 	let changelog;
+	const changelogPath = path.join(factorioDir, "data", "changelog.txt");
 	try {
 		changelog = await fs.readFile(changelogPath, "utf-8");
 	} catch (err: any) {
-		if (err.code === "ENOENT") {
+		if (err.code === "ENOENT" || err.code === "ENOTDIR") {
 			return null;
 		}
 		throw err;
 	}
 
-	for (let line of changelog.split(/[\r\n]+/)) {
-		let index = line.indexOf(":");
+	for (const line of changelog.split(/[\r\n]+/)) {
+		const index = line.indexOf(":");
 		if (index !== -1) {
-			let name = line.slice(0, index).trim();
+			const name = line.slice(0, index).trim();
 			if (name.toLowerCase() === "version") {
-				return line.slice(index + 1).trim();
+				const version = line.slice(index + 1).trim();
+				return lib.isFullVersion(version) ? version : null;
 			}
 		}
 	}
@@ -66,13 +69,113 @@ function versionOrder(a: string, b: string) {
 	return 0;
 }
 
+/**
+ * Find all installed factorio versions and return them as a set
+ *
+ * @param factorioDir - Path to Factorio installation dirs.
+ * @returns Set of installed factorio versions
+ */
+async function listFactorioVersions(factorioDir: string) {
+	const versions = new Set<lib.FullVersion>();
+
+	const directVersion = await getFactorioVersion(factorioDir);
+	if (directVersion !== null) {
+		versions.add(directVersion);
+		return { direct: true, versions };
+	}
+
+	for (const name of await fs.readdir(factorioDir)) {
+		// Avoid any fancy directory detection in order to follow symlinks
+		const fullPath = path.join(factorioDir, name);
+		const version = await getFactorioVersion(fullPath);
+		if (version === null) {
+			continue;
+		}
+
+		versions.add(version);
+	}
+
+	return { direct: false, versions };
+}
+
+/**
+ * Downloads a ZIP file from a URL using native fetch and extracts it
+ *
+ * @param url - The URL of the ZIP file
+ * @param targetDir - Directory to extract the ZIP into
+ * @throws Network error if status of request is not ok
+ */
+async function downloadAndExtractZip(url: string, targetDir: string) {
+	const res = await fetch(url);
+	if (!res.ok) {
+		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+	}
+
+  	// Load the ZIP with JSZip
+	const buffer = Buffer.from(await res.arrayBuffer());
+	const zip = await JSZip.loadAsync(buffer);
+
+  	// Extract each file/folder
+	await fs.ensureDir(targetDir);
+	await Promise.all(
+		Object.values(zip.files).map(async (entry) => {
+			const parts = entry.name.split("/").filter(Boolean);
+			const strippedPath = parts.slice(1).join("/");
+			if (!strippedPath) {
+				return; // This copies the behaviour of tar --strip-components 1
+			}
+
+			const entryPath = path.join(targetDir, strippedPath);
+			if (entry.dir) {
+				await fs.ensureDir(entryPath);
+			} else {
+				await fs.ensureDir(path.dirname(entryPath));
+				await fs.writeFile(entryPath, await entry.async("nodebuffer"));
+			}
+		})
+	);
+}
+
+/**
+ * Downloads a tar.gz file from a URL using native fetch and extracts it
+ *
+ * @param url - The URL of the ZIP file
+ * @param targetDir - Directory to extract the ZIP into
+ * @throws Network error if status of request is not ok
+ */
+async function downloadAndExtractTar(url: string, targetDir: string) {
+	const res = await fetch(url);
+	if (!res.ok) {
+		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+	}
+
+	// Write the buffer to a temp file
+	const tmpFilePath = `${targetDir}.tmp`;
+	const buffer = Buffer.from(await res.arrayBuffer());
+	await fs.writeFile(tmpFilePath, buffer);
+	await fs.ensureDir(targetDir);
+
+	// Execute tar CLI to extract
+	try {
+		await execFileAsync("tar", ["-xf", tmpFilePath, "-C", targetDir, "--strip-components", "1"]);
+	} catch (err: any) {
+		if (err.code === "ENOENT") {
+			throw new Error("error executing tar command, do you have 'xz-utils' installed?");
+		}
+		throw err;
+	} finally {
+		// Clean up temp file
+		await fs.remove(tmpFilePath);
+	}
+}
+
 /**
  * Find factorio data directory of the given version
  *
  * Searches the given factorio dir for an installation of Factorio of
  * the target version.
  *
- * @param factorioDir - Path to Factorio installation dir(s).
+ * @param factorioDir - Path to Factorio installation dirs.
  * @param targetVersion -
  *     Version to look for, supports the special value "latest" for the
  *     latest version available.
@@ -80,40 +183,30 @@ function versionOrder(a: string, b: string) {
  * @internal
  */
 async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion): Promise<[string, lib.FullVersion]> {
-
-	// There are two supported setups: having the factorio dir be the actual
-	// install directory, and having the factorio dir be a folder containing
-	// multiple install directories
-
-	let simpleVersion = await getVersion(path.join(factorioDir, "data", "changelog.txt"));
-	if (simpleVersion !== null) {
-		if (simpleVersion === targetVersion || simpleVersion.startsWith(targetVersion) || targetVersion === "latest") {
-			if (lib.isFullVersion(simpleVersion)) {
-				return [path.join(factorioDir, "data"), simpleVersion];
-			}
+	// Check if this is a direct installation dir
+	const directVersion = await getFactorioVersion(factorioDir);
+	if (directVersion !== null) {
+		if (targetVersion === "latest" || directVersion === targetVersion || directVersion.startsWith(targetVersion)) {
+			return [path.join(factorioDir, "data"), directVersion];
 		}
-
-		throw new Error(
-			`Factorio version ${targetVersion} was requested, but install directory contains ${simpleVersion}`
-		);
+		throw new Error(`Unable to find Factorio version ${targetVersion}`);
 	}
 
-	let versions = new Map<lib.FullVersion, string>();
-	for (let entry of await fs.readdir(factorioDir, { withFileTypes: true })) {
-		if (!entry.isDirectory()) {
-			continue;
-		}
-
-		let version = await getVersion(path.join(factorioDir, entry.name, "data", "changelog.txt"));
-		if (version === null || !lib.isFullVersion(version)) {
+	// Otherwise assume it is a directory of multiple versions
+	const versions = new Map<lib.FullVersion, string>();
+	for (const name of await fs.readdir(factorioDir)) {
+		// Avoid any fancy directory detection in order to follow symlinks
+		const fullPath = path.join(factorioDir, name);
+		const version = await getFactorioVersion(fullPath);
+		if (version === null) {
 			continue;
 		}
 
 		if (version === targetVersion) {
-			return [path.join(factorioDir, entry.name, "data"), version];
+			return [path.join(factorioDir, name, "data"), version];
 		}
 
-		versions.set(version, entry.name);
+		versions.set(version, name);
 	}
 
 	if (!versions.size) {
@@ -794,7 +887,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		try {
 			this._whitelistWatcher = fs.watch(filePath);
 		} catch (err: any) {
-			this._logger.error(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`);
+			this._logger.warn(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`);
 			return;
 		}
 
@@ -871,6 +964,41 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		this._state = "init";
 	}
 
+	/**
+	 * Check of updated versions to download
+	 *
+	 * Should be called before init for the new version to be detected during init
+	 */
+	async checkForUpdates(availableVersions: lib.ExternalFactorioVersion[]) {
+		const targetVersion = this._targetVersion;
+		const latestVersion = availableVersions
+			.filter(v => (
+				targetVersion === "latest" || v.version === targetVersion || v.version.startsWith(targetVersion)
+			))
+			.sort((v1, v2) => versionOrder(v1.version, v2.version))
+			[0];
+
+		if (!latestVersion) {
+			return;
+		}
+
+		const installedVersions = await listFactorioVersions(this._factorioDir);
+		if (!installedVersions.versions.has(latestVersion.version)) {
+			const v = latestVersion.version;
+			// Can't download if not linux or this is a direct install
+			if (installedVersions.direct || process.platform !== "linux") {
+				this._logger.info(`A newer version of factorio is available (${v}) but must be manually downloaded`);
+				return;
+			}
+
+			// Download the newer version
+			this._logger.info(`A newer version of factorio is available (${v}), starting download...`);
+			const dst = path.join(this._factorioDir, latestVersion.version);
+			await downloadAndExtractTar(latestVersion.headlessUrl, dst);
+			this._logger.info(`Downloaded factorio version ${v}`);
+		}
+	}
+
 	/**
 	 * The version of Factorio in use. This will be the actual version if
 	 * "latest" (the default) was specified as `factorioVersion` to the constructor.
@@ -1421,9 +1549,12 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 
 
 // For testing only
-export const _getVersion = getVersion;
+export const _getFactorioVersion = getFactorioVersion;
 export const _versionOrder = versionOrder;
 export const _findVersion = findVersion;
+export const _listFactorioVersions = listFactorioVersions;
+export const _downloadAndExtractZip = downloadAndExtractZip;
+export const _downloadAndExtractTar = downloadAndExtractTar;
 export const _randomDynamicPort = randomDynamicPort;
 export const _generatePassword = generatePassword;
 export const _parseOutput = parseOutput;

packages/lib/browser.ts

@@ -16,3 +16,6 @@ export * from "./src/subscriptions";
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as RateLimiter } from "./src/RateLimiter";
 export { default as ValueCache } from "./src/ValueCache";
+
+import { checkSingletonImport } from "./src/check_singleton_import";
+checkSingletonImport(import.meta.url);

packages/lib/index.ts

@@ -38,3 +38,6 @@ export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";
 export { default as ValueCache } from "./src/ValueCache";
+
+import { checkSingletonImport } from "./src/check_singleton_import";
+checkSingletonImport(__filename);

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@clusterio/lib",
 	"description": "Shared library for Clusterio",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.23",
 	"repository": "https://github.com/clusterio/clusterio",
 	"license": "MIT",
 	"scripts": {

packages/lib/src/check_singleton_import.ts

@@ -0,0 +1,21 @@
+import { InstallationError } from "./errors";
+declare global {
+	// eslint-disable-next-line vars-on-top
+	var _clusterioLibImportedFrom: string;
+}
+
+export function checkSingletonImport(location: string) {
+	if (!global._clusterioLibImportedFrom) {
+		global._clusterioLibImportedFrom = location;
+	} else {
+		throw new InstallationError(`Attempt to import duplicate copy of @clusterio/lib
+
+Importing more than one copy of @clusterio/lib into the same runtime
+will break Clusterio. This typically happens when the package
+manager has installed two different versions of @clusterio/lib
+and assigned different ones to different components of Clusterio.
+
+Previously imported @clusterio/lib was located at ${_clusterioLibImportedFrom}
+This copy of @clusterio/lib is located at ${location}`);
+	}
+}

packages/lib/src/config/definitions.ts

@@ -259,7 +259,6 @@ export interface HostConfigFields {
 	"host.instances_directory": string;
 	"host.controller_url": string;
 	"host.controller_token": string;
-	"host.tls_ca": string | null;
 	"host.public_address": string;
 	"host.factorio_port_range": string;
 	"host.factorio_username": string | null,
@@ -327,12 +326,6 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			type: "string",
 			initialValue: "enter token here",
 		},
-		"host.tls_ca": {
-			description: "Path to Certificate Authority to validate TLS connection to controller against.",
-			restartRequired: true,
-			type: "string",
-			optional: true,
-		},
 		"host.public_address": {
 			description: "Public facing address players should connect to in order to join instances on this host",
 			type: "string",
@@ -624,7 +617,6 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 export interface ControlConfigFields {
 	"control.controller_url": string | null;
 	"control.controller_token": string | null;
-	"control.tls_ca": string | null;
 	"control.max_reconnect_delay": number;
 }
 
@@ -647,11 +639,6 @@ export class ControlConfig extends classes.Config<ControlConfigFields> {
 			type: "string",
 			optional: true,
 		},
-		"control.tls_ca": {
-			description: "Path to Certificate Authority to validate TLS connection to controller against.",
-			type: "string",
-			optional: true,
-		},
 		"control.max_reconnect_delay": {
 			title: "Max Reconnect Delay",
 			description: "Maximum delay to wait before attempting to reconnect WebSocket",

packages/lib/src/data/ExportManifest.ts

@@ -11,21 +11,26 @@ export default class ExportManifest {
 		/**
 		 * Mapping between known assets and their file names.
 		 *
-		 * Currently exported assets are
+		 * Currently exported assets are:
 		 * - settings: JSON of mod settings prototypes.
 		 * - prototypes: JSON of all game prototypes except settings.
-		 * - item-spritesheet: PNG containing icons from the game.
-		 * - item-metadata: JSON of Map entries describing icons in item-spritesheet
 		 * - locale: JSON of flattened Map entries of en locale strings from the game.
+		 * - spritesheet: Single PNG spritesheet containing all icon categories.
+		 * - metadata: JSON array of [name, {x, y, size, category, path?}] entries
+		 *   describing sprite coordinates and category membership. Categories:
+		 *   item, recipe, signal, technology, planet, quality, entity, static.
 		 */
-		public assets: Record<string, string>
+		public assets: Record<string, string>,
+		/** ISO 8601 timestamp of when the export was uploaded. */
+		public exportedAt?: string,
 	) { }
 
 	static jsonSchema = Type.Object({
 		"assets": Type.Record(Type.String(), Type.String()),
+		"exportedAt": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof ExportManifest.jsonSchema>) {
-		return new this(json.assets);
+		return new this(json.assets, json.exportedAt);
 	}
 }

packages/lib/src/data/HostDetails.ts

@@ -0,0 +1,77 @@
+import { Type, Static } from "@sinclair/typebox";
+
+export default class HostDetails {
+	constructor(
+		/** Version of the host software. The controller may warn if there is a mismatch. */
+		public version: string,
+		/** Human-readable display name of the host. */
+		public name: string,
+		/** Unique numeric identifier for this host. */
+		public id: number,
+		/** Whether the host is currently connected to the controller. */
+		public connected: boolean,
+		/** Address from which the host has connected to the controller. */
+		public remoteAddress = "",
+		/** Public-facing IP address of the host, if available. */
+		public publicAddress: string = "",
+		/** Unix timestamp (sec) after which issued tokens are considered valid. */
+		public tokenValidAfter: number = 0,
+		/** Unix timestamp (ms) indicating when this record was last updated. */
+		public updatedAtMs = 0,
+		/** Flag indicating the host record has been removed on the controller. */
+		public isDeleted = false
+	) {}
+
+	static jsonSchema = Type.Object({
+		"version": Type.String(),
+		"name": Type.String(),
+		"id": Type.Integer(),
+		"connected": Type.Boolean(),
+		"remoteAddress": Type.Optional(Type.String()),
+		"publicAddress": Type.Optional(Type.String()),
+		"tokenValidAfter": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"isDeleted": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.version,
+			json.name,
+			json.id,
+			json.connected,
+			json.remoteAddress,
+			json.publicAddress,
+			json.tokenValidAfter,
+			json.updatedAtMs,
+			json.isDeleted
+		);
+	}
+
+	toJSON() {
+		const json = {
+			version: this.version,
+			name: this.name,
+			id: this.id,
+			connected: this.connected,
+		} as Static<typeof HostDetails.jsonSchema>;
+
+		if (this.remoteAddress !== "") {
+			json.remoteAddress = this.remoteAddress;
+		}
+		if (this.publicAddress !== "") {
+			json.publicAddress = this.publicAddress;
+		}
+		if (this.tokenValidAfter !== 0) {
+			json.tokenValidAfter = this.tokenValidAfter;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updatedAtMs = this.updatedAtMs;
+		}
+		if (this.isDeleted !== false) {
+			json.isDeleted = this.isDeleted;
+		}
+
+		return json;
+	}
+}

packages/lib/src/data/InstanceDetails.ts

@@ -0,0 +1,108 @@
+import { Static, Type } from "@sinclair/typebox";
+import { StringEnum } from "./composites";
+import { TargetVersion, TargetVersionSchema } from "./version";
+
+export const InstanceStatus = StringEnum([
+	"unknown",
+	"unassigned",
+	"stopped",
+	"starting",
+	"running",
+	"stopping",
+	"creating_save",
+	"exporting_data",
+	"deleted",
+]);
+
+/**
+ * Current status of the instance. One of:
+ * - `unknown`: Instance is assigned to a host but this host is currently
+ *   not connected to the contreller.
+ * - `unassigned`: Instance is not assigned to a a host and exists only on
+ *   the controller.
+ * - `stopped`: Instance is stopped.
+ * - `starting`: Instance is in the process of starting up.
+ * - `running`: Instance is running normally.
+ * - `stopping`: Instance is in the process of stopping.
+ * - `creating_save`: Instance is in the process of creating a save.
+ * - `exporting_data`: Instance is in the process of exporting game data.
+ * - `deleted`: Instance has been deleted.
+ */
+export type InstanceStatus = Static<typeof InstanceStatus>;
+
+export default class InstanceDetails {
+	constructor(
+		public name: string,
+		public id: number,
+		public assignedHost: number | undefined,
+		public gamePort: number | undefined,
+		public status: InstanceStatus,
+		public factorioVersion: TargetVersion | undefined,
+		public startedAtMs = 0,
+		/** Millisecond Unix timestamp this entry was last updated at */
+		public updatedAtMs = 0,
+		public excludeFromStartAll = false
+	) {}
+
+	get isDeleted() {
+		return this.status === "deleted";
+	}
+
+	static jsonSchema = Type.Object({
+		"name": Type.String(),
+		"id": Type.Integer(),
+		"assignedHost": Type.Optional(Type.Integer()),
+		"gamePort": Type.Optional(Type.Integer()),
+		"status": StringEnum([
+			"unknown", "unassigned", "stopped", "starting", "running", "stopping",
+			"creating_save", "exporting_data", "deleted",
+		]),
+		"factorioVersion": Type.Optional(TargetVersionSchema),
+		"startedAtMs": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"excludeFromStartAll": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.name,
+			json.id,
+			json.assignedHost,
+			json.gamePort,
+			json.status,
+			json.factorioVersion,
+			json.startedAtMs,
+			json.updatedAtMs,
+			json.excludeFromStartAll
+		);
+	}
+
+	toJSON() {
+		const json = {
+			name: this.name,
+			id: this.id,
+			status: this.status,
+		} as Static<typeof InstanceDetails.jsonSchema>;
+
+		if (this.assignedHost !== undefined) {
+			json.assignedHost = this.assignedHost;
+		}
+		if (this.gamePort !== undefined) {
+			json.gamePort = this.gamePort;
+		}
+		if (this.factorioVersion !== undefined) {
+			json.factorioVersion = this.factorioVersion;
+		}
+		if (this.startedAtMs !== 0) {
+			json.startedAtMs = this.startedAtMs;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updatedAtMs = this.updatedAtMs;
+		}
+		if (this.excludeFromStartAll !== false) {
+			json.excludeFromStartAll = this.excludeFromStartAll;
+		}
+
+		return json;
+	}
+}

packages/lib/src/data/ModInfo.ts

@@ -22,11 +22,12 @@ const UnsatisfiedSeverity: Record<ModDependencyUnsatisfiedReason, number> = {
 	"incompatible": 1, "missing_dependency": 2, "wrong_version": 3,
 };
 
-const lettersRegex = /^[a-zA-Z]+$/;
+const depSpecRegex = /^(?:(\?|\(\?\)|!|~|\+) *)?(.+?)(?: *([<>]=?|=) *([0-9.]+))?$/;
 
 export class ModDependency {
-	public type: ModDependencyType;
+	public spec: string;
 	public name: string;
+	public type: ModDependencyType;
 	public version: ModVersionEquality | undefined;
 
 	static getTypeFromPrefix(prefix: string): ModDependencyType {
@@ -42,50 +43,27 @@ export class ModDependency {
 			case "":
 				return "required";
 			default:
-				throw new Error(`Invalid dependency prefix: ${prefix}`);
+				throw new Error(`Invalid dependency prefix "${prefix}"`);
 		}
 	}
 
 	constructor (
-		public specification: string,
+		specification: string,
 	) {
-		const parts = specification.split(" ");
-
-		// Fix mods which include spaces in their name, joins consecutive words together
-		let index = 0;
-		while (index < parts.length - 1) {
-			const current = parts[index];
-			const next = parts[index + 1];
-			if (lettersRegex.test(current) && lettersRegex.test(next)) {
-				parts[index] = `${current} ${next}`;
-				parts.splice(index + 1, 1);
-			} else {
-				index += 1;
-			}
+		const match = depSpecRegex.exec(specification);
+		if (match === null) {
+			throw new Error(`Invalid dependency specification "${specification}"`);
 		}
 
-		// Parse the dependency specification
-		switch (parts.length) {
-			case 1:
-				this.type = "required";
-				this.name = parts[0];
-				break;
-			case 2:
-				this.type = ModDependency.getTypeFromPrefix(parts[0]);
-				this.name = parts[1];
-				break;
-			case 3:
-				this.type = "required";
-				this.name = parts[0];
-				this.version = ModVersionEquality.fromParts(parts[1], parts[2]);
-				break;
-			case 4:
-				this.type = ModDependency.getTypeFromPrefix(parts[0]);
-				this.name = parts[1];
-				this.version = ModVersionEquality.fromParts(parts[2], parts[3]);
-				break;
-			default:
-				throw new Error(`Invalid dependency specification: ${specification}`);
+		// Expand the match to the different parts
+		const [spec, type, name, equality, version] = match;
+		this.type = ModDependency.getTypeFromPrefix(type ?? "");
+		this.spec = spec;
+		this.name = name;
+
+		// Parse the equality and version
+		if ((equality || version) && !this.incompatible) {
+			this.version = ModVersionEquality.fromParts(equality, version);
 		}
 	}
 
@@ -124,7 +102,7 @@ export class ModDependency {
 	}
 
 	toJSON() {
-		return this.specification;
+		return this.spec;
 	}
 }
 
@@ -218,7 +196,7 @@ export default class ModInfo {
 	 * As they would be represented in info.json.
 	 */
 	get dependencySpecifications() {
-		return this.dependencies.map(d => d.specification);
+		return this.dependencies.map(d => d.spec);
 	}
 
 	checkDependencySatisfaction(mods: (ModInfo | ModRecord)[]) {
@@ -308,7 +286,13 @@ export default class ModInfo {
 		if (json.homepage) { modInfo.homepage = json.homepage; }
 		if (json.description) { modInfo.description = json.description; }
 		if (json.factorio_version) { modInfo.factorioVersion = json.factorio_version; }
-		if (json.dependencies) { modInfo.dependencies = json.dependencies.map(d => new ModDependency(d)); }
+
+		// Parse the dependencies
+		try {
+			if (json.dependencies) { modInfo.dependencies = json.dependencies.map(d => new ModDependency(d)); }
+		} catch (err: any) {
+			throw new Error(`Failed for parse dependencies for ${json.name}: ${err}`);
+		}
 
 		// Additional data
 		if (json.size) { modInfo.size = json.size; }

packages/lib/src/data/Role.ts

@@ -1,15 +1,23 @@
 import { Static, Type } from "@sinclair/typebox";
 import { permissions as globalPermissions } from "../permissions";
 
+interface RoleCollection {
+	get(id: number): Role;
+	set(role: Role): void;
+}
+
 /**
  * Represents a collection of granted permissions
  */
 export default class Role {
+	static DefaultAdminRoleId = 0 as const;
+	static DefaultPlayerRoleId = 1 as const;
+
 	constructor(
 		public id: number,
 		public name: string,
 		public description: string,
-		public permissions: Set<string>,
+		public permissions = new Set<string>(),
 		public updatedAtMs = 0,
 		public isDeleted = false,
 	) { }
@@ -45,6 +53,13 @@ export default class Role {
 		};
 	}
 
+	static ensureDefaultPlayerRole(roles: RoleCollection) {
+		const playerRole = roles.get(this.DefaultPlayerRoleId)
+			?? new this(this.DefaultPlayerRoleId, "Player", "Default player role.");
+		playerRole.grantDefaultPermissions();
+		roles.set(playerRole);
+	}
+
 	grantDefaultPermissions() {
 		for (let permission of globalPermissions.values()) {
 			if (permission.grantByDefault) {
@@ -52,4 +67,15 @@ export default class Role {
 			}
 		}
 	}
+
+	static ensureDefaultAdminRole(roles: RoleCollection) {
+		const adminRole = roles.get(this.DefaultAdminRoleId)
+			?? new this(this.DefaultAdminRoleId, "Cluster Admin", "Cluster wide administrator.");
+		adminRole.grantAdminPermissions();
+		roles.set(adminRole);
+	}
+
+	grantAdminPermissions() {
+		this.permissions.add("core.admin");
+	}
 }

packages/lib/src/data/UserDetails.ts

@@ -2,12 +2,10 @@ import { Static, Type } from "@sinclair/typebox";
 
 import PlayerStats from "./PlayerStats";
 
-/**
- * Represeents a user in the cluster
- *
- * Holds data about a Factorio user in the cluster.
- */
-export default class User {
+/** Represents a user in the cluster */
+export default class UserDetails {
+	public playerStats: PlayerStats;
+
 	constructor(
 		/** Factorio user name.  */
 		public name: string,
@@ -27,11 +25,15 @@ export default class User {
 		public updatedAtMs = 0,
 		/** True if this user object has been removed from the cluster.  */
 		public isDeleted = false,
-		/** Combined statistics for the player this user account is tied to.  */
-		public playerStats = new PlayerStats(),
 		/** Per instance statistics for the player this user account is tied to.  */
 		public instanceStats = new Map<number, PlayerStats>(),
 	) {
+		this.playerStats = UserDetails._calculatePlayerStats(this.instanceStats);
+	}
+
+	/** Name of the user, used by event subscriptions and to ensure case insensitivity */
+	get id() {
+		return this.name.toLowerCase();
 	}
 
 	static jsonSchema = Type.Object({
@@ -51,17 +53,13 @@ export default class User {
 		),
 	});
 
-	static fromJSON(json: Static<typeof this.jsonSchema>, _hack?: any) {
-		const roleIds = new Set<number>(json.roles ?? []);
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
 		const instanceStats = new Map(
-			(json.instance_stats ?? []).map(
-				([id, stats]) => [id, PlayerStats.fromJSON(stats)]
-			)
+			(json.instance_stats ?? []).map(([id, stats]) => [id, PlayerStats.fromJSON(stats)])
 		);
-		const playerStats = User._calculatePlayerStats(instanceStats);
 		return new this(
 			json.name,
-			roleIds,
+			new Set<number>(json.roles),
 			new Set(json.instances),
 			json.is_admin,
 			json.is_banned,
@@ -69,13 +67,12 @@ export default class User {
 			json.ban_reason,
 			json.updated_at_ms,
 			json.is_deleted,
-			playerStats,
 			instanceStats,
 		);
 	}
 
 	toJSON() {
-		let json: Static<typeof User.jsonSchema> = {
+		let json: Static<typeof UserDetails.jsonSchema> = {
 			name: this.name,
 		};
 
@@ -119,7 +116,7 @@ export default class User {
 	}
 
 	recalculatePlayerStats() {
-		this.playerStats = User._calculatePlayerStats(this.instanceStats);
+		this.playerStats = UserDetails._calculatePlayerStats(this.instanceStats);
 	}
 
 	static _calculatePlayerStats(instanceStatsMap: Map<number, PlayerStats>) {
@@ -149,44 +146,13 @@ export default class User {
 		}
 		return playerStats;
 	}
-
-	/**
-	 * Merge another user into this one, must have matching ids, user is not deleted
-	 * @param otherUser - User who's details are merged from
-	*/
-	merge(otherUser: User) {
-		if (this.id !== otherUser.id) {
-			throw new Error("Cannot merge users with different ids");
-		}
-
-		// Merge properties
-		this.roleIds = new Set([...this.roleIds, ...otherUser.roleIds]);
-		this.instances = new Set([...this.instances, ...otherUser.instances]);
-		this.isAdmin = this.isAdmin && otherUser.isAdmin; // More secure to use && rather ||
-		this.isBanned = this.isBanned || otherUser.isBanned;
-		this.isBanned = this.isWhitelisted || otherUser.isWhitelisted;
-		this.banReason = this.updatedAtMs > otherUser.updatedAtMs ? this.banReason : otherUser.banReason;
-		this.updatedAtMs = Date.now();
-
-		// Merge instance stats
-		const thisInstanceStats = this.instanceStats;
-		for (const [instanceId, instanceStats] of otherUser.instanceStats.entries()) {
-			if (thisInstanceStats.has(instanceId)) {
-				thisInstanceStats.get(instanceId)!.merge(instanceStats);
-			} else {
-				thisInstanceStats.set(instanceId, instanceStats);
-			}
-		}
-		this.recalculatePlayerStats();
-	}
-
-	/** Name of the user, used by event subscriptions and to ensure case insensitivity */
-	get id() {
-		return this.name.toLowerCase();
-	}
 }
 
-export interface IControllerUser extends User {
+/**
+ * This interface is implemented by User on the controller.
+ * It is exposed here to allow message definitions to create permission methods.
+ */
+export interface IUser extends UserDetails {
 	/**
 	 * Check if a given permission is granted
 	 *

packages/lib/src/data/index.ts

@@ -12,8 +12,10 @@ export { default as ModuleInfo } from "./ModuleInfo";
 export { default as Permission } from "./Permission";
 export { default as PlayerStats } from "./PlayerStats";
 export { default as Role } from "./Role";
-export { default as User } from "./User";
-export type { IControllerUser } from "./User";
+export { default as InstanceDetails, InstanceStatus } from "./InstanceDetails";
+export { default as HostDetails } from "./HostDetails";
+export { default as UserDetails } from "./UserDetails";
+export type { IUser } from "./UserDetails";
 export * from "./composites";
 export * from "./messages_core";
 export * from "./messages_controller";

packages/lib/src/data/messages_controller.ts

@@ -260,6 +260,8 @@ export class SystemInfo {
 		public diskAvailable: number,
 		public canRestart: boolean,
 		public restartRequired: boolean,
+		public systemStartedAtMs: number,
+		public processStartedAtMs: number,
 		/** Millisecond Unix timestamp this entry was last updated at */
 		public updatedAtMs: number,
 		public isDeleted: boolean,
@@ -279,6 +281,8 @@ export class SystemInfo {
 		"diskAvailable": Type.Number(),
 		"canRestart": Type.Boolean(),
 		"restartRequired": Type.Boolean(),
+		"systemStartedAtMs": Type.Number(),
+		"processStartedAtMs": Type.Number(),
 		"updatedAtMs": Type.Number(),
 		"isDeleted": Type.Boolean(),
 	});
@@ -298,6 +302,8 @@ export class SystemInfo {
 			json.diskAvailable,
 			json.canRestart,
 			json.restartRequired,
+			json.systemStartedAtMs,
+			json.processStartedAtMs,
 			json.updatedAtMs,
 			json.isDeleted,
 		);
@@ -396,7 +402,7 @@ export class DebugWsMessageEvent {
 export class FactorioVersionsRequest {
 	declare ["constructor"]: typeof FactorioVersionsRequest;
 	static type = "request" as const;
-	static src = "control" as const;
+	static src = ["control", "instance"] as const;
 	static dst = "controller" as const;
 	static permission = "core.external.get_factorio_versions" as const;
 

packages/lib/src/data/messages_host.ts

@@ -2,6 +2,7 @@ import { Type, Static } from "@sinclair/typebox";
 import { jsonArray, plainJson, StringEnum } from "./composites";
 import { CollectorResultSerialized } from "../prometheus";
 import { HostConfig } from "../config/definitions";
+import HostDetails from "./HostDetails";
 
 export class HostStopRequest {
 	declare ["constructor"]: typeof HostStopRequest;
@@ -82,47 +83,6 @@ export class HostConfigSetPropRequest {
 	}
 }
 
-export class HostDetails {
-	constructor(
-		public version: string,
-		public name: string,
-		public id: number,
-		public connected: boolean,
-		public remoteAddress = "",
-		public publicAddress: string = "",
-		public tokenValidAfter: number = 0,
-		/** Millisecond Unix timestamp this entry was last updated at */
-		public updatedAtMs = 0,
-		public isDeleted = false,
-	) { }
-
-	static jsonSchema = Type.Object({
-		"version": Type.String(),
-		"name": Type.String(),
-		"id": Type.Integer(),
-		"connected": Type.Boolean(),
-		"remoteAddress": Type.Optional(Type.String()),
-		"publicAddress": Type.Optional(Type.String()),
-		"tokenValidAfter": Type.Optional(Type.Number()),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"isDeleted": Type.Optional(Type.Boolean()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(
-			json.version,
-			json.name,
-			json.id,
-			json.connected,
-			json.remoteAddress,
-			json.publicAddress,
-			json.tokenValidAfter,
-			json.updatedAtMs,
-			json.isDeleted,
-		);
-	}
-}
-
 export class HostListRequest {
 	declare ["constructor"]: typeof HostListRequest;
 	static type = "request" as const;

packages/lib/src/data/messages_instance.ts

@@ -2,84 +2,11 @@ import { Type, Static } from "@sinclair/typebox";
 import PlayerStats from "./PlayerStats";
 import { JsonString, StringEnum, jsonArray, plainJson } from "./composites";
 import { InstanceConfig } from "../config";
-import type { IControllerUser } from "./User";
+import type { IUser } from "./UserDetails";
 import type { MessageRequest } from "./messages_core";
 import { CollectorResultSerialized } from "../prometheus";
 import { TargetVersion, TargetVersionSchema, PartialVersion, PartialVersionSchema } from "./version";
-
-
-export const InstanceStatus = StringEnum([
-	"unknown",
-	"unassigned",
-	"stopped",
-	"starting",
-	"running",
-	"stopping",
-	"creating_save",
-	"exporting_data",
-	"deleted",
-]);
-
-/**
- * Current status of the instance. One of:
- * - `unknown`: Instance is assigned to a host but this host is currently
- *   not connected to the contreller.
- * - `unassigned`: Instance is not assigned to a a host and exists only on
- *   the controller.
- * - `stopped`: Instance is stopped.
- * - `starting`: Instance is in the process of starting up.
- * - `running`: Instance is running normally.
- * - `stopping`: Instance is in the process of stopping.
- * - `creating_save`: Instance is in the process of creating a save.
- * - `exporting_data`: Instance is in the process of exporting game data.
- * - `deleted`: Instance has been deleted.
- */
-export type InstanceStatus = Static<typeof InstanceStatus>;
-
-export class InstanceDetails {
-	constructor(
-		public name: string,
-		public id: number,
-		public assignedHost: number | undefined,
-		public gamePort: number | undefined,
-		public status: InstanceStatus,
-		public factorioVersion: TargetVersion | undefined,
-		/** Millisecond Unix timestamp this entry was last updated at */
-		public updatedAtMs = 0,
-		public excludeFromStartAll = false,
-	) { }
-
-	static jsonSchema = Type.Object({
-		"name": Type.String(),
-		"id": Type.Integer(),
-		"assignedHost": Type.Optional(Type.Integer()),
-		"gamePort": Type.Optional(Type.Integer()),
-		"status": StringEnum([
-			"unknown", "unassigned", "stopped", "starting", "running", "stopping",
-			"creating_save", "exporting_data", "deleted",
-		]),
-		"factorioVersion": Type.Optional(TargetVersionSchema),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"excludeFromStartAll": Type.Optional(Type.Boolean()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(
-			json.name,
-			json.id,
-			json.assignedHost,
-			json.gamePort,
-			json.status,
-			json.factorioVersion,
-			json.updatedAtMs,
-			json.excludeFromStartAll,
-		);
-	}
-
-	get isDeleted() {
-		return this.status === "deleted";
-	}
-}
+import InstanceDetails, { InstanceStatus } from "./InstanceDetails";
 
 export class InstanceDetailsGetRequest {
 	declare ["constructor"]: typeof InstanceDetailsGetRequest;
@@ -536,7 +463,7 @@ export class InstanceTransferSaveRequest {
 	static type = "request" as const;
 	static src = ["control", "controller"] as const;
 	static dst = ["controller", "host"] as const;
-	static permission(user: IControllerUser, message: MessageRequest) {
+	static permission(user: IUser, message: MessageRequest) {
 		user.checkPermission("core.instance.save.transfer");
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as { copy: boolean, sourceName: string, targetName: string };

packages/lib/src/data/messages_mod.ts

@@ -403,6 +403,10 @@ export class ModPortalDownloadRequest {
 	}
 }
 
+const ModDependencyResolveErrors = ["incompatible", "unsatisfiable", "invalidDependency", "notFound"] as const;
+export const ModDependencyResolveErrorsSchema = Type.Union(ModDependencyResolveErrors.map(v => Type.Literal(v)));
+export type ModDependencyResolveErrors = Static<typeof ModDependencyResolveErrorsSchema>;
+
 /**
  * Request mods dependencies to be resolved using the Factorio mod portal.
  *
@@ -456,18 +460,26 @@ export class ModDependencyResolveRequest {
 	static Response = class ModDependencyResolveResponse {
 		constructor(
 			public dependencies: ModInfo[],
-			public incompatible: string[],
-			public missing: string[],
+			public errors: Map<string, ModDependencyResolveErrors>,
 		) {}
 
 		static jsonSchema = Type.Object({
 			"dependencies": Type.Array(ModInfo.jsonSchema),
-			"incompatible": Type.Array(Type.String()),
-			"missing": Type.Array(Type.String()),
+			"errors": Type.Record(Type.String(), ModDependencyResolveErrorsSchema),
 		});
 
 		static fromJSON(json: Static<typeof this.jsonSchema>) {
-			return new this(json.dependencies.map(d => ModInfo.fromJSON(d)), json.incompatible, json.missing);
+			return new this(
+				json.dependencies.map(d => ModInfo.fromJSON(d)),
+				new Map(Object.entries(json.errors))
+			);
+		}
+
+		toJSON(): Static<typeof ModDependencyResolveResponse.jsonSchema> {
+			return {
+				dependencies: this.dependencies.map(d => d.toJSON()),
+				errors: Object.fromEntries(this.errors),
+			};
 		}
 	};
 }

packages/lib/src/data/messages_user.ts

@@ -1,5 +1,5 @@
 import { Type, Static } from "@sinclair/typebox";
-import User, { IControllerUser } from "./User";
+import UserDetails, { IUser } from "./UserDetails";
 import { StringEnum, jsonArray, plainJson } from "./composites";
 import { MessageRequest } from "./messages_core";
 
@@ -22,7 +22,7 @@ export class UserGetRequest {
 		return new this(json.name);
 	}
 
-	static Response = User;
+	static Response = UserDetails;
 }
 
 export class UserListRequest {
@@ -31,7 +31,7 @@ export class UserListRequest {
 	static src = "control" as const;
 	static dst = "controller" as const;
 	static permission = "core.user.list" as const;
-	static Response = jsonArray(User);
+	static Response = jsonArray(UserDetails);
 }
 
 export class UserCreateRequest {
@@ -198,15 +198,15 @@ export class UserUpdatesEvent {
 	static permission = "core.user.subscribe" as const;
 
 	constructor(
-		public updates: User[],
+		public updates: UserDetails[],
 	) { }
 
 	static jsonSchema = Type.Object({
-		"updates": Type.Array(User.jsonSchema),
+		"updates": Type.Array(UserDetails.jsonSchema),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.updates.map(update => User.fromJSON(update)));
+		return new this(json.updates.map(update => UserDetails.fromJSON(update)));
 	}
 }
 
@@ -250,7 +250,7 @@ export class UserBulkImportRequest {
 	static type = "request" as const;
 	static src = "control" as const;
 	static dst = "controller" as const;
-	static permission(user: IControllerUser, message: MessageRequest) {
+	static permission(user: IUser, message: MessageRequest) {
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as Static<typeof UserBulkImportRequest.jsonSchema>;
 			// Check if this is a restore or import request

packages/lib/src/data/version.ts

@@ -151,10 +151,10 @@ export class ModVersionEquality {
 
 	static fromParts(equality: string, version: string) {
 		if (!isVersionEquality(equality)) {
-			throw new Error(`Unknown version equality: ${equality}`);
+			throw new Error(`Invalid version equality "${equality}"`);
 		}
 		if (!isPartialVersion(version)) {
-			throw new Error(`Malformed version string: ${version}`);
+			throw new Error(`Invalid version string "${version}"`);
 		}
 		return new this(equality, version);
 	}

packages/lib/src/datastore.ts

@@ -53,7 +53,7 @@ export class JsonDatastoreProvider<
 		private filePath: string,
 		private fromJson: (json: J) => V = v => v as any as V,
 		private migrations: (rawJson: Record<string, unknown>) => Record<string, J> = v => v as Record<string, J>,
-		private finalise: (obj: V) => V = v => v,
+		private finalise: (obj: V, map: Map<string, V>) => V | undefined = v => v,
 	) {
 		super();
 	}
@@ -82,7 +82,24 @@ export class JsonDatastoreProvider<
 		const serialized = this.migrations(rawJson);
 
 		// Convert to data class objects
-		return new Map(Object.entries(serialized).map(([k, v]) => [k, this.finalise(this.fromJson(v))]));
+		let serializedCount = 0;
+		const result = new Map<string, V>();
+		for (const [key, raw] of Object.entries(serialized)) {
+			const final = this.finalise(this.fromJson(raw), result);
+			if (final !== undefined) { result.set(key, final); }
+			serializedCount += 1;
+		}
+
+		// Save a backup if the map does not contain all values (happens where there are duplicates)
+		if (result.size < serializedCount) {
+			const backupPath = `${this.filePath}.${Date.now()}.bak`;
+			logger.warn(
+				`Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}`
+			);
+			await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t"));
+		}
+
+		return result;
 	}
 }
 
@@ -95,7 +112,7 @@ export class JsonIdDatastoreProvider<
 		private filePath: string,
 		private fromJson: (json: J) => V = v => v as any as V,
 		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as Array<J>,
-		private finalise: (obj: V) => V = v => v,
+		private finalise: (obj: V, map: Map<V["id"], V>) => V | undefined = v => v,
 	) {
 		super();
 	}
@@ -124,10 +141,24 @@ export class JsonIdDatastoreProvider<
 		const serialized = this.migrations(rawJson);
 
 		// Convert to data class objects
-		return new Map(serialized.map((e) => {
-			const v = this.finalise(this.fromJson(e));
-			return [v.id, v];
-		}));
+		let serializedCount = 0;
+		const result = new Map<V["id"], V>();
+		for (const raw of serialized) {
+			const final = this.finalise(this.fromJson(raw), result);
+			if (final !== undefined) { result.set(final.id, final); }
+			serializedCount += 1;
+		}
+
+		// Save a backup if the map does not contain all values (happens where there are duplicates)
+		if (result.size < serializedCount) {
+			const backupPath = `${this.filePath}.${Date.now()}.bak`;
+			logger.warn(
+				`Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}`
+			);
+			await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t"));
+		}
+
+		return result;
 	}
 }
 
@@ -259,18 +290,50 @@ export class KeyValueDatastore<
 export class SubscribableDatastore<
 	V extends SubscribableValue,
 > extends Datastore<V["id"], V> {
+	/** Last time an update occurred */
+	lastUpdatedAtMs: number = 0;
+
+	private static maxUpdatedAtMs(values: Iterable<SubscribableValue>) {
+		let max = 0;
+		for (const value of values) {
+			// Prevent one NaN, undefined, +Inf or similar from breaking the datastore.
+			if (Number.isFinite(value.updatedAtMs)) {
+				max = Math.max(max, value.updatedAtMs);
+			}
+		}
+		return max;
+	}
+
+	private incrementLastUpdatedMs() {
+		this.lastUpdatedAtMs = Math.max(Date.now(), this.lastUpdatedAtMs + 1);
+	}
+
+	constructor(
+		provider: DatastoreProvider<V["id"], V> = new MemoryDatastoreProvider(),
+		data = new Map<V["id"], V>(),
+	) {
+		super(provider, data);
+		this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values());
+	}
+
+	async load() {
+		await super.load();
+		this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values());
+	}
+
 	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
 	set(value: V) {
+		this.incrementLastUpdatedMs();
 		this.data.set(value.id, value);
-		value.updatedAtMs = Date.now();
+		value.updatedAtMs = this.lastUpdatedAtMs;
 		this.touch([value]);
 	}
 
 	// Set many values at once from an array of key value pairs
 	setMany(values: V[]) {
-		const nowMs = Date.now();
+		this.incrementLastUpdatedMs();
 		for (const value of values) {
-			value.updatedAtMs = nowMs;
+			value.updatedAtMs = this.lastUpdatedAtMs;
 			this.data.set(value.id, value);
 		}
 		this.touch(values);
@@ -278,18 +341,19 @@ export class SubscribableDatastore<
 
 	// Delete a value in the datastore
 	delete(value: V) {
+		this.incrementLastUpdatedMs();
 		this.data.delete(value.id);
-		value.updatedAtMs = Date.now();
+		value.updatedAtMs = this.lastUpdatedAtMs;
 		value.isDeleted = true;
 		this.touch([value]);
 	}
 
 	// Delete many values at once from an array of values
 	deleteMany(values: V[]) {
-		const nowMs = Date.now();
+		this.incrementLastUpdatedMs();
 		for (const value of values) {
 			value.isDeleted = true;
-			value.updatedAtMs = nowMs;
+			value.updatedAtMs = this.lastUpdatedAtMs;
 			this.data.delete(value.id);
 		}
 		this.touch(values);

packages/lib/src/errors.ts

@@ -10,6 +10,13 @@ export class CommandError extends Error {
 	code = "CommandError";
 }
 
+/**
+ * Thrown if the installation is broken, such as when multiple copies of lib is imported
+ */
+export class InstallationError extends Error {
+	code = "InstallationError";
+}
+
 /**
  * Thrown from requests sent when an error occured handling it
  */

packages/lib/src/external/FactorioVersions.ts

@@ -1,7 +1,7 @@
 import { Static, Type } from "@sinclair/typebox";
-import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data";
+import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data/version";
 
-const ARCHIVE_URL = "https://factorio.com/download/archive";
+const ARCHIVE_URL = "https://factorio.com/download/archive/";
 
 /**
  * Represents the parsed result of a factorio version
@@ -60,7 +60,7 @@ function parseFactorioVersions(html: string): ExternalFactorioVersion[] {
 			versions.set(version, {
 				version: version,
 				stable: kind === "stable",
-				headlessUrl: `www.factorio.com/get-download/${version}/headless/linux64`,
+				headlessUrl: `https://www.factorio.com/get-download/${version}/headless/linux64`,
 			});
 		}
 	}

packages/lib/src/link/connectors.ts

@@ -431,7 +431,6 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
 	constructor(
 		protected _url: string,
 		maxReconnectDelay: number,
-		protected _tlsCa: string | undefined,
 	) {
 		super(undefined as any, new libData.Address(libData.Address.controller, 0));
 		this._backoff = new ExponentialBackoff({ max: maxReconnectDelay });
@@ -543,17 +542,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
 
 		// Open WebSocket to controller
 		logger.verbose(`Connector | connecting to ${url}`);
-
-
-		if (process.env.APP_ENV === "browser") {
-			this._socket = new WebSocket(url) as WebSocketClusterio;
-
-		} else {
-			let options: { ca?: string } = {};
-			if (this._tlsCa) { options.ca = this._tlsCa; }
-			this._socket = new WebSocket(url, options) as WebSocketClusterio;
-		}
-
+		this._socket = new WebSocket(url) as WebSocketClusterio;
 		this._attachSocketHandlers();
 	}
 

packages/lib/src/link/link.ts

@@ -8,7 +8,7 @@ import { BaseConnector, WebSocketBaseConnector } from "./connectors";
 import { strict as assert } from "assert";
 import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin";
 import type {
-	AddressType, JSONDeserialisable, MessageRoutable, MessageRequest, MessageEvent, IControllerUser,
+	AddressType, JSONDeserialisable, MessageRoutable, MessageRequest, MessageEvent, IUser,
 } from "../data";
 
 export interface Request<Req, Res> {
@@ -18,7 +18,7 @@ export interface Request<Req, Res> {
 		type: "request";
 		src: AddressType | readonly AddressType[];
 		dst: AddressType | readonly AddressType[];
-		permission?: null | string | ((user: IControllerUser, message: MessageRequest) => void);
+		permission?: null | string | ((user: IUser, message: MessageRequest) => void);
 		plugin?: string;
 		Response?: JSONDeserialisable<Res>,
 	}
@@ -32,7 +32,7 @@ export interface Event<T> {
 		type: "event";
 		src: AddressType | readonly AddressType[];
 		dst: AddressType | readonly AddressType[];
-		permission?: null | string | ((user: IControllerUser, message: MessageEvent) => void);
+		permission?: null | string | ((user: IUser, message: MessageEvent) => void);
 		plugin?: string;
 	}
 }

packages/lib/src/permissions.ts

@@ -6,9 +6,7 @@
  * @author Hornwitser
  * @module lib/users
  */
-import { Permission, Role } from "./data";
-import { SubscribableDatastore } from "./datastore";
-
+import { Permission } from "./data";
 
 export const permissions = new Map<string, Permission>();
 
@@ -58,54 +56,6 @@ export function definePermission({
 	permissions.set(name, new Permission(name, title, description, grantByDefault));
 }
 
-
-/**
- * Ensure the default admin role has full access
- *
- * Ensures the role with id 0 exisits and has the core.admin permission.
- *
- * @param roles - role storage to modify.
- * @returns the default admin role
- */
-export function ensureDefaultAdminRole(roles: SubscribableDatastore<Role>) {
-	let admin = roles.get(0);
-	if (!admin) {
-		admin = Role.fromJSON({
-			id: 0,
-			name: "Cluster Admin",
-			description: "Cluster wide administrator.",
-			permissions: [],
-		});
-	}
-	admin.permissions.add("core.admin");
-	roles.set(admin);
-	return admin;
-}
-
-/**
- * Ensure the default player role has default accesses
- *
- * Ensures the role with id 1 exists and has all the permissions that are
- * assigned by default.  Note that this may not be the same role as the one
- * that is assigned to new users by default.
- *
- * @param roles - role storage to modify.
- */
-export function ensureDefaultPlayerRole(roles: SubscribableDatastore<Role>) {
-	let player = roles.get(1);
-	if (!player) {
-		player = Role.fromJSON({
-			id: 1,
-			name: "Player",
-			description: "Default player role.",
-			permissions: [],
-		});
-	}
-	player.grantDefaultPermissions();
-	roles.set(player);
-}
-
-
 // Definitions for the built in permissions used in Clusterio.
 // description should answer "this permission allows you to ___"
 definePermission({

packages/lib/src/plugin_loader.ts

@@ -45,6 +45,9 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
 			pluginInfo = require(pluginPath).plugin;
 			pluginPackage = require(path.posix.join(pluginPath, "package.json"));
 		} catch (err: any) {
+			if (err.code === "InstallationError") {
+				throw err;
+			}
 			throw new libErrors.PluginError(pluginName, err);
 		}
 

packages/lib/src/prometheus.ts

@@ -272,18 +272,31 @@ function labelsToKey(labels: LabelValues, metricLabels: string[]) {
 function keyToLabels(key: string) {
 	let labels = new Map<string, string>();
 
-	if (key !== "") {
-		for (let pair of key.split(",")) {
-			let [name, value] = pair.split("=", 2);
-			if (!name || value === undefined) {
-				continue;
+	let pos = 0;
+	let matchLabel = /([^=]+)="(([^\\"]|\\["n\\])+)"/y;
+	while (pos < key.length) {
+		if (pos !== 0) {
+			/* istanbul ignore if */
+			if (key[pos] !== ",") {
+				// Should be unreachable
+				throw Error(`Expected , at pos ${pos} in "${key}"`);
 			}
+			pos += 1;
+		}
+		matchLabel.lastIndex = pos;
+		const match = matchLabel.exec(key);
+		/* istanbul ignore else */
+		if (match) {
+			let [, name, value] = match;
 			labels.set(name, value
-				.slice(1, -1)
 				.replace(/\\"/g, "\"")
 				.replace(/\\n/g, "\n")
 				.replace(/\\\\/g, "\\")
 			);
+			pos += match[0].length;
+		} else {
+			// Should be unreachable
+			throw new Error(`Failed to parse label at pos ${pos} in "${key}"`);
 		}
 	}
 

packages/lib/src/subscriptions.ts

@@ -1,6 +1,6 @@
 import { Type, Static } from "@sinclair/typebox";
 import { Link, Event, EventClass, RequestHandler, WebSocketBaseConnector } from "./link";
-import { Address, MessageRequest, IControllerUser, plainJson, JsonBoolean } from "./data";
+import { Address, MessageRequest, IUser, plainJson, JsonBoolean } from "./data";
 import isDeepStrictEqual from "./is_deep_strict_equal";
 import { RequestError, SessionLost } from "./errors";
 import { logger } from "./logging";
@@ -18,7 +18,7 @@ export class SubscriptionRequest {
 	static type = "request" as const;
 	static src = ["control", "host", "instance"] as const;
 	static dst = "controller" as const;
-	static permission(user: IControllerUser, message: MessageRequest) {
+	static permission(user: IUser, message: MessageRequest) {
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as Static<typeof SubscriptionRequest.jsonSchema>;
 			const entry = Link._eventsByName.get(data[0]);

packages/lib/src/system_collectors.ts

@@ -134,7 +134,12 @@ function minZip<T>(a: T[], b: T[]): [T, T][] {
 
 let previousTotalCpuMs: number[] = [];
 let previousIdleCpuMs: number[] = [];
-export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean, restartRequired: boolean) {
+export async function gatherSystemInfo(
+	id: SystemInfo["id"],
+	canRestart: boolean,
+	restartRequired: boolean,
+	startedAtMs: number
+) {
 	const cpus = os.cpus();
 	const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
 	const currentIdleCpuMs = cpus.map(({ times }) => times.idle);
@@ -152,6 +157,7 @@ export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean
 		diskAvailable = stats.bavail * stats.bsize;
 	}
 
+	const now = Date.now();
 	return new SystemInfo(
 		id,
 		os.hostname(),
@@ -166,7 +172,9 @@ export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean
 		diskAvailable,
 		canRestart,
 		restartRequired,
-		Date.now(),
+		now - (os.uptime() * 1000),
+		startedAtMs,
+		now,
 		false,
 	);
 }

packages/lib/tsconfig.browser.json

@@ -7,6 +7,7 @@
 		"./src/config/*.ts",
 		"./src/data/*.ts",
 		"./src/external/*.ts",
+		"./src/check_singleton_import.ts",
 		"./src/datastore.ts",
 		"./src/errors.ts",
 		"./src/factorio/exchange_string.ts",

packages/lib/tsconfig.node.json

@@ -1,6 +1,6 @@
 {
 	"extends": "../../tsconfig.node.json",
-	"include": ["index.ts", "browser.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"],
+	"include": ["index.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"],
 	"compilerOptions": {
 		"allowJs": true,
 	},

packages/web_ui/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio web interface implementation",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.23",
 	"browser": "src/index.ts",
 	"main": "dist/browser/src/index.js",
 	"scripts": {

packages/web_ui/src/bootstrap.tsx

@@ -137,7 +137,7 @@ export default async function bootstrap() {
 	lib.addPluginConfigFields(pluginInfos);
 
 	let wsUrl = new URL(webRoot, document.location.href);
-	let controlConnector = new ControlConnector(wsUrl.href, 120, undefined);
+	let controlConnector = new ControlConnector(wsUrl.href, 120);
 	let control = new Control(controlConnector, new Map(pluginInfos.map(p => [p.name, p])));
 	control.plugins = await loadPlugins(pluginInfos, control);
 	control.inputComponents = inputComponentsFromPlugins(control.plugins);

packages/web_ui/src/components/ControllerPage.tsx

@@ -8,7 +8,7 @@ import PluginExtra from "./PluginExtra";
 import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
-	MetricDiskUsed, MetricDiskRatio,
+	MetricDiskUsed, MetricDiskRatio, MetricRelativeDate,
 } from "./system_metrics";
 import { useAccount } from "../model/account";
 import { useSystems } from "../model/system";
@@ -101,6 +101,12 @@ export default function ControllerPage() {
 			<Descriptions.Item label="Node.js">{system?.node}</Descriptions.Item>
 			<Descriptions.Item label="OS Kernel">{system?.kernel}</Descriptions.Item>
 			<Descriptions.Item label="Machine">{system?.machine}</Descriptions.Item>
+			<Descriptions.Item label="System Uptime">
+				<MetricRelativeDate timeMs={system?.systemStartedAtMs}/>
+			</Descriptions.Item>
+			<Descriptions.Item label="Process Uptime">
+				<MetricRelativeDate timeMs={system?.processStartedAtMs}/>
+			</Descriptions.Item>
 			<Descriptions.Item label="Hostname" span={2}>{system?.hostname}</Descriptions.Item>
 			<Descriptions.Item label="CPU Model" span={2}>{system?.cpuModel}</Descriptions.Item>
 			<Descriptions.Item label="CPU Usage"><MetricCpuRatio system={system} /></Descriptions.Item>

packages/web_ui/src/components/HostStatusTag.tsx

@@ -0,0 +1,17 @@
+import { Tag } from "antd";
+import React from "react";
+
+import { CheckCircleOutlined, CloseCircleOutlined } from "@ant-design/icons";
+
+const statusOptions = {
+	connected: <Tag color="#389e0d" icon={<CheckCircleOutlined/>} >Connected</Tag>,
+	disconnected: <Tag color="#cf1322" icon={<CloseCircleOutlined/>} >Disconnected</Tag>,
+};
+
+type HostStatusTagProps = {
+	connected: boolean;
+};
+
+export default function HostStatusTag(props: HostStatusTagProps) {
+	return props.connected ? statusOptions.connected : statusOptions.disconnected;
+}

packages/web_ui/src/components/HostViewPage.tsx

@@ -1,5 +1,5 @@
 import React, { useContext, useState } from "react";
-import { Descriptions, Spin, Tag, Typography, Button, Space, Modal, Popconfirm, Flex, Tooltip } from "antd";
+import { Descriptions, Spin, Typography, Button, Space, Popconfirm, Flex, Tooltip } from "antd";
 import { ExclamationCircleOutlined } from "@ant-design/icons";
 import { useParams } from "react-router-dom";
 
@@ -7,6 +7,7 @@ import * as lib from "@clusterio/lib";
 import notify, { notifyErrorHandler } from "../util/notify";
 import ControlContext from "./ControlContext";
 import HostConfigTree from "./HostConfigTree";
+import HostStatusTag from "./HostStatusTag";
 import InstanceList from "./InstanceList";
 import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import { useAccount } from "../model/account";
@@ -17,7 +18,7 @@ import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
-	MetricDiskUsed, MetricDiskRatio,
+	MetricDiskUsed, MetricDiskRatio, MetricRelativeDate,
 } from "./system_metrics";
 import { formatTimestamp } from "../util/time_format";
 import { useSystems } from "../model/system";
@@ -134,21 +135,22 @@ export default function HostViewPage() {
 	return <PageLayout nav={nav}>
 		<PageHeader
 			title={host.name || String(hostId)}
+			status={<HostStatusTag connected={host["connected"]}/>}
 			extra={hostButtons}
 		/>
 		<Descriptions bordered size="small" column={{ xs: 1, md: 2, lg: 2, xl: 2, xxl: 2 }}>
-			<Descriptions.Item label="Name">{host["name"]}</Descriptions.Item>
-			<Descriptions.Item label="Connected">
-				<Tag color={host["connected"] ? "#389e0d" : "#cf1322"}>
-					{host["connected"] ? "Connected" : "Disconnected"}
-				</Tag>
-			</Descriptions.Item>
 			<Descriptions.Item label="Version">{host.version}</Descriptions.Item>
 			<Descriptions.Item label="Node.js">{system?.node}</Descriptions.Item>
 			<Descriptions.Item label="OS Kernel">{system?.kernel}</Descriptions.Item>
 			<Descriptions.Item label="Machine">{system?.machine}</Descriptions.Item>
-			<Descriptions.Item label="Hostname" span={2}>{system?.hostname}</Descriptions.Item>
-			<Descriptions.Item label="Connected From" span={2}>{host.remoteAddress}</Descriptions.Item>
+			<Descriptions.Item label="System Uptime">
+				<MetricRelativeDate timeMs={system?.systemStartedAtMs}/>
+			</Descriptions.Item>
+			<Descriptions.Item label="Process Uptime">
+				<MetricRelativeDate timeMs={system?.processStartedAtMs}/>
+			</Descriptions.Item>
+			<Descriptions.Item label="Hostname">{system?.hostname}</Descriptions.Item>
+			<Descriptions.Item label="Connected From">{host.remoteAddress}</Descriptions.Item>
 			<Descriptions.Item label="CPU Model" span={2}>{system?.cpuModel}</Descriptions.Item>
 			<Descriptions.Item label="CPU Usage"><MetricCpuRatio system={system} /></Descriptions.Item>
 			<Descriptions.Item label="Cores"><MetricCpuUsed system={system} /></Descriptions.Item>

packages/web_ui/src/components/HostsPage.tsx

@@ -1,6 +1,6 @@
 import React, { useContext, useRef, useState, useEffect } from "react";
 import { useNavigate } from "react-router-dom";
-import { Button, Flex, Form, InputNumber, Modal, Progress, Space, Switch, Table, Tag, Tooltip, Typography } from "antd";
+import { Button, Flex, Form, InputNumber, Modal, Space, Switch, Table, Tooltip, Typography } from "antd";
 import { ExclamationCircleOutlined, CopyOutlined } from "@ant-design/icons";
 
 import * as lib from "@clusterio/lib";
@@ -11,6 +11,7 @@ import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import HostStatusTag from "./HostStatusTag";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
 	MetricDiskUsed, MetricDiskRatio,
@@ -56,7 +57,7 @@ function GenerateHostTokenButton() {
 
 		let newToken = await control.send(new lib.HostGenerateTokenRequest(id));
 		setToken(newToken);
-		setHostId(id??null);
+		setHostId(id ?? null);
 	}
 
 	// Generate a new random
@@ -89,13 +90,24 @@ function GenerateHostTokenButton() {
 					}} />
 				</Form.Item>
 			</Form>
+			<Typography.Paragraph>
+				Host IDs are randomly generated and unique, leave blank to use a new ID,
+				 otherwise the ID of an existing host can be found using:
+			</Typography.Paragraph>
+			<div className="codeblock">
+				<CopyButton
+					message="Copied get host id to clipboard"
+					text="npx clusteriohost config get host.id"
+				/>
+				npx clusteriohost config get host.id
+			</div>
 			{token !== null && <>
 				<Typography.Paragraph>
 					Host auth token:
 				</Typography.Paragraph>
 				<div className="codeblock">
 					<CopyButton
-						message={"Copied auth token to clipboard"}
+						message="Copied host token to clipboard"
 						text={token}
 					/>
 					{token}
@@ -105,15 +117,13 @@ function GenerateHostTokenButton() {
 				</Typography.Paragraph>
 				<div className="codeblock">
 					<CopyButton
-						message={"Copied configuration commands to clipboard"}
-						text={`npx clusteriohost config set host.controller_token ${token}
-							   npx clusteriohost config set host.id ${hostId}`}
+						message="Copied set host id to clipboard"
+						text={`npx clusteriohost config set host.id ${token}`}
 					/>
-					<p>npx clusteriohost config set host.controller_token &lt;token&gt;</p>
-					<p>npx clusteriohost config set host.id &lt;hostId&gt;</p>
+					npx clusteriohost config set host.id {token}
 				</div>
 				<Typography.Paragraph>
-					Example host setup commands:
+					A new host can be setup using the following commands:
 				</Typography.Paragraph>
 				<div className="codeblock">
 					<CopyButton
@@ -122,7 +132,7 @@ function GenerateHostTokenButton() {
 mkdir clusterio
 cd clusterio
 npm init "@clusterio" -- --controller-token ${token} --mode "host" --download-headless \
---controller-url ${document.location.origin}/ --host-name "Host ${hostId || "?"}" \
+--controller-url ${document.location.origin}/ --host-name "${hostId ? `Host ${hostId}` : "Host"}" \
 --public-address localhost ${pluginString.length ? "--plugins" : ""} ${pluginString}`
 						}/>
 					<p>&gt; mkdir clusterio</p>
@@ -133,7 +143,7 @@ npm init "@clusterio" -- --controller-token ${token} --mode "host" --download-he
 						--mode "host"
 						--download-headless
 						--controller-url <span className="highlight">{document.location.origin}/ </span>
-						--host-name <span className="highlight">"Host {hostId || "?"}" </span>
+						--host-name <span className="highlight">{hostId ? `Host ${hostId}` : "Host"} </span>
 						--public-address <span className="highlight">localhost </span>
 						{pluginString.length ? "--plugins" : ""} <span className="highlight">{pluginString}</span>
 					</p>
@@ -287,11 +297,7 @@ export default function HostsPage() {
 					title: "Connected",
 					key: "connected",
 					render: (_, host) => <>
-						<Tag
-							color={host.connected ? "#389e0d" : "#cf1322"}
-						>
-							{host.connected ? "Connected" : "Disconnected"}
-						</Tag>
+						<HostStatusTag connected={host["connected"]}/>
 						<RestartRequired system={systems.get(host.id)}/>
 					</>,
 					sorter: (a, b) => Number(a.connected) - Number(b.connected),

packages/web_ui/src/components/IconReferencePage.tsx

@@ -0,0 +1,248 @@
+import React, { useMemo, useState } from "react";
+import { Card, Input, Space, Table, Tag, Typography } from "antd";
+
+import {
+	useItemMetadata,
+	useRecipeMetadata,
+	useSignalMetadata,
+	useTechnologyMetadata,
+	usePlanetMetadata,
+	useQualityMetadata,
+	useEntityMetadata,
+} from "../model/item_metadata";
+import { useExportManifest } from "../model/export_manifest";
+import PageHeader from "./PageHeader";
+import PageLayout from "./PageLayout";
+
+const { Text } = Typography;
+
+type IconEntry = {
+	key: string;
+	category: string;
+	name: string;
+	cssClass: string;
+	size: number;
+	path?: string;
+};
+
+const CATEGORIES = [
+	"item", "recipe", "signal", "technology", "planet", "quality", "entity",
+] as const;
+
+type Category = typeof CATEGORIES[number];
+
+const CATEGORY_LABELS: Record<Category, string> = {
+	item: "Item",
+	recipe: "Recipe",
+	signal: "Signal",
+	technology: "Technology",
+	planet: "Planet",
+	quality: "Quality",
+	entity: "Entity",
+};
+
+function useAllIconEntries() {
+	const item = useItemMetadata();
+	const recipe = useRecipeMetadata();
+	const signal = useSignalMetadata();
+	const technology = useTechnologyMetadata();
+	const planet = usePlanetMetadata();
+	const quality = useQualityMetadata();
+	const entity = useEntityMetadata();
+
+	const byCategory: Record<Category, Map<string, { size: number; path?: string }>> = {
+		item, recipe, signal, technology, planet, quality, entity,
+	};
+
+	return useMemo(() => {
+		const entries: IconEntry[] = [];
+
+		for (const category of CATEGORIES) {
+			const names = [...byCategory[category].keys()].sort();
+			for (const name of names) {
+				const meta = byCategory[category].get(name)!;
+				entries.push({
+					key: `${category}:${name}`,
+					category,
+					name,
+					cssClass: `${category}-${CSS.escape(name)}`,
+					size: meta.size,
+					path: meta.path,
+				});
+			}
+		}
+
+		return entries;
+	}, [item, recipe, signal, technology, planet, quality, entity]);
+}
+
+export default function IconReferencePage() {
+	const [filter, setFilter] = useState("");
+	const [expandedGroups, setExpandedGroups] = useState<Set<string>>(new Set());
+	const allEntries = useAllIconEntries();
+
+	const filtered = useMemo(() => {
+		const q = filter.trim().toLowerCase();
+		if (!q) {
+			return allEntries;
+		}
+		return allEntries.filter(e => e.name.toLowerCase().includes(q) || e.cssClass.toLowerCase().includes(q)
+		);
+	}, [allEntries, filter]);
+
+	const groups = useMemo(() => {
+		const map = new Map<string, IconEntry[]>();
+		for (const cat of CATEGORIES) {
+			map.set(cat, []);
+		}
+		for (const entry of filtered) {
+			map.get(entry.category)!.push(entry);
+		}
+		return CATEGORIES.map(cat => ({ category: cat, entries: map.get(cat)! }))
+			.filter(g => g.entries.length > 0);
+	}, [filtered]);
+
+	const visibleCount = filtered.length;
+	const totalCount = allEntries.length;
+	const autoExpand = visibleCount <= 20;
+
+	const isExpanded = (category: string) => autoExpand || expandedGroups.has(category);
+
+	const toggleGroup = (category: string) => {
+		setExpandedGroups(prev => {
+			const next = new Set(prev);
+			if (next.has(category)) {
+				next.delete(category);
+			} else {
+				next.add(category);
+			}
+			return next;
+		});
+	};
+
+	const columns = [
+		{
+			title: "Icon",
+			dataIndex: "cssClass",
+			key: "icon",
+			width: 48,
+			render: (_: string, record: IconEntry) => (
+				<div className={record.cssClass} style={{ imageRendering: "pixelated" }} />
+			),
+		},
+		{
+			title: "Name",
+			dataIndex: "name",
+			key: "name",
+			render: (name: string) => (
+				<Text style={{ fontSize: 12 }}>{name}</Text>
+			),
+		},
+		{
+			title: "Usage",
+			dataIndex: "cssClass",
+			key: "usage",
+			render: (cssClass: string) => (
+				<Text code style={{ fontSize: 11, color: "#90ee90" }}>
+					{`<div class="${cssClass}" />`}
+				</Text>
+			),
+		},
+		{
+			title: "Path",
+			dataIndex: "path",
+			key: "path",
+			render: (p: string | undefined) => (p
+				? <Text type="secondary" style={{ fontSize: 11 }}>{p}</Text>
+				: null),
+		},
+	];
+
+	const manifest = useExportManifest();
+
+	return (
+		<PageLayout nav={[{ name: "Icon Reference" }]}>
+			<PageHeader title="Icon Reference" />
+			<div style={{ width: "80%", margin: "0 auto" }}>
+				{!manifest && (
+					<Card>
+						<Text type="secondary">
+							No export data available. Run export-data on an instance to generate icons.
+						</Text>
+					</Card>
+				)}
+				{manifest && manifest.exportedAt && (
+					<div style={{ marginBottom: 12, fontSize: 12, lineHeight: 1.8 }}>
+						{manifest.exportedAt && (
+							<div>
+								<Text type="secondary">Exported: </Text>
+								<Text>{new Date(manifest.exportedAt).toLocaleString()}</Text>
+							</div>
+						)}
+						<div>
+							<Text type="secondary">Icons: </Text>
+							<Text>{totalCount}</Text>
+						</div>
+					</div>
+				)}
+				<Card>
+					<Space style={{ marginBottom: 16, display: "flex", alignItems: "center" }}>
+						<Input
+							placeholder="Filter by name or class…"
+							value={filter}
+							onChange={e => setFilter(e.target.value)}
+							allowClear
+							style={{ width: 320 }}
+						/>
+						<Text type="secondary" style={{ fontSize: 12, whiteSpace: "nowrap" }}>
+							{visibleCount === totalCount
+								? `${totalCount} icons`
+								: `${visibleCount} / ${totalCount} icons`}
+						</Text>
+					</Space>
+					{groups.map(({ category, entries }) => {
+						const label = CATEGORY_LABELS[category as Category] ?? category;
+						const expanded = isExpanded(category);
+						return (
+							<div key={category} style={{ marginBottom: 8 }}>
+								<div
+									onClick={() => toggleGroup(category)}
+									style={{
+										display: "flex",
+										alignItems: "center",
+										gap: 8,
+										padding: "6px 12px",
+										background: "#1d1d1d",
+										border: "1px solid #333",
+										borderRadius: expanded ? "4px 4px 0 0" : 4,
+										cursor: "pointer",
+										userSelect: "none",
+									}}
+								>
+									<span style={{ fontSize: 10, color: "#888", width: 12 }}>
+										{expanded ? "▾" : "►"}
+									</span>
+									<Tag style={{ margin: 0 }}>{label}</Tag>
+									<Text type="secondary" style={{ fontSize: 12 }}>
+										{entries.length}
+									</Text>
+								</div>
+								{expanded && (
+									<Table<IconEntry>
+										columns={columns}
+										dataSource={entries}
+										rowKey="key"
+										size="small"
+										showHeader={false}
+										pagination={false}
+										style={{ borderTop: "none" }}
+									/>
+								)}
+							</div>
+						);
+					})}
+				</Card>
+			</div>
+		</PageLayout>
+	);
+}

packages/web_ui/src/components/InstanceRcon.tsx

@@ -1,4 +1,4 @@
-import React, { useContext, useState } from "react";
+import React, { useContext, useRef, useState } from "react";
 import { Input, Typography } from "antd";
 
 import * as lib from "@clusterio/lib";
@@ -17,6 +17,7 @@ export default function InstanceRcon(props: InstanceRconProps) {
 	let control = useContext(ControlContext);
 	let [output, setOutput] = useState<string|null>(null);
 	let [running, setRunning] = useState(false);
+	let resultRef = useRef<HTMLDivElement>(null);
 
 	async function sendCommand(command: string) {
 		if (!command) {
@@ -31,6 +32,8 @@ export default function InstanceRcon(props: InstanceRconProps) {
 				new lib.InstanceSendRconRequest(command),
 			);
 			setOutput(result);
+			const animation = resultRef.current?.children[0]?.getAnimations()[0];
+			if (animation) { animation.currentTime = 0; }
 		} finally {
 			setRunning(false);
 		}
@@ -39,7 +42,7 @@ export default function InstanceRcon(props: InstanceRconProps) {
 	return <>
 		{output && <>
 			<Title level={5}>Rcon result</Title>
-			<Paragraph code className="rcon-result">{output}</Paragraph>
+			<Paragraph ref={resultRef} code className="rcon-result">{output}</Paragraph>
 		</>}
 		<Input.Search
 			disabled={props.disabled}

packages/web_ui/src/components/InstanceStatusTag.tsx

@@ -1,23 +1,27 @@
 import { Tag } from "antd";
 import React from "react";
 
-export const statusColors = {
-	unknown: "#8c8c8c",
-	unassigned: "#eb2f96",
-	stopped: "#cf1322",
-	starting: "#ad8b00",
-	running: "#389e0d",
-	stopping: "#d48806",
-	creating_save: "#096dd9",
-	exporting_data: "#08979c",
-	deleted: "#262626",
+import {
+	QuestionCircleOutlined, ClockCircleOutlined, StopOutlined, LoadingOutlined,
+	CheckCircleOutlined, SaveOutlined, ExportOutlined, DeleteOutlined,
+} from "@ant-design/icons";
+
+export const statusOptions = {
+	unknown: <Tag color="#8c8c8c" icon={<QuestionCircleOutlined />}>Unknown</Tag>,
+	unassigned: <Tag color="#eb2f96" icon={<ClockCircleOutlined />}>Unassigned</Tag>,
+	stopped: <Tag color="#cf1322" icon={<StopOutlined />}>Stopped</Tag>,
+	starting: <Tag color="#ad8b00" icon={<LoadingOutlined />}>Starting</Tag>,
+	running: <Tag color="#389e0d" icon={<CheckCircleOutlined />}>Running</Tag>,
+	stopping: <Tag color="#d48806" icon={<LoadingOutlined />}>Stopping</Tag>,
+	creating_save: <Tag color="#096dd9" icon={<SaveOutlined />}>Creating Save</Tag>,
+	exporting_data: <Tag color="#08979c" icon={<ExportOutlined />}>Exporting Data</Tag>,
+	deleted: <Tag color="#262626" icon={<DeleteOutlined />}>Deleted</Tag>,
 };
 
 type InstanceStatusTag = {
-	status: keyof typeof statusColors;
+	status: keyof typeof statusOptions;
 };
+
 export default function InstanceStatusTag(props: InstanceStatusTag) {
-	return <Tag color={statusColors[props.status]}>
-		{props.status.replace("_", " ")}
-	</Tag>;
+	return statusOptions[props.status];
 }

packages/web_ui/src/components/InstanceViewPage.tsx

@@ -25,6 +25,7 @@ import { useHost } from "../model/host";
 import InstanceStatusTag from "./InstanceStatusTag";
 import Link from "./Link";
 import { instancePublicAddress } from "../util/instance";
+import { MetricRelativeDate } from "./system_metrics";
 
 type MenuItem = Required<MenuProps>["items"][number];
 const { Title } = Typography;
@@ -41,6 +42,7 @@ function InstanceDescription(props: InstanceDescriptionProps) {
 	return <Descriptions
 		bordered
 		size="small"
+		column={{ xs: 1, md: 3, lg: 3, xl: 3, xxl: 3 }}
 	>
 		<Descriptions.Item label="Host">
 			{!assigned
@@ -64,7 +66,9 @@ function InstanceDescription(props: InstanceDescriptionProps) {
 		<Descriptions.Item label="Version">
 			{instance.factorioVersion ?? "unknown"}
 		</Descriptions.Item>
-		<Descriptions.Item label="Status"><InstanceStatusTag status={instance.status!} /></Descriptions.Item>
+		<Descriptions.Item label="Last Start Time">
+			<MetricRelativeDate timeMs={instance.startedAtMs}/>
+		</Descriptions.Item>
 	</Descriptions>;
 }
 
@@ -97,7 +101,7 @@ function InstanceButtons(props: { instance: lib.InstanceDetails }) {
 			instanceButtonMenuItems.push({ type: "divider" });
 		}
 		instanceButtonMenuItems.push({
-			disabled: !["unknown", "unassigned", "stopped"].includes(instance.status!),
+			disabled: !["unknown", "unassigned", "stopped"].includes(instance.status),
 			danger: true,
 			key: "delete",
 			icon: <DeleteOutlined />,
@@ -208,6 +212,7 @@ export default function InstanceViewPage() {
 	return <PageLayout nav={nav}>
 		<PageHeader
 			title={instance.name ?? ""}
+			status={<InstanceStatusTag status={instance.status} />}
 			extra={<InstanceButtons instance={instance} />}
 		/>
 		<InstanceDescription host={host} instance={instance} />

packages/web_ui/src/components/ModDetails.tsx

@@ -98,7 +98,7 @@ export default function ModDetails<T extends lib.ModInfo | lib.ModRecord>(props:
 							{depWarnings.get(e.name) === "wrong_version" && <Tooltip title="Wrong version added.">
 								<FileSyncOutlined style={{ color: "#dd5e14" }} />{" "}
 							</Tooltip>}
-							{e.specification}
+							{e.spec}
 						</Space>
 						<br/>
 					</Fragment>)

packages/web_ui/src/components/ModPackViewPage.tsx

@@ -268,16 +268,28 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 
 	// State for the modal and dependencies
 	const [open, setOpen] = useState(false);
-	const [missing, setMissing] = useState<string[]>([]);
-	const [builtins, setBuiltins] = useState<string[]>([]);
-	const [incompatible, setIncompatible] = useState<string[]>([]);
-	const [factorioVersion, setFactorioVersion] = useState<lib.ApiVersion | null>(null);
-
-	// State for all mods fetched from backend
 	const [error, setError] = useState<Error | null>(null);
 	const [loading, setLoading] = useState<boolean>(false);
-	const [allMods, setAllMods] = useState<lib.ModInfo[]>([]);
-	const [mods, setMods] = useState<lib.ModInfo[]>([]);
+	const [factorioVersion, setFactorioVersion] = useState<lib.ApiVersion | null>(null);
+
+	const [mods, setMods] = useState<lib.ModInfo[]>([]); // All resolved mods and dependencies
+	const [missing, setMissing] = useState<lib.ModInfo[]>([]); // Missing mods and dependencies, excluding builtin
+
+	const [modErrors, setModErrors] = useState({
+		notFound: new Array<string>(), // Mods which were not found on the mod portal, can include builtins
+		incompatible: new Array<string>(), // Mods which are required, but are marked incompatible by another
+		unsatisfiable: new Array<string>(), // Mods which can not have their version range satisfied
+		invalidDependency: new Array<string>(), // Mods with invalid dependency specs
+	});
+
+	function resetModErrors() {
+		setModErrors({
+			notFound: [],
+			incompatible: [],
+			unsatisfiable: [],
+			invalidDependency: [],
+		});
+	}
 
 	// Get a valid factorio version
 	useEffect(() => {
@@ -294,10 +306,8 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 		if (!open || !factorioVersion) {
 			// Clear if closed or no version
 			setMods([]);
-			setAllMods([]);
 			setMissing([]);
-			setBuiltins([]);
-			setIncompatible([]);
+			resetModErrors();
 			return;
 		}
 
@@ -309,21 +319,29 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 			lib.ModDependencyResolveRequest.fromModPackEnabled(props.modPack, props.checkForUpdates)
 		).then(response => {
 			if (canceled) { return; }
-			setMods(response.dependencies.filter(dep => !modInfos.has(`${dep.name}_${dep.version}`)));
-			setIncompatible(response.incompatible);
-			setAllMods(response.dependencies);
-			setMissing(response.missing.filter(name => !props.builtInModNames.includes(name)));
-			setBuiltins(response.missing.filter(name => props.builtInModNames.includes(name)));
+
+			const newModErrors = [...response.errors.entries()]
+				.reduce<typeof modErrors>((acc, [modName, modError]) => {
+					acc[modError].push(modName);
+					return acc;
+				}, {
+					notFound: [],
+					incompatible: [],
+					unsatisfiable: [],
+					invalidDependency: [],
+				});
+
+			setModErrors(newModErrors);
+			setMods(response.dependencies);
+			setMissing(response.dependencies.filter(dep => !modInfos.has(`${dep.name}_${dep.version}`)));
 			setLoading(false);
 		}).catch(err => {
 			if (canceled) { return; }
 			notifyErrorHandler("Error fetching mods dependencies")(err);
 			setError(err);
 			setMods([]);
-			setAllMods([]);
 			setMissing([]);
-			setBuiltins([]);
-			setIncompatible([]);
+			resetModErrors();
 			setLoading(false);
 		});
 
@@ -334,51 +352,56 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 		// Re-fetch when modal opens or factorio version changes
 	}, [open, factorioVersion, control]);
 
-	// Function to handle download from Mod Portal to Controller
-	function handleControllerDownload() {
-		if (!factorioVersion || !mods) {
+	// Fix any mod errors found after resolving dependencies
+	function fixModErrors() {
+		if (!factorioVersion) {
 			return;
 		}
 
-		// Enable any required builtins
-		for (const name of builtins) {
-			props.onChange({
-				type: "mods.set",
-				name: name,
-				value: { ...props.modPack.mods.get(name)!, enabled: true },
-			});
+		// Enable any required builtins, they will be within the not found list
+		for (const modName of modErrors.notFound) {
+			if (props.builtInModNames.includes(modName)) {
+				props.onChange({
+					type: "mods.set",
+					name: modName,
+					value: { ...props.modPack.mods.get(modName)!, enabled: true },
+				});
+			}
 		}
 
-		// Disable any incompatible mods present
-		for (const name of incompatible) {
-			const modRecord = props.modPack.mods.get(name);
+		// Disable any incompatible mods
+		for (const modName of modErrors.incompatible) {
+			const modRecord = props.modPack.mods.get(modName);
 			if (modRecord) {
 				props.onChange({
 					type: "mods.set",
-					name: name,
+					name: modName,
 					value: { ...modRecord, enabled: false },
 				});
 			}
 		}
 
 		// Add all dependencies to the mod pack
-		for (const mod of allMods) {
-			props.onChange({
-				type: "mods.set",
-				name: mod.name,
-				value: {
+		for (const mod of mods) {
+			const existingMod = props.modPack.mods.get(mod.name);
+			if (!existingMod || existingMod.version !== mod.version) {
+				props.onChange({
+					type: "mods.set",
 					name: mod.name,
-					version: mod.version,
-					sha1: mod.sha1,
-					enabled: true,
-				},
-			});
+					value: {
+						name: mod.name,
+						version: mod.version,
+						sha1: mod.sha1,
+						enabled: true,
+					},
+				});
+			}
 		}
 
-		// Download the missing dependencies
+		// Download any missing mods
 		control.send(
 			new lib.ModPortalDownloadRequest(
-				mods.map(mod => ({
+				missing.map(mod => ({
 					name: mod.name, version: new lib.ModVersionEquality("=", mod.version),
 				})),
 				factorioVersion,
@@ -386,12 +409,12 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 		).then(() => {
 			notification.success({
 				message: "Download complete",
-				description: `${mods.length} mods have been downloaded to the controller.`,
+				description: `${missing.length} mods have been downloaded to the controller.`,
 			});
 		}).catch(
 			notifyErrorHandler("Error starting mod download")
 		);
-	};
+	}
 
 	return <>
 		{props.checkForUpdates && <Button
@@ -415,8 +438,11 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 				<Button
 					key="start"
 					type="primary"
-					disabled={loading || mods.length === 0}
-					onClick={() => { setOpen(false); handleControllerDownload(); }}
+					disabled={loading || missing.length === 0}
+					onClick={() => {
+						setOpen(false);
+						fixModErrors();
+					}}
 				>Start Download</Button>,
 			]}
 		>
@@ -429,30 +455,26 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 				style={{ marginBottom: 16 }}
 			/>}
 
-			{/* Display warning message if dependencies could not be found */}
-			{!loading && missing.length > 0 && <Alert
-				message="Failed to Find Dependencies:"
-				description={<ul>
-					{missing.map(name => <li>{name}</li>)}
-				</ul>}
-				type="warning"
-				showIcon
-				style={{ marginBottom: 16 }}
-			/>}
-
-			{/* Display warning message if dependencies are incompatible */}
-			{!loading && incompatible.length > 0 && <Alert
-				message="Incompatible Dependencies Found:"
-				description={<ul>
-					{incompatible.map(name => <li>{name}</li>)}
-				</ul>}
-				type="warning"
-				showIcon
-				style={{ marginBottom: 16 }}
-			/>}
+			{/* Display warning message for mod errors */}
+			{([
+				["notFound", "Failed to find dependencies:"],
+				["incompatible", "Dependencies are incompatible:"],
+				["unsatisfiable", "Dependencies could not be satisfied:"],
+				["invalidDependency", "Dependencies has invalid specifications:"],
+			] as const).map(([prop, message]) => {
+				const modNames = modErrors[prop];
+				return modNames.length === 0 ? null : <Alert
+					showIcon
+					type="warning"
+					message={message}
+					description={<ul>{modNames.map(name => <li>{name}</li>)}</ul>}
+					style={{ marginBottom: 14 }}
+				/>;
+			})}
 
+			{/* Display a table of the missing mods that need to be downloaded */}
 			<Table
-				dataSource={mods}
+				dataSource={missing}
 				rowKey={record => record.name}
 				loading={loading}
 				pagination={{

packages/web_ui/src/components/NpmButton.tsx

@@ -82,7 +82,11 @@ function NpmPluginInstall({ setApplyAction, form, target }: NpmActionProps) {
 	</>;
 }
 
-export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: boolean, disabled?: boolean }) {
+export function NpmButton(props: {
+	target: "controller" | { hostId: number },
+	canRestart?: boolean,
+	disabled?: boolean
+}) {
 	const control = useContext(ControlContext);
 	const account = useAccount();
 	const [open, setOpen] = useState(false);
@@ -93,6 +97,7 @@ export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: bo
 		remoteUpdates: true, pluginUpdates: true, pluginInstall: true,
 	});
 
+	const hostId = props.target === "controller" ? undefined : props.target.hostId;
 	useEffect(() => {
 		if (props.target === "controller") {
 			if (!account.hasPermission("core.controller.get_config")) {
@@ -121,7 +126,7 @@ export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: bo
 				});
 			}).catch(notifyErrorHandler("Failed to fetch host config for remote updates"));
 		}
-	}, [props.target]);
+	}, [hostId]);
 
 	function onValuesChange({ action } : { action?: string }) {
 		if (action) {

packages/web_ui/src/components/PageHeader.tsx

@@ -5,6 +5,7 @@ const { Text } = Typography;
 type PageHeaderProps = {
 	title: string | React.ReactElement;
 	subTitle?: string | React.ReactElement;
+	status?: React.JSX.Element;
 	extra?: React.JSX.Element;
 };
 
@@ -13,6 +14,7 @@ export default function PageHeader(props: PageHeaderProps): React.JSX.Element {
 		<Space>
 			<h2 className="page-header-title">{props.title}</h2>
 			{props.subTitle && <Text type="secondary">{props.subTitle}</Text>}
+			{props.status ?? null}
 		</Space>
 		{props.extra && <div className="page-header-extra">{props.extra}</div>}
 	</div>;

packages/web_ui/src/components/PluginExtra.tsx

@@ -9,7 +9,7 @@ import BaseWebPlugin from "../BaseWebPlugin";
 type PluginExtraProps = {
 	component: string;
 	instance?: lib.InstanceDetails;
-	user?: lib.User;
+	user?: lib.UserDetails;
 	host?: lib.HostDetails;
 	role?: lib.Role;
 };

packages/web_ui/src/components/UsersTable.tsx

@@ -44,7 +44,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 	const data = [...users.values()];
 
 	// Determine online predicate based on instanceId
-	const isUserOnline = (user: lib.User) => {
+	const isUserOnline = (user: lib.UserDetails) => {
 		if (instanceId !== undefined) {
 			return user.instances && user.instances.has(instanceId);
 		}
@@ -58,7 +58,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 		{
 			title: "Name",
 			key: "name",
-			render: (_: any, user: lib.User) => (
+			render: (_: any, user: lib.UserDetails) => (
 				<Space>
 					{user.name}
 					<span>
@@ -69,11 +69,11 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 				</Space>
 			),
 			defaultSortOrder: "ascend",
-			sorter: (a: lib.User, b: lib.User) => strcmp(a.name, b.name),
+			sorter: (a: lib.UserDetails, b: lib.UserDetails) => strcmp(a.name, b.name),
 			filterIcon: (filtered: boolean) => (
 				<SearchOutlined style={{ color: filtered ? "#1677ff" : undefined }} />
 			),
-			onFilter: (value: string | number | boolean, record: lib.User) => record.name
+			onFilter: (value: string | number | boolean, record: lib.UserDetails) => record.name
 				.toLowerCase()
 				.includes((value as string).toLowerCase()),
 			filterDropdownProps: {
@@ -106,8 +106,10 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			key: "roles",
 			filters: roleFilters,
 			filterMultiple: true,
-			onFilter: (value: string | number | boolean, record: lib.User) => record.roleIds.has(value as number),
-			render: (_: any, user: lib.User) => (
+			onFilter: (value: string | number | boolean, record: lib.UserDetails) => (
+				record.roleIds.has(value as number)
+			),
+			render: (_: any, user: lib.UserDetails) => (
 				[...user.roleIds].map((id) => (
 					<Link key={id} to={`/roles/${id}/view`} onClick={(e) => e.stopPropagation()}>
 						<Tag>{(roles.get(id) || { name: id }).name}</Tag>
@@ -119,7 +121,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 
 
 	// Helper to get last seen timestamp
-	function getLastSeenTimestamp(user: lib.User): number | undefined {
+	function getLastSeenTimestamp(user: lib.UserDetails): number | undefined {
 		const stats = instanceId !== undefined ? user.instanceStats.get(instanceId) : user.playerStats;
 		if (!stats) { return undefined; }
 		if (stats.lastLeaveAt && stats.lastLeaveAt.getTime() > (stats.lastJoinAt?.getTime() ?? 0)) {
@@ -136,11 +138,11 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "Play Time",
 				key: "playTime",
-				render: (_: any, user: lib.User) => {
+				render: (_: any, user: lib.UserDetails) => {
 					const instanceStats = user.instanceStats.get(instanceId);
 					return instanceStats?.onlineTimeMs ? formatDuration(instanceStats.onlineTimeMs) : "-";
 				},
-				sorter: (a: lib.User, b: lib.User) => {
+				sorter: (a: lib.UserDetails, b: lib.UserDetails) => {
 					const statsA = a.instanceStats.get(instanceId);
 					const statsB = b.instanceStats.get(instanceId);
 					return (statsA?.onlineTimeMs ?? 0) - (statsB?.onlineTimeMs ?? 0);
@@ -149,11 +151,11 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "Join Count",
 				key: "joinCount",
-				render: (_: any, user: lib.User) => {
+				render: (_: any, user: lib.UserDetails) => {
 					const instanceStats = user.instanceStats.get(instanceId);
 					return instanceStats?.joinCount ?? 0;
 				},
-				sorter: (a: lib.User, b: lib.User) => {
+				sorter: (a: lib.UserDetails, b: lib.UserDetails) => {
 					const statsA = a.instanceStats.get(instanceId);
 					const statsB = b.instanceStats.get(instanceId);
 					return (statsA?.joinCount ?? 0) - (statsB?.joinCount ?? 0);
@@ -162,8 +164,8 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "First Seen",
 				key: "firstSeen",
-				render: (_: any, user: lib.User) => formatFirstSeen(user, instanceId),
-				sorter: (a: lib.User, b: lib.User) => {
+				render: (_: any, user: lib.UserDetails) => formatFirstSeen(user, instanceId),
+				sorter: (a: lib.UserDetails, b: lib.UserDetails) => {
 					const statsA = a.instanceStats.get(instanceId);
 					const statsB = b.instanceStats.get(instanceId);
 					const firstSeenA = statsA?.firstJoinAt?.getTime() ?? 0;
@@ -177,18 +179,18 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "Online time",
 				key: "onlineTime",
-				render: (_: any, user: lib.User) => (user.playerStats?.onlineTimeMs
+				render: (_: any, user: lib.UserDetails) => (user.playerStats?.onlineTimeMs
 					? formatDuration(user.playerStats.onlineTimeMs)
 					: null),
 				// eslint-disable-next-line max-len
-				sorter: (a: lib.User, b: lib.User) => (a.playerStats?.onlineTimeMs ?? 0) - (b.playerStats?.onlineTimeMs ?? 0),
+				sorter: (a: lib.UserDetails, b: lib.UserDetails) => (a.playerStats?.onlineTimeMs ?? 0) - (b.playerStats?.onlineTimeMs ?? 0),
 				responsive: ["lg"],
 			},
 			{
 				title: "First seen",
 				key: "firstSeen",
-				render: (_: any, user: lib.User) => formatFirstSeen(user),
-				sorter: (a: lib.User, b: lib.User) => sortFirstSeen(a, b),
+				render: (_: any, user: lib.UserDetails) => formatFirstSeen(user),
+				sorter: (a: lib.UserDetails, b: lib.UserDetails) => sortFirstSeen(a, b),
 			},
 		);
 	}
@@ -204,7 +206,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 				{ text: "7d", value: "7d" },
 				{ text: "30d", value: "30d" },
 			],
-			onFilter: (value: string | number | boolean, record: lib.User) => {
+			onFilter: (value: string | number | boolean, record: lib.UserDetails) => {
 				const online = isUserOnline(record);
 				if (value === "online") {
 					return online;
@@ -227,8 +229,8 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 						return false;
 				}
 			},
-			sorter: (a: lib.User, b: lib.User) => sortLastSeen(a, b, instanceId, instanceId),
-			render: (_: any, user: lib.User) => formatLastSeen(user, instanceId),
+			sorter: (a: lib.UserDetails, b: lib.UserDetails) => sortLastSeen(a, b, instanceId, instanceId),
+			render: (_: any, user: lib.UserDetails) => formatLastSeen(user, instanceId),
 			responsive: ["lg"],
 		},
 	);

packages/web_ui/src/components/system_metrics.tsx

@@ -63,3 +63,52 @@ export function RestartRequired(props: { system?: lib.SystemInfo }) {
 		<ReloadOutlined style={{ color: "yellow" }}/>
 	</Tooltip>;
 }
+
+/**
+ * Returns a human-readable duration between two dates.
+ *
+ * The output includes up to two most significant time units and omits
+ * leading zero units. Supported units are weeks, days, hours, minutes,
+ * and seconds.
+ *
+ * Examples: "1w 2d", "2d 3h", "1h 2m", "45s"
+ *
+ * @param date1 - The starting date.
+ * @param date2 - The ending date.
+ * @returns A duration string (e.g. "2d 3h").
+ */
+function humanAbsTimeDiff(date1: number, date2 = 0) {
+	let diffMs = Math.abs(date2 - date1);
+	const totalSeconds = Math.floor(diffMs / 1000);
+
+	const weeks = Math.floor(totalSeconds / 604800); // 7 * 24 * 60 * 60
+	const days = Math.floor((totalSeconds % 604800) / 86400);
+	const hours = Math.floor((totalSeconds % 86400) / 3600);
+	const minutes = Math.floor((totalSeconds % 3600) / 60);
+	const seconds = totalSeconds % 60;
+
+	if (weeks > 0) {
+		return days > 0 ? `${weeks}w ${days}d` : `${weeks}w`;
+	}
+
+	if (days > 0) {
+		return hours > 0 ? `${days}d ${hours}h` : `${days}d`;
+	}
+
+	if (hours > 0) {
+		return minutes > 0 ? `${hours}h ${minutes}m` : `${hours}h`;
+	}
+
+	if (minutes > 0) {
+		return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
+	}
+
+	return `${seconds}s`;
+}
+
+export function MetricRelativeDate(props: { timeMs?: number }) {
+	if (!props.timeMs || props.timeMs === 0) {
+		return "N/A";
+	}
+	return `${new Date(props.timeMs).toLocaleString()} (${humanAbsTimeDiff(props.timeMs, Date.now())})`;
+}

packages/web_ui/src/index.ts

@@ -21,7 +21,7 @@ export { default as LogConsole } from "./components/LogConsole";
 export { default as AssignInstanceModal } from "./components/AssignInstanceModal";
 export { default as InstanceList } from "./components/InstanceList";
 export { default as InstanceRcon } from "./components/InstanceRcon";
-export { default as InstanceStatusTag, statusColors } from "./components/InstanceStatusTag";
+export { default as InstanceStatusTag } from "./components/InstanceStatusTag";
 export { default as InstanceControlButton, InstanceControlButtonPermissions } from "./components/InstanceControlButton";
 
 export { default as PageHeader } from "./components/PageHeader";

packages/web_ui/src/model/export_manifest.ts

@@ -4,10 +4,10 @@ import * as lib from "@clusterio/lib";
 import ControlContext from "../components/ControlContext";
 
 let exportManifestCache: lib.ExportManifest|null = null;
-let emptyCache = { assets: {} };
+let loaded = false;
 export function useExportManifest(): lib.ExportManifest|null {
 	const control = useContext(ControlContext);
-	let [exportManifest, setExportManifest] = useState<lib.ExportManifest|null>(exportManifestCache || emptyCache);
+	let [exportManifest, setExportManifest] = useState<lib.ExportManifest|null>(exportManifestCache);
 	useEffect(() => {
 		async function load() {
 			let modPack = await control.send(new lib.ModPackGetDefaultRequest());
@@ -15,9 +15,10 @@ export function useExportManifest(): lib.ExportManifest|null {
 				exportManifestCache = modPack.exportManifest;
 				setExportManifest(exportManifestCache);
 			}
+			loaded = true;
 		}
 
-		if (!exportManifestCache) {
+		if (!loaded) {
 			load();
 		}
 	}, []);

packages/web_ui/src/model/item_metadata.ts

@@ -7,31 +7,51 @@ type Metadata = {
 	x: number;
 	y: number;
 	size: number;
+	category: string;
 	localised_name?: any[];
+	path?: string;
 };
-let itemMetadataCache: Map<string, Metadata> | null = null;
-export function useItemMetadata() {
-	let exportManifest = useExportManifest();
-	let [itemMetadata, setItemMetadata] = useState<Map<string, Metadata>>(itemMetadataCache || new Map());
+
+const metadataCaches = new Map<string, Map<string, Metadata>>();
+let cssInjected = false;
+
+/**
+ * Load the unified spritesheet metadata and inject CSS rules for every icon.
+ * CSS class name: `.{category}-{CSS.escape(name)}`
+ * e.g. `.item-iron-plate`, `.recipe-iron-plate`, `.signal-signal-red`, `.planet-nauvis`
+ */
+function useSpriteMetadata(category: string): Map<string, Metadata> {
+	const exportManifest = useExportManifest();
+	const [metadata, setMetadata] = useState<Map<string, Metadata>>(
+		metadataCaches.get(category) ?? new Map()
+	);
+
 	useEffect(() => {
 		async function load() {
 			if (
 				!exportManifest
-				|| !exportManifest.assets["item-metadata"]
-				|| !exportManifest.assets["item-spritesheet"]
+				|| !exportManifest.assets["metadata"]
+				|| !exportManifest.assets["spritesheet"]
 			) {
 				return;
 			}
-			let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`);
-			if (response.ok) {
-				let data = await response.json();
-				itemMetadataCache = new Map(data);
-				let style = document.createElement("style");
+
+			const response = await fetch(`${staticRoot}static/${exportManifest.assets["metadata"]}`);
+			if (!response.ok) {
+				return;
+			}
+
+			const data: [string, Metadata][] = await response.json();
+
+			if (!cssInjected) {
+				const sheetUrl = `${staticRoot}static/${exportManifest.assets["spritesheet"]}`;
+				const style = document.createElement("style");
 				document.head.appendChild(style);
-				for (let [name, meta] of itemMetadataCache) {
+				for (const [key, meta] of data) {
+					const name = key.includes(".") ? key.slice(key.indexOf(".") + 1) : key;
 					style.sheet!.insertRule(`\
-.item-${CSS.escape(name)} {
-	background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}");
+.${meta.category}-${CSS.escape(name)} {
+	background-image: url("${sheetUrl}");
 	background-repeat: no-repeat;
 	background-position: -${meta.x}px -${meta.y}px;
 	height: ${meta.size}px;
@@ -39,14 +59,32 @@ export function useItemMetadata() {
 }`
 					);
 				}
-				setItemMetadata(itemMetadataCache);
+				cssInjected = true;
 			}
+
+			const cache = new Map<string, Metadata>();
+			for (const [key, meta] of data) {
+				if (meta.category === category) {
+					const name = key.includes(".") ? key.slice(key.indexOf(".") + 1) : key;
+					cache.set(name, meta);
+				}
+			}
+			metadataCaches.set(category, cache);
+			setMetadata(cache);
 		}
 
-		if (!itemMetadataCache) {
+		if (!metadataCaches.has(category)) {
 			load();
 		}
 	}, [exportManifest]);
 
-	return itemMetadata;
+	return metadata;
 }
+
+export function useItemMetadata() { return useSpriteMetadata("item"); }
+export function useRecipeMetadata() { return useSpriteMetadata("recipe"); }
+export function useSignalMetadata() { return useSpriteMetadata("signal"); }
+export function useTechnologyMetadata() { return useSpriteMetadata("technology"); }
+export function usePlanetMetadata() { return useSpriteMetadata("planet"); }
+export function useQualityMetadata() { return useSpriteMetadata("quality"); }
+export function useEntityMetadata() { return useSpriteMetadata("entity"); }

packages/web_ui/src/model/roles.ts

@@ -3,7 +3,7 @@ import ControlContext from "../components/ControlContext";
 
 import * as lib from "@clusterio/lib";
 
-export function useRolesOfUser(user?: lib.User) {
+export function useRolesOfUser(user?: lib.UserDetails) {
 	const [roles, synced] = useRoles();
 	return [
 		user !== undefined ? new Map([...user.roleIds].map(roleId => [roleId, roles.get(roleId)])) : new Map(),

packages/web_ui/src/model/user.tsx

@@ -4,19 +4,19 @@ import ControlContext from "../components/ControlContext";
 
 import * as lib from "@clusterio/lib";
 
-function getInstanceStats(user: lib.User, instanceId?: number) {
+function getInstanceStats(user: lib.UserDetails, instanceId?: number) {
 	if (instanceId === undefined) {
 		return user.playerStats;
 	}
 	return user.instanceStats.get(instanceId);
 }
 
-function calculateFirstSeen(user: lib.User, instanceId?: number) {
+function calculateFirstSeen(user: lib.UserDetails, instanceId?: number) {
 	const stats = getInstanceStats(user, instanceId);
 	return stats?.firstJoinAt?.getTime();
 }
 
-function calculateLastSeen(user: lib.User, instanceId?: number) {
+function calculateLastSeen(user: lib.UserDetails, instanceId?: number) {
 	const stats = getInstanceStats(user, instanceId);
 	if (!stats) {
 		return undefined;
@@ -30,7 +30,7 @@ function calculateLastSeen(user: lib.User, instanceId?: number) {
 	return undefined;
 }
 
-export function formatFirstSeen(user: lib.User, instanceId?: number) {
+export function formatFirstSeen(user: lib.UserDetails, instanceId?: number) {
 	const firstSeen = calculateFirstSeen(user, instanceId);
 	if (firstSeen === undefined) {
 		return undefined;
@@ -38,7 +38,7 @@ export function formatFirstSeen(user: lib.User, instanceId?: number) {
 	return new Date(firstSeen).toLocaleString();
 }
 
-export function formatLastSeen(user: lib.User, instanceId?: number) {
+export function formatLastSeen(user: lib.UserDetails, instanceId?: number) {
 	if (user.instances && [...user.instances].some(id => instanceId === undefined || id === instanceId)) {
 		return <Tag color="green">Online</Tag>;
 	}
@@ -49,14 +49,24 @@ export function formatLastSeen(user: lib.User, instanceId?: number) {
 	return new Date(lastSeen).toLocaleString();
 }
 
-export function sortFirstSeen(userA: lib.User, userB: lib.User, instanceIdA?: number, instanceIdB?: number) {
+export function sortFirstSeen(
+	userA: lib.UserDetails,
+	userB: lib.UserDetails,
+	instanceIdA?: number,
+	instanceIdB?: number
+) {
 	const firstSeenA = calculateFirstSeen(userA, instanceIdA) || 0;
 	const firstSeenB = calculateFirstSeen(userB, instanceIdB) || 0;
 	return firstSeenA - firstSeenB;
 }
 
-export function sortLastSeen(userA: lib.User, userB: lib.User, instanceIdA?: number, instanceIdB?: number) {
-	function epoch(user: lib.User, instanceId?: number) {
+export function sortLastSeen(
+	userA: lib.UserDetails,
+	userB: lib.UserDetails,
+	instanceIdA?: number,
+	instanceIdB?: number
+) {
+	function epoch(user: lib.UserDetails, instanceId?: number) {
 		return user.instances && [...user.instances].some(id => instanceId === undefined || id === instanceId);
 	}
 

packages/web_ui/src/pages.tsx

@@ -16,6 +16,7 @@ import RoleViewPage from "./components/RoleViewPage";
 import PluginsPage from "./components/PluginsPage";
 import PluginViewPage from "./components/PluginViewPage";
 import ErrorPage from "./components/ErrorPage";
+import IconReferencePage from "./components/IconReferencePage";
 
 export const pages: PluginPage[] = [
 	{
@@ -88,6 +89,10 @@ export const pages: PluginPage[] = [
 		sidebarPath: "/plugins",
 		content: <PluginViewPage />,
 	},
+	{
+		path: "/icon-reference",
+		content: <IconReferencePage />,
+	},
 	{
 		path: "/error",
 		content: <ErrorPage throw error={new Error("Example Error Message")} />,