Initial plan

.github/workflows/node.js.yml

@@ -40,12 +40,6 @@ jobs:
       with:
         files: ./coverage/lcov.info
         token: ${{ secrets.CODECOV_TOKEN }}
-    - name: Upload test logs for debugging
-      if: ${{ !cancelled() }}
-      uses: actions/upload-artifact@v4
-      with:
-        name: logs-${{ matrix.node-version }}-${{ matrix.factorio-version }}
-        path: temp/test/logs
 
   lint:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -31,66 +31,6 @@ Many thanks to the following for contributing to this release:
 [@username](https://github.com/username)
 -->
 
-## Version 2.0.0-alpha.23
-
-### Major Features
-
-- Added automatic headless download and update on linux. [#330](https://github.com/clusterio/clusterio/issues/330)
-- Added recipe notification sync to inventory sync. [#506](https://github.com/clusterio/clusterio/issues/506)
-
-### Features
-
-- Added config lock files to prevent process duplication. [#815](https://github.com/clusterio/clusterio/issues/815)
-- Added process and system start times to system metrics. [#826](https://github.com/clusterio/clusterio/issues/826)
-- Added icons to status tags for host and instances. [#827](https://github.com/clusterio/clusterio/issues/827)
-- Extended export-data to produce separate spritesheets for recipes, signals, technologies, planets, qualities, entities, and static UI icons. [#838](https://github.com/clusterio/clusterio/pull/838)
-- Added Icon Reference page to web interface at `/icon-reference` for browsing all exported icons. [#838](https://github.com/clusterio/clusterio/pull/838)
-- Added visual flash to RCON result box to confirm command execution, even when the output is identical to the previous result. [#793](https://github.com/clusterio/clusterio/issues/793)
-
-### Changes
-
-- Login screen now displays the cluster name. [#818](https://github.com/clusterio/clusterio/issues/818)
-- Instance versions can now be set using Major.Minor without specifing a patch version. [#820](https://github.com/clusterio/clusterio/issues/820)
-- Versions for factorio and modpacks can now be selected using a drop down menu. [#820](https://github.com/clusterio/clusterio/issues/820)
-- The controller will now refuse to start if no users were loaded from file. [#814](https://github.com/clusterio/clusterio/issues/814)
-- Improved wording and display of the generate host token modal. [#828](https://github.com/clusterio/clusterio/issues/828)
-- Installations where muliple conflicting clusterio/lib packages have been installed will now fail with an error [#863](https://github.com/clusterio/clusterio/issues/863).
-- Export completion now logs per-category icon counts. [#838](https://github.com/clusterio/clusterio/pull/838)
-
-### Fixes
-
-- Removed debug log statement from equipment grid serialization. [#XXX](https://github.com/clusterio/clusterio/pull/XXX)
-- Fix that the documentation had a dead link to details of the message passing protocol (#812)
-- Fixed case where achievements were not disabled. [#817](https://github.com/clusterio/clusterio/pull/817)
-- Fix headless download on linux install. [#815](https://github.com/clusterio/clusterio/issues/816)
-- Fix option `--no-download-headless` being ignored. [#815](https://github.com/clusterio/clusterio/issues/816)
-- Fixed action buttons overflowing on narrow screens across the web interface. [#810](https://github.com/clusterio/clusterio/issues/810)
-- Fixed player logistics failing to load on servers without unlocked logistics networks. [#1234](https://github.com/clusterio/clusterio/issues/1234)
-- Fixed mod dependency parser breaking with inconsistent spaces. [#834](https://github.com/clusterio/clusterio/issues/834)
-- No longer attempts to sync inventory in single player. [#835](https://github.com/clusterio/clusterio/issues/835)
-- Fixed tests failing when the configured factorio_directory contains stuff other than Factorio installs [#540](https://github.com/clusterio/clusterio/issues/540).
-- Always create a whitelist file so it can be watched for changes. [#865](https://github.com/clusterio/clusterio/issues/865)
-- Added missing await statements which could cause an instance to start with an outdated admin/ban/whitelist. [#869](https://github.com/clusterio/clusterio/issues/869)
-- Fixed metrics breaking when labels contain comma or equal signs [#861](https://github.com/clusterio/clusterio/issues/861)
-- Fixed Host page show repeated errors when viewing an offline local host [#848](https://github.com/clusterio/clusterio/issues/848)
-- Fixed bad mod paths in icon specifications crashing the export. [#841](https://github.com/clusterio/clusterio/issues/841)
-- Fixed prototypes with no icon data causing errors during export. [#841](https://github.com/clusterio/clusterio/issues/841)
-
-### Breaking Changes
-
-- Custom permission methods should use `IUser` rather than `IControllerUser` [#837](https://github.com/clusterio/clusterio/pull/837)
-- `controller.userManager` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837)
-- `controller.instances` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837)
-- `User/Instance/HostInfo` renamed to `User/Instance/HostRecord` [#837](https://github.com/clusterio/clusterio/pull/837)
-- Removed `host.tls_ca` and `control.tls_ca` config options, see [section on self signed certificates](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate) for the replacement for this option [#857](https://github.com/clusterio/clusterio/issues/857)
-
-Many thanks to the following for contributing to this release:  
-[@Cooldude2606](https://github.com/Cooldude2606)
-[@Daniel](https://github.com/Danielv123)
-[@Double](https://github.com/DoubleThoughtTheProgrammer)
-[@Hornwitser](https://github.com/Hornwitser)
-[@Ryan Walker](https://github.com/solarcloud7)
-
 ## Version 2.0.0-alpha.22
 
 ### Major Features

clean.js

@@ -104,7 +104,7 @@ async function main() {
 			removeBuildArtifacts();
 			removeTestArtifacts();
 		})
-		.command("$0", "Remove all build artifacts", () => {}, argv => {
+		.command("fast", "Remove all build artifacts", () => {}, argv => {
 			DRY = argv.dry;
 			removeBuildArtifacts();
 		})

docs/configuration.md

@@ -182,6 +182,14 @@ You can generate an access token with `clusterioctl host generate-token --id <ho
 Defaults to "enter token here".
 
 
+### host.tls_ca
+
+Path to certificate in PEM format to use for validating a TLS connection to the controller.
+If you have a self signed certificate on the controller you will need to copy the certificate to your hosts and set it as the certificate authority with this option.
+
+Defaults to null meaning use Node.js's default set of trusted certificate authorities.
+
+
 ### host.public_address
 
 External address instances hosted on this host can be accessed on.
@@ -405,6 +413,14 @@ You can generate an access token with `clusteriocontroller bootstrap generate-us
 Defaults to null meaning complain about it not being set and exit.
 
 
+### control.tls_ca
+
+Path to certificate in PEM format to use for validating a TLS connection to the controller.
+If you have a self signed certificate on the controller you will need to copy the certificate to the computer you run clusterctl from and set it as the certificate authority with this option.
+
+Defaults to null meaning use Node.js's default set of trusted certificate authorities.
+
+
 ### control.max_reconnect_delay
 
 Max duration in seconds to wait before attempting to reconnect with the controller after the connection is dropped.

docs/setting-up-tls.md

@@ -63,25 +63,10 @@ This creates a private key file named `key.pem` and a public certificate file na
     npx clusteriocontroller config set controller.tls_certificate cert.pem
     npx clusteriocontroller config set controller.tls_private_key key.pem
 
-Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure node to trust it by setting the `NODE_EXTRA_CA_CERTS` environment variable to a path to this file before the host is started. You can do this by passing `--self-signed-cert path/to/cert` to the installer when setting up a host, or manually by editing run-host.cmd, run-host.sh and/or clusteriohost.service by adding the following line next to the one setting `NODE_OPTIONS`: 
+Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure it as the `host.tls_ca` option:
 
-**For run-host.cmd**
+    npx clusteriohost config set host.tls_ca cert.pem
 
-    set "NODE_EXTRA_CA_CERTS=path\to\cert.pem"
+You will also need to copy the `cert.pem` file to all of the computers you want to remotely manage the cluster via clusterctl and configure it as `control.tls_ca` option:
 
-**For run-host.sh**
-
-    export "NODE_EXTRA_CA_CERTS=path/to/cert.pem"
-
-**For clusteriohost.service**
-
-    Environment=NODE_EXTRA_CA_CERTS=path/to/cert.pem
-
-You will also need to copy the `cert.pem` file to all of the computers you want to remotely manage the cluster via clusterctl and set the `NODE_EXTRA_CA_CERTS` environment variable as a path to that file before running clusterioctl. For example:
-
-    # On Windows
-    set "NODE_EXTRA_CA_CERTS=path\to\cert.pem"
-    # On Linux
-    export NODE_EXTRA_CA_CERTS="path/to/cert.pem"
-    # You can now use clusterioctl
-    npx clusterioctl instance list
+    npx clusterioctl control-config set control.tls_ca cert.pem

package.json

@@ -7,7 +7,7 @@
 		"build": "tsc --build",
 		"watch": "tsc --build --watch",
 		"build-mod": "pnpm run -r build-mod --output-dir ../../dist",
-		"test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
+		"test": "pnpm run build && mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
 		"silent-test": "SILENT_TEST=y pnpm test",
 		"fast-test": "FAST_TEST=y pnpm test",
 		"cover": "nyc pnpm test",
@@ -15,7 +15,9 @@
 		"ci-cover": "nyc --reporter=lcovonly pnpm run-script test",
 		"lint": "eslint packages plugins test",
 		"lint-fix": "eslint --fix packages plugins test",
-		"clean": "node ./clean.js",
+		"clean": "node ./clean.js fast",
+		"clean-tests": "node ./clean.js tests",
+		"clean-all": "node ./clean.js all",
 		"docs": "typedoc",
 		"bundle-dependencies": "bundle-dependencies"
 	},
@@ -56,6 +58,7 @@
 		"jszip": "^3.10.1",
 		"mocha": "^11.7.1",
 		"nyc": "^17.1.0",
+		"phin": "^3.7.1",
 		"typedoc": "^0.28.9",
 		"typescript": "^5.9.2",
 		"yargs": "^18.0.0"

packages/controller/controller.ts

@@ -32,7 +32,6 @@ const { ConsoleTransport, levels, logger } = lib;
 
 import Controller from "./src/Controller";
 import UserManager from "./src/UserManager";
-import UserRecord from "./src/UserRecord";
 import { version } from "./package.json";
 
 // globals
@@ -134,22 +133,13 @@ async function handleBootstrapCommand(
 	let subCommand = args._[1];
 
 	const databaseDirectory = controllerConfig.get("controller.database_directory");
-	await UserManager.attemptMigrateLegacyUsersFile(
-		path.join(databaseDirectory, "users.json"),
-		path.join(databaseDirectory, "roles.json"),
-	);
-
 	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 		path.join(databaseDirectory, "roles.json"),
 		lib.Role.fromJSON.bind(lib.Role),
 	).bootstrap());
 
-	const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
-		path.join(databaseDirectory, "users.json"),
-		UserRecord.fromJSON.bind(UserRecord),
-	).bootstrap());
-
-	const userManager = new UserManager(users, roles, controllerConfig);
+	const userManager = new UserManager(controllerConfig, roles);
+	await userManager.load(path.join(databaseDirectory, "users.json"));
 
 	if (subCommand === "create-admin") {
 		await controllerConfigLock.acquire(); // Also needed to write to the database files
@@ -160,12 +150,15 @@ async function handleBootstrapCommand(
 			return;
 		}
 
-		const user = userManager.getOrCreateUser(args.name);
-		user.roleIds.add(lib.Role.DefaultAdminRoleId);
-		user.isAdmin = true;
-		user.saveRecord();
+		let admin = userManager.getByName(args.name);
+		if (!admin) {
+			admin = userManager.createUser(args.name);
+		}
 
-		await userManager.records.save();
+		let adminRole = lib.ensureDefaultAdminRole(roles);
+		admin.roleIds.add(adminRole.id);
+		admin.isAdmin = true;
+		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
 		await controllerConfigLock.release();
 
 	} else if (subCommand === "generate-user-token") {
@@ -438,7 +431,7 @@ async function startup() {
 	);
 
 	// Refuse to start if there are no users loaded
-	if (args.checkUserCount && controller.users.records.size === 0) {
+	if (args.checkUserCount && controller.userManager.users.size === 0) {
 		logger.fatal(
 			"Cannot start controller, no users loaded.\n" +
 			"Try `npx clusteriocontroller bootstrap create-admin <username>`");
@@ -490,7 +483,7 @@ I           version of clusterio.  Expect things to break. I
 `
 	);
 	startup().catch(err => {
-		if (err instanceof lib.StartupError || err.code === "InstallationError") {
+		if (err instanceof lib.StartupError) {
 			logger.fatal(`
 +-------------------------------+
 | Unable to to start controller |

packages/controller/index.ts

@@ -1,15 +1,13 @@
 import { bootstrap } from "./controller";
 export { default as Controller } from "./src/Controller";
+export { default as ControllerUser } from "./src/ControllerUser";
 export { default as ControllerRouter } from "./src/ControllerRouter";
 export { default as ControlConnection } from "./src/ControlConnection";
 export { default as HostConnection } from "./src/HostConnection";
-export { default as InstanceRecord } from "./src/InstanceRecord";
-export { default as InstanceManager } from "./src/InstanceManager";
-export { default as HostRecord } from "./src/HostRecord";
+export { default as InstanceInfo } from "./src/InstanceInfo";
+export { default as HostInfo } from "./src/HostInfo";
 export { default as BaseControllerPlugin } from "./src/BaseControllerPlugin";
 export { default as UserManager } from "./src/UserManager";
-export { default as UserRecord } from "./src/UserRecord";
-export { default as User } from "./src/User";
 export { default as WsServer } from "./src/WsServer";
 
 if (module === require.main) {

packages/controller/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio controller",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.23",
+	"version": "2.0.0-alpha.22b",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --mode production",

packages/controller/src/BaseConnection.ts

@@ -32,7 +32,6 @@ export default class BaseConnection extends lib.Link {
 		this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this));
 		this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this));
 		this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
-		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 
 		this.handle(lib.SubscriptionRequest, this.handleSubscriptionRequest.bind(this));
 		this.connector.on("close", () => {
@@ -107,8 +106,4 @@ export default class BaseConnection extends lib.Link {
 	async handleSubscriptionRequest(request: lib.SubscriptionRequest, src: lib.Address, dst: lib.Address) {
 		return await this._controller.subscriptions.handleRequest(this, request, src, dst);
 	}
-
-	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
-		return await this._controller.factorioVersions.get(request.maxAgeMs);
-	}
 }

packages/controller/src/BaseControllerPlugin.ts

@@ -3,7 +3,7 @@ import type {
 	ModPack, ModInfo, PlayerEvent, PluginNodeEnvInfo,
 } from "@clusterio/lib";
 import type Controller from "./Controller";
-import type InstanceRecord from "./InstanceRecord";
+import type InstanceInfo from "./InstanceInfo";
 import type ControlConnection from "./ControlConnection";
 import type HostConnection from "./HostConnection";
 
@@ -91,7 +91,7 @@ export default class BaseControllerPlugin {
 	 *     The instance that changed.
 	 * @param prev - the previous status of the instance.
 	 */
-	async onInstanceStatusChanged(instance: InstanceRecord, prev?: InstanceStatus) { }
+	async onInstanceStatusChanged(instance: InstanceInfo, prev?: InstanceStatus) { }
 
 	/**
 	 * Called when the value of a controller config field changed.
@@ -117,7 +117,7 @@ export default class BaseControllerPlugin {
 	 * @param curr - The current value of the field.
 	 * @param prev - The previous value of the field.
 	 */
-	async onInstanceConfigFieldChanged(instance: InstanceRecord, field: string, curr: unknown, prev: unknown) { }
+	async onInstanceConfigFieldChanged(instance: InstanceInfo, field: string, curr: unknown, prev: unknown) { }
 
 	/**
 	 * Called before collecting Prometheus metrics
@@ -261,7 +261,7 @@ export default class BaseControllerPlugin {
 	 *     The instance it occured on.
 	 * @param event - Information about the event.
 	 */
-	async onPlayerEvent(instance: InstanceRecord, event: PlayerEvent) { }
+	async onPlayerEvent(instance: InstanceInfo, event: PlayerEvent) { }
 
 	/**
 	 * Broadcast event to all connected hosts

packages/controller/src/ControlConnection.ts

@@ -6,8 +6,7 @@ import * as lib from "@clusterio/lib";
 const { logFilter, logger } = lib;
 
 import BaseConnection from "./BaseConnection";
-import User from "./User";
-import UserRecord from "./UserRecord";
+import ControllerUser from "./ControllerUser";
 import * as routes from "./routes";
 import Controller from "./Controller";
 
@@ -41,7 +40,7 @@ export default class ControlConnection extends BaseConnection {
 		registerData: { version: string },
 		connector: WsServerConnector,
 		controller: Controller,
-		public user: User, // The user making this connection.
+		public user: ControllerUser, // The user making this connection.
 		public id: number
 	) {
 		super(connector, controller);
@@ -127,6 +126,7 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
 		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
 		this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this));
+		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 	}
 
 	validateIngress(message: lib.MessageRequest | lib.MessageEvent) {
@@ -250,7 +250,7 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleInstanceDetailsGetRequest(request: lib.InstanceDetailsGetRequest) {
-		return this._controller.instances.getForRequest(request.instanceId).toInstanceDetails();
+		return this._controller.getRequestInstance(request.instanceId).toInstanceDetails();
 	}
 
 	async handleInstanceDetailsListRequest() {
@@ -268,20 +268,20 @@ export default class ControlConnection extends BaseConnection {
 			instanceConfig.set("instance.assigned_host", null); // New instances are unassigned
 		}
 		instanceConfig.update(request.config, false, "control");
-		await this._controller.instances.createInstance(instanceConfig);
+		await this._controller.instanceCreate(instanceConfig);
 	}
 
 	async handleInstanceDeleteRequest(request: lib.InstanceDeleteRequest) {
-		await this._controller.instances.deleteInstance(request.instanceId);
+		await this._controller.instanceDelete(request.instanceId);
 	}
 
 	async handleInstanceConfigGetRequest(request: lib.InstanceConfigGetRequest) {
-		let instance = this._controller.instances.getForRequest(request.instanceId);
+		let instance = this._controller.getRequestInstance(request.instanceId);
 		return instance.config.toRemote("control");
 	}
 
 	async handleInstanceConfigSetFieldRequest(request: lib.InstanceConfigSetFieldRequest) {
-		let instance = this._controller.instances.getForRequest(request.instanceId);
+		let instance = this._controller.getRequestInstance(request.instanceId);
 		if (request.field === "instance.assigned_host") {
 			throw new lib.RequestError("instance.assigned_host must be set through the assign-host interface");
 		}
@@ -296,14 +296,14 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleInstanceConfigSetPropRequest(request: lib.InstanceConfigSetPropRequest) {
-		let instance = this._controller.instances.getForRequest(request.instanceId);
+		let instance = this._controller.getRequestInstance(request.instanceId);
 		let { field, prop, value } = request;
 		instance.config.setProp(field as keyof lib.InstanceConfigFields, prop, value, "control");
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
 	async handleInstanceAssignRequest(request: lib.InstanceAssignRequest) {
-		await this._controller.instances.assignInstance(request.instanceId, request.hostId ?? null);
+		await this._controller.instanceAssign(request.instanceId, request.hostId);
 	}
 
 	async handleInstanceSaveDetailsListRequest() {
@@ -337,8 +337,8 @@ export default class ControlConnection extends BaseConnection {
 		if (request.sourceInstanceId === request.targetInstanceId) {
 			throw new lib.RequestError("Source and target instance may not be the same");
 		}
-		let sourceInstance = this._controller.instances.getForRequest(request.sourceInstanceId);
-		let targetInstance = this._controller.instances.getForRequest(request.targetInstanceId);
+		let sourceInstance = this._controller.getRequestInstance(request.sourceInstanceId);
+		let targetInstance = this._controller.getRequestInstance(request.targetInstanceId);
 		let sourceHostId = sourceInstance.config.get("instance.assigned_host");
 		let targetHostId = targetInstance.config.get("instance.assigned_host");
 		if (sourceHostId === null) {
@@ -614,21 +614,22 @@ export default class ControlConnection extends BaseConnection {
 		return modInfos;
 	}
 
-	// eslint-disable-next-line complexity
 	async handleModDependencyResolveRequest(request: lib.ModDependencyResolveRequest) {
 		const {factorioVersion, checkForUpdates} = request;
 		const searchQueue = [...request.mods];
 		const localInfos = [...this._controller.modStore.mods()];
 
-		const builtInModNames = lib.ModPack.getBuiltinModNames(factorioVersion);
-		const skipSearch = new Set(builtInModNames);
+		const _builtInModNames = lib.ModPack.getBuiltinModNames(factorioVersion);
+		const builtInModNames = new Set(_builtInModNames);
+		const skipSearch = new Set(_builtInModNames);
 
 		type ModPortalModType = InstanceType<typeof lib.ModPortalGetAllRequest.Response>["mods"][number];
 		type ModPortalReleaseType = NonNullable<ModPortalModType["releases"]>[number];
 		const dependencyRequirements = new Map<string, lib.ModVersionRange>();
 		const optionalRequirements = new Map<string, lib.ModVersionRange>();
 		const candidateReleases = new Map<string, lib.ModInfo>();
-		const errors = new Map<string, lib.ModDependencyResolveErrors>();
+		const incompatible = new Set<string>();
+		const invalid = new Set<string>();
 
 		const _modPortalReleases = new Map<string, ModPortalModType | null>();
 		async function fetchModReleases(modName: string) {
@@ -649,44 +650,10 @@ export default class ControlConnection extends BaseConnection {
 			return releases;
 		}
 
-		function selectModRelease(releases: ModPortalModType | null, versionRange: lib.ModVersionRange) {
-			if (!releases || !releases.releases) {
-				return null;
-			}
-
-			// Select the latest matching version
-			const release = releases.releases
-				.filter(info => versionRange.testVersion(info.version))
-				.reduce<ModPortalReleaseType | undefined>((max, cur) => (
-					max && lib.integerFullVersion(max.version) > lib.integerFullVersion(cur.version) ? max : cur
-				), undefined);
-
-			if (!release) {
-				return null;
-			}
-
-			// Create the mod info class from the selected
-			try {
-				return lib.ModInfo.fromJSON({
-					...release.info_json, // includes dependencies
-					sha1: release.sha1,
-					version: release.version,
-					name: releases.name,
-					author: releases.owner,
-					title: releases.title,
-				});
-			} catch {
-				// Can error if there are invalid dependencies
-				errors.set(releases.name, "invalidDependency");
-				return null;
-			}
-		}
-
 		while (searchQueue.length > 0) {
-			// Dequeue the next mod, skip if incompatible
 			const mod = searchQueue.shift()!;
 			if (mod.incompatible) {
-				errors.set(mod.name, "incompatible");
+				incompatible.add(mod.name);
 				continue;
 			}
 
@@ -707,13 +674,13 @@ export default class ControlConnection extends BaseConnection {
 				versionRange.combineVersion(mod.version);
 				if (!versionRange.valid) {
 					candidateReleases.delete(mod.name);
-					errors.set(mod.name, "unsatisfiable");
+					invalid.add(mod.name);
 				}
 			}
 
-			// Do not add dependencies for errors, skips, optionals, or valid candidates
+			// Do not add dependencies for invalids, optionals, skips, or valid candidates
 			const existingCandidate = candidateReleases.get(mod.name);
-			if (errors.has(mod.name) || skipSearch.has(mod.name) || optionalRequirements.has(mod.name) || (
+			if (invalid.has(mod.name) || optionalRequirements.has(mod.name) || skipSearch.has(mod.name) || (
 				existingCandidate && versionRange.testVersion(existingCandidate.version)
 			)) {
 				continue;
@@ -728,11 +695,28 @@ export default class ControlConnection extends BaseConnection {
 
 			// Check if a mod poral version fits the requirements
 			if (!candidate || checkForUpdates) {
-				const releases = await fetchModReleases(mod.name);
-				const release = selectModRelease(releases, versionRange);
-				// We only select the mod portal release if it is more recent than the local candidate
-				if (release && (!candidate || release.integerVersion > candidate.integerVersion)) {
-					candidate = release;
+				const modReleases = await fetchModReleases(mod.name);
+				if (modReleases && modReleases.releases) {
+					const release = modReleases.releases
+						.filter(info => (
+							info.info_json.factorio_version === factorioVersion
+							&& versionRange.testVersion(info.version)
+						))
+						.reduce<ModPortalReleaseType | undefined>((max, cur) => (
+							max && lib.integerFullVersion(max.version) > lib.integerFullVersion(cur.version) ? max : cur
+						), undefined);
+
+					// We only select the mod portal release if it is more recent than the local candidate
+					if (release && (!candidate || lib.integerFullVersion(release.version) > candidate.integerVersion)) {
+						candidate = lib.ModInfo.fromJSON({
+							...release.info_json, // Also includes dependences
+							sha1: release.sha1,
+							version: release.version,
+							name: modReleases.name,
+							author: modReleases.owner,
+							title: modReleases.title,
+						});
+					}
 				}
 			}
 
@@ -748,22 +732,14 @@ export default class ControlConnection extends BaseConnection {
 			}
 		}
 
-		// Add the not found dependencies as errors
-		for (const modName of dependencyRequirements.keys()) {
-			if (!candidateReleases.has(modName) && !errors.has(modName)) {
-				errors.set(modName, "notFound");
-			}
-		}
-
-		// Remove incompatible dependencies if they aren't required by others
-		for (const [modName, error] of errors) {
-			if (error === "incompatible" && !dependencyRequirements.has(modName)) {
-				errors.delete(modName);
-			}
-		}
-
 		return new lib.ModDependencyResolveRequest.Response(
-			[...candidateReleases.values()], errors,
+			[...candidateReleases.values()],
+			[...invalid.values(), ...[...incompatible.values()].filter(mod => (
+				candidateReleases.has(mod) || builtInModNames.has(mod)
+			))],
+			[...dependencyRequirements.keys()].filter(mod => (
+				!candidateReleases.has(mod) && !incompatible.has(mod) && !invalid.has(mod)
+			)),
 		);
 	}
 
@@ -870,33 +846,35 @@ export default class ControlConnection extends BaseConnection {
 		}
 		this._controller.roles.delete(role);
 
-		for (const user of this._controller.users.values()) {
-			if (user.removeRole(id)) {
+		this._controller.userManager.dirty = true;
+		for (let user of this._controller.userManager.users.values()) {
+			if (user.roleIds.delete(id)) {
 				this._controller.userPermissionsUpdated(user);
 			}
 		}
 	}
 
-	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.UserDetails> {
+	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.User> {
 		let name = request.name;
-		let user = this._controller.users.getByName(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User ${name} does not exist`);
 		}
 
-		return user.toUserDetails();
+		return user;
 	}
 
-	async handleUserListRequest(): Promise<lib.UserDetails[]> {
-		return [...this._controller.users.records.values()].map(user => user.toUserDetails());
+	async handleUserListRequest(): Promise<lib.User[]> {
+		return [...this._controller.userManager.users.values()];
 	}
 
 	async handleUserCreateRequest(request: lib.UserCreateRequest) {
-		this._controller.users.createUser(request.name);
+		let user = this._controller.userManager.createUser(request.name);
+		this._controller.usersUpdated([user]);
 	}
 
 	async handleUserRevokeTokenRequest(request: lib.UserRevokeTokenRequest) {
-		let user = this._controller.users.getByName(request.name);
+		let user = this._controller.userManager.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
@@ -910,10 +888,11 @@ export default class ControlConnection extends BaseConnection {
 				controlConnection.connector.terminate();
 			}
 		}
+		this._controller.usersUpdated([user]);
 	}
 
 	async handleUserUpdateRolesRequest(request: lib.UserUpdateRolesRequest) {
-		let user = this._controller.users.getByName(request.name);
+		let user = this._controller.userManager.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
@@ -924,33 +903,35 @@ export default class ControlConnection extends BaseConnection {
 			}
 		}
 
-		user.set("roleIds", new Set(request.roles));
+		user.roleIds = new Set(request.roles);
 		this._controller.userPermissionsUpdated(user);
+		this._controller.usersUpdated([user]);
 	}
 
 	async handleUserSetAdminRequest(request: lib.UserSetAdminRequest) {
 		let { name, create, admin } = request;
-		let user = this._controller.users.getByName(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.users.createUser(name);
+				user = this._controller.userManager.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
 		}
 
-		user.set("isAdmin", admin);
+		user.isAdmin = admin;
+		this._controller.usersUpdated([user]);
 		this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, admin));
 	}
 
 	async handleUserSetBannedRequest(request: lib.UserSetBannedRequest) {
 		let { name, create, banned, reason } = request;
-		let user = this._controller.users.getByNameMutable(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.users.createUser(name);
+				user = this._controller.userManager.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
@@ -958,34 +939,37 @@ export default class ControlConnection extends BaseConnection {
 
 		user.isBanned = banned;
 		user.banReason = reason;
-		user.saveRecord();
+		this._controller.usersUpdated([user]);
 		this._controller.sendTo("allInstances", new lib.InstanceBanlistUpdateEvent(name, banned, reason));
 	}
 
 	async handleUserSetWhitelistedRequest(request: lib.UserSetWhitelistedRequest) {
 		let { name, create, whitelisted } = request;
-		let user = this._controller.users.getByName(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.users.createUser(name);
+				user = this._controller.userManager.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
 		}
 
-		user.set("isWhitelisted", whitelisted);
+		user.isWhitelisted = whitelisted;
+		this._controller.usersUpdated([user]);
 		this._controller.sendTo("allInstances", new lib.InstanceWhitelistUpdateEvent(name, whitelisted));
 	}
 
 	async handleUserDeleteRequest(request: lib.UserDeleteRequest) {
 		let name = request.name;
-		let user = this._controller.users.getByName(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User '${name}' does not exist`);
 		}
 
-		this._controller.users.deleteUser(user);
+		user.isDeleted = true;
+		this._controller.userManager.users.delete(name);
+		this._controller.usersUpdated([user]);
 
 		if (user.isAdmin) {
 			this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, false));
@@ -998,15 +982,14 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
-	/** This does not do a full restore, rather it demotes / unbanns / unwhitelistes users */
-	async handleUserBulkRestoreRequest(request: lib.UserBulkImportRequest, updated: Map<string, UserRecord>) {
+	async handleUserBulkRestoreRequest(request: lib.UserBulkImportRequest, updated: Map<string, ControllerUser>) {
 		if (request.importType === "users") {
 			// Update all fields for all users
 			const users = request.users as Extract<typeof request.users, { username: string }[]>;
 			const admin = new Set(users.filter(u => u.is_admin).map(u => u.username.toLowerCase()));
 			const banned = new Set(users.filter(u => u.is_banned).map(u => u.username.toLowerCase()));
 			const whitelist = new Set(users.filter(u => u.is_whitelisted).map(u => u.username.toLowerCase()));
-			for (const user of this._controller.users.valuesMutable()) {
+			for (const user of this._controller.userManager.users.values()) {
 				if (user.isAdmin && !admin.has(user.name)) {
 					user.isAdmin = false;
 					updated.set(user.id, user);
@@ -1027,7 +1010,7 @@ export default class ControlConnection extends BaseConnection {
 			const expected = new Set(
 				users.map(u => (typeof u === "string" ? u.toLowerCase() : u.username.toLowerCase()))
 			);
-			for (const user of this._controller.users.valuesMutable()) {
+			for (const user of this._controller.userManager.users.values()) {
 				if (user.isBanned && !expected.has(user.name)) {
 					user.isBanned = false;
 					user.banReason = "";
@@ -1039,7 +1022,7 @@ export default class ControlConnection extends BaseConnection {
 			const users = request.users as string[];
 			const expected = new Set(users.map(u => u.toLowerCase()));
 			const prop = request.importType === "admins" ? "isAdmin" : "isWhitelisted";
-			for (const user of this._controller.users.valuesMutable()) {
+			for (const user of this._controller.userManager.users.values()) {
 				if (user[prop] && !expected.has(user.name)) {
 					user[prop] = false;
 					updated.set(user.id, user);
@@ -1048,9 +1031,11 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
+	// A large number of small if statements are unavoidable when reducing the number of updates sent
+
 	async handleUserBulkImportRequest(request: lib.UserBulkImportRequest) {
 		let backup: undefined | Awaited<ReturnType<ControlConnection["handleUserBulkExportRequest"]>>;
-		const updated = new Map<User["id"], User>();
+		const updated = new Map<ControllerUser["id"], ControllerUser>();
 		if (request.restore) {
 			// Unban / Demote / Unwhitelist players not on the list
 			backup = await this.handleUserBulkExportRequest(new lib.UserBulkExportRequest(request.importType));
@@ -1058,12 +1043,11 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		// Will get a user or attempt to create one
-		// UserManager.getOrCreateUser not used due to inclusion of checkPermission
 		const getUserOrCreate = (username: string) => {
-			const user = this._controller.users.getByNameMutable(username);
-			if (user) { return updated.get(user.id) ?? user; }
+			const user = this._controller.userManager.getByName(username);
+			if (user) { return user; }
 			this.user.checkPermission("core.user.create");
-			return this._controller.users.createUser(username);
+			return this._controller.userManager.createUser(username);
 		};
 
 		// Merge the imported data with existing data, this is strictly additive
@@ -1121,14 +1105,14 @@ export default class ControlConnection extends BaseConnection {
 
 		// Send the necessary update events
 		const updatedUsers = [...updated.values()];
-		this._controller.users.records.setMany(updatedUsers);
+		this._controller.usersUpdated(updatedUsers);
 
 		// Resync the host user lists
 		const adminlist: Set<string> = new Set();
 		const banlist: Map<string, string> = new Map();
 		const whitelist: Set<string> = new Set();
 
-		for (let user of this._controller.users.values()) {
+		for (let user of this._controller.userManager.users.values()) {
 			if (user.isAdmin) {
 				adminlist.add(user.id);
 			}
@@ -1152,7 +1136,7 @@ export default class ControlConnection extends BaseConnection {
 		if (request.exportType === "users") {
 			// Send a full user export
 			const usersToSend = [] as lib.ClusterioUserExport["users"];
-			for (const user of this._controller.users.values()) {
+			for (const user of this._controller.userManager.users.values()) {
 				let send = false;
 				const userToSend = { username: user.name } as typeof usersToSend[0];
 				if (user.isBanned) {
@@ -1178,7 +1162,7 @@ export default class ControlConnection extends BaseConnection {
 		} else if (request.exportType === "bans") {
 			// Bans have extra logic to handle ban reason
 			const usersToSend = [] as Array<string | { username: string, reason: string }>;
-			for (const user of this._controller.users.values()) {
+			for (const user of this._controller.userManager.users.values()) {
 				if (user.isBanned) {
 					if (user.banReason && user.banReason !== "") {
 						usersToSend.push({ username: user.name, reason: user.banReason });
@@ -1193,7 +1177,7 @@ export default class ControlConnection extends BaseConnection {
 		// This is a whitelist or admins export
 		const usersToSend = [] as string[];
 		const prop = request.exportType === "admins" ? "isAdmin" : "isWhitelisted";
-		for (const user of this._controller.users.values()) {
+		for (const user of this._controller.userManager.users.values()) {
 			if (user[prop]) {
 				usersToSend.push(user.name);
 			}
@@ -1239,6 +1223,10 @@ export default class ControlConnection extends BaseConnection {
 		this.connector._socket!.clusterio_ignore_dump = true;
 		this._controller.debugEvents.on("message", this.ws_dumper);
 	}
+
+	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
+		return await this._controller.factorioVersions.get(request.maxAgeMs);
+	}
 }
 
 module.exports = ControlConnection;

packages/controller/src/Controller.ts

@@ -18,17 +18,16 @@ import * as lib from "@clusterio/lib";
 const { logger, Summary, Gauge } = lib;
 
 import HttpCloser from "./HttpCloser";
-import InstanceRecord from "./InstanceRecord";
+import InstanceInfo from "./InstanceInfo";
 import * as metrics from "./metrics";
 import * as routes from "./routes";
-import User from "./User";
-import UserRecord from "./UserRecord";
+import ControllerUser from "./ControllerUser";
 import UserManager from "./UserManager";
 import WsServer from "./WsServer";
-import HostRecord from "./HostRecord";
+import HostConnection from "./HostConnection";
+import HostInfo from "./HostInfo";
 import BaseControllerPlugin from "./BaseControllerPlugin";
 import ControllerRouter from "./ControllerRouter";
-import InstanceManager from "./InstanceManager";
 
 const endpointDurationSummary = new Summary(
 	"clusterio_controller_http_endpoint_duration_seconds",
@@ -80,10 +79,6 @@ export default class Controller {
 
 	/** Event subscription controller */
 	subscriptions = new lib.SubscriptionController();
-	/** User manager for the cluster, also contains access to the user records */
-	users: UserManager;
-	/** Instance manager for the cluster, also contains access to the user records */
-	instances: InstanceManager;
 
 	// Possible states are new, starting, running, stopping, stopped
 	private _state: string = "new";
@@ -101,7 +96,6 @@ export default class Controller {
 
 	devMiddleware: any | null = null;
 
-	startedAtMs = Date.now();
 	autosaveInterval?: ReturnType<typeof setInterval>;
 	systemMetricsInterval?: ReturnType<typeof setInterval>;
 
@@ -112,25 +106,12 @@ export default class Controller {
 	// Cache for mod portal requests
 	modPortalCache = new Map<string, { timestamp: number, data: any[] }>();
 	// Cache for factorio versions
-	factorioVersions = new lib.ValueCache(async () => {
-		try {
-			return await lib.fetchFactorioVersions();
-		} catch (err: any) {
-			logger.warn(`Failed to retrieve Factorio versions ${err.message}`);
-			return [];
-		}
-	});
+	factorioVersions = new lib.ValueCache(lib.fetchFactorioVersions);
 
 	static async bootstrap(config: lib.ControllerConfig) {
 		let databaseDirectory = config.get("controller.database_directory");
 		await fs.ensureDir(databaseDirectory);
 
-		// Migrate combined user and role file from 2.0.0-alpha.22
-		await UserManager.attemptMigrateLegacyUsersFile(
-			path.join(databaseDirectory, "users.json"),
-			path.join(databaseDirectory, "roles.json"),
-		);
-
 		const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "systems.json"),
 			lib.SystemInfo.fromJSON.bind(lib.SystemInfo),
@@ -139,13 +120,13 @@ export default class Controller {
 
 		const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "hosts.json"),
-			HostRecord.fromJSON.bind(HostRecord),
+			HostInfo.fromJSON.bind(HostInfo),
 			this.migrateHosts, this.finaliseHosts,
 		).bootstrap());
 
 		const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "instances.json"),
-			InstanceRecord.fromJSON.bind(InstanceRecord),
+			json => InstanceInfo.fromJSON(json, "controller"),
 			this.migrateInstances, this.finaliseInstances,
 		).bootstrap());
 
@@ -166,11 +147,8 @@ export default class Controller {
 			this.migrateRoles,
 		).bootstrap());
 
-		const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
-			path.join(databaseDirectory, "users.json"),
-			UserRecord.fromJSON.bind(UserRecord),
-			undefined, this.finaliseUsers,
-		).bootstrap());
+		const userManager = new UserManager(config, roles);
+		await userManager.load(path.join(databaseDirectory, "users.json"));
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
@@ -182,8 +160,8 @@ export default class Controller {
 		}
 
 		// Add default roles
-		lib.Role.ensureDefaultAdminRole(roles);
-		lib.Role.ensureDefaultPlayerRole(roles);
+		lib.ensureDefaultAdminRole(roles);
+		lib.ensureDefaultPlayerRole(roles);
 
 		return [
 			systems,
@@ -193,7 +171,7 @@ export default class Controller {
 			modPacks,
 			modStore,
 			roles,
-			users,
+			userManager,
 		] as const;
 	}
 
@@ -213,10 +191,10 @@ export default class Controller {
 		 */
 		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
-		/** Mapping of host id to host record */
-		public hosts = new lib.SubscribableDatastore<HostRecord>(),
-		/** Mapping of instance id to instance record */
-		instances = new lib.SubscribableDatastore<InstanceRecord>(),
+		/** Mapping of host id to host info */
+		public hosts = new lib.SubscribableDatastore<HostInfo>(),
+		/** Mapping of instance id to instance info */
+		public instances = new lib.SubscribableDatastore<InstanceInfo>(),
 		/** Mapping of save id to save details */
 		public saves = new lib.SubscribableDatastore<lib.SaveDetails>(),
 		/** Mapping of mod pack id to mod pack */
@@ -225,8 +203,8 @@ export default class Controller {
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
 		/** Mapping of mod pack id to mod pack */
 		public roles = new lib.SubscribableDatastore<lib.Role>(),
-		/** Mapping of user id to user record */
-		users = new lib.SubscribableDatastore<UserRecord>(),
+		/** User and roles manager for the cluster */
+		public userManager = new UserManager(config, roles),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -239,9 +217,6 @@ export default class Controller {
 		this.trustedProxies = this.parseTrustedProxies();
 		this.wsServer = new WsServer(this);
 
-		this.users = new UserManager(users, roles, config);
-		this.instances = new InstanceManager(instances, this);
-
 		this.modStore.on("change", mod => {
 			this.modsUpdated([mod]);
 		});
@@ -261,11 +236,10 @@ export default class Controller {
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
 		this.hosts.on("update", this.hostsUpdated.bind(this));
-		this.instances.records.on("update", this.instanceUpdated.bind(this));
+		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
 		this.roles.on("update", this.rolesUpdated.bind(this));
-		this.users.records.on("update", this.usersUpdated.bind(this));
 	}
 
 	async start(args: ControllerArgs) {
@@ -330,6 +304,9 @@ export default class Controller {
 			}
 			lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev);
 		});
+		for (let instance of this.instances.values()) {
+			this.addInstanceHooks(instance);
+		}
 
 		// Make sure we're actually going to listen on a port
 		let httpPort = this.config.get("controller.http_port");
@@ -420,7 +397,6 @@ export default class Controller {
 		this.onSystemMetricsIntervalChanged();
 
 		logger.info("Started controller");
-		this.startedAtMs = Date.now();
 		this._state = "running";
 	}
 
@@ -577,7 +553,7 @@ export default class Controller {
 				requests.push(hostConnection.send(new lib.SystemInfoRequest()));
 			}
 			const restartRequired = await this.checkRestartRequired();
-			requests.push(lib.gatherSystemInfo("controller", this.canRestart, restartRequired, this.startedAtMs));
+			requests.push(lib.gatherSystemInfo("controller", this.canRestart, restartRequired));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
 				this.systems.set(metric);
@@ -665,13 +641,16 @@ export default class Controller {
 			this.config.save(),
 			this.systems.save(),
 			this.hosts.save(),
-			this.instances.records.save(),
+			this.instances.save(),
 			this.saves.save(),
 			this.modPacks.save(),
 			this.roles.save(),
-			this.users.records.save(),
 		]);
 
+		if (this.userManager.dirty) {
+			await this.userManager.save(path.join(this.config.get("controller.database_directory"), "users.json"));
+		}
+
 		await lib.invokeHook(this.plugins, "onSaveData");
 	}
 
@@ -684,17 +663,11 @@ export default class Controller {
 			if (!json.restartRequired) { // Added in 2.0.0.alpha.21
 				json.restartRequired = false;
 			}
-			if (json.systemStartedAtMs === undefined) { // Added in 2.0.0.alpha.23
-				json.systemStartedAtMs = Date.now();
-			}
-			if (json.processStartedAtMs === undefined) { // Added in 2.0.0.alpha.23
-				json.processStartedAtMs = Date.now();
-			}
 			return json;
 		});
 	}
 
-	static migrateHosts(rawJson: unknown[]): Static<typeof HostRecord.jsonSchema>[] {
+	static migrateHosts(rawJson: unknown[]): Static<typeof HostInfo.jsonSchema>[] {
 		let serialized = rawJson as any;
 
 		// New format 2.0.0.alpha.19
@@ -705,7 +678,7 @@ export default class Controller {
 		return serialized;
 	}
 
-	static finaliseHosts(host: HostRecord): HostRecord {
+	static finaliseHosts(host: HostInfo): HostInfo {
 		if (host.connected) {
 			host.connected = false;
 			host.updatedAtMs = Date.now();
@@ -713,8 +686,8 @@ export default class Controller {
 		return host;
 	}
 
-	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceRecord.jsonSchema>[] {
-		const serialized = rawJson as Static<typeof InstanceRecord.jsonSchema>[];
+	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceInfo.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof InstanceInfo.jsonSchema>[];
 		return serialized.map(json => {
 			if (!json.config) { // New format 2.0.0.alpha.14
 				return { config: json as any, status: "running" }; // Use running to force updatedAtMs
@@ -723,7 +696,7 @@ export default class Controller {
 		});
 	}
 
-	static finaliseInstances(instance: InstanceRecord): InstanceRecord {
+	static finaliseInstances(instance: InstanceInfo): InstanceInfo {
 		const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown";
 		if (instance.status !== status) {
 			instance.status = status;
@@ -757,20 +730,6 @@ export default class Controller {
 		});
 	}
 
-	static finaliseUsers(user: UserRecord, users: Map<UserRecord["id"], UserRecord>): UserRecord {
-		const existingUser = users.get(user.id);
-		if (existingUser) {
-			// All lowercase ids in 2.0.0.alpha.19
-			if (user.playerStats.onlineTimeMs <= existingUser.playerStats.onlineTimeMs) {
-				// We assume the user with the lower online time is the duplicate
-				existingUser.merge(user);
-				return existingUser;
-			}
-			user.merge(existingUser);
-		}
-		return user;
-	}
-
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -889,7 +848,7 @@ export default class Controller {
 		);
 	}
 
-	hostsUpdated(hosts: HostRecord[]) {
+	hostsUpdated(hosts: HostInfo[]) {
 		let updates = hosts.map(host => host.toHostDetails());
 		this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates));
 	}
@@ -901,6 +860,75 @@ export default class Controller {
 		return hosts.length ? new lib.HostUpdatesEvent(hosts) : null;
 	}
 
+	/**
+	 * Get instance by ID for a request
+	 *
+	 * @param instanceId - ID of instance to get.
+	 * @returns Info for the given instance if it exists
+	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 */
+	getRequestInstance(instanceId:number): InstanceInfo {
+		let instance = this.instances.get(instanceId);
+		if (!instance) {
+			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
+		}
+		return instance;
+	}
+
+	/**
+	 * Create a new instance
+	 *
+	 * Adds common Factorio settings to the provided instance config then
+	 * creates an instance using that config in the cluster.
+	 *
+	 * @example
+	 * let instanceConfig = new lib.InstanceConfig("controller");
+	 * instanceConfig.set("instance.name", "My instance");
+	 * let instance = await controller.instanceAssign(instanceConfig);
+	 * await controller.instanceAssign(instance.id, hostId);
+	 *
+	 * @param instanceConfig -
+	 *     Config to base newly created instance on.
+	 * @returns The created instance
+	 */
+	async instanceCreate(instanceConfig: lib.InstanceConfig): Promise<InstanceInfo> {
+		let instanceId = instanceConfig.get("instance.id");
+		if (this.instances.has(instanceId)) {
+			throw new lib.RequestError(`Instance with ID ${instanceId} already exists`);
+		}
+
+		// Add common settings for the Factorio server
+		let settings = {
+			"name": `${this.config.get("controller.name")} - ${instanceConfig.get("instance.name")}`,
+			"description": `Clusterio instance for ${this.config.get("controller.name")}`,
+			"tags": ["clusterio"],
+			"max_players": 0,
+			"visibility": { "public": true, "lan": true },
+			"game_password": "",
+			"require_user_verification": true,
+			"max_upload_in_kilobytes_per_second": 0,
+			"max_upload_slots": 5,
+			"ignore_player_limit_for_returning_players": false,
+			"allow_commands": "admins-only",
+			"autosave_interval": 10,
+			"autosave_slots": 5,
+			"afk_autokick_interval": 0,
+			"auto_pause": false,
+			"only_admins_can_pause_the_game": true,
+			"autosave_only_on_server": true,
+			"non_blocking_saving": false,
+
+			...instanceConfig.get("factorio.settings"),
+		};
+		instanceConfig.set("factorio.settings", settings);
+
+		let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, undefined, Date.now());
+		this.instances.set(instance);
+		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance);
+		this.addInstanceHooks(instance);
+		return instance;
+	}
+
 	/**
 	 * Removes all saves currently stored for the given instance, if any
 	 * @param instanceId - Id of Instance to clear saves for.
@@ -910,6 +938,84 @@ export default class Controller {
 		this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId));
 	}
 
+	/**
+	 * Change assigned host of an instance
+	 *
+	 * Unassigns instance from existing host if already assigned to one and
+	 * then assigns it to the given host.  This is the only supported way of
+	 * changing the instance.assigned_host config entry.
+	 *
+	 * Note: this will not transfer any files or saves stored on the host
+	 * the instance was previously assgined to the new one.
+	 *
+	 * @param instanceId - ID of Instance to assign.
+	 * @param hostId - ID of host to assign instance to.
+	 */
+	async instanceAssign(instanceId: number, hostId?: number) {
+		let instance = this.getRequestInstance(instanceId);
+
+		// Check if target host is connected
+		let newHostConnection: HostConnection | undefined;
+		if (hostId !== undefined) {
+			newHostConnection = this.wsServer.hostConnections.get(hostId);
+			if (!newHostConnection) {
+				// The case of the host not getting the assign instance message
+				// still have to be handled, so it's not a requirement that the
+				// target host be connected to the controller while doing the
+				// assignment, but it is IMHO a better user experience if this
+				// is the case.
+				throw new lib.RequestError("Target host is not connected to the controller");
+			}
+		}
+
+		// Unassign from currently assigned host if it is connected.
+		let currentAssignedHost = instance.config.get("instance.assigned_host");
+		if (currentAssignedHost !== null && hostId !== currentAssignedHost) {
+			let oldHostConnection = this.wsServer.hostConnections.get(currentAssignedHost);
+			if (oldHostConnection && !oldHostConnection.connector.closing) {
+				await oldHostConnection.send(new lib.InstanceUnassignInternalRequest(instanceId));
+			}
+		}
+
+		// Remove saves recorded from currently assigned host if any
+		this.clearSavesOfInstance(instanceId);
+
+		// Assign to target
+		instance.config.set("instance.assigned_host", hostId ?? null);
+		if (hostId !== undefined && newHostConnection) {
+			await newHostConnection.send(
+				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host"))
+			);
+		} else {
+			instance.status = "unassigned";
+		}
+		this.instances.set(instance);
+	}
+
+	/**
+	 * Delete an instance
+	 *
+	 * Permanently deletes the instance from its assigned host (if assigned)
+	 * and the controller.  This action cannot be undone.
+	 *
+	 * @param instanceId - ID of instance to delete.
+	 */
+	async instanceDelete(instanceId: number) {
+		let instance = this.getRequestInstance(instanceId);
+		let hostId = instance.config.get("instance.assigned_host");
+		if (hostId !== null) {
+			await this.sendTo({ hostId }, new lib.InstanceDeleteInternalRequest(instanceId));
+		}
+
+		let prev = instance.status;
+		this.instances.delete(instance);
+
+		this.clearSavesOfInstance(instanceId);
+		this.userManager.clearStatsOfInstance(instanceId);
+
+		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance, prev);
+	}
+
 	/**
 	 * Notify the config of an instance updated
 	 *
@@ -919,7 +1025,7 @@ export default class Controller {
 	 * @param instance -
 	 *     Instance to nodify the config updated.
 	 */
-	async instanceConfigUpdated(instance: InstanceRecord) {
+	async instanceConfigUpdated(instance: InstanceInfo) {
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId !== null) {
 			let connection = this.wsServer.hostConnections.get(hostId);
@@ -931,7 +1037,16 @@ export default class Controller {
 		}
 	}
 
-	instanceUpdated(instances: InstanceRecord[]) {
+	addInstanceHooks(instance: InstanceInfo) {
+		instance.config.on("fieldChanged", (field: string, curr: any, prev: any) => {
+			instance.updatedAtMs = Date.now();
+			this.instances.set(instance); // Trigger update logic
+
+			lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, field, curr, prev);
+		});
+	}
+
+	instanceDetailsUpdated(instances: InstanceInfo[]) {
 		for (const instance of instances) {
 			if (instance.status === "stopped") {
 				this.subscriptions.unsubscribeAddress(lib.Address.fromShorthand({instanceId: instance.id}));
@@ -983,12 +1098,17 @@ export default class Controller {
 		return mods.length ? new lib.ModUpdatesEvent(mods) : null;
 	}
 
-	usersUpdated(users: UserRecord[]) {
+	usersUpdated(users: ControllerUser[]) {
+		const now = Date.now();
+		for (const user of users) {
+			user.updatedAtMs = now;
+		}
+		this.userManager.dirty = true;
 		this.subscriptions.broadcast(new lib.UserUpdatesEvent(users));
 	}
 
 	async handleUserSubscription(request: lib.SubscriptionRequest) {
-		const users = [...this.users.records.values()].filter(
+		const users = [...this.userManager.users.values()].filter(
 			user => user.updatedAtMs > request.lastRequestTimeMs,
 		);
 		return users.length ? new lib.UserUpdatesEvent(users) : null;
@@ -999,7 +1119,7 @@ export default class Controller {
 	 * permissions for this user may have changed.
 	 * @param user - User permisions updated for.
 	 */
-	userPermissionsUpdated(user: Readonly<User>) {
+	userPermissionsUpdated(user: ControllerUser) {
 		for (let controlConnection of this.wsServer.controlConnections.values()) {
 			if (controlConnection.user === user) {
 				controlConnection.send(
@@ -1075,9 +1195,6 @@ export default class Controller {
 				this.plugins.set(pluginInfo.name, controllerPlugin);
 
 			} catch (err: any) {
-				if (err.code === "InstallationError") {
-					throw err;
-				}
 				throw new lib.PluginError(pluginInfo.name, err);
 			}
 
@@ -1261,7 +1378,7 @@ export default class Controller {
 			}
 
 		} else if (dst.type === lib.Address.instance) {
-			const instance = this.instances.getForRequest(dst.id);
+			let instance = this.getRequestInstance(dst.id);
 			let hostId = instance.config.get("instance.assigned_host");
 			if (hostId === null) {
 				throw new lib.RequestError("Instance is not assigned to a host");
@@ -1296,7 +1413,7 @@ export default class Controller {
 			}
 
 		} else if (dst.type === lib.Address.instance) {
-			const instance = this.instances.get(dst.id);
+			let instance = this.instances.get(dst.id);
 			if (!instance) {
 				throw new Error(`Instance with ID ${dst.id} does not exist`);
 			}
@@ -1355,7 +1472,7 @@ export default class Controller {
 	}
 
 	async sendRequestToHostByInstanceId<Req extends InstanceId, Res>(request: Req & lib.Request<Req, Res>) {
-		let instance = this.instances.getForRequest(request.instanceId);
+		let instance = this.getRequestInstance(request.instanceId);
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId === null) {
 			throw new lib.RequestError("Instance is not assigned to a host");
@@ -1368,7 +1485,7 @@ export default class Controller {
 	}
 
 	sendEventToHostByInstanceId<T extends { instanceId: number }>(event: T & lib.Event<T>) {
-		let instance = this.instances.getForRequest(event.instanceId);
+		let instance = this.getRequestInstance(event.instanceId);
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId === null) {
 			return;

packages/controller/src/ControllerUser.ts

@@ -0,0 +1,96 @@
+import { Static, Type } from "@sinclair/typebox";
+
+import {
+	IControllerUser, PermissionError, PlayerStats,
+	Role, SubscribableDatastore, User, permissions,
+} from "@clusterio/lib";
+
+export default class ControllerUser extends User implements IControllerUser {
+	constructor(
+		private _controllerRoles: SubscribableDatastore<Role>,
+		/** Unix time in seconds the user token must be issued after to be valid.  */
+		public tokenValidAfter = 0,
+		...args: ConstructorParameters<typeof User>
+	) {
+		super(...args);
+	}
+
+	static jsonSchema = Type.Object({
+		...User.jsonSchema.properties,
+		token_valid_after: Type.Optional(Type.Number()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>, controllerRoles: SubscribableDatastore<Role>) {
+		const roleIds = new Set<number>(json.roles ?? []);
+		const instanceStats = new Map(
+			(json.instance_stats ?? []).map(
+				([id, stats]) => [id, PlayerStats.fromJSON(stats)]
+			)
+		);
+		const playerStats = User._calculatePlayerStats(instanceStats);
+		return new this(
+			controllerRoles,
+			json.token_valid_after,
+			json.name,
+			roleIds,
+			new Set(json.instances),
+			json.is_admin,
+			json.is_banned,
+			json.is_whitelisted,
+			json.ban_reason,
+			json.updated_at_ms,
+			json.is_deleted,
+			playerStats,
+			instanceStats,
+		);
+	}
+
+	toJSON(controller?: boolean): Static<typeof ControllerUser.jsonSchema> {
+		if (!controller) {
+			return super.toJSON();
+		}
+
+		const json: Static<typeof ControllerUser.jsonSchema> = super.toJSON();
+		if (this.tokenValidAfter) {
+			json.token_valid_after = this.tokenValidAfter;
+		}
+
+		return json;
+	}
+
+	get roles(): ReadonlySet<Readonly<Role>> {
+		return new Set([...this.roleIds].map(
+			id => this._controllerRoles.get(id)
+		).filter(
+			(r): r is Role => Boolean(r)
+		));
+	}
+
+	/**
+	 * Invalidate current tokens for the user
+	 *
+	 * Sets the tokenValidAfter property to the current time, which causes
+	 * all currently issued tokens for the user to become invalid.
+	 */
+	invalidateToken() {
+		this.tokenValidAfter = Math.floor(Date.now() / 1000);
+	}
+
+	checkPermission(permission: string) {
+		if (!permissions.has(permission)) {
+			throw new Error(`permission ${permission} does not exist`);
+		}
+
+		for (let roleId of this.roleIds) {
+			let role = this._controllerRoles.get(roleId);
+			if (!role) {
+				continue;
+			}
+			if (role.permissions.has("core.admin") || role.permissions.has(permission)) {
+				return;
+			}
+		}
+
+		throw new PermissionError("Permission denied");
+	}
+}

packages/controller/src/HostConnection.ts

@@ -5,8 +5,8 @@ import * as lib from "@clusterio/lib";
 const { logger } = lib;
 
 import BaseConnection from "./BaseConnection";
-import InstanceRecord from "./InstanceRecord";
-import HostRecord from "./HostRecord";
+import InstanceInfo from "./InstanceInfo";
+import HostInfo from "./HostInfo";
 
 
 /**
@@ -16,7 +16,7 @@ import HostRecord from "./HostRecord";
  * @alias module:controller/src/HostConnection
  */
 export default class HostConnection extends BaseConnection {
-	private info: HostRecord;
+	private info: HostInfo;
 	plugins: Map<string, string>;
 
 	constructor(
@@ -30,7 +30,7 @@ export default class HostConnection extends BaseConnection {
 		this.plugins = new Map(Object.entries(registerData.plugins));
 
 		const previousHostInfo = this._controller.hosts.get(registerData.id);
-		this.info = new HostRecord(
+		this.info = new HostInfo(
 			registerData.id,
 			previousHostInfo?.name ?? "",
 			registerData.version,
@@ -57,7 +57,7 @@ export default class HostConnection extends BaseConnection {
 
 		this.connector.on("close", () => {
 			// Update status to unknown for instances on this host.
-			const instances: InstanceRecord[] = [];
+			const instances: InstanceInfo[] = [];
 			for (let instance of this._controller.instances.valuesMutable()) {
 				if (instance.config.get("instance.assigned_host") !== this.id) {
 					continue;
@@ -68,7 +68,7 @@ export default class HostConnection extends BaseConnection {
 				instance.status = "unknown";
 				lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 			}
-			this._controller.instances.records.setMany(instances);
+			this._controller.instances.setMany(instances);
 
 			this.info.connected = false;
 			this._controller.hosts.set(this.info);
@@ -207,14 +207,9 @@ export default class HostConnection extends BaseConnection {
 
 		let prev = instance.status;
 		instance.status = request.status;
-		// If the instance has transitioned to running then updated the started at time
-		if (request.status === "running" && prev === "starting") {
-			instance.startedAtMs = Date.now();
-		}
-
 		instance.gamePort = request.gamePort;
 		instance.factorioVersion = request.factorioVersion;
-		this._controller.instances.records.set(instance);
+		this._controller.instances.set(instance);
 		logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
 		await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 	}
@@ -230,7 +225,7 @@ export default class HostConnection extends BaseConnection {
 		}
 
 		// Assign instances the host has but controller does not
-		const instanceUpdates: InstanceRecord[] = [];
+		const instanceUpdates: InstanceInfo[] = [];
 		for (let instanceData of request.instances) {
 			const instanceConfig = new lib.InstanceConfig("controller");
 			instanceConfig.update(instanceData.config, false, "host");
@@ -251,9 +246,6 @@ export default class HostConnection extends BaseConnection {
 					controllerInstance.status = instanceData.status;
 					controllerInstance.gamePort = instanceData.gamePort;
 					controllerInstance.factorioVersion = instanceData.factorioVersion;
-					if (instanceData.status === "running" && prev === "starting") {
-						controllerInstance.startedAtMs = Date.now();
-					}
 					instanceUpdates.push(controllerInstance);
 					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
 					await lib.invokeHook(
@@ -264,30 +256,31 @@ export default class HostConnection extends BaseConnection {
 			}
 
 			instanceConfig.set("instance.assigned_host", this.id);
-			const newInstance = await this._controller.instances.createInstance(instanceConfig, true);
-			newInstance.factorioVersion = instanceData.factorioVersion;
-			newInstance.gamePort = instanceData.gamePort;
-			newInstance.status = instanceData.status;
+			let newInstance = new InstanceInfo(
+				instanceConfig,
+				instanceData.status,
+				instanceData.gamePort,
+				instanceData.factorioVersion,
+				Date.now(),
+			);
 			instanceUpdates.push(newInstance);
+			this._controller.addInstanceHooks(newInstance);
+			await this.send(
+				new lib.InstanceAssignInternalRequest(
+					instanceConfig.get("instance.id"), instanceConfig.toRemote("host")
+				)
+			);
+			await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance);
 		}
 
-		// We suppress the changes made by createInstance so we must manually make them here
-		this._controller.instances.records.setMany(instanceUpdates);
-		await Promise.all(instanceUpdates.flatMap(newInstance => [
-			lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance),
-			this.send(
-				new lib.InstanceAssignInternalRequest(
-					newInstance.config.get("instance.id"), newInstance.config.toRemote("host")
-				)
-			),
-		]));
+		this._controller.instances.setMany(instanceUpdates);
 
 		// Push lists to make sure they are in sync.
 		let adminlist: Set<string> = new Set();
 		let banlist: Map<string, string> = new Map();
 		let whitelist: Set<string> = new Set();
 
-		for (let user of this._controller.users.values()) {
+		for (let user of this._controller.userManager.users.values()) {
 			if (user.isAdmin) {
 				adminlist.add(user.id);
 			}
@@ -336,18 +329,21 @@ export default class HostConnection extends BaseConnection {
 
 	async handleInstancePlayerUpdateEvent(event: lib.InstancePlayerUpdateEvent, src: lib.Address) {
 		let instanceId = src.id;
-		const user = this._controller.users.getOrCreateUser(event.name);
+		let user = this._controller.userManager.getByName(event.name);
+		if (!user) {
+			user = this._controller.userManager.createUser(event.name);
+		}
 
 		if (event.type === "join") {
-			user.notifyJoin(instanceId);
+			this._controller.userManager.notifyJoin(user, instanceId);
 		} else if (event.type === "leave") {
-			user.notifyLeave(instanceId);
+			this._controller.userManager.notifyLeave(user, instanceId);
 		}
 
 		user.instanceStats.set(instanceId, event.stats);
 
 		user.recalculatePlayerStats();
-		user.saveRecord();
+		this._controller.usersUpdated([user]);
 
 		let instance = this._controller.instances.get(instanceId)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
@@ -363,9 +359,13 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		const user = this._controller.users.getOrCreateUser(event.name);
-		user.set("isAdmin", event.admin);
+		let user = this._controller.userManager.getByName(event.name);
+		if (!user) {
+			user = this._controller.userManager.createUser(event.name);
+		}
 
+		user.isAdmin = event.admin;
+		this._controller.usersUpdated([user]);
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.admin ? "promote" : "demote" as lib.PlayerEvent["type"],
@@ -379,11 +379,14 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		const user = this._controller.users.getOrCreateUser(event.name);
+		let user = this._controller.userManager.getByName(event.name);
+		if (!user) {
+			user = this._controller.userManager.createUser(event.name);
+		}
+
 		user.isBanned = event.banned;
 		user.banReason = event.reason;
-		user.saveRecord();
-
+		this._controller.usersUpdated([user]);
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.banned ? "ban" : "unban" as lib.PlayerEvent["type"],
@@ -398,9 +401,13 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		const user = this._controller.users.getOrCreateUser(event.name);
-		user.set("isWhitelisted", event.whitelisted);
+		let user = this._controller.userManager.getByName(event.name);
+		if (!user) {
+			user = this._controller.userManager.createUser(event.name);
+		}
 
+		user.isWhitelisted = event.whitelisted;
+		this._controller.usersUpdated([user]);
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.whitelisted ? "whitelisted" : "unwhitelisted" as lib.PlayerEvent["type"],

packages/controller/src/HostInfo.ts

@@ -1,8 +1,10 @@
 import { Type, Static } from "@sinclair/typebox";
 import { HostDetails } from "@clusterio/lib";
 
-/** Underlying data class for hosts on the controller */
-export default class HostRecord {
+/**
+ * Info about a host known to the controller.
+ */
+export default class HostInfo {
 	constructor(
 		/** Id of this host */
 		public id: number,
@@ -31,12 +33,12 @@ export default class HostRecord {
 		"name": Type.String(),
 		"version": Type.String(),
 		"plugins": Type.Record(Type.String(), Type.String()),
-		"connected": Type.Optional(Type.Boolean()),
-		"remote_address": Type.Optional(Type.String()),
-		"public_address": Type.Optional(Type.String()),
-		"token_valid_after": Type.Optional(Type.Number()),
-		"updated_at_ms": Type.Optional(Type.Number()),
-		"is_deleted": Type.Optional(Type.Boolean()),
+		"connected": Type.Boolean(),
+		"remote_address": Type.String(),
+		"public_address": Type.String(),
+		"token_valid_after": Type.Number(),
+		"updated_at_ms": Type.Number(),
+		"is_deleted": Type.Boolean(),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
@@ -54,50 +56,19 @@ export default class HostRecord {
 		);
 	}
 
-	toJSON() {
-		const json = {
+	toJSON(): Static<typeof HostInfo.jsonSchema> {
+		return {
 			id: this.id,
 			name: this.name,
 			version: this.version,
 			plugins: Object.fromEntries(this.plugins),
-		} as Static<typeof HostRecord.jsonSchema>;
-
-		if (this.connected !== false) {
-			json.connected = this.connected;
-		}
-		if (this.remoteAddress !== "") {
-			json.remote_address = this.remoteAddress;
-		}
-		if (this.publicAddress !== "") {
-			json.public_address = this.publicAddress;
-		}
-		if (this.tokenValidAfter !== 0) {
-			json.token_valid_after = this.tokenValidAfter;
-		}
-		if (this.updatedAtMs !== 0) {
-			json.updated_at_ms = this.updatedAtMs;
-		}
-		if (this.isDeleted !== false) {
-			json.is_deleted = this.isDeleted;
-		}
-
-		return json;
-	}
-
-
-	static fromHostDetails(details: HostDetails, plugins: Map<string, string>) {
-		return new HostRecord(
-			details.id,
-			details.name,
-			details.version,
-			plugins,
-			details.connected,
-			details.remoteAddress,
-			details.publicAddress,
-			details.tokenValidAfter,
-			details.updatedAtMs,
-			details.isDeleted,
-		);
+			connected: this.connected,
+			remote_address: this.remoteAddress,
+			public_address: this.publicAddress,
+			token_valid_after: this.tokenValidAfter,
+			updated_at_ms: this.updatedAtMs,
+			is_deleted: this.isDeleted,
+		};
 	}
 
 	toHostDetails() {
@@ -114,3 +85,4 @@ export default class HostRecord {
 		);
 	}
 }
+

packages/controller/src/InstanceInfo.ts

@@ -1,22 +1,52 @@
 import * as lib from "@clusterio/lib";
 import { Static, Type } from "@sinclair/typebox";
 
-/** Underlying data class for instances on the controller */
-export default class InstanceRecord {
+
+/**
+ * Runtime status of an instance on the controller
+ * @alias module:controller/src/InstanceInfo
+ */
+export default class InstanceInfo {
 	constructor(
 		public config: lib.InstanceConfig,
 		public status: lib.InstanceStatus,
 		public gamePort?: number,
 		public factorioVersion?: lib.TargetVersion,
-		public startedAtMs = 0,
 		public updatedAtMs = 0,
 	) {
 		this.config = config;
 		this.status = status;
 	}
 
-	get id(): number {
-		return this.config.get("instance.id");
+	static jsonSchema = Type.Object({
+		"config": lib.InstanceConfig.jsonSchema,
+		"status": lib.InstanceStatus,
+		"gamePort": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"factorioVersion": Type.Optional(lib.TargetVersionSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>, location: lib.ConfigLocation) {
+		return new this(
+			lib.InstanceConfig.fromJSON(json.config, location),
+			json.status,
+			json.gamePort,
+			json.factorioVersion,
+			json.updatedAtMs,
+		);
+	}
+
+	toInstanceDetails() {
+		return new lib.InstanceDetails(
+			this.config.get("instance.name"),
+			this.id,
+			this.config.get("instance.assigned_host") ?? undefined,
+			this.gamePort,
+			this.status,
+			this.factorioVersion,
+			this.updatedAtMs,
+			this.config.get("instance.exclude_from_start_all"),
+		);
 	}
 
 	get isDeleted() {
@@ -30,70 +60,8 @@ export default class InstanceRecord {
 		this.status = "deleted";
 	}
 
-	static jsonSchema = Type.Object({
-		"config": lib.InstanceConfig.jsonSchema,
-		"status": lib.InstanceStatus,
-		"gamePort": Type.Optional(Type.Number()),
-		"startedAtMs": Type.Optional(Type.Number()),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"factorioVersion": Type.Optional(lib.TargetVersionSchema),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(
-			lib.InstanceConfig.fromJSON(json.config, "controller"),
-			json.status,
-			json.gamePort,
-			json.factorioVersion,
-			json.startedAtMs,
-			json.updatedAtMs,
-		);
-	}
-
-	toJSON(): Static<typeof InstanceRecord.jsonSchema> {
-		const json: any = {
-			config: this.config.toJSON(),
-			status: this.status,
-		};
-
-		if (this.gamePort !== undefined) {
-			json.gamePort = this.gamePort;
-		}
-		if (this.factorioVersion !== undefined) {
-			json.factorioVersion = this.factorioVersion;
-		}
-		if (this.startedAtMs !== 0) {
-			json.startedAtMs = this.startedAtMs;
-		}
-		if (this.updatedAtMs !== 0) {
-			json.updatedAtMs = this.updatedAtMs;
-		}
-
-		return json;
-	}
-
-	toInstanceDetails() {
-		return new lib.InstanceDetails(
-			this.config.get("instance.name"),
-			this.id,
-			this.config.get("instance.assigned_host") ?? undefined,
-			this.gamePort,
-			this.status,
-			this.factorioVersion,
-			this.startedAtMs,
-			this.updatedAtMs,
-			this.config.get("instance.exclude_from_start_all"),
-		);
-	}
-
-	static fromInstanceDetails(details: lib.InstanceDetails, config: lib.InstanceConfig) {
-		return new InstanceRecord(
-			config,
-			details.status,
-			details.gamePort,
-			details.factorioVersion,
-			details.startedAtMs,
-			details.updatedAtMs,
-		);
+	/** Shorthand for `instance.config.get("instance.id")` */
+	get id():number {
+		return this.config.get("instance.id");
 	}
 }

packages/controller/src/InstanceManager.ts

@@ -1,274 +0,0 @@
-import * as lib from "@clusterio/lib";
-
-import type Controller from "./Controller";
-import type HostConnection from "./HostConnection";
-import InstanceRecord from "./InstanceRecord";
-
-export default class InstanceManager {
-	private static readonly DefaultFactorioSettings = {
-		tags: ["clusterio"],
-		max_players: 0,
-		visibility: { public: true, lan: true },
-		game_password: "",
-		require_user_verification: true,
-		max_upload_in_kilobytes_per_second: 0,
-		max_upload_slots: 5,
-		ignore_player_limit_for_returning_players: false,
-		allow_commands: "admins-only",
-		autosave_interval: 10,
-		autosave_slots: 5,
-		afk_autokick_interval: 0,
-		auto_pause: false,
-		only_admins_can_pause_the_game: true,
-		autosave_only_on_server: true,
-		non_blocking_saving: false,
-	} as const;
-
-	constructor(
-		public readonly records: lib.SubscribableDatastore<InstanceRecord>,
-		private readonly _controller: Controller,
-	) {
-		for (const instance of records.values()) {
-			this.addInstanceHooks(instance);
-		}
-	}
-
-	/* ------------------------------------------------------------------------- */
-	/* Basic Accessors                                                           */
-	/* ------------------------------------------------------------------------- */
-
-	has(id: InstanceRecord["id"]): boolean {
-		return this.records.has(id);
-	}
-
-	getMutable(id: InstanceRecord["id"]): InstanceRecord | undefined {
-		return this.records.get(id);
-	}
-
-	get(id: InstanceRecord["id"]): Readonly<InstanceRecord> | undefined {
-		return this.getMutable(id);
-	}
-
-	valuesMutable(): IterableIterator<InstanceRecord> {
-		return this.records.values();
-	}
-
-	values(): IterableIterator<Readonly<InstanceRecord>> {
-		return this.valuesMutable();
-	}
-
-	/**
-	 * Get instance by ID for a request.
-	 *
-	 * Convenience wrapper around {@link getMutable} that throws
-	 * if the instance does not exist.
-	 *
-	 * @param instanceId - ID of instance to get.
-	 * @returns The instance record.
-	 * @throws {module:lib.RequestError} if the instance does not exist.
-	 */
-	getForRequest(instanceId: number): InstanceRecord {
-		const instance = this.records.get(instanceId);
-		if (!instance) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
-		}
-		return instance;
-	}
-
-	/* ------------------------------------------------------------------------- */
-	/* Internal Helpers                                                          */
-	/* ------------------------------------------------------------------------- */
-
-	private async _notifyStatusChanged(
-		instance: InstanceRecord,
-		prev?: lib.InstanceStatus,
-	): Promise<void> {
-		await lib.invokeHook(
-			this._controller.plugins,
-			"onInstanceStatusChanged",
-			instance,
-			prev,
-		);
-	}
-
-	private async _notifyConfigFieldChanged(
-		instance: InstanceRecord,
-		field: string,
-		curr: unknown,
-		prev: unknown,
-	): Promise<void> {
-		await lib.invokeHook(
-			this._controller.plugins,
-			"onInstanceConfigFieldChanged",
-			instance,
-			field,
-			curr,
-			prev,
-		);
-	}
-
-	/**
-	 * Attach change listeners to an instance record.
-	 *
-	 * Hooks into configuration field changes and ensures the datastore
-	 * is updated and plugins are notified when a config field changes.
-	 *
-	 * @param instance - The instance record to attach hooks to.
-	 * @internal
-	 */
-	addInstanceHooks(instance: InstanceRecord): void {
-		instance.config.on("fieldChanged", (field: string, curr: unknown, prev: unknown) => {
-			this.records.set(instance);
-			this._notifyConfigFieldChanged(instance, field, curr, prev);
-		});
-	}
-
-	/* ------------------------------------------------------------------------- */
-	/* Lifecycle                                                                 */
-	/* ------------------------------------------------------------------------- */
-
-	/**
-	 * Create a new instance.
-	 *
-	 * @example
-	 * const config = new lib.InstanceConfig("controller", { "instance.name": "My instance" });
-	 * const instance = await controller.instances.createInstance(config);
-	 *
-	 * @param instanceConfig - Config to base the newly created instance on.
-	 * @param _suppressChanges - When true, side effects are suppressed, caller takes ownership.
-	 * @returns The created instance record.
-	 * @throws {module:lib.RequestError} if an instance with the same ID already exists.
-	 */
-	async createInstance(
-		instanceConfig: lib.InstanceConfig,
-		_suppressChanges?: boolean,
-	): Promise<InstanceRecord> {
-		const instanceId = instanceConfig.get("instance.id");
-		if (this.records.has(instanceId)) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} already exists`);
-		}
-
-		const controllerName = this._controller.config.get("controller.name");
-		instanceConfig.set("factorio.settings", {
-			name: `${controllerName} - ${instanceConfig.get("instance.name")}`,
-			description: `Clusterio instance for ${controllerName}`,
-			...InstanceManager.DefaultFactorioSettings,
-			...instanceConfig.get("factorio.settings"),
-		});
-
-		const instance = new InstanceRecord(instanceConfig, "unassigned");
-		this.addInstanceHooks(instance);
-
-		if (!_suppressChanges) {
-			this.records.set(instance);
-			await this._notifyStatusChanged(instance);
-		}
-
-		return instance;
-	}
-
-	/**
-	 * Change the assigned host of an instance.
-	 *
-	 * Unassigns the instance from the currently assigned host (if connected)
-	 * and assigns it to the provided host.
-	 *
-	 * This is the only supported way of modifying the
-	 * `instance.assigned_host` configuration entry.
-	 *
-	 * Note:
-	 * - This does NOT transfer any files or saves between hosts.
-	 * - If the instance is already assigned to the specified host, no action is taken.
-	 *
-	 * @param instanceId - ID of instance to assign.
-	 * @param hostId - ID of host to assign the instance to. If undefined,
-	 * the instance will be unassigned.
-	 *
-	 * @throws {module:lib.RequestError} if the instance does not exist.
-	 * @throws {module:lib.RequestError} if the target host is not connected.
-	 */
-	async assignInstance(
-		instanceId: number,
-		hostId: number | null,
-	): Promise<void> {
-		const instance = this.getForRequest(instanceId);
-		const hostConnections = this._controller.wsServer.hostConnections;
-
-		const currentAssignedHost = instance.config.get("instance.assigned_host");
-		if (currentAssignedHost === hostId) {
-			return;
-		}
-
-		let newHostConnection: HostConnection | undefined;
-		if (hostId !== null) {
-			newHostConnection = hostConnections.get(hostId);
-			if (!newHostConnection) {
-				throw new lib.RequestError("Target host is not connected to the controller");
-			}
-		}
-
-		if (currentAssignedHost !== null) {
-			const oldHostConnection = hostConnections.get(currentAssignedHost);
-			if (oldHostConnection && !oldHostConnection.connector.closing) {
-				await oldHostConnection.send(
-					new lib.InstanceUnassignInternalRequest(instanceId),
-				);
-			}
-		}
-
-		this._controller.clearSavesOfInstance(instanceId);
-		instance.config.set("instance.assigned_host", hostId);
-
-		if (newHostConnection) {
-			await newHostConnection.send(
-				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host")),
-			);
-		} else {
-			instance.status = "unassigned";
-		}
-
-		this.records.set(instance);
-	}
-
-	/**
-	 * Unassign an instance from its current host.
-	 *
-	 * Convenience wrapper around {@link assignInstance} that removes
-	 * the assigned host.
-	 *
-	 * @param instanceId - ID of instance to unassign.
-	 */
-	async unassignInstance(instanceId: number): Promise<void> {
-		return this.assignInstance(instanceId, null);
-	}
-
-	/**
-	 * Delete an instance permanently.
-	 *
-	 * Sends a delete request to the assigned host (if any),
-	 * removes the instance from the datastore,
-	 * clears associated saves and user statistics.
-	 *
-	 * @param instanceId - ID of instance to delete.
-	 *
-	 * @throws {module:lib.RequestError} if the instance does not exist.
-	 */
-	async deleteInstance(instanceId: number): Promise<void> {
-		const instance = this.getForRequest(instanceId);
-		const hostId = instance.config.get("instance.assigned_host");
-
-		if (hostId !== null) {
-			await this._controller.sendTo(
-				{ hostId },
-				new lib.InstanceDeleteInternalRequest(instanceId),
-			);
-		}
-
-		const prevStatus = instance.status;
-		this.records.delete(instance);
-		this._controller.clearSavesOfInstance(instanceId);
-		this._controller.users.clearStatsOfInstance(instanceId);
-
-		await this._notifyStatusChanged(instance, prevStatus);
-	}
-}

packages/controller/src/User.ts

@@ -1,146 +0,0 @@
-
-import {
-	IUser, Role, SubscribableDatastore,
-	PermissionError, permissions,
-} from "@clusterio/lib";
-
-import UserRecord from "./UserRecord";
-import { Static } from "@sinclair/typebox";
-
-/**
- * Represents a user as viewed from the controller.
- *
- * All methods are safe to use when readonly.
- * Only use mutable when needing to change multiple values.
- */
-export default class User extends UserRecord implements IUser {
-	constructor(
-		private _controllerUsers: SubscribableDatastore<UserRecord>,
-		private _controllerRoles: SubscribableDatastore<Role>,
-		...args: ConstructorParameters<typeof UserRecord>
-	) {
-		super(...args);
-	}
-
-	static fromJSON(
-		json: Static<typeof this.jsonSchema>,
-		_controllerUsers: SubscribableDatastore<UserRecord>,
-		_controllerRoles: SubscribableDatastore<Role>,
-	) {
-		return this.fromUserRecord(
-			UserRecord.fromJSON(json),
-			_controllerUsers,
-			_controllerRoles,
-		);
-	}
-
-	static fromUserRecord(
-		userRecord: UserRecord,
-		_controllerUsers: SubscribableDatastore<UserRecord>,
-		_controllerRoles: SubscribableDatastore<Role>,
-	): User {
-		return new this(
-			_controllerUsers,
-			_controllerRoles,
-			userRecord.tokenValidAfter,
-			userRecord.name,
-			userRecord.roleIds,
-			userRecord.instances,
-			userRecord.isAdmin,
-			userRecord.isBanned,
-			userRecord.isWhitelisted,
-			userRecord.banReason,
-			userRecord.updatedAtMs,
-			userRecord.isDeleted,
-			userRecord.instanceStats,
-		);
-	}
-
-	/** Roles this user has */
-	get roles(): ReadonlySet<Readonly<Role>> {
-		return new Set([...this.roleIds].map(
-			id => this._controllerRoles.get(id)
-		).filter(
-			(r): r is Role => Boolean(r)
-		));
-	}
-
-	/** Save the underlying user record */
-	saveRecord() {
-		this._controllerUsers.set(this);
-	}
-
-	/** Set a property on the underlying record, then save it. */
-	set<K extends keyof this>(key: K, value: this[K]) {
-		this[key] = value;
-		this.saveRecord();
-	}
-
-	/**
-	 * Invalidate current tokens for the user
-	 *
-	 * Sets the tokenValidAfter property to the current time, which causes
-	 * all currently issued tokens for the user to become invalid.
-	 */
-	invalidateToken() {
-		this.tokenValidAfter = Math.floor(Date.now() / 1000);
-		this.saveRecord();
-	}
-
-	/** Add a role to this user */
-	addRole(roleId: number) {
-		const role = this._controllerRoles.get(roleId);
-		if (!role) {
-			throw new Error(`role ${roleId} does not exist`);
-		}
-		this.roleIds.add(roleId);
-		this.saveRecord();
-	}
-
-	/** Remove a role from user */
-	removeRole(roleId: number) {
-		if (!this.roleIds.delete(roleId)) {
-			return false;
-		}
-		this.saveRecord();
-		return true;
-	}
-
-	/** Set a user as being online on a specific instance */
-	notifyJoin(instanceId: number) {
-		this.instances.add(instanceId);
-		this.saveRecord();
-	}
-
-	/** Set a user as being offline on a specific instance */
-	notifyLeave(instanceId: number) {
-		this.instances.delete(instanceId);
-		this.saveRecord();
-	}
-
-	/** Clear the stats of a player on a specific instance */
-	clearInstanceStats(instanceId: number) {
-		this.instances.delete(instanceId);
-		this.instanceStats.delete(instanceId);
-		this.recalculatePlayerStats();
-		this.saveRecord();
-	}
-
-	checkPermission(permission: string) {
-		if (!permissions.has(permission)) {
-			throw new Error(`permission ${permission} does not exist`);
-		}
-
-		for (const roleId of this.roleIds) {
-			const role = this._controllerRoles.get(roleId);
-			if (!role) {
-				continue;
-			}
-			if (role.permissions.has("core.admin") || role.permissions.has(permission)) {
-				return;
-			}
-		}
-
-		throw new PermissionError("Permission denied");
-	}
-}

packages/controller/src/UserManager.ts

@@ -3,45 +3,65 @@ import jwt from "jsonwebtoken";
 
 import * as lib from "@clusterio/lib";
 
-import User from "./User";
-import UserRecord from "./UserRecord";
+import ControllerUser from "./ControllerUser";
 
 /**
- * Access layer for users on the controller.
- *
- * In performance critical paths the raw records can be accessed directly.
- * Otherwise this class will wrap data in a view class with a set and save method.
- *
+ * Manages users and roles
  * @alias module:controller/src/UserManager
  */
 export default class UserManager {
+	users: Map<ControllerUser["id"], ControllerUser> = new Map();
+	dirty = false;
+
+	/**
+	 * Set of users currently online in the cluster.
+	 */
+	onlineUsers = new Set<ControllerUser>();
+
 	constructor(
-		public records: lib.SubscribableDatastore<UserRecord>,
-		private _controllerRoles: lib.SubscribableDatastore<lib.Role>,
-		private _controllerConfig: lib.ControllerConfig,
+		private _config: lib.ControllerConfig,
+		private _roles: lib.SubscribableDatastore<lib.Role>,
 	) {
 	}
 
-	/**
-	 * Prior to 2.0.0-alpha.22 users and roles were stored in a single file.
-	 * This migration separates them so they can be loaded by datastore.
-	 *
-	 * @param usersFilePath The src and dst file path for the users json file
-	 * @param rolesFilePath The dst for the roles json file
-	 */
-	static async attemptMigrateLegacyUsersFile(usersFilePath: string, rolesFilePath: string) {
+	getByName(name: string) {
+		return this.users.get(name.toLowerCase());
+	}
+
+	async load(filePath: string): Promise<void> {
 		try {
-			const combinedJson = JSON.parse(await fs.readFile(usersFilePath, { encoding: "utf8" }));
-			if (!("roles" in combinedJson)) {
-				return; // Files are already split, no action required
+			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
+			if ("roles" in content) { // Split file format after 2.0.0-alpha.22
+				this._roles.setMany(content.roles.map((serializedRole: any) => lib.Role.fromJSON(serializedRole)));
+				await this._roles.save(); // Force a save to prevent data loss
+				content = content.users;
 			}
-			if (!("users" in combinedJson)) {
-				throw new Error("Legacy users json does not contain users property");
+
+			let duplicates = 0;
+			for (const serializedUser of content) {
+				const user = ControllerUser.fromJSON(serializedUser, this._roles);
+				const existingUser = this.users.get(user.id);
+				if (existingUser) {
+					// Required migration to all lowercase ids in alpha 19
+					duplicates += 1;
+					// We assume the user with the lower online time is the duplicate
+					if (user.playerStats.onlineTimeMs <= existingUser.playerStats.onlineTimeMs) {
+						existingUser.merge(user);
+						continue; // Skip users.set
+					}
+					user.merge(existingUser);
+				}
+				this.users.set(user.id, user);
+			}
+
+			if (duplicates) {
+				const backupPath = `${filePath}.${Date.now()}.bak`;
+				lib.logger.warn(
+					`A total of ${duplicates} users were merged, a backup was written to: ${backupPath}`
+				);
+				await lib.safeOutputFile(backupPath, JSON.stringify(content, null, "\t"));
 			}
 
-			// We intentionally write the roles first because if it fails then the old user file will still contain them
-			await lib.safeOutputFile(rolesFilePath, JSON.stringify(combinedJson.roles));
-			await lib.safeOutputFile(usersFilePath, JSON.stringify(combinedJson.users));
 		} catch (err: any) {
 			if (err.code !== "ENOENT") {
 				throw err;
@@ -49,116 +69,66 @@ export default class UserManager {
 		}
 	}
 
-	/* ------------------------------------------------------------------------- */
-	/* Basic Accessors                                                           */
-	/* ------------------------------------------------------------------------- */
-
-	has(id: UserRecord["id"]) {
-		return this.records.has(id);
-	}
-
-	getMutable(id: UserRecord["id"]) {
-		const userRecord = this.records.getMutable(id);
-		if (!userRecord) {
-			return undefined;
-		}
-
-		return User.fromUserRecord(
-			userRecord,
-			this.records,
-			this._controllerRoles,
+	async save(filePath: string): Promise<void> {
+		this.dirty = false;
+		await lib.safeOutputFile(filePath,
+			JSON.stringify([...this.users.values()].map(user => user.toJSON(true)), null, "\t")
 		);
 	}
 
-	get(id: UserRecord["id"]): Readonly<User> | undefined {
-		return this.getMutable(id);
-	}
-
-	hasName(name: UserRecord["name"]) {
-		return this.has(name.toLowerCase());
-	}
-
-	getByNameMutable(name: UserRecord["name"]) {
-		return this.getMutable(name.toLowerCase());
-	}
-
-	getByName(name: UserRecord["name"]) {
-		return this.get(name.toLowerCase());
-	}
-
-	* valuesMutable(): IterableIterator<User> {
-		const roles = this._controllerRoles;
-		const users = this.records;
-
-		for (const record of users.values()) {
-			yield User.fromUserRecord(record, users, roles);
-		}
-	}
-
-	values(): IterableIterator<Readonly<User>> {
-		return this.valuesMutable();
-	}
-
-	/* ------------------------------------------------------------------------- */
-	/* Lifecycle                                                                 */
-	/* ------------------------------------------------------------------------- */
-
 	/**
 	 * Creates a new user and add it to the user database.
 	 * @param name - Name of the user to create.
 	 * @returns The created user.
 	 */
-	createUser(name: string): User {
+	createUser(name: string): ControllerUser {
 		if (this.getByName(name)) {
 			throw new Error(`User '${name}' already exists`);
 		}
 
 		let roles = new Set<number>();
-		let defaultRoleId = this._controllerConfig.get("controller.default_role_id");
-		if (defaultRoleId !== null && this._controllerRoles.has(defaultRoleId)) {
+		let defaultRoleId = this._config.get("controller.default_role_id");
+		if (defaultRoleId !== null && this._roles.has(defaultRoleId)) {
 			roles.add(defaultRoleId);
 		}
 
-		const user = new User(this.records, this._controllerRoles, 0, name, roles);
-		this.records.set(user);
+		let user = new ControllerUser(this._roles, 0, name, roles);
+		this.users.set(user.id, user);
+		this.dirty = true;
 		return user;
 	}
 
-	/** Get or create a user with a given name. */
-	getOrCreateUser(name: string): User {
-		return this.getByNameMutable(name) ?? this.createUser(name);
-	}
-
-	/**
-	 * Deletes an existing user
-	 *
-	 * @param user - user to delete
-	 */
-	deleteUser(user: UserRecord) {
-		return this.records.delete(user);
-	}
-
-	/* ------------------------------------------------------------------------- */
-	/* Helpers                                                                   */
-	/* ------------------------------------------------------------------------- */
-
 	/**
 	 * Sign access token for the given user
 	 *
 	 * @param user - user to sign token for
 	 * @returns JWT access token for the user.
 	 */
-	signUserToken(user: UserRecord): string {
+	signUserToken(user: ControllerUser): string {
 		return jwt.sign(
 			{ aud: "user", user: user.id },
-			Buffer.from(this._controllerConfig.get("controller.auth_secret"), "base64")
+			Buffer.from(this._config.get("controller.auth_secret"), "base64")
 		);
 	}
 
-	/** Clear all player stats for specific instance */
-	clearStatsOfInstance(instanceId: number) {
-		for (const user of this.values()) {
-			user.clearInstanceStats(instanceId);
+	notifyJoin(user: ControllerUser, instance_id: number) {
+		user.instances.add(instance_id);
+		this.onlineUsers.add(user);
+	}
+
+	notifyLeave(user: ControllerUser, instance_id: number) {
+		user.instances.delete(instance_id);
+		if (!user.instances.size) {
+			this.onlineUsers.delete(user);
 		}
 	}
+
+	clearStatsOfInstance(instanceId: number) {
+		for (const user of this.users.values()) {
+			this.notifyLeave(user, instanceId);
+			user.instanceStats.delete(instanceId);
+			user.recalculatePlayerStats();
+		}
+		this.dirty = true;
+	}
 }

packages/controller/src/UserRecord.ts

@@ -1,102 +0,0 @@
-import { UserDetails } from "@clusterio/lib";
-import { Type, Static } from "@sinclair/typebox";
-
-/** Underlying data class for users on the controller */
-export default class UserRecord extends UserDetails {
-	constructor(
-		/** Unix time in seconds the user token must be issued after to be valid.  */
-		public tokenValidAfter = 0,
-		...args: ConstructorParameters<typeof UserDetails>
-	) {
-		super(...args);
-	}
-
-	static jsonSchema = Type.Object({
-		token_valid_after: Type.Optional(Type.Number()),
-		...UserDetails.jsonSchema.properties,
-	});
-
-	// _a and _b are a ts workaround for allowing User to have extra arguments
-	static fromJSON(json: Static<typeof this.jsonSchema>, _a?: any, _b?: any) {
-		return this.fromUserDetails(
-			UserDetails.fromJSON(json),
-			json.token_valid_after,
-		);
-	}
-
-	toJSON() {
-		const json = super.toJSON() as Static<typeof UserRecord.jsonSchema>;
-
-		if (this.tokenValidAfter > 0) {
-			json.token_valid_after = this.tokenValidAfter;
-		}
-
-		return json;
-	}
-
-	static fromUserDetails(
-		userDetails: UserDetails,
-		tokenValidAfter = 0,
-	) {
-		return new this(
-			tokenValidAfter,
-			userDetails.name,
-			userDetails.roleIds,
-			userDetails.instances,
-			userDetails.isAdmin,
-			userDetails.isBanned,
-			userDetails.isWhitelisted,
-			userDetails.banReason,
-			userDetails.updatedAtMs,
-			userDetails.isDeleted,
-			userDetails.instanceStats,
-		);
-	}
-
-	toUserDetails() {
-		return new UserDetails(
-			this.name,
-			this.roleIds,
-			this.instances,
-			this.isAdmin,
-			this.isBanned,
-			this.isWhitelisted,
-			this.banReason,
-			this.updatedAtMs,
-			this.isDeleted,
-			this.instanceStats,
-		);
-	}
-
-	/**
-	 * Merge another user into this one, must have matching ids, user is not deleted
-	 * @param otherUser - User who's details are merged from
-	*/
-	merge(otherUser: UserDetails) {
-		if (this.id !== otherUser.id) {
-			throw new Error("Cannot merge users with different ids");
-		}
-
-		// Merge properties
-		this.roleIds = new Set([...this.roleIds, ...otherUser.roleIds]);
-		this.instances = new Set([...this.instances, ...otherUser.instances]);
-		this.isAdmin = this.isAdmin && otherUser.isAdmin; // More secure to use && rather ||
-		this.isBanned = this.isBanned || otherUser.isBanned;
-		this.isWhitelisted = this.isWhitelisted || otherUser.isWhitelisted;
-		this.banReason = this.updatedAtMs > otherUser.updatedAtMs ? this.banReason : otherUser.banReason;
-		this.updatedAtMs = Date.now();
-
-		// Merge instance stats
-		const thisInstanceStats = this.instanceStats;
-		for (const [instanceId, instanceStats] of otherUser.instanceStats.entries()) {
-			if (thisInstanceStats.has(instanceId)) {
-				thisInstanceStats.get(instanceId)!.merge(instanceStats);
-			} else {
-				thisInstanceStats.set(instanceId, instanceStats);
-			}
-		}
-
-		// Player stats must be recalculated after modifying instance stats
-		this.recalculatePlayerStats();
-	}
-}

packages/controller/src/WsServer.ts

@@ -334,7 +334,7 @@ ${err.stack}`
 				throw new Error("unexpected JsonWebToken type");
 			}
 
-			user = this.controller.users.getByNameMutable(tokenPayload.user);
+			user = this.controller.userManager.getByName(tokenPayload.user);
 			if (!user) {
 				throw new Error("invalid user");
 			}

packages/controller/src/routes.ts

@@ -189,7 +189,7 @@ function validateUserToken(req: Request, res: Response, next: any) {
 		if (typeof tokenPayload === "string") {
 			throw new Error("unexpected JsonWebToken type");
 		}
-		let user = req.app.locals.controller.users.getByName(tokenPayload.user);
+		let user = req.app.locals.controller.userManager.getByName(tokenPayload.user);
 		if (!user) {
 			throw new Error("invalid user");
 		}
@@ -240,9 +240,9 @@ async function uploadExport(req: Request, res: Response) {
 	let exportFiles = [
 		"export/settings.json",
 		"export/prototypes.json",
+		"export/item-spritesheet.png",
+		"export/item-metadata.json",
 		"export/locale.json",
-		"export/spritesheet.png",
-		"export/metadata.json",
 	];
 
 	let assets: any = {};
@@ -263,7 +263,7 @@ async function uploadExport(req: Request, res: Response) {
 		await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
 	}
 
-	modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString());
+	modPack.exportManifest = new lib.ExportManifest(assets);
 	modPack.fillDefaultSettings(settingPrototypes, logger);
 	res.app.locals.controller.modPacks.set(modPack);
 
@@ -701,6 +701,5 @@ export const webRoutes = [
 	"/roles/:id/view",
 	"/plugins",
 	"/plugins/:name/view",
-	"/icon-reference",
 	"/error",
 ];

packages/create/create.js

@@ -335,7 +335,7 @@ async function migrateRename(args) {
 		let mode = "standalone";
 		if (!hasRunSlave) { mode = "controller"; }
 		if (!hasRunMaster) { mode = "host"; }
-		await writeScripts(mode, "");
+		await writeScripts(mode);
 	}
 
 	logger.info(
@@ -420,7 +420,7 @@ async function groupIdToName(gid) {
 	}
 }
 
-async function writeScripts(mode, selfSignedCert) {
+async function writeScripts(mode) {
 	if (["standalone", "controller"].includes(mode)) {
 		if (process.platform === "win32") {
 			await safeOutputFile(
@@ -474,13 +474,12 @@ WantedBy=multi-user.target
 
 	if (["standalone", "host"].includes(mode)) {
 		if (process.platform === "win32") {
-			let extraCaCertsLine = selfSignedCert ? `set "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
 			await safeOutputFile(
 				"run-host.cmd",
 				`\
 @echo off
 set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%"
-${extraCaCertsLine}:restart
+:restart
 call .\\node_modules\\.bin\\clusteriohost.cmd run --can-restart
 if %errorlevel% equ 0 exit /b
 if %errorlevel% equ 8 exit /b
@@ -488,20 +487,18 @@ goto restart
 `
 			);
 		} else {
-			let extraCaCertsLine = selfSignedCert ? `export "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
 			await safeOutputFile(
 				"run-host.sh",
 				`\
 #!/bin/bash
 export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS"
-${extraCaCertsLine}while true; do
+while true; do
 	./node_modules/.bin/clusteriohost run --can-restart
 	if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi
 done
 `,
 				{ mode: 0o755 },
 			);
-			extraCaCertsLine = selfSignedCert ? `Environment=NODE_EXTRA_CA_CERTS=${selfSignedCert}\n` : "";
 			await safeOutputFile(
 				"systemd/clusteriohost.service",
 				`\
@@ -515,7 +512,7 @@ WorkingDirectory=${process.cwd()}
 KillMode=mixed
 KillSignal=SIGINT
 Environment=NODE_OPTIONS=--enable-source-maps
-${extraCaCertsLine}ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart
+ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart
 Restart=on-failure
 RestartPreventExitStatus=8
 
@@ -842,9 +839,6 @@ async function main() {
 		.option("factorio-dir", {
 			nargs: 1, describe: "Path to Factorio installation [standalone/host]", type: "string",
 		})
-		.option("self-signed-cert", {
-			nargs: 1, describe: "Path to self-signed certificate of the controller [standalone/host]", type: "string",
-		})
 		.option("remote-npm", {
 			nargs: 0, description: "Allow remote updates via npm [standalone/controller/host]", type: "boolean",
 		})
@@ -953,7 +947,7 @@ async function main() {
 
 	if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) {
 		logger.info("Writing run scripts");
-		await writeScripts(answers.mode, args.selfSignedCert);
+		await writeScripts(answers.mode);
 	}
 
 	if (answers.mode === "ctl") {

packages/create/package.json

@@ -3,7 +3,7 @@
 	"description": "Installer for clusterio",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.23",
+	"version": "2.0.0-alpha.22b",
 	"main": "create.js",
 	"bin": {
 		"clusteriocreate": "./create.js"

packages/create/templates/plugin-ts/controller.ts

@@ -1,5 +1,5 @@
 import * as lib from "@clusterio/lib";
-import { BaseControllerPlugin, InstanceRecord } from "@clusterio/controller";
+import { BaseControllerPlugin, InstanceInfo } from "@clusterio/controller";
 //%if multi_context // Messages requires multi context
 
 import {
@@ -37,7 +37,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	}
 //%if instance
 
-	async onInstanceConfigFieldChanged(instance: InstanceRecord, field: string, curr: unknown, prev: unknown) {
+	async onInstanceConfigFieldChanged(instance: InstanceInfo, field: string, curr: unknown, prev: unknown) {
 		this.logger.info(`controller::onInstanceConfigFieldChanged ${instance.id} ${field}`);
 	}
 //%endif
@@ -50,7 +50,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.logger.info("controller::onShutdown");
 	}
 
-	async onPlayerEvent(instance: InstanceRecord, event: lib.PlayerEvent) {
+	async onPlayerEvent(instance: InstanceInfo, event: lib.PlayerEvent) {
 		this.logger.info(`controller::onPlayerEvent ${instance.id} ${JSON.stringify(event)}`);
 	}
 //%if multi_context

packages/ctl/ctl.ts

@@ -27,8 +27,8 @@ import BaseCtlPlugin from "./src/BaseCtlPlugin";
 class ControlConnector extends lib.WebSocketClientConnector {
 	private _token: string;
 
-	constructor(url: string, maxReconnectDelay: number, token: string) {
-		super(url, maxReconnectDelay);
+	constructor(url: string, maxReconnectDelay: number, tlsCa: string | undefined, token: string) {
+		super(url, maxReconnectDelay, tlsCa);
 		this._token = token;
 	}
 
@@ -54,6 +54,8 @@ class ControlConnector extends lib.WebSocketClientConnector {
 export class Control extends lib.Link {
 	/** Control config used for connecting to the controller. */
 	config: lib.ControlConfig;
+	/** Certificate authority used to validate TLS connections to the controller. */
+	tlsCa?: string;
 	/** Mapping of plugin names to their instance for loaded plugins. */
 	plugins: Map<string, BaseCtlPlugin>;
 	/** Keep the control connection alive after the command completes. */
@@ -62,10 +64,12 @@ export class Control extends lib.Link {
 	constructor(
 		connector: ControlConnector,
 		controlConfig: lib.ControlConfig,
+		tlsCa: string | undefined,
 		ctlPlugins: Map<string, BaseCtlPlugin>
 	) {
 		super(connector);
 		this.config = controlConfig;
+		this.tlsCa = tlsCa;
 		this.plugins = ctlPlugins;
 
 		this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this));
@@ -295,13 +299,20 @@ async function startControl() {
 		return;
 	}
 
+	let tlsCa: string | undefined;
+	let tlsCaPath = controlConfig.get("control.tls_ca");
+	if (tlsCaPath) {
+		tlsCa = await fs.readFile(tlsCaPath, "utf8");
+	}
+
 	let controlConnector = new ControlConnector(
 		controlConfig.get("control.controller_url")!,
 		controlConfig.get("control.max_reconnect_delay"),
+		tlsCa,
 		controlConfig.get("control.controller_token")!,
 	);
 
-	let control = new Control(controlConnector, controlConfig, ctlPlugins);
+	let control = new Control(controlConnector, controlConfig, tlsCa, ctlPlugins);
 	try {
 		await controlConnector.connect();
 	} catch (err) {

packages/ctl/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio command line management tool",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.23",
+	"version": "2.0.0-alpha.22",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -24,6 +24,7 @@
 		"set-blocking": "catalog:",
 		"fs-extra": "catalog:",
 		"winston": "catalog:",
+		"phin": "catalog:",
 		"as-table": "^1.0.55",
 		"yargs": "^17.7.2"
 	},

packages/ctl/src/command_ops.ts

@@ -1,4 +0,0 @@
-export function print(...content: any) {
-	// eslint-disable-next-line no-console
-	console.log(...content);
-}

packages/ctl/src/commands_controller.ts

@@ -1,201 +0,0 @@
-import fs from "fs-extra";
-import asTableModule from "as-table";
-import events from "events";
-import os from "os";
-import child_process from "child_process";
-
-import * as lib from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-
-export const controllerCommands = new lib.CommandTree({ name: "controller", description: "Controller management" });
-controllerCommands.add(new lib.Command({
-	definition: ["stop", "Stop controller"],
-	handler: async function(_args: object, control: Control) {
-		await control.send(new lib.ControllerStopRequest());
-	},
-}));
-
-controllerCommands.add(new lib.Command({
-	definition: ["restart", "Restart controller"],
-	handler: async function(_args: object, control: Control) {
-		await control.send(new lib.ControllerRestartRequest());
-	},
-}));
-controllerCommands.add(new lib.Command({
-	definition: ["update", "Update the controller", (yargs) => {
-		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
-	}],
-	handler: async function(args: { restart: boolean }, control: Control) {
-		await control.send(new lib.ControllerUpdateRequest());
-		if (args.restart) {
-			await control.send(new lib.ControllerRestartRequest());
-		} else {
-			print("Controller updated; a restart is required to apply the changes.");
-		}
-	},
-}));
-
-const controllerConfigCommands = new lib.CommandTree({
-	name: "config", alias: ["c"], description: "controller config management",
-});
-
-controllerConfigCommands.add(new lib.Command({
-	definition: ["list", "List controller configuration"],
-	handler: async function(args: object, control: Control) {
-		let config = await control.send(new lib.ControllerConfigGetRequest());
-
-		for (let [name, value] of Object.entries(config)) {
-			print(`${name} ${JSON.stringify(value)}`);
-		}
-	},
-}));
-
-controllerConfigCommands.add(new lib.Command({
-	definition: ["set <field> [value]", "Set field in controller config", (yargs) => {
-		yargs.positional("field", { describe: "Field to set", type: "string" });
-		yargs.positional("value", { describe: "Value to set", type: "string" });
-		yargs.options({
-			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
-		});
-	}],
-	handler: async function(args: { field: string, value?: string, stdin?: boolean }, control: Control) {
-		if (args.stdin) {
-			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
-		} else if (args.value === undefined) {
-			args.value = "";
-		}
-		await control.send(new lib.ControllerConfigSetFieldRequest(args.field, args.value));
-	},
-}));
-
-controllerConfigCommands.add(new lib.Command({
-	definition: ["set-prop <field> <prop> [value]", "Set property of field in controller config", (yargs) => {
-		yargs.positional("field", { describe: "Field to set", type: "string" });
-		yargs.positional("prop", { describe: "Property to set", type: "string" });
-		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
-		yargs.options({
-			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
-		});
-	}],
-	handler: async function(args: { field: string, prop: string, value?: string, stdin?: boolean }, control: Control) {
-		if (args.stdin) {
-			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
-		}
-		let value;
-		try {
-			if (args.value !== undefined) {
-				value = JSON.parse(args.value);
-			}
-		} catch (err: any) {
-			// See note for the instance version of set-prop
-			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
-				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
-			}
-			value = args.value;
-		}
-		await control.send(new lib.ControllerConfigSetPropRequest(args.field, args.prop, value));
-	},
-}));
-controllerConfigCommands.add(new lib.Command({
-	definition: ["edit [editor]", "Edit controller configuration", (yargs) => {
-		yargs.positional("editor", {
-			describe: "Editor to use",
-			type: "string",
-			default: "",
-		});
-	}],
-	handler: async function(args: { editor: string }, control: Control) {
-		let config = await control.send(new lib.ControllerConfigGetRequest());
-		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
-		let editor = await getEditor(args.editor);
-		if (editor === undefined) {
-			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
-							  Try "ctl controller config edit <editor of choice>"`);
-		}
-		let allConfigElements = serializedConfigToString(
-			config, lib.ControllerConfig, {}
-		);
-		await fs.writeFile(tmpFile, allConfigElements);
-		let editorSpawn = child_process.spawn(editor, [tmpFile], {
-			stdio: "inherit",
-			detached: false,
-		});
-		editorSpawn.on("data", (data) => {
-			process.stdout.pipe(data);
-		});
-		await events.once(editorSpawn, "exit");
-		const data = await fs.readFile(tmpFile, "utf8");
-		const final = await configToKeyVal(data);
-		for (let index in final) {
-			if (index in final) {
-				try {
-					await control.send(new lib.ControllerConfigSetFieldRequest(
-						index,
-						final[index],
-					));
-				} catch (err) {
-					// eslint-disable-next-line
-					print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-					print(err);
-					// If the string is empty, it's better to just print "" instead of nothing
-				}
-			}
-		}
-		try {
-			await fs.unlink(tmpFile);
-		} catch (err) {
-			print("err: temporary file", tmpFile, "could not be deleted.");
-			print("This is not fatal, but they may build up over time if the issue persists.");
-		}
-	},
-}));
-
-controllerCommands.add(controllerConfigCommands);
-
-
-const controllerPluginCommands = new lib.CommandTree({
-	name: "plugin", alias: ["p"], description: "controller plugin inspection",
-});
-controllerPluginCommands.add(new lib.Command({
-	definition: ["list", "List plugins on controller"],
-	handler: async function(args: object, control: Control) {
-		let url = new URL(control.config.get("control.controller_url")!);
-		url.pathname += "api/plugins";
-		let response = await fetch(url);
-		print(asTable(await response.json() as []));
-	},
-}));
-controllerPluginCommands.add(new lib.Command({
-	definition: ["update <plugin>", "Update a plugin on the controller", (yargs) => {
-		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
-		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
-	}],
-	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
-		await control.sendTo("controller", new lib.PluginUpdateRequest(args.plugin));
-		if (args.restart) {
-			await control.send(new lib.ControllerRestartRequest());
-		} else {
-			print("Plugin updated; a restart is required to apply the changes.");
-		}
-	},
-}));
-controllerPluginCommands.add(new lib.Command({
-	definition: ["install <plugin>", "Install a plugin on the controller", (yargs) => {
-		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
-		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
-	}],
-	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
-		await control.sendTo("controller", new lib.PluginInstallRequest(args.plugin));
-		if (args.restart) {
-			await control.send(new lib.ControllerRestartRequest());
-		} else {
-			print("Plugin installed; a restart is required to apply the changes.");
-		}
-	},
-}));
-
-controllerCommands.add(controllerPluginCommands);

packages/ctl/src/commands_debug.ts

@@ -1,11 +0,0 @@
-import * as lib from "@clusterio/lib";
-import type { Control } from "../ctl";
-
-export const debugCommands = new lib.CommandTree({ name: "debug", description: "Debugging utilities" });
-debugCommands.add(new lib.Command({
-	definition: ["dump-ws", "Dump WebSocket messages sent and received by controller"],
-	handler: async function(args: object, control: Control) {
-		await control.send(new lib.DebugDumpWsRequest());
-		control.keepOpen = true;
-	},
-}));

packages/ctl/src/commands_host.ts

@@ -1,305 +0,0 @@
-import fs from "fs-extra";
-import asTableModule from "as-table";
-import events from "events";
-import os from "os";
-import child_process from "child_process";
-
-import * as lib from "@clusterio/lib";
-import { logger } from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-
-export const hostCommands = new lib.CommandTree({ name: "host", description: "Host management" });
-hostCommands.add(new lib.Command({
-	definition: ["stop <host>", "Stop the given host", (yargs) => {
-		yargs.positional("host", { describe: "Host to stop", type: "string" });
-	}],
-	handler: async function(args: { host: string }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		await control.sendTo({ hostId }, new lib.HostStopRequest());
-	},
-}));
-
-hostCommands.add(new lib.Command({
-	definition: ["restart <host>", "Restart the given host", (yargs) => {
-		yargs.positional("host", { describe: "Host to restart", type: "string" });
-	}],
-	handler: async function(args: { host: string }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		await control.sendTo({ hostId }, new lib.HostRestartRequest());
-	},
-}));
-
-hostCommands.add(new lib.Command({
-	definition: [["list", "l"], "List hosts connected to the controller"],
-	handler: async function(args: object, control: Control) {
-		let hosts = await control.send(new lib.HostListRequest());
-		print(asTable(hosts));
-	},
-}));
-
-hostCommands.add(new lib.Command({
-	definition: ["generate-token", "Generate token for a host", (yargs) => {
-		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
-	}],
-	handler: async function(args: { id?: number }, control: Control) {
-		let hostId = typeof args.id === "number" ? args.id : undefined;
-		let response = await control.send(new lib.HostGenerateTokenRequest(hostId));
-		print(response);
-	},
-}));
-
-hostCommands.add(new lib.Command({
-	definition: ["revoke-token <host>", "Revoke all tokens for a host", (yargs) => {
-		yargs.positional("host", { describe: "Host to revoke tokens for", type: "string" });
-	}],
-	handler: async function(args: { host: string }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		await control.send(new lib.HostRevokeTokensRequest(hostId));
-	},
-}));
-
-hostCommands.add(new lib.Command({
-	definition: ["create-config", "Create host config", (yargs) => {
-		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
-		yargs.option("name", { type: "string", nargs: 1, describe: "Host name" });
-		yargs.option("generate-token", {
-			type: "boolean", nargs: 0, describe: "Generate authentication token", default: false,
-		});
-		yargs.option("output", {
-			type: "string", nargs: 1, describe: "Path to output config (- for stdout)", default: "config-host.json",
-		});
-	}],
-	handler: async function(
-		args: { id?: number, name?: string, generateToken: boolean, output: string },
-		control: Control
-	) {
-		let config = await control.send(
-			new lib.HostConfigCreateRequest(args.id, args.name, args.generateToken)
-		);
-
-		let content = JSON.stringify(config, null, "\t");
-		if (args.output === "-") {
-			print(content);
-		} else {
-			logger.info(`Writing ${args.output}`);
-			try {
-				await fs.outputFile(args.output, content, { flag: "wx" });
-			} catch (err: any) {
-				if (err.code === "EEXIST") {
-					throw new lib.CommandError(`File ${args.output} already exists`);
-				}
-				throw err;
-			}
-		}
-	},
-}));
-hostCommands.add(new lib.Command({
-	definition: ["update <host>", "Update the host", (yargs) => {
-		yargs.positional("host", { describe: "Host to update", type: "string" });
-		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
-	}],
-	handler: async function(args: { host: string, restart: boolean }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		await control.sendTo({ hostId }, new lib.HostUpdateRequest());
-		if (args.restart) {
-			await control.sendTo({ hostId }, new lib.HostRestartRequest());
-		} else {
-			print("Host updated; a restart is required to apply the changes.");
-		}
-	},
-}));
-
-
-const hostConfigCommands = new lib.CommandTree({
-	name: "config", alias: ["c"], description: "Host config management",
-});
-hostConfigCommands.add(new lib.Command({
-	definition: ["list <host>", "List configuration for a host", (yargs) => {
-		yargs.positional("host", { describe: "Host to list config for", type: "string" });
-	}],
-	handler: async function(args: { host: string }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		let config = await control.sendTo({ hostId }, new lib.HostConfigGetRequest());
-
-		for (let [name, value] of Object.entries(config)) {
-			print(`${name} ${JSON.stringify(value)}`);
-		}
-	},
-}));
-
-hostConfigCommands.add(new lib.Command({
-	definition: ["set <host> <field> [value]", "Set field in host config", (yargs) => {
-		yargs.positional("host", { describe: "Host to set config on", type: "string" });
-		yargs.positional("field", { describe: "Field to set", type: "string" });
-		yargs.positional("value", { describe: "Value to set", type: "string" });
-		yargs.options({
-			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
-		});
-	}],
-	handler: async function(
-		args: { host: string, field: string, value?: string, stdin?: boolean },
-		control: Control,
-	) {
-		let hostId = await lib.resolveHost(control, args.host);
-		if (args.stdin) {
-			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
-		} else if (args.value === undefined) {
-			args.value = "";
-		}
-		await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(args.field, args.value));
-	},
-}));
-
-hostConfigCommands.add(new lib.Command({
-	definition: ["set-prop <host> <field> <prop> [value]", "Set property of field in host config", (yargs) => {
-		yargs.positional("host", { describe: "Host to set config on", type: "string" });
-		yargs.positional("field", { describe: "Field to set", type: "string" });
-		yargs.positional("prop", { describe: "Property to set", type: "string" });
-		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
-		yargs.options({
-			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
-		});
-	}],
-	handler: async function(
-		args: { host: string, field: string, prop: string, value?: string, stdin?: boolean },
-		control: Control
-	) {
-		let hostId = await lib.resolveHost(control, args.host);
-		if (args.stdin) {
-			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
-		}
-		let value;
-		try {
-			if (args.value !== undefined) {
-				value = JSON.parse(args.value);
-			}
-		} catch (err: any) {
-			// See note for the instance version of set-prop
-			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
-				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
-			}
-			value = args.value;
-		}
-		await control.sendTo({ hostId }, new lib.HostConfigSetPropRequest(args.field, args.prop, value));
-	},
-}));
-
-hostConfigCommands.add(new lib.Command({
-	definition: ["edit <host> [editor]", "Edit host configuration", (yargs) => {
-		yargs.positional("host", { describe: "Host to set config on", type: "string" });
-		yargs.positional("editor", {
-			describe: "Editor to use",
-			type: "string",
-			default: "",
-		});
-	}],
-	handler: async function(args: { host: string, editor: string }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		let config = await control.sendTo({ hostId }, new lib.HostConfigGetRequest());
-		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
-		let editor = await getEditor(args.editor);
-		if (editor === undefined) {
-			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
-							  Try "ctl controller config edit <editor of choice>"`);
-		}
-		let disallowedList = {"host.id": 0};
-		let allConfigElements = serializedConfigToString(
-			config,
-			lib.HostConfig,
-			disallowedList
-		);
-		await fs.writeFile(tmpFile, allConfigElements);
-		let editorSpawn = child_process.spawn(editor, [tmpFile], {
-			stdio: "inherit",
-			detached: false,
-		});
-		editorSpawn.on("data", (data) => {
-			process.stdout.pipe(data);
-		});
-		await events.once(editorSpawn, "exit");
-		const data = await fs.readFile(tmpFile, "utf8");
-		const final = await configToKeyVal(data);
-		for (let index in final) {
-			if (index in final) {
-				try {
-					await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(
-						index,
-						final[index],
-					));
-				} catch (err) {
-					// eslint-disable-next-line
-					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-					// If the string is empty, it's better to just print "" instead of nothing
-					print("This message shouldn't normally appear; if the below message does not indicate it");
-					print("was a user mistake, please report it to the clustorio devs.");
-					// added this because it could be a missed entry in disallowedList
-					// i've added all the vanilla clustorio configs, but there may be
-					// some modded ones that need to be added.
-					print(err);
-				}
-			}
-		}
-		try {
-			await fs.unlink(tmpFile);
-		} catch (err) {
-			print("err: temporary file", tmpFile, "could not be deleted.");
-			print("This is not fatal, but they may build up over time if the issue persists.");
-		}
-	},
-}));
-
-hostCommands.add(hostConfigCommands);
-
-
-const hostPluginCommands = new lib.CommandTree({
-	name: "plugin", alias: ["p"], description: "host plugin inspection",
-});
-hostPluginCommands.add(new lib.Command({
-	definition: ["list <host>", "List plugins on a host", (yargs) => {
-		yargs.positional("host", { describe: "Host to list for", type: "string" });
-	}],
-	handler: async function(args: { host: string }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		const plugins = await control.sendTo({ hostId }, new lib.PluginListRequest());
-		print(asTable(plugins.map(p => ({
-			title: p.title, version: p.version, loaded: p.loaded, enabled: p.enabled, npmPackage: p.npmPackage,
-		}))));
-	},
-}));
-hostPluginCommands.add(new lib.Command({
-	definition: ["update <host> <plugin>", "Update a plugin on a host", (yargs) => {
-		yargs.positional("host", { describe: "Host to update on", type: "string" });
-		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
-		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
-	}],
-	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		await control.sendTo({ hostId }, new lib.PluginUpdateRequest(args.plugin));
-		if (args.restart) {
-			await control.sendTo({ hostId }, new lib.HostRestartRequest());
-		} else {
-			print("Plugin updated; a restart is required to apply the changes.");
-		}
-	},
-}));
-hostPluginCommands.add(new lib.Command({
-	definition: ["install <host> <plugin>", "Install a plugin on a host", (yargs) => {
-		yargs.positional("host", { describe: "Host to install on", type: "string" });
-		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
-		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
-	}],
-	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
-		let hostId = await lib.resolveHost(control, args.host);
-		await control.sendTo({ hostId }, new lib.PluginInstallRequest(args.plugin));
-		if (args.restart) {
-			await control.sendTo({ hostId }, new lib.HostRestartRequest());
-		} else {
-			print("Plugin installed; a restart is required to apply the changes.");
-		}
-	},
-}));
-hostCommands.add(hostPluginCommands);

packages/ctl/src/commands_instance.ts

@@ -1,685 +0,0 @@
-import fs from "fs-extra";
-import asTableModule from "as-table";
-import events from "events";
-import path from "path";
-import os from "os";
-import child_process from "child_process";
-import stream from "stream";
-import util from "util";
-
-import * as lib from "@clusterio/lib";
-import { logger } from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-const finished = util.promisify(stream.finished);
-
-export const instanceCommands = new lib.CommandTree({
-	name: "instance", alias: ["i"], description: "Instance management",
-});
-instanceCommands.add(new lib.Command({
-	definition: [["list", "l"], "List instances known to the controller"],
-	handler: async function(args: object, control: Control) {
-		let list = await control.send(new lib.InstanceDetailsListRequest());
-		print(asTable(list));
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["create <name>", "Create an instance", (yargs) => {
-		// XXX TODO: set any specific options?
-		yargs.positional("name", { describe: "Instance name", type: "string" });
-		yargs.options({
-			"id": { type: "number", nargs: 1, describe: "Instance id" },
-			"from": { type: "number", nargs: 1, describe: "Clone config from instance id" },
-		});
-	}],
-	handler: async function(args: { name: string, id?: number, from?: number, }, control: Control) {
-		let instanceConfig = new lib.InstanceConfig("control");
-		if (args.id !== undefined) {
-			instanceConfig.set("instance.id", args.id);
-		}
-		instanceConfig.set("instance.name", args.name);
-		await control.send(new lib.InstanceCreateRequest(
-			instanceConfig.toRemote("controller", [
-				"instance.id", "instance.name",
-			]),
-			args.from
-		));
-	},
-}));
-
-const instanceConfigCommands = new lib.CommandTree({
-	name: "config", alias: ["c"], description: "Instance config management",
-});
-instanceConfigCommands.add(new lib.Command({
-	definition: ["list <instance>", "List configuration for an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to list config for", type: "string" });
-	}],
-	handler: async function(args: { instance: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		let config = await control.send(new lib.InstanceConfigGetRequest(instanceId));
-
-		for (let [name, value] of Object.entries(config)) {
-			print(`${name} ${JSON.stringify(value)}`);
-		}
-	},
-}));
-
-instanceConfigCommands.add(new lib.Command({
-	definition: ["set <instance> <field> [value]", "Set field in instance config", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
-		yargs.positional("field", { describe: "Field to set", type: "string" });
-		yargs.positional("value", { describe: "Value to set", type: "string" });
-		yargs.options({
-			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
-		});
-	}],
-	handler: async function(
-		args: { instance: string, field: string, value?: string, stdin?: boolean },
-		control: Control,
-	) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		if (args.stdin) {
-			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
-		} else if (args.value === undefined) {
-			args.value = "";
-		}
-		await control.send(new lib.InstanceConfigSetFieldRequest(instanceId, args.field, args.value));
-	},
-}));
-
-instanceConfigCommands.add(new lib.Command({
-	definition: ["set-prop <instance> <field> <prop> [value]", "Set property of field in instance config", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
-		yargs.positional("field", { describe: "Field to set", type: "string" });
-		yargs.positional("prop", { describe: "Property to set", type: "string" });
-		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
-		yargs.options({
-			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
-		});
-	}],
-	handler: async function(
-		args: { instance: string, field: string, prop: string, value?: string, stdin?: boolean },
-		control: Control
-	) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		if (args.stdin) {
-			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
-		}
-		let value;
-		try {
-			if (args.value !== undefined) {
-				value = JSON.parse(args.value);
-			}
-		} catch (err: any) {
-			// If this is from stdin or looks like an array, object or string
-			// literal throw the parse error, otherwise assume this is a string.
-			// The resoning behind this is that correctly quoting the string
-			// with the all the layers of quote removal at play is difficult.
-			// See the following table for how to pass "That's a \" quote" in
-			// different environments:
-			// ----------------------- WARNING -------------------------------
-			// This table is most likely invalid due a change in yargs parsing
-			// ---------------------------------------------------------------
-			// cmd              : """""That's a \\"" quote"""""
-			// cmd + npx        : """""""""""That's a \\\\"""" quote"""""""""""
-			// PowerShell       : '"""""That''s a \\"" quote"""""'
-			// PowerShell + npx : '"""""""""""That''s a \\\\"""" quote"""""""""""'
-			// bash             : '""That'\''s a \" quote""'
-			// bash + npx       : '""That'\''s a \" quote""'
-			// bash + npx -s sh : "'\"\"That'\\''s a \\\" quote\"\"'"
-			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
-				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
-			}
-			value = args.value;
-		}
-		await control.send(new lib.InstanceConfigSetPropRequest(instanceId, args.field, args.prop, value));
-	},
-}));
-
-instanceConfigCommands.add(new lib.Command({
-	definition: ["edit <instance> [editor]", "Edit instance configuration", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
-		yargs.positional("editor", {
-			describe: "Editor to use",
-			type: "string",
-			default: "",
-		});
-	}],
-	handler: async function(args: { instance: string, editor: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		let config = await control.send(new lib.InstanceConfigGetRequest(instanceId));
-		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
-		let editor = await getEditor(args.editor);
-		if (editor === undefined) {
-			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
-							  Try "ctl controller config edit <editor of choice>"`);
-		}
-		let disallowedList = {"instance.id": 0, "instance.assigned_host": 0, "factorio.settings": 0};
-		let allConfigElements = serializedConfigToString(
-			config,
-			lib.InstanceConfig,
-			disallowedList
-		);
-		await fs.writeFile(tmpFile, allConfigElements);
-		let editorSpawn = child_process.spawn(editor, [tmpFile], {
-			stdio: "inherit",
-			detached: false,
-		});
-		editorSpawn.on("data", (data) => {
-			process.stdout.pipe(data);
-		});
-		await events.once(editorSpawn, "exit");
-		const data = await fs.readFile(tmpFile, "utf8");
-		const final = await configToKeyVal(data);
-		for (let index in final) {
-			if (index in final) {
-				try {
-					await control.send(new lib.InstanceConfigSetFieldRequest(
-						instanceId,
-						index,
-						final[index],
-					));
-				} catch (err) {
-					// eslint-disable-next-line
-					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-					// If the string is empty, it's better to just print "" instead of nothing
-					print("This message shouldn't normally appear; if the below message does not indicate it");
-					print("was a user mistake, please report it to the clustorio devs.");
-					// added this because it could be a missed entry in disallowedList
-					// i've added all the vanilla clustorio configs, but there may be
-					// some modded ones that need to be added.
-					print(err);
-				}
-			}
-		}
-		try {
-			await fs.unlink(tmpFile);
-		} catch (err) {
-			print("err: temporary file", tmpFile, "could not be deleted.");
-			print("This is not fatal, but they may build up over time if the issue persists.");
-		}
-	},
-}));
-instanceCommands.add(instanceConfigCommands);
-
-instanceCommands.add(new lib.Command({
-	definition: ["assign <instance> [host]", "Assign instance to a host", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to assign", type: "string" });
-		yargs.positional("host", { describe: "Host to assign to or unassign if none", type: "string" });
-	}],
-	handler: async function(args: { instance: string, host?: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		let hostId = args.host ? await lib.resolveHost(control, args.host) : undefined;
-		await control.send(new lib.InstanceAssignRequest(instanceId, hostId));
-	},
-}));
-
-async function loadMapSettings(
-	args: { seed?: number, mapExchangeString?: string, mapGenSettings?: string, mapSettings?: string},
-) {
-	let seed = args.seed;
-	let mapGenSettings;
-	let mapSettings;
-	if (args.mapExchangeString) {
-		let parsed = lib.readMapExchangeString(args.mapExchangeString);
-		mapGenSettings = parsed.map_gen_settings;
-		mapSettings = parsed.map_settings;
-	}
-	if (args.mapGenSettings) {
-		mapGenSettings = JSON.parse((await fs.readFile(args.mapGenSettings)).toString());
-	}
-	if (args.mapSettings) {
-		mapSettings = JSON.parse((await fs.readFile(args.mapSettings)).toString());
-	}
-
-	return {
-		seed,
-		mapGenSettings,
-		mapSettings,
-	};
-}
-
-const instanceSaveCommands = new lib.CommandTree({
-	name: "save", alias: ["s"], description: "Instance save management",
-});
-instanceSaveCommands.add(new lib.Command({
-	definition: ["list <instance>", "list saves on an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to list saves on", type: "string" });
-	}],
-	handler: async function(args: { instance: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		let saves = await control.sendTo("controller", new lib.InstanceSaveDetailsListRequest());
-		saves = saves.filter(save => save.instanceId === instanceId);
-		print(asTable(saves.map(
-			({ mtimeMs, ...rest }) => ({ ...rest, mtime: new Date(mtimeMs).toLocaleString() })
-		)));
-	},
-}));
-
-instanceSaveCommands.add(new lib.Command({
-	definition: ["create <instance> [name]", "Create a new save on an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to create on", type: "string" });
-		yargs.positional("name", { describe: "Name of save to create.", type: "string", default: "world.zip" });
-		yargs.options({
-			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
-			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
-			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
-			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
-		});
-	}],
-	handler: async function(
-		args: {
-			instance: string,
-			name: string,
-			seed?: number,
-			mapExchangeString?: string,
-			mapGenSettings?: string,
-			mapSettings?: string,
-		},
-		control: Control
-	) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		await control.sendTo(
-			{ instanceId },
-			new lib.InstanceCreateSaveRequest(args.name, seed, mapGenSettings, mapSettings),
-		);
-	},
-}));
-
-instanceSaveCommands.add(new lib.Command({
-	definition: ["rename <instance> <old-name> <new-name>", "Rename a save on an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to rename save on", type: "string" });
-		yargs.positional("old-name", { describe: "Old name of save.", type: "string" });
-		yargs.positional("new-name", { describe: "New name of save.", type: "string" });
-	}],
-	handler: async function(args: { instance: string, oldName: string, newName: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.send(new lib.InstanceRenameSaveRequest(instanceId, args.oldName, args.newName));
-	},
-}));
-
-instanceSaveCommands.add(new lib.Command({
-	definition: ["copy <instance> <source> <destination>", "Copy a save on an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to copy save on", type: "string" });
-		yargs.positional("source", { describe: "Save to copy.", type: "string" });
-		yargs.positional("destination", { describe: "Name of copy.", type: "string" });
-	}],
-	handler: async function(args: { instance: string, source: string, destination: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.send(new lib.InstanceCopySaveRequest(instanceId, args.source, args.destination));
-	},
-}));
-
-instanceSaveCommands.add(new lib.Command({
-	definition: ["upload <instance> <filepath>", "Upload a save to an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to upload to", type: "string" });
-		yargs.positional("filepath", { describe: "Path to save to upload", type: "string" });
-		yargs.options({
-			"name": { describe: "Name to give save on server", nargs: 1, type: "string" },
-		});
-	}],
-	handler: async function(args: { instance: string, filepath: string, name?: string }, control: Control) {
-		let filename = args.name || path.basename(args.filepath);
-		if (!filename.endsWith(".zip")) {
-			throw new lib.CommandError("Save name must end with .zip");
-		}
-
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		let url = new URL(control.config.get("control.controller_url")!);
-		url.pathname += "api/upload-save";
-		url.searchParams.append("instance_id", String(instanceId));
-		url.searchParams.append("filename", filename);
-
-		const response = await fetch(url, {
-			method: "POST",
-			body: await fs.createReadStream(args.filepath),
-			duplex: "half",
-			headers: {
-				"X-Access-Token": control.config.get("control.controller_token")!,
-				"Content-Type": "application/zip",
-			},
-		});
-
-		type ResponseType = { errors?: string[], request_errors?: string[], saves?: string[] };
-		const result = await response.json() as ResponseType;
-
-		for (let error of result.errors || []) {
-			logger.error(error);
-		}
-
-		for (let requestError of result.request_errors || []) {
-			logger.error(requestError);
-		}
-
-		if (result.saves && result.saves.length) {
-			logger.info(`Successfully uploaded as ${result.saves[0]}`);
-		}
-
-		if ((result.errors || []).length || (result.request_errors || []).length) {
-			throw new lib.CommandError("Uploading save failed");
-		}
-	},
-}));
-
-instanceSaveCommands.add(new lib.Command({
-	definition: [
-		"transfer <source-instance> <source-save> <target-instance> [target-save]",
-		"Transfer a save between instances",
-		(yargs) => {
-			yargs.positional("source-instance", { describe: "Instance to transfer save from", type: "string" });
-			yargs.positional("source-save", { describe: "Save to transfer.", type: "string" });
-			yargs.positional("target-instance", { describe: "Instance to transfer to", type: "string" });
-			yargs.positional("target-save", { describe: "Name to give transferred save.", type: "string" });
-			yargs.options({
-				"copy": { describe: "Copy instead of moving the save", type: "boolean", default: false },
-			});
-		},
-	],
-	handler: async function(
-		args: {
-			sourceInstance: string,
-			sourceSave: string,
-			targetInstance: string,
-			targetSave: string,
-			copy: boolean,
-		},
-		control: Control
-	) {
-		let sourceInstanceId = await lib.resolveInstance(control, args.sourceInstance);
-		let targetInstanceId = await lib.resolveInstance(control, args.targetInstance);
-		let storedName = await control.send(
-			new lib.InstanceTransferSaveRequest(
-				sourceInstanceId,
-				args.sourceSave,
-				targetInstanceId,
-				args.targetSave || args.sourceSave,
-				args.copy
-			)
-		);
-		print(`Transferred as ${storedName} to ${args.targetInstance}.`);
-	},
-}));
-
-instanceSaveCommands.add(new lib.Command({
-	definition: ["download <instance> <save>", "Download a save from an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to download save from", type: "string" });
-		yargs.positional("save", { describe: "Save to download", type: "string" });
-	}],
-	handler: async function(args: { instance: string, save: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		let streamId = await control.send(new lib.InstanceDownloadSaveRequest(instanceId, args.save));
-
-		let url = new URL(control.config.get("control.controller_url")!);
-		url.pathname += `api/stream/${streamId}`;
-		let response = await fetch(url);
-		if (!response.body) {
-			throw new lib.CommandError("Got empty response from server.");
-		}
-
-		let writeStream;
-		let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip");
-		while (true) {
-			try {
-				writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
-				await events.once(writeStream, "open");
-				break;
-			} catch (err: any) {
-				if (err.code === "EEXIST") {
-					tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
-				} else {
-					throw err;
-				}
-			}
-		}
-		stream.Readable.fromWeb(response.body).pipe(writeStream);
-		await finished(writeStream);
-
-		let filename = await lib.findUnusedName(".", args.save, ".zip");
-		await fs.rename(tempFilename, filename);
-
-		logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`);
-	},
-}));
-
-instanceSaveCommands.add(new lib.Command({
-	definition: ["delete <instance> <save>", "Delete a save from an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to delete save from", type: "string" });
-		yargs.positional("save", { describe: "Save to delete", type: "string" });
-	}],
-	handler: async function(args: { instance: string, save: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.send(new lib.InstanceDeleteSaveRequest(instanceId, args.save));
-	},
-}));
-instanceCommands.add(instanceSaveCommands);
-
-instanceCommands.add(new lib.Command({
-	definition: ["export-data <instance>", "Export item icons and locale from instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to export from", type: "string" });
-	}],
-	handler: async function(args: { instance: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		await control.sendTo({ instanceId }, new lib.InstanceExportDataRequest());
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["extract-players <instance>", "Extract players from running save into the cluster.", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to extract players and online time from", type: "string" });
-	}],
-	handler: async function(args: { instance: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		await control.sendTo({ instanceId }, new lib.InstanceExtractPlayersRequest());
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["start <instance>", "Start instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to start", type: "string" });
-		yargs.options({
-			"save": { describe: "Save to load, defaults to latest", nargs: 1, type: "string" },
-			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: { instance: string, save?: string, keepOpen: boolean }, control: Control) {
-		const instanceId = await lib.resolveInstance(control, args.instance);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		await control.sendTo({ instanceId }, new lib.InstanceStartRequest(args.save));
-		control.keepOpen = args.keepOpen;
-	},
-}));
-instanceCommands.add(new lib.Command({
-	definition: ["restart <instance>", "Restart instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to restart", type: "string" });
-		yargs.options({
-			"save": { describe: "Save to load, defaults to latest", nargs: 1, type: "string" },
-			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: { instance: string, save?: string, keepOpen: boolean }, control: Control) {
-		const instanceId = await lib.resolveInstance(control, args.instance);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		await control.sendTo({ instanceId }, new lib.InstanceRestartRequest(args.save));
-		control.keepOpen = args.keepOpen;
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["start-all", "Start all stopped instances (respects exclude_from_start_all setting)", (yargs) => {
-		yargs.options({
-			"force": {
-				describe: "Start instances even if they have exclude_from_start_all set",
-				nargs: 0,
-				type: "boolean",
-				default: false,
-			},
-		});
-	}],
-	handler: async function(args: { force: boolean }, control: Control) {
-		// Get all instances
-		const instances = await control.send(new lib.InstanceDetailsListRequest());
-
-		// Filter instances that should be started
-		const instancesToStart = instances.filter((instance: lib.InstanceDetails) => {
-			// Only start instances that are currently stopped
-			if (instance.status !== "stopped") {
-				return false;
-			}
-
-			// Respect exclude_from_start_all setting unless --force is used
-			if (!args.force && instance.excludeFromStartAll) {
-				return false;
-			}
-
-			return true;
-		});
-
-		if (instancesToStart.length === 0) {
-			print("No instances to start.");
-			return;
-		}
-
-		print(`Starting ${instancesToStart.length} instance(s):`);
-		for (const instance of instancesToStart) {
-			print(`  - ${instance.name} (ID: ${instance.id})`);
-		}
-
-		// Start all filtered instances
-		const startPromises = instancesToStart.map(async (instance: lib.InstanceDetails) => {
-			try {
-				await control.sendTo({ instanceId: instance.id }, new lib.InstanceStartRequest());
-				print(`✓ Started ${instance.name}`);
-			} catch (error) {
-				print(`✗ Failed to start ${instance.name}: ${error}`);
-			}
-		});
-
-		await Promise.all(startPromises);
-		print("Start all operation completed.");
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["load-scenario <instance> <scenario>", "Start instance by loading a scenario", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to start", type: "string" });
-		yargs.positional("scenario", { describe: "Scenario to load", type: "string" });
-		yargs.options({
-			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
-			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
-			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
-			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
-			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(
-		args: {
-			instance: string,
-			scenario: string,
-			seed?: number,
-			mapExchangeString?: string,
-			mapGenSettings?: string,
-			mapSettings?: string,
-			keepOpen: boolean,
-		},
-		control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
-		await control.sendTo(
-			{ instanceId },
-			new lib.InstanceLoadScenarioRequest(args.scenario, seed, mapGenSettings, mapSettings),
-		);
-		control.keepOpen = args.keepOpen;
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["stop <instance>", "Stop instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to stop", type: "string" });
-	}],
-	handler: async function(args: { instance: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		await control.sendTo({ instanceId }, new lib.InstanceStopRequest());
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["stop-all", "Stop all running instances", () => {}],
-	handler: async function(_args, control: Control) {
-		// Get all instances
-		const instances = await control.send(new lib.InstanceDetailsListRequest());
-
-		// Filter instances that should be stopped
-		const instancesToStop = instances
-			.filter((instance: lib.InstanceDetails) => ["starting", "running"].includes(instance.status));
-
-		if (instancesToStop.length === 0) {
-			print("No instances to stop.");
-			return;
-		}
-
-		print(`Stopping ${instancesToStop.length} instance(s):`);
-		for (const instance of instancesToStop) {
-			print(`  - ${instance.name} (ID: ${instance.id})`);
-		}
-
-		// Stop all filtered instances
-		const stopPromises = instancesToStop.map(async (instance: lib.InstanceDetails) => {
-			try {
-				await control.sendTo({ instanceId: instance.id }, new lib.InstanceStopRequest());
-				print(`✓ Stopped ${instance.name}`);
-			} catch (error) {
-				print(`✗ Failed to stop ${instance.name}: ${error}`);
-			}
-		});
-
-		await Promise.all(stopPromises);
-		print("Stop all operation completed.");
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["kill <instance>", "Kill instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to kill", type: "string" });
-	}],
-	handler: async function(args: { instance: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.setLogSubscriptions({ instanceIds: [instanceId] });
-		await control.sendTo({ instanceId }, new lib.InstanceKillRequest());
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["delete <instance>", "Delete instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to delete", type: "string" });
-	}],
-	handler: async function(args: { instance: string }, control: Control) {
-		let instanceId = await lib.resolveInstance(control, args.instance);
-		await control.send(new lib.InstanceDeleteRequest(instanceId));
-	},
-}));
-
-instanceCommands.add(new lib.Command({
-	definition: ["send-rcon <instance> <command>", "Send RCON command", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to send to", type: "string" });
-		yargs.positional("command", { describe: "command to send", type: "string" });
-	}],
-	handler: async function(args: { instance: string, command: string }, control: Control) {
-		let result = await control.sendTo(
-			{ instanceId: await lib.resolveInstance(control, args.instance) },
-			new lib.InstanceSendRconRequest(args.command),
-		);
-
-		// Factorio includes a newline in its response output.
-		process.stdout.write(result);
-	},
-}));

packages/ctl/src/commands_log.ts

@@ -1,90 +0,0 @@
-import winston from "winston";
-
-import * as lib from "@clusterio/lib";
-import { ConsoleTransport, levels, logger } from "@clusterio/lib";
-import type { Control } from "../ctl";
-
-export const logCommands = new lib.CommandTree({ name: "log", description: "Log inspection" });
-logCommands.add(new lib.Command({
-	definition: ["follow", "follow cluster log", (yargs) => {
-		yargs.options({
-			"all": { describe: "Follow the whole cluster log", nargs: 0, type: "boolean", default: false },
-			"controller": { describe: "Follow log of the controller", nargs: 0, type: "boolean", default: false },
-			"host": { describe: "Follow log of given host", nargs: 1, type: "string", default: null },
-			"instance": { describe: "Follow log of given instance", nargs: 1, type: "string", default: null },
-		});
-	}],
-	handler: async function(
-		args: { all: boolean, controller: boolean, host: string | null, instance: string | null },
-		control: Control
-	) {
-		if (!args.all && !args.controller && !args.host && !args.instance) {
-			logger.error("At least one of --all, --controller, --host and --instance must be passed");
-			process.exitCode = 1;
-			return;
-		}
-		let instanceIds = args.instance ? [await lib.resolveInstance(control, args.instance)] : [];
-		let hostIds = args.host ? [await lib.resolveHost(control, args.host)] : [];
-		await control.setLogSubscriptions({ all: args.all, controller: args.controller, hostIds, instanceIds });
-		control.keepOpen = true;
-	},
-}));
-
-logCommands.add(new lib.Command({
-	definition: ["query", "Query cluster log", (yargs) => {
-		yargs.options({
-			"all": { describe: "Query the whole cluster log", nargs: 0, type: "boolean", default: false },
-			"controller": { describe: "Query log of the controller", nargs: 0, type: "boolean", default: false },
-			"host": { describe: "Query log of given host", nargs: 1, type: "string", default: null },
-			"instance": { describe: "Query log of given instance", nargs: 1, type: "string", default: null },
-			"max-level": { describe: "Maximum log level to return", nargs: 1, type: "string", default: undefined },
-			"limit": { describe: "Max number of entries to return", nargs: 1, type: "number", default: 1000 },
-			"start": { describe: "Limit from the start instead of the end", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(
-		args: {
-			all: boolean,
-			controller: boolean,
-			host: string | null,
-			instance: string | null,
-			maxLevel?: string,
-			limit: number,
-			start: boolean,
-		},
-		control: Control
-	) {
-		if (!args.all && !args.controller && !args.host && !args.instance) {
-			logger.error("At least one of --all, --controller, --host and --instance must be passed");
-			process.exitCode = 1;
-			return;
-		}
-		let instanceIds = args.instance ? [await lib.resolveInstance(control, args.instance)] : [];
-		let hostIds = args.host ? [await lib.resolveHost(control, args.host)] : [];
-		let result = await control.send(new lib.LogQueryRequest(
-			args.all,
-			args.controller,
-			hostIds,
-			instanceIds,
-			args.maxLevel as keyof typeof levels,
-			args.limit,
-			args.start ? "asc" : "desc",
-		));
-
-		if (!args.start) {
-			result.log.reverse();
-		}
-
-		let stdoutLogger = winston.createLogger({
-			level: "verbose",
-			levels,
-			format: new lib.TerminalFormat({ showTimestamp: true }),
-			transports: [
-				new ConsoleTransport({ errorLevels: [], warnLevels: [] }),
-			],
-		});
-		for (let info of result.log) {
-			stdoutLogger.log(info as any);
-		}
-	},
-}));

packages/ctl/src/commands_mod.ts

@@ -1,237 +0,0 @@
-import fs from "fs-extra";
-import asTableModule from "as-table";
-import events from "events";
-import path from "path";
-import stream from "stream";
-import util from "util";
-
-import * as lib from "@clusterio/lib";
-import { logger } from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-const finished = util.promisify(stream.finished);
-
-export const modCommands = new lib.CommandTree({ name: "mod", description: "Manage uploaded mods" });
-modCommands.add(new lib.Command({
-	definition: ["show <name> <mod-version>", "Show details for a mod stored in the cluster", (yargs) => {
-		yargs.positional("name", { describe: "Mod name to show details for", type: "string" });
-		yargs.positional("mod-version", { describe: "Version of the mod", type: "string" });
-	}],
-	handler: async function(args: { name: string, modVersion: string }, control: Control) {
-		if (!lib.isFullVersion(args.modVersion)) {
-			throw new lib.CommandError("mod-version must match format digit.digit.digit");
-		}
-		let modInfo = await control.send(new lib.ModGetRequest(args.name, args.modVersion));
-		for (let [field, value] of Object.entries(modInfo)) {
-			if (value instanceof Array) {
-				print(`${field}:`);
-				for (let entry of value) {
-					print(`  ${entry}`);
-				}
-			} else {
-				print(`${field}: ${value}`);
-			}
-		}
-	},
-}));
-
-modCommands.add(new lib.Command({
-	definition: [["list", "l"], "List mods stored in the cluster", (yargs) => {
-		yargs.options({
-			"fields": {
-				describe: "Fields to show, supports 'all'.",
-				array: true,
-				type: "string",
-				default: ["name", "version", "title", "factorio_version"],
-			},
-		});
-	}],
-	handler: async function(args: { fields: string[] }, control: Control) {
-		let mods = await control.send(new lib.ModListRequest());
-		if (!args.fields.includes("all")) {
-			for (let entry of mods) {
-				for (let field of Object.keys(entry)) {
-					if (!args.fields.includes(field)) {
-						// @ts-expect-error terrible hack
-						delete entry[field];
-					}
-				}
-			}
-		}
-		print(asTable(mods));
-	},
-}));
-
-modCommands.add(new lib.Command({
-	definition: ["search <factorio-version> [query]", "Search mods stored in the cluster", (yargs) => {
-		yargs.positional("factorio-version", { describe: "Major version of Factorio to search for", type: "string" });
-		yargs.positional("query", { describe: "Search query", type: "string", default: "" });
-		yargs.options({
-			"page": {
-				describe: "Result page to show",
-				type: "number",
-				default: 1,
-			},
-			"page-size": {
-				describe: "Results per page to show",
-				type: "number",
-				default: 10,
-			},
-			"sort": {
-				describe: "sort results by given field",
-				type: "string",
-			},
-			"sort-order": {
-				describe: "order to sort results in (asc/desc)",
-				type: "string",
-				default: "asc",
-			},
-			"fields": {
-				describe: "Fields to show, supports 'all'.",
-				array: true,
-				type: "string",
-				default: ["name", "version", "title", "factorio_version"],
-			},
-		});
-	}],
-	handler: async function(
-		args: {
-			factorioVersion: string,
-			query: string,
-			page: number,
-			pageSize: number,
-			sort?: string,
-			sortOrder: "asc" | "desc",
-			fields: string[],
-		},
-		control: Control
-	) {
-		let response = await control.send(new lib.ModSearchRequest(
-			args.query,
-			lib.normaliseApiVersion(args.factorioVersion as any),
-			args.page,
-			args.pageSize,
-			args.sort,
-			args.sortOrder
-		));
-		let results = response.results.flatMap(result => result.versions);
-		if (!args.fields.includes("all")) {
-			for (let entry of results) {
-				for (let field of Object.keys(entry)) {
-					if (!args.fields.includes(field)) {
-						// @ts-expect-error terrible hack
-						delete entry[field];
-					}
-				}
-			}
-		}
-		for (let issue of response.queryIssues) {
-			print(issue);
-		}
-		print(`page ${args.page} of ${response.pageCount} (${response.resultCount} results)`);
-		print(asTable(results));
-	},
-}));
-
-modCommands.add(new lib.Command({
-	definition: ["upload <file>", "Upload mod to the cluster", (yargs) => {
-		yargs.positional("file", { describe: "File to upload", type: "string" });
-	}],
-	handler: async function(args: { file: string }, control: Control) {
-		let filename = path.basename(args.file);
-		if (!filename.endsWith(".zip")) {
-			throw new lib.CommandError("Mod filename must end with .zip");
-		}
-
-		let url = new URL(control.config.get("control.controller_url")!);
-		url.pathname += "api/upload-mod";
-		url.searchParams.append("filename", filename);
-
-		let response = await fetch(url, {
-			method: "POST",
-			body: fs.createReadStream(args.file),
-			duplex: "half",
-			headers: {
-				"X-Access-Token": control.config.get("control.controller_token")!,
-				"Content-Type": "application/zip",
-			},
-		});
-
-		type ResponseType = { errors?: [], request_errors?: [], mods: Parameters<typeof lib.ModInfo.fromJSON>[0][]};
-		const result = await response.json() as ResponseType;
-
-		for (let error of result.errors || []) {
-			logger.error(error);
-		}
-
-		for (let requestError of result.request_errors || []) {
-			logger.error(requestError);
-		}
-
-		if (result.mods && result.mods.length) {
-			const mod = lib.ModInfo.fromJSON(result.mods[0]);
-			logger.info(`Successfully uploaded ${mod.filename}`);
-		}
-
-		if ((result.errors || []).length || (result.request_errors || []).length) {
-			throw new lib.CommandError("Uploading mod failed");
-		}
-	},
-}));
-
-modCommands.add(new lib.Command({
-	definition: ["download <name> <mod-version>", "Download a mod from the cluster", (yargs) => {
-		yargs.positional("name", { describe: "Internal name of mod to download", type: "string" });
-		yargs.positional("mod-version", { describe: "Version of mod to download", type: "string" });
-	}],
-	handler: async function(args: { name: string, modVersion: string }, control: Control) {
-		if (!lib.isFullVersion(args.modVersion)) {
-			throw new lib.CommandError("mod-version must match format digit.digit.digit");
-		}
-		let streamId = await control.send(new lib.ModDownloadRequest(args.name, args.modVersion));
-
-		let url = new URL(control.config.get("control.controller_url")!);
-		url.pathname += `api/stream/${streamId}`;
-		let response = await fetch(url);
-		if (!response.body) {
-			throw new lib.CommandError("Got empty response from server");
-		}
-
-		let writeStream;
-		let filename = `${args.name}_${args.modVersion}.zip`;
-		let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
-		while (true) {
-			try {
-				writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
-				await events.once(writeStream, "open");
-				break;
-			} catch (err: any) {
-				if (err.code === "EEXIST") {
-					tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
-				} else {
-					throw err;
-				}
-			}
-		}
-		stream.Readable.fromWeb(response.body).pipe(writeStream);
-		await finished(writeStream);
-		await fs.rename(tempFilename, filename);
-
-		logger.info(`Downloaded ${filename}`);
-	},
-}));
-
-modCommands.add(new lib.Command({
-	definition: ["delete <name> <mod-version>", "Delete a mod stored in the cluster", (yargs) => {
-		yargs.positional("name", { describe: "Name of mod to delete", type: "string" });
-		yargs.positional("mod-version", { describe: "Version of mod to delete", type: "string" });
-	}],
-	handler: async function(args: { name: string, modVersion: string }, control: Control) {
-		if (!lib.isFullVersion(args.modVersion)) {
-			throw new lib.CommandError("mod-version must match format digit.digit.digit");
-		}
-		await control.send(new lib.ModDeleteRequest(args.name, args.modVersion));
-	},
-}));

packages/ctl/src/commands_mod_pack.ts

@@ -1,315 +0,0 @@
-import asTableModule from "as-table";
-
-import * as lib from "@clusterio/lib";
-import { logger } from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-
-export const modPackCommands = new lib.CommandTree({ name: "mod-pack", description: "Mod Pack" });
-modPackCommands.add(new lib.Command({
-	definition: ["show <mod-pack>", "Show details of mod pack", (yargs) => {
-		yargs.positional("mod-pack", { describe: "Mod pack to show", type: "string" });
-	}],
-	handler: async function(args: { modPack: string }, control: Control) {
-		let mods = new Map(
-			(await control.send(new lib.ModListRequest())).map(m => [m.id, m])
-		);
-		let modPack = await control.send(
-			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
-		);
-
-		for (let [field, value] of Object.entries(modPack)) {
-			if (field === "mods") {
-				print(`${field}:`);
-				for (let [name, entry] of value) {
-					const mod = mods.get(`${name}_${entry.version}`);
-					const missing = !mod && "? ";
-					const badChecksum = mod && entry.sha1 && entry.sha1 !== mod.sha1 && "! ";
-					const warn = missing || badChecksum || "";
-					const enabled = entry.enabled ? "" : "(disabled) ";
-					print(`  ${warn}${enabled}${name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`);
-				}
-			} else if (field === "settings") {
-				print(`${field}:`);
-				for (let [scope, settings] of Object.entries(value)) {
-					print(`  ${scope}:`);
-					for (let [setting, settingValue] of settings as any) {
-						print(`    ${setting}: ${JSON.stringify(settingValue.value)}`);
-					}
-				}
-			} else if (field === "exportManifest") {
-				print(`${field}:`);
-				if (value && value.assets) {
-					print("  assets:");
-					for (let [name, fileName] of Object.entries(value.assets)) {
-						print(`    ${name}: ${fileName}`);
-					}
-				}
-
-			} else {
-				print(`${field}: ${value}`);
-			}
-		}
-	},
-}));
-
-modPackCommands.add(new lib.Command({
-	definition: [["list", "l"], "List mod packs in the cluster"],
-	handler: async function(args: object, control: Control) {
-		let modPacks = await control.send(new lib.ModPackListRequest());
-		let fields = ["id", "name", "factorioVersion"];
-		for (let entry of modPacks) {
-			for (let field of Object.keys(entry)) {
-				if (!fields.includes(field)) {
-					// @ts-expect-error terrible hack
-					delete entry[field];
-				}
-			}
-		}
-		print(asTable(modPacks));
-	},
-}));
-
-function setModPackSettings(
-	modPack: lib.ModPack,
-	args: {
-		boolSetting?: string[],
-		intSetting?: string[],
-		doubleSetting?: string[],
-		stringSetting?: string[],
-		colorSetting?: string[],
-	},
-) {
-	function doSettings(settings: string[], cast: (value: string) => (string | number | boolean)) {
-		for (let i = 0; i + 2 < settings.length; i += 3) {
-			const [scope, name, value] = settings.slice(i, i + 3);
-			if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
-				throw new lib.CommandError(
-					`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
-				);
-			}
-			const checkedScope = scope as "startup" | "runtime-global" | "runtime-per-user";
-			modPack.settings[checkedScope].set(name, { value: cast(value) });
-		}
-	}
-
-	doSettings(args.boolSetting || [], value => {
-		if (!["true", "false"].includes(value)) {
-			throw new lib.CommandError(`boolean value must be one of true or false, not ${value}`);
-		}
-		return value === "true";
-	});
-	doSettings(args.intSetting || [], value => {
-		// eslint-disable-next-line radix
-		let number = Number.parseInt(value);
-		if (isNaN(number)) {
-			throw new lib.CommandError(`int value must be an integer, not ${value}`);
-		}
-		return number;
-	});
-	doSettings(args.doubleSetting || [], value => {
-		let number = Number.parseFloat(value);
-		if (isNaN(number)) {
-			throw new lib.CommandError(`double value must be a number, not ${value}`);
-		}
-		return number;
-	});
-	doSettings(args.stringSetting || [], value => value);
-	doSettings(args.colorSetting || [], value => {
-		let color = JSON.parse(value);
-		if (typeof color !== "object" || color === null) {
-			throw new lib.CommandError("color value must be a JSON object with r, g, b, a properties");
-		}
-		return color;
-	});
-}
-
-function setModPackMods(modPack: lib.ModPack, mods: string[] | undefined) {
-	for (let mod of mods || []) {
-		const [name, version, sha1] = mod.split(":");
-		if (!version) {
-			throw new lib.CommandError("Added mod must be formatted as name:version or name:version:sha1");
-		}
-		if (!lib.isFullVersion(version)) {
-			throw new lib.CommandError("version must match the format digit.digit.digit");
-		}
-		if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) {
-			throw new lib.CommandError("sha1 must be a 40 digit lower case hex string");
-		}
-		modPack.mods.set(name, { name, enabled: true, version, sha1 });
-	}
-}
-
-function setModPackModsEnabled(modPack: lib.ModPack, mods: string[] | undefined, enabled: boolean) {
-	for (let mod of mods || []) {
-		if (!modPack.mods.has(mod)) {
-			throw new lib.CommandError(`Mod named ${mod} does not exist in the mod pack`);
-		}
-		modPack.mods.get(mod)!.enabled = enabled;
-	}
-}
-
-modPackCommands.add(new lib.Command({
-	definition: ["create <name> <factorio-version>", "Create mod pack", (yargs) => {
-		yargs.positional("name", { describe: "Name of mod pack to create", type: "string" });
-		yargs.positional("factorio-version", { describe: "Version of factorio the mod pack is for", type: "string" });
-		yargs.options({
-			"description": { describe: "Description for mod pack", type: "string" },
-			"mods": { describe: "Mods in the form of name:version[:sha1]", array: true, type: "string" },
-			"disabled-mods": { describe: "Mods that are in the pack but not enabled", array: true, type: "string" },
-			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
-			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
-			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
-			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
-			"color-setting": { describe: "Set color setting", array: true, nargs: 3, type: "string" },
-		});
-	}],
-	handler: async function(
-		args: {
-			name: string,
-			factorioVersion: string,
-			description?: string,
-			mods?: string[],
-			disabledMods?: string[],
-			boolSetting?: string[],
-			intSetting?: string[],
-			doubleSetting?: string[],
-			stringSetting?: string[],
-			colorSetting?: string[],
-		},
-		control: Control,
-	) {
-		const modPack = new lib.ModPack();
-		modPack.name = args.name;
-		if (args.description) { modPack.description = args.description; }
-		if (args.factorioVersion) {
-			if (!lib.isPartialVersion(args.factorioVersion)) {
-				throw new lib.CommandError("factorio-version must match the format digit.digit[.digit]");
-			}
-			modPack.factorioVersion = args.factorioVersion;
-		}
-		setModPackMods(modPack, args.mods);
-		setModPackModsEnabled(modPack, args.disabledMods, false);
-		setModPackSettings(modPack, args);
-		await control.send(new lib.ModPackCreateRequest(modPack));
-		print(`Created mod pack ${modPack.name} (${modPack.id})`);
-	},
-}));
-
-modPackCommands.add(new lib.Command({
-	definition: ["import <string>", "Import mod pack string", (yargs) => {
-		yargs.positional("string", { describe: "Mod pack string to import", type: "string" });
-	}],
-	handler: async function(args: { string: string }, control: Control) {
-		const modPack = lib.ModPack.fromModPackString(args.string);
-		await control.send(new lib.ModPackCreateRequest(modPack));
-		print(`Created mod pack ${modPack.name} (${modPack.id})`);
-	},
-}));
-
-modPackCommands.add(new lib.Command({
-	definition: ["export <mod-pack>", "Export mod pack string", (yargs) => {
-		yargs.positional("string", { describe: "Mod pack to export", type: "string" });
-	}],
-	handler: async function(args: { modPack: string }, control: Control) {
-		const modPack = await control.send(
-			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
-		);
-		print(modPack.toModPackString());
-	},
-}));
-
-modPackCommands.add(new lib.Command({
-	definition: ["edit <mod-pack>", "Edit mod pack", (yargs) => {
-		yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" });
-		yargs.options({
-			"name": { describe: "New name for mod pack", type: "string" },
-			"description": { describe: "New description for mod pack", type: "string" },
-			"factorio-version": { describe: "Set version of factorio the mod pack is for", type: "string" },
-			"add-mods": { describe: "Mods in the form of name:version[:sha1] to add", array: true, type: "string" },
-			"enable-mods": { describe: "Mods to set as enabled", array: true, type: "string" },
-			"disable-mods": { describe: "Mods to set as disabled", array: true, type: "string" },
-			"remove-mods": { describe: "Name of mods to remove", array: true, type: "string" },
-			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
-			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
-			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
-			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
-			"color-setting": { describe: "Set color setting", array: true, nargs: 3, type: "string" },
-			"remove-setting": { describe: "Remove a setting", array: true, nargs: 2, type: "string" },
-		});
-	}],
-	handler: async function(
-		args: {
-			modPack: string,
-			name?: string,
-			description?: string,
-			factorioVersion?: string,
-			addMods?: string[],
-			enableMods?: string[],
-			disableMods?: string[],
-			removeMods?: string[],
-			boolSetting?: string[],
-			intSetting?: string[],
-			doubleSetting?: string[],
-			stringSetting?: string[],
-			colorSetting?: string[],
-			removeSetting?: string[],
-		},
-		control: Control,
-	) {
-		const modPack = await control.send(
-			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
-		);
-
-		if (args.name) { modPack.name = args.name; }
-		if (args.description) { modPack.description = args.description; }
-		if (args.factorioVersion) {
-			if (!lib.isPartialVersion(args.factorioVersion)) {
-				throw new lib.CommandError("factorio-version must match the format digit.digit[.digit]");
-			}
-			modPack.factorioVersion = args.factorioVersion;
-		}
-
-		setModPackMods(modPack, args.addMods);
-		for (let name of args.removeMods || []) {
-			if (modPack.mods.has(name)) {
-				modPack.mods.delete(name);
-			} else {
-				logger.warn(`Mod ${name} did not exist on ${modPack.name}`);
-			}
-		}
-		setModPackModsEnabled(modPack, args.disableMods, false);
-		setModPackModsEnabled(modPack, args.enableMods, true);
-
-		setModPackSettings(modPack, args);
-		if (args.removeSetting) {
-			for (let i = 0; i + 1 < args.removeSetting.length; i += 2) {
-				const [scope, name] = args.removeSetting.slice(i, i + 2);
-				if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
-					throw new lib.CommandError(
-						`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
-					);
-				}
-				const checkedScope = scope as "startup" | "runtime-global" | "runtime-per-user";
-				if (modPack.settings[checkedScope].has(name)) {
-					modPack.settings[checkedScope].delete(name);
-				} else {
-					logger.warn(`Mod setting ${scope} ${name} did not exist on ${modPack.name}`);
-				}
-			}
-		}
-		await control.send(new lib.ModPackUpdateRequest(modPack));
-	},
-}));
-
-modPackCommands.add(new lib.Command({
-	definition: ["delete <mod-pack>", "Delete mod pack", (yargs) => {
-		yargs.positional("mod-pack", { describe: "Mod pack to delete", type: "string" });
-	}],
-	handler: async function(args: { modPack: string }, control: Control) {
-		const id = await lib.resolveModPack(control, args.modPack);
-		await control.send(new lib.ModPackDeleteRequest(id));
-	},
-}));

packages/ctl/src/commands_permission.ts

@@ -1,16 +0,0 @@
-import asTableModule from "as-table";
-
-import * as lib from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-
-export const permissionCommands = new lib.CommandTree({ name: "permission", description: "Permission inspection" });
-permissionCommands.add(new lib.Command({
-	definition: [["list", "l"], "List permissions in the cluster"],
-	handler: async function(args: object, control: Control) {
-		let permissions = await control.send(new lib.PermissionListRequest());
-		print(asTable(permissions));
-	},
-}));

packages/ctl/src/commands_role.ts

@@ -1,100 +0,0 @@
-import asTableModule from "as-table";
-
-import * as lib from "@clusterio/lib";
-import { logger } from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-
-export const roleCommands = new lib.CommandTree({ name: "role", description: "Role management" });
-roleCommands.add(new lib.Command({
-	definition: [["list", "l"], "List roles in the cluster"],
-	handler: async function(args: object, control: Control) {
-		let roles = await control.send(new lib.RoleListRequest());
-		print(asTable(roles));
-	},
-}));
-
-roleCommands.add(new lib.Command({
-	definition: ["create <name>", "Create a new role", (yargs) => {
-		yargs.positional("name", { describe: "Name of role to create", type: "string" });
-		yargs.options({
-			"description": { describe: "Description for role", nargs: 1, type: "string", default: "" },
-			"permissions": { describe: "Permissions role grants", nargs: 1, array: true, type: "string", default: [] },
-		});
-	}],
-	handler: async function(args: { name: string, description: string, permissions: string[] }, control: Control) {
-		let id = await control.send(new lib.RoleCreateRequest(
-			args.name,
-			args.description,
-			args.permissions,
-		));
-		logger.info(`Created role ID ${id}`);
-	},
-}));
-
-roleCommands.add(new lib.Command({
-	definition: ["edit <role>", "Edit existing role", (yargs) => {
-		yargs.positional("role", { describe: "Role to edit", type: "string" });
-		yargs.options({
-			"name": { describe: "New name for role", nargs: 1, type: "string" },
-			"description": { describe: "New description for role", nargs: 1, type: "string" },
-			"set-perms": { describe: "Set permissions for role", array: true, type: "string" },
-			"add-perms": { describe: "Add permissions to role", array: true, type: "string", conflicts: "set-perms" },
-			"remove-perms": {
-				describe: "Remove permissions from role", array: true, type: "string", conflicts: "set-perms",
-			},
-			"grant-default": { describe: "Add default permissions to role", nargs: 0, type: "boolean" },
-		});
-	}],
-	handler: async function(
-		args: {
-			role: string,
-			name?: string,
-			description?: string,
-			setPerms?: string[],
-			addPerms?: string[],
-			removePerms?: string[],
-			grantDefault?: boolean,
-		},
-		control: Control
-	) {
-		let role = await lib.retrieveRole(control, args.role);
-
-		if (args.name !== undefined) {
-			role.name = args.name;
-		}
-		if (args.description !== undefined) {
-			role.description = args.description;
-		}
-		if (args.addPerms) {
-			for (const perm of args.addPerms) {
-				role.permissions.add(perm);
-			}
-		}
-		if (args.removePerms) {
-			for (let perm of args.removePerms) {
-				role.permissions.delete(perm);
-			}
-		}
-		if (args.setPerms !== undefined) {
-			role.permissions = new Set(args.setPerms);
-		}
-		await control.send(new lib.RoleUpdateRequest(role.id, role.name, role.description, [...role.permissions]));
-
-		if (args.grantDefault) {
-			await control.send(new lib.RoleGrantDefaultPermissionsRequest(role.id));
-		}
-	},
-}));
-
-roleCommands.add(new lib.Command({
-	definition: ["delete <role>", "Delete role", (yargs) => {
-		yargs.positional("role", { describe: "Role to delete", type: "string" });
-	}],
-	handler: async function(args: { role: string }, control: Control) {
-		let role = await lib.retrieveRole(control, args.role);
-		await control.send(new lib.RoleDeleteRequest(role.id));
-	},
-}));

packages/ctl/src/commands_user.ts

@@ -1,304 +0,0 @@
-import fs from "fs-extra";
-import asTableModule from "as-table";
-import path from "path";
-
-import * as lib from "@clusterio/lib";
-import type { Control } from "../ctl";
-import { print } from "./command_ops";
-
-const asTable = asTableModule.configure({ delimiter: " | " });
-
-export const userCommands = new lib.CommandTree({ name: "user", alias: ["u"], description: "User management" });
-userCommands.add(new lib.Command({
-	definition: ["show <name>", "Show details for one user", (yargs) => {
-		yargs.positional("name", { decribe: "Name of user to show", type: "string" });
-		yargs.options({
-			"instance-stats": { describe: "include per-instance stats", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: { name: string, instanceStats: boolean }, control: Control) {
-		let user = await control.send(new lib.UserGetRequest(args.name));
-		Object.assign(user, user.playerStats);
-		// @ts-expect-error Terrible hack
-		delete user.playerStats;
-		let instanceStats = user.instanceStats;
-		// @ts-expect-error Terrible hack
-		delete user.instanceStats;
-		print(asTable(Object.entries(user).map(([property, value]) => ({ property, value }))));
-
-		if (args.instanceStats) {
-			let instances = await control.send(new lib.InstanceDetailsListRequest());
-			function instanceName(id: number) {
-				let instance = instances.find(i => i.id === id);
-				if (instance) {
-					return instance.name;
-				}
-				return "<deleted>";
-			}
-			for (let [id, playerInstanceStats] of instanceStats || []) {
-				print();
-				print(`Instance ${instanceName(id)} (${id}):`);
-				print(asTable(Object.entries(playerInstanceStats).map(([property, value]) => ({ property, value }))));
-			}
-		}
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: [["list", "l"], "List user in the cluster", (yargs) => {
-		yargs.options({
-			"stats": { describe: "include user stats", nargs: 0, type: "boolean", default: false },
-			"attributes": { describe: "include admin/whitelisted/banned", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: { stats: boolean, attributes: boolean }, control: Control) {
-		let users = await control.send(new lib.UserListRequest());
-		for (let user of users) {
-			if (args.stats) {
-				Object.assign(user, user.playerStats);
-			}
-			// @ts-expect-error Terrible hack
-			delete user.playerStats;
-			// @ts-expect-error Terrible hack
-			delete user.isDeleted;
-			// @ts-expect-error Terrible hack
-			delete user.banReason;
-			// @ts-expect-error Terrible hack
-			delete user.instanceStats;
-			if (!args.attributes) {
-				// @ts-expect-error Terrible hack
-				delete user.isAdmin;
-				// @ts-expect-error Terrible hack
-				delete user.isWhitelisted;
-				// @ts-expect-error Terrible hack
-				delete user.isBanned;
-			}
-		}
-		print(asTable(users));
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["create <name>", "Create a user", (yargs) => {
-		yargs.positional("name", { describe: "Name of user to create", type: "string" });
-	}],
-	handler: async function(args: { name: string }, control: Control) {
-		await control.send(new lib.UserCreateRequest(args.name));
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["revoke-token <name>", "Revoke token for user", (yargs) => {
-		yargs.positional("name", { describe: "Name of user to revoke token for", type: "string" });
-	}],
-	handler: async function(args: { name: string }, control: Control) {
-		await control.send(new lib.UserRevokeTokenRequest(args.name));
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["set-admin <user>", "Promote or demote a user to admin", (yargs) => {
-		yargs.positional("user", { describe: "Name of user set admin status for", type: "string" });
-		yargs.options({
-			"revoke": { describe: "Revoke admin status", nargs: 0, type: "boolean", default: false },
-			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: { user: string, revoke: boolean, create: boolean }, control: Control) {
-		await control.send(new lib.UserSetAdminRequest(args.user, args.create, !args.revoke));
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["set-whitelisted <user>", "Add or remove user from the whitelist", (yargs) => {
-		yargs.positional("user", { describe: "Name of user to set whitelist status for", type: "string" });
-		yargs.options({
-			"remove": { describe: "Remove from whitelist", nargs: 0, type: "boolean", default: false },
-			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: { user: string, remove: boolean, create: boolean }, control: Control) {
-		await control.send(new lib.UserSetWhitelistedRequest(args.user, args.create, !args.remove));
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["set-banned <user>", "Ban or pardon user from banlist", (yargs) => {
-		yargs.positional("user", { describe: "Name of user to set ban status for", type: "string" });
-		yargs.options({
-			"pardon": { describe: "Remove from banlist", nargs: 0, type: "boolean", default: false },
-			"reason": { describe: "Ban reason", nargs: 1, type: "string", default: "" },
-			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(
-		args: {
-			user: string,
-			pardon: boolean,
-			reason: string,
-			create: boolean
-		},
-		control: Control
-	) {
-		await control.send(new lib.UserSetBannedRequest(args.user, args.create, !args.pardon, args.reason));
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["set-roles <user> [roles...]", "Replace user roles", (yargs) => {
-		yargs.positional("user", { describe: "Name of user to change roles for", type: "string" });
-		yargs.positional("roles", { describe: "roles to assign", type: "string" });
-	}],
-	handler: async function(args: { user: string, roles: string[] }, control: Control) {
-		let roles = await control.send(new lib.RoleListRequest());
-
-		let resolvedRoles = [];
-		for (let roleName of args.roles) {
-			if (/^-?\d+$/.test(roleName)) {
-				let roleId = parseInt(roleName, 10);
-				resolvedRoles.push(roleId);
-
-			} else {
-				let found = false;
-				for (let role of roles) {
-					if (role.name === roleName) {
-						resolvedRoles.push(role.id);
-						found = true;
-						break;
-					}
-				}
-
-				if (!found) {
-					throw new lib.CommandError(`No role named ${roleName}`);
-				}
-			}
-		}
-
-		await control.send(new lib.UserUpdateRolesRequest(args.user, resolvedRoles));
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["delete <user>", "Delete user", (yargs) => {
-		yargs.positional("user", { describe: "Name of user to delete", type: "string" });
-	}],
-	handler: async function(args: { user: string }, control: Control) {
-		await control.send(new lib.UserDeleteRequest(args.user));
-	},
-}));
-
-async function handleImportOrRestore(args: {
-	filepath: string,
-	users: boolean,
-	bans: boolean,
-	admins: boolean,
-	whitelist: boolean,
-	noBackup?: boolean,
-}, control: Control, restore: boolean) {
-	let importType: ConstructorParameters<typeof lib.UserBulkImportRequest>[0];
-	const data = JSON.parse((await fs.readFile(args.filepath)).toString());
-	const optionCount = [args.users, args.bans, args.admins, args.whitelist]
-		.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
-
-	if (optionCount > 1) {
-		throw new lib.CommandError("Can not specify multiple options");
-	}
-
-	// Assign based on options or attempt to guess based on filename
-	const filename = path.basename(args.filepath);
-	if (args.users || (optionCount === 0 && data.export_version)) {
-		importType = "users";
-	} else if (args.bans || (optionCount === 0 && filename.includes("ban"))) {
-		importType = "bans";
-	} else if (args.admins || (optionCount === 0 && filename.includes("admin"))) {
-		importType = "admins";
-	} else if (args.whitelist || (optionCount === 0 && filename.includes("whitelist"))) {
-		importType = "whitelist";
-	} else {
-		throw new lib.CommandError("Unknown json file, please specify an option");
-	}
-
-	const backup = await control.send(
-		new lib.UserBulkImportRequest(importType, importType === "users" ? data.users : data, restore)
-	);
-
-	print(`${restore ? "Restored" : "Imported"} ${importType} from ${args.filepath}`);
-	if (restore && backup && !args.noBackup) {
-		await fs.writeJSON(`${importType}-backup.json`, backup, { spaces: 2 });
-		print(`Wrote backup to ${importType}-backup.json`);
-	}
-}
-
-userCommands.add(new lib.Command({
-	definition: ["import <filepath>", "Import user data from a file", (yargs) => {
-		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
-		yargs.options({
-			"users": { describe: "Import users json", nargs: 0, type: "boolean", default: false },
-			"bans": { describe: "Import banlist json", nargs: 0, type: "boolean", default: false },
-			"admins": { describe: "Import adminlist json", nargs: 0, type: "boolean", default: false },
-			"whitelist": { describe: "Import whitelist json", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: any, control: any) {
-		await handleImportOrRestore(args, control, false);
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["restore <filepath>", "Restore user data from a file", (yargs) => {
-		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
-		yargs.options({
-			"users": { describe: "Restore users json", nargs: 0, type: "boolean", default: false },
-			"bans": { describe: "Restore banlist json", nargs: 0, type: "boolean", default: false },
-			"admins": { describe: "Restore adminlist json", nargs: 0, type: "boolean", default: false },
-			"whitelist": { describe: "Restore whitelist json", nargs: 0, type: "boolean", default: false },
-			"no-backup": { describe: "Don't save a backup to the cwd", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: any, control: any) {
-		await handleImportOrRestore(args, control, true);
-	},
-}));
-
-userCommands.add(new lib.Command({
-	definition: ["export <filepath>", "Export user data to a file", (yargs) => {
-		yargs.positional("filepath", { describe: "Path to the file to save to", type: "string" });
-		yargs.options({
-			"users": { describe: "Export users json", nargs: 0, type: "boolean", default: false },
-			"bans": { describe: "Export banlist json", nargs: 0, type: "boolean", default: false },
-			"admins": { describe: "Export adminlist json", nargs: 0, type: "boolean", default: false },
-			"whitelist": { describe: "Export whitelist json", nargs: 0, type: "boolean", default: false },
-		});
-	}],
-	handler: async function(args: {
-		filepath: string,
-		users: boolean,
-		bans: boolean,
-		admins: boolean,
-		whitelist: boolean,
-	}, control: Control) {
-		let exportType = "users" as ConstructorParameters<typeof lib.UserBulkExportRequest>[0];
-		const optionCount = [args.bans, args.admins, args.whitelist]
-			.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
-
-		if (optionCount > 1) {
-			throw new lib.CommandError("Can not specify multiple options");
-		}
-
-		// Assign based on options or attempt to guess based on filename
-		if (args.users) {
-			exportType = "users";
-		} else if (args.bans) {
-			exportType = "bans";
-		} else if (args.admins) {
-			exportType = "admins";
-		} else if (args.whitelist) {
-			exportType = "whitelist";
-		}
-
-		await fs.writeJSON(args.filepath, await control.send(
-			new lib.UserBulkExportRequest(exportType)
-		), { spaces: 2 });
-		print(`Exported ${exportType} to ${args.filepath}`);
-	},
-}));

packages/ctl/src/config_ops.ts

@@ -1,58 +0,0 @@
-import * as lib from "@clusterio/lib";
-
-export async function getEditor(argsEditor: string) {
-	// eslint-disable-next-line
-	return argsEditor || process.env.EDITOR || process.env.VISUAL || undefined
-	// needed for the process.env statements to not be flagged by eslint
-	// priority for editors is CLI argument > env.EDITOR > env.VISUAL
-}
-
-export async function configToKeyVal(data: string) {
-	let final: Record<string, string> = {};
-	let splitData = data.split(/\r?\n/);
-	// split on newlines
-	let filtered = splitData.filter((value) => value[0] !== "#").filter((a) => a);
-	// the last filter removes empty elements left by the first. Not done on one line due to readability.
-	for (let index in filtered) {
-		if (index in filtered) {
-			let split = filtered[index].split("=");
-			let finalIndex = filtered[index][0].trim();
-			// split on the = we added earlier, giving us both value and key
-			let part = "";
-			try {
-				part = split[1].trim();
-				// it's a string if we can read it
-			} catch (err) {
-				// if we can't read it, it's a empty field and therefor null
-				part = "";
-			}
-			final[finalIndex] = part;
-		}
-	}
-	return final;
-}
-
-export function serializedConfigToString(
-	serializedConfig: lib.ConfigSchema,
-	configGroup: typeof lib.ControllerConfig | typeof lib.HostConfig | typeof lib.InstanceConfig,
-	disallowedList: Record<string, unknown>,
-) {
-	let allConfigElements = "";
-	for (let [name, value] of Object.entries(serializedConfig)) {
-		if (name in disallowedList) {
-			continue;
-		}
-		let desc = "";
-		try {
-			desc += (configGroup.fieldDefinitions as any)[name]!.description;
-		} catch (err) {
-			desc += "No description found";
-		}
-		// split onto two lines for readability and es-lint
-		if (String(value) === "null") {
-			value = "";
-		}
-		allConfigElements += `${name} = ${value}\n\n`;
-	}
-	return allConfigElements;
-}

packages/host/host.ts

@@ -51,11 +51,13 @@ void new lib.Gauge(
 export class HostConnector extends lib.WebSocketClientConnector {
 	constructor(
 		public hostConfig: lib.HostConfig,
+		tlsCa: string | undefined,
 		public pluginInfos: lib.PluginNodeEnvInfo[]
 	) {
 		super(
 			hostConfig.get("host.controller_url"),
 			hostConfig.get("host.max_reconnect_delay"),
+			tlsCa
 		);
 	}
 
@@ -228,10 +230,17 @@ async function startHost() {
 		return;
 	}
 
-	let hostConnector = new HostConnector(hostConfig, pluginInfos);
+	let tlsCa: string | undefined;
+	let tlsCaPath = hostConfig.get("host.tls_ca");
+	if (tlsCaPath) {
+		tlsCa = await fs.readFile(tlsCaPath, "utf8");
+	}
+
+	let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos);
 	host = new Host(
 		hostConnector,
 		hostConfig,
+		tlsCa,
 		pluginInfos,
 		Boolean(args.canRestart),
 		Boolean(args.recovery),
@@ -282,7 +291,6 @@ async function startHost() {
 	});
 
 	await hostConnector.connect();
-	host.startedAtMs = Date.now();
 	logger.info("Started host");
 }
 
@@ -299,7 +307,7 @@ I           version of clusterio.  Expect things to break. I
 		if (err instanceof lib.WebSocketError) {
 			// Already handled by connector.on("error") within Host
 
-		} else if (err instanceof lib.StartupError || err.code === "InstallationError") {
+		} else if (err instanceof lib.StartupError) {
 			logger.fatal(`
 +----------------------------------+
 | Unable to to start clusteriohost |

packages/host/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio host server",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.23",
+	"version": "2.0.0-alpha.22b",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -26,6 +26,7 @@
 		"set-blocking": "catalog:",
 		"fs-extra": "catalog:",
 		"jszip": "catalog:",
+		"phin": "catalog:",
 		"winston": "catalog:",
 		"winston-daily-rotate-file": "catalog:",
 		"jimp": "^0.22.8",

packages/host/src/Host.ts

@@ -3,6 +3,7 @@ import path from "path";
 import events from "events";
 import pidusage from "pidusage";
 import setBlocking from "set-blocking";
+import phin from "phin";
 import stream from "stream";
 import util from "util";
 
@@ -31,7 +32,7 @@ function checkRequestSaveName(name: string) {
  *
  * Looks through all sub-dirs of the provided directory for valid
  * instance definitions and return a mapping of instance id to
- * instance configs and the root instance paths.
+ * instanceInfo objects.
  *
  * @param instancesDir - Directory containing instances
  * @returns
@@ -39,7 +40,7 @@ function checkRequestSaveName(name: string) {
  * @internal
  */
 async function discoverInstances(instancesDir: string) {
-	const instances = new Map<number, UnloadedInstance>();
+	let instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
 	for (let entry of await fs.readdir(instancesDir, { withFileTypes: true })) {
 		if (entry.isDirectory()) {
 			let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
@@ -59,25 +60,21 @@ async function discoverInstances(instancesDir: string) {
 				continue;
 			}
 
-			if (instances.has(instanceConfig.get("instance.id"))) {
+			if (instanceInfos.has(instanceConfig.get("instance.id"))) {
 				logger.warn(`Ignoring instance with duplicate id in folder ${entry.name}`);
 				continue;
 			}
 
 			logger.verbose(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`);
-			instances.set(instanceConfig.get("instance.id"), {
+			instanceInfos.set(instanceConfig.get("instance.id"), {
 				path: instancePath,
 				config: instanceConfig,
 			});
 		}
 	}
-	return instances;
+	return instanceInfos;
 }
 
-// Requests that wake up instances are responsible for making sure the
-// instance status is correctly updated even if they fail with an error. This
-// means they should typically use try catch on anything that can throw
-// and call notifyExit upon any errors.
 const instanceStartingMessages = new Set([
 	lib.InstanceStartRequest.name,
 	lib.InstanceLoadScenarioRequest.name,
@@ -117,10 +114,10 @@ export class HostRouter {
 			dst.type === lib.Address.controller
 			|| dst.type === lib.Address.control
 			|| dst.type === lib.Address.host && dst.id !== this.host.config.get("host.id")
-			|| dst.type === lib.Address.instance && !this.host.assignedInstances.has(dst.id)
+			|| dst.type === lib.Address.instance && !this.host.instanceInfos.has(dst.id)
 		) {
 			nextHop = this.host;
-		} else if (dst.type === lib.Address.instance && this.host.assignedInstances.has(dst.id)) {
+		} else if (dst.type === lib.Address.instance && this.host.instanceInfos.has(dst.id)) {
 			nextHop = this.host.instanceConnections.get(dst.id)!;
 		}
 
@@ -247,12 +244,6 @@ export class HostRouter {
 	}
 }
 
-// Instance information required to location and load an instance
-type UnloadedInstance = {
-	path: string,
-	config: lib.InstanceConfig,
-};
-
 /**
  * Handles running the host
  *
@@ -262,18 +253,19 @@ export default class Host extends lib.Link {
 	declare "connector": HostConnector;
 
 	router = new HostRouter(this);
+	/**
+	 * Certificate authority used to validate TLS connections to the controller.
+	 */
+	tlsCa?: string;
 	pluginInfos: lib.PluginNodeEnvInfo[];
 	config: lib.HostConfig;
 
 	/** Mapping of plugin name to loaded plugin */
 	plugins: Map<string, BaseHostPlugin> = new Map();
 
-	/** A map from instance id to instance connection. Only present when instance is running. */
 	instanceConnections = new Map<number, InstanceConnection>();
-	/** A map of instances that exist in the instance directory of the host */
-	discoveredInstances = new Map<number, UnloadedInstance>();
-	/** A map of instances that are assigned to this host */
-	assignedInstances = new Map<number, UnloadedInstance>();
+	discoveredInstanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
+	instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
 
 	adminlist = new Set<string>();
 	banlist = new Map<string, string>();
@@ -286,8 +278,6 @@ export default class Host extends lib.Link {
 	_disconnecting = false;
 	_shuttingDown = false;
 
-	startedAtMs = Date.now();
-
 	static instanceConfigWarning = {
 		"_warning": "Changes to this file will be overwritten by the controller's copy.",
 	};
@@ -303,6 +293,7 @@ export default class Host extends lib.Link {
 	constructor(
 		connector: HostConnector,
 		hostConfig: lib.HostConfig,
+		tlsCa: string | undefined,
 		pluginInfos: lib.PluginNodeEnvInfo[],
 		/**
 		 * If true indicates that there is a process monitor present that
@@ -317,6 +308,7 @@ export default class Host extends lib.Link {
 		public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
 	) {
 		super(connector);
+		this.tlsCa = tlsCa;
 
 		this.pluginInfos = pluginInfos;
 		this.config = hostConfig;
@@ -444,9 +436,6 @@ export default class Host extends lib.Link {
 				this.plugins.set(pluginInfo.name, hostPlugin);
 
 			} catch (err: any) {
-				if (err.code === "InstallationError") {
-					throw err;
-				}
 				throw new lib.PluginError(pluginInfo.name, err);
 			}
 
@@ -591,17 +580,18 @@ export default class Host extends lib.Link {
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		let response = await fetch(url);
-		if (!response.body) {
-			throw new Error(`Empty body downloading stream ${streamId} from controller`);
-		}
+		let response = await phin({
+			url, method: "GET",
+			core: { ca: this.tlsCa } as object,
+			stream: true,
+		});
 
 		const file = lib.ModInfo.filename(mod.name, mod.version);
 		const filePath = path.join(this.modStore.modsDirectory, file);
 		const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip");
 		const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" });
 		await events.once(writeStream, "open");
-		stream.Readable.fromWeb(response.body).pipe(writeStream);
+		response.pipe(writeStream);
 		await finished(writeStream);
 		await fs.rename(tempFilePath, filePath);
 
@@ -660,16 +650,16 @@ export default class Host extends lib.Link {
 
 	async handleInstanceAssignInternalRequest(request: lib.InstanceAssignInternalRequest) {
 		let { instanceId, config } = request;
-		let instance = this.assignedInstances.get(instanceId);
-		if (instance) {
-			instance.config.update(config, true, "controller");
-			logger.verbose(`Updated config for ${instance.path}`, this.instanceLogMeta(instanceId, instance));
+		let instanceInfo = this.instanceInfos.get(instanceId);
+		if (instanceInfo) {
+			instanceInfo.config.update(config, true, "controller");
+			logger.verbose(`Updated config for ${instanceInfo.path}`, this.instanceLogMeta(instanceId, instanceInfo));
 
 		} else {
-			instance = this.discoveredInstances.get(instanceId);
-			if (instance) {
-				await Instance.populate_folders(instance.path);
-				instance.config.update(config, true, "controller");
+			instanceInfo = this.discoveredInstanceInfos.get(instanceId);
+			if (instanceInfo) {
+				await Instance.populate_folders(instanceInfo.path);
+				instanceInfo.config.update(config, true, "controller");
 
 			} else {
 				let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
@@ -680,16 +670,16 @@ export default class Host extends lib.Link {
 
 				logger.info(`Creating ${instanceDir}`);
 				await Instance.populate_folders(instanceDir);
-				instance = {
+				instanceInfo = {
 					path: instanceDir,
 					config: instanceConfig,
 				};
 
-				this.discoveredInstances.set(instanceId, instance);
+				this.discoveredInstanceInfos.set(instanceId, instanceInfo);
 			}
 
-			this.assignedInstances.set(instanceId, instance);
-			logger.verbose(`assigned instance ${instance.config.get("instance.name")}`);
+			this.instanceInfos.set(instanceId, instanceInfo);
+			logger.verbose(`assigned instance ${instanceInfo.config.get("instance.name")}`);
 		}
 
 		// Somewhat hacky, but in the event of a lost session the status is
@@ -704,45 +694,45 @@ export default class Host extends lib.Link {
 				instanceConnection
 					? instanceConnection.instance.server.version
 					// TODO: factorio.version is not validated
-					: instance.config.get("factorio.version") as lib.TargetVersion,
+					: instanceInfo.config.get("factorio.version") as lib.TargetVersion,
 			)
 		);
 
 		// Send the new list of saves for this assigned instance to the controller.
-		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
 
 		// save a copy of the instance config
-		await instance.config.save();
+		await instanceInfo.config.save();
 	}
 
 	async handleInstanceUnassignInternalRequest(request: lib.InstanceUnassignInternalRequest) {
 		let instanceId = request.instanceId;
-		let instance = this.assignedInstances.get(instanceId);
-		if (instance) {
+		let instanceInfo = this.instanceInfos.get(instanceId);
+		if (instanceInfo) {
 			let instanceConnection = this.instanceConnections.get(instanceId);
 			if (instanceConnection && ["starting", "running"].includes(instanceConnection.instance.status)) {
 				await instanceConnection.send(new lib.InstanceStopRequest());
 			}
 
-			this.assignedInstances.delete(instanceId);
-			logger.verbose(`unassigned instance ${instance.config.get("instance.name")}`);
+			this.instanceInfos.delete(instanceId);
+			logger.verbose(`unassigned instance ${instanceInfo.config.get("instance.name")}`);
 		}
 	}
 
-	instanceLogMeta(instanceId: number, instance?: { config: lib.InstanceConfig }) {
-		instance = instance || this.assignedInstances.get(instanceId);
-		if (!instance) {
+	instanceLogMeta(instanceId: number, instanceInfo?: { config: lib.InstanceConfig }) {
+		instanceInfo = instanceInfo || this.instanceInfos.get(instanceId);
+		if (!instanceInfo) {
 			return { instance_id: instanceId, instance_name: String(instanceId) };
 		}
-		return { instance_id: instanceId, instance_name: instance.config.get("instance.name") };
+		return { instance_id: instanceId, instance_name: instanceInfo.config.get("instance.name") };
 	}
 
-	getRequestInstance(instanceId: number) {
-		let instance = this.assignedInstances.get(instanceId);
-		if (!instance) {
+	getRequestInstanceInfo(instanceId: number) {
+		let instanceInfo = this.instanceInfos.get(instanceId);
+		if (!instanceInfo) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
 		}
-		return instance;
+		return instanceInfo;
 	}
 
 	/**
@@ -751,7 +741,7 @@ export default class Host extends lib.Link {
 	 * @returns Assigned game port or undefined if it does not exist.
 	 */
 	gamePort(instanceId: number) {
-		const instance = this.discoveredInstances.get(instanceId);
+		const instance = this.discoveredInstanceInfos.get(instanceId);
 		if (!instance) {
 			return undefined;
 		}
@@ -764,7 +754,7 @@ export default class Host extends lib.Link {
 
 	assignGamePort(instanceId: number) {
 		const availablePorts = lib.parseRanges(this.config.get("host.factorio_port_range"), 1, 2 ** 16 - 1);
-		for (const [id, instance] of this.assignedInstances) {
+		for (const [id, instance] of this.instanceInfos) {
 			if (id === instanceId) {
 				continue;
 			}
@@ -776,7 +766,7 @@ export default class Host extends lib.Link {
 				availablePorts.delete(port);
 			}
 		}
-		const instanceToAssign = this.assignedInstances.get(instanceId)!;
+		const instanceToAssign = this.instanceInfos.get(instanceId)!;
 		const assignedPort = instanceToAssign.config.get("factorio.host_assigned_game_port");
 		if (assignedPort !== null && availablePorts.has(assignedPort)) {
 			return assignedPort;
@@ -796,7 +786,7 @@ export default class Host extends lib.Link {
 	 * @returns connection to instance.
 	 */
 	async _connectInstance(instanceId: number) {
-		let unloadedInstance = this.getRequestInstance(instanceId);
+		let instanceInfo = this.getRequestInstanceInfo(instanceId);
 		if (this.instanceConnections.has(instanceId)) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
 		}
@@ -805,9 +795,7 @@ export default class Host extends lib.Link {
 		let instanceAddress = new lib.Address(lib.Address.instance, instanceId);
 		let [connectionClient, connectionServer] = lib.VirtualConnector.makePair(instanceAddress, hostAddress);
 		let instance = new Instance(
-			this, connectionClient,
-			unloadedInstance.path, this.config.get("host.factorio_directory"),
-			unloadedInstance.config
+			this, connectionClient, instanceInfo.path, this.config.get("host.factorio_directory"), instanceInfo.config
 		);
 		let instanceConnection = new InstanceConnection(connectionServer, this, instance);
 
@@ -819,7 +807,7 @@ export default class Host extends lib.Link {
 
 	async handleSystemInfoRequest() {
 		const restartRequired = await this.checkRestartRequired();
-		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, restartRequired, this.startedAtMs);
+		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, restartRequired);
 	}
 
 	async checkRestartRequired() {
@@ -906,19 +894,19 @@ export default class Host extends lib.Link {
 		src: lib.Address,
 		dst: lib.Address
 	) {
-		let instance = this.getRequestInstance(dst.id);
-		return await Instance.listSaves(dst.id, path.join(instance.path, "saves"), null);
+		let instanceInfo = this.getRequestInstanceInfo(dst.id);
+		return await Instance.listSaves(dst.id, path.join(instanceInfo.path, "saves"), null);
 	}
 
 	async handleInstanceRenameSaveRequest(request: lib.InstanceRenameSaveRequest) {
 		let { instanceId, oldName, newName } = request;
 		checkRequestSaveName(oldName);
 		checkRequestSaveName(newName);
-		let instance = this.getRequestInstance(instanceId);
+		let instanceInfo = this.getRequestInstanceInfo(instanceId);
 		try {
 			await fs.move(
-				path.join(instance.path, "saves", oldName),
-				path.join(instance.path, "saves", newName),
+				path.join(instanceInfo.path, "saves", oldName),
+				path.join(instanceInfo.path, "saves", newName),
 				{ overwrite: false },
 			);
 		} catch (err: any) {
@@ -927,18 +915,18 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
 	}
 
 	async handleInstanceCopySaveRequest(request: lib.InstanceCopySaveRequest) {
 		let { instanceId, source, destination } = request;
 		checkRequestSaveName(source);
 		checkRequestSaveName(destination);
-		let instance = this.getRequestInstance(instanceId);
+		let instanceInfo = this.getRequestInstanceInfo(instanceId);
 		try {
 			await fs.copy(
-				path.join(instance.path, "saves", source),
-				path.join(instance.path, "saves", destination),
+				path.join(instanceInfo.path, "saves", source),
+				path.join(instanceInfo.path, "saves", destination),
 				{ overwrite: false, errorOnExist: true },
 			);
 		} catch (err: any) {
@@ -947,33 +935,33 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
 	}
 
 	async handleInstanceTransferSaveRequest(request: lib.InstanceTransferSaveRequest) {
 		let { sourceName, targetName, copy, sourceInstanceId, targetInstanceId } = request;
 		checkRequestSaveName(sourceName);
 		checkRequestSaveName(targetName);
-		let sourceInstance = this.getRequestInstance(sourceInstanceId);
-		let targetInstance = this.getRequestInstance(targetInstanceId);
+		let sourceInstanceInfo = this.getRequestInstanceInfo(sourceInstanceId);
+		let targetInstanceInfo = this.getRequestInstanceInfo(targetInstanceId);
 
 		// For consistency with remote transfer initiated through pullSave the
 		// target is renamed if it already exists.
 		targetName = await lib.findUnusedName(
-			path.join(targetInstance.path, "saves"), targetName, ".zip"
+			path.join(targetInstanceInfo.path, "saves"), targetName, ".zip"
 		);
 
 		try {
 			if (copy) {
 				await fs.copy(
-					path.join(sourceInstance.path, "saves", sourceName),
-					path.join(targetInstance.path, "saves", targetName),
+					path.join(sourceInstanceInfo.path, "saves", sourceName),
+					path.join(targetInstanceInfo.path, "saves", targetName),
 					{ overwrite: true },
 				);
 			} else {
 				await fs.move(
-					path.join(sourceInstance.path, "saves", sourceName),
-					path.join(targetInstance.path, "saves", targetName),
+					path.join(sourceInstanceInfo.path, "saves", sourceName),
+					path.join(targetInstanceInfo.path, "saves", targetName),
 					{ overwrite: true },
 				);
 			}
@@ -983,8 +971,8 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(sourceInstanceId, path.join(sourceInstance.path, "saves"));
-		await this.sendSaveListUpdate(targetInstanceId, path.join(targetInstance.path, "saves"));
+		await this.sendSaveListUpdate(sourceInstanceId, path.join(sourceInstanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(targetInstanceId, path.join(targetInstanceInfo.path, "saves"));
 
 		return targetName;
 	}
@@ -1004,34 +992,38 @@ export default class Host extends lib.Link {
 	async handleInstanceDeleteSaveRequest(request: lib.InstanceDeleteSaveRequest) {
 		let { instanceId, name } = request;
 		checkRequestSaveName(name);
-		let instance = this.getRequestInstance(instanceId);
+		let instanceInfo = this.getRequestInstanceInfo(instanceId);
 
 		try {
-			await fs.unlink(path.join(instance.path, "saves", name));
+			await fs.unlink(path.join(instanceInfo.path, "saves", name));
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
 				throw new lib.RequestError(`${name} does not exist`);
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
 	}
 
 	async handleInstancePullSaveRequest(request: lib.InstancePullSaveRequest) {
 		let { instanceId, streamId, name } = request;
 		checkRequestSaveName(name);
-		let instance = this.getRequestInstance(instanceId);
+		let instanceInfo = this.getRequestInstanceInfo(instanceId);
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		let response = await fetch(url);
+		let response = await phin({
+			url, method: "GET",
+			core: { ca: this.tlsCa } as object,
+			stream: true,
+		});
 
-		if (response.status !== 200 || !response.body) {
-			let body = Buffer.from(await response.arrayBuffer());
-			throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`);
+		if (response.statusCode !== 200) {
+			let content = await lib.readStream(response);
+			throw new lib.RequestError(`Stream returned ${response.statusCode}: ${content.toString()}`);
 		}
 
-		let savesDir = path.join(instance.path, "saves");
+		let savesDir = path.join(instanceInfo.path, "saves");
 		let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip");
 		let writeStream: NodeJS.WritableStream;
 		while (true) {
@@ -1047,7 +1039,7 @@ export default class Host extends lib.Link {
 				}
 			}
 		}
-		stream.Readable.fromWeb(response.body).pipe(writeStream);
+		response.pipe(writeStream);
 		await finished(writeStream);
 
 		name = await lib.findUnusedName(savesDir, name, ".zip");
@@ -1060,13 +1052,12 @@ export default class Host extends lib.Link {
 	async handleInstancePushSaveRequest(request: lib.InstancePushSaveRequest) {
 		let { instanceId, streamId, name } = request;
 		checkRequestSaveName(name);
-		let instance = this.getRequestInstance(instanceId);
+		let instanceInfo = this.getRequestInstanceInfo(instanceId);
 
-		let content = fs.createReadStream(path.join(instance.path, "saves", name));
+		let content: Buffer;
 		try {
-			// TODO: Due to a bug in fs-extra the ready event is not emitted here. Replace
-			// "open" with "ready" when fs-extra is replaced with node's fs module.
-			await events.once(content, "open");
+			// phin doesn't support streaming requests :(
+			content = await fs.readFile(path.join(instanceInfo.path, "saves", name));
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
 				throw new lib.RequestError(`${name} does not exist`);
@@ -1076,10 +1067,10 @@ export default class Host extends lib.Link {
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		fetch(url, {
-			method: "PUT",
-			body: content,
-			duplex: "half",
+		phin({
+			url, method: "PUT",
+			core: { ca: this.tlsCa } as object,
+			data: content,
 		}).catch(err => {
 			logger.error(`Error pushing save to controller:\n${err.stack}`, this.instanceLogMeta(instanceId));
 		});
@@ -1091,14 +1082,14 @@ export default class Host extends lib.Link {
 			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
 		}
 
-		let instance = this.discoveredInstances.get(instanceId);
-		if (!instance) {
+		let instanceInfo = this.discoveredInstanceInfos.get(instanceId);
+		if (!instanceInfo) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
 		}
 
-		this.discoveredInstances.delete(instanceId);
-		this.assignedInstances.delete(instanceId);
-		await fs.remove(instance.path);
+		this.discoveredInstanceInfos.delete(instanceId);
+		this.instanceInfos.delete(instanceId);
+		await fs.remove(instanceInfo.path);
 	}
 
 	async handleHostUpdateRequest(request: lib.HostUpdateRequest) {
@@ -1148,12 +1139,12 @@ export default class Host extends lib.Link {
 	 * the host and controller with the new list of instances.
 	 */
 	async updateInstances() {
-		this.discoveredInstances = await discoverInstances(this.config.get("host.instances_directory"));
+		this.discoveredInstanceInfos = await discoverInstances(this.config.get("host.instances_directory"));
 		let list = [];
-		for (let [instanceId, instance] of this.discoveredInstances) {
+		for (let [instanceId, instanceInfo] of this.discoveredInstanceInfos) {
 			let instanceConnection = this.instanceConnections.get(instanceId);
 			list.push(new lib.HostInstanceUpdate(
-				instance.config.toRemote("controller"),
+				instanceInfo.config.toRemote("controller"),
 				instanceConnection ? instanceConnection.instance.status : "stopped",
 				instanceConnection ? instanceConnection.instance.server.gamePort : this.gamePort(instanceId),
 				instanceConnection ? instanceConnection.instance.server.version : undefined,
@@ -1165,10 +1156,10 @@ export default class Host extends lib.Link {
 		if (this._startup) {
 			this._startup = false;
 
-			for (let [instanceId, instance] of this.assignedInstances) {
-				if (instance.config.get("instance.auto_start")) {
+			for (let [instanceId, instanceInfo] of this.instanceInfos) {
+				if (instanceInfo.config.get("instance.auto_start")) {
 					if (this.recoveryMode) {
-						logger.warn(`Recovery | skipping auto startup for ${instance.config.get("instance.name")}`);
+						logger.warn(`Recovery | skipping auto startup for ${instanceInfo.config.get("instance.name")}`);
 						continue;
 					}
 					try {
@@ -1176,8 +1167,8 @@ export default class Host extends lib.Link {
 						await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
 					} catch (err: any) {
 						logger.error(
-							`Error during auto startup for ${instance.config.get("instance.name")}:\n${err.stack}`,
-							this.instanceLogMeta(instanceId, instance)
+							`Error during auto startup for ${instanceInfo.config.get("instance.name")}:\n${err.stack}`,
+							this.instanceLogMeta(instanceId, instanceInfo)
 						);
 					}
 				}
@@ -1225,7 +1216,7 @@ export default class Host extends lib.Link {
 		await this.config.save();
 		// Save host side in case host_assigned_game_port changed in the config.
 		await Promise.all(
-			[...this.assignedInstances.values()].map(instance => instance.config.save())
+			[...this.instanceInfos.values()].map(instanceInfo => instanceInfo.config.save())
 		);
 
 		try {

packages/host/src/Instance.ts

@@ -1,6 +1,7 @@
 import fs from "fs-extra";
 import path from "path";
 import pidusage from "pidusage";
+import phin from "phin";
 import util from "util";
 import type { Static } from "@sinclair/typebox";
 import { exec } from "child_process";
@@ -637,12 +638,9 @@ end`.replace(/\r?\n/g, " ");
 	}
 
 	async init(pluginInfos: lib.PluginNodeEnvInfo[]) {
-		// After changing the status this function is responsible for calling notifyExit if it throws.
 		this.notifyStatus("starting");
 		try {
-			const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest());
 			await this._loadStats();
-			await this.server.checkForUpdates(factorioVersions);
 			await this.server.init();
 		} catch (err) {
 			this.notifyExit();
@@ -845,7 +843,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_adminlist") !== "disabled") {
 			this.logger.verbose("Writing server-adminlist.json");
-			await lib.safeOutputFile(
+			lib.safeOutputFile(
 				this.server.writePath("server-adminlist.json"),
 				JSON.stringify([...this._host.adminlist], null, "\t")
 			);
@@ -853,7 +851,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_banlist") !== "disabled") {
 			this.logger.verbose("Writing server-banlist.json");
-			await lib.safeOutputFile(
+			lib.safeOutputFile(
 				this.server.writePath("server-banlist.json"),
 				JSON.stringify([...this._host.banlist].map(
 					([username, reason]) => ({ username, reason })
@@ -863,21 +861,10 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_whitelist") !== "disabled") {
 			this.logger.verbose("Writing server-whitelist.json");
-			await lib.safeOutputFile(
+			lib.safeOutputFile(
 				this.server.writePath("server-whitelist.json"),
 				JSON.stringify([...this._host.whitelist], null, "\t")
 			);
-		} else {
-			// Always create the whitelist even if unused
-			// This allows it to be watched for changes without a server restart
-			try {
-				// wx will throw an error if the file already exists to prevent overwrite
-				await fs.writeFile(this.server.writePath("server-whitelist.json"), "\n[]\n", { flag: "wx" });
-			} catch (err: any) {
-				if (err.code !== "EEXIST") {
-					throw err;
-				}
-			}
 		}
 
 		await this.syncMods();
@@ -1303,18 +1290,17 @@ end`.replace(/\r?\n/g, " ");
 			let url = new URL(this._host.config.get("host.controller_url"));
 			url.pathname += "api/upload-export";
 			url.searchParams.set("mod_pack_id", String(this.activeModPack.id));
-			let response = await fetch(url, {
-				method: "PUT",
-				body: content,
-				duplex: "half",
+			let response = await phin({
+				url, method: "PUT",
+				data: content,
+				core: { ca: this._host.tlsCa } as object,
 				headers: {
 					"Content-Type": "application/zip",
 					"x-access-token": this._host.config.get("host.controller_token"),
 				},
 			});
-			if (response.status !== 200) {
-				const body = Buffer.from(await response.arrayBuffer());
-				throw Error(`Upload failed: ${response.status} ${response.statusText}: ${body.toString()}`);
+			if (response.statusCode !== 200) {
+				throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`);
 			}
 
 		} finally {

packages/host/src/export.ts

@@ -98,8 +98,7 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
 async function loadFile(server: FactorioServer, modVersions: Map<string, string>, modPath: string) {
 	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
 	if (!match) {
-		server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`);
-		return null;
+		throw new Error(`Bad mod path ${match}`);
 	}
 
 	let [, mod, filePath] = match;
@@ -152,6 +151,7 @@ async function loadIcon(
 			iconCache.set(iconPath, icon);
 		} else {
 			icon = null;
+			server._logger.warn(`${iconPath} not found`);
 		}
 		iconCache.set(iconPath, icon);
 	}
@@ -258,14 +258,6 @@ function fixIcons(item: ItemPrototype) {
 	return item;
 }
 
-/** Get the primary icon path from a prototype (first layer for layered, .icon for simple). */
-function getPrimaryIconPath(item: ItemPrototype): string | undefined {
-	if (item.icons) {
-		return (item as LayeredIconSpecification).icons[0]?.icon;
-	}
-	return (item as SimpleIconSpecification).icon;
-}
-
 const itemTypes = new Set([
 	"item",
 	"ammo",
@@ -288,153 +280,87 @@ const itemTypes = new Set([
 	"armor",
 	"mining-tool",
 	"repair-tool",
+
 	// XXX Bad hack to get icons for fluids. These should be treated as a
 	// separate namespace from items as they may be named the same as items.
 	"fluid",
+	// TODO consider adding virtual signals and recipes
 ]);
 
-const recipeTypes = new Set(["recipe"]);
-const signalTypes = new Set(["virtual-signal"]);
-const technologyTypes = new Set(["technology"]);
-const planetTypes = new Set(["planet", "space-location"]);
-const qualityTypes = new Set(["quality"]);
-
-// Entity types to export. Entities whose name matches an already-exported
-const entityTypes = new Set([
-	"assembling-machine", "furnace", "inserter", "transport-belt", "underground-belt",
-	"splitter", "loader", "loader-1x1", "mining-drill", "container", "logistic-container",
-	"electric-pole", "pipe", "pipe-to-ground", "pump", "storage-tank", "boiler",
-	"generator", "solar-panel", "accumulator", "reactor", "heat-pipe", "lab", "beacon",
-	"roboport", "lamp", "radar", "wall", "gate", "land-mine", "turret", "ammo-turret",
-	"electric-turret", "fluid-turret", "artillery-turret", "rocket-silo", "car",
-	"spider-vehicle", "locomotive", "cargo-wagon", "fluid-wagon", "artillery-wagon",
-	"train-stop", "rail-signal", "rail-chain-signal", "straight-rail", "curved-rail-a",
-	"curved-rail-b", "half-diagonal-rail", "elevated-straight-rail", "elevated-curved-rail-a",
-	"elevated-curved-rail-b", "elevated-half-diagonal-rail", "rail-ramp", "rail-support",
-	"space-platform-hub", "cargo-bay", "asteroid-collector", "thruster", "cargo-pod",
-	"cargo-landing-pad", "asteroid",
-	"construction-robot", "logistic-robot", "combat-robot", "capture-robot",
-	"unit", "unit-spawner", "spider-unit", "segmented-unit",
-	"tree", "plant", "fish", "resource", "cliff", "simple-entity",
-	"simple-entity-with-owner", "simple-entity-with-force",
-	"arithmetic-combinator", "decider-combinator", "constant-combinator",
-	"selector-combinator", "programmable-speaker", "power-switch", "display-panel",
-	"character",
-	"offshore-pump", "valve", "lightning-attractor",
-	"fusion-generator", "fusion-reactor", "burner-generator",
-	"agricultural-tower", "market", "lane-splitter",
-]);
-
-function filterPrototypes(prototypes: Prototypes, allowedTypes: Set<string>): ItemPrototype[] {
+function filterItems(prototypes: Prototypes): ItemPrototype[] {
 	return Object.entries(prototypes)
-		.filter(([type]) => allowedTypes.has(type))
+		.filter(([type, _]) => itemTypes.has(type))
 		.flatMap(([_, typePrototypes]) => Object.values(typePrototypes) as ItemPrototype[])
-		.map(fixIcons);
-}
-
-/** Shared state for building a unified spritesheet across all categories. */
-interface SheetState {
-	sheet: Jimp;
-	metadata: Map<string, { x: number, y: number, size: number, category: string, path?: string }>;
-	iconCache: IconCache;
-	simpleIcons: Map<string, number>;
-	pos: number;
-}
-
-const SHEET_ICON_SIZE = 32;
-const SHEET_WIDTH = 1024;
-
-async function createSheetState(): Promise<SheetState> {
-	return {
-		sheet: await Jimp.create(SHEET_WIDTH, SHEET_ICON_SIZE),
-		metadata: new Map(),
-		iconCache: new Map(),
-		simpleIcons: new Map(),
-		pos: 0,
-	};
-}
-
-/** Grow the spritesheet vertically if needed to fit another row. */
-async function growSheet(state: SheetState) {
-	const row = Math.floor(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
-	const neededHeight = (row + 1) * SHEET_ICON_SIZE;
-	if (neededHeight > state.sheet.bitmap.height) {
-		const extended = await Jimp.create(SHEET_WIDTH, neededHeight);
-		extended.composite(state.sheet, 0, 0);
-		(state.sheet as any).bitmap = (extended as any).bitmap;
-	}
+		.map(fixIcons)
+	;
 }
 
 /**
- * Export item icons for a single category onto the shared spritesheet.
+ * Export item icons and data
  *
- * @param server - The server to load icons from.
- * @param modVersions - Mapping of mod name to version.
- * @param items - Array of item prototypes to export.
- * @param category - Category name for metadata.
- * @param state - Shared spritesheet state (mutated in place).
- * @returns Number of icons exported for this category.
+ * Assembles and packs the icons for the item prototypes given into a single
+ * spritesheet and json file with meta data.
+ *
+ * @param server -
+ *     The server to generate the export mod for.
+ * @param modVersions -
+ *     Mapping of mod name to versions to get icons from.
+ * @param items - Array of item prototypes.
+ * @return Item spritesheet and metadata.
  * @internal
  */
-async function exportItems(
-	server: FactorioServer,
-	modVersions: Map<string, string>,
-	items: ItemPrototype[],
-	category: string,
-	state: SheetState,
-) {
-	let count = 0;
-	for (let item of items) {
-		// Skip prototypes with no icon data at all
-		if (!item.icons && !item.icon) {
-			continue;
-		}
+async function exportItems(server: FactorioServer, modVersions: Map<string, string>, items: ItemPrototype[]) {
 
+	// Size to render icons at
+	let size = 32;
+
+	// Width of spritesheet
+	let width = 1024;
+
+	let rows = Math.ceil(items.length / (width / size));
+	let iconSheet = await Jimp.create(width, rows * size);
+	let itemData = new Map();
+	let pos = 0;
+
+	let iconCache: IconCache = new Map();
+	let simpleIcons = new Map();
+	for (let item of items) {
 		let icon: Jimp | null = null;
 		let iconPos: number | undefined;
 		if (item.icons) {
-			icon = await loadLayeredIcon(
-				server, modVersions, item as LayeredIconSpecification, SHEET_ICON_SIZE, state.iconCache,
-			);
-			iconPos = state.pos;
+			icon = await loadLayeredIcon(server, modVersions, item as LayeredIconSpecification, size, iconCache);
+			iconPos = pos;
 
 		} else {
-			iconPos = state.simpleIcons.get(item.icon as string);
+			iconPos = simpleIcons.get(item.icon);
 			if (iconPos === undefined) {
-				icon = await loadSimpleIcon(
-					server, modVersions, item as SimpleIconSpecification, SHEET_ICON_SIZE, state.iconCache,
-				);
+				icon = await loadSimpleIcon(server, modVersions, item as SimpleIconSpecification, size, iconCache);
 				if (icon) {
-					iconPos = state.pos;
-					state.simpleIcons.set(item.icon as string, state.pos);
+					iconPos = pos;
+					simpleIcons.set(item.icon, pos);
 				}
 			}
 		}
 
 		if (iconPos !== undefined) {
-			const iconPath = getPrimaryIconPath(item);
-			state.metadata.set(`${category}.${item.name}`, {
-				x: iconPos * SHEET_ICON_SIZE % SHEET_WIDTH,
-				y: Math.floor(iconPos / (SHEET_WIDTH / SHEET_ICON_SIZE)) * SHEET_ICON_SIZE,
-				size: SHEET_ICON_SIZE,
-				category,
-				...(iconPath ? { path: iconPath } : {}),
+			itemData.set(item.name, {
+				x: iconPos * size % width,
+				y: Math.floor(iconPos / (width / size)) * size,
+				size,
+				localised_name: item.localised_name,
+				localised_description: item.localised_description,
 			});
 		}
 
 		if (icon) {
-			await growSheet(state);
-			state.sheet.composite(
-				icon,
-				state.pos * SHEET_ICON_SIZE % SHEET_WIDTH,
-				Math.floor(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE)) * SHEET_ICON_SIZE,
-			);
-			state.pos += 1;
-			count += 1;
+			iconSheet.composite(icon, pos * size % width, Math.floor(pos / (width / size)) * size);
+			pos += 1;
 		}
 	}
 
-	return count;
+	iconSheet.crop(0, 0, width, Math.ceil(pos / (width / size)) * size);
+
+	return { iconSheet, itemData };
 }
 
 /**
@@ -579,16 +505,7 @@ export async function exportData(server: FactorioServer) {
 		}
 	}
 
-	const categories: { name: string, types: Set<string> }[] = [
-		{ name: "item", types: itemTypes },
-		{ name: "recipe", types: recipeTypes },
-		{ name: "signal", types: signalTypes },
-		{ name: "technology", types: technologyTypes },
-		{ name: "planet", types: planetTypes },
-		{ name: "quality", types: qualityTypes },
-		{ name: "entity", types: entityTypes },
-	];
-
+	let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes));
 	let locale = await exportLocale(server, modVersions, modOrder, "en");
 
 	// Free up the memory used by zip files loaded during the export.
@@ -597,28 +514,9 @@ export async function exportData(server: FactorioServer) {
 	let zip = new JSZip();
 	zip.file("export/settings.json", JSON.stringify(settings));
 	zip.file("export/prototypes.json", JSON.stringify(prototypes));
+	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
+	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
 	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
-
-	// Build a single unified spritesheet across all prototype categories.
-	const state = await createSheetState();
-	for (const category of categories) {
-		const categoryItems = filterPrototypes(prototypes, category.types);
-
-		if (categoryItems.length === 0) {
-			continue;
-		}
-		const count = await exportItems(server, modVersions, categoryItems, category.name, state);
-		server._logger.info(`Exported ${count} ${category.name} icons`);
-	}
-
-	// Crop sheet to actual used height and write single spritesheet + metadata.
-	const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
-	state.sheet.crop(0, 0, SHEET_WIDTH, Math.max(usedRows * SHEET_ICON_SIZE, 1));
-	zip.file("export/spritesheet.png", await state.sheet.getBufferAsync(Jimp.MIME_PNG));
-	zip.file("export/metadata.json", JSON.stringify([...state.metadata.entries()]));
-
-	server._logger.info(`Export complete: ${state.metadata.size} icons on ${usedRows} row(s)`);
-
 	return zip;
 }
 

packages/host/src/server.ts

@@ -2,7 +2,6 @@
 import fs from "fs-extra";
 import child_process from "child_process";
 import path from "path";
-import JSZip from "jszip";
 import events from "events";
 import util from "util";
 import crypto from "crypto";
@@ -10,33 +9,31 @@ import { Rcon } from "rcon-client";
 
 import * as lib from "@clusterio/lib";
 
-const execFileAsync = util.promisify(child_process.execFile);
 
 /**
  * Determines the version of Factorio the datadir is pointing to by
  * reading the changelog.txt in it.
  *
- * @param factorioDir - Path factorio dir containing data/changelog.txt.
+ * @param changelogPath - Path to changelog.txt.
+ * @internal
  */
-async function getFactorioVersion(factorioDir: string) {
+async function getVersion(changelogPath: string) {
 	let changelog;
-	const changelogPath = path.join(factorioDir, "data", "changelog.txt");
 	try {
 		changelog = await fs.readFile(changelogPath, "utf-8");
 	} catch (err: any) {
-		if (err.code === "ENOENT" || err.code === "ENOTDIR") {
+		if (err.code === "ENOENT") {
 			return null;
 		}
 		throw err;
 	}
 
-	for (const line of changelog.split(/[\r\n]+/)) {
-		const index = line.indexOf(":");
+	for (let line of changelog.split(/[\r\n]+/)) {
+		let index = line.indexOf(":");
 		if (index !== -1) {
-			const name = line.slice(0, index).trim();
+			let name = line.slice(0, index).trim();
 			if (name.toLowerCase() === "version") {
-				const version = line.slice(index + 1).trim();
-				return lib.isFullVersion(version) ? version : null;
+				return line.slice(index + 1).trim();
 			}
 		}
 	}
@@ -69,113 +66,13 @@ function versionOrder(a: string, b: string) {
 	return 0;
 }
 
-/**
- * Find all installed factorio versions and return them as a set
- *
- * @param factorioDir - Path to Factorio installation dirs.
- * @returns Set of installed factorio versions
- */
-async function listFactorioVersions(factorioDir: string) {
-	const versions = new Set<lib.FullVersion>();
-
-	const directVersion = await getFactorioVersion(factorioDir);
-	if (directVersion !== null) {
-		versions.add(directVersion);
-		return { direct: true, versions };
-	}
-
-	for (const name of await fs.readdir(factorioDir)) {
-		// Avoid any fancy directory detection in order to follow symlinks
-		const fullPath = path.join(factorioDir, name);
-		const version = await getFactorioVersion(fullPath);
-		if (version === null) {
-			continue;
-		}
-
-		versions.add(version);
-	}
-
-	return { direct: false, versions };
-}
-
-/**
- * Downloads a ZIP file from a URL using native fetch and extracts it
- *
- * @param url - The URL of the ZIP file
- * @param targetDir - Directory to extract the ZIP into
- * @throws Network error if status of request is not ok
- */
-async function downloadAndExtractZip(url: string, targetDir: string) {
-	const res = await fetch(url);
-	if (!res.ok) {
-		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
-	}
-
-  	// Load the ZIP with JSZip
-	const buffer = Buffer.from(await res.arrayBuffer());
-	const zip = await JSZip.loadAsync(buffer);
-
-  	// Extract each file/folder
-	await fs.ensureDir(targetDir);
-	await Promise.all(
-		Object.values(zip.files).map(async (entry) => {
-			const parts = entry.name.split("/").filter(Boolean);
-			const strippedPath = parts.slice(1).join("/");
-			if (!strippedPath) {
-				return; // This copies the behaviour of tar --strip-components 1
-			}
-
-			const entryPath = path.join(targetDir, strippedPath);
-			if (entry.dir) {
-				await fs.ensureDir(entryPath);
-			} else {
-				await fs.ensureDir(path.dirname(entryPath));
-				await fs.writeFile(entryPath, await entry.async("nodebuffer"));
-			}
-		})
-	);
-}
-
-/**
- * Downloads a tar.gz file from a URL using native fetch and extracts it
- *
- * @param url - The URL of the ZIP file
- * @param targetDir - Directory to extract the ZIP into
- * @throws Network error if status of request is not ok
- */
-async function downloadAndExtractTar(url: string, targetDir: string) {
-	const res = await fetch(url);
-	if (!res.ok) {
-		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
-	}
-
-	// Write the buffer to a temp file
-	const tmpFilePath = `${targetDir}.tmp`;
-	const buffer = Buffer.from(await res.arrayBuffer());
-	await fs.writeFile(tmpFilePath, buffer);
-	await fs.ensureDir(targetDir);
-
-	// Execute tar CLI to extract
-	try {
-		await execFileAsync("tar", ["-xf", tmpFilePath, "-C", targetDir, "--strip-components", "1"]);
-	} catch (err: any) {
-		if (err.code === "ENOENT") {
-			throw new Error("error executing tar command, do you have 'xz-utils' installed?");
-		}
-		throw err;
-	} finally {
-		// Clean up temp file
-		await fs.remove(tmpFilePath);
-	}
-}
-
 /**
  * Find factorio data directory of the given version
  *
  * Searches the given factorio dir for an installation of Factorio of
  * the target version.
  *
- * @param factorioDir - Path to Factorio installation dirs.
+ * @param factorioDir - Path to Factorio installation dir(s).
  * @param targetVersion -
  *     Version to look for, supports the special value "latest" for the
  *     latest version available.
@@ -183,30 +80,40 @@ async function downloadAndExtractTar(url: string, targetDir: string) {
  * @internal
  */
 async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion): Promise<[string, lib.FullVersion]> {
-	// Check if this is a direct installation dir
-	const directVersion = await getFactorioVersion(factorioDir);
-	if (directVersion !== null) {
-		if (targetVersion === "latest" || directVersion === targetVersion || directVersion.startsWith(targetVersion)) {
-			return [path.join(factorioDir, "data"), directVersion];
+
+	// There are two supported setups: having the factorio dir be the actual
+	// install directory, and having the factorio dir be a folder containing
+	// multiple install directories
+
+	let simpleVersion = await getVersion(path.join(factorioDir, "data", "changelog.txt"));
+	if (simpleVersion !== null) {
+		if (simpleVersion === targetVersion || simpleVersion.startsWith(targetVersion) || targetVersion === "latest") {
+			if (lib.isFullVersion(simpleVersion)) {
+				return [path.join(factorioDir, "data"), simpleVersion];
+			}
 		}
-		throw new Error(`Unable to find Factorio version ${targetVersion}`);
+
+		throw new Error(
+			`Factorio version ${targetVersion} was requested, but install directory contains ${simpleVersion}`
+		);
 	}
 
-	// Otherwise assume it is a directory of multiple versions
-	const versions = new Map<lib.FullVersion, string>();
-	for (const name of await fs.readdir(factorioDir)) {
-		// Avoid any fancy directory detection in order to follow symlinks
-		const fullPath = path.join(factorioDir, name);
-		const version = await getFactorioVersion(fullPath);
-		if (version === null) {
+	let versions = new Map<lib.FullVersion, string>();
+	for (let entry of await fs.readdir(factorioDir, { withFileTypes: true })) {
+		if (!entry.isDirectory()) {
+			continue;
+		}
+
+		let version = await getVersion(path.join(factorioDir, entry.name, "data", "changelog.txt"));
+		if (version === null || !lib.isFullVersion(version)) {
 			continue;
 		}
 
 		if (version === targetVersion) {
-			return [path.join(factorioDir, name, "data"), version];
+			return [path.join(factorioDir, entry.name, "data"), version];
 		}
 
-		versions.set(version, name);
+		versions.set(version, entry.name);
 	}
 
 	if (!versions.size) {
@@ -887,7 +794,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		try {
 			this._whitelistWatcher = fs.watch(filePath);
 		} catch (err: any) {
-			this._logger.warn(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`);
+			this._logger.error(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`);
 			return;
 		}
 
@@ -964,41 +871,6 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		this._state = "init";
 	}
 
-	/**
-	 * Check of updated versions to download
-	 *
-	 * Should be called before init for the new version to be detected during init
-	 */
-	async checkForUpdates(availableVersions: lib.ExternalFactorioVersion[]) {
-		const targetVersion = this._targetVersion;
-		const latestVersion = availableVersions
-			.filter(v => (
-				targetVersion === "latest" || v.version === targetVersion || v.version.startsWith(targetVersion)
-			))
-			.sort((v1, v2) => versionOrder(v1.version, v2.version))
-			[0];
-
-		if (!latestVersion) {
-			return;
-		}
-
-		const installedVersions = await listFactorioVersions(this._factorioDir);
-		if (!installedVersions.versions.has(latestVersion.version)) {
-			const v = latestVersion.version;
-			// Can't download if not linux or this is a direct install
-			if (installedVersions.direct || process.platform !== "linux") {
-				this._logger.info(`A newer version of factorio is available (${v}) but must be manually downloaded`);
-				return;
-			}
-
-			// Download the newer version
-			this._logger.info(`A newer version of factorio is available (${v}), starting download...`);
-			const dst = path.join(this._factorioDir, latestVersion.version);
-			await downloadAndExtractTar(latestVersion.headlessUrl, dst);
-			this._logger.info(`Downloaded factorio version ${v}`);
-		}
-	}
-
 	/**
 	 * The version of Factorio in use. This will be the actual version if
 	 * "latest" (the default) was specified as `factorioVersion` to the constructor.
@@ -1549,12 +1421,9 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 
 
 // For testing only
-export const _getFactorioVersion = getFactorioVersion;
+export const _getVersion = getVersion;
 export const _versionOrder = versionOrder;
 export const _findVersion = findVersion;
-export const _listFactorioVersions = listFactorioVersions;
-export const _downloadAndExtractZip = downloadAndExtractZip;
-export const _downloadAndExtractTar = downloadAndExtractTar;
 export const _randomDynamicPort = randomDynamicPort;
 export const _generatePassword = generatePassword;
 export const _parseOutput = parseOutput;

packages/lib/browser.ts

@@ -16,6 +16,3 @@ export * from "./src/subscriptions";
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as RateLimiter } from "./src/RateLimiter";
 export { default as ValueCache } from "./src/ValueCache";
-
-import { checkSingletonImport } from "./src/check_singleton_import";
-checkSingletonImport(import.meta.url);

packages/lib/index.ts

@@ -38,6 +38,3 @@ export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";
 export { default as ValueCache } from "./src/ValueCache";
-
-import { checkSingletonImport } from "./src/check_singleton_import";
-checkSingletonImport(__filename);

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@clusterio/lib",
 	"description": "Shared library for Clusterio",
-	"version": "2.0.0-alpha.23",
+	"version": "2.0.0-alpha.22b",
 	"repository": "https://github.com/clusterio/clusterio",
 	"license": "MIT",
 	"scripts": {

packages/lib/src/check_singleton_import.ts

@@ -1,21 +0,0 @@
-import { InstallationError } from "./errors";
-declare global {
-	// eslint-disable-next-line vars-on-top
-	var _clusterioLibImportedFrom: string;
-}
-
-export function checkSingletonImport(location: string) {
-	if (!global._clusterioLibImportedFrom) {
-		global._clusterioLibImportedFrom = location;
-	} else {
-		throw new InstallationError(`Attempt to import duplicate copy of @clusterio/lib
-
-Importing more than one copy of @clusterio/lib into the same runtime
-will break Clusterio. This typically happens when the package
-manager has installed two different versions of @clusterio/lib
-and assigned different ones to different components of Clusterio.
-
-Previously imported @clusterio/lib was located at ${_clusterioLibImportedFrom}
-This copy of @clusterio/lib is located at ${location}`);
-	}
-}

packages/lib/src/config/definitions.ts

@@ -259,6 +259,7 @@ export interface HostConfigFields {
 	"host.instances_directory": string;
 	"host.controller_url": string;
 	"host.controller_token": string;
+	"host.tls_ca": string | null;
 	"host.public_address": string;
 	"host.factorio_port_range": string;
 	"host.factorio_username": string | null,
@@ -326,6 +327,12 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			type: "string",
 			initialValue: "enter token here",
 		},
+		"host.tls_ca": {
+			description: "Path to Certificate Authority to validate TLS connection to controller against.",
+			restartRequired: true,
+			type: "string",
+			optional: true,
+		},
 		"host.public_address": {
 			description: "Public facing address players should connect to in order to join instances on this host",
 			type: "string",
@@ -617,6 +624,7 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 export interface ControlConfigFields {
 	"control.controller_url": string | null;
 	"control.controller_token": string | null;
+	"control.tls_ca": string | null;
 	"control.max_reconnect_delay": number;
 }
 
@@ -639,6 +647,11 @@ export class ControlConfig extends classes.Config<ControlConfigFields> {
 			type: "string",
 			optional: true,
 		},
+		"control.tls_ca": {
+			description: "Path to Certificate Authority to validate TLS connection to controller against.",
+			type: "string",
+			optional: true,
+		},
 		"control.max_reconnect_delay": {
 			title: "Max Reconnect Delay",
 			description: "Maximum delay to wait before attempting to reconnect WebSocket",

packages/lib/src/data/ExportManifest.ts

@@ -11,26 +11,21 @@ export default class ExportManifest {
 		/**
 		 * Mapping between known assets and their file names.
 		 *
-		 * Currently exported assets are:
+		 * Currently exported assets are
 		 * - settings: JSON of mod settings prototypes.
 		 * - prototypes: JSON of all game prototypes except settings.
+		 * - item-spritesheet: PNG containing icons from the game.
+		 * - item-metadata: JSON of Map entries describing icons in item-spritesheet
 		 * - locale: JSON of flattened Map entries of en locale strings from the game.
-		 * - spritesheet: Single PNG spritesheet containing all icon categories.
-		 * - metadata: JSON array of [name, {x, y, size, category, path?}] entries
-		 *   describing sprite coordinates and category membership. Categories:
-		 *   item, recipe, signal, technology, planet, quality, entity, static.
 		 */
-		public assets: Record<string, string>,
-		/** ISO 8601 timestamp of when the export was uploaded. */
-		public exportedAt?: string,
+		public assets: Record<string, string>
 	) { }
 
 	static jsonSchema = Type.Object({
 		"assets": Type.Record(Type.String(), Type.String()),
-		"exportedAt": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof ExportManifest.jsonSchema>) {
-		return new this(json.assets, json.exportedAt);
+		return new this(json.assets);
 	}
 }

packages/lib/src/data/HostDetails.ts

@@ -1,77 +0,0 @@
-import { Type, Static } from "@sinclair/typebox";
-
-export default class HostDetails {
-	constructor(
-		/** Version of the host software. The controller may warn if there is a mismatch. */
-		public version: string,
-		/** Human-readable display name of the host. */
-		public name: string,
-		/** Unique numeric identifier for this host. */
-		public id: number,
-		/** Whether the host is currently connected to the controller. */
-		public connected: boolean,
-		/** Address from which the host has connected to the controller. */
-		public remoteAddress = "",
-		/** Public-facing IP address of the host, if available. */
-		public publicAddress: string = "",
-		/** Unix timestamp (sec) after which issued tokens are considered valid. */
-		public tokenValidAfter: number = 0,
-		/** Unix timestamp (ms) indicating when this record was last updated. */
-		public updatedAtMs = 0,
-		/** Flag indicating the host record has been removed on the controller. */
-		public isDeleted = false
-	) {}
-
-	static jsonSchema = Type.Object({
-		"version": Type.String(),
-		"name": Type.String(),
-		"id": Type.Integer(),
-		"connected": Type.Boolean(),
-		"remoteAddress": Type.Optional(Type.String()),
-		"publicAddress": Type.Optional(Type.String()),
-		"tokenValidAfter": Type.Optional(Type.Number()),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"isDeleted": Type.Optional(Type.Boolean()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(
-			json.version,
-			json.name,
-			json.id,
-			json.connected,
-			json.remoteAddress,
-			json.publicAddress,
-			json.tokenValidAfter,
-			json.updatedAtMs,
-			json.isDeleted
-		);
-	}
-
-	toJSON() {
-		const json = {
-			version: this.version,
-			name: this.name,
-			id: this.id,
-			connected: this.connected,
-		} as Static<typeof HostDetails.jsonSchema>;
-
-		if (this.remoteAddress !== "") {
-			json.remoteAddress = this.remoteAddress;
-		}
-		if (this.publicAddress !== "") {
-			json.publicAddress = this.publicAddress;
-		}
-		if (this.tokenValidAfter !== 0) {
-			json.tokenValidAfter = this.tokenValidAfter;
-		}
-		if (this.updatedAtMs !== 0) {
-			json.updatedAtMs = this.updatedAtMs;
-		}
-		if (this.isDeleted !== false) {
-			json.isDeleted = this.isDeleted;
-		}
-
-		return json;
-	}
-}

packages/lib/src/data/InstanceDetails.ts

@@ -1,108 +0,0 @@
-import { Static, Type } from "@sinclair/typebox";
-import { StringEnum } from "./composites";
-import { TargetVersion, TargetVersionSchema } from "./version";
-
-export const InstanceStatus = StringEnum([
-	"unknown",
-	"unassigned",
-	"stopped",
-	"starting",
-	"running",
-	"stopping",
-	"creating_save",
-	"exporting_data",
-	"deleted",
-]);
-
-/**
- * Current status of the instance. One of:
- * - `unknown`: Instance is assigned to a host but this host is currently
- *   not connected to the contreller.
- * - `unassigned`: Instance is not assigned to a a host and exists only on
- *   the controller.
- * - `stopped`: Instance is stopped.
- * - `starting`: Instance is in the process of starting up.
- * - `running`: Instance is running normally.
- * - `stopping`: Instance is in the process of stopping.
- * - `creating_save`: Instance is in the process of creating a save.
- * - `exporting_data`: Instance is in the process of exporting game data.
- * - `deleted`: Instance has been deleted.
- */
-export type InstanceStatus = Static<typeof InstanceStatus>;
-
-export default class InstanceDetails {
-	constructor(
-		public name: string,
-		public id: number,
-		public assignedHost: number | undefined,
-		public gamePort: number | undefined,
-		public status: InstanceStatus,
-		public factorioVersion: TargetVersion | undefined,
-		public startedAtMs = 0,
-		/** Millisecond Unix timestamp this entry was last updated at */
-		public updatedAtMs = 0,
-		public excludeFromStartAll = false
-	) {}
-
-	get isDeleted() {
-		return this.status === "deleted";
-	}
-
-	static jsonSchema = Type.Object({
-		"name": Type.String(),
-		"id": Type.Integer(),
-		"assignedHost": Type.Optional(Type.Integer()),
-		"gamePort": Type.Optional(Type.Integer()),
-		"status": StringEnum([
-			"unknown", "unassigned", "stopped", "starting", "running", "stopping",
-			"creating_save", "exporting_data", "deleted",
-		]),
-		"factorioVersion": Type.Optional(TargetVersionSchema),
-		"startedAtMs": Type.Optional(Type.Number()),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"excludeFromStartAll": Type.Optional(Type.Boolean()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(
-			json.name,
-			json.id,
-			json.assignedHost,
-			json.gamePort,
-			json.status,
-			json.factorioVersion,
-			json.startedAtMs,
-			json.updatedAtMs,
-			json.excludeFromStartAll
-		);
-	}
-
-	toJSON() {
-		const json = {
-			name: this.name,
-			id: this.id,
-			status: this.status,
-		} as Static<typeof InstanceDetails.jsonSchema>;
-
-		if (this.assignedHost !== undefined) {
-			json.assignedHost = this.assignedHost;
-		}
-		if (this.gamePort !== undefined) {
-			json.gamePort = this.gamePort;
-		}
-		if (this.factorioVersion !== undefined) {
-			json.factorioVersion = this.factorioVersion;
-		}
-		if (this.startedAtMs !== 0) {
-			json.startedAtMs = this.startedAtMs;
-		}
-		if (this.updatedAtMs !== 0) {
-			json.updatedAtMs = this.updatedAtMs;
-		}
-		if (this.excludeFromStartAll !== false) {
-			json.excludeFromStartAll = this.excludeFromStartAll;
-		}
-
-		return json;
-	}
-}

packages/lib/src/data/ModInfo.ts

@@ -22,12 +22,11 @@ const UnsatisfiedSeverity: Record<ModDependencyUnsatisfiedReason, number> = {
 	"incompatible": 1, "missing_dependency": 2, "wrong_version": 3,
 };
 
-const depSpecRegex = /^(?:(\?|\(\?\)|!|~|\+) *)?(.+?)(?: *([<>]=?|=) *([0-9.]+))?$/;
+const lettersRegex = /^[a-zA-Z]+$/;
 
 export class ModDependency {
-	public spec: string;
-	public name: string;
 	public type: ModDependencyType;
+	public name: string;
 	public version: ModVersionEquality | undefined;
 
 	static getTypeFromPrefix(prefix: string): ModDependencyType {
@@ -43,27 +42,50 @@ export class ModDependency {
 			case "":
 				return "required";
 			default:
-				throw new Error(`Invalid dependency prefix "${prefix}"`);
+				throw new Error(`Invalid dependency prefix: ${prefix}`);
 		}
 	}
 
 	constructor (
-		specification: string,
+		public specification: string,
 	) {
-		const match = depSpecRegex.exec(specification);
-		if (match === null) {
-			throw new Error(`Invalid dependency specification "${specification}"`);
+		const parts = specification.split(" ");
+
+		// Fix mods which include spaces in their name, joins consecutive words together
+		let index = 0;
+		while (index < parts.length - 1) {
+			const current = parts[index];
+			const next = parts[index + 1];
+			if (lettersRegex.test(current) && lettersRegex.test(next)) {
+				parts[index] = `${current} ${next}`;
+				parts.splice(index + 1, 1);
+			} else {
+				index += 1;
+			}
 		}
 
-		// Expand the match to the different parts
-		const [spec, type, name, equality, version] = match;
-		this.type = ModDependency.getTypeFromPrefix(type ?? "");
-		this.spec = spec;
-		this.name = name;
-
-		// Parse the equality and version
-		if ((equality || version) && !this.incompatible) {
-			this.version = ModVersionEquality.fromParts(equality, version);
+		// Parse the dependency specification
+		switch (parts.length) {
+			case 1:
+				this.type = "required";
+				this.name = parts[0];
+				break;
+			case 2:
+				this.type = ModDependency.getTypeFromPrefix(parts[0]);
+				this.name = parts[1];
+				break;
+			case 3:
+				this.type = "required";
+				this.name = parts[0];
+				this.version = ModVersionEquality.fromParts(parts[1], parts[2]);
+				break;
+			case 4:
+				this.type = ModDependency.getTypeFromPrefix(parts[0]);
+				this.name = parts[1];
+				this.version = ModVersionEquality.fromParts(parts[2], parts[3]);
+				break;
+			default:
+				throw new Error(`Invalid dependency specification: ${specification}`);
 		}
 	}
 
@@ -102,7 +124,7 @@ export class ModDependency {
 	}
 
 	toJSON() {
-		return this.spec;
+		return this.specification;
 	}
 }
 
@@ -196,7 +218,7 @@ export default class ModInfo {
 	 * As they would be represented in info.json.
 	 */
 	get dependencySpecifications() {
-		return this.dependencies.map(d => d.spec);
+		return this.dependencies.map(d => d.specification);
 	}
 
 	checkDependencySatisfaction(mods: (ModInfo | ModRecord)[]) {
@@ -286,13 +308,7 @@ export default class ModInfo {
 		if (json.homepage) { modInfo.homepage = json.homepage; }
 		if (json.description) { modInfo.description = json.description; }
 		if (json.factorio_version) { modInfo.factorioVersion = json.factorio_version; }
-
-		// Parse the dependencies
-		try {
-			if (json.dependencies) { modInfo.dependencies = json.dependencies.map(d => new ModDependency(d)); }
-		} catch (err: any) {
-			throw new Error(`Failed for parse dependencies for ${json.name}: ${err}`);
-		}
+		if (json.dependencies) { modInfo.dependencies = json.dependencies.map(d => new ModDependency(d)); }
 
 		// Additional data
 		if (json.size) { modInfo.size = json.size; }

packages/lib/src/data/Role.ts

@@ -1,23 +1,15 @@
 import { Static, Type } from "@sinclair/typebox";
 import { permissions as globalPermissions } from "../permissions";
 
-interface RoleCollection {
-	get(id: number): Role;
-	set(role: Role): void;
-}
-
 /**
  * Represents a collection of granted permissions
  */
 export default class Role {
-	static DefaultAdminRoleId = 0 as const;
-	static DefaultPlayerRoleId = 1 as const;
-
 	constructor(
 		public id: number,
 		public name: string,
 		public description: string,
-		public permissions = new Set<string>(),
+		public permissions: Set<string>,
 		public updatedAtMs = 0,
 		public isDeleted = false,
 	) { }
@@ -53,13 +45,6 @@ export default class Role {
 		};
 	}
 
-	static ensureDefaultPlayerRole(roles: RoleCollection) {
-		const playerRole = roles.get(this.DefaultPlayerRoleId)
-			?? new this(this.DefaultPlayerRoleId, "Player", "Default player role.");
-		playerRole.grantDefaultPermissions();
-		roles.set(playerRole);
-	}
-
 	grantDefaultPermissions() {
 		for (let permission of globalPermissions.values()) {
 			if (permission.grantByDefault) {
@@ -67,15 +52,4 @@ export default class Role {
 			}
 		}
 	}
-
-	static ensureDefaultAdminRole(roles: RoleCollection) {
-		const adminRole = roles.get(this.DefaultAdminRoleId)
-			?? new this(this.DefaultAdminRoleId, "Cluster Admin", "Cluster wide administrator.");
-		adminRole.grantAdminPermissions();
-		roles.set(adminRole);
-	}
-
-	grantAdminPermissions() {
-		this.permissions.add("core.admin");
-	}
 }

packages/lib/src/data/User.ts

@@ -2,10 +2,12 @@ import { Static, Type } from "@sinclair/typebox";
 
 import PlayerStats from "./PlayerStats";
 
-/** Represents a user in the cluster */
-export default class UserDetails {
-	public playerStats: PlayerStats;
-
+/**
+ * Represeents a user in the cluster
+ *
+ * Holds data about a Factorio user in the cluster.
+ */
+export default class User {
 	constructor(
 		/** Factorio user name.  */
 		public name: string,
@@ -25,15 +27,11 @@ export default class UserDetails {
 		public updatedAtMs = 0,
 		/** True if this user object has been removed from the cluster.  */
 		public isDeleted = false,
+		/** Combined statistics for the player this user account is tied to.  */
+		public playerStats = new PlayerStats(),
 		/** Per instance statistics for the player this user account is tied to.  */
 		public instanceStats = new Map<number, PlayerStats>(),
 	) {
-		this.playerStats = UserDetails._calculatePlayerStats(this.instanceStats);
-	}
-
-	/** Name of the user, used by event subscriptions and to ensure case insensitivity */
-	get id() {
-		return this.name.toLowerCase();
 	}
 
 	static jsonSchema = Type.Object({
@@ -53,13 +51,17 @@ export default class UserDetails {
 		),
 	});
 
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
+	static fromJSON(json: Static<typeof this.jsonSchema>, _hack?: any) {
+		const roleIds = new Set<number>(json.roles ?? []);
 		const instanceStats = new Map(
-			(json.instance_stats ?? []).map(([id, stats]) => [id, PlayerStats.fromJSON(stats)])
+			(json.instance_stats ?? []).map(
+				([id, stats]) => [id, PlayerStats.fromJSON(stats)]
+			)
 		);
+		const playerStats = User._calculatePlayerStats(instanceStats);
 		return new this(
 			json.name,
-			new Set<number>(json.roles),
+			roleIds,
 			new Set(json.instances),
 			json.is_admin,
 			json.is_banned,
@@ -67,12 +69,13 @@ export default class UserDetails {
 			json.ban_reason,
 			json.updated_at_ms,
 			json.is_deleted,
+			playerStats,
 			instanceStats,
 		);
 	}
 
 	toJSON() {
-		let json: Static<typeof UserDetails.jsonSchema> = {
+		let json: Static<typeof User.jsonSchema> = {
 			name: this.name,
 		};
 
@@ -116,7 +119,7 @@ export default class UserDetails {
 	}
 
 	recalculatePlayerStats() {
-		this.playerStats = UserDetails._calculatePlayerStats(this.instanceStats);
+		this.playerStats = User._calculatePlayerStats(this.instanceStats);
 	}
 
 	static _calculatePlayerStats(instanceStatsMap: Map<number, PlayerStats>) {
@@ -146,13 +149,44 @@ export default class UserDetails {
 		}
 		return playerStats;
 	}
+
+	/**
+	 * Merge another user into this one, must have matching ids, user is not deleted
+	 * @param otherUser - User who's details are merged from
+	*/
+	merge(otherUser: User) {
+		if (this.id !== otherUser.id) {
+			throw new Error("Cannot merge users with different ids");
+		}
+
+		// Merge properties
+		this.roleIds = new Set([...this.roleIds, ...otherUser.roleIds]);
+		this.instances = new Set([...this.instances, ...otherUser.instances]);
+		this.isAdmin = this.isAdmin && otherUser.isAdmin; // More secure to use && rather ||
+		this.isBanned = this.isBanned || otherUser.isBanned;
+		this.isBanned = this.isWhitelisted || otherUser.isWhitelisted;
+		this.banReason = this.updatedAtMs > otherUser.updatedAtMs ? this.banReason : otherUser.banReason;
+		this.updatedAtMs = Date.now();
+
+		// Merge instance stats
+		const thisInstanceStats = this.instanceStats;
+		for (const [instanceId, instanceStats] of otherUser.instanceStats.entries()) {
+			if (thisInstanceStats.has(instanceId)) {
+				thisInstanceStats.get(instanceId)!.merge(instanceStats);
+			} else {
+				thisInstanceStats.set(instanceId, instanceStats);
+			}
+		}
+		this.recalculatePlayerStats();
+	}
+
+	/** Name of the user, used by event subscriptions and to ensure case insensitivity */
+	get id() {
+		return this.name.toLowerCase();
+	}
 }
 
-/**
- * This interface is implemented by User on the controller.
- * It is exposed here to allow message definitions to create permission methods.
- */
-export interface IUser extends UserDetails {
+export interface IControllerUser extends User {
 	/**
 	 * Check if a given permission is granted
 	 *

packages/lib/src/data/index.ts

@@ -12,10 +12,8 @@ export { default as ModuleInfo } from "./ModuleInfo";
 export { default as Permission } from "./Permission";
 export { default as PlayerStats } from "./PlayerStats";
 export { default as Role } from "./Role";
-export { default as InstanceDetails, InstanceStatus } from "./InstanceDetails";
-export { default as HostDetails } from "./HostDetails";
-export { default as UserDetails } from "./UserDetails";
-export type { IUser } from "./UserDetails";
+export { default as User } from "./User";
+export type { IControllerUser } from "./User";
 export * from "./composites";
 export * from "./messages_core";
 export * from "./messages_controller";

packages/lib/src/data/messages_controller.ts

@@ -260,8 +260,6 @@ export class SystemInfo {
 		public diskAvailable: number,
 		public canRestart: boolean,
 		public restartRequired: boolean,
-		public systemStartedAtMs: number,
-		public processStartedAtMs: number,
 		/** Millisecond Unix timestamp this entry was last updated at */
 		public updatedAtMs: number,
 		public isDeleted: boolean,
@@ -281,8 +279,6 @@ export class SystemInfo {
 		"diskAvailable": Type.Number(),
 		"canRestart": Type.Boolean(),
 		"restartRequired": Type.Boolean(),
-		"systemStartedAtMs": Type.Number(),
-		"processStartedAtMs": Type.Number(),
 		"updatedAtMs": Type.Number(),
 		"isDeleted": Type.Boolean(),
 	});
@@ -302,8 +298,6 @@ export class SystemInfo {
 			json.diskAvailable,
 			json.canRestart,
 			json.restartRequired,
-			json.systemStartedAtMs,
-			json.processStartedAtMs,
 			json.updatedAtMs,
 			json.isDeleted,
 		);
@@ -402,7 +396,7 @@ export class DebugWsMessageEvent {
 export class FactorioVersionsRequest {
 	declare ["constructor"]: typeof FactorioVersionsRequest;
 	static type = "request" as const;
-	static src = ["control", "instance"] as const;
+	static src = "control" as const;
 	static dst = "controller" as const;
 	static permission = "core.external.get_factorio_versions" as const;
 

packages/lib/src/data/messages_host.ts

@@ -2,7 +2,6 @@ import { Type, Static } from "@sinclair/typebox";
 import { jsonArray, plainJson, StringEnum } from "./composites";
 import { CollectorResultSerialized } from "../prometheus";
 import { HostConfig } from "../config/definitions";
-import HostDetails from "./HostDetails";
 
 export class HostStopRequest {
 	declare ["constructor"]: typeof HostStopRequest;
@@ -83,6 +82,47 @@ export class HostConfigSetPropRequest {
 	}
 }
 
+export class HostDetails {
+	constructor(
+		public version: string,
+		public name: string,
+		public id: number,
+		public connected: boolean,
+		public remoteAddress = "",
+		public publicAddress: string = "",
+		public tokenValidAfter: number = 0,
+		/** Millisecond Unix timestamp this entry was last updated at */
+		public updatedAtMs = 0,
+		public isDeleted = false,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"version": Type.String(),
+		"name": Type.String(),
+		"id": Type.Integer(),
+		"connected": Type.Boolean(),
+		"remoteAddress": Type.Optional(Type.String()),
+		"publicAddress": Type.Optional(Type.String()),
+		"tokenValidAfter": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"isDeleted": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.version,
+			json.name,
+			json.id,
+			json.connected,
+			json.remoteAddress,
+			json.publicAddress,
+			json.tokenValidAfter,
+			json.updatedAtMs,
+			json.isDeleted,
+		);
+	}
+}
+
 export class HostListRequest {
 	declare ["constructor"]: typeof HostListRequest;
 	static type = "request" as const;

packages/lib/src/data/messages_instance.ts

@@ -2,11 +2,84 @@ import { Type, Static } from "@sinclair/typebox";
 import PlayerStats from "./PlayerStats";
 import { JsonString, StringEnum, jsonArray, plainJson } from "./composites";
 import { InstanceConfig } from "../config";
-import type { IUser } from "./UserDetails";
+import type { IControllerUser } from "./User";
 import type { MessageRequest } from "./messages_core";
 import { CollectorResultSerialized } from "../prometheus";
 import { TargetVersion, TargetVersionSchema, PartialVersion, PartialVersionSchema } from "./version";
-import InstanceDetails, { InstanceStatus } from "./InstanceDetails";
+
+
+export const InstanceStatus = StringEnum([
+	"unknown",
+	"unassigned",
+	"stopped",
+	"starting",
+	"running",
+	"stopping",
+	"creating_save",
+	"exporting_data",
+	"deleted",
+]);
+
+/**
+ * Current status of the instance. One of:
+ * - `unknown`: Instance is assigned to a host but this host is currently
+ *   not connected to the contreller.
+ * - `unassigned`: Instance is not assigned to a a host and exists only on
+ *   the controller.
+ * - `stopped`: Instance is stopped.
+ * - `starting`: Instance is in the process of starting up.
+ * - `running`: Instance is running normally.
+ * - `stopping`: Instance is in the process of stopping.
+ * - `creating_save`: Instance is in the process of creating a save.
+ * - `exporting_data`: Instance is in the process of exporting game data.
+ * - `deleted`: Instance has been deleted.
+ */
+export type InstanceStatus = Static<typeof InstanceStatus>;
+
+export class InstanceDetails {
+	constructor(
+		public name: string,
+		public id: number,
+		public assignedHost: number | undefined,
+		public gamePort: number | undefined,
+		public status: InstanceStatus,
+		public factorioVersion: TargetVersion | undefined,
+		/** Millisecond Unix timestamp this entry was last updated at */
+		public updatedAtMs = 0,
+		public excludeFromStartAll = false,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"name": Type.String(),
+		"id": Type.Integer(),
+		"assignedHost": Type.Optional(Type.Integer()),
+		"gamePort": Type.Optional(Type.Integer()),
+		"status": StringEnum([
+			"unknown", "unassigned", "stopped", "starting", "running", "stopping",
+			"creating_save", "exporting_data", "deleted",
+		]),
+		"factorioVersion": Type.Optional(TargetVersionSchema),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"excludeFromStartAll": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.name,
+			json.id,
+			json.assignedHost,
+			json.gamePort,
+			json.status,
+			json.factorioVersion,
+			json.updatedAtMs,
+			json.excludeFromStartAll,
+		);
+	}
+
+	get isDeleted() {
+		return this.status === "deleted";
+	}
+}
 
 export class InstanceDetailsGetRequest {
 	declare ["constructor"]: typeof InstanceDetailsGetRequest;
@@ -463,7 +536,7 @@ export class InstanceTransferSaveRequest {
 	static type = "request" as const;
 	static src = ["control", "controller"] as const;
 	static dst = ["controller", "host"] as const;
-	static permission(user: IUser, message: MessageRequest) {
+	static permission(user: IControllerUser, message: MessageRequest) {
 		user.checkPermission("core.instance.save.transfer");
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as { copy: boolean, sourceName: string, targetName: string };

packages/lib/src/data/messages_mod.ts

@@ -403,10 +403,6 @@ export class ModPortalDownloadRequest {
 	}
 }
 
-const ModDependencyResolveErrors = ["incompatible", "unsatisfiable", "invalidDependency", "notFound"] as const;
-export const ModDependencyResolveErrorsSchema = Type.Union(ModDependencyResolveErrors.map(v => Type.Literal(v)));
-export type ModDependencyResolveErrors = Static<typeof ModDependencyResolveErrorsSchema>;
-
 /**
  * Request mods dependencies to be resolved using the Factorio mod portal.
  *
@@ -460,26 +456,18 @@ export class ModDependencyResolveRequest {
 	static Response = class ModDependencyResolveResponse {
 		constructor(
 			public dependencies: ModInfo[],
-			public errors: Map<string, ModDependencyResolveErrors>,
+			public incompatible: string[],
+			public missing: string[],
 		) {}
 
 		static jsonSchema = Type.Object({
 			"dependencies": Type.Array(ModInfo.jsonSchema),
-			"errors": Type.Record(Type.String(), ModDependencyResolveErrorsSchema),
+			"incompatible": Type.Array(Type.String()),
+			"missing": Type.Array(Type.String()),
 		});
 
 		static fromJSON(json: Static<typeof this.jsonSchema>) {
-			return new this(
-				json.dependencies.map(d => ModInfo.fromJSON(d)),
-				new Map(Object.entries(json.errors))
-			);
-		}
-
-		toJSON(): Static<typeof ModDependencyResolveResponse.jsonSchema> {
-			return {
-				dependencies: this.dependencies.map(d => d.toJSON()),
-				errors: Object.fromEntries(this.errors),
-			};
+			return new this(json.dependencies.map(d => ModInfo.fromJSON(d)), json.incompatible, json.missing);
 		}
 	};
 }

packages/lib/src/data/messages_user.ts

@@ -1,5 +1,5 @@
 import { Type, Static } from "@sinclair/typebox";
-import UserDetails, { IUser } from "./UserDetails";
+import User, { IControllerUser } from "./User";
 import { StringEnum, jsonArray, plainJson } from "./composites";
 import { MessageRequest } from "./messages_core";
 
@@ -22,7 +22,7 @@ export class UserGetRequest {
 		return new this(json.name);
 	}
 
-	static Response = UserDetails;
+	static Response = User;
 }
 
 export class UserListRequest {
@@ -31,7 +31,7 @@ export class UserListRequest {
 	static src = "control" as const;
 	static dst = "controller" as const;
 	static permission = "core.user.list" as const;
-	static Response = jsonArray(UserDetails);
+	static Response = jsonArray(User);
 }
 
 export class UserCreateRequest {
@@ -198,15 +198,15 @@ export class UserUpdatesEvent {
 	static permission = "core.user.subscribe" as const;
 
 	constructor(
-		public updates: UserDetails[],
+		public updates: User[],
 	) { }
 
 	static jsonSchema = Type.Object({
-		"updates": Type.Array(UserDetails.jsonSchema),
+		"updates": Type.Array(User.jsonSchema),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.updates.map(update => UserDetails.fromJSON(update)));
+		return new this(json.updates.map(update => User.fromJSON(update)));
 	}
 }
 
@@ -250,7 +250,7 @@ export class UserBulkImportRequest {
 	static type = "request" as const;
 	static src = "control" as const;
 	static dst = "controller" as const;
-	static permission(user: IUser, message: MessageRequest) {
+	static permission(user: IControllerUser, message: MessageRequest) {
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as Static<typeof UserBulkImportRequest.jsonSchema>;
 			// Check if this is a restore or import request

packages/lib/src/data/version.ts

@@ -151,10 +151,10 @@ export class ModVersionEquality {
 
 	static fromParts(equality: string, version: string) {
 		if (!isVersionEquality(equality)) {
-			throw new Error(`Invalid version equality "${equality}"`);
+			throw new Error(`Unknown version equality: ${equality}`);
 		}
 		if (!isPartialVersion(version)) {
-			throw new Error(`Invalid version string "${version}"`);
+			throw new Error(`Malformed version string: ${version}`);
 		}
 		return new this(equality, version);
 	}

packages/lib/src/datastore.ts

@@ -53,7 +53,7 @@ export class JsonDatastoreProvider<
 		private filePath: string,
 		private fromJson: (json: J) => V = v => v as any as V,
 		private migrations: (rawJson: Record<string, unknown>) => Record<string, J> = v => v as Record<string, J>,
-		private finalise: (obj: V, map: Map<string, V>) => V | undefined = v => v,
+		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}
@@ -82,24 +82,7 @@ export class JsonDatastoreProvider<
 		const serialized = this.migrations(rawJson);
 
 		// Convert to data class objects
-		let serializedCount = 0;
-		const result = new Map<string, V>();
-		for (const [key, raw] of Object.entries(serialized)) {
-			const final = this.finalise(this.fromJson(raw), result);
-			if (final !== undefined) { result.set(key, final); }
-			serializedCount += 1;
-		}
-
-		// Save a backup if the map does not contain all values (happens where there are duplicates)
-		if (result.size < serializedCount) {
-			const backupPath = `${this.filePath}.${Date.now()}.bak`;
-			logger.warn(
-				`Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}`
-			);
-			await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t"));
-		}
-
-		return result;
+		return new Map(Object.entries(serialized).map(([k, v]) => [k, this.finalise(this.fromJson(v))]));
 	}
 }
 
@@ -112,7 +95,7 @@ export class JsonIdDatastoreProvider<
 		private filePath: string,
 		private fromJson: (json: J) => V = v => v as any as V,
 		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as Array<J>,
-		private finalise: (obj: V, map: Map<V["id"], V>) => V | undefined = v => v,
+		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}
@@ -141,24 +124,10 @@ export class JsonIdDatastoreProvider<
 		const serialized = this.migrations(rawJson);
 
 		// Convert to data class objects
-		let serializedCount = 0;
-		const result = new Map<V["id"], V>();
-		for (const raw of serialized) {
-			const final = this.finalise(this.fromJson(raw), result);
-			if (final !== undefined) { result.set(final.id, final); }
-			serializedCount += 1;
-		}
-
-		// Save a backup if the map does not contain all values (happens where there are duplicates)
-		if (result.size < serializedCount) {
-			const backupPath = `${this.filePath}.${Date.now()}.bak`;
-			logger.warn(
-				`Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}`
-			);
-			await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t"));
-		}
-
-		return result;
+		return new Map(serialized.map((e) => {
+			const v = this.finalise(this.fromJson(e));
+			return [v.id, v];
+		}));
 	}
 }
 
@@ -290,50 +259,18 @@ export class KeyValueDatastore<
 export class SubscribableDatastore<
 	V extends SubscribableValue,
 > extends Datastore<V["id"], V> {
-	/** Last time an update occurred */
-	lastUpdatedAtMs: number = 0;
-
-	private static maxUpdatedAtMs(values: Iterable<SubscribableValue>) {
-		let max = 0;
-		for (const value of values) {
-			// Prevent one NaN, undefined, +Inf or similar from breaking the datastore.
-			if (Number.isFinite(value.updatedAtMs)) {
-				max = Math.max(max, value.updatedAtMs);
-			}
-		}
-		return max;
-	}
-
-	private incrementLastUpdatedMs() {
-		this.lastUpdatedAtMs = Math.max(Date.now(), this.lastUpdatedAtMs + 1);
-	}
-
-	constructor(
-		provider: DatastoreProvider<V["id"], V> = new MemoryDatastoreProvider(),
-		data = new Map<V["id"], V>(),
-	) {
-		super(provider, data);
-		this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values());
-	}
-
-	async load() {
-		await super.load();
-		this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values());
-	}
-
 	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
 	set(value: V) {
-		this.incrementLastUpdatedMs();
 		this.data.set(value.id, value);
-		value.updatedAtMs = this.lastUpdatedAtMs;
+		value.updatedAtMs = Date.now();
 		this.touch([value]);
 	}
 
 	// Set many values at once from an array of key value pairs
 	setMany(values: V[]) {
-		this.incrementLastUpdatedMs();
+		const nowMs = Date.now();
 		for (const value of values) {
-			value.updatedAtMs = this.lastUpdatedAtMs;
+			value.updatedAtMs = nowMs;
 			this.data.set(value.id, value);
 		}
 		this.touch(values);
@@ -341,19 +278,18 @@ export class SubscribableDatastore<
 
 	// Delete a value in the datastore
 	delete(value: V) {
-		this.incrementLastUpdatedMs();
 		this.data.delete(value.id);
-		value.updatedAtMs = this.lastUpdatedAtMs;
+		value.updatedAtMs = Date.now();
 		value.isDeleted = true;
 		this.touch([value]);
 	}
 
 	// Delete many values at once from an array of values
 	deleteMany(values: V[]) {
-		this.incrementLastUpdatedMs();
+		const nowMs = Date.now();
 		for (const value of values) {
 			value.isDeleted = true;
-			value.updatedAtMs = this.lastUpdatedAtMs;
+			value.updatedAtMs = nowMs;
 			this.data.delete(value.id);
 		}
 		this.touch(values);

packages/lib/src/errors.ts

@@ -10,13 +10,6 @@ export class CommandError extends Error {
 	code = "CommandError";
 }
 
-/**
- * Thrown if the installation is broken, such as when multiple copies of lib is imported
- */
-export class InstallationError extends Error {
-	code = "InstallationError";
-}
-
 /**
  * Thrown from requests sent when an error occured handling it
  */

packages/lib/src/external/FactorioVersions.ts

@@ -1,7 +1,7 @@
 import { Static, Type } from "@sinclair/typebox";
-import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data/version";
+import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data";
 
-const ARCHIVE_URL = "https://factorio.com/download/archive/";
+const ARCHIVE_URL = "https://factorio.com/download/archive";
 
 /**
  * Represents the parsed result of a factorio version
@@ -60,7 +60,7 @@ function parseFactorioVersions(html: string): ExternalFactorioVersion[] {
 			versions.set(version, {
 				version: version,
 				stable: kind === "stable",
-				headlessUrl: `https://www.factorio.com/get-download/${version}/headless/linux64`,
+				headlessUrl: `www.factorio.com/get-download/${version}/headless/linux64`,
 			});
 		}
 	}

packages/lib/src/link/connectors.ts

@@ -431,6 +431,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
 	constructor(
 		protected _url: string,
 		maxReconnectDelay: number,
+		protected _tlsCa: string | undefined,
 	) {
 		super(undefined as any, new libData.Address(libData.Address.controller, 0));
 		this._backoff = new ExponentialBackoff({ max: maxReconnectDelay });
@@ -542,7 +543,17 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
 
 		// Open WebSocket to controller
 		logger.verbose(`Connector | connecting to ${url}`);
-		this._socket = new WebSocket(url) as WebSocketClusterio;
+
+
+		if (process.env.APP_ENV === "browser") {
+			this._socket = new WebSocket(url) as WebSocketClusterio;
+
+		} else {
+			let options: { ca?: string } = {};
+			if (this._tlsCa) { options.ca = this._tlsCa; }
+			this._socket = new WebSocket(url, options) as WebSocketClusterio;
+		}
+
 		this._attachSocketHandlers();
 	}
 

packages/lib/src/link/link.ts

@@ -8,7 +8,7 @@ import { BaseConnector, WebSocketBaseConnector } from "./connectors";
 import { strict as assert } from "assert";
 import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin";
 import type {
-	AddressType, JSONDeserialisable, MessageRoutable, MessageRequest, MessageEvent, IUser,
+	AddressType, JSONDeserialisable, MessageRoutable, MessageRequest, MessageEvent, IControllerUser,
 } from "../data";
 
 export interface Request<Req, Res> {
@@ -18,7 +18,7 @@ export interface Request<Req, Res> {
 		type: "request";
 		src: AddressType | readonly AddressType[];
 		dst: AddressType | readonly AddressType[];
-		permission?: null | string | ((user: IUser, message: MessageRequest) => void);
+		permission?: null | string | ((user: IControllerUser, message: MessageRequest) => void);
 		plugin?: string;
 		Response?: JSONDeserialisable<Res>,
 	}
@@ -32,7 +32,7 @@ export interface Event<T> {
 		type: "event";
 		src: AddressType | readonly AddressType[];
 		dst: AddressType | readonly AddressType[];
-		permission?: null | string | ((user: IUser, message: MessageEvent) => void);
+		permission?: null | string | ((user: IControllerUser, message: MessageEvent) => void);
 		plugin?: string;
 	}
 }

packages/lib/src/permissions.ts

@@ -6,7 +6,9 @@
  * @author Hornwitser
  * @module lib/users
  */
-import { Permission } from "./data";
+import { Permission, Role } from "./data";
+import { SubscribableDatastore } from "./datastore";
+
 
 export const permissions = new Map<string, Permission>();
 
@@ -56,6 +58,54 @@ export function definePermission({
 	permissions.set(name, new Permission(name, title, description, grantByDefault));
 }
 
+
+/**
+ * Ensure the default admin role has full access
+ *
+ * Ensures the role with id 0 exisits and has the core.admin permission.
+ *
+ * @param roles - role storage to modify.
+ * @returns the default admin role
+ */
+export function ensureDefaultAdminRole(roles: SubscribableDatastore<Role>) {
+	let admin = roles.get(0);
+	if (!admin) {
+		admin = Role.fromJSON({
+			id: 0,
+			name: "Cluster Admin",
+			description: "Cluster wide administrator.",
+			permissions: [],
+		});
+	}
+	admin.permissions.add("core.admin");
+	roles.set(admin);
+	return admin;
+}
+
+/**
+ * Ensure the default player role has default accesses
+ *
+ * Ensures the role with id 1 exists and has all the permissions that are
+ * assigned by default.  Note that this may not be the same role as the one
+ * that is assigned to new users by default.
+ *
+ * @param roles - role storage to modify.
+ */
+export function ensureDefaultPlayerRole(roles: SubscribableDatastore<Role>) {
+	let player = roles.get(1);
+	if (!player) {
+		player = Role.fromJSON({
+			id: 1,
+			name: "Player",
+			description: "Default player role.",
+			permissions: [],
+		});
+	}
+	player.grantDefaultPermissions();
+	roles.set(player);
+}
+
+
 // Definitions for the built in permissions used in Clusterio.
 // description should answer "this permission allows you to ___"
 definePermission({

packages/lib/src/plugin_loader.ts

@@ -45,9 +45,6 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
 			pluginInfo = require(pluginPath).plugin;
 			pluginPackage = require(path.posix.join(pluginPath, "package.json"));
 		} catch (err: any) {
-			if (err.code === "InstallationError") {
-				throw err;
-			}
 			throw new libErrors.PluginError(pluginName, err);
 		}
 

packages/lib/src/prometheus.ts

@@ -272,31 +272,18 @@ function labelsToKey(labels: LabelValues, metricLabels: string[]) {
 function keyToLabels(key: string) {
 	let labels = new Map<string, string>();
 
-	let pos = 0;
-	let matchLabel = /([^=]+)="(([^\\"]|\\["n\\])+)"/y;
-	while (pos < key.length) {
-		if (pos !== 0) {
-			/* istanbul ignore if */
-			if (key[pos] !== ",") {
-				// Should be unreachable
-				throw Error(`Expected , at pos ${pos} in "${key}"`);
+	if (key !== "") {
+		for (let pair of key.split(",")) {
+			let [name, value] = pair.split("=", 2);
+			if (!name || value === undefined) {
+				continue;
 			}
-			pos += 1;
-		}
-		matchLabel.lastIndex = pos;
-		const match = matchLabel.exec(key);
-		/* istanbul ignore else */
-		if (match) {
-			let [, name, value] = match;
 			labels.set(name, value
+				.slice(1, -1)
 				.replace(/\\"/g, "\"")
 				.replace(/\\n/g, "\n")
 				.replace(/\\\\/g, "\\")
 			);
-			pos += match[0].length;
-		} else {
-			// Should be unreachable
-			throw new Error(`Failed to parse label at pos ${pos} in "${key}"`);
 		}
 	}
 

packages/lib/src/subscriptions.ts

@@ -1,6 +1,6 @@
 import { Type, Static } from "@sinclair/typebox";
 import { Link, Event, EventClass, RequestHandler, WebSocketBaseConnector } from "./link";
-import { Address, MessageRequest, IUser, plainJson, JsonBoolean } from "./data";
+import { Address, MessageRequest, IControllerUser, plainJson, JsonBoolean } from "./data";
 import isDeepStrictEqual from "./is_deep_strict_equal";
 import { RequestError, SessionLost } from "./errors";
 import { logger } from "./logging";
@@ -18,7 +18,7 @@ export class SubscriptionRequest {
 	static type = "request" as const;
 	static src = ["control", "host", "instance"] as const;
 	static dst = "controller" as const;
-	static permission(user: IUser, message: MessageRequest) {
+	static permission(user: IControllerUser, message: MessageRequest) {
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as Static<typeof SubscriptionRequest.jsonSchema>;
 			const entry = Link._eventsByName.get(data[0]);

packages/lib/src/system_collectors.ts

@@ -134,12 +134,7 @@ function minZip<T>(a: T[], b: T[]): [T, T][] {
 
 let previousTotalCpuMs: number[] = [];
 let previousIdleCpuMs: number[] = [];
-export async function gatherSystemInfo(
-	id: SystemInfo["id"],
-	canRestart: boolean,
-	restartRequired: boolean,
-	startedAtMs: number
-) {
+export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean, restartRequired: boolean) {
 	const cpus = os.cpus();
 	const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
 	const currentIdleCpuMs = cpus.map(({ times }) => times.idle);
@@ -157,7 +152,6 @@ export async function gatherSystemInfo(
 		diskAvailable = stats.bavail * stats.bsize;
 	}
 
-	const now = Date.now();
 	return new SystemInfo(
 		id,
 		os.hostname(),
@@ -172,9 +166,7 @@ export async function gatherSystemInfo(
 		diskAvailable,
 		canRestart,
 		restartRequired,
-		now - (os.uptime() * 1000),
-		startedAtMs,
-		now,
+		Date.now(),
 		false,
 	);
 }

packages/lib/tsconfig.browser.json

@@ -7,7 +7,6 @@
 		"./src/config/*.ts",
 		"./src/data/*.ts",
 		"./src/external/*.ts",
-		"./src/check_singleton_import.ts",
 		"./src/datastore.ts",
 		"./src/errors.ts",
 		"./src/factorio/exchange_string.ts",

packages/lib/tsconfig.node.json

@@ -1,6 +1,6 @@
 {
 	"extends": "../../tsconfig.node.json",
-	"include": ["index.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"],
+	"include": ["index.ts", "browser.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"],
 	"compilerOptions": {
 		"allowJs": true,
 	},

packages/web_ui/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio web interface implementation",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.23",
+	"version": "2.0.0-alpha.22b",
 	"browser": "src/index.ts",
 	"main": "dist/browser/src/index.js",
 	"scripts": {

packages/web_ui/src/bootstrap.tsx

@@ -137,7 +137,7 @@ export default async function bootstrap() {
 	lib.addPluginConfigFields(pluginInfos);
 
 	let wsUrl = new URL(webRoot, document.location.href);
-	let controlConnector = new ControlConnector(wsUrl.href, 120);
+	let controlConnector = new ControlConnector(wsUrl.href, 120, undefined);
 	let control = new Control(controlConnector, new Map(pluginInfos.map(p => [p.name, p])));
 	control.plugins = await loadPlugins(pluginInfos, control);
 	control.inputComponents = inputComponentsFromPlugins(control.plugins);

packages/web_ui/src/components/ControllerPage.tsx

@@ -8,7 +8,7 @@ import PluginExtra from "./PluginExtra";
 import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
-	MetricDiskUsed, MetricDiskRatio, MetricRelativeDate,
+	MetricDiskUsed, MetricDiskRatio,
 } from "./system_metrics";
 import { useAccount } from "../model/account";
 import { useSystems } from "../model/system";
@@ -101,12 +101,6 @@ export default function ControllerPage() {
 			<Descriptions.Item label="Node.js">{system?.node}</Descriptions.Item>
 			<Descriptions.Item label="OS Kernel">{system?.kernel}</Descriptions.Item>
 			<Descriptions.Item label="Machine">{system?.machine}</Descriptions.Item>
-			<Descriptions.Item label="System Uptime">
-				<MetricRelativeDate timeMs={system?.systemStartedAtMs}/>
-			</Descriptions.Item>
-			<Descriptions.Item label="Process Uptime">
-				<MetricRelativeDate timeMs={system?.processStartedAtMs}/>
-			</Descriptions.Item>
 			<Descriptions.Item label="Hostname" span={2}>{system?.hostname}</Descriptions.Item>
 			<Descriptions.Item label="CPU Model" span={2}>{system?.cpuModel}</Descriptions.Item>
 			<Descriptions.Item label="CPU Usage"><MetricCpuRatio system={system} /></Descriptions.Item>

packages/web_ui/src/components/HostStatusTag.tsx

@@ -1,17 +0,0 @@
-import { Tag } from "antd";
-import React from "react";
-
-import { CheckCircleOutlined, CloseCircleOutlined } from "@ant-design/icons";
-
-const statusOptions = {
-	connected: <Tag color="#389e0d" icon={<CheckCircleOutlined/>} >Connected</Tag>,
-	disconnected: <Tag color="#cf1322" icon={<CloseCircleOutlined/>} >Disconnected</Tag>,
-};
-
-type HostStatusTagProps = {
-	connected: boolean;
-};
-
-export default function HostStatusTag(props: HostStatusTagProps) {
-	return props.connected ? statusOptions.connected : statusOptions.disconnected;
-}

packages/web_ui/src/components/HostViewPage.tsx

@@ -1,5 +1,5 @@
 import React, { useContext, useState } from "react";
-import { Descriptions, Spin, Typography, Button, Space, Popconfirm, Flex, Tooltip } from "antd";
+import { Descriptions, Spin, Tag, Typography, Button, Space, Modal, Popconfirm, Flex, Tooltip } from "antd";
 import { ExclamationCircleOutlined } from "@ant-design/icons";
 import { useParams } from "react-router-dom";
 
@@ -7,7 +7,6 @@ import * as lib from "@clusterio/lib";
 import notify, { notifyErrorHandler } from "../util/notify";
 import ControlContext from "./ControlContext";
 import HostConfigTree from "./HostConfigTree";
-import HostStatusTag from "./HostStatusTag";
 import InstanceList from "./InstanceList";
 import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import { useAccount } from "../model/account";
@@ -18,7 +17,7 @@ import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
-	MetricDiskUsed, MetricDiskRatio, MetricRelativeDate,
+	MetricDiskUsed, MetricDiskRatio,
 } from "./system_metrics";
 import { formatTimestamp } from "../util/time_format";
 import { useSystems } from "../model/system";
@@ -135,22 +134,21 @@ export default function HostViewPage() {
 	return <PageLayout nav={nav}>
 		<PageHeader
 			title={host.name || String(hostId)}
-			status={<HostStatusTag connected={host["connected"]}/>}
 			extra={hostButtons}
 		/>
 		<Descriptions bordered size="small" column={{ xs: 1, md: 2, lg: 2, xl: 2, xxl: 2 }}>
+			<Descriptions.Item label="Name">{host["name"]}</Descriptions.Item>
+			<Descriptions.Item label="Connected">
+				<Tag color={host["connected"] ? "#389e0d" : "#cf1322"}>
+					{host["connected"] ? "Connected" : "Disconnected"}
+				</Tag>
+			</Descriptions.Item>
 			<Descriptions.Item label="Version">{host.version}</Descriptions.Item>
 			<Descriptions.Item label="Node.js">{system?.node}</Descriptions.Item>
 			<Descriptions.Item label="OS Kernel">{system?.kernel}</Descriptions.Item>
 			<Descriptions.Item label="Machine">{system?.machine}</Descriptions.Item>
-			<Descriptions.Item label="System Uptime">
-				<MetricRelativeDate timeMs={system?.systemStartedAtMs}/>
-			</Descriptions.Item>
-			<Descriptions.Item label="Process Uptime">
-				<MetricRelativeDate timeMs={system?.processStartedAtMs}/>
-			</Descriptions.Item>
-			<Descriptions.Item label="Hostname">{system?.hostname}</Descriptions.Item>
-			<Descriptions.Item label="Connected From">{host.remoteAddress}</Descriptions.Item>
+			<Descriptions.Item label="Hostname" span={2}>{system?.hostname}</Descriptions.Item>
+			<Descriptions.Item label="Connected From" span={2}>{host.remoteAddress}</Descriptions.Item>
 			<Descriptions.Item label="CPU Model" span={2}>{system?.cpuModel}</Descriptions.Item>
 			<Descriptions.Item label="CPU Usage"><MetricCpuRatio system={system} /></Descriptions.Item>
 			<Descriptions.Item label="Cores"><MetricCpuUsed system={system} /></Descriptions.Item>

packages/web_ui/src/components/HostsPage.tsx

@@ -1,6 +1,6 @@
 import React, { useContext, useRef, useState, useEffect } from "react";
 import { useNavigate } from "react-router-dom";
-import { Button, Flex, Form, InputNumber, Modal, Space, Switch, Table, Tooltip, Typography } from "antd";
+import { Button, Flex, Form, InputNumber, Modal, Progress, Space, Switch, Table, Tag, Tooltip, Typography } from "antd";
 import { ExclamationCircleOutlined, CopyOutlined } from "@ant-design/icons";
 
 import * as lib from "@clusterio/lib";
@@ -11,7 +11,6 @@ import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
-import HostStatusTag from "./HostStatusTag";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
 	MetricDiskUsed, MetricDiskRatio,
@@ -57,7 +56,7 @@ function GenerateHostTokenButton() {
 
 		let newToken = await control.send(new lib.HostGenerateTokenRequest(id));
 		setToken(newToken);
-		setHostId(id ?? null);
+		setHostId(id??null);
 	}
 
 	// Generate a new random
@@ -90,24 +89,13 @@ function GenerateHostTokenButton() {
 					}} />
 				</Form.Item>
 			</Form>
-			<Typography.Paragraph>
-				Host IDs are randomly generated and unique, leave blank to use a new ID,
-				 otherwise the ID of an existing host can be found using:
-			</Typography.Paragraph>
-			<div className="codeblock">
-				<CopyButton
-					message="Copied get host id to clipboard"
-					text="npx clusteriohost config get host.id"
-				/>
-				npx clusteriohost config get host.id
-			</div>
 			{token !== null && <>
 				<Typography.Paragraph>
 					Host auth token:
 				</Typography.Paragraph>
 				<div className="codeblock">
 					<CopyButton
-						message="Copied host token to clipboard"
+						message={"Copied auth token to clipboard"}
 						text={token}
 					/>
 					{token}
@@ -117,13 +105,15 @@ function GenerateHostTokenButton() {
 				</Typography.Paragraph>
 				<div className="codeblock">
 					<CopyButton
-						message="Copied set host id to clipboard"
-						text={`npx clusteriohost config set host.id ${token}`}
+						message={"Copied configuration commands to clipboard"}
+						text={`npx clusteriohost config set host.controller_token ${token}
+							   npx clusteriohost config set host.id ${hostId}`}
 					/>
-					npx clusteriohost config set host.id {token}
+					<p>npx clusteriohost config set host.controller_token &lt;token&gt;</p>
+					<p>npx clusteriohost config set host.id &lt;hostId&gt;</p>
 				</div>
 				<Typography.Paragraph>
-					A new host can be setup using the following commands:
+					Example host setup commands:
 				</Typography.Paragraph>
 				<div className="codeblock">
 					<CopyButton
@@ -132,7 +122,7 @@ function GenerateHostTokenButton() {
 mkdir clusterio
 cd clusterio
 npm init "@clusterio" -- --controller-token ${token} --mode "host" --download-headless \
---controller-url ${document.location.origin}/ --host-name "${hostId ? `Host ${hostId}` : "Host"}" \
+--controller-url ${document.location.origin}/ --host-name "Host ${hostId || "?"}" \
 --public-address localhost ${pluginString.length ? "--plugins" : ""} ${pluginString}`
 						}/>
 					<p>&gt; mkdir clusterio</p>
@@ -143,7 +133,7 @@ npm init "@clusterio" -- --controller-token ${token} --mode "host" --download-he
 						--mode "host"
 						--download-headless
 						--controller-url <span className="highlight">{document.location.origin}/ </span>
-						--host-name <span className="highlight">{hostId ? `Host ${hostId}` : "Host"} </span>
+						--host-name <span className="highlight">"Host {hostId || "?"}" </span>
 						--public-address <span className="highlight">localhost </span>
 						{pluginString.length ? "--plugins" : ""} <span className="highlight">{pluginString}</span>
 					</p>
@@ -297,7 +287,11 @@ export default function HostsPage() {
 					title: "Connected",
 					key: "connected",
 					render: (_, host) => <>
-						<HostStatusTag connected={host["connected"]}/>
+						<Tag
+							color={host.connected ? "#389e0d" : "#cf1322"}
+						>
+							{host.connected ? "Connected" : "Disconnected"}
+						</Tag>
 						<RestartRequired system={systems.get(host.id)}/>
 					</>,
 					sorter: (a, b) => Number(a.connected) - Number(b.connected),

packages/web_ui/src/components/IconReferencePage.tsx

@@ -1,248 +0,0 @@
-import React, { useMemo, useState } from "react";
-import { Card, Input, Space, Table, Tag, Typography } from "antd";
-
-import {
-	useItemMetadata,
-	useRecipeMetadata,
-	useSignalMetadata,
-	useTechnologyMetadata,
-	usePlanetMetadata,
-	useQualityMetadata,
-	useEntityMetadata,
-} from "../model/item_metadata";
-import { useExportManifest } from "../model/export_manifest";
-import PageHeader from "./PageHeader";
-import PageLayout from "./PageLayout";
-
-const { Text } = Typography;
-
-type IconEntry = {
-	key: string;
-	category: string;
-	name: string;
-	cssClass: string;
-	size: number;
-	path?: string;
-};
-
-const CATEGORIES = [
-	"item", "recipe", "signal", "technology", "planet", "quality", "entity",
-] as const;
-
-type Category = typeof CATEGORIES[number];
-
-const CATEGORY_LABELS: Record<Category, string> = {
-	item: "Item",
-	recipe: "Recipe",
-	signal: "Signal",
-	technology: "Technology",
-	planet: "Planet",
-	quality: "Quality",
-	entity: "Entity",
-};
-
-function useAllIconEntries() {
-	const item = useItemMetadata();
-	const recipe = useRecipeMetadata();
-	const signal = useSignalMetadata();
-	const technology = useTechnologyMetadata();
-	const planet = usePlanetMetadata();
-	const quality = useQualityMetadata();
-	const entity = useEntityMetadata();
-
-	const byCategory: Record<Category, Map<string, { size: number; path?: string }>> = {
-		item, recipe, signal, technology, planet, quality, entity,
-	};
-
-	return useMemo(() => {
-		const entries: IconEntry[] = [];
-
-		for (const category of CATEGORIES) {
-			const names = [...byCategory[category].keys()].sort();
-			for (const name of names) {
-				const meta = byCategory[category].get(name)!;
-				entries.push({
-					key: `${category}:${name}`,
-					category,
-					name,
-					cssClass: `${category}-${CSS.escape(name)}`,
-					size: meta.size,
-					path: meta.path,
-				});
-			}
-		}
-
-		return entries;
-	}, [item, recipe, signal, technology, planet, quality, entity]);
-}
-
-export default function IconReferencePage() {
-	const [filter, setFilter] = useState("");
-	const [expandedGroups, setExpandedGroups] = useState<Set<string>>(new Set());
-	const allEntries = useAllIconEntries();
-
-	const filtered = useMemo(() => {
-		const q = filter.trim().toLowerCase();
-		if (!q) {
-			return allEntries;
-		}
-		return allEntries.filter(e => e.name.toLowerCase().includes(q) || e.cssClass.toLowerCase().includes(q)
-		);
-	}, [allEntries, filter]);
-
-	const groups = useMemo(() => {
-		const map = new Map<string, IconEntry[]>();
-		for (const cat of CATEGORIES) {
-			map.set(cat, []);
-		}
-		for (const entry of filtered) {
-			map.get(entry.category)!.push(entry);
-		}
-		return CATEGORIES.map(cat => ({ category: cat, entries: map.get(cat)! }))
-			.filter(g => g.entries.length > 0);
-	}, [filtered]);
-
-	const visibleCount = filtered.length;
-	const totalCount = allEntries.length;
-	const autoExpand = visibleCount <= 20;
-
-	const isExpanded = (category: string) => autoExpand || expandedGroups.has(category);
-
-	const toggleGroup = (category: string) => {
-		setExpandedGroups(prev => {
-			const next = new Set(prev);
-			if (next.has(category)) {
-				next.delete(category);
-			} else {
-				next.add(category);
-			}
-			return next;
-		});
-	};
-
-	const columns = [
-		{
-			title: "Icon",
-			dataIndex: "cssClass",
-			key: "icon",
-			width: 48,
-			render: (_: string, record: IconEntry) => (
-				<div className={record.cssClass} style={{ imageRendering: "pixelated" }} />
-			),
-		},
-		{
-			title: "Name",
-			dataIndex: "name",
-			key: "name",
-			render: (name: string) => (
-				<Text style={{ fontSize: 12 }}>{name}</Text>
-			),
-		},
-		{
-			title: "Usage",
-			dataIndex: "cssClass",
-			key: "usage",
-			render: (cssClass: string) => (
-				<Text code style={{ fontSize: 11, color: "#90ee90" }}>
-					{`<div class="${cssClass}" />`}
-				</Text>
-			),
-		},
-		{
-			title: "Path",
-			dataIndex: "path",
-			key: "path",
-			render: (p: string | undefined) => (p
-				? <Text type="secondary" style={{ fontSize: 11 }}>{p}</Text>
-				: null),
-		},
-	];
-
-	const manifest = useExportManifest();
-
-	return (
-		<PageLayout nav={[{ name: "Icon Reference" }]}>
-			<PageHeader title="Icon Reference" />
-			<div style={{ width: "80%", margin: "0 auto" }}>
-				{!manifest && (
-					<Card>
-						<Text type="secondary">
-							No export data available. Run export-data on an instance to generate icons.
-						</Text>
-					</Card>
-				)}
-				{manifest && manifest.exportedAt && (
-					<div style={{ marginBottom: 12, fontSize: 12, lineHeight: 1.8 }}>
-						{manifest.exportedAt && (
-							<div>
-								<Text type="secondary">Exported: </Text>
-								<Text>{new Date(manifest.exportedAt).toLocaleString()}</Text>
-							</div>
-						)}
-						<div>
-							<Text type="secondary">Icons: </Text>
-							<Text>{totalCount}</Text>
-						</div>
-					</div>
-				)}
-				<Card>
-					<Space style={{ marginBottom: 16, display: "flex", alignItems: "center" }}>
-						<Input
-							placeholder="Filter by name or class…"
-							value={filter}
-							onChange={e => setFilter(e.target.value)}
-							allowClear
-							style={{ width: 320 }}
-						/>
-						<Text type="secondary" style={{ fontSize: 12, whiteSpace: "nowrap" }}>
-							{visibleCount === totalCount
-								? `${totalCount} icons`
-								: `${visibleCount} / ${totalCount} icons`}
-						</Text>
-					</Space>
-					{groups.map(({ category, entries }) => {
-						const label = CATEGORY_LABELS[category as Category] ?? category;
-						const expanded = isExpanded(category);
-						return (
-							<div key={category} style={{ marginBottom: 8 }}>
-								<div
-									onClick={() => toggleGroup(category)}
-									style={{
-										display: "flex",
-										alignItems: "center",
-										gap: 8,
-										padding: "6px 12px",
-										background: "#1d1d1d",
-										border: "1px solid #333",
-										borderRadius: expanded ? "4px 4px 0 0" : 4,
-										cursor: "pointer",
-										userSelect: "none",
-									}}
-								>
-									<span style={{ fontSize: 10, color: "#888", width: 12 }}>
-										{expanded ? "▾" : "►"}
-									</span>
-									<Tag style={{ margin: 0 }}>{label}</Tag>
-									<Text type="secondary" style={{ fontSize: 12 }}>
-										{entries.length}
-									</Text>
-								</div>
-								{expanded && (
-									<Table<IconEntry>
-										columns={columns}
-										dataSource={entries}
-										rowKey="key"
-										size="small"
-										showHeader={false}
-										pagination={false}
-										style={{ borderTop: "none" }}
-									/>
-								)}
-							</div>
-						);
-					})}
-				</Card>
-			</div>
-		</PageLayout>
-	);
-}

packages/web_ui/src/components/InstanceRcon.tsx

@@ -1,4 +1,4 @@
-import React, { useContext, useRef, useState } from "react";
+import React, { useContext, useState } from "react";
 import { Input, Typography } from "antd";
 
 import * as lib from "@clusterio/lib";
@@ -17,7 +17,6 @@ export default function InstanceRcon(props: InstanceRconProps) {
 	let control = useContext(ControlContext);
 	let [output, setOutput] = useState<string|null>(null);
 	let [running, setRunning] = useState(false);
-	let resultRef = useRef<HTMLDivElement>(null);
 
 	async function sendCommand(command: string) {
 		if (!command) {
@@ -32,8 +31,6 @@ export default function InstanceRcon(props: InstanceRconProps) {
 				new lib.InstanceSendRconRequest(command),
 			);
 			setOutput(result);
-			const animation = resultRef.current?.children[0]?.getAnimations()[0];
-			if (animation) { animation.currentTime = 0; }
 		} finally {
 			setRunning(false);
 		}
@@ -42,7 +39,7 @@ export default function InstanceRcon(props: InstanceRconProps) {
 	return <>
 		{output && <>
 			<Title level={5}>Rcon result</Title>
-			<Paragraph ref={resultRef} code className="rcon-result">{output}</Paragraph>
+			<Paragraph code className="rcon-result">{output}</Paragraph>
 		</>}
 		<Input.Search
 			disabled={props.disabled}

packages/web_ui/src/components/InstanceStatusTag.tsx

@@ -1,27 +1,23 @@
 import { Tag } from "antd";
 import React from "react";
 
-import {
-	QuestionCircleOutlined, ClockCircleOutlined, StopOutlined, LoadingOutlined,
-	CheckCircleOutlined, SaveOutlined, ExportOutlined, DeleteOutlined,
-} from "@ant-design/icons";
-
-export const statusOptions = {
-	unknown: <Tag color="#8c8c8c" icon={<QuestionCircleOutlined />}>Unknown</Tag>,
-	unassigned: <Tag color="#eb2f96" icon={<ClockCircleOutlined />}>Unassigned</Tag>,
-	stopped: <Tag color="#cf1322" icon={<StopOutlined />}>Stopped</Tag>,
-	starting: <Tag color="#ad8b00" icon={<LoadingOutlined />}>Starting</Tag>,
-	running: <Tag color="#389e0d" icon={<CheckCircleOutlined />}>Running</Tag>,
-	stopping: <Tag color="#d48806" icon={<LoadingOutlined />}>Stopping</Tag>,
-	creating_save: <Tag color="#096dd9" icon={<SaveOutlined />}>Creating Save</Tag>,
-	exporting_data: <Tag color="#08979c" icon={<ExportOutlined />}>Exporting Data</Tag>,
-	deleted: <Tag color="#262626" icon={<DeleteOutlined />}>Deleted</Tag>,
+export const statusColors = {
+	unknown: "#8c8c8c",
+	unassigned: "#eb2f96",
+	stopped: "#cf1322",
+	starting: "#ad8b00",
+	running: "#389e0d",
+	stopping: "#d48806",
+	creating_save: "#096dd9",
+	exporting_data: "#08979c",
+	deleted: "#262626",
 };
 
 type InstanceStatusTag = {
-	status: keyof typeof statusOptions;
+	status: keyof typeof statusColors;
 };
-
 export default function InstanceStatusTag(props: InstanceStatusTag) {
-	return statusOptions[props.status];
+	return <Tag color={statusColors[props.status]}>
+		{props.status.replace("_", " ")}
+	</Tag>;
 }

packages/web_ui/src/components/InstanceViewPage.tsx

@@ -25,7 +25,6 @@ import { useHost } from "../model/host";
 import InstanceStatusTag from "./InstanceStatusTag";
 import Link from "./Link";
 import { instancePublicAddress } from "../util/instance";
-import { MetricRelativeDate } from "./system_metrics";
 
 type MenuItem = Required<MenuProps>["items"][number];
 const { Title } = Typography;
@@ -42,7 +41,6 @@ function InstanceDescription(props: InstanceDescriptionProps) {
 	return <Descriptions
 		bordered
 		size="small"
-		column={{ xs: 1, md: 3, lg: 3, xl: 3, xxl: 3 }}
 	>
 		<Descriptions.Item label="Host">
 			{!assigned
@@ -66,9 +64,7 @@ function InstanceDescription(props: InstanceDescriptionProps) {
 		<Descriptions.Item label="Version">
 			{instance.factorioVersion ?? "unknown"}
 		</Descriptions.Item>
-		<Descriptions.Item label="Last Start Time">
-			<MetricRelativeDate timeMs={instance.startedAtMs}/>
-		</Descriptions.Item>
+		<Descriptions.Item label="Status"><InstanceStatusTag status={instance.status!} /></Descriptions.Item>
 	</Descriptions>;
 }
 
@@ -101,7 +97,7 @@ function InstanceButtons(props: { instance: lib.InstanceDetails }) {
 			instanceButtonMenuItems.push({ type: "divider" });
 		}
 		instanceButtonMenuItems.push({
-			disabled: !["unknown", "unassigned", "stopped"].includes(instance.status),
+			disabled: !["unknown", "unassigned", "stopped"].includes(instance.status!),
 			danger: true,
 			key: "delete",
 			icon: <DeleteOutlined />,
@@ -212,7 +208,6 @@ export default function InstanceViewPage() {
 	return <PageLayout nav={nav}>
 		<PageHeader
 			title={instance.name ?? ""}
-			status={<InstanceStatusTag status={instance.status} />}
 			extra={<InstanceButtons instance={instance} />}
 		/>
 		<InstanceDescription host={host} instance={instance} />

packages/web_ui/src/components/ModDetails.tsx

@@ -98,7 +98,7 @@ export default function ModDetails<T extends lib.ModInfo | lib.ModRecord>(props:
 							{depWarnings.get(e.name) === "wrong_version" && <Tooltip title="Wrong version added.">
 								<FileSyncOutlined style={{ color: "#dd5e14" }} />{" "}
 							</Tooltip>}
-							{e.spec}
+							{e.specification}
 						</Space>
 						<br/>
 					</Fragment>)

packages/web_ui/src/components/ModPackViewPage.tsx

@@ -268,28 +268,16 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 
 	// State for the modal and dependencies
 	const [open, setOpen] = useState(false);
-	const [error, setError] = useState<Error | null>(null);
-	const [loading, setLoading] = useState<boolean>(false);
+	const [missing, setMissing] = useState<string[]>([]);
+	const [builtins, setBuiltins] = useState<string[]>([]);
+	const [incompatible, setIncompatible] = useState<string[]>([]);
 	const [factorioVersion, setFactorioVersion] = useState<lib.ApiVersion | null>(null);
 
-	const [mods, setMods] = useState<lib.ModInfo[]>([]); // All resolved mods and dependencies
-	const [missing, setMissing] = useState<lib.ModInfo[]>([]); // Missing mods and dependencies, excluding builtin
-
-	const [modErrors, setModErrors] = useState({
-		notFound: new Array<string>(), // Mods which were not found on the mod portal, can include builtins
-		incompatible: new Array<string>(), // Mods which are required, but are marked incompatible by another
-		unsatisfiable: new Array<string>(), // Mods which can not have their version range satisfied
-		invalidDependency: new Array<string>(), // Mods with invalid dependency specs
-	});
-
-	function resetModErrors() {
-		setModErrors({
-			notFound: [],
-			incompatible: [],
-			unsatisfiable: [],
-			invalidDependency: [],
-		});
-	}
+	// State for all mods fetched from backend
+	const [error, setError] = useState<Error | null>(null);
+	const [loading, setLoading] = useState<boolean>(false);
+	const [allMods, setAllMods] = useState<lib.ModInfo[]>([]);
+	const [mods, setMods] = useState<lib.ModInfo[]>([]);
 
 	// Get a valid factorio version
 	useEffect(() => {
@@ -306,8 +294,10 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 		if (!open || !factorioVersion) {
 			// Clear if closed or no version
 			setMods([]);
+			setAllMods([]);
 			setMissing([]);
-			resetModErrors();
+			setBuiltins([]);
+			setIncompatible([]);
 			return;
 		}
 
@@ -319,29 +309,21 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 			lib.ModDependencyResolveRequest.fromModPackEnabled(props.modPack, props.checkForUpdates)
 		).then(response => {
 			if (canceled) { return; }
-
-			const newModErrors = [...response.errors.entries()]
-				.reduce<typeof modErrors>((acc, [modName, modError]) => {
-					acc[modError].push(modName);
-					return acc;
-				}, {
-					notFound: [],
-					incompatible: [],
-					unsatisfiable: [],
-					invalidDependency: [],
-				});
-
-			setModErrors(newModErrors);
-			setMods(response.dependencies);
-			setMissing(response.dependencies.filter(dep => !modInfos.has(`${dep.name}_${dep.version}`)));
+			setMods(response.dependencies.filter(dep => !modInfos.has(`${dep.name}_${dep.version}`)));
+			setIncompatible(response.incompatible);
+			setAllMods(response.dependencies);
+			setMissing(response.missing.filter(name => !props.builtInModNames.includes(name)));
+			setBuiltins(response.missing.filter(name => props.builtInModNames.includes(name)));
 			setLoading(false);
 		}).catch(err => {
 			if (canceled) { return; }
 			notifyErrorHandler("Error fetching mods dependencies")(err);
 			setError(err);
 			setMods([]);
+			setAllMods([]);
 			setMissing([]);
-			resetModErrors();
+			setBuiltins([]);
+			setIncompatible([]);
 			setLoading(false);
 		});
 
@@ -352,56 +334,51 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 		// Re-fetch when modal opens or factorio version changes
 	}, [open, factorioVersion, control]);
 
-	// Fix any mod errors found after resolving dependencies
-	function fixModErrors() {
-		if (!factorioVersion) {
+	// Function to handle download from Mod Portal to Controller
+	function handleControllerDownload() {
+		if (!factorioVersion || !mods) {
 			return;
 		}
 
-		// Enable any required builtins, they will be within the not found list
-		for (const modName of modErrors.notFound) {
-			if (props.builtInModNames.includes(modName)) {
-				props.onChange({
-					type: "mods.set",
-					name: modName,
-					value: { ...props.modPack.mods.get(modName)!, enabled: true },
-				});
-			}
+		// Enable any required builtins
+		for (const name of builtins) {
+			props.onChange({
+				type: "mods.set",
+				name: name,
+				value: { ...props.modPack.mods.get(name)!, enabled: true },
+			});
 		}
 
-		// Disable any incompatible mods
-		for (const modName of modErrors.incompatible) {
-			const modRecord = props.modPack.mods.get(modName);
+		// Disable any incompatible mods present
+		for (const name of incompatible) {
+			const modRecord = props.modPack.mods.get(name);
 			if (modRecord) {
 				props.onChange({
 					type: "mods.set",
-					name: modName,
+					name: name,
 					value: { ...modRecord, enabled: false },
 				});
 			}
 		}
 
 		// Add all dependencies to the mod pack
-		for (const mod of mods) {
-			const existingMod = props.modPack.mods.get(mod.name);
-			if (!existingMod || existingMod.version !== mod.version) {
-				props.onChange({
-					type: "mods.set",
+		for (const mod of allMods) {
+			props.onChange({
+				type: "mods.set",
+				name: mod.name,
+				value: {
 					name: mod.name,
-					value: {
-						name: mod.name,
-						version: mod.version,
-						sha1: mod.sha1,
-						enabled: true,
-					},
-				});
-			}
+					version: mod.version,
+					sha1: mod.sha1,
+					enabled: true,
+				},
+			});
 		}
 
-		// Download any missing mods
+		// Download the missing dependencies
 		control.send(
 			new lib.ModPortalDownloadRequest(
-				missing.map(mod => ({
+				mods.map(mod => ({
 					name: mod.name, version: new lib.ModVersionEquality("=", mod.version),
 				})),
 				factorioVersion,
@@ -409,12 +386,12 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 		).then(() => {
 			notification.success({
 				message: "Download complete",
-				description: `${missing.length} mods have been downloaded to the controller.`,
+				description: `${mods.length} mods have been downloaded to the controller.`,
 			});
 		}).catch(
 			notifyErrorHandler("Error starting mod download")
 		);
-	}
+	};
 
 	return <>
 		{props.checkForUpdates && <Button
@@ -438,11 +415,8 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 				<Button
 					key="start"
 					type="primary"
-					disabled={loading || missing.length === 0}
-					onClick={() => {
-						setOpen(false);
-						fixModErrors();
-					}}
+					disabled={loading || mods.length === 0}
+					onClick={() => { setOpen(false); handleControllerDownload(); }}
 				>Start Download</Button>,
 			]}
 		>
@@ -455,26 +429,30 @@ function DownloadDependenciesButton(props: DownloadDependenciesProps) {
 				style={{ marginBottom: 16 }}
 			/>}
 
-			{/* Display warning message for mod errors */}
-			{([
-				["notFound", "Failed to find dependencies:"],
-				["incompatible", "Dependencies are incompatible:"],
-				["unsatisfiable", "Dependencies could not be satisfied:"],
-				["invalidDependency", "Dependencies has invalid specifications:"],
-			] as const).map(([prop, message]) => {
-				const modNames = modErrors[prop];
-				return modNames.length === 0 ? null : <Alert
-					showIcon
-					type="warning"
-					message={message}
-					description={<ul>{modNames.map(name => <li>{name}</li>)}</ul>}
-					style={{ marginBottom: 14 }}
-				/>;
-			})}
+			{/* Display warning message if dependencies could not be found */}
+			{!loading && missing.length > 0 && <Alert
+				message="Failed to Find Dependencies:"
+				description={<ul>
+					{missing.map(name => <li>{name}</li>)}
+				</ul>}
+				type="warning"
+				showIcon
+				style={{ marginBottom: 16 }}
+			/>}
+
+			{/* Display warning message if dependencies are incompatible */}
+			{!loading && incompatible.length > 0 && <Alert
+				message="Incompatible Dependencies Found:"
+				description={<ul>
+					{incompatible.map(name => <li>{name}</li>)}
+				</ul>}
+				type="warning"
+				showIcon
+				style={{ marginBottom: 16 }}
+			/>}
 
-			{/* Display a table of the missing mods that need to be downloaded */}
 			<Table
-				dataSource={missing}
+				dataSource={mods}
 				rowKey={record => record.name}
 				loading={loading}
 				pagination={{

packages/web_ui/src/components/NpmButton.tsx

@@ -82,11 +82,7 @@ function NpmPluginInstall({ setApplyAction, form, target }: NpmActionProps) {
 	</>;
 }
 
-export function NpmButton(props: {
-	target: "controller" | { hostId: number },
-	canRestart?: boolean,
-	disabled?: boolean
-}) {
+export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: boolean, disabled?: boolean }) {
 	const control = useContext(ControlContext);
 	const account = useAccount();
 	const [open, setOpen] = useState(false);
@@ -97,7 +93,6 @@ export function NpmButton(props: {
 		remoteUpdates: true, pluginUpdates: true, pluginInstall: true,
 	});
 
-	const hostId = props.target === "controller" ? undefined : props.target.hostId;
 	useEffect(() => {
 		if (props.target === "controller") {
 			if (!account.hasPermission("core.controller.get_config")) {
@@ -126,7 +121,7 @@ export function NpmButton(props: {
 				});
 			}).catch(notifyErrorHandler("Failed to fetch host config for remote updates"));
 		}
-	}, [hostId]);
+	}, [props.target]);
 
 	function onValuesChange({ action } : { action?: string }) {
 		if (action) {

packages/web_ui/src/components/PageHeader.tsx

@@ -5,7 +5,6 @@ const { Text } = Typography;
 type PageHeaderProps = {
 	title: string | React.ReactElement;
 	subTitle?: string | React.ReactElement;
-	status?: React.JSX.Element;
 	extra?: React.JSX.Element;
 };
 
@@ -14,7 +13,6 @@ export default function PageHeader(props: PageHeaderProps): React.JSX.Element {
 		<Space>
 			<h2 className="page-header-title">{props.title}</h2>
 			{props.subTitle && <Text type="secondary">{props.subTitle}</Text>}
-			{props.status ?? null}
 		</Space>
 		{props.extra && <div className="page-header-extra">{props.extra}</div>}
 	</div>;

packages/web_ui/src/components/PluginExtra.tsx

@@ -9,7 +9,7 @@ import BaseWebPlugin from "../BaseWebPlugin";
 type PluginExtraProps = {
 	component: string;
 	instance?: lib.InstanceDetails;
-	user?: lib.UserDetails;
+	user?: lib.User;
 	host?: lib.HostDetails;
 	role?: lib.Role;
 };

packages/web_ui/src/components/UsersTable.tsx

@@ -44,7 +44,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 	const data = [...users.values()];
 
 	// Determine online predicate based on instanceId
-	const isUserOnline = (user: lib.UserDetails) => {
+	const isUserOnline = (user: lib.User) => {
 		if (instanceId !== undefined) {
 			return user.instances && user.instances.has(instanceId);
 		}
@@ -58,7 +58,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 		{
 			title: "Name",
 			key: "name",
-			render: (_: any, user: lib.UserDetails) => (
+			render: (_: any, user: lib.User) => (
 				<Space>
 					{user.name}
 					<span>
@@ -69,11 +69,11 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 				</Space>
 			),
 			defaultSortOrder: "ascend",
-			sorter: (a: lib.UserDetails, b: lib.UserDetails) => strcmp(a.name, b.name),
+			sorter: (a: lib.User, b: lib.User) => strcmp(a.name, b.name),
 			filterIcon: (filtered: boolean) => (
 				<SearchOutlined style={{ color: filtered ? "#1677ff" : undefined }} />
 			),
-			onFilter: (value: string | number | boolean, record: lib.UserDetails) => record.name
+			onFilter: (value: string | number | boolean, record: lib.User) => record.name
 				.toLowerCase()
 				.includes((value as string).toLowerCase()),
 			filterDropdownProps: {
@@ -106,10 +106,8 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			key: "roles",
 			filters: roleFilters,
 			filterMultiple: true,
-			onFilter: (value: string | number | boolean, record: lib.UserDetails) => (
-				record.roleIds.has(value as number)
-			),
-			render: (_: any, user: lib.UserDetails) => (
+			onFilter: (value: string | number | boolean, record: lib.User) => record.roleIds.has(value as number),
+			render: (_: any, user: lib.User) => (
 				[...user.roleIds].map((id) => (
 					<Link key={id} to={`/roles/${id}/view`} onClick={(e) => e.stopPropagation()}>
 						<Tag>{(roles.get(id) || { name: id }).name}</Tag>
@@ -121,7 +119,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 
 
 	// Helper to get last seen timestamp
-	function getLastSeenTimestamp(user: lib.UserDetails): number | undefined {
+	function getLastSeenTimestamp(user: lib.User): number | undefined {
 		const stats = instanceId !== undefined ? user.instanceStats.get(instanceId) : user.playerStats;
 		if (!stats) { return undefined; }
 		if (stats.lastLeaveAt && stats.lastLeaveAt.getTime() > (stats.lastJoinAt?.getTime() ?? 0)) {
@@ -138,11 +136,11 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "Play Time",
 				key: "playTime",
-				render: (_: any, user: lib.UserDetails) => {
+				render: (_: any, user: lib.User) => {
 					const instanceStats = user.instanceStats.get(instanceId);
 					return instanceStats?.onlineTimeMs ? formatDuration(instanceStats.onlineTimeMs) : "-";
 				},
-				sorter: (a: lib.UserDetails, b: lib.UserDetails) => {
+				sorter: (a: lib.User, b: lib.User) => {
 					const statsA = a.instanceStats.get(instanceId);
 					const statsB = b.instanceStats.get(instanceId);
 					return (statsA?.onlineTimeMs ?? 0) - (statsB?.onlineTimeMs ?? 0);
@@ -151,11 +149,11 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "Join Count",
 				key: "joinCount",
-				render: (_: any, user: lib.UserDetails) => {
+				render: (_: any, user: lib.User) => {
 					const instanceStats = user.instanceStats.get(instanceId);
 					return instanceStats?.joinCount ?? 0;
 				},
-				sorter: (a: lib.UserDetails, b: lib.UserDetails) => {
+				sorter: (a: lib.User, b: lib.User) => {
 					const statsA = a.instanceStats.get(instanceId);
 					const statsB = b.instanceStats.get(instanceId);
 					return (statsA?.joinCount ?? 0) - (statsB?.joinCount ?? 0);
@@ -164,8 +162,8 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "First Seen",
 				key: "firstSeen",
-				render: (_: any, user: lib.UserDetails) => formatFirstSeen(user, instanceId),
-				sorter: (a: lib.UserDetails, b: lib.UserDetails) => {
+				render: (_: any, user: lib.User) => formatFirstSeen(user, instanceId),
+				sorter: (a: lib.User, b: lib.User) => {
 					const statsA = a.instanceStats.get(instanceId);
 					const statsB = b.instanceStats.get(instanceId);
 					const firstSeenA = statsA?.firstJoinAt?.getTime() ?? 0;
@@ -179,18 +177,18 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 			{
 				title: "Online time",
 				key: "onlineTime",
-				render: (_: any, user: lib.UserDetails) => (user.playerStats?.onlineTimeMs
+				render: (_: any, user: lib.User) => (user.playerStats?.onlineTimeMs
 					? formatDuration(user.playerStats.onlineTimeMs)
 					: null),
 				// eslint-disable-next-line max-len
-				sorter: (a: lib.UserDetails, b: lib.UserDetails) => (a.playerStats?.onlineTimeMs ?? 0) - (b.playerStats?.onlineTimeMs ?? 0),
+				sorter: (a: lib.User, b: lib.User) => (a.playerStats?.onlineTimeMs ?? 0) - (b.playerStats?.onlineTimeMs ?? 0),
 				responsive: ["lg"],
 			},
 			{
 				title: "First seen",
 				key: "firstSeen",
-				render: (_: any, user: lib.UserDetails) => formatFirstSeen(user),
-				sorter: (a: lib.UserDetails, b: lib.UserDetails) => sortFirstSeen(a, b),
+				render: (_: any, user: lib.User) => formatFirstSeen(user),
+				sorter: (a: lib.User, b: lib.User) => sortFirstSeen(a, b),
 			},
 		);
 	}
@@ -206,7 +204,7 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 				{ text: "7d", value: "7d" },
 				{ text: "30d", value: "30d" },
 			],
-			onFilter: (value: string | number | boolean, record: lib.UserDetails) => {
+			onFilter: (value: string | number | boolean, record: lib.User) => {
 				const online = isUserOnline(record);
 				if (value === "online") {
 					return online;
@@ -229,8 +227,8 @@ export default function UsersTable({ instanceId, onlyOnline = false, pagination,
 						return false;
 				}
 			},
-			sorter: (a: lib.UserDetails, b: lib.UserDetails) => sortLastSeen(a, b, instanceId, instanceId),
-			render: (_: any, user: lib.UserDetails) => formatLastSeen(user, instanceId),
+			sorter: (a: lib.User, b: lib.User) => sortLastSeen(a, b, instanceId, instanceId),
+			render: (_: any, user: lib.User) => formatLastSeen(user, instanceId),
 			responsive: ["lg"],
 		},
 	);

packages/web_ui/src/components/system_metrics.tsx

@@ -63,52 +63,3 @@ export function RestartRequired(props: { system?: lib.SystemInfo }) {
 		<ReloadOutlined style={{ color: "yellow" }}/>
 	</Tooltip>;
 }
-
-/**
- * Returns a human-readable duration between two dates.
- *
- * The output includes up to two most significant time units and omits
- * leading zero units. Supported units are weeks, days, hours, minutes,
- * and seconds.
- *
- * Examples: "1w 2d", "2d 3h", "1h 2m", "45s"
- *
- * @param date1 - The starting date.
- * @param date2 - The ending date.
- * @returns A duration string (e.g. "2d 3h").
- */
-function humanAbsTimeDiff(date1: number, date2 = 0) {
-	let diffMs = Math.abs(date2 - date1);
-	const totalSeconds = Math.floor(diffMs / 1000);
-
-	const weeks = Math.floor(totalSeconds / 604800); // 7 * 24 * 60 * 60
-	const days = Math.floor((totalSeconds % 604800) / 86400);
-	const hours = Math.floor((totalSeconds % 86400) / 3600);
-	const minutes = Math.floor((totalSeconds % 3600) / 60);
-	const seconds = totalSeconds % 60;
-
-	if (weeks > 0) {
-		return days > 0 ? `${weeks}w ${days}d` : `${weeks}w`;
-	}
-
-	if (days > 0) {
-		return hours > 0 ? `${days}d ${hours}h` : `${days}d`;
-	}
-
-	if (hours > 0) {
-		return minutes > 0 ? `${hours}h ${minutes}m` : `${hours}h`;
-	}
-
-	if (minutes > 0) {
-		return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
-	}
-
-	return `${seconds}s`;
-}
-
-export function MetricRelativeDate(props: { timeMs?: number }) {
-	if (!props.timeMs || props.timeMs === 0) {
-		return "N/A";
-	}
-	return `${new Date(props.timeMs).toLocaleString()} (${humanAbsTimeDiff(props.timeMs, Date.now())})`;
-}

packages/web_ui/src/index.ts

@@ -21,7 +21,7 @@ export { default as LogConsole } from "./components/LogConsole";
 export { default as AssignInstanceModal } from "./components/AssignInstanceModal";
 export { default as InstanceList } from "./components/InstanceList";
 export { default as InstanceRcon } from "./components/InstanceRcon";
-export { default as InstanceStatusTag } from "./components/InstanceStatusTag";
+export { default as InstanceStatusTag, statusColors } from "./components/InstanceStatusTag";
 export { default as InstanceControlButton, InstanceControlButtonPermissions } from "./components/InstanceControlButton";
 
 export { default as PageHeader } from "./components/PageHeader";

packages/web_ui/src/model/export_manifest.ts

@@ -4,10 +4,10 @@ import * as lib from "@clusterio/lib";
 import ControlContext from "../components/ControlContext";
 
 let exportManifestCache: lib.ExportManifest|null = null;
-let loaded = false;
+let emptyCache = { assets: {} };
 export function useExportManifest(): lib.ExportManifest|null {
 	const control = useContext(ControlContext);
-	let [exportManifest, setExportManifest] = useState<lib.ExportManifest|null>(exportManifestCache);
+	let [exportManifest, setExportManifest] = useState<lib.ExportManifest|null>(exportManifestCache || emptyCache);
 	useEffect(() => {
 		async function load() {
 			let modPack = await control.send(new lib.ModPackGetDefaultRequest());
@@ -15,10 +15,9 @@ export function useExportManifest(): lib.ExportManifest|null {
 				exportManifestCache = modPack.exportManifest;
 				setExportManifest(exportManifestCache);
 			}
-			loaded = true;
 		}
 
-		if (!loaded) {
+		if (!exportManifestCache) {
 			load();
 		}
 	}, []);

packages/web_ui/src/model/item_metadata.ts

@@ -7,51 +7,31 @@ type Metadata = {
 	x: number;
 	y: number;
 	size: number;
-	category: string;
 	localised_name?: any[];
-	path?: string;
 };
-
-const metadataCaches = new Map<string, Map<string, Metadata>>();
-let cssInjected = false;
-
-/**
- * Load the unified spritesheet metadata and inject CSS rules for every icon.
- * CSS class name: `.{category}-{CSS.escape(name)}`
- * e.g. `.item-iron-plate`, `.recipe-iron-plate`, `.signal-signal-red`, `.planet-nauvis`
- */
-function useSpriteMetadata(category: string): Map<string, Metadata> {
-	const exportManifest = useExportManifest();
-	const [metadata, setMetadata] = useState<Map<string, Metadata>>(
-		metadataCaches.get(category) ?? new Map()
-	);
-
+let itemMetadataCache: Map<string, Metadata> | null = null;
+export function useItemMetadata() {
+	let exportManifest = useExportManifest();
+	let [itemMetadata, setItemMetadata] = useState<Map<string, Metadata>>(itemMetadataCache || new Map());
 	useEffect(() => {
 		async function load() {
 			if (
 				!exportManifest
-				|| !exportManifest.assets["metadata"]
-				|| !exportManifest.assets["spritesheet"]
+				|| !exportManifest.assets["item-metadata"]
+				|| !exportManifest.assets["item-spritesheet"]
 			) {
 				return;
 			}
-
-			const response = await fetch(`${staticRoot}static/${exportManifest.assets["metadata"]}`);
-			if (!response.ok) {
-				return;
-			}
-
-			const data: [string, Metadata][] = await response.json();
-
-			if (!cssInjected) {
-				const sheetUrl = `${staticRoot}static/${exportManifest.assets["spritesheet"]}`;
-				const style = document.createElement("style");
+			let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`);
+			if (response.ok) {
+				let data = await response.json();
+				itemMetadataCache = new Map(data);
+				let style = document.createElement("style");
 				document.head.appendChild(style);
-				for (const [key, meta] of data) {
-					const name = key.includes(".") ? key.slice(key.indexOf(".") + 1) : key;
+				for (let [name, meta] of itemMetadataCache) {
 					style.sheet!.insertRule(`\
-.${meta.category}-${CSS.escape(name)} {
-	background-image: url("${sheetUrl}");
+.item-${CSS.escape(name)} {
+	background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}");
 	background-repeat: no-repeat;
 	background-position: -${meta.x}px -${meta.y}px;
 	height: ${meta.size}px;
@@ -59,32 +39,14 @@ function useSpriteMetadata(category: string): Map<string, Metadata> {
 }`
 					);
 				}
-				cssInjected = true;
+				setItemMetadata(itemMetadataCache);
 			}
-
-			const cache = new Map<string, Metadata>();
-			for (const [key, meta] of data) {
-				if (meta.category === category) {
-					const name = key.includes(".") ? key.slice(key.indexOf(".") + 1) : key;
-					cache.set(name, meta);
-				}
-			}
-			metadataCaches.set(category, cache);
-			setMetadata(cache);
 		}
 
-		if (!metadataCaches.has(category)) {
+		if (!itemMetadataCache) {
 			load();
 		}
 	}, [exportManifest]);
 
-	return metadata;
+	return itemMetadata;
 }
-
-export function useItemMetadata() { return useSpriteMetadata("item"); }
-export function useRecipeMetadata() { return useSpriteMetadata("recipe"); }
-export function useSignalMetadata() { return useSpriteMetadata("signal"); }
-export function useTechnologyMetadata() { return useSpriteMetadata("technology"); }
-export function usePlanetMetadata() { return useSpriteMetadata("planet"); }
-export function useQualityMetadata() { return useSpriteMetadata("quality"); }
-export function useEntityMetadata() { return useSpriteMetadata("entity"); }

packages/web_ui/src/model/roles.ts

@@ -3,7 +3,7 @@ import ControlContext from "../components/ControlContext";
 
 import * as lib from "@clusterio/lib";
 
-export function useRolesOfUser(user?: lib.UserDetails) {
+export function useRolesOfUser(user?: lib.User) {
 	const [roles, synced] = useRoles();
 	return [
 		user !== undefined ? new Map([...user.roleIds].map(roleId => [roleId, roles.get(roleId)])) : new Map(),

packages/web_ui/src/model/user.tsx

@@ -4,19 +4,19 @@ import ControlContext from "../components/ControlContext";
 
 import * as lib from "@clusterio/lib";
 
-function getInstanceStats(user: lib.UserDetails, instanceId?: number) {
+function getInstanceStats(user: lib.User, instanceId?: number) {
 	if (instanceId === undefined) {
 		return user.playerStats;
 	}
 	return user.instanceStats.get(instanceId);
 }
 
-function calculateFirstSeen(user: lib.UserDetails, instanceId?: number) {
+function calculateFirstSeen(user: lib.User, instanceId?: number) {
 	const stats = getInstanceStats(user, instanceId);
 	return stats?.firstJoinAt?.getTime();
 }
 
-function calculateLastSeen(user: lib.UserDetails, instanceId?: number) {
+function calculateLastSeen(user: lib.User, instanceId?: number) {
 	const stats = getInstanceStats(user, instanceId);
 	if (!stats) {
 		return undefined;
@@ -30,7 +30,7 @@ function calculateLastSeen(user: lib.UserDetails, instanceId?: number) {
 	return undefined;
 }
 
-export function formatFirstSeen(user: lib.UserDetails, instanceId?: number) {
+export function formatFirstSeen(user: lib.User, instanceId?: number) {
 	const firstSeen = calculateFirstSeen(user, instanceId);
 	if (firstSeen === undefined) {
 		return undefined;
@@ -38,7 +38,7 @@ export function formatFirstSeen(user: lib.UserDetails, instanceId?: number) {
 	return new Date(firstSeen).toLocaleString();
 }
 
-export function formatLastSeen(user: lib.UserDetails, instanceId?: number) {
+export function formatLastSeen(user: lib.User, instanceId?: number) {
 	if (user.instances && [...user.instances].some(id => instanceId === undefined || id === instanceId)) {
 		return <Tag color="green">Online</Tag>;
 	}
@@ -49,24 +49,14 @@ export function formatLastSeen(user: lib.UserDetails, instanceId?: number) {
 	return new Date(lastSeen).toLocaleString();
 }
 
-export function sortFirstSeen(
-	userA: lib.UserDetails,
-	userB: lib.UserDetails,
-	instanceIdA?: number,
-	instanceIdB?: number
-) {
+export function sortFirstSeen(userA: lib.User, userB: lib.User, instanceIdA?: number, instanceIdB?: number) {
 	const firstSeenA = calculateFirstSeen(userA, instanceIdA) || 0;
 	const firstSeenB = calculateFirstSeen(userB, instanceIdB) || 0;
 	return firstSeenA - firstSeenB;
 }
 
-export function sortLastSeen(
-	userA: lib.UserDetails,
-	userB: lib.UserDetails,
-	instanceIdA?: number,
-	instanceIdB?: number
-) {
-	function epoch(user: lib.UserDetails, instanceId?: number) {
+export function sortLastSeen(userA: lib.User, userB: lib.User, instanceIdA?: number, instanceIdB?: number) {
+	function epoch(user: lib.User, instanceId?: number) {
 		return user.instances && [...user.instances].some(id => instanceId === undefined || id === instanceId);
 	}
 

packages/web_ui/src/pages.tsx

@@ -16,7 +16,6 @@ import RoleViewPage from "./components/RoleViewPage";
 import PluginsPage from "./components/PluginsPage";
 import PluginViewPage from "./components/PluginViewPage";
 import ErrorPage from "./components/ErrorPage";
-import IconReferencePage from "./components/IconReferencePage";
 
 export const pages: PluginPage[] = [
 	{
@@ -89,10 +88,6 @@ export const pages: PluginPage[] = [
 		sidebarPath: "/plugins",
 		content: <PluginViewPage />,
 	},
-	{
-		path: "/icon-reference",
-		content: <IconReferencePage />,
-	},
 	{
 		path: "/error",
 		content: <ErrorPage throw error={new Error("Example Error Message")} />,