2.0.0-alpha.24

Add variables specifying the repository and branch that the changelog
script parser operates on in order to make it easier to use this script
on other projects.

.github/pull_request_template.md

@@ -1,10 +1,15 @@
-Describe why and what has been changed by this pullrequest, feel free to also ask for input on decisions you are unsure about etc.
-The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferrentially link to the associated issues. If there are no relevant issues, link to the pull request itself.
+<!--
+Describe why and what has been changed by this pull request, feel free to also ask for input on decisions you are unsure about etc.
 
-## Changelog
+The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferentially link to the associated issues. If there are no relevant issues, link to the pull request itself.
+
+If there are no relevant changelog entries replace the code block below with "None." followed by an optional reason for why it is not needed.
+-->
+
+### Changelog
 ```
 ### Features
-- Added doodad to web interface. [#999](https://github.com/clusterio/clusterio/issues/999)
+- Added doodad to web interface. #999
 
 ### Changes
 - ...
@@ -12,6 +17,6 @@ The changelog lines below will be copied directly from merged pull requests to t
 ### Fixes
 - ...
 
-### Breaking changes
+### Breaking Changes
 - ...
 ```

.github/workflows/node.js.yml

@@ -40,6 +40,12 @@ jobs:
       with:
         files: ./coverage/lcov.info
         token: ${{ secrets.CODECOV_TOKEN }}
+    - name: Upload test logs for debugging
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: logs-${{ matrix.node-version }}-${{ matrix.factorio-version }}
+        path: temp/test/logs
 
   lint:
     runs-on: ubuntu-latest

.github/workflows/publish.yml

@@ -13,8 +13,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-        with:
-          version: 8
       - uses: actions/setup-node@v4
         with:
           node-version: 22.x
@@ -33,8 +31,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-        with:
-          version: 8
       - uses: actions/setup-node@v4
         with:
           node-version: 22.x

.gitignore

@@ -29,22 +29,13 @@ dist/
 
 node_modules
 
-Factorio*
-!*Factorio*.ts
-!*Factorio*.tsx
-!*Factorio*.js
-/factorio
-!/test/file/factorio
-!/test/file/html/*.html
-!/test/lib/factorio
-!/packages/lib/src/factorio
+/factorio*
 /instances
 /mods
 /database
 /plugin-list.json
-/config-controller.json
-/config-control.json
-/config-host.json
+/config-*.json
+/config-*.json.lock
 pnpm-lock.yaml
 mod_cache.json
 /static/

CHANGELOG.md

@@ -31,6 +31,110 @@ Many thanks to the following for contributing to this release:
 [@username](https://github.com/username)
 -->
 
+## Version 2.0.0-alpha.24
+
+### Features
+
+- Added `FactorioIcon` component for showing icons from the game in the web UI. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Added `useDefaultModPack` hook to the web UI. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Added types for the exported metadata for mod packs. [#526](https://github.com/clusterio/clusterio/issues/526)
+- Added config value validation beyond simple type checks. [#856](https://github.com/clusterio/clusterio/pull/856)
+- Added config staging to allow setting many values at once. [#856](https://github.com/clusterio/clusterio/pull/856)
+- Added a warning when attempting to set player autosave slots lower than autosave slots. [#855](https://github.com/clusterio/clusterio/issues/855)
+- Added `controller.authSecret` as the type of config field `controller.auth_secret` was inconsistent. [#856](https://github.com/clusterio/clusterio/pull/856)
+- Added support for the `node:` prefix in import statements in web UI code. [#881](https://github.com/clusterio/clusterio/pull/881)
+- Added config validation to load plugin to error when save patching or script commands are disabled [#884](https://github.com/clusterio/clusterio/issues/884)
+
+### Fixes
+
+- Fixed host printing a deprecation warning on startup. [#855](https://github.com/clusterio/clusterio/issues/855)
+- Fixed exported prototype metadata missing localised_name. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Fixed headless download option in installer failing with `TypeError: callback is not a function`. [#860](https://github.com/clusterio/clusterio/issues/860)
+- Fixed controller crashing if a host attempted to download a mod that had been deleted from the controller's filesystem while the controller was running. [#860](https://github.com/clusterio/clusterio/issues/860)
+
+### Changes
+
+- Moved `/icon-reference` to `/mods/mod-packs/:id/icon-reference` and linked it from the mod pack page. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Made the `useAccount`, `useModPacks`, `useMods`, `useRoles`, `useSaves`, `useSystems`, `useUsers` and related hooks in the web UI available to plugins. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Exported prototypes are now based on the `defines.prototypes` mapping from the game. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Removed usage of fs-extra. [#860](https://github.com/clusterio/clusterio/issues/860)
+
+### Breaking Changes
+
+- Removed `useExportManifest` from the web UI. Use `.exportManifest` on a mod pack from one of the `useModPack` hooks instead. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Replaced the 8 `useTypeMetadata` hooks in the web UI such as `useItemMetadata` with `useExportPrototypeMetadata`. [#875](https://github.com/clusterio/clusterio/pull/875)
+- `.item-name` css classes are no longer created in the web UI, and will not render icons from the game. Use the `FactorioIcon` component instead and pass it a prototype from `useExportPrototypeMetadata`. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Renamed `useLocale` hook in web UI to `useExportLocale` which now requires the mod pack to load the exported locale from as an argument. [#875](https://github.com/clusterio/clusterio/pull/875)
+- Fluids are no longer merged with items in the exported prototype metadata for mod packs. [#875](https://github.com/clusterio/clusterio/pull/875)
+
+### Meta
+
+- Changelog entries are now parsed with a script from pull request descriptions. [#871](https://github.com/clusterio/clusterio/pull/871)
+
+Many thanks to the following for contributing to this release:  
+[@Cooldude2606](https://github.com/Cooldude2606)
+[@Hornwitser](https://github.com/Hornwitser)
+
+## Version 2.0.0-alpha.23
+
+### Major Features
+
+- Added automatic headless download and update on linux. [#330](https://github.com/clusterio/clusterio/issues/330)
+- Added recipe notification sync to inventory sync. [#506](https://github.com/clusterio/clusterio/issues/506)
+
+### Features
+
+- Added config lock files to prevent process duplication. [#815](https://github.com/clusterio/clusterio/issues/815)
+- Added process and system start times to system metrics. [#826](https://github.com/clusterio/clusterio/issues/826)
+- Added icons to status tags for host and instances. [#827](https://github.com/clusterio/clusterio/issues/827)
+- Extended export-data to produce icons for recipes, signals, technologies, planets, qualities and entities. [#838](https://github.com/clusterio/clusterio/pull/838)
+- Added Icon Reference page to web interface at `/icon-reference` for browsing all exported icons. [#838](https://github.com/clusterio/clusterio/pull/838)
+- Added visual flash to RCON result box to confirm command execution, even when the output is identical to the previous result. [#793](https://github.com/clusterio/clusterio/issues/793)
+
+### Changes
+
+- Login screen now displays the cluster name. [#818](https://github.com/clusterio/clusterio/issues/818)
+- Instance versions can now be set using Major.Minor without specifying a patch version. [#820](https://github.com/clusterio/clusterio/issues/820)
+- Versions for factorio and mod packs can now be selected using a drop down menu. [#820](https://github.com/clusterio/clusterio/issues/820)
+- The controller will now refuse to start if no users were loaded from file. [#814](https://github.com/clusterio/clusterio/issues/814)
+- Improved wording and display of the generate host token modal. [#828](https://github.com/clusterio/clusterio/issues/828)
+- Installations where multiple conflicting clusterio/lib packages have been installed will now fail with an error [#863](https://github.com/clusterio/clusterio/issues/863).
+- Export completion now logs per-category icon counts. [#838](https://github.com/clusterio/clusterio/pull/838)
+
+### Fixes
+
+- Removed debug log statement from equipment grid serialization. [#807](https://github.com/clusterio/clusterio/pull/807)
+- Fix that the documentation had a dead link to details of the message passing protocol [#812](https://github.com/clusterio/clusterio/issues/812)
+- Fixed case where achievements were not disabled. [#817](https://github.com/clusterio/clusterio/pull/817)
+- Fix headless download on linux install. [#816](https://github.com/clusterio/clusterio/issues/816)
+- Fix option `--no-download-headless` being ignored. [#816](https://github.com/clusterio/clusterio/issues/816)
+- Fixed action buttons overflowing on narrow screens across the web interface. [#810](https://github.com/clusterio/clusterio/issues/810)
+- Fixed player logistics failing to load on servers without unlocked logistics networks. [#806](https://github.com/clusterio/clusterio/issues/806)
+- Fixed mod dependency parser breaking with inconsistent spaces. [#834](https://github.com/clusterio/clusterio/issues/834)
+- No longer attempts to sync inventory in single player. [#835](https://github.com/clusterio/clusterio/issues/835)
+- Fixed tests failing when the configured factorio_directory contains stuff other than Factorio installs [#540](https://github.com/clusterio/clusterio/issues/540).
+- Always create a whitelist file so it can be watched for changes. [#865](https://github.com/clusterio/clusterio/issues/865)
+- Added missing await statements which could cause an instance to start with an outdated admin/ban/whitelist. [#869](https://github.com/clusterio/clusterio/issues/869)
+- Fixed metrics breaking when labels contain comma or equal signs [#861](https://github.com/clusterio/clusterio/issues/861)
+- Fixed Host page show repeated errors when viewing an offline local host [#848](https://github.com/clusterio/clusterio/issues/848)
+- Fixed bad mod paths in icon specifications crashing the export. [#841](https://github.com/clusterio/clusterio/issues/841)
+- Fixed prototypes with no icon data causing errors during export. [#841](https://github.com/clusterio/clusterio/issues/841)
+
+### Breaking Changes
+
+- Custom permission methods should use `IUser` rather than `IControllerUser` [#837](https://github.com/clusterio/clusterio/pull/837)
+- `controller.userManager` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837)
+- `controller.instances` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837)
+- `User/Instance/HostInfo` renamed to `User/Instance/HostRecord` [#837](https://github.com/clusterio/clusterio/pull/837)
+- Removed `host.tls_ca` and `control.tls_ca` config options, see [section on self signed certificates](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate) for the replacement for this option [#857](https://github.com/clusterio/clusterio/issues/857)
+
+Many thanks to the following for contributing to this release:  
+[@Cooldude2606](https://github.com/Cooldude2606)
+[@Daniel](https://github.com/Danielv123)
+[@Double](https://github.com/DoubleThoughtTheProgrammer)
+[@Hornwitser](https://github.com/Hornwitser)
+[@Ryan Walker](https://github.com/solarcloud7)
+
 ## Version 2.0.0-alpha.22
 
 ### Major Features

clean.js

@@ -8,7 +8,7 @@
 
 "use strict";
 const path = require("path");
-const fs = require("fs-extra");
+const fs = require("node:fs/promises");
 const yargs = require("yargs");
 
 const npmPackage = require("./package.json");
@@ -24,13 +24,15 @@ let DRY = false;
  * @param {string} filePath The path to remove.
  */
 async function tryRemove(filePath) {
-	if (await fs.exists(filePath)) {
-		// eslint-disable-next-line no-console
+	try {
+		await fs.access(filePath);
 		console.log(filePath);
+	} catch {
+		return;
 	}
 
 	if (!DRY) {
-		await fs.remove(filePath);
+		await fs.rm(filePath, { force: true, recursive: true, maxRetries: 10 });
 	}
 }
 
@@ -104,7 +106,7 @@ async function main() {
 			removeBuildArtifacts();
 			removeTestArtifacts();
 		})
-		.command("fast", "Remove all build artifacts", () => {}, argv => {
+		.command("$0", "Remove all build artifacts", () => {}, argv => {
 			DRY = argv.dry;
 			removeBuildArtifacts();
 		})

docs/configuration.md

@@ -182,14 +182,6 @@ You can generate an access token with `clusterioctl host generate-token --id <ho
 Defaults to "enter token here".
 
 
-### host.tls_ca
-
-Path to certificate in PEM format to use for validating a TLS connection to the controller.
-If you have a self signed certificate on the controller you will need to copy the certificate to your hosts and set it as the certificate authority with this option.
-
-Defaults to null meaning use Node.js's default set of trusted certificate authorities.
-
-
 ### host.public_address
 
 External address instances hosted on this host can be accessed on.
@@ -413,14 +405,6 @@ You can generate an access token with `clusteriocontroller bootstrap generate-us
 Defaults to null meaning complain about it not being set and exit.
 
 
-### control.tls_ca
-
-Path to certificate in PEM format to use for validating a TLS connection to the controller.
-If you have a self signed certificate on the controller you will need to copy the certificate to the computer you run clusterctl from and set it as the certificate authority with this option.
-
-Defaults to null meaning use Node.js's default set of trusted certificate authorities.
-
-
 ### control.max_reconnect_delay
 
 Max duration in seconds to wait before attempting to reconnect with the controller after the connection is dropped.

docs/contributing.md

@@ -137,6 +137,8 @@ You can also pass the `--set-upstream` (or `-u`) flag to make `git push` do the
 After your feature branch have been merged you should consider it final.
 If there are more changes you want to have merged or mistakes you need to fix you must start a new feature branch and submit a new pull request.
 
+Pull request descriptions should contain a `### Changelog` section with additions for the changelog contained inside a code block, see the [CHANGELOG.md](/CHANGELOG.md) file for the formatting of changelog entries and possible sections to use.
+If there are no changelog entries to add, remove the code block and start the changelog section with the text `None.`, followed by an optional reason for why it is not is needed.
 
 ### Editing your Feature Branch
 

docs/release-workflow.md

@@ -5,7 +5,9 @@
 - CHANGELOG.md in the repository root
 - The changelog.txt file in mod directories
 
-Pull requests will usually have suggestions for changelog entries.
+Pull requests will usually have suggestions for changelog entries. These can be automatically parsed and formatted by running `node scripts/pr_to_changelog.mts`, though you will have to manually sort through which should go in the changelog in the repository root and which should go in the changelogs in mod directories.
+
+If the script fails due to GitHub's 60/hour public access rate limit, you can set the `GH_TOKEN` environment variable to a fine-grained personal access token which has a much higher rate limits.
 
 ## Packages
 

docs/setting-up-tls.md

@@ -63,10 +63,25 @@ This creates a private key file named `key.pem` and a public certificate file na
     npx clusteriocontroller config set controller.tls_certificate cert.pem
     npx clusteriocontroller config set controller.tls_private_key key.pem
 
-Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure it as the `host.tls_ca` option:
+Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure node to trust it by setting the `NODE_EXTRA_CA_CERTS` environment variable to a path to this file before the host is started. You can do this by passing `--self-signed-cert path/to/cert` to the installer when setting up a host, or manually by editing run-host.cmd, run-host.sh and/or clusteriohost.service by adding the following line next to the one setting `NODE_OPTIONS`: 
 
-    npx clusteriohost config set host.tls_ca cert.pem
+**For run-host.cmd**
 
-You will also need to copy the `cert.pem` file to all of the computers you want to remotely manage the cluster via clusterctl and configure it as `control.tls_ca` option:
+    set "NODE_EXTRA_CA_CERTS=path\to\cert.pem"
 
-    npx clusterioctl control-config set control.tls_ca cert.pem
+**For run-host.sh**
+
+    export "NODE_EXTRA_CA_CERTS=path/to/cert.pem"
+
+**For clusteriohost.service**
+
+    Environment=NODE_EXTRA_CA_CERTS=path/to/cert.pem
+
+You will also need to copy the `cert.pem` file to all of the computers you want to remotely manage the cluster via clusterctl and set the `NODE_EXTRA_CA_CERTS` environment variable as a path to that file before running clusterioctl. For example:
+
+    # On Windows
+    set "NODE_EXTRA_CA_CERTS=path\to\cert.pem"
+    # On Linux
+    export NODE_EXTRA_CA_CERTS="path/to/cert.pem"
+    # You can now use clusterioctl
+    npx clusterioctl instance list

package.json

@@ -7,7 +7,7 @@
 		"build": "tsc --build",
 		"watch": "tsc --build --watch",
 		"build-mod": "pnpm run -r build-mod --output-dir ../../dist",
-		"test": "pnpm run build && mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
+		"test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
 		"silent-test": "SILENT_TEST=y pnpm test",
 		"fast-test": "FAST_TEST=y pnpm test",
 		"cover": "nyc pnpm test",
@@ -15,9 +15,7 @@
 		"ci-cover": "nyc --reporter=lcovonly pnpm run-script test",
 		"lint": "eslint packages plugins test",
 		"lint-fix": "eslint --fix packages plugins test",
-		"clean": "node ./clean.js fast",
-		"clean-tests": "node ./clean.js tests",
-		"clean-all": "node ./clean.js all",
+		"clean": "node ./clean.js",
 		"docs": "typedoc",
 		"bundle-dependencies": "bundle-dependencies"
 	},
@@ -46,19 +44,18 @@
 		"@eslint/js": "^9.32.0",
 		"@stylistic/eslint-plugin": "^5.2.2",
 		"@swc/core": "^1.13.3",
+		"@types/node": "catalog:",
 		"@typescript-eslint/eslint-plugin": "^8.38.0",
 		"@typescript-eslint/parser": "^8.38.0",
 		"eslint": "^9.32.0",
 		"eslint-plugin-node": "^11.1.0",
 		"express": "^5.1.0",
 		"form-data": "^4.0.4",
-		"fs-extra": "^11.3.0",
 		"globals": "^16.3.0",
 		"jsonwebtoken": "^9.0.2",
 		"jszip": "^3.10.1",
 		"mocha": "^11.7.1",
 		"nyc": "^17.1.0",
-		"phin": "^3.7.1",
 		"typedoc": "^0.28.9",
 		"typescript": "^5.9.2",
 		"yargs": "^18.0.0"

packages/controller/controller.ts

@@ -15,7 +15,7 @@
  */
 
 import path from "path";
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 
 import crypto from "crypto";
 import setBlocking from "set-blocking";
@@ -32,6 +32,7 @@ const { ConsoleTransport, levels, logger } = lib;
 
 import Controller from "./src/Controller";
 import UserManager from "./src/UserManager";
+import UserRecord from "./src/UserRecord";
 import { version } from "./package.json";
 
 // globals
@@ -133,13 +134,22 @@ async function handleBootstrapCommand(
 	let subCommand = args._[1];
 
 	const databaseDirectory = controllerConfig.get("controller.database_directory");
+	await UserManager.attemptMigrateLegacyUsersFile(
+		path.join(databaseDirectory, "users.json"),
+		path.join(databaseDirectory, "roles.json"),
+	);
+
 	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 		path.join(databaseDirectory, "roles.json"),
 		lib.Role.fromJSON.bind(lib.Role),
 	).bootstrap());
 
-	const userManager = new UserManager(controllerConfig, roles);
-	await userManager.load(path.join(databaseDirectory, "users.json"));
+	const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		path.join(databaseDirectory, "users.json"),
+		UserRecord.fromJSON.bind(UserRecord),
+	).bootstrap());
+
+	const userManager = new UserManager(users, roles, controllerConfig);
 
 	if (subCommand === "create-admin") {
 		await controllerConfigLock.acquire(); // Also needed to write to the database files
@@ -150,15 +160,12 @@ async function handleBootstrapCommand(
 			return;
 		}
 
-		let admin = userManager.getByName(args.name);
-		if (!admin) {
-			admin = userManager.createUser(args.name);
-		}
+		const user = userManager.getOrCreateUser(args.name);
+		user.roleIds.add(lib.Role.DefaultAdminRoleId);
+		user.isAdmin = true;
+		user.saveRecord();
 
-		let adminRole = lib.ensureDefaultAdminRole(roles);
-		admin.roleIds.add(adminRole.id);
-		admin.isAdmin = true;
-		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
+		await userManager.records.save();
 		await controllerConfigLock.release();
 
 	} else if (subCommand === "generate-user-token") {
@@ -175,7 +182,8 @@ async function handleBootstrapCommand(
 		// eslint-disable-next-line no-console
 		console.log(jwt.sign(
 			{ aud: "host", host: args.id },
-			Buffer.from(controllerConfig.get("controller.auth_secret"), "base64")
+			// Auth secret is never null after startup
+			Buffer.from(controllerConfig.get("controller.auth_secret")!, "base64")
 		));
 
 	} else if (subCommand === "create-ctl-config") {
@@ -431,7 +439,7 @@ async function startup() {
 	);
 
 	// Refuse to start if there are no users loaded
-	if (args.checkUserCount && controller.userManager.users.size === 0) {
+	if (args.checkUserCount && controller.users.records.size === 0) {
 		logger.fatal(
 			"Cannot start controller, no users loaded.\n" +
 			"Try `npx clusteriocontroller bootstrap create-admin <username>`");
@@ -483,7 +491,7 @@ I           version of clusterio.  Expect things to break. I
 `
 	);
 	startup().catch(err => {
-		if (err instanceof lib.StartupError) {
+		if (err instanceof lib.StartupError || err.code === "InstallationError") {
 			logger.fatal(`
 +-------------------------------+
 | Unable to to start controller |

packages/controller/index.ts

@@ -1,13 +1,15 @@
 import { bootstrap } from "./controller";
 export { default as Controller } from "./src/Controller";
-export { default as ControllerUser } from "./src/ControllerUser";
 export { default as ControllerRouter } from "./src/ControllerRouter";
 export { default as ControlConnection } from "./src/ControlConnection";
 export { default as HostConnection } from "./src/HostConnection";
-export { default as InstanceInfo } from "./src/InstanceInfo";
-export { default as HostInfo } from "./src/HostInfo";
+export { default as InstanceRecord } from "./src/InstanceRecord";
+export { default as InstanceManager } from "./src/InstanceManager";
+export { default as HostRecord } from "./src/HostRecord";
 export { default as BaseControllerPlugin } from "./src/BaseControllerPlugin";
 export { default as UserManager } from "./src/UserManager";
+export { default as UserRecord } from "./src/UserRecord";
+export { default as User } from "./src/User";
 export { default as WsServer } from "./src/WsServer";
 
 if (module === require.main) {

packages/controller/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio controller",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.24",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --mode production",
@@ -22,40 +22,38 @@
 	"dependencies": {
 		"@clusterio/lib": "workspace:*",
 		"@sinclair/typebox": "catalog:",
-		"set-blocking": "catalog:",
-		"jsonwebtoken": "catalog:",
-		"express": "catalog:",
-		"fs-extra": "catalog:",
-		"ws": "catalog:",
-		"jszip": "catalog:",
-		"winston": "catalog:",
-		"winston-daily-rotate-file": "catalog:",
 		"busboy": "^1.6.0",
 		"compression": "^1.8.0",
+		"express": "catalog:",
 		"finalhandler": "^1.3.1",
+		"jsonwebtoken": "catalog:",
+		"jszip": "catalog:",
+		"set-blocking": "catalog:",
+		"winston": "catalog:",
+		"winston-daily-rotate-file": "catalog:",
+		"ws": "catalog:",
 		"yargs": "^17.7.2"
 	},
 	"devDependencies": {
 		"@clusterio/web_ui": "workspace:*",
-		"@types/jsonwebtoken": "catalog:",
-		"@types/set-blocking": "catalog:",
-		"@types/finalhandler": "catalog:",
+		"@swc/core": "catalog:",
+		"@types/busboy": "^1.5.0",
+		"@types/compression": "^1.7.2",
 		"@types/express": "catalog:",
-		"@types/fs-extra": "catalog:",
+		"@types/finalhandler": "catalog:",
+		"@types/jsonwebtoken": "catalog:",
 		"@types/node": "catalog:",
+		"@types/set-blocking": "catalog:",
 		"@types/ws": "catalog:",
 		"@types/yargs": "^17.0.33",
-		"@types/compression": "^1.7.2",
-		"@types/busboy": "^1.5.0",
-		"@swc/core": "catalog:",
 		"react": "catalog:",
 		"react-dom": "catalog:",
 		"react-router-dom": "catalog:",
 		"typescript": "catalog:",
 		"webpack": "catalog:",
 		"webpack-cli": "catalog:",
-		"webpack-merge": "catalog:",
-		"webpack-dev-middleware": "^6.1.1"
+		"webpack-dev-middleware": "^6.1.1",
+		"webpack-merge": "catalog:"
 	},
 	"publishConfig": {
 		"access": "public"

packages/controller/src/BaseConnection.ts

@@ -1,5 +1,5 @@
 import path from "path";
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 
 import type Controller from "./Controller";
 import type WsServerConnector from "./WsServerConnector";
@@ -32,6 +32,7 @@ export default class BaseConnection extends lib.Link {
 		this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this));
 		this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this));
 		this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
+		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 
 		this.handle(lib.SubscriptionRequest, this.handleSubscriptionRequest.bind(this));
 		this.connector.on("close", () => {
@@ -93,10 +94,12 @@ export default class BaseConnection extends lib.Link {
 	async handleModDownloadRequest(request: lib.ModDownloadRequest) {
 		let mod = this.getMod(request);
 		let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename);
+		// Open source file before creating the stream in case it throws.
+		let source = (await fs.open(modPath)).createReadStream();
 
 		let stream = await routes.createProxyStream(this._controller.app);
 		stream.filename = mod.filename;
-		stream.source = fs.createReadStream(modPath);
+		stream.source = source;
 		stream.mime = "application/zip";
 		stream.size = String(mod.size);
 
@@ -106,4 +109,8 @@ export default class BaseConnection extends lib.Link {
 	async handleSubscriptionRequest(request: lib.SubscriptionRequest, src: lib.Address, dst: lib.Address) {
 		return await this._controller.subscriptions.handleRequest(this, request, src, dst);
 	}
+
+	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
+		return await this._controller.factorioVersions.get(request.maxAgeMs);
+	}
 }

packages/controller/src/BaseControllerPlugin.ts

@@ -3,7 +3,7 @@ import type {
 	ModPack, ModInfo, PlayerEvent, PluginNodeEnvInfo,
 } from "@clusterio/lib";
 import type Controller from "./Controller";
-import type InstanceInfo from "./InstanceInfo";
+import type InstanceRecord from "./InstanceRecord";
 import type ControlConnection from "./ControlConnection";
 import type HostConnection from "./HostConnection";
 
@@ -91,7 +91,7 @@ export default class BaseControllerPlugin {
 	 *     The instance that changed.
 	 * @param prev - the previous status of the instance.
 	 */
-	async onInstanceStatusChanged(instance: InstanceInfo, prev?: InstanceStatus) { }
+	async onInstanceStatusChanged(instance: InstanceRecord, prev?: InstanceStatus) { }
 
 	/**
 	 * Called when the value of a controller config field changed.
@@ -117,7 +117,7 @@ export default class BaseControllerPlugin {
 	 * @param curr - The current value of the field.
 	 * @param prev - The previous value of the field.
 	 */
-	async onInstanceConfigFieldChanged(instance: InstanceInfo, field: string, curr: unknown, prev: unknown) { }
+	async onInstanceConfigFieldChanged(instance: InstanceRecord, field: string, curr: unknown, prev: unknown) { }
 
 	/**
 	 * Called before collecting Prometheus metrics
@@ -261,7 +261,7 @@ export default class BaseControllerPlugin {
 	 *     The instance it occured on.
 	 * @param event - Information about the event.
 	 */
-	async onPlayerEvent(instance: InstanceInfo, event: PlayerEvent) { }
+	async onPlayerEvent(instance: InstanceRecord, event: PlayerEvent) { }
 
 	/**
 	 * Broadcast event to all connected hosts

packages/controller/src/ControlConnection.ts

@@ -6,7 +6,8 @@ import * as lib from "@clusterio/lib";
 const { logFilter, logger } = lib;
 
 import BaseConnection from "./BaseConnection";
-import ControllerUser from "./ControllerUser";
+import User from "./User";
+import UserRecord from "./UserRecord";
 import * as routes from "./routes";
 import Controller from "./Controller";
 
@@ -40,7 +41,7 @@ export default class ControlConnection extends BaseConnection {
 		registerData: { version: string },
 		connector: WsServerConnector,
 		controller: Controller,
-		public user: ControllerUser, // The user making this connection.
+		public user: User, // The user making this connection.
 		public id: number
 	) {
 		super(connector, controller);
@@ -72,6 +73,7 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.ControllerRestartRequest, this.handleControllerRestartRequest.bind(this));
 		this.handle(lib.ControllerUpdateRequest, this.handleControllerUpdateRequest.bind(this));
 		this.handle(lib.ControllerConfigGetRequest, this.handleControllerConfigGetRequest.bind(this));
+		this.handle(lib.ControllerConfigSetRequest, this.handleControllerConfigSetRequest.bind(this));
 		this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this));
 		this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this));
 		this.handle(lib.HostRevokeTokensRequest, this.handleHostRevokeTokensRequest.bind(this));
@@ -83,6 +85,7 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.InstanceCreateRequest, this.handleInstanceCreateRequest.bind(this));
 		this.handle(lib.InstanceDeleteRequest, this.handleInstanceDeleteRequest.bind(this));
 		this.handle(lib.InstanceConfigGetRequest, this.handleInstanceConfigGetRequest.bind(this));
+		this.handle(lib.InstanceConfigSetRequest, this.handleInstanceConfigSetRequest.bind(this));
 		this.handle(lib.InstanceConfigSetFieldRequest, this.handleInstanceConfigSetFieldRequest.bind(this));
 		this.handle(lib.InstanceConfigSetPropRequest, this.handleInstanceConfigSetPropRequest.bind(this));
 		this.handle(lib.InstanceAssignRequest, this.handleInstanceAssignRequest.bind(this));
@@ -126,7 +129,6 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
 		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
 		this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this));
-		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 	}
 
 	validateIngress(message: lib.MessageRequest | lib.MessageEvent) {
@@ -195,6 +197,26 @@ export default class ControlConnection extends BaseConnection {
 		return this._controller.config.toRemote("control");
 	}
 
+	async handleControllerConfigSetRequest(request: lib.ControllerConfigSetRequest) {
+		try {
+			for (const [fieldName, fieldValue] of Object.entries(request.fields)) {
+				if (typeof fieldValue === "object") {
+					for (const [propName, propValue] of Object.entries(fieldValue)) {
+						this._controller.config.stageProp(
+							fieldName as keyof lib.ControllerConfigFields, propName, propValue, "control"
+						);
+					}
+				} else {
+					this._controller.config.stage(fieldName as keyof lib.ControllerConfigFields, fieldValue, "control");
+				}
+			}
+
+			this._controller.config.commitStaging();
+		} finally {
+			this._controller.config.revertStaging();
+		}
+	}
+
 	async handleControllerConfigSetFieldRequest(request: lib.ControllerConfigSetFieldRequest) {
 		this._controller.config.set(request.field as keyof lib.ControllerConfigFields, request.value, "control");
 	}
@@ -250,7 +272,7 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleInstanceDetailsGetRequest(request: lib.InstanceDetailsGetRequest) {
-		return this._controller.getRequestInstance(request.instanceId).toInstanceDetails();
+		return this._controller.instances.getForRequest(request.instanceId).toInstanceDetails();
 	}
 
 	async handleInstanceDetailsListRequest() {
@@ -268,42 +290,68 @@ export default class ControlConnection extends BaseConnection {
 			instanceConfig.set("instance.assigned_host", null); // New instances are unassigned
 		}
 		instanceConfig.update(request.config, false, "control");
-		await this._controller.instanceCreate(instanceConfig);
+		await this._controller.instances.createInstance(instanceConfig);
 	}
 
 	async handleInstanceDeleteRequest(request: lib.InstanceDeleteRequest) {
-		await this._controller.instanceDelete(request.instanceId);
+		await this._controller.instances.deleteInstance(request.instanceId);
 	}
 
 	async handleInstanceConfigGetRequest(request: lib.InstanceConfigGetRequest) {
-		let instance = this._controller.getRequestInstance(request.instanceId);
+		let instance = this._controller.instances.getForRequest(request.instanceId);
 		return instance.config.toRemote("control");
 	}
 
-	async handleInstanceConfigSetFieldRequest(request: lib.InstanceConfigSetFieldRequest) {
-		let instance = this._controller.getRequestInstance(request.instanceId);
-		if (request.field === "instance.assigned_host") {
+	private static validateInstanceConfigSetField(fieldName: string) {
+		if (fieldName === "instance.assigned_host") {
 			throw new lib.RequestError("instance.assigned_host must be set through the assign-host interface");
 		}
 
-		if (request.field === "instance.id") {
+		if (fieldName === "instance.id") {
 			// XXX is this worth implementing?  It's race condition galore.
 			throw new lib.RequestError("Setting instance.id is not supported");
 		}
+	}
 
+	async handleInstanceConfigSetRequest(request: lib.InstanceConfigSetRequest) {
+		const instance = this._controller.instances.getForRequest(request.instanceId);
+		try {
+			for (const [fieldName, fieldValue] of Object.entries(request.fields)) {
+				ControlConnection.validateInstanceConfigSetField(fieldName);
+				if (typeof fieldValue === "object") {
+					for (const [propName, propValue] of Object.entries(fieldValue)) {
+						instance.config.stageProp(
+							fieldName as keyof lib.InstanceConfigFields, propName, propValue, "control"
+						);
+					}
+				} else {
+					instance.config.stage(fieldName as keyof lib.InstanceConfigFields, fieldValue, "control");
+				}
+			}
+
+			instance.config.commitStaging();
+		} finally {
+			instance.config.revertStaging();
+		}
+	}
+
+	async handleInstanceConfigSetFieldRequest(request: lib.InstanceConfigSetFieldRequest) {
+		const instance = this._controller.instances.getForRequest(request.instanceId);
+		ControlConnection.validateInstanceConfigSetField(request.field);
 		instance.config.set(request.field as keyof lib.InstanceConfigFields, request.value, "control");
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
 	async handleInstanceConfigSetPropRequest(request: lib.InstanceConfigSetPropRequest) {
-		let instance = this._controller.getRequestInstance(request.instanceId);
+		let instance = this._controller.instances.getForRequest(request.instanceId);
 		let { field, prop, value } = request;
+		ControlConnection.validateInstanceConfigSetField(field);
 		instance.config.setProp(field as keyof lib.InstanceConfigFields, prop, value, "control");
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
 	async handleInstanceAssignRequest(request: lib.InstanceAssignRequest) {
-		await this._controller.instanceAssign(request.instanceId, request.hostId);
+		await this._controller.instances.assignInstance(request.instanceId, request.hostId ?? null);
 	}
 
 	async handleInstanceSaveDetailsListRequest() {
@@ -337,8 +385,8 @@ export default class ControlConnection extends BaseConnection {
 		if (request.sourceInstanceId === request.targetInstanceId) {
 			throw new lib.RequestError("Source and target instance may not be the same");
 		}
-		let sourceInstance = this._controller.getRequestInstance(request.sourceInstanceId);
-		let targetInstance = this._controller.getRequestInstance(request.targetInstanceId);
+		let sourceInstance = this._controller.instances.getForRequest(request.sourceInstanceId);
+		let targetInstance = this._controller.instances.getForRequest(request.targetInstanceId);
 		let sourceHostId = sourceInstance.config.get("instance.assigned_host");
 		let targetHostId = targetInstance.config.get("instance.assigned_host");
 		if (sourceHostId === null) {
@@ -614,22 +662,21 @@ export default class ControlConnection extends BaseConnection {
 		return modInfos;
 	}
 
+	// eslint-disable-next-line complexity
 	async handleModDependencyResolveRequest(request: lib.ModDependencyResolveRequest) {
 		const {factorioVersion, checkForUpdates} = request;
 		const searchQueue = [...request.mods];
 		const localInfos = [...this._controller.modStore.mods()];
 
-		const _builtInModNames = lib.ModPack.getBuiltinModNames(factorioVersion);
-		const builtInModNames = new Set(_builtInModNames);
-		const skipSearch = new Set(_builtInModNames);
+		const builtInModNames = lib.ModPack.getBuiltinModNames(factorioVersion);
+		const skipSearch = new Set(builtInModNames);
 
 		type ModPortalModType = InstanceType<typeof lib.ModPortalGetAllRequest.Response>["mods"][number];
 		type ModPortalReleaseType = NonNullable<ModPortalModType["releases"]>[number];
 		const dependencyRequirements = new Map<string, lib.ModVersionRange>();
 		const optionalRequirements = new Map<string, lib.ModVersionRange>();
 		const candidateReleases = new Map<string, lib.ModInfo>();
-		const incompatible = new Set<string>();
-		const invalid = new Set<string>();
+		const errors = new Map<string, lib.ModDependencyResolveErrors>();
 
 		const _modPortalReleases = new Map<string, ModPortalModType | null>();
 		async function fetchModReleases(modName: string) {
@@ -650,10 +697,44 @@ export default class ControlConnection extends BaseConnection {
 			return releases;
 		}
 
+		function selectModRelease(releases: ModPortalModType | null, versionRange: lib.ModVersionRange) {
+			if (!releases || !releases.releases) {
+				return null;
+			}
+
+			// Select the latest matching version
+			const release = releases.releases
+				.filter(info => versionRange.testVersion(info.version))
+				.reduce<ModPortalReleaseType | undefined>((max, cur) => (
+					max && lib.integerFullVersion(max.version) > lib.integerFullVersion(cur.version) ? max : cur
+				), undefined);
+
+			if (!release) {
+				return null;
+			}
+
+			// Create the mod info class from the selected
+			try {
+				return lib.ModInfo.fromJSON({
+					...release.info_json, // includes dependencies
+					sha1: release.sha1,
+					version: release.version,
+					name: releases.name,
+					author: releases.owner,
+					title: releases.title,
+				});
+			} catch {
+				// Can error if there are invalid dependencies
+				errors.set(releases.name, "invalidDependency");
+				return null;
+			}
+		}
+
 		while (searchQueue.length > 0) {
+			// Dequeue the next mod, skip if incompatible
 			const mod = searchQueue.shift()!;
 			if (mod.incompatible) {
-				incompatible.add(mod.name);
+				errors.set(mod.name, "incompatible");
 				continue;
 			}
 
@@ -674,13 +755,13 @@ export default class ControlConnection extends BaseConnection {
 				versionRange.combineVersion(mod.version);
 				if (!versionRange.valid) {
 					candidateReleases.delete(mod.name);
-					invalid.add(mod.name);
+					errors.set(mod.name, "unsatisfiable");
 				}
 			}
 
-			// Do not add dependencies for invalids, optionals, skips, or valid candidates
+			// Do not add dependencies for errors, skips, optionals, or valid candidates
 			const existingCandidate = candidateReleases.get(mod.name);
-			if (invalid.has(mod.name) || optionalRequirements.has(mod.name) || skipSearch.has(mod.name) || (
+			if (errors.has(mod.name) || skipSearch.has(mod.name) || optionalRequirements.has(mod.name) || (
 				existingCandidate && versionRange.testVersion(existingCandidate.version)
 			)) {
 				continue;
@@ -695,28 +776,11 @@ export default class ControlConnection extends BaseConnection {
 
 			// Check if a mod poral version fits the requirements
 			if (!candidate || checkForUpdates) {
-				const modReleases = await fetchModReleases(mod.name);
-				if (modReleases && modReleases.releases) {
-					const release = modReleases.releases
-						.filter(info => (
-							info.info_json.factorio_version === factorioVersion
-							&& versionRange.testVersion(info.version)
-						))
-						.reduce<ModPortalReleaseType | undefined>((max, cur) => (
-							max && lib.integerFullVersion(max.version) > lib.integerFullVersion(cur.version) ? max : cur
-						), undefined);
-
-					// We only select the mod portal release if it is more recent than the local candidate
-					if (release && (!candidate || lib.integerFullVersion(release.version) > candidate.integerVersion)) {
-						candidate = lib.ModInfo.fromJSON({
-							...release.info_json, // Also includes dependences
-							sha1: release.sha1,
-							version: release.version,
-							name: modReleases.name,
-							author: modReleases.owner,
-							title: modReleases.title,
-						});
-					}
+				const releases = await fetchModReleases(mod.name);
+				const release = selectModRelease(releases, versionRange);
+				// We only select the mod portal release if it is more recent than the local candidate
+				if (release && (!candidate || release.integerVersion > candidate.integerVersion)) {
+					candidate = release;
 				}
 			}
 
@@ -732,14 +796,22 @@ export default class ControlConnection extends BaseConnection {
 			}
 		}
 
+		// Add the not found dependencies as errors
+		for (const modName of dependencyRequirements.keys()) {
+			if (!candidateReleases.has(modName) && !errors.has(modName)) {
+				errors.set(modName, "notFound");
+			}
+		}
+
+		// Remove incompatible dependencies if they aren't required by others
+		for (const [modName, error] of errors) {
+			if (error === "incompatible" && !dependencyRequirements.has(modName)) {
+				errors.delete(modName);
+			}
+		}
+
 		return new lib.ModDependencyResolveRequest.Response(
-			[...candidateReleases.values()],
-			[...invalid.values(), ...[...incompatible.values()].filter(mod => (
-				candidateReleases.has(mod) || builtInModNames.has(mod)
-			))],
-			[...dependencyRequirements.keys()].filter(mod => (
-				!candidateReleases.has(mod) && !incompatible.has(mod) && !invalid.has(mod)
-			)),
+			[...candidateReleases.values()], errors,
 		);
 	}
 
@@ -846,35 +918,33 @@ export default class ControlConnection extends BaseConnection {
 		}
 		this._controller.roles.delete(role);
 
-		this._controller.userManager.dirty = true;
-		for (let user of this._controller.userManager.users.values()) {
-			if (user.roleIds.delete(id)) {
+		for (const user of this._controller.users.values()) {
+			if (user.removeRole(id)) {
 				this._controller.userPermissionsUpdated(user);
 			}
 		}
 	}
 
-	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.User> {
+	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.UserDetails> {
 		let name = request.name;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User ${name} does not exist`);
 		}
 
-		return user;
+		return user.toUserDetails();
 	}
 
-	async handleUserListRequest(): Promise<lib.User[]> {
-		return [...this._controller.userManager.users.values()];
+	async handleUserListRequest(): Promise<lib.UserDetails[]> {
+		return [...this._controller.users.records.values()].map(user => user.toUserDetails());
 	}
 
 	async handleUserCreateRequest(request: lib.UserCreateRequest) {
-		let user = this._controller.userManager.createUser(request.name);
-		this._controller.usersUpdated([user]);
+		this._controller.users.createUser(request.name);
 	}
 
 	async handleUserRevokeTokenRequest(request: lib.UserRevokeTokenRequest) {
-		let user = this._controller.userManager.getByName(request.name);
+		let user = this._controller.users.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
@@ -888,11 +958,10 @@ export default class ControlConnection extends BaseConnection {
 				controlConnection.connector.terminate();
 			}
 		}
-		this._controller.usersUpdated([user]);
 	}
 
 	async handleUserUpdateRolesRequest(request: lib.UserUpdateRolesRequest) {
-		let user = this._controller.userManager.getByName(request.name);
+		let user = this._controller.users.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
@@ -903,35 +972,33 @@ export default class ControlConnection extends BaseConnection {
 			}
 		}
 
-		user.roleIds = new Set(request.roles);
+		user.set("roleIds", new Set(request.roles));
 		this._controller.userPermissionsUpdated(user);
-		this._controller.usersUpdated([user]);
 	}
 
 	async handleUserSetAdminRequest(request: lib.UserSetAdminRequest) {
 		let { name, create, admin } = request;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.userManager.createUser(name);
+				user = this._controller.users.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
 		}
 
-		user.isAdmin = admin;
-		this._controller.usersUpdated([user]);
+		user.set("isAdmin", admin);
 		this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, admin));
 	}
 
 	async handleUserSetBannedRequest(request: lib.UserSetBannedRequest) {
 		let { name, create, banned, reason } = request;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByNameMutable(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.userManager.createUser(name);
+				user = this._controller.users.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
@@ -939,37 +1006,34 @@ export default class ControlConnection extends BaseConnection {
 
 		user.isBanned = banned;
 		user.banReason = reason;
-		this._controller.usersUpdated([user]);
+		user.saveRecord();
 		this._controller.sendTo("allInstances", new lib.InstanceBanlistUpdateEvent(name, banned, reason));
 	}
 
 	async handleUserSetWhitelistedRequest(request: lib.UserSetWhitelistedRequest) {
 		let { name, create, whitelisted } = request;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
-				user = this._controller.userManager.createUser(name);
+				user = this._controller.users.createUser(name);
 			} else {
 				throw new lib.RequestError(`User '${name}' does not exist`);
 			}
 		}
 
-		user.isWhitelisted = whitelisted;
-		this._controller.usersUpdated([user]);
+		user.set("isWhitelisted", whitelisted);
 		this._controller.sendTo("allInstances", new lib.InstanceWhitelistUpdateEvent(name, whitelisted));
 	}
 
 	async handleUserDeleteRequest(request: lib.UserDeleteRequest) {
 		let name = request.name;
-		let user = this._controller.userManager.getByName(name);
+		let user = this._controller.users.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User '${name}' does not exist`);
 		}
 
-		user.isDeleted = true;
-		this._controller.userManager.users.delete(name);
-		this._controller.usersUpdated([user]);
+		this._controller.users.deleteUser(user);
 
 		if (user.isAdmin) {
 			this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, false));
@@ -982,14 +1046,15 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
-	async handleUserBulkRestoreRequest(request: lib.UserBulkImportRequest, updated: Map<string, ControllerUser>) {
+	/** This does not do a full restore, rather it demotes / unbanns / unwhitelistes users */
+	async handleUserBulkRestoreRequest(request: lib.UserBulkImportRequest, updated: Map<string, UserRecord>) {
 		if (request.importType === "users") {
 			// Update all fields for all users
 			const users = request.users as Extract<typeof request.users, { username: string }[]>;
 			const admin = new Set(users.filter(u => u.is_admin).map(u => u.username.toLowerCase()));
 			const banned = new Set(users.filter(u => u.is_banned).map(u => u.username.toLowerCase()));
 			const whitelist = new Set(users.filter(u => u.is_whitelisted).map(u => u.username.toLowerCase()));
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.valuesMutable()) {
 				if (user.isAdmin && !admin.has(user.name)) {
 					user.isAdmin = false;
 					updated.set(user.id, user);
@@ -1010,7 +1075,7 @@ export default class ControlConnection extends BaseConnection {
 			const expected = new Set(
 				users.map(u => (typeof u === "string" ? u.toLowerCase() : u.username.toLowerCase()))
 			);
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.valuesMutable()) {
 				if (user.isBanned && !expected.has(user.name)) {
 					user.isBanned = false;
 					user.banReason = "";
@@ -1022,7 +1087,7 @@ export default class ControlConnection extends BaseConnection {
 			const users = request.users as string[];
 			const expected = new Set(users.map(u => u.toLowerCase()));
 			const prop = request.importType === "admins" ? "isAdmin" : "isWhitelisted";
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.valuesMutable()) {
 				if (user[prop] && !expected.has(user.name)) {
 					user[prop] = false;
 					updated.set(user.id, user);
@@ -1031,11 +1096,9 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
-	// A large number of small if statements are unavoidable when reducing the number of updates sent
-
 	async handleUserBulkImportRequest(request: lib.UserBulkImportRequest) {
 		let backup: undefined | Awaited<ReturnType<ControlConnection["handleUserBulkExportRequest"]>>;
-		const updated = new Map<ControllerUser["id"], ControllerUser>();
+		const updated = new Map<User["id"], User>();
 		if (request.restore) {
 			// Unban / Demote / Unwhitelist players not on the list
 			backup = await this.handleUserBulkExportRequest(new lib.UserBulkExportRequest(request.importType));
@@ -1043,11 +1106,12 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		// Will get a user or attempt to create one
+		// UserManager.getOrCreateUser not used due to inclusion of checkPermission
 		const getUserOrCreate = (username: string) => {
-			const user = this._controller.userManager.getByName(username);
-			if (user) { return user; }
+			const user = this._controller.users.getByNameMutable(username);
+			if (user) { return updated.get(user.id) ?? user; }
 			this.user.checkPermission("core.user.create");
-			return this._controller.userManager.createUser(username);
+			return this._controller.users.createUser(username);
 		};
 
 		// Merge the imported data with existing data, this is strictly additive
@@ -1105,14 +1169,14 @@ export default class ControlConnection extends BaseConnection {
 
 		// Send the necessary update events
 		const updatedUsers = [...updated.values()];
-		this._controller.usersUpdated(updatedUsers);
+		this._controller.users.records.setMany(updatedUsers);
 
 		// Resync the host user lists
 		const adminlist: Set<string> = new Set();
 		const banlist: Map<string, string> = new Map();
 		const whitelist: Set<string> = new Set();
 
-		for (let user of this._controller.userManager.users.values()) {
+		for (let user of this._controller.users.values()) {
 			if (user.isAdmin) {
 				adminlist.add(user.id);
 			}
@@ -1136,7 +1200,7 @@ export default class ControlConnection extends BaseConnection {
 		if (request.exportType === "users") {
 			// Send a full user export
 			const usersToSend = [] as lib.ClusterioUserExport["users"];
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.values()) {
 				let send = false;
 				const userToSend = { username: user.name } as typeof usersToSend[0];
 				if (user.isBanned) {
@@ -1162,7 +1226,7 @@ export default class ControlConnection extends BaseConnection {
 		} else if (request.exportType === "bans") {
 			// Bans have extra logic to handle ban reason
 			const usersToSend = [] as Array<string | { username: string, reason: string }>;
-			for (const user of this._controller.userManager.users.values()) {
+			for (const user of this._controller.users.values()) {
 				if (user.isBanned) {
 					if (user.banReason && user.banReason !== "") {
 						usersToSend.push({ username: user.name, reason: user.banReason });
@@ -1177,7 +1241,7 @@ export default class ControlConnection extends BaseConnection {
 		// This is a whitelist or admins export
 		const usersToSend = [] as string[];
 		const prop = request.exportType === "admins" ? "isAdmin" : "isWhitelisted";
-		for (const user of this._controller.userManager.users.values()) {
+		for (const user of this._controller.users.values()) {
 			if (user[prop]) {
 				usersToSend.push(user.name);
 			}
@@ -1223,10 +1287,6 @@ export default class ControlConnection extends BaseConnection {
 		this.connector._socket!.clusterio_ignore_dump = true;
 		this._controller.debugEvents.on("message", this.ws_dumper);
 	}
-
-	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
-		return await this._controller.factorioVersions.get(request.maxAgeMs);
-	}
 }
 
 module.exports = ControlConnection;

packages/controller/src/Controller.ts

@@ -7,7 +7,7 @@ import finalhandler from "finalhandler";
 
 import compression from "compression";
 import events from "events";
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import http from "http";
 import https from "https";
 import jwt from "jsonwebtoken";
@@ -18,16 +18,17 @@ import * as lib from "@clusterio/lib";
 const { logger, Summary, Gauge } = lib;
 
 import HttpCloser from "./HttpCloser";
-import InstanceInfo from "./InstanceInfo";
+import InstanceRecord from "./InstanceRecord";
 import * as metrics from "./metrics";
 import * as routes from "./routes";
-import ControllerUser from "./ControllerUser";
+import User from "./User";
+import UserRecord from "./UserRecord";
 import UserManager from "./UserManager";
 import WsServer from "./WsServer";
-import HostConnection from "./HostConnection";
-import HostInfo from "./HostInfo";
+import HostRecord from "./HostRecord";
 import BaseControllerPlugin from "./BaseControllerPlugin";
 import ControllerRouter from "./ControllerRouter";
+import InstanceManager from "./InstanceManager";
 
 const endpointDurationSummary = new Summary(
 	"clusterio_controller_http_endpoint_duration_seconds",
@@ -79,6 +80,10 @@ export default class Controller {
 
 	/** Event subscription controller */
 	subscriptions = new lib.SubscriptionController();
+	/** User manager for the cluster, also contains access to the user records */
+	users: UserManager;
+	/** Instance manager for the cluster, also contains access to the user records */
+	instances: InstanceManager;
 
 	// Possible states are new, starting, running, stopping, stopped
 	private _state: string = "new";
@@ -96,6 +101,7 @@ export default class Controller {
 
 	devMiddleware: any | null = null;
 
+	startedAtMs = Date.now();
 	autosaveInterval?: ReturnType<typeof setInterval>;
 	systemMetricsInterval?: ReturnType<typeof setInterval>;
 
@@ -106,11 +112,24 @@ export default class Controller {
 	// Cache for mod portal requests
 	modPortalCache = new Map<string, { timestamp: number, data: any[] }>();
 	// Cache for factorio versions
-	factorioVersions = new lib.ValueCache(lib.fetchFactorioVersions);
+	factorioVersions = new lib.ValueCache(async () => {
+		try {
+			return await lib.fetchFactorioVersions();
+		} catch (err: any) {
+			logger.warn(`Failed to retrieve Factorio versions ${err.message}`);
+			return [];
+		}
+	});
 
 	static async bootstrap(config: lib.ControllerConfig) {
 		let databaseDirectory = config.get("controller.database_directory");
-		await fs.ensureDir(databaseDirectory);
+		await fs.mkdir(databaseDirectory, { recursive: true });
+
+		// Migrate combined user and role file from 2.0.0-alpha.22
+		await UserManager.attemptMigrateLegacyUsersFile(
+			path.join(databaseDirectory, "users.json"),
+			path.join(databaseDirectory, "roles.json"),
+		);
 
 		const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "systems.json"),
@@ -120,13 +139,13 @@ export default class Controller {
 
 		const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "hosts.json"),
-			HostInfo.fromJSON.bind(HostInfo),
+			HostRecord.fromJSON.bind(HostRecord),
 			this.migrateHosts, this.finaliseHosts,
 		).bootstrap());
 
 		const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "instances.json"),
-			json => InstanceInfo.fromJSON(json, "controller"),
+			InstanceRecord.fromJSON.bind(InstanceRecord),
 			this.migrateInstances, this.finaliseInstances,
 		).bootstrap());
 
@@ -147,11 +166,14 @@ export default class Controller {
 			this.migrateRoles,
 		).bootstrap());
 
-		const userManager = new UserManager(config, roles);
-		await userManager.load(path.join(databaseDirectory, "users.json"));
+		const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "users.json"),
+			UserRecord.fromJSON.bind(UserRecord),
+			undefined, this.finaliseUsers,
+		).bootstrap());
 
 		let modsDirectory = config.get("controller.mods_directory");
-		await fs.ensureDir(modsDirectory);
+		await fs.mkdir(modsDirectory, { recursive: true });
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
 		// Add default mod packs
@@ -160,8 +182,8 @@ export default class Controller {
 		}
 
 		// Add default roles
-		lib.ensureDefaultAdminRole(roles);
-		lib.ensureDefaultPlayerRole(roles);
+		lib.Role.ensureDefaultAdminRole(roles);
+		lib.Role.ensureDefaultPlayerRole(roles);
 
 		return [
 			systems,
@@ -171,7 +193,7 @@ export default class Controller {
 			modPacks,
 			modStore,
 			roles,
-			userManager,
+			users,
 		] as const;
 	}
 
@@ -191,10 +213,10 @@ export default class Controller {
 		 */
 		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
-		/** Mapping of host id to host info */
-		public hosts = new lib.SubscribableDatastore<HostInfo>(),
-		/** Mapping of instance id to instance info */
-		public instances = new lib.SubscribableDatastore<InstanceInfo>(),
+		/** Mapping of host id to host record */
+		public hosts = new lib.SubscribableDatastore<HostRecord>(),
+		/** Mapping of instance id to instance record */
+		instances = new lib.SubscribableDatastore<InstanceRecord>(),
 		/** Mapping of save id to save details */
 		public saves = new lib.SubscribableDatastore<lib.SaveDetails>(),
 		/** Mapping of mod pack id to mod pack */
@@ -203,8 +225,8 @@ export default class Controller {
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
 		/** Mapping of mod pack id to mod pack */
 		public roles = new lib.SubscribableDatastore<lib.Role>(),
-		/** User and roles manager for the cluster */
-		public userManager = new UserManager(config, roles),
+		/** Mapping of user id to user record */
+		users = new lib.SubscribableDatastore<UserRecord>(),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -217,6 +239,9 @@ export default class Controller {
 		this.trustedProxies = this.parseTrustedProxies();
 		this.wsServer = new WsServer(this);
 
+		this.users = new UserManager(users, roles, config);
+		this.instances = new InstanceManager(instances, this);
+
 		this.modStore.on("change", mod => {
 			this.modsUpdated([mod]);
 		});
@@ -236,10 +261,16 @@ export default class Controller {
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
 		this.hosts.on("update", this.hostsUpdated.bind(this));
-		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
+		this.instances.records.on("update", this.instanceUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
 		this.roles.on("update", this.rolesUpdated.bind(this));
+		this.users.records.on("update", this.usersUpdated.bind(this));
+	}
+
+	get authSecret() {
+		// Auth secret is never null after startup
+		return Buffer.from(this.config.get("controller.auth_secret")!, "base64");
 	}
 
 	async start(args: ControllerArgs) {
@@ -304,9 +335,6 @@ export default class Controller {
 			}
 			lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev);
 		});
-		for (let instance of this.instances.values()) {
-			this.addInstanceHooks(instance);
-		}
 
 		// Make sure we're actually going to listen on a port
 		let httpPort = this.config.get("controller.http_port");
@@ -397,6 +425,7 @@ export default class Controller {
 		this.onSystemMetricsIntervalChanged();
 
 		logger.info("Started controller");
+		this.startedAtMs = Date.now();
 		this._state = "running";
 	}
 
@@ -553,7 +582,7 @@ export default class Controller {
 				requests.push(hostConnection.send(new lib.SystemInfoRequest()));
 			}
 			const restartRequired = await this.checkRestartRequired();
-			requests.push(lib.gatherSystemInfo("controller", this.canRestart, restartRequired));
+			requests.push(lib.gatherSystemInfo("controller", this.canRestart, restartRequired, this.startedAtMs));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
 				this.systems.set(metric);
@@ -573,7 +602,7 @@ export default class Controller {
 			// First check the clusterio version
 			const runningVersion = this.config.get("controller.version");
 			const packageJsonPath = require.resolve("@clusterio/controller/package.json");
-			const packageJson = await fs.readJSON(packageJsonPath);
+			const packageJson = JSON.parse(await fs.readFile(packageJsonPath, "utf8"));
 			if (runningVersion !== packageJson.version) {
 				this.config.restartRequired = true;
 				return true;
@@ -590,7 +619,7 @@ export default class Controller {
 
 				packageName = pluginInfo.npmPackage ?? pluginInfo.name;
 				const pluginPackageJsonPath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json"));
-				const pluginPackageJson = await fs.readJSON(pluginPackageJsonPath);
+				const pluginPackageJson = JSON.parse(await fs.readFile(pluginPackageJsonPath, "utf8"));
 				if (pluginInfo.version !== pluginPackageJson.version) {
 					this.config.restartRequired = true;
 					return true;
@@ -641,16 +670,13 @@ export default class Controller {
 			this.config.save(),
 			this.systems.save(),
 			this.hosts.save(),
-			this.instances.save(),
+			this.instances.records.save(),
 			this.saves.save(),
 			this.modPacks.save(),
 			this.roles.save(),
+			this.users.records.save(),
 		]);
 
-		if (this.userManager.dirty) {
-			await this.userManager.save(path.join(this.config.get("controller.database_directory"), "users.json"));
-		}
-
 		await lib.invokeHook(this.plugins, "onSaveData");
 	}
 
@@ -663,11 +689,17 @@ export default class Controller {
 			if (!json.restartRequired) { // Added in 2.0.0.alpha.21
 				json.restartRequired = false;
 			}
+			if (json.systemStartedAtMs === undefined) { // Added in 2.0.0.alpha.23
+				json.systemStartedAtMs = Date.now();
+			}
+			if (json.processStartedAtMs === undefined) { // Added in 2.0.0.alpha.23
+				json.processStartedAtMs = Date.now();
+			}
 			return json;
 		});
 	}
 
-	static migrateHosts(rawJson: unknown[]): Static<typeof HostInfo.jsonSchema>[] {
+	static migrateHosts(rawJson: unknown[]): Static<typeof HostRecord.jsonSchema>[] {
 		let serialized = rawJson as any;
 
 		// New format 2.0.0.alpha.19
@@ -678,7 +710,7 @@ export default class Controller {
 		return serialized;
 	}
 
-	static finaliseHosts(host: HostInfo): HostInfo {
+	static finaliseHosts(host: HostRecord): HostRecord {
 		if (host.connected) {
 			host.connected = false;
 			host.updatedAtMs = Date.now();
@@ -686,8 +718,8 @@ export default class Controller {
 		return host;
 	}
 
-	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceInfo.jsonSchema>[] {
-		const serialized = rawJson as Static<typeof InstanceInfo.jsonSchema>[];
+	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceRecord.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof InstanceRecord.jsonSchema>[];
 		return serialized.map(json => {
 			if (!json.config) { // New format 2.0.0.alpha.14
 				return { config: json as any, status: "running" }; // Use running to force updatedAtMs
@@ -696,7 +728,7 @@ export default class Controller {
 		});
 	}
 
-	static finaliseInstances(instance: InstanceInfo): InstanceInfo {
+	static finaliseInstances(instance: InstanceRecord): InstanceRecord {
 		const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown";
 		if (instance.status !== status) {
 			instance.status = status;
@@ -730,6 +762,20 @@ export default class Controller {
 		});
 	}
 
+	static finaliseUsers(user: UserRecord, users: Map<UserRecord["id"], UserRecord>): UserRecord {
+		const existingUser = users.get(user.id);
+		if (existingUser) {
+			// All lowercase ids in 2.0.0.alpha.19
+			if (user.playerStats.onlineTimeMs <= existingUser.playerStats.onlineTimeMs) {
+				// We assume the user with the lower online time is the duplicate
+				existingUser.merge(user);
+				return existingUser;
+			}
+			user.merge(existingUser);
+		}
+		return user;
+	}
+
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -842,13 +888,10 @@ export default class Controller {
 	 * @param hostId - ID of host to generate a token for.
 	 */
 	generateHostToken(hostId: number): string {
-		return jwt.sign(
-			{ aud: "host", host: hostId },
-			Buffer.from(this.config.get("controller.auth_secret"), "base64")
-		);
+		return jwt.sign({ aud: "host", host: hostId }, this.authSecret);
 	}
 
-	hostsUpdated(hosts: HostInfo[]) {
+	hostsUpdated(hosts: HostRecord[]) {
 		let updates = hosts.map(host => host.toHostDetails());
 		this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates));
 	}
@@ -860,75 +903,6 @@ export default class Controller {
 		return hosts.length ? new lib.HostUpdatesEvent(hosts) : null;
 	}
 
-	/**
-	 * Get instance by ID for a request
-	 *
-	 * @param instanceId - ID of instance to get.
-	 * @returns Info for the given instance if it exists
-	 * @throws {module:lib.RequestError} if the instance does not exist.
-	 */
-	getRequestInstance(instanceId:number): InstanceInfo {
-		let instance = this.instances.get(instanceId);
-		if (!instance) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
-		}
-		return instance;
-	}
-
-	/**
-	 * Create a new instance
-	 *
-	 * Adds common Factorio settings to the provided instance config then
-	 * creates an instance using that config in the cluster.
-	 *
-	 * @example
-	 * let instanceConfig = new lib.InstanceConfig("controller");
-	 * instanceConfig.set("instance.name", "My instance");
-	 * let instance = await controller.instanceAssign(instanceConfig);
-	 * await controller.instanceAssign(instance.id, hostId);
-	 *
-	 * @param instanceConfig -
-	 *     Config to base newly created instance on.
-	 * @returns The created instance
-	 */
-	async instanceCreate(instanceConfig: lib.InstanceConfig): Promise<InstanceInfo> {
-		let instanceId = instanceConfig.get("instance.id");
-		if (this.instances.has(instanceId)) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} already exists`);
-		}
-
-		// Add common settings for the Factorio server
-		let settings = {
-			"name": `${this.config.get("controller.name")} - ${instanceConfig.get("instance.name")}`,
-			"description": `Clusterio instance for ${this.config.get("controller.name")}`,
-			"tags": ["clusterio"],
-			"max_players": 0,
-			"visibility": { "public": true, "lan": true },
-			"game_password": "",
-			"require_user_verification": true,
-			"max_upload_in_kilobytes_per_second": 0,
-			"max_upload_slots": 5,
-			"ignore_player_limit_for_returning_players": false,
-			"allow_commands": "admins-only",
-			"autosave_interval": 10,
-			"autosave_slots": 5,
-			"afk_autokick_interval": 0,
-			"auto_pause": false,
-			"only_admins_can_pause_the_game": true,
-			"autosave_only_on_server": true,
-			"non_blocking_saving": false,
-
-			...instanceConfig.get("factorio.settings"),
-		};
-		instanceConfig.set("factorio.settings", settings);
-
-		let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, undefined, Date.now());
-		this.instances.set(instance);
-		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance);
-		this.addInstanceHooks(instance);
-		return instance;
-	}
-
 	/**
 	 * Removes all saves currently stored for the given instance, if any
 	 * @param instanceId - Id of Instance to clear saves for.
@@ -938,84 +912,6 @@ export default class Controller {
 		this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId));
 	}
 
-	/**
-	 * Change assigned host of an instance
-	 *
-	 * Unassigns instance from existing host if already assigned to one and
-	 * then assigns it to the given host.  This is the only supported way of
-	 * changing the instance.assigned_host config entry.
-	 *
-	 * Note: this will not transfer any files or saves stored on the host
-	 * the instance was previously assgined to the new one.
-	 *
-	 * @param instanceId - ID of Instance to assign.
-	 * @param hostId - ID of host to assign instance to.
-	 */
-	async instanceAssign(instanceId: number, hostId?: number) {
-		let instance = this.getRequestInstance(instanceId);
-
-		// Check if target host is connected
-		let newHostConnection: HostConnection | undefined;
-		if (hostId !== undefined) {
-			newHostConnection = this.wsServer.hostConnections.get(hostId);
-			if (!newHostConnection) {
-				// The case of the host not getting the assign instance message
-				// still have to be handled, so it's not a requirement that the
-				// target host be connected to the controller while doing the
-				// assignment, but it is IMHO a better user experience if this
-				// is the case.
-				throw new lib.RequestError("Target host is not connected to the controller");
-			}
-		}
-
-		// Unassign from currently assigned host if it is connected.
-		let currentAssignedHost = instance.config.get("instance.assigned_host");
-		if (currentAssignedHost !== null && hostId !== currentAssignedHost) {
-			let oldHostConnection = this.wsServer.hostConnections.get(currentAssignedHost);
-			if (oldHostConnection && !oldHostConnection.connector.closing) {
-				await oldHostConnection.send(new lib.InstanceUnassignInternalRequest(instanceId));
-			}
-		}
-
-		// Remove saves recorded from currently assigned host if any
-		this.clearSavesOfInstance(instanceId);
-
-		// Assign to target
-		instance.config.set("instance.assigned_host", hostId ?? null);
-		if (hostId !== undefined && newHostConnection) {
-			await newHostConnection.send(
-				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host"))
-			);
-		} else {
-			instance.status = "unassigned";
-		}
-		this.instances.set(instance);
-	}
-
-	/**
-	 * Delete an instance
-	 *
-	 * Permanently deletes the instance from its assigned host (if assigned)
-	 * and the controller.  This action cannot be undone.
-	 *
-	 * @param instanceId - ID of instance to delete.
-	 */
-	async instanceDelete(instanceId: number) {
-		let instance = this.getRequestInstance(instanceId);
-		let hostId = instance.config.get("instance.assigned_host");
-		if (hostId !== null) {
-			await this.sendTo({ hostId }, new lib.InstanceDeleteInternalRequest(instanceId));
-		}
-
-		let prev = instance.status;
-		this.instances.delete(instance);
-
-		this.clearSavesOfInstance(instanceId);
-		this.userManager.clearStatsOfInstance(instanceId);
-
-		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance, prev);
-	}
-
 	/**
 	 * Notify the config of an instance updated
 	 *
@@ -1025,7 +921,7 @@ export default class Controller {
 	 * @param instance -
 	 *     Instance to nodify the config updated.
 	 */
-	async instanceConfigUpdated(instance: InstanceInfo) {
+	async instanceConfigUpdated(instance: InstanceRecord) {
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId !== null) {
 			let connection = this.wsServer.hostConnections.get(hostId);
@@ -1037,16 +933,7 @@ export default class Controller {
 		}
 	}
 
-	addInstanceHooks(instance: InstanceInfo) {
-		instance.config.on("fieldChanged", (field: string, curr: any, prev: any) => {
-			instance.updatedAtMs = Date.now();
-			this.instances.set(instance); // Trigger update logic
-
-			lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, field, curr, prev);
-		});
-	}
-
-	instanceDetailsUpdated(instances: InstanceInfo[]) {
+	instanceUpdated(instances: InstanceRecord[]) {
 		for (const instance of instances) {
 			if (instance.status === "stopped") {
 				this.subscriptions.unsubscribeAddress(lib.Address.fromShorthand({instanceId: instance.id}));
@@ -1098,17 +985,12 @@ export default class Controller {
 		return mods.length ? new lib.ModUpdatesEvent(mods) : null;
 	}
 
-	usersUpdated(users: ControllerUser[]) {
-		const now = Date.now();
-		for (const user of users) {
-			user.updatedAtMs = now;
-		}
-		this.userManager.dirty = true;
+	usersUpdated(users: UserRecord[]) {
 		this.subscriptions.broadcast(new lib.UserUpdatesEvent(users));
 	}
 
 	async handleUserSubscription(request: lib.SubscriptionRequest) {
-		const users = [...this.userManager.users.values()].filter(
+		const users = [...this.users.records.values()].filter(
 			user => user.updatedAtMs > request.lastRequestTimeMs,
 		);
 		return users.length ? new lib.UserUpdatesEvent(users) : null;
@@ -1119,7 +1001,7 @@ export default class Controller {
 	 * permissions for this user may have changed.
 	 * @param user - User permisions updated for.
 	 */
-	userPermissionsUpdated(user: ControllerUser) {
+	userPermissionsUpdated(user: Readonly<User>) {
 		for (let controlConnection of this.wsServer.controlConnections.values()) {
 			if (controlConnection.user === user) {
 				controlConnection.send(
@@ -1195,6 +1077,9 @@ export default class Controller {
 				this.plugins.set(pluginInfo.name, controllerPlugin);
 
 			} catch (err: any) {
+				if (err.code === "InstallationError") {
+					throw err;
+				}
 				throw new lib.PluginError(pluginInfo.name, err);
 			}
 
@@ -1378,7 +1263,7 @@ export default class Controller {
 			}
 
 		} else if (dst.type === lib.Address.instance) {
-			let instance = this.getRequestInstance(dst.id);
+			const instance = this.instances.getForRequest(dst.id);
 			let hostId = instance.config.get("instance.assigned_host");
 			if (hostId === null) {
 				throw new lib.RequestError("Instance is not assigned to a host");
@@ -1413,7 +1298,7 @@ export default class Controller {
 			}
 
 		} else if (dst.type === lib.Address.instance) {
-			let instance = this.instances.get(dst.id);
+			const instance = this.instances.get(dst.id);
 			if (!instance) {
 				throw new Error(`Instance with ID ${dst.id} does not exist`);
 			}
@@ -1472,7 +1357,7 @@ export default class Controller {
 	}
 
 	async sendRequestToHostByInstanceId<Req extends InstanceId, Res>(request: Req & lib.Request<Req, Res>) {
-		let instance = this.getRequestInstance(request.instanceId);
+		let instance = this.instances.getForRequest(request.instanceId);
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId === null) {
 			throw new lib.RequestError("Instance is not assigned to a host");
@@ -1485,7 +1370,7 @@ export default class Controller {
 	}
 
 	sendEventToHostByInstanceId<T extends { instanceId: number }>(event: T & lib.Event<T>) {
-		let instance = this.getRequestInstance(event.instanceId);
+		let instance = this.instances.getForRequest(event.instanceId);
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId === null) {
 			return;

packages/controller/src/ControllerUser.ts

@@ -1,96 +0,0 @@
-import { Static, Type } from "@sinclair/typebox";
-
-import {
-	IControllerUser, PermissionError, PlayerStats,
-	Role, SubscribableDatastore, User, permissions,
-} from "@clusterio/lib";
-
-export default class ControllerUser extends User implements IControllerUser {
-	constructor(
-		private _controllerRoles: SubscribableDatastore<Role>,
-		/** Unix time in seconds the user token must be issued after to be valid.  */
-		public tokenValidAfter = 0,
-		...args: ConstructorParameters<typeof User>
-	) {
-		super(...args);
-	}
-
-	static jsonSchema = Type.Object({
-		...User.jsonSchema.properties,
-		token_valid_after: Type.Optional(Type.Number()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>, controllerRoles: SubscribableDatastore<Role>) {
-		const roleIds = new Set<number>(json.roles ?? []);
-		const instanceStats = new Map(
-			(json.instance_stats ?? []).map(
-				([id, stats]) => [id, PlayerStats.fromJSON(stats)]
-			)
-		);
-		const playerStats = User._calculatePlayerStats(instanceStats);
-		return new this(
-			controllerRoles,
-			json.token_valid_after,
-			json.name,
-			roleIds,
-			new Set(json.instances),
-			json.is_admin,
-			json.is_banned,
-			json.is_whitelisted,
-			json.ban_reason,
-			json.updated_at_ms,
-			json.is_deleted,
-			playerStats,
-			instanceStats,
-		);
-	}
-
-	toJSON(controller?: boolean): Static<typeof ControllerUser.jsonSchema> {
-		if (!controller) {
-			return super.toJSON();
-		}
-
-		const json: Static<typeof ControllerUser.jsonSchema> = super.toJSON();
-		if (this.tokenValidAfter) {
-			json.token_valid_after = this.tokenValidAfter;
-		}
-
-		return json;
-	}
-
-	get roles(): ReadonlySet<Readonly<Role>> {
-		return new Set([...this.roleIds].map(
-			id => this._controllerRoles.get(id)
-		).filter(
-			(r): r is Role => Boolean(r)
-		));
-	}
-
-	/**
-	 * Invalidate current tokens for the user
-	 *
-	 * Sets the tokenValidAfter property to the current time, which causes
-	 * all currently issued tokens for the user to become invalid.
-	 */
-	invalidateToken() {
-		this.tokenValidAfter = Math.floor(Date.now() / 1000);
-	}
-
-	checkPermission(permission: string) {
-		if (!permissions.has(permission)) {
-			throw new Error(`permission ${permission} does not exist`);
-		}
-
-		for (let roleId of this.roleIds) {
-			let role = this._controllerRoles.get(roleId);
-			if (!role) {
-				continue;
-			}
-			if (role.permissions.has("core.admin") || role.permissions.has(permission)) {
-				return;
-			}
-		}
-
-		throw new PermissionError("Permission denied");
-	}
-}

packages/controller/src/HostConnection.ts

@@ -5,8 +5,8 @@ import * as lib from "@clusterio/lib";
 const { logger } = lib;
 
 import BaseConnection from "./BaseConnection";
-import InstanceInfo from "./InstanceInfo";
-import HostInfo from "./HostInfo";
+import InstanceRecord from "./InstanceRecord";
+import HostRecord from "./HostRecord";
 
 
 /**
@@ -16,7 +16,7 @@ import HostInfo from "./HostInfo";
  * @alias module:controller/src/HostConnection
  */
 export default class HostConnection extends BaseConnection {
-	private info: HostInfo;
+	private info: HostRecord;
 	plugins: Map<string, string>;
 
 	constructor(
@@ -30,7 +30,7 @@ export default class HostConnection extends BaseConnection {
 		this.plugins = new Map(Object.entries(registerData.plugins));
 
 		const previousHostInfo = this._controller.hosts.get(registerData.id);
-		this.info = new HostInfo(
+		this.info = new HostRecord(
 			registerData.id,
 			previousHostInfo?.name ?? "",
 			registerData.version,
@@ -57,7 +57,7 @@ export default class HostConnection extends BaseConnection {
 
 		this.connector.on("close", () => {
 			// Update status to unknown for instances on this host.
-			const instances: InstanceInfo[] = [];
+			const instances: InstanceRecord[] = [];
 			for (let instance of this._controller.instances.valuesMutable()) {
 				if (instance.config.get("instance.assigned_host") !== this.id) {
 					continue;
@@ -68,7 +68,7 @@ export default class HostConnection extends BaseConnection {
 				instance.status = "unknown";
 				lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 			}
-			this._controller.instances.setMany(instances);
+			this._controller.instances.records.setMany(instances);
 
 			this.info.connected = false;
 			this._controller.hosts.set(this.info);
@@ -207,9 +207,14 @@ export default class HostConnection extends BaseConnection {
 
 		let prev = instance.status;
 		instance.status = request.status;
+		// If the instance has transitioned to running then updated the started at time
+		if (request.status === "running" && prev === "starting") {
+			instance.startedAtMs = Date.now();
+		}
+
 		instance.gamePort = request.gamePort;
 		instance.factorioVersion = request.factorioVersion;
-		this._controller.instances.set(instance);
+		this._controller.instances.records.set(instance);
 		logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
 		await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 	}
@@ -225,7 +230,7 @@ export default class HostConnection extends BaseConnection {
 		}
 
 		// Assign instances the host has but controller does not
-		const instanceUpdates: InstanceInfo[] = [];
+		const instanceUpdates: InstanceRecord[] = [];
 		for (let instanceData of request.instances) {
 			const instanceConfig = new lib.InstanceConfig("controller");
 			instanceConfig.update(instanceData.config, false, "host");
@@ -246,6 +251,9 @@ export default class HostConnection extends BaseConnection {
 					controllerInstance.status = instanceData.status;
 					controllerInstance.gamePort = instanceData.gamePort;
 					controllerInstance.factorioVersion = instanceData.factorioVersion;
+					if (instanceData.status === "running" && prev === "starting") {
+						controllerInstance.startedAtMs = Date.now();
+					}
 					instanceUpdates.push(controllerInstance);
 					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
 					await lib.invokeHook(
@@ -256,31 +264,30 @@ export default class HostConnection extends BaseConnection {
 			}
 
 			instanceConfig.set("instance.assigned_host", this.id);
-			let newInstance = new InstanceInfo(
-				instanceConfig,
-				instanceData.status,
-				instanceData.gamePort,
-				instanceData.factorioVersion,
-				Date.now(),
-			);
+			const newInstance = await this._controller.instances.createInstance(instanceConfig, true);
+			newInstance.factorioVersion = instanceData.factorioVersion;
+			newInstance.gamePort = instanceData.gamePort;
+			newInstance.status = instanceData.status;
 			instanceUpdates.push(newInstance);
-			this._controller.addInstanceHooks(newInstance);
-			await this.send(
-				new lib.InstanceAssignInternalRequest(
-					instanceConfig.get("instance.id"), instanceConfig.toRemote("host")
-				)
-			);
-			await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance);
 		}
 
-		this._controller.instances.setMany(instanceUpdates);
+		// We suppress the changes made by createInstance so we must manually make them here
+		this._controller.instances.records.setMany(instanceUpdates);
+		await Promise.all(instanceUpdates.flatMap(newInstance => [
+			lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance),
+			this.send(
+				new lib.InstanceAssignInternalRequest(
+					newInstance.config.get("instance.id"), newInstance.config.toRemote("host")
+				)
+			),
+		]));
 
 		// Push lists to make sure they are in sync.
 		let adminlist: Set<string> = new Set();
 		let banlist: Map<string, string> = new Map();
 		let whitelist: Set<string> = new Set();
 
-		for (let user of this._controller.userManager.users.values()) {
+		for (let user of this._controller.users.values()) {
 			if (user.isAdmin) {
 				adminlist.add(user.id);
 			}
@@ -329,21 +336,18 @@ export default class HostConnection extends BaseConnection {
 
 	async handleInstancePlayerUpdateEvent(event: lib.InstancePlayerUpdateEvent, src: lib.Address) {
 		let instanceId = src.id;
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
+		const user = this._controller.users.getOrCreateUser(event.name);
 
 		if (event.type === "join") {
-			this._controller.userManager.notifyJoin(user, instanceId);
+			user.notifyJoin(instanceId);
 		} else if (event.type === "leave") {
-			this._controller.userManager.notifyLeave(user, instanceId);
+			user.notifyLeave(instanceId);
 		}
 
 		user.instanceStats.set(instanceId, event.stats);
 
 		user.recalculatePlayerStats();
-		this._controller.usersUpdated([user]);
+		user.saveRecord();
 
 		let instance = this._controller.instances.get(instanceId)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
@@ -359,13 +363,9 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
+		const user = this._controller.users.getOrCreateUser(event.name);
+		user.set("isAdmin", event.admin);
 
-		user.isAdmin = event.admin;
-		this._controller.usersUpdated([user]);
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.admin ? "promote" : "demote" as lib.PlayerEvent["type"],
@@ -379,14 +379,11 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
-
+		const user = this._controller.users.getOrCreateUser(event.name);
 		user.isBanned = event.banned;
 		user.banReason = event.reason;
-		this._controller.usersUpdated([user]);
+		user.saveRecord();
+
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.banned ? "ban" : "unban" as lib.PlayerEvent["type"],
@@ -401,13 +398,9 @@ export default class HostConnection extends BaseConnection {
 			return;
 		}
 
-		let user = this._controller.userManager.getByName(event.name);
-		if (!user) {
-			user = this._controller.userManager.createUser(event.name);
-		}
+		const user = this._controller.users.getOrCreateUser(event.name);
+		user.set("isWhitelisted", event.whitelisted);
 
-		user.isWhitelisted = event.whitelisted;
-		this._controller.usersUpdated([user]);
 		const instance = this._controller.instances.get(src.id)!;
 		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
 			type: event.whitelisted ? "whitelisted" : "unwhitelisted" as lib.PlayerEvent["type"],

packages/controller/src/HostRecord.ts

@@ -1,10 +1,8 @@
 import { Type, Static } from "@sinclair/typebox";
 import { HostDetails } from "@clusterio/lib";
 
-/**
- * Info about a host known to the controller.
- */
-export default class HostInfo {
+/** Underlying data class for hosts on the controller */
+export default class HostRecord {
 	constructor(
 		/** Id of this host */
 		public id: number,
@@ -33,12 +31,12 @@ export default class HostInfo {
 		"name": Type.String(),
 		"version": Type.String(),
 		"plugins": Type.Record(Type.String(), Type.String()),
-		"connected": Type.Boolean(),
-		"remote_address": Type.String(),
-		"public_address": Type.String(),
-		"token_valid_after": Type.Number(),
-		"updated_at_ms": Type.Number(),
-		"is_deleted": Type.Boolean(),
+		"connected": Type.Optional(Type.Boolean()),
+		"remote_address": Type.Optional(Type.String()),
+		"public_address": Type.Optional(Type.String()),
+		"token_valid_after": Type.Optional(Type.Number()),
+		"updated_at_ms": Type.Optional(Type.Number()),
+		"is_deleted": Type.Optional(Type.Boolean()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
@@ -56,19 +54,50 @@ export default class HostInfo {
 		);
 	}
 
-	toJSON(): Static<typeof HostInfo.jsonSchema> {
-		return {
+	toJSON() {
+		const json = {
 			id: this.id,
 			name: this.name,
 			version: this.version,
 			plugins: Object.fromEntries(this.plugins),
-			connected: this.connected,
-			remote_address: this.remoteAddress,
-			public_address: this.publicAddress,
-			token_valid_after: this.tokenValidAfter,
-			updated_at_ms: this.updatedAtMs,
-			is_deleted: this.isDeleted,
-		};
+		} as Static<typeof HostRecord.jsonSchema>;
+
+		if (this.connected !== false) {
+			json.connected = this.connected;
+		}
+		if (this.remoteAddress !== "") {
+			json.remote_address = this.remoteAddress;
+		}
+		if (this.publicAddress !== "") {
+			json.public_address = this.publicAddress;
+		}
+		if (this.tokenValidAfter !== 0) {
+			json.token_valid_after = this.tokenValidAfter;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updated_at_ms = this.updatedAtMs;
+		}
+		if (this.isDeleted !== false) {
+			json.is_deleted = this.isDeleted;
+		}
+
+		return json;
+	}
+
+
+	static fromHostDetails(details: HostDetails, plugins: Map<string, string>) {
+		return new HostRecord(
+			details.id,
+			details.name,
+			details.version,
+			plugins,
+			details.connected,
+			details.remoteAddress,
+			details.publicAddress,
+			details.tokenValidAfter,
+			details.updatedAtMs,
+			details.isDeleted,
+		);
 	}
 
 	toHostDetails() {
@@ -85,4 +114,3 @@ export default class HostInfo {
 		);
 	}
 }
-

packages/controller/src/InstanceManager.ts

@@ -0,0 +1,274 @@
+import * as lib from "@clusterio/lib";
+
+import type Controller from "./Controller";
+import type HostConnection from "./HostConnection";
+import InstanceRecord from "./InstanceRecord";
+
+export default class InstanceManager {
+	private static readonly DefaultFactorioSettings = {
+		tags: ["clusterio"],
+		max_players: 0,
+		visibility: { public: true, lan: true },
+		game_password: "",
+		require_user_verification: true,
+		max_upload_in_kilobytes_per_second: 0,
+		max_upload_slots: 5,
+		ignore_player_limit_for_returning_players: false,
+		allow_commands: "admins-only",
+		autosave_interval: 10,
+		autosave_slots: 5,
+		afk_autokick_interval: 0,
+		auto_pause: false,
+		only_admins_can_pause_the_game: true,
+		autosave_only_on_server: true,
+		non_blocking_saving: false,
+	} as const;
+
+	constructor(
+		public readonly records: lib.SubscribableDatastore<InstanceRecord>,
+		private readonly _controller: Controller,
+	) {
+		for (const instance of records.values()) {
+			this.addInstanceHooks(instance);
+		}
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Basic Accessors                                                           */
+	/* ------------------------------------------------------------------------- */
+
+	has(id: InstanceRecord["id"]): boolean {
+		return this.records.has(id);
+	}
+
+	getMutable(id: InstanceRecord["id"]): InstanceRecord | undefined {
+		return this.records.get(id);
+	}
+
+	get(id: InstanceRecord["id"]): Readonly<InstanceRecord> | undefined {
+		return this.getMutable(id);
+	}
+
+	valuesMutable(): IterableIterator<InstanceRecord> {
+		return this.records.values();
+	}
+
+	values(): IterableIterator<Readonly<InstanceRecord>> {
+		return this.valuesMutable();
+	}
+
+	/**
+	 * Get instance by ID for a request.
+	 *
+	 * Convenience wrapper around {@link getMutable} that throws
+	 * if the instance does not exist.
+	 *
+	 * @param instanceId - ID of instance to get.
+	 * @returns The instance record.
+	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 */
+	getForRequest(instanceId: number): InstanceRecord {
+		const instance = this.records.get(instanceId);
+		if (!instance) {
+			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
+		}
+		return instance;
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Internal Helpers                                                          */
+	/* ------------------------------------------------------------------------- */
+
+	private async _notifyStatusChanged(
+		instance: InstanceRecord,
+		prev?: lib.InstanceStatus,
+	): Promise<void> {
+		await lib.invokeHook(
+			this._controller.plugins,
+			"onInstanceStatusChanged",
+			instance,
+			prev,
+		);
+	}
+
+	private async _notifyConfigFieldChanged(
+		instance: InstanceRecord,
+		field: string,
+		curr: unknown,
+		prev: unknown,
+	): Promise<void> {
+		await lib.invokeHook(
+			this._controller.plugins,
+			"onInstanceConfigFieldChanged",
+			instance,
+			field,
+			curr,
+			prev,
+		);
+	}
+
+	/**
+	 * Attach change listeners to an instance record.
+	 *
+	 * Hooks into configuration field changes and ensures the datastore
+	 * is updated and plugins are notified when a config field changes.
+	 *
+	 * @param instance - The instance record to attach hooks to.
+	 * @internal
+	 */
+	addInstanceHooks(instance: InstanceRecord): void {
+		instance.config.on("fieldChanged", (field: string, curr: unknown, prev: unknown) => {
+			this.records.set(instance);
+			this._notifyConfigFieldChanged(instance, field, curr, prev);
+		});
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Lifecycle                                                                 */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Create a new instance.
+	 *
+	 * @example
+	 * const config = new lib.InstanceConfig("controller", { "instance.name": "My instance" });
+	 * const instance = await controller.instances.createInstance(config);
+	 *
+	 * @param instanceConfig - Config to base the newly created instance on.
+	 * @param _suppressChanges - When true, side effects are suppressed, caller takes ownership.
+	 * @returns The created instance record.
+	 * @throws {module:lib.RequestError} if an instance with the same ID already exists.
+	 */
+	async createInstance(
+		instanceConfig: lib.InstanceConfig,
+		_suppressChanges?: boolean,
+	): Promise<InstanceRecord> {
+		const instanceId = instanceConfig.get("instance.id");
+		if (this.records.has(instanceId)) {
+			throw new lib.RequestError(`Instance with ID ${instanceId} already exists`);
+		}
+
+		const controllerName = this._controller.config.get("controller.name");
+		instanceConfig.set("factorio.settings", {
+			name: `${controllerName} - ${instanceConfig.get("instance.name")}`,
+			description: `Clusterio instance for ${controllerName}`,
+			...InstanceManager.DefaultFactorioSettings,
+			...instanceConfig.get("factorio.settings"),
+		});
+
+		const instance = new InstanceRecord(instanceConfig, "unassigned");
+		this.addInstanceHooks(instance);
+
+		if (!_suppressChanges) {
+			this.records.set(instance);
+			await this._notifyStatusChanged(instance);
+		}
+
+		return instance;
+	}
+
+	/**
+	 * Change the assigned host of an instance.
+	 *
+	 * Unassigns the instance from the currently assigned host (if connected)
+	 * and assigns it to the provided host.
+	 *
+	 * This is the only supported way of modifying the
+	 * `instance.assigned_host` configuration entry.
+	 *
+	 * Note:
+	 * - This does NOT transfer any files or saves between hosts.
+	 * - If the instance is already assigned to the specified host, no action is taken.
+	 *
+	 * @param instanceId - ID of instance to assign.
+	 * @param hostId - ID of host to assign the instance to. If undefined,
+	 * the instance will be unassigned.
+	 *
+	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 * @throws {module:lib.RequestError} if the target host is not connected.
+	 */
+	async assignInstance(
+		instanceId: number,
+		hostId: number | null,
+	): Promise<void> {
+		const instance = this.getForRequest(instanceId);
+		const hostConnections = this._controller.wsServer.hostConnections;
+
+		const currentAssignedHost = instance.config.get("instance.assigned_host");
+		if (currentAssignedHost === hostId) {
+			return;
+		}
+
+		let newHostConnection: HostConnection | undefined;
+		if (hostId !== null) {
+			newHostConnection = hostConnections.get(hostId);
+			if (!newHostConnection) {
+				throw new lib.RequestError("Target host is not connected to the controller");
+			}
+		}
+
+		if (currentAssignedHost !== null) {
+			const oldHostConnection = hostConnections.get(currentAssignedHost);
+			if (oldHostConnection && !oldHostConnection.connector.closing) {
+				await oldHostConnection.send(
+					new lib.InstanceUnassignInternalRequest(instanceId),
+				);
+			}
+		}
+
+		this._controller.clearSavesOfInstance(instanceId);
+		instance.config.set("instance.assigned_host", hostId);
+
+		if (newHostConnection) {
+			await newHostConnection.send(
+				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host")),
+			);
+		} else {
+			instance.status = "unassigned";
+		}
+
+		this.records.set(instance);
+	}
+
+	/**
+	 * Unassign an instance from its current host.
+	 *
+	 * Convenience wrapper around {@link assignInstance} that removes
+	 * the assigned host.
+	 *
+	 * @param instanceId - ID of instance to unassign.
+	 */
+	async unassignInstance(instanceId: number): Promise<void> {
+		return this.assignInstance(instanceId, null);
+	}
+
+	/**
+	 * Delete an instance permanently.
+	 *
+	 * Sends a delete request to the assigned host (if any),
+	 * removes the instance from the datastore,
+	 * clears associated saves and user statistics.
+	 *
+	 * @param instanceId - ID of instance to delete.
+	 *
+	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 */
+	async deleteInstance(instanceId: number): Promise<void> {
+		const instance = this.getForRequest(instanceId);
+		const hostId = instance.config.get("instance.assigned_host");
+
+		if (hostId !== null) {
+			await this._controller.sendTo(
+				{ hostId },
+				new lib.InstanceDeleteInternalRequest(instanceId),
+			);
+		}
+
+		const prevStatus = instance.status;
+		this.records.delete(instance);
+		this._controller.clearSavesOfInstance(instanceId);
+		this._controller.users.clearStatsOfInstance(instanceId);
+
+		await this._notifyStatusChanged(instance, prevStatus);
+	}
+}

packages/controller/src/InstanceRecord.ts

@@ -1,52 +1,22 @@
 import * as lib from "@clusterio/lib";
 import { Static, Type } from "@sinclair/typebox";
 
-
-/**
- * Runtime status of an instance on the controller
- * @alias module:controller/src/InstanceInfo
- */
-export default class InstanceInfo {
+/** Underlying data class for instances on the controller */
+export default class InstanceRecord {
 	constructor(
 		public config: lib.InstanceConfig,
 		public status: lib.InstanceStatus,
 		public gamePort?: number,
 		public factorioVersion?: lib.TargetVersion,
+		public startedAtMs = 0,
 		public updatedAtMs = 0,
 	) {
 		this.config = config;
 		this.status = status;
 	}
 
-	static jsonSchema = Type.Object({
-		"config": lib.InstanceConfig.jsonSchema,
-		"status": lib.InstanceStatus,
-		"gamePort": Type.Optional(Type.Number()),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"factorioVersion": Type.Optional(lib.TargetVersionSchema),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>, location: lib.ConfigLocation) {
-		return new this(
-			lib.InstanceConfig.fromJSON(json.config, location),
-			json.status,
-			json.gamePort,
-			json.factorioVersion,
-			json.updatedAtMs,
-		);
-	}
-
-	toInstanceDetails() {
-		return new lib.InstanceDetails(
-			this.config.get("instance.name"),
-			this.id,
-			this.config.get("instance.assigned_host") ?? undefined,
-			this.gamePort,
-			this.status,
-			this.factorioVersion,
-			this.updatedAtMs,
-			this.config.get("instance.exclude_from_start_all"),
-		);
+	get id(): number {
+		return this.config.get("instance.id");
 	}
 
 	get isDeleted() {
@@ -60,8 +30,70 @@ export default class InstanceInfo {
 		this.status = "deleted";
 	}
 
-	/** Shorthand for `instance.config.get("instance.id")` */
-	get id():number {
-		return this.config.get("instance.id");
+	static jsonSchema = Type.Object({
+		"config": lib.InstanceConfig.jsonSchema,
+		"status": lib.InstanceStatus,
+		"gamePort": Type.Optional(Type.Number()),
+		"startedAtMs": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"factorioVersion": Type.Optional(lib.TargetVersionSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			lib.InstanceConfig.fromJSON(json.config, "controller"),
+			json.status,
+			json.gamePort,
+			json.factorioVersion,
+			json.startedAtMs,
+			json.updatedAtMs,
+		);
+	}
+
+	toJSON(): Static<typeof InstanceRecord.jsonSchema> {
+		const json: any = {
+			config: this.config.toJSON(),
+			status: this.status,
+		};
+
+		if (this.gamePort !== undefined) {
+			json.gamePort = this.gamePort;
+		}
+		if (this.factorioVersion !== undefined) {
+			json.factorioVersion = this.factorioVersion;
+		}
+		if (this.startedAtMs !== 0) {
+			json.startedAtMs = this.startedAtMs;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updatedAtMs = this.updatedAtMs;
+		}
+
+		return json;
+	}
+
+	toInstanceDetails() {
+		return new lib.InstanceDetails(
+			this.config.get("instance.name"),
+			this.id,
+			this.config.get("instance.assigned_host") ?? undefined,
+			this.gamePort,
+			this.status,
+			this.factorioVersion,
+			this.startedAtMs,
+			this.updatedAtMs,
+			this.config.get("instance.exclude_from_start_all"),
+		);
+	}
+
+	static fromInstanceDetails(details: lib.InstanceDetails, config: lib.InstanceConfig) {
+		return new InstanceRecord(
+			config,
+			details.status,
+			details.gamePort,
+			details.factorioVersion,
+			details.startedAtMs,
+			details.updatedAtMs,
+		);
 	}
 }

packages/controller/src/User.ts

@@ -0,0 +1,146 @@
+
+import {
+	IUser, Role, SubscribableDatastore,
+	PermissionError, permissions,
+} from "@clusterio/lib";
+
+import UserRecord from "./UserRecord";
+import { Static } from "@sinclair/typebox";
+
+/**
+ * Represents a user as viewed from the controller.
+ *
+ * All methods are safe to use when readonly.
+ * Only use mutable when needing to change multiple values.
+ */
+export default class User extends UserRecord implements IUser {
+	constructor(
+		private _controllerUsers: SubscribableDatastore<UserRecord>,
+		private _controllerRoles: SubscribableDatastore<Role>,
+		...args: ConstructorParameters<typeof UserRecord>
+	) {
+		super(...args);
+	}
+
+	static fromJSON(
+		json: Static<typeof this.jsonSchema>,
+		_controllerUsers: SubscribableDatastore<UserRecord>,
+		_controllerRoles: SubscribableDatastore<Role>,
+	) {
+		return this.fromUserRecord(
+			UserRecord.fromJSON(json),
+			_controllerUsers,
+			_controllerRoles,
+		);
+	}
+
+	static fromUserRecord(
+		userRecord: UserRecord,
+		_controllerUsers: SubscribableDatastore<UserRecord>,
+		_controllerRoles: SubscribableDatastore<Role>,
+	): User {
+		return new this(
+			_controllerUsers,
+			_controllerRoles,
+			userRecord.tokenValidAfter,
+			userRecord.name,
+			userRecord.roleIds,
+			userRecord.instances,
+			userRecord.isAdmin,
+			userRecord.isBanned,
+			userRecord.isWhitelisted,
+			userRecord.banReason,
+			userRecord.updatedAtMs,
+			userRecord.isDeleted,
+			userRecord.instanceStats,
+		);
+	}
+
+	/** Roles this user has */
+	get roles(): ReadonlySet<Readonly<Role>> {
+		return new Set([...this.roleIds].map(
+			id => this._controllerRoles.get(id)
+		).filter(
+			(r): r is Role => Boolean(r)
+		));
+	}
+
+	/** Save the underlying user record */
+	saveRecord() {
+		this._controllerUsers.set(this);
+	}
+
+	/** Set a property on the underlying record, then save it. */
+	set<K extends keyof this>(key: K, value: this[K]) {
+		this[key] = value;
+		this.saveRecord();
+	}
+
+	/**
+	 * Invalidate current tokens for the user
+	 *
+	 * Sets the tokenValidAfter property to the current time, which causes
+	 * all currently issued tokens for the user to become invalid.
+	 */
+	invalidateToken() {
+		this.tokenValidAfter = Math.floor(Date.now() / 1000);
+		this.saveRecord();
+	}
+
+	/** Add a role to this user */
+	addRole(roleId: number) {
+		const role = this._controllerRoles.get(roleId);
+		if (!role) {
+			throw new Error(`role ${roleId} does not exist`);
+		}
+		this.roleIds.add(roleId);
+		this.saveRecord();
+	}
+
+	/** Remove a role from user */
+	removeRole(roleId: number) {
+		if (!this.roleIds.delete(roleId)) {
+			return false;
+		}
+		this.saveRecord();
+		return true;
+	}
+
+	/** Set a user as being online on a specific instance */
+	notifyJoin(instanceId: number) {
+		this.instances.add(instanceId);
+		this.saveRecord();
+	}
+
+	/** Set a user as being offline on a specific instance */
+	notifyLeave(instanceId: number) {
+		this.instances.delete(instanceId);
+		this.saveRecord();
+	}
+
+	/** Clear the stats of a player on a specific instance */
+	clearInstanceStats(instanceId: number) {
+		this.instances.delete(instanceId);
+		this.instanceStats.delete(instanceId);
+		this.recalculatePlayerStats();
+		this.saveRecord();
+	}
+
+	checkPermission(permission: string) {
+		if (!permissions.has(permission)) {
+			throw new Error(`permission ${permission} does not exist`);
+		}
+
+		for (const roleId of this.roleIds) {
+			const role = this._controllerRoles.get(roleId);
+			if (!role) {
+				continue;
+			}
+			if (role.permissions.has("core.admin") || role.permissions.has(permission)) {
+				return;
+			}
+		}
+
+		throw new PermissionError("Permission denied");
+	}
+}

packages/controller/src/UserManager.ts

@@ -1,67 +1,47 @@
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import jwt from "jsonwebtoken";
 
 import * as lib from "@clusterio/lib";
 
-import ControllerUser from "./ControllerUser";
+import User from "./User";
+import UserRecord from "./UserRecord";
 
 /**
- * Manages users and roles
+ * Access layer for users on the controller.
+ *
+ * In performance critical paths the raw records can be accessed directly.
+ * Otherwise this class will wrap data in a view class with a set and save method.
+ *
  * @alias module:controller/src/UserManager
  */
 export default class UserManager {
-	users: Map<ControllerUser["id"], ControllerUser> = new Map();
-	dirty = false;
-
-	/**
-	 * Set of users currently online in the cluster.
-	 */
-	onlineUsers = new Set<ControllerUser>();
-
 	constructor(
-		private _config: lib.ControllerConfig,
-		private _roles: lib.SubscribableDatastore<lib.Role>,
+		public records: lib.SubscribableDatastore<UserRecord>,
+		private _controllerRoles: lib.SubscribableDatastore<lib.Role>,
+		private _controllerConfig: lib.ControllerConfig,
 	) {
 	}
 
-	getByName(name: string) {
-		return this.users.get(name.toLowerCase());
-	}
-
-	async load(filePath: string): Promise<void> {
+	/**
+	 * Prior to 2.0.0-alpha.22 users and roles were stored in a single file.
+	 * This migration separates them so they can be loaded by datastore.
+	 *
+	 * @param usersFilePath The src and dst file path for the users json file
+	 * @param rolesFilePath The dst for the roles json file
+	 */
+	static async attemptMigrateLegacyUsersFile(usersFilePath: string, rolesFilePath: string) {
 		try {
-			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-			if ("roles" in content) { // Split file format after 2.0.0-alpha.22
-				this._roles.setMany(content.roles.map((serializedRole: any) => lib.Role.fromJSON(serializedRole)));
-				await this._roles.save(); // Force a save to prevent data loss
-				content = content.users;
+			const combinedJson = JSON.parse(await fs.readFile(usersFilePath, { encoding: "utf8" }));
+			if (!("roles" in combinedJson)) {
+				return; // Files are already split, no action required
 			}
-
-			let duplicates = 0;
-			for (const serializedUser of content) {
-				const user = ControllerUser.fromJSON(serializedUser, this._roles);
-				const existingUser = this.users.get(user.id);
-				if (existingUser) {
-					// Required migration to all lowercase ids in alpha 19
-					duplicates += 1;
-					// We assume the user with the lower online time is the duplicate
-					if (user.playerStats.onlineTimeMs <= existingUser.playerStats.onlineTimeMs) {
-						existingUser.merge(user);
-						continue; // Skip users.set
-					}
-					user.merge(existingUser);
-				}
-				this.users.set(user.id, user);
-			}
-
-			if (duplicates) {
-				const backupPath = `${filePath}.${Date.now()}.bak`;
-				lib.logger.warn(
-					`A total of ${duplicates} users were merged, a backup was written to: ${backupPath}`
-				);
-				await lib.safeOutputFile(backupPath, JSON.stringify(content, null, "\t"));
+			if (!("users" in combinedJson)) {
+				throw new Error("Legacy users json does not contain users property");
 			}
 
+			// We intentionally write the roles first because if it fails then the old user file will still contain them
+			await lib.safeOutputFile(rolesFilePath, JSON.stringify(combinedJson.roles));
+			await lib.safeOutputFile(usersFilePath, JSON.stringify(combinedJson.users));
 		} catch (err: any) {
 			if (err.code !== "ENOENT") {
 				throw err;
@@ -69,66 +49,117 @@ export default class UserManager {
 		}
 	}
 
-	async save(filePath: string): Promise<void> {
-		this.dirty = false;
-		await lib.safeOutputFile(filePath,
-			JSON.stringify([...this.users.values()].map(user => user.toJSON(true)), null, "\t")
+	/* ------------------------------------------------------------------------- */
+	/* Basic Accessors                                                           */
+	/* ------------------------------------------------------------------------- */
+
+	has(id: UserRecord["id"]) {
+		return this.records.has(id);
+	}
+
+	getMutable(id: UserRecord["id"]) {
+		const userRecord = this.records.getMutable(id);
+		if (!userRecord) {
+			return undefined;
+		}
+
+		return User.fromUserRecord(
+			userRecord,
+			this.records,
+			this._controllerRoles,
 		);
 	}
 
+	get(id: UserRecord["id"]): Readonly<User> | undefined {
+		return this.getMutable(id);
+	}
+
+	hasName(name: UserRecord["name"]) {
+		return this.has(name.toLowerCase());
+	}
+
+	getByNameMutable(name: UserRecord["name"]) {
+		return this.getMutable(name.toLowerCase());
+	}
+
+	getByName(name: UserRecord["name"]) {
+		return this.get(name.toLowerCase());
+	}
+
+	* valuesMutable(): IterableIterator<User> {
+		const roles = this._controllerRoles;
+		const users = this.records;
+
+		for (const record of users.values()) {
+			yield User.fromUserRecord(record, users, roles);
+		}
+	}
+
+	values(): IterableIterator<Readonly<User>> {
+		return this.valuesMutable();
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Lifecycle                                                                 */
+	/* ------------------------------------------------------------------------- */
+
 	/**
 	 * Creates a new user and add it to the user database.
 	 * @param name - Name of the user to create.
 	 * @returns The created user.
 	 */
-	createUser(name: string): ControllerUser {
+	createUser(name: string): User {
 		if (this.getByName(name)) {
 			throw new Error(`User '${name}' already exists`);
 		}
 
 		let roles = new Set<number>();
-		let defaultRoleId = this._config.get("controller.default_role_id");
-		if (defaultRoleId !== null && this._roles.has(defaultRoleId)) {
+		let defaultRoleId = this._controllerConfig.get("controller.default_role_id");
+		if (defaultRoleId !== null && this._controllerRoles.has(defaultRoleId)) {
 			roles.add(defaultRoleId);
 		}
 
-		let user = new ControllerUser(this._roles, 0, name, roles);
-		this.users.set(user.id, user);
-		this.dirty = true;
+		const user = new User(this.records, this._controllerRoles, 0, name, roles);
+		this.records.set(user);
 		return user;
 	}
 
+	/** Get or create a user with a given name. */
+	getOrCreateUser(name: string): User {
+		return this.getByNameMutable(name) ?? this.createUser(name);
+	}
+
+	/**
+	 * Deletes an existing user
+	 *
+	 * @param user - user to delete
+	 */
+	deleteUser(user: UserRecord) {
+		return this.records.delete(user);
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Helpers                                                                   */
+	/* ------------------------------------------------------------------------- */
+
 	/**
 	 * Sign access token for the given user
 	 *
 	 * @param user - user to sign token for
 	 * @returns JWT access token for the user.
 	 */
-	signUserToken(user: ControllerUser): string {
+	signUserToken(user: UserRecord): string {
 		return jwt.sign(
 			{ aud: "user", user: user.id },
-			Buffer.from(this._config.get("controller.auth_secret"), "base64")
+			// auth secret is never null after startup
+			Buffer.from(this._controllerConfig.get("controller.auth_secret")!, "base64")
 		);
 	}
 
-	notifyJoin(user: ControllerUser, instance_id: number) {
-		user.instances.add(instance_id);
-		this.onlineUsers.add(user);
-	}
-
-	notifyLeave(user: ControllerUser, instance_id: number) {
-		user.instances.delete(instance_id);
-		if (!user.instances.size) {
-			this.onlineUsers.delete(user);
-		}
-	}
-
+	/** Clear all player stats for specific instance */
 	clearStatsOfInstance(instanceId: number) {
-		for (const user of this.users.values()) {
-			this.notifyLeave(user, instanceId);
-			user.instanceStats.delete(instanceId);
-			user.recalculatePlayerStats();
+		for (const user of this.values()) {
+			user.clearInstanceStats(instanceId);
 		}
-		this.dirty = true;
 	}
 }

packages/controller/src/UserRecord.ts

@@ -0,0 +1,102 @@
+import { UserDetails } from "@clusterio/lib";
+import { Type, Static } from "@sinclair/typebox";
+
+/** Underlying data class for users on the controller */
+export default class UserRecord extends UserDetails {
+	constructor(
+		/** Unix time in seconds the user token must be issued after to be valid.  */
+		public tokenValidAfter = 0,
+		...args: ConstructorParameters<typeof UserDetails>
+	) {
+		super(...args);
+	}
+
+	static jsonSchema = Type.Object({
+		token_valid_after: Type.Optional(Type.Number()),
+		...UserDetails.jsonSchema.properties,
+	});
+
+	// _a and _b are a ts workaround for allowing User to have extra arguments
+	static fromJSON(json: Static<typeof this.jsonSchema>, _a?: any, _b?: any) {
+		return this.fromUserDetails(
+			UserDetails.fromJSON(json),
+			json.token_valid_after,
+		);
+	}
+
+	toJSON() {
+		const json = super.toJSON() as Static<typeof UserRecord.jsonSchema>;
+
+		if (this.tokenValidAfter > 0) {
+			json.token_valid_after = this.tokenValidAfter;
+		}
+
+		return json;
+	}
+
+	static fromUserDetails(
+		userDetails: UserDetails,
+		tokenValidAfter = 0,
+	) {
+		return new this(
+			tokenValidAfter,
+			userDetails.name,
+			userDetails.roleIds,
+			userDetails.instances,
+			userDetails.isAdmin,
+			userDetails.isBanned,
+			userDetails.isWhitelisted,
+			userDetails.banReason,
+			userDetails.updatedAtMs,
+			userDetails.isDeleted,
+			userDetails.instanceStats,
+		);
+	}
+
+	toUserDetails() {
+		return new UserDetails(
+			this.name,
+			this.roleIds,
+			this.instances,
+			this.isAdmin,
+			this.isBanned,
+			this.isWhitelisted,
+			this.banReason,
+			this.updatedAtMs,
+			this.isDeleted,
+			this.instanceStats,
+		);
+	}
+
+	/**
+	 * Merge another user into this one, must have matching ids, user is not deleted
+	 * @param otherUser - User who's details are merged from
+	*/
+	merge(otherUser: UserDetails) {
+		if (this.id !== otherUser.id) {
+			throw new Error("Cannot merge users with different ids");
+		}
+
+		// Merge properties
+		this.roleIds = new Set([...this.roleIds, ...otherUser.roleIds]);
+		this.instances = new Set([...this.instances, ...otherUser.instances]);
+		this.isAdmin = this.isAdmin && otherUser.isAdmin; // More secure to use && rather ||
+		this.isBanned = this.isBanned || otherUser.isBanned;
+		this.isWhitelisted = this.isWhitelisted || otherUser.isWhitelisted;
+		this.banReason = this.updatedAtMs > otherUser.updatedAtMs ? this.banReason : otherUser.banReason;
+		this.updatedAtMs = Date.now();
+
+		// Merge instance stats
+		const thisInstanceStats = this.instanceStats;
+		for (const [instanceId, instanceStats] of otherUser.instanceStats.entries()) {
+			if (thisInstanceStats.has(instanceId)) {
+				thisInstanceStats.get(instanceId)!.merge(instanceStats);
+			} else {
+				thisInstanceStats.set(instanceId, instanceStats);
+			}
+		}
+
+		// Player stats must be recalculated after modifying instance stats
+		this.recalculatePlayerStats();
+	}
+}

packages/controller/src/WsServer.ts

@@ -223,10 +223,7 @@ ${err.stack}`
 	createSession(dst: lib.Address): [ WsServerConnector, string ] {
 		let sessionId = this.nextSessionId;
 		this.nextSessionId += 1;
-		let sessionToken = jwt.sign(
-			{ aud: this.sessionAud, sid: sessionId },
-			Buffer.from(this.controller.config.get("controller.auth_secret"), "base64"),
-		);
+		let sessionToken = jwt.sign({ aud: this.sessionAud, sid: sessionId }, this.controller.authSecret);
 		let sessionTimeout = this.controller.config.get("controller.session_timeout");
 		let heartbeatInterval = this.controller.config.get("controller.heartbeat_interval");
 		let connector = new WsServerConnector(dst, sessionId, sessionTimeout, heartbeatInterval);
@@ -247,13 +244,8 @@ ${err.stack}`
 		}
 
 		try {
-			let tokenPayload = jwt.verify(
-				data.token,
-				Buffer.from(this.controller.config.get("controller.auth_secret"), "base64"),
-				// migrate: allow pre-rename tokens issued to hosts before alpha-14
-				{ audience: ["host", "slave"] }
-			);
-
+			// migrate: allow pre-rename tokens issued to hosts before alpha-14
+			const tokenPayload = jwt.verify(data.token, this.controller.authSecret, { audience: ["host", "slave"] });
 			if (typeof tokenPayload === "string") {
 				throw new Error("unexpected JsonWebToken type");
 			}
@@ -324,17 +316,12 @@ ${err.stack}`
 	async registerControl(data: lib.RegisterControlData, socket: WebSocket, req: IncomingMessage) {
 		let user;
 		try {
-			let tokenPayload = jwt.verify(
-				data.token,
-				Buffer.from(this.controller.config.get("controller.auth_secret"), "base64"),
-				{ audience: "user" }
-			);
-
+			const tokenPayload = jwt.verify(data.token, this.controller.authSecret, { audience: "user" });
 			if (typeof tokenPayload === "string") {
 				throw new Error("unexpected JsonWebToken type");
 			}
 
-			user = this.controller.userManager.getByName(tokenPayload.user);
+			user = this.controller.users.getByNameMutable(tokenPayload.user);
 			if (!user) {
 				throw new Error("invalid user");
 			}
@@ -376,12 +363,7 @@ ${err.stack}`
 	resumeSession(data: lib.ResumeData, socket: WebSocket, req: IncomingMessage) {
 		let connector;
 		try {
-			let payload = jwt.verify(
-				data.sessionToken,
-				Buffer.from(this.controller.config.get("controller.auth_secret"), "base64"),
-				{ audience: this.sessionAud }
-			);
-
+			const payload = jwt.verify(data.sessionToken, this.controller.authSecret, { audience: this.sessionAud });
 			if (typeof payload === "string") {
 				throw new Error("unexpected JsonWebToken type");
 			}

packages/controller/src/routes.ts

@@ -1,10 +1,10 @@
 import type { Application, Request, Response } from "express";
-import type Controller from "./Controller";
+import Controller from "./Controller";
 
 import busboy from "busboy";
 import crypto from "crypto";
 import events from "events";
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import JSZip from "jszip";
 import jwt from "jsonwebtoken";
 import path from "path";
@@ -148,12 +148,7 @@ function validateHostToken(req: Request, res: Response, next: any) {
 	}
 
 	try {
-		const tokenPayload = jwt.verify(
-			token,
-			Buffer.from(req.app.locals.controller.config.get("controller.auth_secret"), "base64"),
-			{ audience: "host" }
-		);
-
+		const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "host" });
 		if (typeof tokenPayload === "string") {
 			throw new Error("unexpected JsonWebToken type");
 		}
@@ -180,16 +175,11 @@ function validateUserToken(req: Request, res: Response, next: any) {
 	}
 
 	try {
-		let tokenPayload = jwt.verify(
-			token,
-			Buffer.from(req.app.locals.controller.config.get("controller.auth_secret"), "base64"),
-			{ audience: "user" }
-		);
-
+		const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "user" });
 		if (typeof tokenPayload === "string") {
 			throw new Error("unexpected JsonWebToken type");
 		}
-		let user = req.app.locals.controller.userManager.getByName(tokenPayload.user);
+		let user = req.app.locals.controller.users.getByName(tokenPayload.user);
 		if (!user) {
 			throw new Error("invalid user");
 		}
@@ -240,13 +230,14 @@ async function uploadExport(req: Request, res: Response) {
 	let exportFiles = [
 		"export/settings.json",
 		"export/prototypes.json",
-		"export/item-spritesheet.png",
-		"export/item-metadata.json",
 		"export/locale.json",
+		"export/defines.json",
+		"export/spritesheet.png",
+		"export/metadata.json",
 	];
 
 	let assets: any = {};
-	let settingPrototypes = {};
+	let settingPrototypes: lib.ExportSettings = {};
 	for (let filePath of exportFiles) {
 		let file = zip.file(filePath);
 		if (!file) {
@@ -263,7 +254,7 @@ async function uploadExport(req: Request, res: Response) {
 		await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
 	}
 
-	modPack.exportManifest = new lib.ExportManifest(assets);
+	modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString());
 	modPack.fillDefaultSettings(settingPrototypes, logger);
 	res.app.locals.controller.modPacks.set(modPack);
 
@@ -552,30 +543,16 @@ async function uploadMod(req: Request, res: Response) {
 		}
 
 		const modsDirectory = req.app.locals.controller.config.get("controller.mods_directory");
-		let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
+		const filePath = path.join(modsDirectory, filename);
+		const [tempFilePath, writeStream] = await lib.createTempWriteStream(filePath);
 
-		let writeStream;
 		try {
-
-			while (true) {
-				try {
-					writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" });
-					await events.once(writeStream, "open");
-					break;
-				} catch (err: any) {
-					if (err.code === "EEXIST") {
-						tempFilename = await lib.findUnusedName(modsDirectory, tempFilename, ".tmp.zip");
-					} else {
-						throw err;
-					}
-				}
-			}
 			stream.pipe(writeStream);
 			await finished(writeStream);
 
-			const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
+			const modInfo = await lib.ModInfo.fromModFile(tempFilePath);
 			checkModName(modInfo.filename);
-			await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
+			await fs.rename(tempFilePath, path.join(modsDirectory, modInfo.filename));
 			req.app.locals.controller.modStore.addMod(modInfo);
 			mods.push(modInfo.toJSON());
 
@@ -585,15 +562,13 @@ async function uploadMod(req: Request, res: Response) {
 			stream.resume();
 
 			// Attempt to clean up.
-			if (writeStream) {
-				writeStream.destroy();
-			}
+			writeStream.destroy();
 
 			try {
-				await fs.unlink(path.join(modsDirectory, tempFilename));
+				await fs.unlink(tempFilePath);
 			} catch (unlinkErr: any) {
 				if (unlinkErr.code !== "ENOENT") {
-					logger.error(`Error removing ${tempFilename}: ${unlinkErr.message}`);
+					logger.error(`Error removing ${tempFilePath}: ${unlinkErr.message}`);
 				}
 			}
 		}
@@ -695,6 +670,7 @@ export const webRoutes = [
 	"/instances/:id/view",
 	"/mods",
 	"/mods/mod-packs/:id/view",
+	"/mods/mod-packs/:id/icon-reference",
 	"/users",
 	"/users/:id/view",
 	"/roles",

packages/create/create.js

@@ -2,7 +2,7 @@
 
 "use strict";
 const child_process = require("child_process");
-const fs = require("fs-extra");
+const fs = require("node:fs/promises");
 const inquirer = require("inquirer");
 const os = require("os");
 const path = require("path");
@@ -97,11 +97,24 @@ class LineSplitter extends stream.Transform {
 
 class InstallError extends Error { }
 
+async function pathExists(filePath) {
+	try {
+		await fs.access(filePath);
+	} catch (err) {
+		if (err.code === "ENOENT" || err.code === "ENOTDIR") {
+			return false;
+		}
+	}
+	return true;
+}
 
 async function safeOutputFile(file, data, options={}) {
 	let { dir, name, ext } = path.parse(file);
 	let temporary = path.join(dir, `${name}.tmp${ext}`);
-	await fs.outputFile(temporary, data, options);
+	if (dir) {
+		await fs.mkdir(dir, { recursive: true });
+	}
+	await fs.writeFile(temporary, data, { ...options, flush: true });
 	await fs.rename(temporary, file);
 }
 
@@ -221,7 +234,7 @@ async function migrateRename(args) {
 	}
 
 	async function migrateConfig(source, destination) {
-		if (await fs.pathExists(source) && !await fs.pathExists(destination)) {
+		if (await pathExists(source) && !await pathExists(destination)) {
 			await safeOutputFile(destination, JSON.stringify(
 				renameConfig(JSON.parse(await fs.readFile(source))), null, "\t"
 			));
@@ -233,7 +246,7 @@ async function migrateRename(args) {
 	await migrateConfig("config-slave.json", "config-host.json");
 
 	let instancesFile = path.join("database", "instances.json");
-	if (await fs.pathExists(instancesFile)) {
+	if (await pathExists(instancesFile)) {
 		await safeOutputFile(instancesFile, JSON.stringify(
 			JSON.parse(await fs.readFile(instancesFile)).map(renameConfig), null, "\t"
 		));
@@ -241,7 +254,7 @@ async function migrateRename(args) {
 	}
 
 	let usersFile = path.join("database", "users.json");
-	if (await fs.pathExists(usersFile)) {
+	if (await pathExists(usersFile)) {
 		let users = JSON.parse(await fs.readFile(usersFile));
 		for (let role of users["roles"]) {
 			role["permissions"] = role["permissions"].map(rename);
@@ -258,7 +271,7 @@ async function migrateRename(args) {
 
 	async function migrateLog(inputFile, outputFile) {
 		let lineStream = new LineSplitter({ readableObjectMode: true });
-		let fileStream = fs.createReadStream(inputFile);
+		let fileStream = (await fs.open(inputFile)).createReadStream();
 		fileStream.pipe(lineStream);
 		let lines = [];
 
@@ -278,7 +291,7 @@ async function migrateRename(args) {
 	}
 
 	async function migrateLogsDir(inputLogs, outputLogs) {
-		if (await fs.pathExists(inputLogs)) {
+		if (await pathExists(inputLogs)) {
 			logger.info(`Migrating ${inputLogs} to ${outputLogs}`);
 			for (let logFile of await fs.readdir(inputLogs)) {
 				if (!logFile.endsWith(".log")) {
@@ -286,7 +299,7 @@ async function migrateRename(args) {
 				}
 				let inputFile = path.join(inputLogs, logFile);
 				let outputFile = path.join(outputLogs, rename(logFile));
-				if (!await fs.pathExists(outputFile)) {
+				if (!await pathExists(outputFile)) {
 					await migrateLog(inputFile, outputFile);
 					logger.verbose(`Migrated ${inputFile} to ${outputFile}`);
 				}
@@ -297,16 +310,16 @@ async function migrateRename(args) {
 	await migrateLogsDir(path.join("logs", "master"), path.join("logs", "controller"));
 	await migrateLogsDir(path.join("logs", "slave"), path.join("logs", "host"));
 	if (
-		await fs.pathExists(path.join("logs", "cluster"))
-		&& !await fs.pathExists(path.join("logs", "cluster-prerename"))
+		await pathExists(path.join("logs", "cluster"))
+		&& !await pathExists(path.join("logs", "cluster-prerename"))
 	) {
 		await fs.rename(path.join("logs", "cluster"), path.join("logs", "cluster-prerename"));
 		await migrateLogsDir(path.join("logs", "cluster-prerename"), path.join("logs", "cluster"));
 	}
 
 	if (
-		await fs.pathExists("sharedMods")
-		&& !await fs.pathExists("mods")
+		await pathExists("sharedMods")
+		&& !await pathExists("mods")
 	) {
 		logger.info("Moving sharedMods/ to mods/");
 		await fs.rename("sharedMods", "mods");
@@ -328,14 +341,14 @@ async function migrateRename(args) {
 		await execFile(`npm${scriptExt}`, ["update"]);
 	}
 
-	const hasRunMaster = await fs.pathExists("run-master.sh") || await fs.pathExists("run-master.cmd");
-	const hasRunSlave = await fs.pathExists("run-slave.sh") || await fs.pathExists("run-slave.cmd");
+	const hasRunMaster = await pathExists("run-master.sh") || await pathExists("run-master.cmd");
+	const hasRunSlave = await pathExists("run-slave.sh") || await pathExists("run-slave.cmd");
 	if (hasRunMaster || hasRunSlave) {
 		logger.info("Writing run scripts");
 		let mode = "standalone";
 		if (!hasRunSlave) { mode = "controller"; }
 		if (!hasRunMaster) { mode = "host"; }
-		await writeScripts(mode);
+		await writeScripts(mode, "");
 	}
 
 	logger.info(
@@ -354,7 +367,7 @@ async function migrateRename(args) {
 
 async function installClusterio(mode, plugins) {
 	try {
-		await fs.outputFile("package.json", JSON.stringify({
+		await fs.writeFile("package.json", JSON.stringify({
 			name: "clusterio-install",
 			private: true,
 		}, null, 2), { flag: "wx" });
@@ -420,7 +433,7 @@ async function groupIdToName(gid) {
 	}
 }
 
-async function writeScripts(mode) {
+async function writeScripts(mode, selfSignedCert) {
 	if (["standalone", "controller"].includes(mode)) {
 		if (process.platform === "win32") {
 			await safeOutputFile(
@@ -474,12 +487,13 @@ WantedBy=multi-user.target
 
 	if (["standalone", "host"].includes(mode)) {
 		if (process.platform === "win32") {
+			let extraCaCertsLine = selfSignedCert ? `set "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
 			await safeOutputFile(
 				"run-host.cmd",
 				`\
 @echo off
 set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%"
-:restart
+${extraCaCertsLine}:restart
 call .\\node_modules\\.bin\\clusteriohost.cmd run --can-restart
 if %errorlevel% equ 0 exit /b
 if %errorlevel% equ 8 exit /b
@@ -487,18 +501,20 @@ goto restart
 `
 			);
 		} else {
+			let extraCaCertsLine = selfSignedCert ? `export "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
 			await safeOutputFile(
 				"run-host.sh",
 				`\
 #!/bin/bash
 export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS"
-while true; do
+${extraCaCertsLine}while true; do
 	./node_modules/.bin/clusteriohost run --can-restart
 	if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi
 done
 `,
 				{ mode: 0o755 },
 			);
+			extraCaCertsLine = selfSignedCert ? `Environment=NODE_EXTRA_CA_CERTS=${selfSignedCert}\n` : "";
 			await safeOutputFile(
 				"systemd/clusteriohost.service",
 				`\
@@ -512,7 +528,7 @@ WorkingDirectory=${process.cwd()}
 KillMode=mixed
 KillSignal=SIGINT
 Environment=NODE_OPTIONS=--enable-source-maps
-ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart
+${extraCaCertsLine}ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart
 Restart=on-failure
 RestartPreventExitStatus=8
 
@@ -640,7 +656,7 @@ async function inquirerMissingArgs(args) {
 		let locations = factorioLocations[process.platform] || [];
 		let foundLocations = [];
 		for (let location of locations) {
-			if (await fs.pathExists(path.join(location, "data", "changelog.txt"))) {
+			if (await pathExists(path.join(location, "data", "changelog.txt"))) {
 				foundLocations.push(location);
 			}
 		}
@@ -684,7 +700,7 @@ async function inquirerMissingArgs(args) {
 				},
 			], answers);
 		} else if (answers.factorioDir === "factorio") {
-			await fs.ensureDir("factorio");
+			await fs.mkdir("factorio", { recursive: true });
 		}
 	}
 
@@ -769,20 +785,21 @@ async function downloadLinuxServer() {
 	const factorioDir = `factorio/${version}/`;
 	const tmpFactorioDir = tmpDir + version;
 
-	if (await fs.pathExists(factorioDir)) {
+	if (await pathExists(factorioDir)) {
 		logger.warn(`setting downloadDir to ${factorioDir}, but not downloading because already existing`);
 	} else {
-		await fs.emptyDir(tmpDir);
+		await fs.rm(tmpDir, { force: true, recursive: true, maxRetries: 10 });
+		await fs.mkdir(tmpDir, { recursive: true });
 
 		logger.info("Downloading latest Factorio server release. This may take a while.");
-		const writeStream = fs.createWriteStream(tmpArchivePath);
+		const writeStream = (await fs.open(tmpArchivePath, "w")).createWriteStream();
 		stream.Readable.fromWeb(res.body).pipe(writeStream);
 
 		await finished(writeStream);
 
 		await fs.rename(tmpArchivePath, archivePath);
 		try {
-			await fs.ensureDir(tmpFactorioDir);
+			await fs.mkdir(tmpFactorioDir, { recursive: true });
 			await execFile("tar", [
 				"xf", archivePath, "-C", tmpFactorioDir, "--strip-components", "1",
 			]);
@@ -791,7 +808,7 @@ async function downloadLinuxServer() {
 			throw e;
 		}
 
-		await fs.ensureDir("factorio");
+		await fs.mkdir("factorio", { recursive: true });
 		await fs.rename(tmpFactorioDir, factorioDir);
 		await fs.unlink(archivePath);
 		await fs.rm(tmpDir);
@@ -839,6 +856,9 @@ async function main() {
 		.option("factorio-dir", {
 			nargs: 1, describe: "Path to Factorio installation [standalone/host]", type: "string",
 		})
+		.option("self-signed-cert", {
+			nargs: 1, describe: "Path to self-signed certificate of the controller [standalone/host]", type: "string",
+		})
 		.option("remote-npm", {
 			nargs: 0, description: "Allow remote updates via npm [standalone/controller/host]", type: "boolean",
 		})
@@ -947,7 +967,7 @@ async function main() {
 
 	if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) {
 		logger.info("Writing run scripts");
-		await writeScripts(answers.mode);
+		await writeScripts(answers.mode, args.selfSignedCert);
 	}
 
 	if (answers.mode === "ctl") {

packages/create/package.json

@@ -3,7 +3,7 @@
 	"description": "Installer for clusterio",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.24",
 	"main": "create.js",
 	"bin": {
 		"clusteriocreate": "./create.js"
@@ -17,7 +17,6 @@
 	},
 	"dependencies": {
 		"chalk": "catalog:",
-		"fs-extra": "catalog:",
 		"inquirer": "^8.2.4",
 		"yargs": "^17.7.2"
 	},

packages/create/template.js

@@ -1,6 +1,6 @@
 "use strict";
 const { logger } = require("./logging");
-const fs = require("fs-extra");
+const fs = require("node:fs/promises");
 const path = require("path");
 
 const _package = require("./package.json");
@@ -61,9 +61,15 @@ async function copyTemplateFile(src, dst, properties) {
 			return include ? content : "";
 		});
 
+	// Create folder for output file
+	const dir = path.parse(dst);
+	if (dir) {
+		await fs.mkdir(dir, { recursive: true });
+	}
+
 	// Attempt to write the output file, warn if it already exists
 	try {
-		await fs.outputFile(dst, outputData, { flag: "wx" });
+		await fs.writeFile(dst, outputData, { flag: "wx" });
 	} catch (err) {
 		if (err.code === "EEXIST") {
 			logger.warn(`Could not create file ${dst} because it already exists`);

packages/create/templates/plugin-ts/controller.ts

@@ -1,5 +1,5 @@
 import * as lib from "@clusterio/lib";
-import { BaseControllerPlugin, InstanceInfo } from "@clusterio/controller";
+import { BaseControllerPlugin, InstanceRecord } from "@clusterio/controller";
 //%if multi_context // Messages requires multi context
 
 import {
@@ -37,7 +37,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	}
 //%if instance
 
-	async onInstanceConfigFieldChanged(instance: InstanceInfo, field: string, curr: unknown, prev: unknown) {
+	async onInstanceConfigFieldChanged(instance: InstanceRecord, field: string, curr: unknown, prev: unknown) {
 		this.logger.info(`controller::onInstanceConfigFieldChanged ${instance.id} ${field}`);
 	}
 //%endif
@@ -50,7 +50,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.logger.info("controller::onShutdown");
 	}
 
-	async onPlayerEvent(instance: InstanceInfo, event: lib.PlayerEvent) {
+	async onPlayerEvent(instance: InstanceRecord, event: lib.PlayerEvent) {
 		this.logger.info(`controller::onPlayerEvent ${instance.id} ${JSON.stringify(event)}`);
 	}
 //%if multi_context

packages/ctl/ctl.ts

@@ -4,7 +4,7 @@
  * Command line interface for controlling a Clusterio cluster
  * @module ctl/ctl
  */
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import yargs, { type Argv } from "yargs";
 import path from "path";
 import { version } from "./package.json";
@@ -27,8 +27,8 @@ import BaseCtlPlugin from "./src/BaseCtlPlugin";
 class ControlConnector extends lib.WebSocketClientConnector {
 	private _token: string;
 
-	constructor(url: string, maxReconnectDelay: number, tlsCa: string | undefined, token: string) {
-		super(url, maxReconnectDelay, tlsCa);
+	constructor(url: string, maxReconnectDelay: number, token: string) {
+		super(url, maxReconnectDelay);
 		this._token = token;
 	}
 
@@ -54,8 +54,6 @@ class ControlConnector extends lib.WebSocketClientConnector {
 export class Control extends lib.Link {
 	/** Control config used for connecting to the controller. */
 	config: lib.ControlConfig;
-	/** Certificate authority used to validate TLS connections to the controller. */
-	tlsCa?: string;
 	/** Mapping of plugin names to their instance for loaded plugins. */
 	plugins: Map<string, BaseCtlPlugin>;
 	/** Keep the control connection alive after the command completes. */
@@ -64,12 +62,10 @@ export class Control extends lib.Link {
 	constructor(
 		connector: ControlConnector,
 		controlConfig: lib.ControlConfig,
-		tlsCa: string | undefined,
 		ctlPlugins: Map<string, BaseCtlPlugin>
 	) {
 		super(connector);
 		this.config = controlConfig;
-		this.tlsCa = tlsCa;
 		this.plugins = ctlPlugins;
 
 		this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this));
@@ -299,20 +295,13 @@ async function startControl() {
 		return;
 	}
 
-	let tlsCa: string | undefined;
-	let tlsCaPath = controlConfig.get("control.tls_ca");
-	if (tlsCaPath) {
-		tlsCa = await fs.readFile(tlsCaPath, "utf8");
-	}
-
 	let controlConnector = new ControlConnector(
 		controlConfig.get("control.controller_url")!,
 		controlConfig.get("control.max_reconnect_delay"),
-		tlsCa,
 		controlConfig.get("control.controller_token")!,
 	);
 
-	let control = new Control(controlConnector, controlConfig, tlsCa, ctlPlugins);
+	let control = new Control(controlConnector, controlConfig, ctlPlugins);
 	try {
 		await controlConnector.connect();
 	} catch (err) {

packages/ctl/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio command line management tool",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22",
+	"version": "2.0.0-alpha.24",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -21,18 +21,15 @@
 	},
 	"dependencies": {
 		"@clusterio/lib": "workspace:*",
-		"set-blocking": "catalog:",
-		"fs-extra": "catalog:",
-		"winston": "catalog:",
-		"phin": "catalog:",
 		"as-table": "^1.0.55",
+		"set-blocking": "catalog:",
+		"winston": "catalog:",
 		"yargs": "^17.7.2"
 	},
 	"publishConfig": {
 		"access": "public"
 	},
 	"devDependencies": {
-		"@types/fs-extra": "catalog:",
 		"@types/node": "catalog:",
 		"@types/set-blocking": "catalog:",
 		"@types/yargs": "^17.0.33",

packages/ctl/src/command_ops.ts

@@ -0,0 +1,4 @@
+export function print(...content: any) {
+	// eslint-disable-next-line no-console
+	console.log(...content);
+}

packages/ctl/src/commands_controller.ts

@@ -0,0 +1,201 @@
+import fs from "node:fs/promises";
+import asTableModule from "as-table";
+import events from "events";
+import os from "os";
+import child_process from "child_process";
+
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const controllerCommands = new lib.CommandTree({ name: "controller", description: "Controller management" });
+controllerCommands.add(new lib.Command({
+	definition: ["stop", "Stop controller"],
+	handler: async function(_args: object, control: Control) {
+		await control.send(new lib.ControllerStopRequest());
+	},
+}));
+
+controllerCommands.add(new lib.Command({
+	definition: ["restart", "Restart controller"],
+	handler: async function(_args: object, control: Control) {
+		await control.send(new lib.ControllerRestartRequest());
+	},
+}));
+controllerCommands.add(new lib.Command({
+	definition: ["update", "Update the controller", (yargs) => {
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { restart: boolean }, control: Control) {
+		await control.send(new lib.ControllerUpdateRequest());
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Controller updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+
+const controllerConfigCommands = new lib.CommandTree({
+	name: "config", alias: ["c"], description: "controller config management",
+});
+
+controllerConfigCommands.add(new lib.Command({
+	definition: ["list", "List controller configuration"],
+	handler: async function(args: object, control: Control) {
+		let config = await control.send(new lib.ControllerConfigGetRequest());
+
+		for (let [name, value] of Object.entries(config)) {
+			print(`${name} ${JSON.stringify(value)}`);
+		}
+	},
+}));
+
+controllerConfigCommands.add(new lib.Command({
+	definition: ["set <field> [value]", "Set field in controller config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args: { field: string, value?: string, stdin?: boolean }, control: Control) {
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await control.send(new lib.ControllerConfigSetFieldRequest(args.field, args.value));
+	},
+}));
+
+controllerConfigCommands.add(new lib.Command({
+	definition: ["set-prop <field> <prop> [value]", "Set property of field in controller config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args: { field: string, prop: string, value?: string, stdin?: boolean }, control: Control) {
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let value;
+		try {
+			if (args.value !== undefined) {
+				value = JSON.parse(args.value);
+			}
+		} catch (err: any) {
+			// See note for the instance version of set-prop
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
+				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			value = args.value;
+		}
+		await control.send(new lib.ControllerConfigSetPropRequest(args.field, args.prop, value));
+	},
+}));
+controllerConfigCommands.add(new lib.Command({
+	definition: ["edit [editor]", "Edit controller configuration", (yargs) => {
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args: { editor: string }, control: Control) {
+		let config = await control.send(new lib.ControllerConfigGetRequest());
+		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === undefined) {
+			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl controller config edit <editor of choice>"`);
+		}
+		let allConfigElements = serializedConfigToString(
+			config, lib.ControllerConfig, {}
+		);
+		await fs.writeFile(tmpFile, allConfigElements);
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+			process.stdout.pipe(data);
+		});
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.ControllerConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					print(err);
+					// If the string is empty, it's better to just print "" instead of nothing
+				}
+			}
+		}
+		try {
+			await fs.unlink(tmpFile);
+		} catch (err) {
+			print("err: temporary file", tmpFile, "could not be deleted.");
+			print("This is not fatal, but they may build up over time if the issue persists.");
+		}
+	},
+}));
+
+controllerCommands.add(controllerConfigCommands);
+
+
+const controllerPluginCommands = new lib.CommandTree({
+	name: "plugin", alias: ["p"], description: "controller plugin inspection",
+});
+controllerPluginCommands.add(new lib.Command({
+	definition: ["list", "List plugins on controller"],
+	handler: async function(args: object, control: Control) {
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += "api/plugins";
+		let response = await fetch(url);
+		print(asTable(await response.json() as []));
+	},
+}));
+controllerPluginCommands.add(new lib.Command({
+	definition: ["update <plugin>", "Update a plugin on the controller", (yargs) => {
+		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
+		await control.sendTo("controller", new lib.PluginUpdateRequest(args.plugin));
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Plugin updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+controllerPluginCommands.add(new lib.Command({
+	definition: ["install <plugin>", "Install a plugin on the controller", (yargs) => {
+		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { plugin: string, restart: boolean }, control: Control) {
+		await control.sendTo("controller", new lib.PluginInstallRequest(args.plugin));
+		if (args.restart) {
+			await control.send(new lib.ControllerRestartRequest());
+		} else {
+			print("Plugin installed; a restart is required to apply the changes.");
+		}
+	},
+}));
+
+controllerCommands.add(controllerPluginCommands);

packages/ctl/src/commands_debug.ts

@@ -0,0 +1,11 @@
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+
+export const debugCommands = new lib.CommandTree({ name: "debug", description: "Debugging utilities" });
+debugCommands.add(new lib.Command({
+	definition: ["dump-ws", "Dump WebSocket messages sent and received by controller"],
+	handler: async function(args: object, control: Control) {
+		await control.send(new lib.DebugDumpWsRequest());
+		control.keepOpen = true;
+	},
+}));

packages/ctl/src/commands_host.ts

@@ -0,0 +1,305 @@
+import fs from "node:fs/promises";
+import asTableModule from "as-table";
+import events from "events";
+import os from "os";
+import child_process from "child_process";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const hostCommands = new lib.CommandTree({ name: "host", description: "Host management" });
+hostCommands.add(new lib.Command({
+	definition: ["stop <host>", "Stop the given host", (yargs) => {
+		yargs.positional("host", { describe: "Host to stop", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostStopRequest());
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["restart <host>", "Restart the given host", (yargs) => {
+		yargs.positional("host", { describe: "Host to restart", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostRestartRequest());
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: [["list", "l"], "List hosts connected to the controller"],
+	handler: async function(args: object, control: Control) {
+		let hosts = await control.send(new lib.HostListRequest());
+		print(asTable(hosts));
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["generate-token", "Generate token for a host", (yargs) => {
+		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
+	}],
+	handler: async function(args: { id?: number }, control: Control) {
+		let hostId = typeof args.id === "number" ? args.id : undefined;
+		let response = await control.send(new lib.HostGenerateTokenRequest(hostId));
+		print(response);
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["revoke-token <host>", "Revoke all tokens for a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to revoke tokens for", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.send(new lib.HostRevokeTokensRequest(hostId));
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["create-config", "Create host config", (yargs) => {
+		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
+		yargs.option("name", { type: "string", nargs: 1, describe: "Host name" });
+		yargs.option("generate-token", {
+			type: "boolean", nargs: 0, describe: "Generate authentication token", default: false,
+		});
+		yargs.option("output", {
+			type: "string", nargs: 1, describe: "Path to output config (- for stdout)", default: "config-host.json",
+		});
+	}],
+	handler: async function(
+		args: { id?: number, name?: string, generateToken: boolean, output: string },
+		control: Control
+	) {
+		let config = await control.send(
+			new lib.HostConfigCreateRequest(args.id, args.name, args.generateToken)
+		);
+
+		let content = JSON.stringify(config, null, "\t");
+		if (args.output === "-") {
+			print(content);
+		} else {
+			logger.info(`Writing ${args.output}`);
+			try {
+				await fs.writeFile(args.output, content, { flag: "wx" });
+			} catch (err: any) {
+				if (err.code === "EEXIST") {
+					throw new lib.CommandError(`File ${args.output} already exists`);
+				}
+				throw err;
+			}
+		}
+	},
+}));
+hostCommands.add(new lib.Command({
+	definition: ["update <host>", "Update the host", (yargs) => {
+		yargs.positional("host", { describe: "Host to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostUpdateRequest());
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Host updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+
+
+const hostConfigCommands = new lib.CommandTree({
+	name: "config", alias: ["c"], description: "Host config management",
+});
+hostConfigCommands.add(new lib.Command({
+	definition: ["list <host>", "List configuration for a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to list config for", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		let config = await control.sendTo({ hostId }, new lib.HostConfigGetRequest());
+
+		for (let [name, value] of Object.entries(config)) {
+			print(`${name} ${JSON.stringify(value)}`);
+		}
+	},
+}));
+
+hostConfigCommands.add(new lib.Command({
+	definition: ["set <host> <field> [value]", "Set field in host config", (yargs) => {
+		yargs.positional("host", { describe: "Host to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { host: string, field: string, value?: string, stdin?: boolean },
+		control: Control,
+	) {
+		let hostId = await lib.resolveHost(control, args.host);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(args.field, args.value));
+	},
+}));
+
+hostConfigCommands.add(new lib.Command({
+	definition: ["set-prop <host> <field> <prop> [value]", "Set property of field in host config", (yargs) => {
+		yargs.positional("host", { describe: "Host to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { host: string, field: string, prop: string, value?: string, stdin?: boolean },
+		control: Control
+	) {
+		let hostId = await lib.resolveHost(control, args.host);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let value;
+		try {
+			if (args.value !== undefined) {
+				value = JSON.parse(args.value);
+			}
+		} catch (err: any) {
+			// See note for the instance version of set-prop
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
+				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			value = args.value;
+		}
+		await control.sendTo({ hostId }, new lib.HostConfigSetPropRequest(args.field, args.prop, value));
+	},
+}));
+
+hostConfigCommands.add(new lib.Command({
+	definition: ["edit <host> [editor]", "Edit host configuration", (yargs) => {
+		yargs.positional("host", { describe: "Host to set config on", type: "string" });
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args: { host: string, editor: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		let config = await control.sendTo({ hostId }, new lib.HostConfigGetRequest());
+		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === undefined) {
+			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl controller config edit <editor of choice>"`);
+		}
+		let disallowedList = {"host.id": 0};
+		let allConfigElements = serializedConfigToString(
+			config,
+			lib.HostConfig,
+			disallowedList
+		);
+		await fs.writeFile(tmpFile, allConfigElements);
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+			process.stdout.pipe(data);
+		});
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
+				}
+			}
+		}
+		try {
+			await fs.unlink(tmpFile);
+		} catch (err) {
+			print("err: temporary file", tmpFile, "could not be deleted.");
+			print("This is not fatal, but they may build up over time if the issue persists.");
+		}
+	},
+}));
+
+hostCommands.add(hostConfigCommands);
+
+
+const hostPluginCommands = new lib.CommandTree({
+	name: "plugin", alias: ["p"], description: "host plugin inspection",
+});
+hostPluginCommands.add(new lib.Command({
+	definition: ["list <host>", "List plugins on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to list for", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		const plugins = await control.sendTo({ hostId }, new lib.PluginListRequest());
+		print(asTable(plugins.map(p => ({
+			title: p.title, version: p.version, loaded: p.loaded, enabled: p.enabled, npmPackage: p.npmPackage,
+		}))));
+	},
+}));
+hostPluginCommands.add(new lib.Command({
+	definition: ["update <host> <plugin>", "Update a plugin on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to update on", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.PluginUpdateRequest(args.plugin));
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Plugin updated; a restart is required to apply the changes.");
+		}
+	},
+}));
+hostPluginCommands.add(new lib.Command({
+	definition: ["install <host> <plugin>", "Install a plugin on a host", (yargs) => {
+		yargs.positional("host", { describe: "Host to install on", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
+	}],
+	handler: async function(args: { host: string, plugin: string, restart: boolean }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.PluginInstallRequest(args.plugin));
+		if (args.restart) {
+			await control.sendTo({ hostId }, new lib.HostRestartRequest());
+		} else {
+			print("Plugin installed; a restart is required to apply the changes.");
+		}
+	},
+}));
+hostCommands.add(hostPluginCommands);

packages/ctl/src/commands_instance.ts

@@ -0,0 +1,661 @@
+import fs from "node:fs/promises";
+import asTableModule from "as-table";
+import events from "events";
+import path from "path";
+import os from "os";
+import child_process from "child_process";
+import stream from "stream";
+import util from "util";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+const finished = util.promisify(stream.finished);
+
+export const instanceCommands = new lib.CommandTree({
+	name: "instance", alias: ["i"], description: "Instance management",
+});
+instanceCommands.add(new lib.Command({
+	definition: [["list", "l"], "List instances known to the controller"],
+	handler: async function(args: object, control: Control) {
+		let list = await control.send(new lib.InstanceDetailsListRequest());
+		print(asTable(list));
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["create <name>", "Create an instance", (yargs) => {
+		// XXX TODO: set any specific options?
+		yargs.positional("name", { describe: "Instance name", type: "string" });
+		yargs.options({
+			"id": { type: "number", nargs: 1, describe: "Instance id" },
+			"from": { type: "number", nargs: 1, describe: "Clone config from instance id" },
+		});
+	}],
+	handler: async function(args: { name: string, id?: number, from?: number, }, control: Control) {
+		let instanceConfig = new lib.InstanceConfig("control");
+		if (args.id !== undefined) {
+			instanceConfig.set("instance.id", args.id);
+		}
+		instanceConfig.set("instance.name", args.name);
+		await control.send(new lib.InstanceCreateRequest(
+			instanceConfig.toRemote("controller", [
+				"instance.id", "instance.name",
+			]),
+			args.from
+		));
+	},
+}));
+
+const instanceConfigCommands = new lib.CommandTree({
+	name: "config", alias: ["c"], description: "Instance config management",
+});
+instanceConfigCommands.add(new lib.Command({
+	definition: ["list <instance>", "List configuration for an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to list config for", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let config = await control.send(new lib.InstanceConfigGetRequest(instanceId));
+
+		for (let [name, value] of Object.entries(config)) {
+			print(`${name} ${JSON.stringify(value)}`);
+		}
+	},
+}));
+
+instanceConfigCommands.add(new lib.Command({
+	definition: ["set <instance> <field> [value]", "Set field in instance config", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { instance: string, field: string, value?: string, stdin?: boolean },
+		control: Control,
+	) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await control.send(new lib.InstanceConfigSetFieldRequest(instanceId, args.field, args.value));
+	},
+}));
+
+instanceConfigCommands.add(new lib.Command({
+	definition: ["set-prop <instance> <field> <prop> [value]", "Set property of field in instance config", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: { instance: string, field: string, prop: string, value?: string, stdin?: boolean },
+		control: Control
+	) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		if (args.stdin) {
+			args.value = (await lib.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let value;
+		try {
+			if (args.value !== undefined) {
+				value = JSON.parse(args.value);
+			}
+		} catch (err: any) {
+			// If this is from stdin or looks like an array, object or string
+			// literal throw the parse error, otherwise assume this is a string.
+			// The resoning behind this is that correctly quoting the string
+			// with the all the layers of quote removal at play is difficult.
+			// See the following table for how to pass "That's a \" quote" in
+			// different environments:
+			// ----------------------- WARNING -------------------------------
+			// This table is most likely invalid due a change in yargs parsing
+			// ---------------------------------------------------------------
+			// cmd              : """""That's a \\"" quote"""""
+			// cmd + npx        : """""""""""That's a \\\\"""" quote"""""""""""
+			// PowerShell       : '"""""That''s a \\"" quote"""""'
+			// PowerShell + npx : '"""""""""""That''s a \\\\"""" quote"""""""""""'
+			// bash             : '""That'\''s a \" quote""'
+			// bash + npx       : '""That'\''s a \" quote""'
+			// bash + npx -s sh : "'\"\"That'\\''s a \\\" quote\"\"'"
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value!)) {
+				throw new lib.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			value = args.value;
+		}
+		await control.send(new lib.InstanceConfigSetPropRequest(instanceId, args.field, args.prop, value));
+	},
+}));
+
+instanceConfigCommands.add(new lib.Command({
+	definition: ["edit <instance> [editor]", "Edit instance configuration", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args: { instance: string, editor: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let config = await control.send(new lib.InstanceConfigGetRequest(instanceId));
+		let tmpFile = await lib.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === undefined) {
+			throw new lib.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl controller config edit <editor of choice>"`);
+		}
+		let disallowedList = {"instance.id": 0, "instance.assigned_host": 0, "factorio.settings": 0};
+		let allConfigElements = serializedConfigToString(
+			config,
+			lib.InstanceConfig,
+			disallowedList
+		);
+		await fs.writeFile(tmpFile, allConfigElements);
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+			process.stdout.pipe(data);
+		});
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.InstanceConfigSetFieldRequest(
+						instanceId,
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
+				}
+			}
+		}
+		try {
+			await fs.unlink(tmpFile);
+		} catch (err) {
+			print("err: temporary file", tmpFile, "could not be deleted.");
+			print("This is not fatal, but they may build up over time if the issue persists.");
+		}
+	},
+}));
+instanceCommands.add(instanceConfigCommands);
+
+instanceCommands.add(new lib.Command({
+	definition: ["assign <instance> [host]", "Assign instance to a host", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to assign", type: "string" });
+		yargs.positional("host", { describe: "Host to assign to or unassign if none", type: "string" });
+	}],
+	handler: async function(args: { instance: string, host?: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let hostId = args.host ? await lib.resolveHost(control, args.host) : undefined;
+		await control.send(new lib.InstanceAssignRequest(instanceId, hostId));
+	},
+}));
+
+async function loadMapSettings(
+	args: { seed?: number, mapExchangeString?: string, mapGenSettings?: string, mapSettings?: string},
+) {
+	let seed = args.seed;
+	let mapGenSettings;
+	let mapSettings;
+	if (args.mapExchangeString) {
+		let parsed = lib.readMapExchangeString(args.mapExchangeString);
+		mapGenSettings = parsed.map_gen_settings;
+		mapSettings = parsed.map_settings;
+	}
+	if (args.mapGenSettings) {
+		mapGenSettings = JSON.parse((await fs.readFile(args.mapGenSettings)).toString());
+	}
+	if (args.mapSettings) {
+		mapSettings = JSON.parse((await fs.readFile(args.mapSettings)).toString());
+	}
+
+	return {
+		seed,
+		mapGenSettings,
+		mapSettings,
+	};
+}
+
+const instanceSaveCommands = new lib.CommandTree({
+	name: "save", alias: ["s"], description: "Instance save management",
+});
+instanceSaveCommands.add(new lib.Command({
+	definition: ["list <instance>", "list saves on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to list saves on", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let saves = await control.sendTo("controller", new lib.InstanceSaveDetailsListRequest());
+		saves = saves.filter(save => save.instanceId === instanceId);
+		print(asTable(saves.map(
+			({ mtimeMs, ...rest }) => ({ ...rest, mtime: new Date(mtimeMs).toLocaleString() })
+		)));
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["create <instance> [name]", "Create a new save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to create on", type: "string" });
+		yargs.positional("name", { describe: "Name of save to create.", type: "string", default: "world.zip" });
+		yargs.options({
+			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
+			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
+			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
+			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(
+		args: {
+			instance: string,
+			name: string,
+			seed?: number,
+			mapExchangeString?: string,
+			mapGenSettings?: string,
+			mapSettings?: string,
+		},
+		control: Control
+	) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo(
+			{ instanceId },
+			new lib.InstanceCreateSaveRequest(args.name, seed, mapGenSettings, mapSettings),
+		);
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["rename <instance> <old-name> <new-name>", "Rename a save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to rename save on", type: "string" });
+		yargs.positional("old-name", { describe: "Old name of save.", type: "string" });
+		yargs.positional("new-name", { describe: "New name of save.", type: "string" });
+	}],
+	handler: async function(args: { instance: string, oldName: string, newName: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceRenameSaveRequest(instanceId, args.oldName, args.newName));
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["copy <instance> <source> <destination>", "Copy a save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to copy save on", type: "string" });
+		yargs.positional("source", { describe: "Save to copy.", type: "string" });
+		yargs.positional("destination", { describe: "Name of copy.", type: "string" });
+	}],
+	handler: async function(args: { instance: string, source: string, destination: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceCopySaveRequest(instanceId, args.source, args.destination));
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["upload <instance> <filepath>", "Upload a save to an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to upload to", type: "string" });
+		yargs.positional("filepath", { describe: "Path to save to upload", type: "string" });
+		yargs.options({
+			"name": { describe: "Name to give save on server", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(args: { instance: string, filepath: string, name?: string }, control: Control) {
+		let filename = args.name || path.basename(args.filepath);
+		if (!filename.endsWith(".zip")) {
+			throw new lib.CommandError("Save name must end with .zip");
+		}
+
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += "api/upload-save";
+		url.searchParams.append("instance_id", String(instanceId));
+		url.searchParams.append("filename", filename);
+
+		const response = await fetch(url, {
+			method: "POST",
+			body: (await fs.open(args.filepath)).createReadStream(),
+			duplex: "half",
+			headers: {
+				"X-Access-Token": control.config.get("control.controller_token")!,
+				"Content-Type": "application/zip",
+			},
+		});
+
+		type ResponseType = { errors?: string[], request_errors?: string[], saves?: string[] };
+		const result = await response.json() as ResponseType;
+
+		for (let error of result.errors || []) {
+			logger.error(error);
+		}
+
+		for (let requestError of result.request_errors || []) {
+			logger.error(requestError);
+		}
+
+		if (result.saves && result.saves.length) {
+			logger.info(`Successfully uploaded as ${result.saves[0]}`);
+		}
+
+		if ((result.errors || []).length || (result.request_errors || []).length) {
+			throw new lib.CommandError("Uploading save failed");
+		}
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: [
+		"transfer <source-instance> <source-save> <target-instance> [target-save]",
+		"Transfer a save between instances",
+		(yargs) => {
+			yargs.positional("source-instance", { describe: "Instance to transfer save from", type: "string" });
+			yargs.positional("source-save", { describe: "Save to transfer.", type: "string" });
+			yargs.positional("target-instance", { describe: "Instance to transfer to", type: "string" });
+			yargs.positional("target-save", { describe: "Name to give transferred save.", type: "string" });
+			yargs.options({
+				"copy": { describe: "Copy instead of moving the save", type: "boolean", default: false },
+			});
+		},
+	],
+	handler: async function(
+		args: {
+			sourceInstance: string,
+			sourceSave: string,
+			targetInstance: string,
+			targetSave: string,
+			copy: boolean,
+		},
+		control: Control
+	) {
+		let sourceInstanceId = await lib.resolveInstance(control, args.sourceInstance);
+		let targetInstanceId = await lib.resolveInstance(control, args.targetInstance);
+		let storedName = await control.send(
+			new lib.InstanceTransferSaveRequest(
+				sourceInstanceId,
+				args.sourceSave,
+				targetInstanceId,
+				args.targetSave || args.sourceSave,
+				args.copy
+			)
+		);
+		print(`Transferred as ${storedName} to ${args.targetInstance}.`);
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["download <instance> <save>", "Download a save from an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to download save from", type: "string" });
+		yargs.positional("save", { describe: "Save to download", type: "string" });
+	}],
+	handler: async function(args: { instance: string, save: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		let streamId = await control.send(new lib.InstanceDownloadSaveRequest(instanceId, args.save));
+
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += `api/stream/${streamId}`;
+		const filename = await lib.downloadFile(url, args.save, "rename");
+
+		logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`);
+	},
+}));
+
+instanceSaveCommands.add(new lib.Command({
+	definition: ["delete <instance> <save>", "Delete a save from an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to delete save from", type: "string" });
+		yargs.positional("save", { describe: "Save to delete", type: "string" });
+	}],
+	handler: async function(args: { instance: string, save: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceDeleteSaveRequest(instanceId, args.save));
+	},
+}));
+instanceCommands.add(instanceSaveCommands);
+
+instanceCommands.add(new lib.Command({
+	definition: ["export-data <instance>", "Export item icons and locale from instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to export from", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceExportDataRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["extract-players <instance>", "Extract players from running save into the cluster.", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to extract players and online time from", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceExtractPlayersRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["start <instance>", "Start instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to start", type: "string" });
+		yargs.options({
+			"save": { describe: "Save to load, defaults to latest", nargs: 1, type: "string" },
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { instance: string, save?: string, keepOpen: boolean }, control: Control) {
+		const instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceStartRequest(args.save));
+		control.keepOpen = args.keepOpen;
+	},
+}));
+instanceCommands.add(new lib.Command({
+	definition: ["restart <instance>", "Restart instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to restart", type: "string" });
+		yargs.options({
+			"save": { describe: "Save to load, defaults to latest", nargs: 1, type: "string" },
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { instance: string, save?: string, keepOpen: boolean }, control: Control) {
+		const instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceRestartRequest(args.save));
+		control.keepOpen = args.keepOpen;
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["start-all", "Start all stopped instances (respects exclude_from_start_all setting)", (yargs) => {
+		yargs.options({
+			"force": {
+				describe: "Start instances even if they have exclude_from_start_all set",
+				nargs: 0,
+				type: "boolean",
+				default: false,
+			},
+		});
+	}],
+	handler: async function(args: { force: boolean }, control: Control) {
+		// Get all instances
+		const instances = await control.send(new lib.InstanceDetailsListRequest());
+
+		// Filter instances that should be started
+		const instancesToStart = instances.filter((instance: lib.InstanceDetails) => {
+			// Only start instances that are currently stopped
+			if (instance.status !== "stopped") {
+				return false;
+			}
+
+			// Respect exclude_from_start_all setting unless --force is used
+			if (!args.force && instance.excludeFromStartAll) {
+				return false;
+			}
+
+			return true;
+		});
+
+		if (instancesToStart.length === 0) {
+			print("No instances to start.");
+			return;
+		}
+
+		print(`Starting ${instancesToStart.length} instance(s):`);
+		for (const instance of instancesToStart) {
+			print(`  - ${instance.name} (ID: ${instance.id})`);
+		}
+
+		// Start all filtered instances
+		const startPromises = instancesToStart.map(async (instance: lib.InstanceDetails) => {
+			try {
+				await control.sendTo({ instanceId: instance.id }, new lib.InstanceStartRequest());
+				print(`✓ Started ${instance.name}`);
+			} catch (error) {
+				print(`✗ Failed to start ${instance.name}: ${error}`);
+			}
+		});
+
+		await Promise.all(startPromises);
+		print("Start all operation completed.");
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["load-scenario <instance> <scenario>", "Start instance by loading a scenario", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to start", type: "string" });
+		yargs.positional("scenario", { describe: "Scenario to load", type: "string" });
+		yargs.options({
+			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
+			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
+			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
+			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(
+		args: {
+			instance: string,
+			scenario: string,
+			seed?: number,
+			mapExchangeString?: string,
+			mapGenSettings?: string,
+			mapSettings?: string,
+			keepOpen: boolean,
+		},
+		control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
+		await control.sendTo(
+			{ instanceId },
+			new lib.InstanceLoadScenarioRequest(args.scenario, seed, mapGenSettings, mapSettings),
+		);
+		control.keepOpen = args.keepOpen;
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["stop <instance>", "Stop instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to stop", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceStopRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["stop-all", "Stop all running instances", () => {}],
+	handler: async function(_args, control: Control) {
+		// Get all instances
+		const instances = await control.send(new lib.InstanceDetailsListRequest());
+
+		// Filter instances that should be stopped
+		const instancesToStop = instances
+			.filter((instance: lib.InstanceDetails) => ["starting", "running"].includes(instance.status));
+
+		if (instancesToStop.length === 0) {
+			print("No instances to stop.");
+			return;
+		}
+
+		print(`Stopping ${instancesToStop.length} instance(s):`);
+		for (const instance of instancesToStop) {
+			print(`  - ${instance.name} (ID: ${instance.id})`);
+		}
+
+		// Stop all filtered instances
+		const stopPromises = instancesToStop.map(async (instance: lib.InstanceDetails) => {
+			try {
+				await control.sendTo({ instanceId: instance.id }, new lib.InstanceStopRequest());
+				print(`✓ Stopped ${instance.name}`);
+			} catch (error) {
+				print(`✗ Failed to stop ${instance.name}: ${error}`);
+			}
+		});
+
+		await Promise.all(stopPromises);
+		print("Stop all operation completed.");
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["kill <instance>", "Kill instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to kill", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instanceIds: [instanceId] });
+		await control.sendTo({ instanceId }, new lib.InstanceKillRequest());
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["delete <instance>", "Delete instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to delete", type: "string" });
+	}],
+	handler: async function(args: { instance: string }, control: Control) {
+		let instanceId = await lib.resolveInstance(control, args.instance);
+		await control.send(new lib.InstanceDeleteRequest(instanceId));
+	},
+}));
+
+instanceCommands.add(new lib.Command({
+	definition: ["send-rcon <instance> <command>", "Send RCON command", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to send to", type: "string" });
+		yargs.positional("command", { describe: "command to send", type: "string" });
+	}],
+	handler: async function(args: { instance: string, command: string }, control: Control) {
+		let result = await control.sendTo(
+			{ instanceId: await lib.resolveInstance(control, args.instance) },
+			new lib.InstanceSendRconRequest(args.command),
+		);
+
+		// Factorio includes a newline in its response output.
+		process.stdout.write(result);
+	},
+}));

packages/ctl/src/commands_log.ts

@@ -0,0 +1,90 @@
+import winston from "winston";
+
+import * as lib from "@clusterio/lib";
+import { ConsoleTransport, levels, logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+
+export const logCommands = new lib.CommandTree({ name: "log", description: "Log inspection" });
+logCommands.add(new lib.Command({
+	definition: ["follow", "follow cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Follow the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"controller": { describe: "Follow log of the controller", nargs: 0, type: "boolean", default: false },
+			"host": { describe: "Follow log of given host", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Follow log of given instance", nargs: 1, type: "string", default: null },
+		});
+	}],
+	handler: async function(
+		args: { all: boolean, controller: boolean, host: string | null, instance: string | null },
+		control: Control
+	) {
+		if (!args.all && !args.controller && !args.host && !args.instance) {
+			logger.error("At least one of --all, --controller, --host and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instanceIds = args.instance ? [await lib.resolveInstance(control, args.instance)] : [];
+		let hostIds = args.host ? [await lib.resolveHost(control, args.host)] : [];
+		await control.setLogSubscriptions({ all: args.all, controller: args.controller, hostIds, instanceIds });
+		control.keepOpen = true;
+	},
+}));
+
+logCommands.add(new lib.Command({
+	definition: ["query", "Query cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Query the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"controller": { describe: "Query log of the controller", nargs: 0, type: "boolean", default: false },
+			"host": { describe: "Query log of given host", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Query log of given instance", nargs: 1, type: "string", default: null },
+			"max-level": { describe: "Maximum log level to return", nargs: 1, type: "string", default: undefined },
+			"limit": { describe: "Max number of entries to return", nargs: 1, type: "number", default: 1000 },
+			"start": { describe: "Limit from the start instead of the end", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(
+		args: {
+			all: boolean,
+			controller: boolean,
+			host: string | null,
+			instance: string | null,
+			maxLevel?: string,
+			limit: number,
+			start: boolean,
+		},
+		control: Control
+	) {
+		if (!args.all && !args.controller && !args.host && !args.instance) {
+			logger.error("At least one of --all, --controller, --host and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instanceIds = args.instance ? [await lib.resolveInstance(control, args.instance)] : [];
+		let hostIds = args.host ? [await lib.resolveHost(control, args.host)] : [];
+		let result = await control.send(new lib.LogQueryRequest(
+			args.all,
+			args.controller,
+			hostIds,
+			instanceIds,
+			args.maxLevel as keyof typeof levels,
+			args.limit,
+			args.start ? "asc" : "desc",
+		));
+
+		if (!args.start) {
+			result.log.reverse();
+		}
+
+		let stdoutLogger = winston.createLogger({
+			level: "verbose",
+			levels,
+			format: new lib.TerminalFormat({ showTimestamp: true }),
+			transports: [
+				new ConsoleTransport({ errorLevels: [], warnLevels: [] }),
+			],
+		});
+		for (let info of result.log) {
+			stdoutLogger.log(info as any);
+		}
+	},
+}));

packages/ctl/src/commands_mod.ts

@@ -0,0 +1,215 @@
+import fs from "node:fs/promises";
+import asTableModule from "as-table";
+import events from "events";
+import path from "path";
+import stream from "stream";
+import util from "util";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+const finished = util.promisify(stream.finished);
+
+export const modCommands = new lib.CommandTree({ name: "mod", description: "Manage uploaded mods" });
+modCommands.add(new lib.Command({
+	definition: ["show <name> <mod-version>", "Show details for a mod stored in the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Mod name to show details for", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of the mod", type: "string" });
+	}],
+	handler: async function(args: { name: string, modVersion: string }, control: Control) {
+		if (!lib.isFullVersion(args.modVersion)) {
+			throw new lib.CommandError("mod-version must match format digit.digit.digit");
+		}
+		let modInfo = await control.send(new lib.ModGetRequest(args.name, args.modVersion));
+		for (let [field, value] of Object.entries(modInfo)) {
+			if (value instanceof Array) {
+				print(`${field}:`);
+				for (let entry of value) {
+					print(`  ${entry}`);
+				}
+			} else {
+				print(`${field}: ${value}`);
+			}
+		}
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: [["list", "l"], "List mods stored in the cluster", (yargs) => {
+		yargs.options({
+			"fields": {
+				describe: "Fields to show, supports 'all'.",
+				array: true,
+				type: "string",
+				default: ["name", "version", "title", "factorio_version"],
+			},
+		});
+	}],
+	handler: async function(args: { fields: string[] }, control: Control) {
+		let mods = await control.send(new lib.ModListRequest());
+		if (!args.fields.includes("all")) {
+			for (let entry of mods) {
+				for (let field of Object.keys(entry)) {
+					if (!args.fields.includes(field)) {
+						// @ts-expect-error terrible hack
+						delete entry[field];
+					}
+				}
+			}
+		}
+		print(asTable(mods));
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["search <factorio-version> [query]", "Search mods stored in the cluster", (yargs) => {
+		yargs.positional("factorio-version", { describe: "Major version of Factorio to search for", type: "string" });
+		yargs.positional("query", { describe: "Search query", type: "string", default: "" });
+		yargs.options({
+			"page": {
+				describe: "Result page to show",
+				type: "number",
+				default: 1,
+			},
+			"page-size": {
+				describe: "Results per page to show",
+				type: "number",
+				default: 10,
+			},
+			"sort": {
+				describe: "sort results by given field",
+				type: "string",
+			},
+			"sort-order": {
+				describe: "order to sort results in (asc/desc)",
+				type: "string",
+				default: "asc",
+			},
+			"fields": {
+				describe: "Fields to show, supports 'all'.",
+				array: true,
+				type: "string",
+				default: ["name", "version", "title", "factorio_version"],
+			},
+		});
+	}],
+	handler: async function(
+		args: {
+			factorioVersion: string,
+			query: string,
+			page: number,
+			pageSize: number,
+			sort?: string,
+			sortOrder: "asc" | "desc",
+			fields: string[],
+		},
+		control: Control
+	) {
+		let response = await control.send(new lib.ModSearchRequest(
+			args.query,
+			lib.normaliseApiVersion(args.factorioVersion as any),
+			args.page,
+			args.pageSize,
+			args.sort,
+			args.sortOrder
+		));
+		let results = response.results.flatMap(result => result.versions);
+		if (!args.fields.includes("all")) {
+			for (let entry of results) {
+				for (let field of Object.keys(entry)) {
+					if (!args.fields.includes(field)) {
+						// @ts-expect-error terrible hack
+						delete entry[field];
+					}
+				}
+			}
+		}
+		for (let issue of response.queryIssues) {
+			print(issue);
+		}
+		print(`page ${args.page} of ${response.pageCount} (${response.resultCount} results)`);
+		print(asTable(results));
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["upload <file>", "Upload mod to the cluster", (yargs) => {
+		yargs.positional("file", { describe: "File to upload", type: "string" });
+	}],
+	handler: async function(args: { file: string }, control: Control) {
+		let filename = path.basename(args.file);
+		if (!filename.endsWith(".zip")) {
+			throw new lib.CommandError("Mod filename must end with .zip");
+		}
+
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += "api/upload-mod";
+		url.searchParams.append("filename", filename);
+
+		let response = await fetch(url, {
+			method: "POST",
+			body: (await fs.open(args.file)).createReadStream(),
+			duplex: "half",
+			headers: {
+				"X-Access-Token": control.config.get("control.controller_token")!,
+				"Content-Type": "application/zip",
+			},
+		});
+
+		type ResponseType = { errors?: [], request_errors?: [], mods: Parameters<typeof lib.ModInfo.fromJSON>[0][]};
+		const result = await response.json() as ResponseType;
+
+		for (let error of result.errors || []) {
+			logger.error(error);
+		}
+
+		for (let requestError of result.request_errors || []) {
+			logger.error(requestError);
+		}
+
+		if (result.mods && result.mods.length) {
+			const mod = lib.ModInfo.fromJSON(result.mods[0]);
+			logger.info(`Successfully uploaded ${mod.filename}`);
+		}
+
+		if ((result.errors || []).length || (result.request_errors || []).length) {
+			throw new lib.CommandError("Uploading mod failed");
+		}
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["download <name> <mod-version>", "Download a mod from the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Internal name of mod to download", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of mod to download", type: "string" });
+	}],
+	handler: async function(args: { name: string, modVersion: string }, control: Control) {
+		if (!lib.isFullVersion(args.modVersion)) {
+			throw new lib.CommandError("mod-version must match format digit.digit.digit");
+		}
+		let streamId = await control.send(new lib.ModDownloadRequest(args.name, args.modVersion));
+
+		let url = new URL(control.config.get("control.controller_url")!);
+		url.pathname += `api/stream/${streamId}`;
+		let filename = `${args.name}_${args.modVersion}.zip`;
+		await lib.downloadFile(url, filename, "overwrite");
+
+		logger.info(`Downloaded ${filename}`);
+	},
+}));
+
+modCommands.add(new lib.Command({
+	definition: ["delete <name> <mod-version>", "Delete a mod stored in the cluster", (yargs) => {
+		yargs.positional("name", { describe: "Name of mod to delete", type: "string" });
+		yargs.positional("mod-version", { describe: "Version of mod to delete", type: "string" });
+	}],
+	handler: async function(args: { name: string, modVersion: string }, control: Control) {
+		if (!lib.isFullVersion(args.modVersion)) {
+			throw new lib.CommandError("mod-version must match format digit.digit.digit");
+		}
+		await control.send(new lib.ModDeleteRequest(args.name, args.modVersion));
+	},
+}));

packages/ctl/src/commands_mod_pack.ts

@@ -0,0 +1,315 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const modPackCommands = new lib.CommandTree({ name: "mod-pack", description: "Mod Pack" });
+modPackCommands.add(new lib.Command({
+	definition: ["show <mod-pack>", "Show details of mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to show", type: "string" });
+	}],
+	handler: async function(args: { modPack: string }, control: Control) {
+		let mods = new Map(
+			(await control.send(new lib.ModListRequest())).map(m => [m.id, m])
+		);
+		let modPack = await control.send(
+			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
+		);
+
+		for (let [field, value] of Object.entries(modPack)) {
+			if (field === "mods") {
+				print(`${field}:`);
+				for (let [name, entry] of value) {
+					const mod = mods.get(`${name}_${entry.version}`);
+					const missing = !mod && "? ";
+					const badChecksum = mod && entry.sha1 && entry.sha1 !== mod.sha1 && "! ";
+					const warn = missing || badChecksum || "";
+					const enabled = entry.enabled ? "" : "(disabled) ";
+					print(`  ${warn}${enabled}${name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`);
+				}
+			} else if (field === "settings") {
+				print(`${field}:`);
+				for (let [scope, settings] of Object.entries(value)) {
+					print(`  ${scope}:`);
+					for (let [setting, settingValue] of settings as any) {
+						print(`    ${setting}: ${JSON.stringify(settingValue.value)}`);
+					}
+				}
+			} else if (field === "exportManifest") {
+				print(`${field}:`);
+				if (value && value.assets) {
+					print("  assets:");
+					for (let [name, fileName] of Object.entries(value.assets)) {
+						print(`    ${name}: ${fileName}`);
+					}
+				}
+
+			} else {
+				print(`${field}: ${value}`);
+			}
+		}
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: [["list", "l"], "List mod packs in the cluster"],
+	handler: async function(args: object, control: Control) {
+		let modPacks = await control.send(new lib.ModPackListRequest());
+		let fields = ["id", "name", "factorioVersion"];
+		for (let entry of modPacks) {
+			for (let field of Object.keys(entry)) {
+				if (!fields.includes(field)) {
+					// @ts-expect-error terrible hack
+					delete entry[field];
+				}
+			}
+		}
+		print(asTable(modPacks));
+	},
+}));
+
+function setModPackSettings(
+	modPack: lib.ModPack,
+	args: {
+		boolSetting?: string[],
+		intSetting?: string[],
+		doubleSetting?: string[],
+		stringSetting?: string[],
+		colorSetting?: string[],
+	},
+) {
+	function doSettings(settings: string[], cast: (value: string) => (string | number | boolean)) {
+		for (let i = 0; i + 2 < settings.length; i += 3) {
+			const [scope, name, value] = settings.slice(i, i + 3);
+			if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
+				throw new lib.CommandError(
+					`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
+				);
+			}
+			const checkedScope = scope as "startup" | "runtime-global" | "runtime-per-user";
+			modPack.settings[checkedScope].set(name, { value: cast(value) });
+		}
+	}
+
+	doSettings(args.boolSetting || [], value => {
+		if (!["true", "false"].includes(value)) {
+			throw new lib.CommandError(`boolean value must be one of true or false, not ${value}`);
+		}
+		return value === "true";
+	});
+	doSettings(args.intSetting || [], value => {
+		// eslint-disable-next-line radix
+		let number = Number.parseInt(value);
+		if (isNaN(number)) {
+			throw new lib.CommandError(`int value must be an integer, not ${value}`);
+		}
+		return number;
+	});
+	doSettings(args.doubleSetting || [], value => {
+		let number = Number.parseFloat(value);
+		if (isNaN(number)) {
+			throw new lib.CommandError(`double value must be a number, not ${value}`);
+		}
+		return number;
+	});
+	doSettings(args.stringSetting || [], value => value);
+	doSettings(args.colorSetting || [], value => {
+		let color = JSON.parse(value);
+		if (typeof color !== "object" || color === null) {
+			throw new lib.CommandError("color value must be a JSON object with r, g, b, a properties");
+		}
+		return color;
+	});
+}
+
+function setModPackMods(modPack: lib.ModPack, mods: string[] | undefined) {
+	for (let mod of mods || []) {
+		const [name, version, sha1] = mod.split(":");
+		if (!version) {
+			throw new lib.CommandError("Added mod must be formatted as name:version or name:version:sha1");
+		}
+		if (!lib.isFullVersion(version)) {
+			throw new lib.CommandError("version must match the format digit.digit.digit");
+		}
+		if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) {
+			throw new lib.CommandError("sha1 must be a 40 digit lower case hex string");
+		}
+		modPack.mods.set(name, { name, enabled: true, version, sha1 });
+	}
+}
+
+function setModPackModsEnabled(modPack: lib.ModPack, mods: string[] | undefined, enabled: boolean) {
+	for (let mod of mods || []) {
+		if (!modPack.mods.has(mod)) {
+			throw new lib.CommandError(`Mod named ${mod} does not exist in the mod pack`);
+		}
+		modPack.mods.get(mod)!.enabled = enabled;
+	}
+}
+
+modPackCommands.add(new lib.Command({
+	definition: ["create <name> <factorio-version>", "Create mod pack", (yargs) => {
+		yargs.positional("name", { describe: "Name of mod pack to create", type: "string" });
+		yargs.positional("factorio-version", { describe: "Version of factorio the mod pack is for", type: "string" });
+		yargs.options({
+			"description": { describe: "Description for mod pack", type: "string" },
+			"mods": { describe: "Mods in the form of name:version[:sha1]", array: true, type: "string" },
+			"disabled-mods": { describe: "Mods that are in the pack but not enabled", array: true, type: "string" },
+			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
+			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
+			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
+			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
+			"color-setting": { describe: "Set color setting", array: true, nargs: 3, type: "string" },
+		});
+	}],
+	handler: async function(
+		args: {
+			name: string,
+			factorioVersion: string,
+			description?: string,
+			mods?: string[],
+			disabledMods?: string[],
+			boolSetting?: string[],
+			intSetting?: string[],
+			doubleSetting?: string[],
+			stringSetting?: string[],
+			colorSetting?: string[],
+		},
+		control: Control,
+	) {
+		const modPack = new lib.ModPack();
+		modPack.name = args.name;
+		if (args.description) { modPack.description = args.description; }
+		if (args.factorioVersion) {
+			if (!lib.isPartialVersion(args.factorioVersion)) {
+				throw new lib.CommandError("factorio-version must match the format digit.digit[.digit]");
+			}
+			modPack.factorioVersion = args.factorioVersion;
+		}
+		setModPackMods(modPack, args.mods);
+		setModPackModsEnabled(modPack, args.disabledMods, false);
+		setModPackSettings(modPack, args);
+		await control.send(new lib.ModPackCreateRequest(modPack));
+		print(`Created mod pack ${modPack.name} (${modPack.id})`);
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["import <string>", "Import mod pack string", (yargs) => {
+		yargs.positional("string", { describe: "Mod pack string to import", type: "string" });
+	}],
+	handler: async function(args: { string: string }, control: Control) {
+		const modPack = lib.ModPack.fromModPackString(args.string);
+		await control.send(new lib.ModPackCreateRequest(modPack));
+		print(`Created mod pack ${modPack.name} (${modPack.id})`);
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["export <mod-pack>", "Export mod pack string", (yargs) => {
+		yargs.positional("string", { describe: "Mod pack to export", type: "string" });
+	}],
+	handler: async function(args: { modPack: string }, control: Control) {
+		const modPack = await control.send(
+			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
+		);
+		print(modPack.toModPackString());
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["edit <mod-pack>", "Edit mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" });
+		yargs.options({
+			"name": { describe: "New name for mod pack", type: "string" },
+			"description": { describe: "New description for mod pack", type: "string" },
+			"factorio-version": { describe: "Set version of factorio the mod pack is for", type: "string" },
+			"add-mods": { describe: "Mods in the form of name:version[:sha1] to add", array: true, type: "string" },
+			"enable-mods": { describe: "Mods to set as enabled", array: true, type: "string" },
+			"disable-mods": { describe: "Mods to set as disabled", array: true, type: "string" },
+			"remove-mods": { describe: "Name of mods to remove", array: true, type: "string" },
+			"bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" },
+			"int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" },
+			"double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" },
+			"string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" },
+			"color-setting": { describe: "Set color setting", array: true, nargs: 3, type: "string" },
+			"remove-setting": { describe: "Remove a setting", array: true, nargs: 2, type: "string" },
+		});
+	}],
+	handler: async function(
+		args: {
+			modPack: string,
+			name?: string,
+			description?: string,
+			factorioVersion?: string,
+			addMods?: string[],
+			enableMods?: string[],
+			disableMods?: string[],
+			removeMods?: string[],
+			boolSetting?: string[],
+			intSetting?: string[],
+			doubleSetting?: string[],
+			stringSetting?: string[],
+			colorSetting?: string[],
+			removeSetting?: string[],
+		},
+		control: Control,
+	) {
+		const modPack = await control.send(
+			new lib.ModPackGetRequest(await lib.resolveModPack(control, args.modPack))
+		);
+
+		if (args.name) { modPack.name = args.name; }
+		if (args.description) { modPack.description = args.description; }
+		if (args.factorioVersion) {
+			if (!lib.isPartialVersion(args.factorioVersion)) {
+				throw new lib.CommandError("factorio-version must match the format digit.digit[.digit]");
+			}
+			modPack.factorioVersion = args.factorioVersion;
+		}
+
+		setModPackMods(modPack, args.addMods);
+		for (let name of args.removeMods || []) {
+			if (modPack.mods.has(name)) {
+				modPack.mods.delete(name);
+			} else {
+				logger.warn(`Mod ${name} did not exist on ${modPack.name}`);
+			}
+		}
+		setModPackModsEnabled(modPack, args.disableMods, false);
+		setModPackModsEnabled(modPack, args.enableMods, true);
+
+		setModPackSettings(modPack, args);
+		if (args.removeSetting) {
+			for (let i = 0; i + 1 < args.removeSetting.length; i += 2) {
+				const [scope, name] = args.removeSetting.slice(i, i + 2);
+				if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) {
+					throw new lib.CommandError(
+						`Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}`
+					);
+				}
+				const checkedScope = scope as "startup" | "runtime-global" | "runtime-per-user";
+				if (modPack.settings[checkedScope].has(name)) {
+					modPack.settings[checkedScope].delete(name);
+				} else {
+					logger.warn(`Mod setting ${scope} ${name} did not exist on ${modPack.name}`);
+				}
+			}
+		}
+		await control.send(new lib.ModPackUpdateRequest(modPack));
+	},
+}));
+
+modPackCommands.add(new lib.Command({
+	definition: ["delete <mod-pack>", "Delete mod pack", (yargs) => {
+		yargs.positional("mod-pack", { describe: "Mod pack to delete", type: "string" });
+	}],
+	handler: async function(args: { modPack: string }, control: Control) {
+		const id = await lib.resolveModPack(control, args.modPack);
+		await control.send(new lib.ModPackDeleteRequest(id));
+	},
+}));

packages/ctl/src/commands_permission.ts

@@ -0,0 +1,16 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const permissionCommands = new lib.CommandTree({ name: "permission", description: "Permission inspection" });
+permissionCommands.add(new lib.Command({
+	definition: [["list", "l"], "List permissions in the cluster"],
+	handler: async function(args: object, control: Control) {
+		let permissions = await control.send(new lib.PermissionListRequest());
+		print(asTable(permissions));
+	},
+}));

packages/ctl/src/commands_role.ts

@@ -0,0 +1,100 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const roleCommands = new lib.CommandTree({ name: "role", description: "Role management" });
+roleCommands.add(new lib.Command({
+	definition: [["list", "l"], "List roles in the cluster"],
+	handler: async function(args: object, control: Control) {
+		let roles = await control.send(new lib.RoleListRequest());
+		print(asTable(roles));
+	},
+}));
+
+roleCommands.add(new lib.Command({
+	definition: ["create <name>", "Create a new role", (yargs) => {
+		yargs.positional("name", { describe: "Name of role to create", type: "string" });
+		yargs.options({
+			"description": { describe: "Description for role", nargs: 1, type: "string", default: "" },
+			"permissions": { describe: "Permissions role grants", nargs: 1, array: true, type: "string", default: [] },
+		});
+	}],
+	handler: async function(args: { name: string, description: string, permissions: string[] }, control: Control) {
+		let id = await control.send(new lib.RoleCreateRequest(
+			args.name,
+			args.description,
+			args.permissions,
+		));
+		logger.info(`Created role ID ${id}`);
+	},
+}));
+
+roleCommands.add(new lib.Command({
+	definition: ["edit <role>", "Edit existing role", (yargs) => {
+		yargs.positional("role", { describe: "Role to edit", type: "string" });
+		yargs.options({
+			"name": { describe: "New name for role", nargs: 1, type: "string" },
+			"description": { describe: "New description for role", nargs: 1, type: "string" },
+			"set-perms": { describe: "Set permissions for role", array: true, type: "string" },
+			"add-perms": { describe: "Add permissions to role", array: true, type: "string", conflicts: "set-perms" },
+			"remove-perms": {
+				describe: "Remove permissions from role", array: true, type: "string", conflicts: "set-perms",
+			},
+			"grant-default": { describe: "Add default permissions to role", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(
+		args: {
+			role: string,
+			name?: string,
+			description?: string,
+			setPerms?: string[],
+			addPerms?: string[],
+			removePerms?: string[],
+			grantDefault?: boolean,
+		},
+		control: Control
+	) {
+		let role = await lib.retrieveRole(control, args.role);
+
+		if (args.name !== undefined) {
+			role.name = args.name;
+		}
+		if (args.description !== undefined) {
+			role.description = args.description;
+		}
+		if (args.addPerms) {
+			for (const perm of args.addPerms) {
+				role.permissions.add(perm);
+			}
+		}
+		if (args.removePerms) {
+			for (let perm of args.removePerms) {
+				role.permissions.delete(perm);
+			}
+		}
+		if (args.setPerms !== undefined) {
+			role.permissions = new Set(args.setPerms);
+		}
+		await control.send(new lib.RoleUpdateRequest(role.id, role.name, role.description, [...role.permissions]));
+
+		if (args.grantDefault) {
+			await control.send(new lib.RoleGrantDefaultPermissionsRequest(role.id));
+		}
+	},
+}));
+
+roleCommands.add(new lib.Command({
+	definition: ["delete <role>", "Delete role", (yargs) => {
+		yargs.positional("role", { describe: "Role to delete", type: "string" });
+	}],
+	handler: async function(args: { role: string }, control: Control) {
+		let role = await lib.retrieveRole(control, args.role);
+		await control.send(new lib.RoleDeleteRequest(role.id));
+	},
+}));

packages/ctl/src/commands_user.ts

@@ -0,0 +1,304 @@
+import fs from "node:fs/promises";
+import asTableModule from "as-table";
+import path from "path";
+
+import * as lib from "@clusterio/lib";
+import type { Control } from "../ctl";
+import { print } from "./command_ops";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const userCommands = new lib.CommandTree({ name: "user", alias: ["u"], description: "User management" });
+userCommands.add(new lib.Command({
+	definition: ["show <name>", "Show details for one user", (yargs) => {
+		yargs.positional("name", { decribe: "Name of user to show", type: "string" });
+		yargs.options({
+			"instance-stats": { describe: "include per-instance stats", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { name: string, instanceStats: boolean }, control: Control) {
+		let user = await control.send(new lib.UserGetRequest(args.name));
+		Object.assign(user, user.playerStats);
+		// @ts-expect-error Terrible hack
+		delete user.playerStats;
+		let instanceStats = user.instanceStats;
+		// @ts-expect-error Terrible hack
+		delete user.instanceStats;
+		print(asTable(Object.entries(user).map(([property, value]) => ({ property, value }))));
+
+		if (args.instanceStats) {
+			let instances = await control.send(new lib.InstanceDetailsListRequest());
+			function instanceName(id: number) {
+				let instance = instances.find(i => i.id === id);
+				if (instance) {
+					return instance.name;
+				}
+				return "<deleted>";
+			}
+			for (let [id, playerInstanceStats] of instanceStats || []) {
+				print();
+				print(`Instance ${instanceName(id)} (${id}):`);
+				print(asTable(Object.entries(playerInstanceStats).map(([property, value]) => ({ property, value }))));
+			}
+		}
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: [["list", "l"], "List user in the cluster", (yargs) => {
+		yargs.options({
+			"stats": { describe: "include user stats", nargs: 0, type: "boolean", default: false },
+			"attributes": { describe: "include admin/whitelisted/banned", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { stats: boolean, attributes: boolean }, control: Control) {
+		let users = await control.send(new lib.UserListRequest());
+		for (let user of users) {
+			if (args.stats) {
+				Object.assign(user, user.playerStats);
+			}
+			// @ts-expect-error Terrible hack
+			delete user.playerStats;
+			// @ts-expect-error Terrible hack
+			delete user.isDeleted;
+			// @ts-expect-error Terrible hack
+			delete user.banReason;
+			// @ts-expect-error Terrible hack
+			delete user.instanceStats;
+			if (!args.attributes) {
+				// @ts-expect-error Terrible hack
+				delete user.isAdmin;
+				// @ts-expect-error Terrible hack
+				delete user.isWhitelisted;
+				// @ts-expect-error Terrible hack
+				delete user.isBanned;
+			}
+		}
+		print(asTable(users));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["create <name>", "Create a user", (yargs) => {
+		yargs.positional("name", { describe: "Name of user to create", type: "string" });
+	}],
+	handler: async function(args: { name: string }, control: Control) {
+		await control.send(new lib.UserCreateRequest(args.name));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["revoke-token <name>", "Revoke token for user", (yargs) => {
+		yargs.positional("name", { describe: "Name of user to revoke token for", type: "string" });
+	}],
+	handler: async function(args: { name: string }, control: Control) {
+		await control.send(new lib.UserRevokeTokenRequest(args.name));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-admin <user>", "Promote or demote a user to admin", (yargs) => {
+		yargs.positional("user", { describe: "Name of user set admin status for", type: "string" });
+		yargs.options({
+			"revoke": { describe: "Revoke admin status", nargs: 0, type: "boolean", default: false },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { user: string, revoke: boolean, create: boolean }, control: Control) {
+		await control.send(new lib.UserSetAdminRequest(args.user, args.create, !args.revoke));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-whitelisted <user>", "Add or remove user from the whitelist", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to set whitelist status for", type: "string" });
+		yargs.options({
+			"remove": { describe: "Remove from whitelist", nargs: 0, type: "boolean", default: false },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { user: string, remove: boolean, create: boolean }, control: Control) {
+		await control.send(new lib.UserSetWhitelistedRequest(args.user, args.create, !args.remove));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-banned <user>", "Ban or pardon user from banlist", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to set ban status for", type: "string" });
+		yargs.options({
+			"pardon": { describe: "Remove from banlist", nargs: 0, type: "boolean", default: false },
+			"reason": { describe: "Ban reason", nargs: 1, type: "string", default: "" },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(
+		args: {
+			user: string,
+			pardon: boolean,
+			reason: string,
+			create: boolean
+		},
+		control: Control
+	) {
+		await control.send(new lib.UserSetBannedRequest(args.user, args.create, !args.pardon, args.reason));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["set-roles <user> [roles...]", "Replace user roles", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to change roles for", type: "string" });
+		yargs.positional("roles", { describe: "roles to assign", type: "string" });
+	}],
+	handler: async function(args: { user: string, roles: string[] }, control: Control) {
+		let roles = await control.send(new lib.RoleListRequest());
+
+		let resolvedRoles = [];
+		for (let roleName of args.roles) {
+			if (/^-?\d+$/.test(roleName)) {
+				let roleId = parseInt(roleName, 10);
+				resolvedRoles.push(roleId);
+
+			} else {
+				let found = false;
+				for (let role of roles) {
+					if (role.name === roleName) {
+						resolvedRoles.push(role.id);
+						found = true;
+						break;
+					}
+				}
+
+				if (!found) {
+					throw new lib.CommandError(`No role named ${roleName}`);
+				}
+			}
+		}
+
+		await control.send(new lib.UserUpdateRolesRequest(args.user, resolvedRoles));
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["delete <user>", "Delete user", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to delete", type: "string" });
+	}],
+	handler: async function(args: { user: string }, control: Control) {
+		await control.send(new lib.UserDeleteRequest(args.user));
+	},
+}));
+
+async function handleImportOrRestore(args: {
+	filepath: string,
+	users: boolean,
+	bans: boolean,
+	admins: boolean,
+	whitelist: boolean,
+	noBackup?: boolean,
+}, control: Control, restore: boolean) {
+	let importType: ConstructorParameters<typeof lib.UserBulkImportRequest>[0];
+	const data = JSON.parse((await fs.readFile(args.filepath)).toString());
+	const optionCount = [args.users, args.bans, args.admins, args.whitelist]
+		.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
+
+	if (optionCount > 1) {
+		throw new lib.CommandError("Can not specify multiple options");
+	}
+
+	// Assign based on options or attempt to guess based on filename
+	const filename = path.basename(args.filepath);
+	if (args.users || (optionCount === 0 && data.export_version)) {
+		importType = "users";
+	} else if (args.bans || (optionCount === 0 && filename.includes("ban"))) {
+		importType = "bans";
+	} else if (args.admins || (optionCount === 0 && filename.includes("admin"))) {
+		importType = "admins";
+	} else if (args.whitelist || (optionCount === 0 && filename.includes("whitelist"))) {
+		importType = "whitelist";
+	} else {
+		throw new lib.CommandError("Unknown json file, please specify an option");
+	}
+
+	const backup = await control.send(
+		new lib.UserBulkImportRequest(importType, importType === "users" ? data.users : data, restore)
+	);
+
+	print(`${restore ? "Restored" : "Imported"} ${importType} from ${args.filepath}`);
+	if (restore && backup && !args.noBackup) {
+		await fs.writeFile(`${importType}-backup.json`, JSON.stringify(backup, undefined, "\t"));
+		print(`Wrote backup to ${importType}-backup.json`);
+	}
+}
+
+userCommands.add(new lib.Command({
+	definition: ["import <filepath>", "Import user data from a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
+		yargs.options({
+			"users": { describe: "Import users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Import banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Import adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Import whitelist json", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: any, control: any) {
+		await handleImportOrRestore(args, control, false);
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["restore <filepath>", "Restore user data from a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to import", type: "string" });
+		yargs.options({
+			"users": { describe: "Restore users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Restore banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Restore adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Restore whitelist json", nargs: 0, type: "boolean", default: false },
+			"no-backup": { describe: "Don't save a backup to the cwd", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: any, control: any) {
+		await handleImportOrRestore(args, control, true);
+	},
+}));
+
+userCommands.add(new lib.Command({
+	definition: ["export <filepath>", "Export user data to a file", (yargs) => {
+		yargs.positional("filepath", { describe: "Path to the file to save to", type: "string" });
+		yargs.options({
+			"users": { describe: "Export users json", nargs: 0, type: "boolean", default: false },
+			"bans": { describe: "Export banlist json", nargs: 0, type: "boolean", default: false },
+			"admins": { describe: "Export adminlist json", nargs: 0, type: "boolean", default: false },
+			"whitelist": { describe: "Export whitelist json", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: {
+		filepath: string,
+		users: boolean,
+		bans: boolean,
+		admins: boolean,
+		whitelist: boolean,
+	}, control: Control) {
+		let exportType = "users" as ConstructorParameters<typeof lib.UserBulkExportRequest>[0];
+		const optionCount = [args.bans, args.admins, args.whitelist]
+			.reduce((acc, bool) => (bool ? acc + 1 : acc), 0);
+
+		if (optionCount > 1) {
+			throw new lib.CommandError("Can not specify multiple options");
+		}
+
+		// Assign based on options or attempt to guess based on filename
+		if (args.users) {
+			exportType = "users";
+		} else if (args.bans) {
+			exportType = "bans";
+		} else if (args.admins) {
+			exportType = "admins";
+		} else if (args.whitelist) {
+			exportType = "whitelist";
+		}
+
+		await fs.writeFile(args.filepath, JSON.stringify(await control.send(
+			new lib.UserBulkExportRequest(exportType)
+		), undefined, "\t"));
+		print(`Exported ${exportType} to ${args.filepath}`);
+	},
+}));

packages/ctl/src/config_ops.ts

@@ -0,0 +1,58 @@
+import * as lib from "@clusterio/lib";
+
+export async function getEditor(argsEditor: string) {
+	// eslint-disable-next-line
+	return argsEditor || process.env.EDITOR || process.env.VISUAL || undefined
+	// needed for the process.env statements to not be flagged by eslint
+	// priority for editors is CLI argument > env.EDITOR > env.VISUAL
+}
+
+export async function configToKeyVal(data: string) {
+	let final: Record<string, string> = {};
+	let splitData = data.split(/\r?\n/);
+	// split on newlines
+	let filtered = splitData.filter((value) => value[0] !== "#").filter((a) => a);
+	// the last filter removes empty elements left by the first. Not done on one line due to readability.
+	for (let index in filtered) {
+		if (index in filtered) {
+			let split = filtered[index].split("=");
+			let finalIndex = filtered[index][0].trim();
+			// split on the = we added earlier, giving us both value and key
+			let part = "";
+			try {
+				part = split[1].trim();
+				// it's a string if we can read it
+			} catch (err) {
+				// if we can't read it, it's a empty field and therefor null
+				part = "";
+			}
+			final[finalIndex] = part;
+		}
+	}
+	return final;
+}
+
+export function serializedConfigToString(
+	serializedConfig: lib.ConfigSchema,
+	configGroup: typeof lib.ControllerConfig | typeof lib.HostConfig | typeof lib.InstanceConfig,
+	disallowedList: Record<string, unknown>,
+) {
+	let allConfigElements = "";
+	for (let [name, value] of Object.entries(serializedConfig)) {
+		if (name in disallowedList) {
+			continue;
+		}
+		let desc = "";
+		try {
+			desc += (configGroup.fieldDefinitions as any)[name]!.description;
+		} catch (err) {
+			desc += "No description found";
+		}
+		// split onto two lines for readability and es-lint
+		if (String(value) === "null") {
+			value = "";
+		}
+		allConfigElements += `${name} = ${value}\n\n`;
+	}
+	return allConfigElements;
+}

packages/host/host.ts

@@ -12,7 +12,7 @@
  * @example
  * npx clusteriohost run
  */
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import path from "path";
 import yargs from "yargs";
 import setBlocking from "set-blocking";
@@ -51,13 +51,11 @@ void new lib.Gauge(
 export class HostConnector extends lib.WebSocketClientConnector {
 	constructor(
 		public hostConfig: lib.HostConfig,
-		tlsCa: string | undefined,
 		public pluginInfos: lib.PluginNodeEnvInfo[]
 	) {
 		super(
 			hostConfig.get("host.controller_url"),
 			hostConfig.get("host.max_reconnect_delay"),
-			tlsCa
 		);
 	}
 
@@ -201,9 +199,9 @@ async function startHost() {
 	await hostConfigLock.acquire();
 
 	// If we get here the command was run
-	await fs.ensureDir(hostConfig.get("host.instances_directory"));
-	await fs.ensureDir(hostConfig.get("host.mods_directory"));
-	await fs.ensureDir("modules");
+	await fs.mkdir(hostConfig.get("host.instances_directory"), { recursive: true });
+	await fs.mkdir(hostConfig.get("host.mods_directory"), { recursive: true });
+	await fs.mkdir("modules", { recursive: true });
 
 	// Set the process title, shows up as the title of the CMD window on windows
 	// and as the process name in ps/top on linux.
@@ -230,17 +228,10 @@ async function startHost() {
 		return;
 	}
 
-	let tlsCa: string | undefined;
-	let tlsCaPath = hostConfig.get("host.tls_ca");
-	if (tlsCaPath) {
-		tlsCa = await fs.readFile(tlsCaPath, "utf8");
-	}
-
-	let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos);
+	let hostConnector = new HostConnector(hostConfig, pluginInfos);
 	host = new Host(
 		hostConnector,
 		hostConfig,
-		tlsCa,
 		pluginInfos,
 		Boolean(args.canRestart),
 		Boolean(args.recovery),
@@ -291,6 +282,7 @@ async function startHost() {
 	});
 
 	await hostConnector.connect();
+	host.startedAtMs = Date.now();
 	logger.info("Started host");
 }
 
@@ -307,7 +299,7 @@ I           version of clusterio.  Expect things to break. I
 		if (err instanceof lib.WebSocketError) {
 			// Already handled by connector.on("error") within Host
 
-		} else if (err instanceof lib.StartupError) {
+		} else if (err instanceof lib.StartupError || err.code === "InstallationError") {
 			logger.fatal(`
 +----------------------------------+
 | Unable to to start clusteriohost |

packages/host/lua/export/control.lua

@@ -9,4 +9,5 @@ script.on_init(function()
 	for name, mod_setting in pairs(mod_settings) do
 		send_json("mod_setting_mod", { name = name, mod = mod_setting.mod })
 	end
+	send_json("defines", defines)
 end)

packages/host/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio host server",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.24",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -23,16 +23,14 @@
 	"dependencies": {
 		"@clusterio/lib": "workspace:*",
 		"@sinclair/typebox": "catalog:",
-		"set-blocking": "catalog:",
-		"fs-extra": "catalog:",
+		"jimp": "^1.6.0",
 		"jszip": "catalog:",
-		"phin": "catalog:",
-		"winston": "catalog:",
-		"winston-daily-rotate-file": "catalog:",
-		"jimp": "^0.22.8",
 		"pidusage": "^3.0.2",
 		"rcon-client": "^4.2.5",
 		"semver": "^7.7.1",
+		"set-blocking": "catalog:",
+		"winston": "catalog:",
+		"winston-daily-rotate-file": "catalog:",
 		"yargs": "^17.7.2"
 	},
 	"publishConfig": {
@@ -40,10 +38,9 @@
 	},
 	"devDependencies": {
 		"@types/node": "catalog:",
-		"@types/fs-extra": "catalog:",
-		"@types/set-blocking": "catalog:",
 		"@types/pidusage": "^2.0.2",
 		"@types/semver": "^7.5.0",
+		"@types/set-blocking": "catalog:",
 		"@types/yargs": "^17.0.33",
 		"typescript": "catalog:"
 	}

packages/host/src/Host.ts

@@ -1,9 +1,8 @@
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import path from "path";
 import events from "events";
 import pidusage from "pidusage";
 import setBlocking from "set-blocking";
-import phin from "phin";
 import stream from "stream";
 import util from "util";
 
@@ -32,7 +31,7 @@ function checkRequestSaveName(name: string) {
  *
  * Looks through all sub-dirs of the provided directory for valid
  * instance definitions and return a mapping of instance id to
- * instanceInfo objects.
+ * instance configs and the root instance paths.
  *
  * @param instancesDir - Directory containing instances
  * @returns
@@ -40,7 +39,7 @@ function checkRequestSaveName(name: string) {
  * @internal
  */
 async function discoverInstances(instancesDir: string) {
-	let instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
+	const instances = new Map<number, UnloadedInstance>();
 	for (let entry of await fs.readdir(instancesDir, { withFileTypes: true })) {
 		if (entry.isDirectory()) {
 			let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
@@ -60,21 +59,25 @@ async function discoverInstances(instancesDir: string) {
 				continue;
 			}
 
-			if (instanceInfos.has(instanceConfig.get("instance.id"))) {
+			if (instances.has(instanceConfig.get("instance.id"))) {
 				logger.warn(`Ignoring instance with duplicate id in folder ${entry.name}`);
 				continue;
 			}
 
 			logger.verbose(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`);
-			instanceInfos.set(instanceConfig.get("instance.id"), {
+			instances.set(instanceConfig.get("instance.id"), {
 				path: instancePath,
 				config: instanceConfig,
 			});
 		}
 	}
-	return instanceInfos;
+	return instances;
 }
 
+// Requests that wake up instances are responsible for making sure the
+// instance status is correctly updated even if they fail with an error. This
+// means they should typically use try catch on anything that can throw
+// and call notifyExit upon any errors.
 const instanceStartingMessages = new Set([
 	lib.InstanceStartRequest.name,
 	lib.InstanceLoadScenarioRequest.name,
@@ -114,10 +117,10 @@ export class HostRouter {
 			dst.type === lib.Address.controller
 			|| dst.type === lib.Address.control
 			|| dst.type === lib.Address.host && dst.id !== this.host.config.get("host.id")
-			|| dst.type === lib.Address.instance && !this.host.instanceInfos.has(dst.id)
+			|| dst.type === lib.Address.instance && !this.host.assignedInstances.has(dst.id)
 		) {
 			nextHop = this.host;
-		} else if (dst.type === lib.Address.instance && this.host.instanceInfos.has(dst.id)) {
+		} else if (dst.type === lib.Address.instance && this.host.assignedInstances.has(dst.id)) {
 			nextHop = this.host.instanceConnections.get(dst.id)!;
 		}
 
@@ -244,6 +247,12 @@ export class HostRouter {
 	}
 }
 
+// Instance information required to location and load an instance
+type UnloadedInstance = {
+	path: string,
+	config: lib.InstanceConfig,
+};
+
 /**
  * Handles running the host
  *
@@ -253,19 +262,18 @@ export default class Host extends lib.Link {
 	declare "connector": HostConnector;
 
 	router = new HostRouter(this);
-	/**
-	 * Certificate authority used to validate TLS connections to the controller.
-	 */
-	tlsCa?: string;
 	pluginInfos: lib.PluginNodeEnvInfo[];
 	config: lib.HostConfig;
 
 	/** Mapping of plugin name to loaded plugin */
 	plugins: Map<string, BaseHostPlugin> = new Map();
 
+	/** A map from instance id to instance connection. Only present when instance is running. */
 	instanceConnections = new Map<number, InstanceConnection>();
-	discoveredInstanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
-	instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
+	/** A map of instances that exist in the instance directory of the host */
+	discoveredInstances = new Map<number, UnloadedInstance>();
+	/** A map of instances that are assigned to this host */
+	assignedInstances = new Map<number, UnloadedInstance>();
 
 	adminlist = new Set<string>();
 	banlist = new Map<string, string>();
@@ -278,13 +286,15 @@ export default class Host extends lib.Link {
 	_disconnecting = false;
 	_shuttingDown = false;
 
+	startedAtMs = Date.now();
+
 	static instanceConfigWarning = {
 		"_warning": "Changes to this file will be overwritten by the controller's copy.",
 	};
 
 	static async bootstrap(hostConfig: lib.HostConfig) {
 		const modsDirectory = hostConfig.get("host.mods_directory");
-		await fs.ensureDir(modsDirectory);
+		await fs.mkdir(modsDirectory, { recursive: true });
 		return [
 			await lib.ModStore.fromDirectory(modsDirectory),
 		] as const;
@@ -293,7 +303,6 @@ export default class Host extends lib.Link {
 	constructor(
 		connector: HostConnector,
 		hostConfig: lib.HostConfig,
-		tlsCa: string | undefined,
 		pluginInfos: lib.PluginNodeEnvInfo[],
 		/**
 		 * If true indicates that there is a process monitor present that
@@ -308,7 +317,6 @@ export default class Host extends lib.Link {
 		public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
 	) {
 		super(connector);
-		this.tlsCa = tlsCa;
 
 		this.pluginInfos = pluginInfos;
 		this.config = hostConfig;
@@ -379,6 +387,7 @@ export default class Host extends lib.Link {
 		this.handle(lib.HostRestartRequest, this.handleHostRestartRequest.bind(this));
 		this.handle(lib.HostUpdateRequest, this.handleHostUpdateRequest.bind(this));
 		this.handle(lib.HostConfigGetRequest, this.handleHostConfigGetRequest.bind(this));
+		this.handle(lib.HostConfigSetRequest, this.handleHostConfigSetRequest.bind(this));
 		this.handle(lib.HostConfigSetFieldRequest, this.handleHostConfigSetFieldRequest.bind(this));
 		this.handle(lib.HostConfigSetPropRequest, this.handleHostConfigSetPropRequest.bind(this));
 		this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this));
@@ -436,6 +445,9 @@ export default class Host extends lib.Link {
 				this.plugins.set(pluginInfo.name, hostPlugin);
 
 			} catch (err: any) {
+				if (err.code === "InstallationError") {
+					throw err;
+				}
 				throw new lib.PluginError(pluginInfo.name, err);
 			}
 
@@ -492,18 +504,41 @@ export default class Host extends lib.Link {
 		return this.config.toRemote("control");
 	}
 
-	async handleHostConfigSetFieldRequest(request: lib.HostConfigSetFieldRequest) {
-		if (request.field === "host.id") {
+	private static validateHostConfigSetField(fieldName: string) {
+		if (fieldName === "host.id") {
 			// Changing host.id at runtime is not worth the effort to
 			// support and will break a lot of things if it is allowed.
 			throw new lib.RequestError("Setting 'host.id' while host is running is not supported");
 		}
+	}
 
+	async handleHostConfigSetRequest(request: lib.HostConfigSetRequest) {
+		try {
+			for (const [fieldName, fieldValue] of Object.entries(request.fields)) {
+				Host.validateHostConfigSetField(fieldName);
+				if (typeof fieldValue === "object") {
+					for (const [propName, propValue] of Object.entries(fieldValue)) {
+						this.config.stageProp(fieldName as keyof lib.HostConfigFields, propName, propValue, "control");
+					}
+				} else {
+					this.config.stage(fieldName as keyof lib.HostConfigFields, fieldValue, "control");
+				}
+			}
+
+			this.config.commitStaging();
+		} finally {
+			this.config.revertStaging();
+		}
+	}
+
+	async handleHostConfigSetFieldRequest(request: lib.HostConfigSetFieldRequest) {
+		Host.validateHostConfigSetField(request.field);
 		this.config.set(request.field as keyof lib.HostConfigFields, request.value, "control");
 	}
 
 	async handleHostConfigSetPropRequest(request: lib.HostConfigSetPropRequest) {
 		let { field, prop, value } = request;
+		Host.validateHostConfigSetField(field);
 		this.config.setProp(field as keyof lib.HostConfigFields, prop, value, "control");
 	}
 
@@ -580,20 +615,10 @@ export default class Host extends lib.Link {
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		let response = await phin({
-			url, method: "GET",
-			core: { ca: this.tlsCa } as object,
-			stream: true,
-		});
 
 		const file = lib.ModInfo.filename(mod.name, mod.version);
 		const filePath = path.join(this.modStore.modsDirectory, file);
-		const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip");
-		const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" });
-		await events.once(writeStream, "open");
-		response.pipe(writeStream);
-		await finished(writeStream);
-		await fs.rename(tempFilePath, filePath);
+		await lib.downloadFile(url, filePath, "overwrite");
 
 		const modInfo = await this.modStore.loadFile(file);
 		if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
@@ -650,16 +675,16 @@ export default class Host extends lib.Link {
 
 	async handleInstanceAssignInternalRequest(request: lib.InstanceAssignInternalRequest) {
 		let { instanceId, config } = request;
-		let instanceInfo = this.instanceInfos.get(instanceId);
-		if (instanceInfo) {
-			instanceInfo.config.update(config, true, "controller");
-			logger.verbose(`Updated config for ${instanceInfo.path}`, this.instanceLogMeta(instanceId, instanceInfo));
+		let instance = this.assignedInstances.get(instanceId);
+		if (instance) {
+			instance.config.update(config, true, "controller");
+			logger.verbose(`Updated config for ${instance.path}`, this.instanceLogMeta(instanceId, instance));
 
 		} else {
-			instanceInfo = this.discoveredInstanceInfos.get(instanceId);
-			if (instanceInfo) {
-				await Instance.populate_folders(instanceInfo.path);
-				instanceInfo.config.update(config, true, "controller");
+			instance = this.discoveredInstances.get(instanceId);
+			if (instance) {
+				await Instance.populate_folders(instance.path);
+				instance.config.update(config, true, "controller");
 
 			} else {
 				let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning);
@@ -670,16 +695,16 @@ export default class Host extends lib.Link {
 
 				logger.info(`Creating ${instanceDir}`);
 				await Instance.populate_folders(instanceDir);
-				instanceInfo = {
+				instance = {
 					path: instanceDir,
 					config: instanceConfig,
 				};
 
-				this.discoveredInstanceInfos.set(instanceId, instanceInfo);
+				this.discoveredInstances.set(instanceId, instance);
 			}
 
-			this.instanceInfos.set(instanceId, instanceInfo);
-			logger.verbose(`assigned instance ${instanceInfo.config.get("instance.name")}`);
+			this.assignedInstances.set(instanceId, instance);
+			logger.verbose(`assigned instance ${instance.config.get("instance.name")}`);
 		}
 
 		// Somewhat hacky, but in the event of a lost session the status is
@@ -694,45 +719,45 @@ export default class Host extends lib.Link {
 				instanceConnection
 					? instanceConnection.instance.server.version
 					// TODO: factorio.version is not validated
-					: instanceInfo.config.get("factorio.version") as lib.TargetVersion,
+					: instance.config.get("factorio.version") as lib.TargetVersion,
 			)
 		);
 
 		// Send the new list of saves for this assigned instance to the controller.
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 
 		// save a copy of the instance config
-		await instanceInfo.config.save();
+		await instance.config.save();
 	}
 
 	async handleInstanceUnassignInternalRequest(request: lib.InstanceUnassignInternalRequest) {
 		let instanceId = request.instanceId;
-		let instanceInfo = this.instanceInfos.get(instanceId);
-		if (instanceInfo) {
+		let instance = this.assignedInstances.get(instanceId);
+		if (instance) {
 			let instanceConnection = this.instanceConnections.get(instanceId);
 			if (instanceConnection && ["starting", "running"].includes(instanceConnection.instance.status)) {
 				await instanceConnection.send(new lib.InstanceStopRequest());
 			}
 
-			this.instanceInfos.delete(instanceId);
-			logger.verbose(`unassigned instance ${instanceInfo.config.get("instance.name")}`);
+			this.assignedInstances.delete(instanceId);
+			logger.verbose(`unassigned instance ${instance.config.get("instance.name")}`);
 		}
 	}
 
-	instanceLogMeta(instanceId: number, instanceInfo?: { config: lib.InstanceConfig }) {
-		instanceInfo = instanceInfo || this.instanceInfos.get(instanceId);
-		if (!instanceInfo) {
+	instanceLogMeta(instanceId: number, instance?: { config: lib.InstanceConfig }) {
+		instance = instance || this.assignedInstances.get(instanceId);
+		if (!instance) {
 			return { instance_id: instanceId, instance_name: String(instanceId) };
 		}
-		return { instance_id: instanceId, instance_name: instanceInfo.config.get("instance.name") };
+		return { instance_id: instanceId, instance_name: instance.config.get("instance.name") };
 	}
 
-	getRequestInstanceInfo(instanceId: number) {
-		let instanceInfo = this.instanceInfos.get(instanceId);
-		if (!instanceInfo) {
+	getRequestInstance(instanceId: number) {
+		let instance = this.assignedInstances.get(instanceId);
+		if (!instance) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
 		}
-		return instanceInfo;
+		return instance;
 	}
 
 	/**
@@ -741,7 +766,7 @@ export default class Host extends lib.Link {
 	 * @returns Assigned game port or undefined if it does not exist.
 	 */
 	gamePort(instanceId: number) {
-		const instance = this.discoveredInstanceInfos.get(instanceId);
+		const instance = this.discoveredInstances.get(instanceId);
 		if (!instance) {
 			return undefined;
 		}
@@ -754,7 +779,7 @@ export default class Host extends lib.Link {
 
 	assignGamePort(instanceId: number) {
 		const availablePorts = lib.parseRanges(this.config.get("host.factorio_port_range"), 1, 2 ** 16 - 1);
-		for (const [id, instance] of this.instanceInfos) {
+		for (const [id, instance] of this.assignedInstances) {
 			if (id === instanceId) {
 				continue;
 			}
@@ -766,7 +791,7 @@ export default class Host extends lib.Link {
 				availablePorts.delete(port);
 			}
 		}
-		const instanceToAssign = this.instanceInfos.get(instanceId)!;
+		const instanceToAssign = this.assignedInstances.get(instanceId)!;
 		const assignedPort = instanceToAssign.config.get("factorio.host_assigned_game_port");
 		if (assignedPort !== null && availablePorts.has(assignedPort)) {
 			return assignedPort;
@@ -786,7 +811,7 @@ export default class Host extends lib.Link {
 	 * @returns connection to instance.
 	 */
 	async _connectInstance(instanceId: number) {
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let unloadedInstance = this.getRequestInstance(instanceId);
 		if (this.instanceConnections.has(instanceId)) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
 		}
@@ -795,7 +820,9 @@ export default class Host extends lib.Link {
 		let instanceAddress = new lib.Address(lib.Address.instance, instanceId);
 		let [connectionClient, connectionServer] = lib.VirtualConnector.makePair(instanceAddress, hostAddress);
 		let instance = new Instance(
-			this, connectionClient, instanceInfo.path, this.config.get("host.factorio_directory"), instanceInfo.config
+			this, connectionClient,
+			unloadedInstance.path, this.config.get("host.factorio_directory"),
+			unloadedInstance.config
 		);
 		let instanceConnection = new InstanceConnection(connectionServer, this, instance);
 
@@ -807,7 +834,7 @@ export default class Host extends lib.Link {
 
 	async handleSystemInfoRequest() {
 		const restartRequired = await this.checkRestartRequired();
-		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, restartRequired);
+		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, restartRequired, this.startedAtMs);
 	}
 
 	async checkRestartRequired() {
@@ -820,7 +847,7 @@ export default class Host extends lib.Link {
 			// First check the clusterio version
 			const runningVersion = this.config.get("host.version");
 			const packageJsonPath = require.resolve("@clusterio/host/package.json");
-			const packageJson = await fs.readJSON(packageJsonPath);
+			const packageJson = JSON.parse(await fs.readFile(packageJsonPath, "utf8"));
 			if (runningVersion !== packageJson.version) {
 				this.config.restartRequired = true;
 				return true;
@@ -837,7 +864,7 @@ export default class Host extends lib.Link {
 
 				packageName = pluginInfo.npmPackage ?? pluginInfo.name;
 				const pluginPackageJsonPath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json"));
-				const pluginPackageJson = await fs.readJSON(pluginPackageJsonPath);
+				const pluginPackageJson = JSON.parse(await fs.readFile(pluginPackageJsonPath, "utf8"));
 				if (pluginInfo.version !== pluginPackageJson.version) {
 					this.config.restartRequired = true;
 					return true;
@@ -894,40 +921,45 @@ export default class Host extends lib.Link {
 		src: lib.Address,
 		dst: lib.Address
 	) {
-		let instanceInfo = this.getRequestInstanceInfo(dst.id);
-		return await Instance.listSaves(dst.id, path.join(instanceInfo.path, "saves"), null);
+		let instance = this.getRequestInstance(dst.id);
+		return await Instance.listSaves(dst.id, path.join(instance.path, "saves"), null);
 	}
 
 	async handleInstanceRenameSaveRequest(request: lib.InstanceRenameSaveRequest) {
 		let { instanceId, oldName, newName } = request;
 		checkRequestSaveName(oldName);
 		checkRequestSaveName(newName);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 		try {
-			await fs.move(
-				path.join(instanceInfo.path, "saves", oldName),
-				path.join(instanceInfo.path, "saves", newName),
-				{ overwrite: false },
+			// Link save in the new location.
+			await fs.link(
+				path.join(instance.path, "saves", oldName),
+				path.join(instance.path, "saves", newName),
 			);
+			// Unlink save in old location.
+			await fs.unlink(path.join(instance.path, "saves", oldName));
 		} catch (err: any) {
+			if (err.code === "EEXIST") {
+				throw new lib.RequestError(`${newName} already exists`);
+			}
 			if (err.code === "ENOENT") {
 				throw new lib.RequestError(`${oldName} does not exist`);
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 	}
 
 	async handleInstanceCopySaveRequest(request: lib.InstanceCopySaveRequest) {
 		let { instanceId, source, destination } = request;
 		checkRequestSaveName(source);
 		checkRequestSaveName(destination);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 		try {
-			await fs.copy(
-				path.join(instanceInfo.path, "saves", source),
-				path.join(instanceInfo.path, "saves", destination),
-				{ overwrite: false, errorOnExist: true },
+			await fs.copyFile(
+				path.join(instance.path, "saves", source),
+				path.join(instance.path, "saves", destination),
+				fs.constants.COPYFILE_EXCL | fs.constants.COPYFILE_FICLONE
 			);
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
@@ -935,34 +967,33 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 	}
 
 	async handleInstanceTransferSaveRequest(request: lib.InstanceTransferSaveRequest) {
 		let { sourceName, targetName, copy, sourceInstanceId, targetInstanceId } = request;
 		checkRequestSaveName(sourceName);
 		checkRequestSaveName(targetName);
-		let sourceInstanceInfo = this.getRequestInstanceInfo(sourceInstanceId);
-		let targetInstanceInfo = this.getRequestInstanceInfo(targetInstanceId);
+		let sourceInstance = this.getRequestInstance(sourceInstanceId);
+		let targetInstance = this.getRequestInstance(targetInstanceId);
 
 		// For consistency with remote transfer initiated through pullSave the
 		// target is renamed if it already exists.
 		targetName = await lib.findUnusedName(
-			path.join(targetInstanceInfo.path, "saves"), targetName, ".zip"
+			path.join(targetInstance.path, "saves"), targetName, ".zip"
 		);
 
 		try {
 			if (copy) {
-				await fs.copy(
-					path.join(sourceInstanceInfo.path, "saves", sourceName),
-					path.join(targetInstanceInfo.path, "saves", targetName),
-					{ overwrite: true },
+				await fs.copyFile(
+					path.join(sourceInstance.path, "saves", sourceName),
+					path.join(targetInstance.path, "saves", targetName),
+					fs.constants.COPYFILE_FICLONE
 				);
 			} else {
-				await fs.move(
-					path.join(sourceInstanceInfo.path, "saves", sourceName),
-					path.join(targetInstanceInfo.path, "saves", targetName),
-					{ overwrite: true },
+				await fs.rename(
+					path.join(sourceInstance.path, "saves", sourceName),
+					path.join(targetInstance.path, "saves", targetName),
 				);
 			}
 		} catch (err: any) {
@@ -971,8 +1002,8 @@ export default class Host extends lib.Link {
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(sourceInstanceId, path.join(sourceInstanceInfo.path, "saves"));
-		await this.sendSaveListUpdate(targetInstanceId, path.join(targetInstanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(sourceInstanceId, path.join(sourceInstance.path, "saves"));
+		await this.sendSaveListUpdate(targetInstanceId, path.join(targetInstance.path, "saves"));
 
 		return targetName;
 	}
@@ -992,59 +1023,29 @@ export default class Host extends lib.Link {
 	async handleInstanceDeleteSaveRequest(request: lib.InstanceDeleteSaveRequest) {
 		let { instanceId, name } = request;
 		checkRequestSaveName(name);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 
 		try {
-			await fs.unlink(path.join(instanceInfo.path, "saves", name));
+			await fs.unlink(path.join(instance.path, "saves", name));
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
 				throw new lib.RequestError(`${name} does not exist`);
 			}
 			throw err;
 		}
-		await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves"));
+		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 	}
 
 	async handleInstancePullSaveRequest(request: lib.InstancePullSaveRequest) {
 		let { instanceId, streamId, name } = request;
 		checkRequestSaveName(name);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		let response = await phin({
-			url, method: "GET",
-			core: { ca: this.tlsCa } as object,
-			stream: true,
-		});
-
-		if (response.statusCode !== 200) {
-			let content = await lib.readStream(response);
-			throw new lib.RequestError(`Stream returned ${response.statusCode}: ${content.toString()}`);
-		}
-
-		let savesDir = path.join(instanceInfo.path, "saves");
-		let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip");
-		let writeStream: NodeJS.WritableStream;
-		while (true) {
-			try {
-				writeStream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" });
-				await events.once(writeStream, "open");
-				break;
-			} catch (err: any) {
-				if (err.code === "EEXIST") {
-					tempFilename = await lib.findUnusedName(savesDir, tempFilename, ".tmp.zip");
-				} else {
-					throw err;
-				}
-			}
-		}
-		response.pipe(writeStream);
-		await finished(writeStream);
-
-		name = await lib.findUnusedName(savesDir, name, ".zip");
-		await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, name));
-
+		let savesDir = path.join(instance.path, "saves");
+		const savedPath = await lib.downloadFile(url, path.join(savesDir, name), "rename");
+		name = path.basename(savedPath);
 		await this.sendSaveListUpdate(instanceId, savesDir);
 		return name;
 	}
@@ -1052,12 +1053,11 @@ export default class Host extends lib.Link {
 	async handleInstancePushSaveRequest(request: lib.InstancePushSaveRequest) {
 		let { instanceId, streamId, name } = request;
 		checkRequestSaveName(name);
-		let instanceInfo = this.getRequestInstanceInfo(instanceId);
+		let instance = this.getRequestInstance(instanceId);
 
-		let content: Buffer;
+		let content;
 		try {
-			// phin doesn't support streaming requests :(
-			content = await fs.readFile(path.join(instanceInfo.path, "saves", name));
+			content = (await fs.open(path.join(instance.path, "saves", name))).createReadStream();
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
 				throw new lib.RequestError(`${name} does not exist`);
@@ -1067,10 +1067,10 @@ export default class Host extends lib.Link {
 
 		let url = new URL(this.config.get("host.controller_url"));
 		url.pathname += `api/stream/${streamId}`;
-		phin({
-			url, method: "PUT",
-			core: { ca: this.tlsCa } as object,
-			data: content,
+		fetch(url, {
+			method: "PUT",
+			body: content,
+			duplex: "half",
 		}).catch(err => {
 			logger.error(`Error pushing save to controller:\n${err.stack}`, this.instanceLogMeta(instanceId));
 		});
@@ -1082,14 +1082,14 @@ export default class Host extends lib.Link {
 			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
 		}
 
-		let instanceInfo = this.discoveredInstanceInfos.get(instanceId);
-		if (!instanceInfo) {
+		let instance = this.discoveredInstances.get(instanceId);
+		if (!instance) {
 			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
 		}
 
-		this.discoveredInstanceInfos.delete(instanceId);
-		this.instanceInfos.delete(instanceId);
-		await fs.remove(instanceInfo.path);
+		this.discoveredInstances.delete(instanceId);
+		this.assignedInstances.delete(instanceId);
+		await fs.rm(instance.path, { recursive: true, maxRetries: 10 });
 	}
 
 	async handleHostUpdateRequest(request: lib.HostUpdateRequest) {
@@ -1139,12 +1139,12 @@ export default class Host extends lib.Link {
 	 * the host and controller with the new list of instances.
 	 */
 	async updateInstances() {
-		this.discoveredInstanceInfos = await discoverInstances(this.config.get("host.instances_directory"));
+		this.discoveredInstances = await discoverInstances(this.config.get("host.instances_directory"));
 		let list = [];
-		for (let [instanceId, instanceInfo] of this.discoveredInstanceInfos) {
+		for (let [instanceId, instance] of this.discoveredInstances) {
 			let instanceConnection = this.instanceConnections.get(instanceId);
 			list.push(new lib.HostInstanceUpdate(
-				instanceInfo.config.toRemote("controller"),
+				instance.config.toRemote("controller"),
 				instanceConnection ? instanceConnection.instance.status : "stopped",
 				instanceConnection ? instanceConnection.instance.server.gamePort : this.gamePort(instanceId),
 				instanceConnection ? instanceConnection.instance.server.version : undefined,
@@ -1156,10 +1156,10 @@ export default class Host extends lib.Link {
 		if (this._startup) {
 			this._startup = false;
 
-			for (let [instanceId, instanceInfo] of this.instanceInfos) {
-				if (instanceInfo.config.get("instance.auto_start")) {
+			for (let [instanceId, instance] of this.assignedInstances) {
+				if (instance.config.get("instance.auto_start")) {
 					if (this.recoveryMode) {
-						logger.warn(`Recovery | skipping auto startup for ${instanceInfo.config.get("instance.name")}`);
+						logger.warn(`Recovery | skipping auto startup for ${instance.config.get("instance.name")}`);
 						continue;
 					}
 					try {
@@ -1167,8 +1167,8 @@ export default class Host extends lib.Link {
 						await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
 					} catch (err: any) {
 						logger.error(
-							`Error during auto startup for ${instanceInfo.config.get("instance.name")}:\n${err.stack}`,
-							this.instanceLogMeta(instanceId, instanceInfo)
+							`Error during auto startup for ${instance.config.get("instance.name")}:\n${err.stack}`,
+							this.instanceLogMeta(instanceId, instance)
 						);
 					}
 				}
@@ -1216,7 +1216,7 @@ export default class Host extends lib.Link {
 		await this.config.save();
 		// Save host side in case host_assigned_game_port changed in the config.
 		await Promise.all(
-			[...this.instanceInfos.values()].map(instanceInfo => instanceInfo.config.save())
+			[...this.assignedInstances.values()].map(instance => instance.config.save())
 		);
 
 		try {

packages/host/src/Instance.ts

@@ -1,7 +1,6 @@
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import path from "path";
 import pidusage from "pidusage";
-import phin from "phin";
 import util from "util";
 import type { Static } from "@sinclair/typebox";
 import { exec } from "child_process";
@@ -638,9 +637,12 @@ end`.replace(/\r?\n/g, " ");
 	}
 
 	async init(pluginInfos: lib.PluginNodeEnvInfo[]) {
+		// After changing the status this function is responsible for calling notifyExit if it throws.
 		this.notifyStatus("starting");
 		try {
+			const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest());
 			await this._loadStats();
+			await this.server.checkForUpdates(factorioVersions);
 			await this.server.init();
 		} catch (err) {
 			this.notifyExit();
@@ -753,8 +755,8 @@ end`.replace(/\r?\n/g, " ");
 	 * @param instanceDir -
 	 */
 	static async populate_folders(instanceDir: string) {
-		await fs.ensureDir(path.join(instanceDir, "script-output"));
-		await fs.ensureDir(path.join(instanceDir, "saves"));
+		await fs.mkdir(path.join(instanceDir, "script-output"), { recursive: true });
+		await fs.mkdir(path.join(instanceDir, "saves"), { recursive: true });
 	}
 
 	/**
@@ -784,7 +786,7 @@ end`.replace(/\r?\n/g, " ");
 
 		const mods = await this._host.fetchMods(modPack.mods.values());
 
-		await fs.ensureDir(this.path("mods"));
+		await fs.mkdir(this.path("mods"), { recursive: true });
 
 		// Remove all files
 		for (let entry of await fs.readdir(this.path("mods"), { withFileTypes: true })) {
@@ -823,12 +825,12 @@ end`.replace(/\r?\n/g, " ");
 		}
 
 		// Write mod-list.json
-		await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({
+		await fs.writeFile(this.path("mods", "mod-list.json"), JSON.stringify({
 			mods: [...this.activeModPack.mods.values()],
 		}, null, "\t"));
 
 		// Write mod-settings.dat
-		await fs.outputFile(this.path("mods", "mod-settings.dat"), this.activeModPack.toModSettingsDat());
+		await fs.writeFile(this.path("mods", "mod-settings.dat"), this.activeModPack.toModSettingsDat());
 	}
 
 	/**
@@ -843,7 +845,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_adminlist") !== "disabled") {
 			this.logger.verbose("Writing server-adminlist.json");
-			lib.safeOutputFile(
+			await lib.safeOutputFile(
 				this.server.writePath("server-adminlist.json"),
 				JSON.stringify([...this._host.adminlist], null, "\t")
 			);
@@ -851,7 +853,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_banlist") !== "disabled") {
 			this.logger.verbose("Writing server-banlist.json");
-			lib.safeOutputFile(
+			await lib.safeOutputFile(
 				this.server.writePath("server-banlist.json"),
 				JSON.stringify([...this._host.banlist].map(
 					([username, reason]) => ({ username, reason })
@@ -861,10 +863,21 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_whitelist") !== "disabled") {
 			this.logger.verbose("Writing server-whitelist.json");
-			lib.safeOutputFile(
+			await lib.safeOutputFile(
 				this.server.writePath("server-whitelist.json"),
 				JSON.stringify([...this._host.whitelist], null, "\t")
 			);
+		} else {
+			// Always create the whitelist even if unused
+			// This allows it to be watched for changes without a server restart
+			try {
+				// wx will throw an error if the file already exists to prevent overwrite
+				await fs.writeFile(this.server.writePath("server-whitelist.json"), "\n[]\n", { flag: "wx" });
+			} catch (err: any) {
+				if (err.code !== "EEXIST") {
+					throw err;
+				}
+			}
 		}
 
 		await this.syncMods();
@@ -908,7 +921,12 @@ end`.replace(/\r?\n/g, " ");
 				now.getUTCMinutes().toLocaleString("en", { minimumIntegerDigits: 2 }),
 				saveName,
 			);
-			await fs.copy(this.path("saves", saveName), this.path("saves", newName));
+			// This will overwrite the save if this runs twice in the same minute.
+			await fs.copyFile(
+				this.path("saves", saveName),
+				this.path("saves", newName),
+				fs.constants.COPYFILE_FICLONE,
+			);
 			saveName = newName;
 		}
 
@@ -1290,17 +1308,18 @@ end`.replace(/\r?\n/g, " ");
 			let url = new URL(this._host.config.get("host.controller_url"));
 			url.pathname += "api/upload-export";
 			url.searchParams.set("mod_pack_id", String(this.activeModPack.id));
-			let response = await phin({
-				url, method: "PUT",
-				data: content,
-				core: { ca: this._host.tlsCa } as object,
+			let response = await fetch(url, {
+				method: "PUT",
+				body: content,
+				duplex: "half",
 				headers: {
 					"Content-Type": "application/zip",
 					"x-access-token": this._host.config.get("host.controller_token"),
 				},
 			});
-			if (response.statusCode !== 200) {
-				throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`);
+			if (response.status !== 200) {
+				const body = Buffer.from(await response.arrayBuffer());
+				throw Error(`Upload failed: ${response.status} ${response.statusText}: ${body.toString()}`);
 			}
 
 		} finally {

packages/host/src/export.ts

@@ -1,19 +1,16 @@
 // Export of item icons and locale
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import path from "path";
-import Jimp from "jimp";
+import { Jimp, JimpMime, type JimpInstance } from "jimp";
 import JSZip from "jszip";
 
 import * as lib from "@clusterio/lib";
 import * as libBuildMod from "@clusterio/lib/dist/node/build_mod";
 import type { FactorioServer } from "./server";
+import { definesPrototypesFallback } from "./export_fallback";
 
-interface Prototype {
-	[index: string]: unknown,
-	type: string,
-	name: string,
-}
 interface SimpleIconSpecification {
+	icons?: never;
 	icon: string;
 	icon_size?: number;
 	icon_mipmaps?: number;
@@ -32,11 +29,9 @@ interface LayeredIconSpecification {
 	icon_mipmaps?: number;
 }
 type IconSpecification = LayeredIconSpecification | SimpleIconSpecification;
-type ItemPrototype = Prototype & IconSpecification;
+type PrototypeWithIcon = lib.FactorioPrototypeBase & IconSpecification;
 
 
-type Prototypes = Record<string, Record<string, Prototype>>
-
 /**
  * Generate the export mod needed for exportData
  *
@@ -69,7 +64,7 @@ async function generateExportMod(server: FactorioServer) {
 	});
 }
 
-let zipCache = new Map();
+let zipCache = new Map<string, JSZip>();
 async function loadZip(server: FactorioServer, modVersions: Map<string, string>, mod: string) {
 	let modVersion = modVersions.get(mod);
 	if (!modVersion) {
@@ -83,7 +78,7 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
 		zipCache.set(zipPath, zip);
 	}
 
-	return zip.folder(lib.findRoot(zip));
+	return zip.folder(lib.findRoot(zip))!;
 }
 
 /**
@@ -98,7 +93,8 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
 async function loadFile(server: FactorioServer, modVersions: Map<string, string>, modPath: string) {
 	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
 	if (!match) {
-		throw new Error(`Bad mod path ${match}`);
+		server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`);
+		return null;
 	}
 
 	let [, mod, filePath] = match;
@@ -133,7 +129,8 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
 	return await file.async("nodebuffer");
 }
 
-type IconCache = Map<string, Jimp | null>;
+type Image = JimpInstance;
+type IconCache = Map<string, Image | null>;
 async function loadIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
@@ -146,12 +143,11 @@ async function loadIcon(
 	if (icon === undefined) {
 		let fileContent = await loadFile(server, modVersions, iconPath);
 		if (fileContent) {
-			icon = await Jimp.read(fileContent);
-			icon.crop(0, 0, iconSize, iconSize);
+			icon = await Jimp.fromBuffer(fileContent) as Image;
+			icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize });
 			iconCache.set(iconPath, icon);
 		} else {
 			icon = null;
-			server._logger.warn(`${iconPath} not found`);
 		}
 		iconCache.set(iconPath, icon);
 	}
@@ -184,7 +180,7 @@ async function loadLayeredIcon(
 	iconCache: IconCache,
 ) {
 	let baseLayerSize = (item.icons[0].icon_size || item.icon_size) ?? 64;
-	let icon = await Jimp.create(size, size);
+	let icon = new Jimp({ width: size, height: size });
 
 	// The scaling factor of the base layer
 	let baseLayerScale = item.icons[0].scale || 32 / baseLayerSize;
@@ -246,7 +242,7 @@ async function loadLayeredIcon(
 	return icon;
 }
 
-function fixIcons(item: ItemPrototype) {
+function fixIcons(item: PrototypeWithIcon) {
 	const icons = item.icons;
 	if (typeof icons === "object" && !(icons instanceof Array) && icons !== null) {
 		// It's possible to specify icons as an object with arbitrary keys
@@ -258,109 +254,146 @@ function fixIcons(item: ItemPrototype) {
 	return item;
 }
 
-const itemTypes = new Set([
+const prototypesWithIcon = [
 	"item",
-	"ammo",
-	"capsule",
-	"gun",
-	"item-with-entity-data",
-	"item-with-label",
-	"item-with-inventory",
-	"blueprint-book",
-	"item-with-tags",
-	"selection-tool",
-	"blueprint",
-	"copy-paste-tool",
-	"deconstruction-item",
-	"upgrade-item",
-	"module",
-	"rail-planner",
-	"spidertron-remote",
-	"tool",
-	"armor",
-	"mining-tool",
-	"repair-tool",
-
-	// XXX Bad hack to get icons for fluids. These should be treated as a
-	// separate namespace from items as they may be named the same as items.
 	"fluid",
-	// TODO consider adding virtual signals and recipes
-]);
+	"recipe",
+	"virtual-signal",
+	"technology",
+	"space-location",
+	"quality",
+	"entity",
+] as const;
 
-function filterItems(prototypes: Prototypes): ItemPrototype[] {
+function filterAndFixPrototypesWithIcons(prototypes: lib.FactorioPrototypes, allowedTypes: Set<string>) {
 	return Object.entries(prototypes)
-		.filter(([type, _]) => itemTypes.has(type))
-		.flatMap(([_, typePrototypes]) => Object.values(typePrototypes) as ItemPrototype[])
-		.map(fixIcons)
-	;
+		.filter(([type]) => allowedTypes.has(type))
+		.flatMap(([_, typePrototypes]) => Object.values(typePrototypes) as PrototypeWithIcon[])
+		.map(fixIcons);
 }
 
-/**
- * Export item icons and data
- *
- * Assembles and packs the icons for the item prototypes given into a single
- * spritesheet and json file with meta data.
- *
- * @param server -
- *     The server to generate the export mod for.
- * @param modVersions -
- *     Mapping of mod name to versions to get icons from.
- * @param items - Array of item prototypes.
- * @return Item spritesheet and metadata.
- * @internal
- */
-async function exportItems(server: FactorioServer, modVersions: Map<string, string>, items: ItemPrototype[]) {
+/** Shared state for building a unified spritesheet across all categories. */
+interface SheetState {
+	sheet: Image;
+	metadata: Record<string, lib.ExportMetadataEntry[]>;
+	iconCache: IconCache;
+	simpleIcons: Map<string, number>;
+	pos: number;
+}
 
-	// Size to render icons at
-	let size = 32;
+const SHEET_ICON_SIZE = 32;
+const SHEET_WIDTH = 1024;
 
-	// Width of spritesheet
-	let width = 1024;
+async function createSheetState(): Promise<SheetState> {
+	return {
+		sheet: new Jimp({ width: SHEET_WIDTH, height: SHEET_ICON_SIZE }),
+		metadata: Object.create(null),
+		iconCache: new Map(),
+		simpleIcons: new Map(),
+		pos: 0,
+	};
+}
 
-	let rows = Math.ceil(items.length / (width / size));
-	let iconSheet = await Jimp.create(width, rows * size);
-	let itemData = new Map();
-	let pos = 0;
+/** Grow the spritesheet vertically if needed to fit another row. */
+async function growSheet(state: SheetState) {
+	const row = Math.floor(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
+	const neededHeight = (row + 1) * SHEET_ICON_SIZE;
+	if (neededHeight > state.sheet.bitmap.height) {
+		const extended = new Jimp({ width: SHEET_WIDTH, height: neededHeight });
+		extended.composite(state.sheet, 0, 0);
+		(state.sheet as any).bitmap = (extended as any).bitmap;
+	}
+}
 
-	let iconCache: IconCache = new Map();
-	let simpleIcons = new Map();
-	for (let item of items) {
-		let icon: Jimp | null = null;
-		let iconPos: number | undefined;
-		if (item.icons) {
-			icon = await loadLayeredIcon(server, modVersions, item as LayeredIconSpecification, size, iconCache);
-			iconPos = pos;
+async function exportIcon(
+	server: FactorioServer,
+	modVersions: Map<string, string>,
+	state: SheetState,
+	prototype: PrototypeWithIcon,
+) {
+	let icon: Image | null = null;
+	let iconPos: number | undefined;
+	let iconData: lib.ExportMetadataEntry["icon"];
+	let iconPath: string | undefined;
 
-		} else {
-			iconPos = simpleIcons.get(item.icon);
-			if (iconPos === undefined) {
-				icon = await loadSimpleIcon(server, modVersions, item as SimpleIconSpecification, size, iconCache);
-				if (icon) {
-					iconPos = pos;
-					simpleIcons.set(item.icon, pos);
-				}
+	if (prototype.icons) {
+		icon = await loadLayeredIcon(
+			server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
+		);
+		iconPos = state.pos;
+		iconPath = prototype.icons[0].icon;
+
+	} else if (prototype.icon) {
+		iconPos = state.simpleIcons.get(prototype.icon);
+		iconPath = prototype.icon;
+		if (iconPos === undefined) {
+			icon = await loadSimpleIcon(
+				server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
+			);
+			if (icon) {
+				iconPos = state.pos;
+				state.simpleIcons.set(prototype.icon, state.pos);
 			}
 		}
-
-		if (iconPos !== undefined) {
-			itemData.set(item.name, {
-				x: iconPos * size % width,
-				y: Math.floor(iconPos / (width / size)) * size,
-				size,
-				localised_name: item.localised_name,
-				localised_description: item.localised_description,
-			});
-		}
-
-		if (icon) {
-			iconSheet.composite(icon, pos * size % width, Math.floor(pos / (width / size)) * size);
-			pos += 1;
-		}
 	}
 
-	iconSheet.crop(0, 0, width, Math.ceil(pos / (width / size)) * size);
+	if (iconPos !== undefined) {
+		iconData = {
+			x: iconPos * SHEET_ICON_SIZE % SHEET_WIDTH,
+			y: Math.floor(iconPos / (SHEET_WIDTH / SHEET_ICON_SIZE)) * SHEET_ICON_SIZE,
+			size: SHEET_ICON_SIZE,
+			path: iconPath,
+		};
+	}
 
-	return { iconSheet, itemData };
+	if (icon) {
+		await growSheet(state);
+		state.sheet.composite(
+			icon,
+			state.pos * SHEET_ICON_SIZE % SHEET_WIDTH,
+			Math.floor(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE)) * SHEET_ICON_SIZE,
+		);
+		state.pos += 1;
+	}
+
+	return iconData;
+}
+
+/**
+ * Export prototype icons for a single base prototype onto the shared spritesheet.
+ *
+ * @param server - The server to load icons from.
+ * @param modVersions - Mapping of mod name to version.
+ * @param prototypes - Array of prototypes with icon to export for this base type.
+ * @param baseName - name of base type (used in metadata).
+ * @param state - Shared spritesheet state (mutated in place).
+ * @returns Number of icons exported for this base type.
+ * @internal
+ */
+async function exportIcons(
+	server: FactorioServer,
+	modVersions: Map<string, string>,
+	prototypes: PrototypeWithIcon[],
+	baseName: string,
+	state: SheetState,
+) {
+	const metadata = state.metadata[baseName] ?? [];
+	state.metadata[baseName] ??= metadata;
+	let count = 0;
+	for (let prototype of prototypes) {
+		const icon = await exportIcon(server, modVersions, state, prototype);
+		if (icon) {
+			count += 1;
+		}
+		metadata.push({
+			name: prototype.name,
+			type: prototype.type,
+			localised_name: prototype.localised_name,
+			icon,
+		});
+	}
+
+	return count;
 }
 
 /**
@@ -431,7 +464,7 @@ async function exportLocale(
 }
 
 /**
- * Export the locale and item icons for the given factorio server
+ * Export locale, icons and metadata for the given factorio server
  *
  * @param server - The server to export the data from.
  * @returns zip file with exported data.
@@ -439,12 +472,13 @@ async function exportLocale(
 export async function exportData(server: FactorioServer) {
 	await generateExportMod(server);
 
-	let settings: Prototypes = {};
-	let prototypes: Prototypes = {};
+	let settings: lib.ExportSettings = {};
+	let prototypes: lib.ExportPrototypes = {};
 	let modVersions = new Map();
 	let modOrder: string[] = [];
+	let defines: lib.FactorioDefines | undefined;
 
-	function add(obj: Prototypes, prototype: Prototype) {
+	function add(obj: lib.FactorioPrototypes, prototype: lib.FactorioPrototypeBase) {
 		if (!Object.prototype.hasOwnProperty.call(obj, prototype.type)) {
 			obj[prototype.type] = {};
 		}
@@ -463,6 +497,7 @@ export async function exportData(server: FactorioServer) {
 		server._logger.error(`Unable to find ${name} in settings prototypes`);
 	});
 	server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); });
+	server.on("ipc-defines", data => { defines = data; });
 	server.on("output", parsed => {
 		if (parsed.format === "seconds" && parsed.type === "generic") {
 			let match = /^Checksum of (.*): \d+$/.exec(parsed.message);
@@ -483,6 +518,15 @@ export async function exportData(server: FactorioServer) {
 		throw new Error("No prototypes got exported");
 	}
 
+	if (!defines) {
+		throw new Error("No defines got exported");
+	}
+
+	// compat with versions prior to 1.1.31
+	if (!defines.prototypes) {
+		defines.prototypes = definesPrototypesFallback;
+	}
+
 	// Some mod authors put leading zeros into the versions of their zip files.
 	let splitter = /^(.*)_(\d+)\.(\d+)\.(\d+)\.zip?$/;
 	for (let entry of await fs.readdir(server.writePath("mods"))) {
@@ -505,18 +549,41 @@ export async function exportData(server: FactorioServer) {
 		}
 	}
 
-	let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes));
 	let locale = await exportLocale(server, modVersions, modOrder, "en");
 
 	// Free up the memory used by zip files loaded during the export.
 	zipCache.clear();
 
 	let zip = new JSZip();
-	zip.file("export/settings.json", JSON.stringify(settings));
-	zip.file("export/prototypes.json", JSON.stringify(prototypes));
-	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
-	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
-	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
+	zip.file("export/settings.json", JSON.stringify(settings satisfies lib.ExportSettings));
+	zip.file("export/prototypes.json", JSON.stringify(prototypes satisfies lib.ExportPrototypes));
+	zip.file("export/locale.json", JSON.stringify([...locale.entries()] satisfies lib.ExportLocale));
+	zip.file("export/defines.json", JSON.stringify(defines satisfies lib.ExportDefines));
+
+	// Build a single unified spritesheet across all prototypes with icons.
+	const state = await createSheetState();
+	for (const baseName of prototypesWithIcon) {
+		if (!Object.prototype.hasOwnProperty.call(defines.prototypes, baseName)) {
+			continue;
+		}
+		const subTypeNames = new Set(Object.keys(defines.prototypes[baseName]));
+		const prototypesWithIcons = filterAndFixPrototypesWithIcons(prototypes, subTypeNames);
+
+		if (prototypesWithIcons.length === 0) {
+			continue;
+		}
+		const count = await exportIcons(server, modVersions, prototypesWithIcons, baseName, state);
+		server._logger.info(`Exported ${count} ${baseName} icons`);
+	}
+
+	// Crop sheet to actual used height and write single spritesheet + metadata.
+	const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
+	state.sheet.crop({ x: 0, y: 0, w: SHEET_WIDTH, h: Math.max(usedRows * SHEET_ICON_SIZE, 1) });
+	zip.file("export/spritesheet.png", await state.sheet.getBuffer(JimpMime.png));
+	zip.file("export/metadata.json", JSON.stringify(state.metadata satisfies lib.ExportMetadata));
+
+	server._logger.info(`Export complete: ${state.pos} icons on ${usedRows} row(s)`);
+
 	return zip;
 }
 

packages/host/src/export_fallback.ts

@@ -0,0 +1,294 @@
+// defines.prototypes from Factorio 1.1.31, the first version that has this.
+export const definesPrototypesFallback = {
+	"achievement": {
+		"achievement": 0,
+		"build-entity-achievement": 0,
+		"combat-robot-count": 0,
+		"construct-with-robots-achievement": 0,
+		"deconstruct-with-robots-achievement": 0,
+		"deliver-by-robots-achievement": 0,
+		"dont-build-entity-achievement": 0,
+		"dont-craft-manually-achievement": 0,
+		"dont-use-entity-in-energy-production-achievement": 0,
+		"finish-the-game-achievement": 0,
+		"group-attack-achievement": 0,
+		"kill-achievement": 0,
+		"player-damaged-achievement": 0,
+		"produce-achievement": 0,
+		"produce-per-hour-achievement": 0,
+		"research-achievement": 0,
+		"train-path-achievement": 0,
+	},
+	"ambient-sound": {
+		"ambient-sound": 0,
+	},
+	"ammo-category": {
+		"ammo-category": 0,
+	},
+	"animation": {
+		"animation": 0,
+	},
+	"autoplace-control": {
+		"autoplace-control": 0,
+	},
+	"custom-input": {
+		"custom-input": 0,
+	},
+	"damage-type": {
+		"damage-type": 0,
+	},
+	"decorative": {
+		"optimized-decorative": 0,
+	},
+	"editor-controller": {
+		"editor-controller": 0,
+	},
+	"entity": {
+		"accumulator": 0,
+		"ammo-turret": 0,
+		"arithmetic-combinator": 0,
+		"arrow": 0,
+		"artillery-flare": 0,
+		"artillery-projectile": 0,
+		"artillery-turret": 0,
+		"artillery-wagon": 0,
+		"assembling-machine": 0,
+		"beacon": 0,
+		"beam": 0,
+		"boiler": 0,
+		"burner-generator": 0,
+		"car": 0,
+		"cargo-wagon": 0,
+		"character": 0,
+		"character-corpse": 0,
+		"cliff": 0,
+		"combat-robot": 0,
+		"constant-combinator": 0,
+		"construction-robot": 0,
+		"container": 0,
+		"corpse": 0,
+		"curved-rail": 0,
+		"decider-combinator": 0,
+		"deconstructible-tile-proxy": 0,
+		"electric-energy-interface": 0,
+		"electric-pole": 0,
+		"electric-turret": 0,
+		"entity-ghost": 0,
+		"explosion": 0,
+		"fire": 0,
+		"fish": 0,
+		"flame-thrower-explosion": 0,
+		"fluid-turret": 0,
+		"fluid-wagon": 0,
+		"flying-text": 0,
+		"furnace": 0,
+		"gate": 0,
+		"generator": 0,
+		"heat-interface": 0,
+		"heat-pipe": 0,
+		"highlight-box": 0,
+		"infinity-container": 0,
+		"infinity-pipe": 0,
+		"inserter": 0,
+		"item-entity": 0,
+		"item-request-proxy": 0,
+		"lab": 0,
+		"lamp": 0,
+		"land-mine": 0,
+		"leaf-particle": 0,
+		"linked-belt": 0,
+		"linked-container": 0,
+		"loader": 0,
+		"loader-1x1": 0,
+		"locomotive": 0,
+		"logistic-container": 0,
+		"logistic-robot": 0,
+		"market": 0,
+		"mining-drill": 0,
+		"offshore-pump": 0,
+		"particle": 0,
+		"particle-source": 0,
+		"pipe": 0,
+		"pipe-to-ground": 0,
+		"player-port": 0,
+		"power-switch": 0,
+		"programmable-speaker": 0,
+		"projectile": 0,
+		"pump": 0,
+		"radar": 0,
+		"rail-chain-signal": 0,
+		"rail-remnants": 0,
+		"rail-signal": 0,
+		"reactor": 0,
+		"resource": 0,
+		"roboport": 0,
+		"rocket-silo": 0,
+		"rocket-silo-rocket": 0,
+		"rocket-silo-rocket-shadow": 0,
+		"simple-entity": 0,
+		"simple-entity-with-force": 0,
+		"simple-entity-with-owner": 0,
+		"smoke": 0,
+		"smoke-with-trigger": 0,
+		"solar-panel": 0,
+		"speech-bubble": 0,
+		"spider-leg": 0,
+		"spider-vehicle": 0,
+		"splitter": 0,
+		"sticker": 0,
+		"storage-tank": 0,
+		"straight-rail": 0,
+		"stream": 0,
+		"tile-ghost": 0,
+		"train-stop": 0,
+		"transport-belt": 0,
+		"tree": 0,
+		"turret": 0,
+		"underground-belt": 0,
+		"unit": 0,
+		"unit-spawner": 0,
+		"wall": 0,
+	},
+	"equipment": {
+		"active-defense-equipment": 0,
+		"battery-equipment": 0,
+		"belt-immunity-equipment": 0,
+		"energy-shield-equipment": 0,
+		"generator-equipment": 0,
+		"movement-bonus-equipment": 0,
+		"night-vision-equipment": 0,
+		"roboport-equipment": 0,
+		"solar-panel-equipment": 0,
+	},
+	"equipment-category": {
+		"equipment-category": 0,
+	},
+	"equipment-grid": {
+		"equipment-grid": 0,
+	},
+	"fluid": {
+		"fluid": 0,
+	},
+	"font": {
+		"font": 0,
+	},
+	"fuel-category": {
+		"fuel-category": 0,
+	},
+	"god-controller": {
+		"god-controller": 0,
+	},
+	"gui-style": {
+		"gui-style": 0,
+	},
+	"item": {
+		"ammo": 0,
+		"armor": 0,
+		"blueprint": 0,
+		"blueprint-book": 0,
+		"capsule": 0,
+		"copy-paste-tool": 0,
+		"deconstruction-item": 0,
+		"gun": 0,
+		"item": 0,
+		"item-with-entity-data": 0,
+		"item-with-inventory": 0,
+		"item-with-label": 0,
+		"item-with-tags": 0,
+		"mining-tool": 0,
+		"module": 0,
+		"rail-planner": 0,
+		"repair-tool": 0,
+		"selection-tool": 0,
+		"spidertron-remote": 0,
+		"tool": 0,
+		"upgrade-item": 0,
+	},
+	"item-group": {
+		"item-group": 0,
+	},
+	"item-subgroup": {
+		"item-subgroup": 0,
+	},
+	"map-gen-presets": {
+		"map-gen-presets": 0,
+	},
+	"map-settings": {
+		"map-settings": 0,
+	},
+	"module-category": {
+		"module-category": 0,
+	},
+	"mouse-cursor": {
+		"mouse-cursor": 0,
+	},
+	"noise-expression": {
+		"noise-expression": 0,
+	},
+	"noise-layer": {
+		"noise-layer": 0,
+	},
+	"particle": {
+		"optimized-particle": 0,
+	},
+	"recipe": {
+		"recipe": 0,
+	},
+	"recipe-category": {
+		"recipe-category": 0,
+	},
+	"resource-category": {
+		"resource-category": 0,
+	},
+	"shortcut": {
+		"shortcut": 0,
+	},
+	"sound": {
+		"sound": 0,
+	},
+	"spectator-controller": {
+		"spectator-controller": 0,
+	},
+	"sprite": {
+		"sprite": 0,
+	},
+	"technology": {
+		"technology": 0,
+	},
+	"tile": {
+		"tile": 0,
+	},
+	"tile-effect": {
+		"tile-effect": 0,
+	},
+	"tips-and-tricks-item": {
+		"tips-and-tricks-item": 0,
+	},
+	"tips-and-tricks-item-category": {
+		"tips-and-tricks-item-category": 0,
+	},
+	"trigger-target-type": {
+		"trigger-target-type": 0,
+	},
+	"trivial-smoke": {
+		"trivial-smoke": 0,
+	},
+	"tutorial": {
+		"tutorial": 0,
+	},
+	"utility-constants": {
+		"utility-constants": 0,
+	},
+	"utility-sounds": {
+		"utility-sounds": 0,
+	},
+	"utility-sprites": {
+		"utility-sprites": 0,
+	},
+	"virtual-signal": {
+		"virtual-signal": 0,
+	},
+	"wind-sound": {
+		"wind-sound": 0,
+	},
+};

packages/host/src/patch.ts

@@ -1,7 +1,7 @@
 // Library for patching Factorio saves with scenario code.
 
 import events from "events";
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import JSZip from "jszip";
 import path from "path";
 import semver from "semver";
@@ -93,7 +93,9 @@ export class SaveModule {
 	static async fromPlugin(plugin: BaseInstancePlugin) {
 		let pluginPackagePath = require.resolve(path.posix.join(plugin.info.requirePath, "package.json"));
 		let moduleDirectory = path.join(path.dirname(pluginPackagePath), "module");
-		if (!await fs.pathExists(moduleDirectory)) {
+		try {
+			await fs.access(moduleDirectory);
+		} catch (err: any) {
 			return null;
 		}
 
@@ -431,15 +433,13 @@ export async function patch(savePath: string, modules: SaveModule[]) {
 	root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
 
 	// Write back the save
-	let tempSavePath = savePath.replace(/(\.zip)?$/, ".tmp.zip");
 	let stream = zip.generateNodeStream({ compression: "DEFLATE" });
-	let fd = await fs.open(tempSavePath, "w");
+	const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
 	try {
-		let pipe = stream.pipe(fs.createWriteStream("", { fd, autoClose: false }));
+		let pipe = stream.pipe(writeStream);
 		await events.once(pipe, "finish");
-		await fs.fsync(fd);
 	} finally {
-		await fs.close(fd);
+		writeStream.destroy();
 	}
 	await fs.rename(tempSavePath, savePath);
 }

packages/host/src/server.ts

@@ -1,7 +1,9 @@
 // Factorio Server interface
-import fs from "fs-extra";
+import fs from "node:fs/promises";
+import { type FSWatcher, watch as fsWatch, writeFileSync } from "node:fs";
 import child_process from "child_process";
 import path from "path";
+import JSZip from "jszip";
 import events from "events";
 import util from "util";
 import crypto from "crypto";
@@ -9,31 +11,33 @@ import { Rcon } from "rcon-client";
 
 import * as lib from "@clusterio/lib";
 
+const execFileAsync = util.promisify(child_process.execFile);
 
 /**
  * Determines the version of Factorio the datadir is pointing to by
  * reading the changelog.txt in it.
  *
- * @param changelogPath - Path to changelog.txt.
- * @internal
+ * @param factorioDir - Path factorio dir containing data/changelog.txt.
  */
-async function getVersion(changelogPath: string) {
+async function getFactorioVersion(factorioDir: string) {
 	let changelog;
+	const changelogPath = path.join(factorioDir, "data", "changelog.txt");
 	try {
 		changelog = await fs.readFile(changelogPath, "utf-8");
 	} catch (err: any) {
-		if (err.code === "ENOENT") {
+		if (err.code === "ENOENT" || err.code === "ENOTDIR") {
 			return null;
 		}
 		throw err;
 	}
 
-	for (let line of changelog.split(/[\r\n]+/)) {
-		let index = line.indexOf(":");
+	for (const line of changelog.split(/[\r\n]+/)) {
+		const index = line.indexOf(":");
 		if (index !== -1) {
-			let name = line.slice(0, index).trim();
+			const name = line.slice(0, index).trim();
 			if (name.toLowerCase() === "version") {
-				return line.slice(index + 1).trim();
+				const version = line.slice(index + 1).trim();
+				return lib.isFullVersion(version) ? version : null;
 			}
 		}
 	}
@@ -66,13 +70,113 @@ function versionOrder(a: string, b: string) {
 	return 0;
 }
 
+/**
+ * Find all installed factorio versions and return them as a set
+ *
+ * @param factorioDir - Path to Factorio installation dirs.
+ * @returns Set of installed factorio versions
+ */
+async function listFactorioVersions(factorioDir: string) {
+	const versions = new Set<lib.FullVersion>();
+
+	const directVersion = await getFactorioVersion(factorioDir);
+	if (directVersion !== null) {
+		versions.add(directVersion);
+		return { direct: true, versions };
+	}
+
+	for (const name of await fs.readdir(factorioDir)) {
+		// Avoid any fancy directory detection in order to follow symlinks
+		const fullPath = path.join(factorioDir, name);
+		const version = await getFactorioVersion(fullPath);
+		if (version === null) {
+			continue;
+		}
+
+		versions.add(version);
+	}
+
+	return { direct: false, versions };
+}
+
+/**
+ * Downloads a ZIP file from a URL using native fetch and extracts it
+ *
+ * @param url - The URL of the ZIP file
+ * @param targetDir - Directory to extract the ZIP into
+ * @throws Network error if status of request is not ok
+ */
+async function downloadAndExtractZip(url: string, targetDir: string) {
+	const res = await fetch(url);
+	if (!res.ok) {
+		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+	}
+
+  	// Load the ZIP with JSZip
+	const buffer = Buffer.from(await res.arrayBuffer());
+	const zip = await JSZip.loadAsync(buffer);
+
+  	// Extract each file/folder
+	await fs.mkdir(targetDir, { recursive: true });
+	await Promise.all(
+		Object.values(zip.files).map(async (entry) => {
+			const parts = entry.name.split("/").filter(Boolean);
+			const strippedPath = parts.slice(1).join("/");
+			if (!strippedPath) {
+				return; // This copies the behaviour of tar --strip-components 1
+			}
+
+			const entryPath = path.join(targetDir, strippedPath);
+			if (entry.dir) {
+				await fs.mkdir(entryPath, { recursive: true });
+			} else {
+				await fs.mkdir(path.dirname(entryPath), { recursive: true });
+				await fs.writeFile(entryPath, await entry.async("nodebuffer"));
+			}
+		})
+	);
+}
+
+/**
+ * Downloads a tar.gz file from a URL using native fetch and extracts it
+ *
+ * @param url - The URL of the ZIP file
+ * @param targetDir - Directory to extract the ZIP into
+ * @throws Network error if status of request is not ok
+ */
+async function downloadAndExtractTar(url: string, targetDir: string) {
+	const res = await fetch(url);
+	if (!res.ok) {
+		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+	}
+
+	// Write the buffer to a temp file
+	const tmpFilePath = `${targetDir}.tmp`;
+	const buffer = Buffer.from(await res.arrayBuffer());
+	await fs.writeFile(tmpFilePath, buffer);
+	await fs.mkdir(targetDir, { recursive: true });
+
+	// Execute tar CLI to extract
+	try {
+		await execFileAsync("tar", ["-xf", tmpFilePath, "-C", targetDir, "--strip-components", "1"]);
+	} catch (err: any) {
+		if (err.code === "ENOENT") {
+			throw new Error("error executing tar command, do you have 'xz-utils' installed?");
+		}
+		throw err;
+	} finally {
+		// Clean up temp file
+		await fs.rm(tmpFilePath);
+	}
+}
+
 /**
  * Find factorio data directory of the given version
  *
  * Searches the given factorio dir for an installation of Factorio of
  * the target version.
  *
- * @param factorioDir - Path to Factorio installation dir(s).
+ * @param factorioDir - Path to Factorio installation dirs.
  * @param targetVersion -
  *     Version to look for, supports the special value "latest" for the
  *     latest version available.
@@ -80,40 +184,30 @@ function versionOrder(a: string, b: string) {
  * @internal
  */
 async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion): Promise<[string, lib.FullVersion]> {
-
-	// There are two supported setups: having the factorio dir be the actual
-	// install directory, and having the factorio dir be a folder containing
-	// multiple install directories
-
-	let simpleVersion = await getVersion(path.join(factorioDir, "data", "changelog.txt"));
-	if (simpleVersion !== null) {
-		if (simpleVersion === targetVersion || simpleVersion.startsWith(targetVersion) || targetVersion === "latest") {
-			if (lib.isFullVersion(simpleVersion)) {
-				return [path.join(factorioDir, "data"), simpleVersion];
-			}
+	// Check if this is a direct installation dir
+	const directVersion = await getFactorioVersion(factorioDir);
+	if (directVersion !== null) {
+		if (targetVersion === "latest" || directVersion === targetVersion || directVersion.startsWith(targetVersion)) {
+			return [path.join(factorioDir, "data"), directVersion];
 		}
-
-		throw new Error(
-			`Factorio version ${targetVersion} was requested, but install directory contains ${simpleVersion}`
-		);
+		throw new Error(`Unable to find Factorio version ${targetVersion}`);
 	}
 
-	let versions = new Map<lib.FullVersion, string>();
-	for (let entry of await fs.readdir(factorioDir, { withFileTypes: true })) {
-		if (!entry.isDirectory()) {
-			continue;
-		}
-
-		let version = await getVersion(path.join(factorioDir, entry.name, "data", "changelog.txt"));
-		if (version === null || !lib.isFullVersion(version)) {
+	// Otherwise assume it is a directory of multiple versions
+	const versions = new Map<lib.FullVersion, string>();
+	for (const name of await fs.readdir(factorioDir)) {
+		// Avoid any fancy directory detection in order to follow symlinks
+		const fullPath = path.join(factorioDir, name);
+		const version = await getFactorioVersion(fullPath);
+		if (version === null) {
 			continue;
 		}
 
 		if (version === targetVersion) {
-			return [path.join(factorioDir, entry.name, "data"), version];
+			return [path.join(factorioDir, name, "data"), version];
 		}
 
-		versions.set(version, entry.name);
+		versions.set(version, name);
 	}
 
 	if (!versions.size) {
@@ -538,7 +632,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 
 	// Due to inconsistencies in the factorio api, we must manually watch the whitelist
 	// https://forums.factorio.com/viewtopic.php?t=123673
-	_whitelistWatcher: fs.FSWatcher | null = null;
+	_whitelistWatcher: FSWatcher | null = null;
 	_whitelist = new Set<string>();
 
 	_logger: lib.Logger;
@@ -792,9 +886,9 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 
 		const filePath = this.writePath("server-whitelist.json");
 		try {
-			this._whitelistWatcher = fs.watch(filePath);
+			this._whitelistWatcher = fsWatch(filePath);
 		} catch (err: any) {
-			this._logger.error(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`);
+			this._logger.warn(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`);
 			return;
 		}
 
@@ -814,7 +908,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 
 			let newWhitelistJson;
 			try {
-				newWhitelistJson = await fs.readJSON(filePath);
+				newWhitelistJson = JSON.parse(await fs.readFile(filePath, "utf8"));
 			} catch (err: any) {
 				this._logger.error(`Unable to read whitelist, bidirectional sync will not be available:\n${err}`);
 				return;
@@ -871,6 +965,41 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		this._state = "init";
 	}
 
+	/**
+	 * Check of updated versions to download
+	 *
+	 * Should be called before init for the new version to be detected during init
+	 */
+	async checkForUpdates(availableVersions: lib.ExternalFactorioVersion[]) {
+		const targetVersion = this._targetVersion;
+		const latestVersion = availableVersions
+			.filter(v => (
+				targetVersion === "latest" || v.version === targetVersion || v.version.startsWith(targetVersion)
+			))
+			.sort((v1, v2) => versionOrder(v1.version, v2.version))
+			[0];
+
+		if (!latestVersion) {
+			return;
+		}
+
+		const installedVersions = await listFactorioVersions(this._factorioDir);
+		if (!installedVersions.versions.has(latestVersion.version)) {
+			const v = latestVersion.version;
+			// Can't download if not linux or this is a direct install
+			if (installedVersions.direct || process.platform !== "linux") {
+				this._logger.info(`A newer version of factorio is available (${v}) but must be manually downloaded`);
+				return;
+			}
+
+			// Download the newer version
+			this._logger.info(`A newer version of factorio is available (${v}), starting download...`);
+			const dst = path.join(this._factorioDir, latestVersion.version);
+			await downloadAndExtractTar(latestVersion.headlessUrl, dst);
+			this._logger.info(`Downloaded factorio version ${v}`);
+		}
+	}
+
 	/**
 	 * The version of Factorio in use. This will be the actual version if
 	 * "latest" (the default) was specified as `factorioVersion` to the constructor.
@@ -1399,20 +1528,20 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		});
 		// Must be sync to allow process spawn without awaiting
 		// eslint-disable-next-line node/no-sync
-		fs.writeFileSync(this.writePath("config.ini"), content);
+		writeFileSync(this.writePath("config.ini"), content);
 	}
 
 	_writeMapSettingsSync(mapGenSettings?: object, mapSettings?: object) {
 		// Must be sync to allow process spawn without awaiting
 		if (mapGenSettings) {
 			// eslint-disable-next-line node/no-sync
-			fs.writeFileSync(
+			writeFileSync(
 				this.writePath("map-gen-settings.json"), JSON.stringify(mapGenSettings, null, "\t")
 			);
 		}
 		if (mapSettings) {
 			// eslint-disable-next-line node/no-sync
-			fs.writeFileSync(
+			writeFileSync(
 				this.writePath("map-settings.json"), JSON.stringify(mapSettings, null, "\t")
 			);
 		}
@@ -1421,9 +1550,12 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 
 
 // For testing only
-export const _getVersion = getVersion;
+export const _getFactorioVersion = getFactorioVersion;
 export const _versionOrder = versionOrder;
 export const _findVersion = findVersion;
+export const _listFactorioVersions = listFactorioVersions;
+export const _downloadAndExtractZip = downloadAndExtractZip;
+export const _downloadAndExtractTar = downloadAndExtractTar;
 export const _randomDynamicPort = randomDynamicPort;
 export const _generatePassword = generatePassword;
 export const _parseOutput = parseOutput;

packages/lib/browser.ts

@@ -2,6 +2,7 @@ export * from "./src/api";
 export * from "./src/config";
 export * from "./src/data";
 export * from "./src/errors";
+export * from "./src/export";
 export * from "./src/factorio/exchange_string";
 export * from "./src/helpers";
 export * from "./src/ini";
@@ -16,3 +17,7 @@ export * from "./src/subscriptions";
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as RateLimiter } from "./src/RateLimiter";
 export { default as ValueCache } from "./src/ValueCache";
+export { default as isDeepStrictEqual } from "./src/is_deep_strict_equal";
+
+import { checkSingletonImport } from "./src/check_singleton_import";
+checkSingletonImport(import.meta.url);

packages/lib/build_mod.js

@@ -1,9 +1,7 @@
 /* eslint-disable no-console */
 "use strict";
-const events = require("events");
-const fs = require("fs-extra");
+const fs = require("node:fs/promises");
 const JSZip = require("jszip");
-const klaw = require("klaw");
 const path = require("path");
 const stream = require("stream");
 const util = require("util");
@@ -14,30 +12,36 @@ const finished = util.promisify(stream.finished);
 
 async function buildMod(args, info, modName) {
 	if (args.build) {
-		await fs.ensureDir(args.outputDir);
+		await fs.mkdir(args.outputDir, { recursive: true });
 		modName = args.modName ?? `${info.name}_${info.version}`;
 
 		if (args.pack) {
 			let zip = new JSZip();
-			let walker = klaw(args.sourceDir)
-				.on("data", item => {
-					if (item.stats.isFile()) {
-						// On Windows the path created uses backslashes as the directory sepparator
-						// but the zip file needs to use forward slashes.  We can't use the posix
-						// version of relative here as it doesn't work with Windows style paths.
-						let basePath = path.relative(args.sourceDir, item.path).replace(/\\/g, "/");
-						zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
+			for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) {
+				if (dirent.isFile()) {
+					const itemPath = path.join(dirent.parentPath, dirent.name);
+					// On Windows the path created uses backslashes as the directory sepparator
+					// but the zip file needs to use forward slashes.  We can't use the posix
+					// version of relative here as it doesn't work with Windows style paths.
+					let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/");
+					// eslint-disable-next-line max-depth
+					if (basePath === "info.json") {
+						// The info.json file is overridden later.
+						continue;
 					}
-				})
-			;
-			await events.once(walker, "end");
+					zip.file(path.posix.join(modName, basePath), (await fs.open(itemPath)).createReadStream());
+				}
+			}
 
 			for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
 				let filePath = path.join(args.sourceDir, ...pathParts);
-				if (!await fs.pathExists(filePath)) {
-					throw new Error(`Additional file ${filePath} does not exist`);
+				let fileStream;
+				try {
+					fileStream = (await fs.open(filePath)).createReadStream();
+				} catch (err) {
+					throw new Error(`Error reading additional file ${filePath}`, { cause: err });
 				}
-				zip.file(path.posix.join(modName, fileName), fs.createReadStream(filePath));
+				zip.file(path.posix.join(modName, fileName), fileStream);
 			}
 			delete info.additional_files;
 
@@ -45,20 +49,28 @@ async function buildMod(args, info, modName) {
 
 			let modPath = path.join(args.outputDir, `${modName}.zip`);
 			console.log(`Writing ${modPath}`);
-			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
+			let writeStream = zip.generateNodeStream().pipe(
+				(await fs.open(modPath, "w")).createWriteStream()
+			);
 			await finished(writeStream);
 
 		} else {
 			let modDir = path.join(args.outputDir, modName);
-			if (await fs.exists(modDir)) {
-				console.log(`Removing existing build ${modDir}`);
-				await fs.remove(modDir);
-			}
+			console.log(`Removing potentially existing build ${modDir}`);
+			await fs.rm(modDir, { force: true, recursive: true, maxRetries: 10 });
 			console.log(`Building ${modDir}`);
-			await fs.copy(args.sourceDir, modDir);
+			await fs.cp(args.sourceDir, modDir, {
+				errorOnExists: true,
+				mode: fs.constants.COPYFILE_FICLONE,
+				recursive: true,
+			});
 			for (let [fileName, pathParts] of Object.entries(info.additional_files) || []) {
 				let filePath = path.join(...pathParts);
-				await fs.copy(filePath, path.join(modDir, fileName));
+				await fs.copyFile(
+					filePath,
+					path.join(modDir, fileName),
+					fs.constants.COPYFILE_EXCL | fs.constants.COPYFILE_FICLONE,
+				);
 			}
 			delete info.additional_files;
 
@@ -87,7 +99,7 @@ async function build(args) {
 				if (name === info.name) {
 					let modPath = path.join(args.outputDir, entry);
 					console.log(`Removing ${modPath}`);
-					await fs.remove(modPath);
+					await fs.rm(modPath, { force: true, recursive: true, maxRetries: 10 });
 				}
 			}
 		}

packages/lib/index.ts

@@ -9,6 +9,7 @@ export * from "./src/config";
 export * from "./src/data";
 export * from "./src/database";
 export * from "./src/errors";
+export * from "./src/export";
 export * from "./src/external";
 export * from "./src/factorio";
 export * from "./src/file_ops";
@@ -38,3 +39,7 @@ export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";
 export { default as ValueCache } from "./src/ValueCache";
+export { default as isDeepStrictEqual } from "./src/is_deep_strict_equal";
+
+import { checkSingletonImport } from "./src/check_singleton_import";
+checkSingletonImport(__filename);

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@clusterio/lib",
 	"description": "Shared library for Clusterio",
-	"version": "2.0.0-alpha.22b",
+	"version": "2.0.0-alpha.24",
 	"repository": "https://github.com/clusterio/clusterio",
 	"license": "MIT",
 	"scripts": {
@@ -19,28 +19,25 @@
 	},
 	"dependencies": {
 		"@sinclair/typebox": "catalog:",
-		"set-blocking": "catalog:",
-		"fs-extra": "catalog:",
-		"jszip": "catalog:",
+		"ajv": "^8.17.1",
 		"chalk": "catalog:",
-		"ws": "catalog:",
+		"fast-deep-equal": "^3.1.3",
+		"jszip": "catalog:",
+		"set-blocking": "catalog:",
+		"triple-beam": "^1.4.1",
 		"winston": "catalog:",
 		"winston-transport": "catalog:",
-		"fast-deep-equal": "^3.1.3",
-		"triple-beam": "^1.4.1",
-		"ajv": "^8.17.1",
-		"klaw": "^4.1.0",
+		"ws": "catalog:",
 		"yargs": "^17.7.2"
 	},
 	"publishConfig": {
 		"access": "public"
 	},
 	"devDependencies": {
-		"@types/fs-extra": "catalog:",
 		"@types/node": "catalog:",
-		"@types/ws": "catalog:",
 		"@types/set-blocking": "catalog:",
 		"@types/triple-beam": "^1.3.2",
+		"@types/ws": "catalog:",
 		"@types/yargs": "^17.0.33",
 		"typescript": "catalog:"
 	}

packages/lib/scripts/compile_validator.js

@@ -1,5 +1,5 @@
 "use strict";
-const fs = require("fs-extra");
+const fs = require("node:fs");
 const path = require("path");
 const Ajv = require("ajv");
 const standaloneCode = require("ajv/dist/standalone").default;
@@ -7,7 +7,7 @@ const standaloneCode = require("ajv/dist/standalone").default;
 /* eslint-disable node/no-sync */
 
 // Provide an empty file for message_core to load
-fs.outputFileSync(path.join(__dirname, "../dist/node/src/data/message_validate.js"), "");
+fs.writeFileSync(path.join(__dirname, "../dist/node/src/data/message_validate.js"), "");
 
 const { Message } = require("../dist/node/src/data/messages_core");
 
@@ -23,7 +23,7 @@ const ajv = new Ajv({
 const validate = ajv.compile(Message.jsonSchema);
 let moduleCode = standaloneCode(ajv, validate);
 function write(output) {
-	fs.outputFileSync(
+	fs.writeFileSync(
 		path.join(__dirname, output),
 		`// generated by scripts/compile_validator.js\n// @ts-nocheck\n/* eslint-disable */\n${moduleCode}`
 	);

packages/lib/src/LockFile.ts

@@ -1,5 +1,5 @@
-import { FileHandle, open, unlink, readFile } from "fs/promises";
-import { unlinkSync } from "fs";
+import { FileHandle, open, unlink, readFile } from "node:fs/promises";
+import { unlinkSync } from "node:fs";
 import assert from "assert";
 
 /**

packages/lib/src/ModStore.ts

@@ -1,10 +1,9 @@
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import path from "path";
 import events from "events";
 import { Static } from "@sinclair/typebox";
 import { logger } from "./logging";
-import { safeOutputFile } from "./file_ops";
-import { Writable } from "stream";
+import { downloadFile, safeOutputFile } from "./file_ops";
 import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod";
 import { ModInfo, ModPack, FullVersion, ApiVersion, ModVersionEquality } from "./data";
 
@@ -155,33 +154,6 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
 		return new this(modsDirectory, files);
 	}
 
-	/**
-	 * Stream a url's content into a file.
-	 *
-	 * @param url - Web path to stream from.
-	 * @param filePath - File path to stream to.
-	 */
-	private static async downloadFile(url: string | URL, filePath: string) {
-		try {
-			const response = await fetch(url);
-
-			if (!response.ok) {
-				throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-			}
-
-			if (!response.body) {
-				throw new Error("Response body is missing.");
-			}
-
-			const fileStream = fs.createWriteStream(filePath);
-			const writer = Writable.toWeb(fileStream);
-			await response.body.pipeTo(writer);
-		} catch (error) {
-			logger.error("Error downloading file:", error);
-			throw error;
-		}
-	}
-
 	/**
 	 * Download a matching version from a selection of available versions.
 	 *
@@ -202,27 +174,16 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
 		}
 
 		try {
-			const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`);
+			const url = new URL(selectedRelease.download_url, "https://mods.factorio.com/");
 			url.searchParams.set("username", username);
 			url.searchParams.set("token", token);
-			const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
-			const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`;
+			const filePath = path.join(this.modsDirectory, selectedRelease.file_name);
 
-			await ModStore.downloadFile(url, tempFileName);
-			await fs.rename(tempFileName, finalFileName);
+			await downloadFile(url, filePath, "overwrite");
 			await this.loadFile(selectedRelease.file_name);
 
 		} catch (err: any) {
 			logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
-			// Attempt to clean up temp file if it exists
-			try {
-				const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
-				await fs.unlink(tempFileName);
-			} catch (unlinkErr: any) {
-				if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist
-					logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`);
-				}
-			}
 		}
 	}
 

packages/lib/src/check_singleton_import.ts

@@ -0,0 +1,21 @@
+import { InstallationError } from "./errors";
+declare global {
+	// eslint-disable-next-line vars-on-top
+	var _clusterioLibImportedFrom: string;
+}
+
+export function checkSingletonImport(location: string) {
+	if (!global._clusterioLibImportedFrom) {
+		global._clusterioLibImportedFrom = location;
+	} else {
+		throw new InstallationError(`Attempt to import duplicate copy of @clusterio/lib
+
+Importing more than one copy of @clusterio/lib into the same runtime
+will break Clusterio. This typically happens when the package
+manager has installed two different versions of @clusterio/lib
+and assigned different ones to different components of Clusterio.
+
+Previously imported @clusterio/lib was located at ${_clusterioLibImportedFrom}
+This copy of @clusterio/lib is located at ${location}`);
+	}
+}

packages/lib/src/config/classes.ts

@@ -1,6 +1,6 @@
 // Configuration classes
 import { Type, Static } from "@sinclair/typebox";
-import * as fs from "fs-extra";
+import * as fs from "node:fs/promises";
 import events from "events";
 
 import isDeepStrictEqual from "../is_deep_strict_equal";
@@ -8,6 +8,7 @@ import { basicType } from "../helpers";
 import * as libSchema from "../schema";
 import { StringEnum } from "../data/composites";
 import { safeOutputFile } from "../file_ops";
+import * as validators from "./validators";
 
 const ConfigLocation = StringEnum(["controller", "host", "control"]);
 export type ConfigLocation = Static<typeof ConfigLocation>;
@@ -63,14 +64,24 @@ export const ConfigSchema = Type.Record(Type.String(), Type.Union([
 ]));
 export type ConfigSchema = Static<typeof ConfigSchema>;
 
-type FieldType = "boolean" | "string" | "number" | "object";
 type FieldValue = null | boolean | string | number | object;
-export interface FieldDefinition {
+
+type FieldType<T extends FieldValue> =
+	T extends boolean ? "boolean" :
+		T extends string ? "string" :
+			T extends number ? "number" :
+				T extends object ? "object" :
+					never;
+
+export interface FieldDefinition<
+	Type extends FieldValue = any,
+	Fields extends Record<keyof Fields, FieldValue> = any,
+> {
 	/**
 	 * string declaring the type of the config value, supports boolean,
 	 * string, number and object.
 	 */
-	type: FieldType;
+	type: FieldType<Type>;
 	/**
 	 * Text used to identify the config field in user interfaces.
 	 */
@@ -79,6 +90,16 @@ export interface FieldDefinition {
 	 * Text used to describe the config field in user interfaces.
 	 */
 	description?: string;
+	/**
+	 * Validation function ran to confirm the value is consistent with other config values.
+	 * Should throw an error if the value in invalid / inconsistent
+	 */
+	validator?: (value: Type, config: Config<Fields>) => void;
+	/**
+	 * Names the config fields that the validator depends on.
+	 * This will ensure it is checked when these fields are changed.
+	 */
+	dependsOn?: string[];
 	/**
 	 * Value to use for the autocomplete attribute in the Web UI's input element.
 	 * Useful for making browsers behave properly with credential inputs.
@@ -93,7 +114,7 @@ export interface FieldDefinition {
 	/**
 	 * True if this field can be set to null.  Defaults to false.
 	 */
-	optional?: boolean;
+	optional?: Type extends null ? true : false;
 	/**
 	 * Value this config field should take in a newly initialized config.
 	 * This can also be an function returning the value to use.
@@ -137,8 +158,10 @@ export interface FieldDefinition {
 	hidden?: boolean;
 }
 
-export type ConfigDefs<Fields> = {
-	[Prop in keyof Fields]: FieldDefinition;
+export type ConfigDefs<
+	Fields extends Record<keyof Fields, FieldValue>
+> = {
+	[Prop in keyof Fields]: FieldDefinition<Fields[Prop], Fields>;
 }
 
 function defaultValue(def: FieldDefinition) {
@@ -177,14 +200,17 @@ export enum ConfigAccess {
  * @extends events.EventEmitter
  */
 export class Config<
-	Fields extends { [Field in keyof Fields]: FieldValue },
+	Fields extends Record<keyof Fields, FieldValue>
 > extends events.EventEmitter<ConfigEvents> {
 	/**
 	 * Mapping of config name to field meta data
 	 * Note that mutating this object will lead to unexpected behaviour.
 	 */
 	declare static fieldDefinitions: ConfigDefs<any>;
-	declare ["constructor"]: typeof Config;
+	declare ["constructor"]: typeof Config<Fields>;
+
+	/** A set of common validators and validator factories */
+	static validators = validators;
 
 	/**
 	 * Handle migration between clusterio versions
@@ -204,6 +230,8 @@ export class Config<
 	private fields: Fields;
 	private _unknownFields: Record<string, FieldValue> = {};
 
+	/** Set to true during validators to prevent edits */
+	private _readonly = false; // TODO replace with "using config.asReadonly()"
 	/** Set to true when a field in the config is changed. */
 	private dirty = false;
 	/** Set to true when a 'restart required' field in the config is changed. */
@@ -244,6 +272,110 @@ export class Config<
 		}
 	}
 
+	/** A staging version of the config with changes tracked, invalided by revertStaging */
+	private _staging?: { config: Config<Fields>, changes: Set<string>};
+
+	/** Get the latest version of staging */
+	// TODO replace with "using config.startStating()"
+	get staging(): Config<Fields> {
+		if (this._staging) { return this._staging.config; }
+		const config = new this.constructor(this.location, this.fields);
+		const changes = new Set<string>();
+		config.on("fieldChanged", name => changes.add(name));
+		this._staging = { config, changes };
+		return config;
+	}
+
+	private static _fieldDependents: Map<string, {
+		name: string,
+		def: FieldDefinition & { validator: NonNullable<FieldDefinition["validator"]> }
+	}[]>;
+
+	/** Map of all fields whose validators depend on the given field */
+	static get fieldDependents(): typeof this._fieldDependents {
+		if (this._fieldDependents) {
+			return this._fieldDependents;
+		}
+
+		const dependents = new Map<string, { name: string, def: FieldDefinition }[]>();
+
+		for (const [name, def] of Object.entries(this.fieldDefinitions)) {
+			if (!def.validator || !def.dependsOn) { continue; }
+
+			for (const otherName of def.dependsOn) {
+				const fieldDependents = dependents.get(otherName);
+				if (fieldDependents) {
+					fieldDependents.push({name, def});
+				} else {
+					dependents.set(otherName, [{name, def}]);
+				}
+			}
+		}
+
+		// The type is correct because of "!def.validator", not clear why this doesn't infer
+		this._fieldDependents = dependents as any;
+		return this._fieldDependents;
+	}
+
+	/**
+	 * Coerce and validate a value according to a field definition.
+	 *
+	 * @param name - Name of the field being validated.
+	 * @param def - Definition describing the field type and constraints.
+	 * @param value - Value to validate and coerce.
+	 * @throws {InvalidValue} if the value cannot be coerced to the correct type
+	 * @returns The validated and coerced value.
+	 * @private
+	 */
+	static _coerceValue<T extends FieldDefinition>(name: string, def: T, value: unknown) {
+		// Empty strings are treated as null
+		if (value === "") {
+			value = null;
+		}
+
+		// If def is an enum then it must be a value within it
+		if (def.enum && !def.enum.includes(value)) {
+			throw new InvalidValue(`Expected one of [${def.enum.join(", ")}], not ${value}`);
+		}
+
+		// If it is a string, then attempt to coerce it
+		if (typeof value === "string") {
+			if (def.type === "boolean") {
+				if (value === "true") {
+					value = true;
+				} else if (value === "false") {
+					value = false;
+				}
+
+			} else if (def.type === "number") {
+				const numberRegExp = /^[+-]?(Infinity|\d+\.(\d+)?([eE][+-]?\d+)?|\.?\d+([eE][+-]?\d+)?)$/;
+				if (numberRegExp.test(value.trim())) {
+					value = Number.parseFloat(value);
+				}
+
+			} else if (def.type === "object") {
+				try {
+					value = JSON.parse(value);
+				} catch (err: any) {
+					throw new InvalidValue(`Error parsing value for ${name}: ${err.message}`);
+				}
+			}
+		}
+
+		// Check if it is allowed to be null
+		if (value === null) {
+			if (!def.optional) {
+				throw new InvalidValue(`Field ${name} cannot be null`);
+			}
+
+		// Check if is the correct type
+		} else if (basicType(value) !== def.type) {
+			throw new InvalidValue(`Expected type of ${name} to be ${def.type}, not ${basicType(value)}`);
+		}
+
+		return value as FieldType<T["type"]>;
+	}
+
 	/**
 	 * Check access for a field
 	 *
@@ -288,8 +420,14 @@ export class Config<
 	 * @param notify - False to skip setting dirty flags and emitting events
 	 */
 	_set(name: keyof Fields & string, value: FieldValue, notify: boolean = true) {
+		if (this._readonly) {
+			throw new Error("config is readonly and cannot have values changed");
+		}
+
 		const prev = this.fields[name];
 		this.fields[name] = value as any;
+		this.revertStaging();
+
 		if (notify && !isDeepStrictEqual(value, prev)) {
 			const def = this.constructor.fieldDefinitions[name];
 			if (def.restartRequired) {
@@ -308,7 +446,7 @@ export class Config<
 	 * @returns Instance of this config
 	 */
 	static async fromFile(location: ConfigLocation, filepath: string) {
-		const fields = await fs.readJson(filepath, { encoding: "utf8" });
+		const fields = JSON.parse(await fs.readFile(filepath, { encoding: "utf8" }));
 		return this.fromJSON(fields, location, filepath);
 	}
 
@@ -508,55 +646,50 @@ export class Config<
 	 * @throws {InvalidValue} if value is not allowed for the field.
 	 */
 	set<Field extends keyof Fields & string>(name: Field, newValue: Fields[Field], remote = this.location) {
-		let value: FieldValue = newValue;
-		let def = this.constructor.fieldDefinitions[name];
+		const def = this.constructor.fieldDefinitions[name];
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
+
 		this._checkAccess(name, def, remote, ConfigAccess.write, true);
 		if (remote !== this.location) {
 			this._checkAccess(name, def, this.location, ConfigAccess.write, true);
 		}
 
-		// Empty strings are treates as null
-		if (value === "") {
-			value = null;
-		}
+		const value = Config._coerceValue(name, def, newValue);
 
-		if (def.enum && !def.enum.includes(value)) {
-			throw new InvalidValue(`Expected one of [${def.enum.join(", ")}], not ${value}`);
-		}
-
-		if (typeof value === "string") {
-			if (def.type === "boolean") {
-				if (value === "true") {
-					value = true;
-				} else if (value === "false") {
-					value = false;
-				}
-
-			} else if (def.type === "number") {
-				let numberRegExp = /^[+-]?(Infinity|\d+\.(\d+)?([eE][+-]?\d+)?|\.?\d+([eE][+-]?\d+)?)$/;
-				if (numberRegExp.test(value.trim())) {
-					value = Number.parseFloat(value);
-				}
-
-			} else if (def.type === "object") {
+		const _readonly = this._readonly;
+		this.staging._readonly = true;
+		this._readonly = true;
+		try {
+			if (def.validator) {
 				try {
-					value = JSON.parse(value);
+					def.validator(value, this);
 				} catch (err: any) {
-					throw new InvalidValue(`Error parsing value for ${name}: ${err.message}`);
+					throw new InvalidValue(`Failed validation of ${name}: ${err.message}`);
 				}
 			}
+		} finally {
+			this._readonly = _readonly;
+			this.staging._readonly = _readonly;
 		}
 
-		if (value === null) {
-			if (!def.optional) {
-				throw new InvalidValue(`Field ${name} cannot be null`);
+		const dependents = this.constructor.fieldDependents.get(name);
+		if (dependents) {
+			// This mimics stage + commit but optimised for a single value
+			this.staging._set(name, value);
+			this.staging._readonly = true;
+			try {
+				for (const dependent of dependents) {
+					try {
+						dependent.def.validator(this.get(dependent.name as any, remote), this.staging);
+					} catch (err: any) {
+						throw new InvalidValue(`Failed validation of dependent ${dependent.name}: ${err.message}`);
+					}
+				}
+			} finally {
+				this.revertStaging();
 			}
-
-		} else if (basicType(value) !== def.type) {
-			throw new InvalidValue(`Expected type of ${name} to be ${def.type}, not ${basicType(value)}`);
 		}
 
 		this._set(name, value);
@@ -585,6 +718,7 @@ export class Config<
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
+
 		this._checkAccess(name, def, remote, ConfigAccess.write, true);
 		if (remote !== this.location) {
 			this._checkAccess(name, def, this.location, ConfigAccess.write, true);
@@ -603,6 +737,164 @@ export class Config<
 			delete updated[prop];
 		}
 
+		const _readonly = this._readonly;
+		this.staging._readonly = true;
+		this._readonly = true;
+		try {
+			if (def.validator) {
+				try {
+					def.validator(updated, this);
+				} catch (err: any) {
+					throw new InvalidValue(`Failed validation of ${name}: ${err.message}`);
+				}
+			}
+		} finally {
+			this._readonly = _readonly;
+			this.staging._readonly = _readonly;
+		}
+
+		const dependents = this.constructor.fieldDependents.get(name);
+		if (dependents) {
+			// This mimics stage + commit but optimised for a single value
+			this.staging._set(name, updated);
+			this.staging._readonly = true;
+			try {
+				for (const dependent of dependents) {
+					try {
+						dependent.def.validator(this.get(dependent.name as any, remote), this.staging);
+					} catch (err: any) {
+						throw new InvalidValue(`Failed validation of dependent ${dependent.name}: ${err.message}`);
+					}
+				}
+			} finally {
+				this.revertStaging();
+			}
+		}
+
 		this._set(name, updated);
 	}
+
+	/**
+	 * Similar to set but only validates the field type and not other constraints.
+	 * The value is stored in staging and must be committed for the change to take effect.
+	 *
+	 * @param name - Name of field to set.
+	 * @param newValue - Value to set for field.
+	 * @param remote - Location this field is being set from.
+	 * @throws {InvalidField} if field is not defined.
+	 * @throws {InvalidValue} if value is not allowed for the field.
+	 */
+	stage<Field extends keyof Fields & string>(name: Field, newValue: Fields[Field], remote = this.location) {
+		if (this._readonly) {
+			throw new Error("config is readonly and cannot have values changed");
+		}
+
+		const def = this.constructor.fieldDefinitions[name];
+		if (!def) {
+			throw new InvalidField(`No field named '${name}'`);
+		}
+
+		this._checkAccess(name, def, remote, ConfigAccess.write, true);
+		if (remote !== this.location) {
+			this._checkAccess(name, def, this.location, ConfigAccess.write, true);
+		}
+
+		const value = Config._coerceValue(name, def, newValue);
+		this.staging._set(name, value);
+	}
+
+	/**
+	 * Similar to setProp but only validates the field type and not other constraints.
+	 * The value is stored in staging and must be committed for the change to take effect.
+	 *
+	 * @param name - Name of field to set property on.
+	 * @param prop - Name of property to set on field.
+	 * @param value - the value to set the property to.
+	 * @param remote - Location this property is being set from
+	 * @throws {InvalidField} if field is not defined.
+	 * @throws {InvalidValue} if field is not an object.
+	 */
+	stageProp<Field extends keyof Fields & string>(
+		name: Field,
+		prop: string,
+		value?: unknown,
+		remote = this.location,
+	) {
+		if (this._readonly) {
+			throw new Error("config is readonly and cannot have values changed");
+		}
+
+		const def = this.constructor.fieldDefinitions[name];
+		if (!def) {
+			throw new InvalidField(`No field named '${name}'`);
+		}
+
+		this._checkAccess(name, def, remote, ConfigAccess.write, true);
+		if (remote !== this.location) {
+			this._checkAccess(name, def, this.location, ConfigAccess.write, true);
+		}
+
+		if (def.type !== "object") {
+			throw new InvalidValue(`Cannot set property on non-object field '${name}'`);
+		}
+
+		const prev = this.staging.get(name) as Record<string, unknown>;
+		const updated = {...prev || {}};
+
+		if (value !== undefined) {
+			updated[prop] = value;
+		} else {
+			delete updated[prop];
+		}
+
+		this.staging._set(name, updated);
+	}
+
+	/**
+	 * Commit all changes in stating after they have been validated.
+	 *
+	 * @throws {InvalidValue} if a value is not allowed for its field.
+	 */
+	commitStaging() {
+		if (this._readonly) {
+			throw new Error("config is readonly and cannot have values changed");
+		}
+
+		const staging = this._staging;
+		if (!staging) {
+			return; // Nothing to commit
+		}
+
+		// Validate all fields using the current state of staging
+		const config = staging.config;
+		const _readonly = this._readonly;
+		config._readonly = true;
+		try {
+			for (const [fieldName, fieldDef] of Object.entries(this.constructor.fieldDefinitions)) {
+				if (fieldDef.validator) {
+					try {
+						fieldDef.validator(config.fields[fieldName as keyof Fields], config);
+					} catch (err: any) {
+						throw new InvalidValue(`Failed validation of ${fieldName}: ${err.message}`);
+					}
+				}
+			}
+		} finally {
+			config._readonly = _readonly;
+		}
+
+		// Validation success, set all values which have changed
+		for (const fieldName of staging.changes) {
+			const field = fieldName as keyof Fields & string;
+			this._set(field, config.fields[field]);
+		}
+	}
+
+	/** Revert all changes in staging */
+	revertStaging() {
+		if (this._readonly) {
+			throw new Error("config is readonly and cannot have values changed");
+		}
+		this._staging = undefined;
+	}
 }

packages/lib/src/config/definitions.ts

@@ -1,7 +1,9 @@
 // Core definitions for the configuration system
 import * as classes from "./classes";
+import * as validators from "./validators";
 import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin";
 import { Static } from "@sinclair/typebox";
+import { isTargetVersion } from "../data/version";
 
 type configFromJSON<T> = (...args: Parameters<typeof classes.Config.fromJSON>) => T;
 type configFromFile<T> = (...args: Parameters<typeof classes.Config.fromFile>) => Promise<T>;
@@ -18,7 +20,7 @@ export interface ControllerConfigFields {
 	"controller.public_url": string | null;
 	"controller.tls_certificate": string | null;
 	"controller.tls_private_key": string | null;
-	"controller.auth_secret": string;
+	"controller.auth_secret": string | null;
 	"controller.heartbeat_interval": number;
 	"controller.session_timeout": number;
 	"controller.metrics_timeout": number;
@@ -73,18 +75,21 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			restartRequired: true,
 			type: "string",
 			initialValue: "mods",
+			validator: validators.filePath,
 		},
 		"controller.database_directory": {
 			title: "Database directory",
 			description: "Directory where item and configuration data is stored.",
 			type: "string",
 			initialValue: "database",
+			validator: validators.filePath,
 		},
 		"controller.autosave_interval": {
 			title: "Autosave Interval",
 			description: "Interval in seconds to autosave data in memory to disk.",
 			type: "number",
 			initialValue: 60,
+			validator: validators.greaterThanZero,
 		},
 		"controller.http_port": {
 			title: "HTTP Port",
@@ -93,6 +98,10 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			type: "number",
 			optional: true,
 			initialValue: 8080,
+			validator: validators.optional(validators.all(
+				validators.integer,
+				validators.greaterThanZero
+			)),
 		},
 		"controller.https_port": {
 			title: "HTTPS Port",
@@ -100,6 +109,10 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			restartRequired: true,
 			type: "number",
 			optional: true,
+			validator: validators.optional(validators.all(
+				validators.integer,
+				validators.greaterThanZero
+			)),
 		},
 		"controller.bind_address": {
 			title: "Bind Address",
@@ -107,6 +120,7 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			restartRequired: true,
 			type: "string",
 			optional: true,
+			// TODO extendedValidation, valid addresses including IPV4, IPV6, and localhost
 		},
 		"controller.trusted_proxies": {
 			title: "Trusted Proxies",
@@ -114,12 +128,14 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 				"Comma separated list of IP addresses and/or CIDR blocks to trust the X-Forwarded-For header on",
 			type: "string",
 			optional: true,
+			// TODO extendedValidation, see parseTrustedProxies
 		},
 		"controller.public_url": {
 			title: "Public URL",
 			description: "Public facing URL the controller is hosted on, including the protocol.",
 			type: "string",
 			optional: true,
+			// TODO extendedValidation, valid url including the protocol
 		},
 		"controller.tls_certificate": {
 			title: "TLS Certificate",
@@ -127,6 +143,7 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			restartRequired: true,
 			type: "string",
 			optional: true,
+			validator: validators.optional(validators.filePath),
 		},
 		"controller.tls_private_key": {
 			title: "TLS Private Key",
@@ -134,6 +151,7 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			restartRequired: true,
 			type: "string",
 			optional: true,
+			validator: validators.optional(validators.filePath),
 		},
 		"controller.auth_secret": {
 			access: ["controller"],
@@ -151,30 +169,35 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			description: "Interval heartbeats are sent out on WebSocket connections.",
 			type: "number",
 			initialValue: 15,
+			validator: validators.greaterThanZero,
 		},
 		"controller.session_timeout": {
 			title: "Session Timeout",
 			description: "Time in seconds before giving up resuming a dropped WebSocket session.",
 			type: "number",
 			initialValue: 60,
+			validator: validators.greaterThanZero,
 		},
 		"controller.metrics_timeout": {
 			title: "Metrics Timeout",
 			description: "Timeout in seconds for metrics gathering from hosts.",
 			type: "number",
 			initialValue: 8,
+			validator: validators.greaterThanZero,
 		},
 		"controller.system_metrics_interval": {
 			title: "System Metrics Interval",
 			description: "Interval in seconds to collect and update system metrics for the Web UI",
 			type: "number",
 			initialValue: 10,
+			validator: validators.greaterThanEqualZero,
 		},
 		"controller.proxy_stream_timeout": {
 			title: "Proxy Stream Timeout",
 			description: "Timeout in seconds for proxy streams to start flowing.",
 			type: "number",
 			initialValue: 15,
+			validator: validators.greaterThanZero,
 		},
 		"controller.factorio_username": {
 			title: "Factorio Username",
@@ -219,12 +242,14 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			description: "Duration in minutes to cache mod portal API responses.",
 			type: "number",
 			initialValue: 30,
+			validator: validators.greaterThanEqualZero, // Zero is allowed here to disable cache
 		},
 		"controller.mod_portal_page_size": {
 			title: "Mod Portal Page Size",
 			description: "Maximum number of results per page when querying the Factorio mod portal API.",
 			type: "number",
 			initialValue: 1000,
+			validator: validators.greaterThanZero,
 		},
 		"controller.allow_remote_updates": {
 			description: "When true, allows a remote event to trigger a clusterio update via npm",
@@ -259,7 +284,6 @@ export interface HostConfigFields {
 	"host.instances_directory": string;
 	"host.controller_url": string;
 	"host.controller_token": string;
-	"host.tls_ca": string | null;
 	"host.public_address": string;
 	"host.factorio_port_range": string;
 	"host.factorio_username": string | null,
@@ -300,6 +324,7 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			description: "Path to directory to look for factorio installs",
 			type: "string",
 			initialValue: "factorio",
+			validator: validators.filePath,
 		},
 		"host.mods_directory": {
 			title: "Mods Directory",
@@ -307,18 +332,21 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			restartRequired: true,
 			type: "string",
 			initialValue: "mods",
+			validator: validators.filePath,
 		},
 		"host.instances_directory": {
 			description: "Path to directory to store instances in.",
 			restartRequired: true,
 			type: "string",
 			initialValue: "instances",
+			validator: validators.filePath,
 		},
 		"host.controller_url": {
 			description: "URL to connect to the controller at",
 			restartRequired: true,
 			type: "string",
 			initialValue: "http://localhost:8080/",
+			// TODO extendedValidation, valid url including the protocol
 		},
 		"host.controller_token": {
 			access: ["host"],
@@ -326,12 +354,7 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			restartRequired: true,
 			type: "string",
 			initialValue: "enter token here",
-		},
-		"host.tls_ca": {
-			description: "Path to Certificate Authority to validate TLS connection to controller against.",
-			restartRequired: true,
-			type: "string",
-			optional: true,
+			// TODO extendedValidation, valid JWT with matching host id
 		},
 		"host.public_address": {
 			description: "Public facing address players should connect to in order to join instances on this host",
@@ -345,6 +368,7 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 				"ranges separated with a dash.",
 			type: "string",
 			initialValue: "34100-34199",
+			// TODO extendedValidation, port range
 		},
 		"host.factorio_username": {
 			title: "Factorio Username",
@@ -370,6 +394,7 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			description: "Maximum delay to wait before attempting to reconnect WebSocket",
 			type: "number",
 			initialValue: 60,
+			validator: validators.greaterThanZero,
 		},
 		"host.allow_remote_updates": {
 			description: "When true, allows a remote event to trigger a clusterio update via npm",
@@ -463,6 +488,7 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			type: "number",
 			optional: true,
 			hidden: true,
+			// Internal value, extendedValidation not used
 		},
 		"instance.auto_start": {
 			description: "Automatically start this instance when the host hosting it is started up",
@@ -480,6 +506,11 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			type: "string",
 			initialValue: "latest",
 			inputComponent: "target_version",
+			validator: function(value) {
+				if (!isTargetVersion(value)) {
+					throw new Error("Value must be be 'latest', or match X.Y, or match X.Y.Z");
+				}
+			},
 		},
 		"factorio.executable_path": {
 			description:
@@ -488,6 +519,7 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			restartRequired: true,
 			type: "string",
 			optional: true,
+			validator: validators.optional(validators.filePath),
 		},
 		"factorio.shutdown_timeout": {
 			description:
@@ -495,24 +527,27 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 				"the process. Set to 0 to disable.",
 			type: "number",
 			initialValue: 300,
-			optional: true,
+			validator: validators.greaterThanEqualZero,
 		},
 		"factorio.game_port": {
 			description: "UDP port to run game on, uses a port in host.factorio_port_range if null",
 			restartRequired: true,
 			type: "number",
 			optional: true,
+			validator: validators.optional(validators.greaterThanEqualZero),
 		},
 		"factorio.host_assigned_game_port": {
 			access: ["host"],
 			type: "number",
 			optional: true,
+			// Internal value, extendedValidation not used
 		},
 		"factorio.rcon_port": {
 			description: "TCP port to run RCON on, uses a random port if null",
 			restartRequired: true,
 			type: "number",
 			optional: true,
+			validator: validators.optional(validators.greaterThanEqualZero),
 		},
 		"factorio.rcon_password": {
 			credential: ["host", "controller"],
@@ -527,6 +562,14 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 				"autosave pool with this many slots. Requires autosaves to be enabled to work. Set to 0 to disable.",
 			type: "number",
 			initialValue: 5,
+			dependsOn: ["factorio.settings", "factorio.settings.autosave_slots"],
+			validator: function(value, config) {
+				const factorioSettings = config.get("factorio.settings");
+				const autosaveSlots = factorioSettings.autosave_slots;
+				if (typeof autosaveSlots === "number" && value < autosaveSlots) {
+					throw new Error("Value cannot be less than the number of autosave slots");
+				}
+			},
 		},
 		"factorio.mod_pack_id": {
 			title: "Mod Pack",
@@ -575,6 +618,7 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			],
 			type: "object",
 			initialValue: {}, // See create instance handler in controller.
+			// TODO extendedValidation, we could check keys and types here if we wanted
 		},
 		"factorio.verbose_logging": {
 			description: "Enable verbose logging on the Factorio server",
@@ -617,6 +661,7 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			restartRequired: true,
 			type: "number",
 			initialValue: 5,
+			validator: validators.greaterThanZero,
 		},
 	};
 }
@@ -624,7 +669,6 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 export interface ControlConfigFields {
 	"control.controller_url": string | null;
 	"control.controller_token": string | null;
-	"control.tls_ca": string | null;
 	"control.max_reconnect_delay": number;
 }
 
@@ -647,22 +691,18 @@ export class ControlConfig extends classes.Config<ControlConfigFields> {
 			type: "string",
 			optional: true,
 		},
-		"control.tls_ca": {
-			description: "Path to Certificate Authority to validate TLS connection to controller against.",
-			type: "string",
-			optional: true,
-		},
 		"control.max_reconnect_delay": {
 			title: "Max Reconnect Delay",
 			description: "Maximum delay to wait before attempting to reconnect WebSocket",
 			type: "number",
 			initialValue: 60,
+			validator: validators.greaterThanEqualZero,
 		},
 	};
 }
 
 
-function validateFields(
+function validateFieldNames(
 	pluginName: string,
 	fields: Record<string, classes.FieldDefinition>,
 ) {
@@ -676,36 +716,73 @@ function validateFields(
 }
 
 /**
- * Add config fields defined by the provided plugin infos
+ * Add the config fields defined by the by a specific plugin infos
+ */
+export function addPluginFieldDefinitions(
+	pluginInfo: PluginNodeEnvInfo | PluginWebpackEnvInfo,
+	kind: "controllerConfigFields" | "hostConfigFields" | "instanceConfigFields",
+	Config: typeof classes.Config<any>
+) {
+	// Define plugin.load_plugin
+	const fieldDef: classes.FieldDefinition = {
+		title: "Load Plugin",
+		type: "boolean",
+		initialValue: true,
+		restartRequired: true,
+	};
+
+	// The validator is dynamic based on the feature requirements of the plugin
+	if (kind === "instanceConfigFields" && pluginInfo.features) {
+		fieldDef.dependsOn = [];
+
+		const savePatching = pluginInfo.features.includes("SavePatching");
+		if (savePatching) {
+			fieldDef.dependsOn.push("factorio.enable_save_patching");
+		}
+
+		const scriptCommands = pluginInfo.features.includes("ScriptCommands");
+		if (scriptCommands) {
+			fieldDef.dependsOn.push("factorio.enable_script_commands");
+		}
+
+		if (scriptCommands || savePatching) {
+			fieldDef.validator = (enabled, config) => {
+				if (!enabled) {
+					return;
+				}
+				if (savePatching && !config.get("factorio.enable_save_patching")) {
+					throw new Error(`Plugin ${pluginInfo.name} requires save patching`);
+				}
+				if (scriptCommands && !config.get("factorio.enable_script_commands")) {
+					throw new Error(`Plugin ${pluginInfo.name} requires script commands`);
+				}
+			};
+		}
+	}
+
+	Config.fieldDefinitions[`${pluginInfo.name}.load_plugin`] = fieldDef;
+
+	// Merge in other fields from the plugin
+	const fields = pluginInfo[kind];
+	if (fields) {
+		validateFieldNames(pluginInfo.name, fields);
+		Object.assign(Config.fieldDefinitions, fields);
+	}
+}
+
+/**
+ * Add all config fields defined by the provided plugin infos
  *
  * @param {Array<Object>} pluginInfos - Array of plugin info objects.
  */
 export function addPluginConfigFields(pluginInfos: PluginNodeEnvInfo[] | PluginWebpackEnvInfo[]) {
-	function pluginConfig(
-		pluginInfo: PluginNodeEnvInfo | PluginWebpackEnvInfo,
-		kind: "controllerConfigFields" | "hostConfigFields" | "instanceConfigFields",
-		Config: typeof ControllerConfig | typeof HostConfig | typeof InstanceConfig,
-	) {
-		(Config.fieldDefinitions as any)[`${pluginInfo.name}.load_plugin`] = {
-			title: "Load Plugin",
-			restartRequired: true,
-			type: "boolean",
-			initialValue: true,
-		};
-
-		const fields = pluginInfo[kind];
-		if (fields) {
-			validateFields(pluginInfo.name, fields);
-			Object.assign(Config.fieldDefinitions, fields);
-		}
-	}
 	for (let pluginInfo of pluginInfos) {
-		pluginConfig(pluginInfo, "controllerConfigFields", ControllerConfig);
+		addPluginFieldDefinitions(pluginInfo, "controllerConfigFields", ControllerConfig);
 		if (pluginInfo.hostEntrypoint || pluginInfo.instanceEntrypoint) {
-			pluginConfig(pluginInfo, "hostConfigFields", HostConfig);
+			addPluginFieldDefinitions(pluginInfo, "hostConfigFields", HostConfig);
 		}
 		if (pluginInfo.instanceEntrypoint) {
-			pluginConfig(pluginInfo, "instanceConfigFields", InstanceConfig);
+			addPluginFieldDefinitions(pluginInfo, "instanceConfigFields", InstanceConfig);
 		}
 	}
 }

packages/lib/src/config/validators.ts

@@ -0,0 +1,66 @@
+import path from "path";
+
+/** Combine multiple validators into a single one. */
+export function all<V, C> (
+	...validators: ((value: V, config: C) => void)[]
+) {
+	return function (value: V, config: C) {
+		for (const validator of validators) {
+			validator(value, config);
+		}
+	};
+}
+
+/** Skip validation of optional fields when null */
+export function optional<V, C> (
+	validator: (value: V, config: C) => void
+) {
+	return function (value: V | null, config: C) {
+		if (value === null) { return; }
+		validator(value, config);
+	};
+}
+
+/** Value must be greater than the minimum value */
+export function greaterThan(min: number) {
+	return function (value: number) {
+		if (value <= min) {
+			throw new Error(`Value must be greater than ${min}`);
+		}
+	};
+}
+
+/** Value must be greater than 0 */
+export const greaterThanZero = greaterThan(0);
+
+/** Value must be greater than or equal to the minimum value */
+export function greaterThanEqual(min: number) {
+	return function (value: number) {
+		if (value < min) {
+			throw new Error(`Value must be greater than ${min}`);
+		}
+	};
+}
+
+/** Value must be greater than or equal to 0 */
+export const greaterThanEqualZero = greaterThanEqual(0);
+
+/** Value must be an integer value */
+export function integer(value: number) {
+	if (!Number.isInteger(value)) {
+		throw new Error("Value must be an integer");
+	}
+}
+
+/**
+ * Value must be a valid file path, it does not need to exist
+ * On posix this will only error for non string values, which isn't very useful
+ * However, this could be expanded in the future to check system OS
+ */
+export function filePath(value: string) {
+	try {
+		path.resolve(value);
+	} catch (err: any) {
+		throw new Error(`Value must be a valid path: ${err.message}`);
+	}
+}

packages/lib/src/data/ExportManifest.ts

@@ -11,21 +11,26 @@ export default class ExportManifest {
 		/**
 		 * Mapping between known assets and their file names.
 		 *
-		 * Currently exported assets are
+		 * Currently exported assets are:
 		 * - settings: JSON of mod settings prototypes.
 		 * - prototypes: JSON of all game prototypes except settings.
-		 * - item-spritesheet: PNG containing icons from the game.
-		 * - item-metadata: JSON of Map entries describing icons in item-spritesheet
 		 * - locale: JSON of flattened Map entries of en locale strings from the game.
+		 * - spritesheet: Single PNG spritesheet containing all icon categories.
+		 * - metadata: JSON array of [name, {x, y, size, category, path?}] entries
+		 *   describing sprite coordinates and category membership. Categories:
+		 *   item, recipe, signal, technology, planet, quality, entity, static.
 		 */
-		public assets: Record<string, string>
+		public assets: Record<string, string>,
+		/** ISO 8601 timestamp of when the export was uploaded. */
+		public exportedAt?: string,
 	) { }
 
 	static jsonSchema = Type.Object({
 		"assets": Type.Record(Type.String(), Type.String()),
+		"exportedAt": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof ExportManifest.jsonSchema>) {
-		return new this(json.assets);
+		return new this(json.assets, json.exportedAt);
 	}
 }

packages/lib/src/data/HostDetails.ts

@@ -0,0 +1,77 @@
+import { Type, Static } from "@sinclair/typebox";
+
+export default class HostDetails {
+	constructor(
+		/** Version of the host software. The controller may warn if there is a mismatch. */
+		public version: string,
+		/** Human-readable display name of the host. */
+		public name: string,
+		/** Unique numeric identifier for this host. */
+		public id: number,
+		/** Whether the host is currently connected to the controller. */
+		public connected: boolean,
+		/** Address from which the host has connected to the controller. */
+		public remoteAddress = "",
+		/** Public-facing IP address of the host, if available. */
+		public publicAddress: string = "",
+		/** Unix timestamp (sec) after which issued tokens are considered valid. */
+		public tokenValidAfter: number = 0,
+		/** Unix timestamp (ms) indicating when this record was last updated. */
+		public updatedAtMs = 0,
+		/** Flag indicating the host record has been removed on the controller. */
+		public isDeleted = false
+	) {}
+
+	static jsonSchema = Type.Object({
+		"version": Type.String(),
+		"name": Type.String(),
+		"id": Type.Integer(),
+		"connected": Type.Boolean(),
+		"remoteAddress": Type.Optional(Type.String()),
+		"publicAddress": Type.Optional(Type.String()),
+		"tokenValidAfter": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"isDeleted": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.version,
+			json.name,
+			json.id,
+			json.connected,
+			json.remoteAddress,
+			json.publicAddress,
+			json.tokenValidAfter,
+			json.updatedAtMs,
+			json.isDeleted
+		);
+	}
+
+	toJSON() {
+		const json = {
+			version: this.version,
+			name: this.name,
+			id: this.id,
+			connected: this.connected,
+		} as Static<typeof HostDetails.jsonSchema>;
+
+		if (this.remoteAddress !== "") {
+			json.remoteAddress = this.remoteAddress;
+		}
+		if (this.publicAddress !== "") {
+			json.publicAddress = this.publicAddress;
+		}
+		if (this.tokenValidAfter !== 0) {
+			json.tokenValidAfter = this.tokenValidAfter;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updatedAtMs = this.updatedAtMs;
+		}
+		if (this.isDeleted !== false) {
+			json.isDeleted = this.isDeleted;
+		}
+
+		return json;
+	}
+}

packages/lib/src/data/InstanceDetails.ts

@@ -0,0 +1,108 @@
+import { Static, Type } from "@sinclair/typebox";
+import { StringEnum } from "./composites";
+import { TargetVersion, TargetVersionSchema } from "./version";
+
+export const InstanceStatus = StringEnum([
+	"unknown",
+	"unassigned",
+	"stopped",
+	"starting",
+	"running",
+	"stopping",
+	"creating_save",
+	"exporting_data",
+	"deleted",
+]);
+
+/**
+ * Current status of the instance. One of:
+ * - `unknown`: Instance is assigned to a host but this host is currently
+ *   not connected to the contreller.
+ * - `unassigned`: Instance is not assigned to a a host and exists only on
+ *   the controller.
+ * - `stopped`: Instance is stopped.
+ * - `starting`: Instance is in the process of starting up.
+ * - `running`: Instance is running normally.
+ * - `stopping`: Instance is in the process of stopping.
+ * - `creating_save`: Instance is in the process of creating a save.
+ * - `exporting_data`: Instance is in the process of exporting game data.
+ * - `deleted`: Instance has been deleted.
+ */
+export type InstanceStatus = Static<typeof InstanceStatus>;
+
+export default class InstanceDetails {
+	constructor(
+		public name: string,
+		public id: number,
+		public assignedHost: number | undefined,
+		public gamePort: number | undefined,
+		public status: InstanceStatus,
+		public factorioVersion: TargetVersion | undefined,
+		public startedAtMs = 0,
+		/** Millisecond Unix timestamp this entry was last updated at */
+		public updatedAtMs = 0,
+		public excludeFromStartAll = false
+	) {}
+
+	get isDeleted() {
+		return this.status === "deleted";
+	}
+
+	static jsonSchema = Type.Object({
+		"name": Type.String(),
+		"id": Type.Integer(),
+		"assignedHost": Type.Optional(Type.Integer()),
+		"gamePort": Type.Optional(Type.Integer()),
+		"status": StringEnum([
+			"unknown", "unassigned", "stopped", "starting", "running", "stopping",
+			"creating_save", "exporting_data", "deleted",
+		]),
+		"factorioVersion": Type.Optional(TargetVersionSchema),
+		"startedAtMs": Type.Optional(Type.Number()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"excludeFromStartAll": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.name,
+			json.id,
+			json.assignedHost,
+			json.gamePort,
+			json.status,
+			json.factorioVersion,
+			json.startedAtMs,
+			json.updatedAtMs,
+			json.excludeFromStartAll
+		);
+	}
+
+	toJSON() {
+		const json = {
+			name: this.name,
+			id: this.id,
+			status: this.status,
+		} as Static<typeof InstanceDetails.jsonSchema>;
+
+		if (this.assignedHost !== undefined) {
+			json.assignedHost = this.assignedHost;
+		}
+		if (this.gamePort !== undefined) {
+			json.gamePort = this.gamePort;
+		}
+		if (this.factorioVersion !== undefined) {
+			json.factorioVersion = this.factorioVersion;
+		}
+		if (this.startedAtMs !== 0) {
+			json.startedAtMs = this.startedAtMs;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updatedAtMs = this.updatedAtMs;
+		}
+		if (this.excludeFromStartAll !== false) {
+			json.excludeFromStartAll = this.excludeFromStartAll;
+		}
+
+		return json;
+	}
+}

packages/lib/src/data/ModInfo.ts

@@ -1,4 +1,4 @@
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import JSZip from "jszip";
 import { Type, Static } from "@sinclair/typebox";
 
@@ -22,11 +22,12 @@ const UnsatisfiedSeverity: Record<ModDependencyUnsatisfiedReason, number> = {
 	"incompatible": 1, "missing_dependency": 2, "wrong_version": 3,
 };
 
-const lettersRegex = /^[a-zA-Z]+$/;
+const depSpecRegex = /^(?:(\?|\(\?\)|!|~|\+) *)?(.+?)(?: *([<>]=?|=) *([0-9.]+))?$/;
 
 export class ModDependency {
-	public type: ModDependencyType;
+	public spec: string;
 	public name: string;
+	public type: ModDependencyType;
 	public version: ModVersionEquality | undefined;
 
 	static getTypeFromPrefix(prefix: string): ModDependencyType {
@@ -42,50 +43,27 @@ export class ModDependency {
 			case "":
 				return "required";
 			default:
-				throw new Error(`Invalid dependency prefix: ${prefix}`);
+				throw new Error(`Invalid dependency prefix "${prefix}"`);
 		}
 	}
 
 	constructor (
-		public specification: string,
+		specification: string,
 	) {
-		const parts = specification.split(" ");
-
-		// Fix mods which include spaces in their name, joins consecutive words together
-		let index = 0;
-		while (index < parts.length - 1) {
-			const current = parts[index];
-			const next = parts[index + 1];
-			if (lettersRegex.test(current) && lettersRegex.test(next)) {
-				parts[index] = `${current} ${next}`;
-				parts.splice(index + 1, 1);
-			} else {
-				index += 1;
-			}
+		const match = depSpecRegex.exec(specification);
+		if (match === null) {
+			throw new Error(`Invalid dependency specification "${specification}"`);
 		}
 
-		// Parse the dependency specification
-		switch (parts.length) {
-			case 1:
-				this.type = "required";
-				this.name = parts[0];
-				break;
-			case 2:
-				this.type = ModDependency.getTypeFromPrefix(parts[0]);
-				this.name = parts[1];
-				break;
-			case 3:
-				this.type = "required";
-				this.name = parts[0];
-				this.version = ModVersionEquality.fromParts(parts[1], parts[2]);
-				break;
-			case 4:
-				this.type = ModDependency.getTypeFromPrefix(parts[0]);
-				this.name = parts[1];
-				this.version = ModVersionEquality.fromParts(parts[2], parts[3]);
-				break;
-			default:
-				throw new Error(`Invalid dependency specification: ${specification}`);
+		// Expand the match to the different parts
+		const [spec, type, name, equality, version] = match;
+		this.type = ModDependency.getTypeFromPrefix(type ?? "");
+		this.spec = spec;
+		this.name = name;
+
+		// Parse the equality and version
+		if ((equality || version) && !this.incompatible) {
+			this.version = ModVersionEquality.fromParts(equality, version);
 		}
 	}
 
@@ -124,7 +102,7 @@ export class ModDependency {
 	}
 
 	toJSON() {
-		return this.specification;
+		return this.spec;
 	}
 }
 
@@ -218,7 +196,7 @@ export default class ModInfo {
 	 * As they would be represented in info.json.
 	 */
 	get dependencySpecifications() {
-		return this.dependencies.map(d => d.specification);
+		return this.dependencies.map(d => d.spec);
 	}
 
 	checkDependencySatisfaction(mods: (ModInfo | ModRecord)[]) {
@@ -308,7 +286,13 @@ export default class ModInfo {
 		if (json.homepage) { modInfo.homepage = json.homepage; }
 		if (json.description) { modInfo.description = json.description; }
 		if (json.factorio_version) { modInfo.factorioVersion = json.factorio_version; }
-		if (json.dependencies) { modInfo.dependencies = json.dependencies.map(d => new ModDependency(d)); }
+
+		// Parse the dependencies
+		try {
+			if (json.dependencies) { modInfo.dependencies = json.dependencies.map(d => new ModDependency(d)); }
+		} catch (err: any) {
+			throw new Error(`Failed for parse dependencies for ${json.name}: ${err}`);
+		}
 
 		// Additional data
 		if (json.size) { modInfo.size = json.size; }

packages/lib/src/data/ModPack.ts

@@ -5,6 +5,7 @@ import * as libSchema from "../schema";
 import * as libLuaTools from "../lua_tools";
 
 import ExportManifest from "./ExportManifest";
+import type { ExportSettings } from "../export";
 import type { Logger } from "../logging";
 
 import ModInfo, { ModDependencyUnsatisfiedReason } from "./ModInfo";
@@ -357,7 +358,7 @@ export default class ModPack {
 	 *     Setting prototypes exported from the game.
 	 * @param logger - Logger used to report warnings on.
 	 */
-	fillDefaultSettings(settingPrototypes: Record<string, object>, logger: Logger) {
+	fillDefaultSettings(settingPrototypes: ExportSettings, logger: Logger) {
 		const knownTypes = ["bool-setting", "int-setting", "double-setting", "string-setting", "color-setting"];
 		let prototypes = Object.entries(settingPrototypes)
 			.filter(([type, _]) => knownTypes.includes(type))
@@ -370,7 +371,7 @@ export default class ModPack {
 				logger.warn(`Ignoring ${prototype.name} with unknown setting_type ${settingType}`);
 				continue;
 			}
-			if (prototype.default_value === undefined) {
+			if ((prototype as any).default_value === undefined) {
 				logger.warn(`Ignoring ${prototype.name} with missing default_value`);
 				continue;
 			}
@@ -389,7 +390,7 @@ export default class ModPack {
 			if (type === "color-setting") {
 				value = libLuaTools.normalizeColor(value as object);
 			}
-			this.settings[settingType].set(prototype.name, { value });
+			this.settings[settingType].set(prototype.name, { value } as ModSetting);
 		}
 	}
 

packages/lib/src/data/Role.ts

@@ -1,15 +1,23 @@
 import { Static, Type } from "@sinclair/typebox";
 import { permissions as globalPermissions } from "../permissions";
 
+interface RoleCollection {
+	get(id: number): Role;
+	set(role: Role): void;
+}
+
 /**
  * Represents a collection of granted permissions
  */
 export default class Role {
+	static DefaultAdminRoleId = 0 as const;
+	static DefaultPlayerRoleId = 1 as const;
+
 	constructor(
 		public id: number,
 		public name: string,
 		public description: string,
-		public permissions: Set<string>,
+		public permissions = new Set<string>(),
 		public updatedAtMs = 0,
 		public isDeleted = false,
 	) { }
@@ -45,6 +53,13 @@ export default class Role {
 		};
 	}
 
+	static ensureDefaultPlayerRole(roles: RoleCollection) {
+		const playerRole = roles.get(this.DefaultPlayerRoleId)
+			?? new this(this.DefaultPlayerRoleId, "Player", "Default player role.");
+		playerRole.grantDefaultPermissions();
+		roles.set(playerRole);
+	}
+
 	grantDefaultPermissions() {
 		for (let permission of globalPermissions.values()) {
 			if (permission.grantByDefault) {
@@ -52,4 +67,15 @@ export default class Role {
 			}
 		}
 	}
+
+	static ensureDefaultAdminRole(roles: RoleCollection) {
+		const adminRole = roles.get(this.DefaultAdminRoleId)
+			?? new this(this.DefaultAdminRoleId, "Cluster Admin", "Cluster wide administrator.");
+		adminRole.grantAdminPermissions();
+		roles.set(adminRole);
+	}
+
+	grantAdminPermissions() {
+		this.permissions.add("core.admin");
+	}
 }

packages/lib/src/data/UserDetails.ts

@@ -2,12 +2,10 @@ import { Static, Type } from "@sinclair/typebox";
 
 import PlayerStats from "./PlayerStats";
 
-/**
- * Represeents a user in the cluster
- *
- * Holds data about a Factorio user in the cluster.
- */
-export default class User {
+/** Represents a user in the cluster */
+export default class UserDetails {
+	public playerStats: PlayerStats;
+
 	constructor(
 		/** Factorio user name.  */
 		public name: string,
@@ -27,11 +25,15 @@ export default class User {
 		public updatedAtMs = 0,
 		/** True if this user object has been removed from the cluster.  */
 		public isDeleted = false,
-		/** Combined statistics for the player this user account is tied to.  */
-		public playerStats = new PlayerStats(),
 		/** Per instance statistics for the player this user account is tied to.  */
 		public instanceStats = new Map<number, PlayerStats>(),
 	) {
+		this.playerStats = UserDetails._calculatePlayerStats(this.instanceStats);
+	}
+
+	/** Name of the user, used by event subscriptions and to ensure case insensitivity */
+	get id() {
+		return this.name.toLowerCase();
 	}
 
 	static jsonSchema = Type.Object({
@@ -51,17 +53,13 @@ export default class User {
 		),
 	});
 
-	static fromJSON(json: Static<typeof this.jsonSchema>, _hack?: any) {
-		const roleIds = new Set<number>(json.roles ?? []);
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
 		const instanceStats = new Map(
-			(json.instance_stats ?? []).map(
-				([id, stats]) => [id, PlayerStats.fromJSON(stats)]
-			)
+			(json.instance_stats ?? []).map(([id, stats]) => [id, PlayerStats.fromJSON(stats)])
 		);
-		const playerStats = User._calculatePlayerStats(instanceStats);
 		return new this(
 			json.name,
-			roleIds,
+			new Set<number>(json.roles),
 			new Set(json.instances),
 			json.is_admin,
 			json.is_banned,
@@ -69,13 +67,12 @@ export default class User {
 			json.ban_reason,
 			json.updated_at_ms,
 			json.is_deleted,
-			playerStats,
 			instanceStats,
 		);
 	}
 
 	toJSON() {
-		let json: Static<typeof User.jsonSchema> = {
+		let json: Static<typeof UserDetails.jsonSchema> = {
 			name: this.name,
 		};
 
@@ -119,7 +116,7 @@ export default class User {
 	}
 
 	recalculatePlayerStats() {
-		this.playerStats = User._calculatePlayerStats(this.instanceStats);
+		this.playerStats = UserDetails._calculatePlayerStats(this.instanceStats);
 	}
 
 	static _calculatePlayerStats(instanceStatsMap: Map<number, PlayerStats>) {
@@ -149,44 +146,13 @@ export default class User {
 		}
 		return playerStats;
 	}
-
-	/**
-	 * Merge another user into this one, must have matching ids, user is not deleted
-	 * @param otherUser - User who's details are merged from
-	*/
-	merge(otherUser: User) {
-		if (this.id !== otherUser.id) {
-			throw new Error("Cannot merge users with different ids");
-		}
-
-		// Merge properties
-		this.roleIds = new Set([...this.roleIds, ...otherUser.roleIds]);
-		this.instances = new Set([...this.instances, ...otherUser.instances]);
-		this.isAdmin = this.isAdmin && otherUser.isAdmin; // More secure to use && rather ||
-		this.isBanned = this.isBanned || otherUser.isBanned;
-		this.isBanned = this.isWhitelisted || otherUser.isWhitelisted;
-		this.banReason = this.updatedAtMs > otherUser.updatedAtMs ? this.banReason : otherUser.banReason;
-		this.updatedAtMs = Date.now();
-
-		// Merge instance stats
-		const thisInstanceStats = this.instanceStats;
-		for (const [instanceId, instanceStats] of otherUser.instanceStats.entries()) {
-			if (thisInstanceStats.has(instanceId)) {
-				thisInstanceStats.get(instanceId)!.merge(instanceStats);
-			} else {
-				thisInstanceStats.set(instanceId, instanceStats);
-			}
-		}
-		this.recalculatePlayerStats();
-	}
-
-	/** Name of the user, used by event subscriptions and to ensure case insensitivity */
-	get id() {
-		return this.name.toLowerCase();
-	}
 }
 
-export interface IControllerUser extends User {
+/**
+ * This interface is implemented by User on the controller.
+ * It is exposed here to allow message definitions to create permission methods.
+ */
+export interface IUser extends UserDetails {
 	/**
 	 * Check if a given permission is granted
 	 *

packages/lib/src/data/index.ts

@@ -12,8 +12,10 @@ export { default as ModuleInfo } from "./ModuleInfo";
 export { default as Permission } from "./Permission";
 export { default as PlayerStats } from "./PlayerStats";
 export { default as Role } from "./Role";
-export { default as User } from "./User";
-export type { IControllerUser } from "./User";
+export { default as InstanceDetails, InstanceStatus } from "./InstanceDetails";
+export { default as HostDetails } from "./HostDetails";
+export { default as UserDetails } from "./UserDetails";
+export type { IUser } from "./UserDetails";
 export * from "./composites";
 export * from "./messages_core";
 export * from "./messages_controller";

packages/lib/src/data/messages_controller.ts

@@ -37,6 +37,28 @@ export class ControllerConfigGetRequest {
 	static Response = plainJson(ControllerConfig.jsonSchema);
 }
 
+export class ControllerConfigSetRequest {
+	declare ["constructor"]: typeof ControllerConfigSetRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.controller.update_config" as const;
+
+	constructor(
+		public fields: Record<string, string | Record<string, unknown>>,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"fields": Type.Record(Type.String(), Type.Union([
+			Type.String(), Type.Record(Type.String(), Type.Unknown()),
+		])),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.fields);
+	}
+}
+
 export class ControllerConfigSetFieldRequest {
 	declare ["constructor"]: typeof ControllerConfigSetFieldRequest;
 	static type = "request" as const;
@@ -260,6 +282,8 @@ export class SystemInfo {
 		public diskAvailable: number,
 		public canRestart: boolean,
 		public restartRequired: boolean,
+		public systemStartedAtMs: number,
+		public processStartedAtMs: number,
 		/** Millisecond Unix timestamp this entry was last updated at */
 		public updatedAtMs: number,
 		public isDeleted: boolean,
@@ -279,6 +303,8 @@ export class SystemInfo {
 		"diskAvailable": Type.Number(),
 		"canRestart": Type.Boolean(),
 		"restartRequired": Type.Boolean(),
+		"systemStartedAtMs": Type.Number(),
+		"processStartedAtMs": Type.Number(),
 		"updatedAtMs": Type.Number(),
 		"isDeleted": Type.Boolean(),
 	});
@@ -298,6 +324,8 @@ export class SystemInfo {
 			json.diskAvailable,
 			json.canRestart,
 			json.restartRequired,
+			json.systemStartedAtMs,
+			json.processStartedAtMs,
 			json.updatedAtMs,
 			json.isDeleted,
 		);
@@ -396,7 +424,7 @@ export class DebugWsMessageEvent {
 export class FactorioVersionsRequest {
 	declare ["constructor"]: typeof FactorioVersionsRequest;
 	static type = "request" as const;
-	static src = "control" as const;
+	static src = ["control", "instance"] as const;
 	static dst = "controller" as const;
 	static permission = "core.external.get_factorio_versions" as const;
 

packages/lib/src/data/messages_host.ts

@@ -2,6 +2,7 @@ import { Type, Static } from "@sinclair/typebox";
 import { jsonArray, plainJson, StringEnum } from "./composites";
 import { CollectorResultSerialized } from "../prometheus";
 import { HostConfig } from "../config/definitions";
+import HostDetails from "./HostDetails";
 
 export class HostStopRequest {
 	declare ["constructor"]: typeof HostStopRequest;
@@ -36,6 +37,28 @@ export class HostConfigGetRequest {
 	static Response = plainJson(HostConfig.jsonSchema);
 }
 
+export class HostConfigSetRequest {
+	declare ["constructor"]: typeof HostConfigSetRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.update_config" as const;
+
+	constructor(
+		public fields: Record<string, string | Record<string, unknown>>,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"fields": Type.Record(Type.String(), Type.Union([
+			Type.String(), Type.Record(Type.String(), Type.Unknown()),
+		])),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.fields);
+	}
+}
+
 export class HostConfigSetFieldRequest {
 	declare ["constructor"]: typeof HostConfigSetFieldRequest;
 	static type = "request" as const;
@@ -82,47 +105,6 @@ export class HostConfigSetPropRequest {
 	}
 }
 
-export class HostDetails {
-	constructor(
-		public version: string,
-		public name: string,
-		public id: number,
-		public connected: boolean,
-		public remoteAddress = "",
-		public publicAddress: string = "",
-		public tokenValidAfter: number = 0,
-		/** Millisecond Unix timestamp this entry was last updated at */
-		public updatedAtMs = 0,
-		public isDeleted = false,
-	) { }
-
-	static jsonSchema = Type.Object({
-		"version": Type.String(),
-		"name": Type.String(),
-		"id": Type.Integer(),
-		"connected": Type.Boolean(),
-		"remoteAddress": Type.Optional(Type.String()),
-		"publicAddress": Type.Optional(Type.String()),
-		"tokenValidAfter": Type.Optional(Type.Number()),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"isDeleted": Type.Optional(Type.Boolean()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(
-			json.version,
-			json.name,
-			json.id,
-			json.connected,
-			json.remoteAddress,
-			json.publicAddress,
-			json.tokenValidAfter,
-			json.updatedAtMs,
-			json.isDeleted,
-		);
-	}
-}
-
 export class HostListRequest {
 	declare ["constructor"]: typeof HostListRequest;
 	static type = "request" as const;

packages/lib/src/data/messages_instance.ts

@@ -2,84 +2,11 @@ import { Type, Static } from "@sinclair/typebox";
 import PlayerStats from "./PlayerStats";
 import { JsonString, StringEnum, jsonArray, plainJson } from "./composites";
 import { InstanceConfig } from "../config";
-import type { IControllerUser } from "./User";
+import type { IUser } from "./UserDetails";
 import type { MessageRequest } from "./messages_core";
 import { CollectorResultSerialized } from "../prometheus";
 import { TargetVersion, TargetVersionSchema, PartialVersion, PartialVersionSchema } from "./version";
-
-
-export const InstanceStatus = StringEnum([
-	"unknown",
-	"unassigned",
-	"stopped",
-	"starting",
-	"running",
-	"stopping",
-	"creating_save",
-	"exporting_data",
-	"deleted",
-]);
-
-/**
- * Current status of the instance. One of:
- * - `unknown`: Instance is assigned to a host but this host is currently
- *   not connected to the contreller.
- * - `unassigned`: Instance is not assigned to a a host and exists only on
- *   the controller.
- * - `stopped`: Instance is stopped.
- * - `starting`: Instance is in the process of starting up.
- * - `running`: Instance is running normally.
- * - `stopping`: Instance is in the process of stopping.
- * - `creating_save`: Instance is in the process of creating a save.
- * - `exporting_data`: Instance is in the process of exporting game data.
- * - `deleted`: Instance has been deleted.
- */
-export type InstanceStatus = Static<typeof InstanceStatus>;
-
-export class InstanceDetails {
-	constructor(
-		public name: string,
-		public id: number,
-		public assignedHost: number | undefined,
-		public gamePort: number | undefined,
-		public status: InstanceStatus,
-		public factorioVersion: TargetVersion | undefined,
-		/** Millisecond Unix timestamp this entry was last updated at */
-		public updatedAtMs = 0,
-		public excludeFromStartAll = false,
-	) { }
-
-	static jsonSchema = Type.Object({
-		"name": Type.String(),
-		"id": Type.Integer(),
-		"assignedHost": Type.Optional(Type.Integer()),
-		"gamePort": Type.Optional(Type.Integer()),
-		"status": StringEnum([
-			"unknown", "unassigned", "stopped", "starting", "running", "stopping",
-			"creating_save", "exporting_data", "deleted",
-		]),
-		"factorioVersion": Type.Optional(TargetVersionSchema),
-		"updatedAtMs": Type.Optional(Type.Number()),
-		"excludeFromStartAll": Type.Optional(Type.Boolean()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(
-			json.name,
-			json.id,
-			json.assignedHost,
-			json.gamePort,
-			json.status,
-			json.factorioVersion,
-			json.updatedAtMs,
-			json.excludeFromStartAll,
-		);
-	}
-
-	get isDeleted() {
-		return this.status === "deleted";
-	}
-}
+import InstanceDetails, { InstanceStatus } from "./InstanceDetails";
 
 export class InstanceDetailsGetRequest {
 	declare ["constructor"]: typeof InstanceDetailsGetRequest;
@@ -175,6 +102,30 @@ export class InstanceConfigGetRequest {
 	}
 }
 
+export class InstanceConfigSetRequest {
+	declare ["constructor"]: typeof InstanceConfigSetRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.instance.update_config" as const;
+
+	constructor(
+		public instanceId: number,
+		public fields: Record<string, string | Record<string, unknown>>,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"instanceId": Type.Integer(),
+		"fields": Type.Record(Type.String(), Type.Union([
+			Type.String(), Type.Record(Type.String(), Type.Unknown()),
+		])),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.instanceId, json.fields);
+	}
+}
+
 export class InstanceConfigSetFieldRequest {
 	declare ["constructor"]: typeof InstanceConfigSetFieldRequest;
 	static type = "request" as const;
@@ -536,7 +487,7 @@ export class InstanceTransferSaveRequest {
 	static type = "request" as const;
 	static src = ["control", "controller"] as const;
 	static dst = ["controller", "host"] as const;
-	static permission(user: IControllerUser, message: MessageRequest) {
+	static permission(user: IUser, message: MessageRequest) {
 		user.checkPermission("core.instance.save.transfer");
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as { copy: boolean, sourceName: string, targetName: string };

packages/lib/src/data/messages_mod.ts

@@ -403,6 +403,10 @@ export class ModPortalDownloadRequest {
 	}
 }
 
+const ModDependencyResolveErrors = ["incompatible", "unsatisfiable", "invalidDependency", "notFound"] as const;
+export const ModDependencyResolveErrorsSchema = Type.Union(ModDependencyResolveErrors.map(v => Type.Literal(v)));
+export type ModDependencyResolveErrors = Static<typeof ModDependencyResolveErrorsSchema>;
+
 /**
  * Request mods dependencies to be resolved using the Factorio mod portal.
  *
@@ -456,18 +460,26 @@ export class ModDependencyResolveRequest {
 	static Response = class ModDependencyResolveResponse {
 		constructor(
 			public dependencies: ModInfo[],
-			public incompatible: string[],
-			public missing: string[],
+			public errors: Map<string, ModDependencyResolveErrors>,
 		) {}
 
 		static jsonSchema = Type.Object({
 			"dependencies": Type.Array(ModInfo.jsonSchema),
-			"incompatible": Type.Array(Type.String()),
-			"missing": Type.Array(Type.String()),
+			"errors": Type.Record(Type.String(), ModDependencyResolveErrorsSchema),
 		});
 
 		static fromJSON(json: Static<typeof this.jsonSchema>) {
-			return new this(json.dependencies.map(d => ModInfo.fromJSON(d)), json.incompatible, json.missing);
+			return new this(
+				json.dependencies.map(d => ModInfo.fromJSON(d)),
+				new Map(Object.entries(json.errors))
+			);
+		}
+
+		toJSON(): Static<typeof ModDependencyResolveResponse.jsonSchema> {
+			return {
+				dependencies: this.dependencies.map(d => d.toJSON()),
+				errors: Object.fromEntries(this.errors),
+			};
 		}
 	};
 }

packages/lib/src/data/messages_user.ts

@@ -1,5 +1,5 @@
 import { Type, Static } from "@sinclair/typebox";
-import User, { IControllerUser } from "./User";
+import UserDetails, { IUser } from "./UserDetails";
 import { StringEnum, jsonArray, plainJson } from "./composites";
 import { MessageRequest } from "./messages_core";
 
@@ -22,7 +22,7 @@ export class UserGetRequest {
 		return new this(json.name);
 	}
 
-	static Response = User;
+	static Response = UserDetails;
 }
 
 export class UserListRequest {
@@ -31,7 +31,7 @@ export class UserListRequest {
 	static src = "control" as const;
 	static dst = "controller" as const;
 	static permission = "core.user.list" as const;
-	static Response = jsonArray(User);
+	static Response = jsonArray(UserDetails);
 }
 
 export class UserCreateRequest {
@@ -198,15 +198,15 @@ export class UserUpdatesEvent {
 	static permission = "core.user.subscribe" as const;
 
 	constructor(
-		public updates: User[],
+		public updates: UserDetails[],
 	) { }
 
 	static jsonSchema = Type.Object({
-		"updates": Type.Array(User.jsonSchema),
+		"updates": Type.Array(UserDetails.jsonSchema),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.updates.map(update => User.fromJSON(update)));
+		return new this(json.updates.map(update => UserDetails.fromJSON(update)));
 	}
 }
 
@@ -250,7 +250,7 @@ export class UserBulkImportRequest {
 	static type = "request" as const;
 	static src = "control" as const;
 	static dst = "controller" as const;
-	static permission(user: IControllerUser, message: MessageRequest) {
+	static permission(user: IUser, message: MessageRequest) {
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as Static<typeof UserBulkImportRequest.jsonSchema>;
 			// Check if this is a restore or import request

packages/lib/src/data/version.ts

@@ -151,10 +151,10 @@ export class ModVersionEquality {
 
 	static fromParts(equality: string, version: string) {
 		if (!isVersionEquality(equality)) {
-			throw new Error(`Unknown version equality: ${equality}`);
+			throw new Error(`Invalid version equality "${equality}"`);
 		}
 		if (!isPartialVersion(version)) {
-			throw new Error(`Malformed version string: ${version}`);
+			throw new Error(`Invalid version string "${version}"`);
 		}
 		return new this(equality, version);
 	}

packages/lib/src/database.ts

@@ -4,7 +4,7 @@
  * @module lib/database
  * @author Hornwitser
  */
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 
 import * as libFileOps from "./file_ops";
 import { basicType } from "./helpers";

packages/lib/src/datastore.ts

@@ -1,5 +1,5 @@
 import path from "path";
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import { EventEmitter } from "events";
 
 import { safeOutputFile } from "./file_ops";
@@ -53,7 +53,7 @@ export class JsonDatastoreProvider<
 		private filePath: string,
 		private fromJson: (json: J) => V = v => v as any as V,
 		private migrations: (rawJson: Record<string, unknown>) => Record<string, J> = v => v as Record<string, J>,
-		private finalise: (obj: V) => V = v => v,
+		private finalise: (obj: V, map: Map<string, V>) => V | undefined = v => v,
 	) {
 		super();
 	}
@@ -82,7 +82,24 @@ export class JsonDatastoreProvider<
 		const serialized = this.migrations(rawJson);
 
 		// Convert to data class objects
-		return new Map(Object.entries(serialized).map(([k, v]) => [k, this.finalise(this.fromJson(v))]));
+		let serializedCount = 0;
+		const result = new Map<string, V>();
+		for (const [key, raw] of Object.entries(serialized)) {
+			const final = this.finalise(this.fromJson(raw), result);
+			if (final !== undefined) { result.set(key, final); }
+			serializedCount += 1;
+		}
+
+		// Save a backup if the map does not contain all values (happens where there are duplicates)
+		if (result.size < serializedCount) {
+			const backupPath = `${this.filePath}.${Date.now()}.bak`;
+			logger.warn(
+				`Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}`
+			);
+			await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t"));
+		}
+
+		return result;
 	}
 }
 
@@ -95,7 +112,7 @@ export class JsonIdDatastoreProvider<
 		private filePath: string,
 		private fromJson: (json: J) => V = v => v as any as V,
 		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as Array<J>,
-		private finalise: (obj: V) => V = v => v,
+		private finalise: (obj: V, map: Map<V["id"], V>) => V | undefined = v => v,
 	) {
 		super();
 	}
@@ -124,10 +141,24 @@ export class JsonIdDatastoreProvider<
 		const serialized = this.migrations(rawJson);
 
 		// Convert to data class objects
-		return new Map(serialized.map((e) => {
-			const v = this.finalise(this.fromJson(e));
-			return [v.id, v];
-		}));
+		let serializedCount = 0;
+		const result = new Map<V["id"], V>();
+		for (const raw of serialized) {
+			const final = this.finalise(this.fromJson(raw), result);
+			if (final !== undefined) { result.set(final.id, final); }
+			serializedCount += 1;
+		}
+
+		// Save a backup if the map does not contain all values (happens where there are duplicates)
+		if (result.size < serializedCount) {
+			const backupPath = `${this.filePath}.${Date.now()}.bak`;
+			logger.warn(
+				`Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}`
+			);
+			await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t"));
+		}
+
+		return result;
 	}
 }
 
@@ -259,18 +290,50 @@ export class KeyValueDatastore<
 export class SubscribableDatastore<
 	V extends SubscribableValue,
 > extends Datastore<V["id"], V> {
+	/** Last time an update occurred */
+	lastUpdatedAtMs: number = 0;
+
+	private static maxUpdatedAtMs(values: Iterable<SubscribableValue>) {
+		let max = 0;
+		for (const value of values) {
+			// Prevent one NaN, undefined, +Inf or similar from breaking the datastore.
+			if (Number.isFinite(value.updatedAtMs)) {
+				max = Math.max(max, value.updatedAtMs);
+			}
+		}
+		return max;
+	}
+
+	private incrementLastUpdatedMs() {
+		this.lastUpdatedAtMs = Math.max(Date.now(), this.lastUpdatedAtMs + 1);
+	}
+
+	constructor(
+		provider: DatastoreProvider<V["id"], V> = new MemoryDatastoreProvider(),
+		data = new Map<V["id"], V>(),
+	) {
+		super(provider, data);
+		this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values());
+	}
+
+	async load() {
+		await super.load();
+		this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values());
+	}
+
 	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
 	set(value: V) {
+		this.incrementLastUpdatedMs();
 		this.data.set(value.id, value);
-		value.updatedAtMs = Date.now();
+		value.updatedAtMs = this.lastUpdatedAtMs;
 		this.touch([value]);
 	}
 
 	// Set many values at once from an array of key value pairs
 	setMany(values: V[]) {
-		const nowMs = Date.now();
+		this.incrementLastUpdatedMs();
 		for (const value of values) {
-			value.updatedAtMs = nowMs;
+			value.updatedAtMs = this.lastUpdatedAtMs;
 			this.data.set(value.id, value);
 		}
 		this.touch(values);
@@ -278,18 +341,19 @@ export class SubscribableDatastore<
 
 	// Delete a value in the datastore
 	delete(value: V) {
+		this.incrementLastUpdatedMs();
 		this.data.delete(value.id);
-		value.updatedAtMs = Date.now();
+		value.updatedAtMs = this.lastUpdatedAtMs;
 		value.isDeleted = true;
 		this.touch([value]);
 	}
 
 	// Delete many values at once from an array of values
 	deleteMany(values: V[]) {
-		const nowMs = Date.now();
+		this.incrementLastUpdatedMs();
 		for (const value of values) {
 			value.isDeleted = true;
-			value.updatedAtMs = nowMs;
+			value.updatedAtMs = this.lastUpdatedAtMs;
 			this.data.delete(value.id);
 		}
 		this.touch(values);

packages/lib/src/errors.ts

@@ -10,6 +10,13 @@ export class CommandError extends Error {
 	code = "CommandError";
 }
 
+/**
+ * Thrown if the installation is broken, such as when multiple copies of lib is imported
+ */
+export class InstallationError extends Error {
+	code = "InstallationError";
+}
+
 /**
  * Thrown from requests sent when an error occured handling it
  */

packages/lib/src/export.ts

@@ -0,0 +1,129 @@
+import { FactorioColor } from "./lua_tools";
+
+/**
+ * A localised string.
+ * @see [LocalisedString](https://lua-api.factorio.com/stable/types/LocalisedString.html)
+ */
+export type FactorioLocalisedString = string | [string, ...FactorioLocalisedString[]];
+
+/**
+ * Shared properties of all prototypes.
+ * @see [PrototypeBase](https://lua-api.factorio.com/stable/types/PrototypeBase.html)
+ */
+export interface FactorioPrototypeBase {
+	[index: string]: unknown,
+	type: string,
+	name: string,
+	order?: string,
+	localised_name?: FactorioLocalisedString,
+	localised_description?: FactorioLocalisedString,
+	factoriopedia_description?: FactorioLocalisedString,
+	subgroup?: string,
+	hidden?: boolean,
+	hidden_in_factoriopedia?: boolean,
+	parameter?: boolean,
+	factoriopedia_simulation?: unknown,
+}
+
+/**
+ * Stage the mod setting is part of.
+ * @see [settings_type property](https://wiki.factorio.com/Tutorial:Mod_settings#The_setting_type_property)
+ */
+export type FactorioModSettingType = "startup" | "runtime-global" | "runtime-per-user";
+
+/**
+ * Boolean mod setting prototype.
+ * @see [bool-setting](https://wiki.factorio.com/Tutorial:Mod_settings#bool-setting)
+ */
+export interface FactorioBoolSettingPrototype extends FactorioPrototypeBase{
+	type: "bool-setting",
+	setting_type?: FactorioModSettingType,
+	default_value: boolean,
+	forced_value?: boolean,
+}
+
+/**
+ * Integer mod setting prototype.
+ * @see [int-setting](https://wiki.factorio.com/Tutorial:Mod_settings#int-setting)
+ */
+export interface FactorioIntSettingPrototype extends FactorioPrototypeBase{
+	type: "int-setting",
+	setting_type?: FactorioModSettingType,
+	default_value: number,
+	minimum_value?: number,
+	maximum_value?: number,
+	allowed_values?: number[],
+}
+
+/**
+ * Real number mod setting prototype.
+ * @see [double-setting](https://wiki.factorio.com/Tutorial:Mod_settings#double-setting)
+ */
+export interface FactorioDoubleSettingPrototype extends FactorioPrototypeBase{
+	type: "double-setting",
+	setting_type?: FactorioModSettingType,
+	default_value: number,
+	minimum_value?: number,
+	maximum_value?: number,
+	allowed_values?: number[],
+}
+
+/**
+ * String mod setting prototype.
+ * @see [string-setting](https://wiki.factorio.com/Tutorial:Mod_settings#string-setting)
+ */
+export interface FactorioStringSettingPrototype extends FactorioPrototypeBase{
+	type: "string-setting",
+	setting_type?: FactorioModSettingType,
+	default_value: string,
+	allow_black?: boolean,
+	auto_trim?: string,
+	allowed_values?: string[],
+}
+
+/**
+ * Color mod setting prototype.
+ * @see [color-setting](https://wiki.factorio.com/Tutorial:Mod_settings#color-setting)
+ */
+export interface FactorioColorSettingPrototype extends FactorioPrototypeBase{
+	type: "color-setting",
+	setting_type?: FactorioModSettingType,
+	default_value: FactorioColor,
+	forced_value?: boolean,
+}
+
+export type FactorioModSettingPrototype =
+	| FactorioBoolSettingPrototype
+	| FactorioIntSettingPrototype
+	| FactorioDoubleSettingPrototype
+	| FactorioStringSettingPrototype
+	| FactorioColorSettingPrototype
+;
+
+/**
+ * A list of prototypes as it appears in Factorio's Lua API.
+ * @see [Data.raw](https://lua-api.factorio.com/stable/types/Data.html#raw)
+ */
+export type FactorioPrototypes<Prototype = FactorioPrototypeBase> = Record<string, Record<string, Prototype>>
+export type FactorioDefinesValue = number | string | { [index: string]: FactorioDefinesValue };
+export type FactorioDefines = {
+	[index: string]: FactorioDefinesValue,
+	prototypes: Record<string, Record<string, number>>,
+};
+
+export type ExportSettings = FactorioPrototypes<FactorioModSettingPrototype>;
+export type ExportPrototypes = FactorioPrototypes;
+export type ExportLocale = [string, string][];
+export interface ExportMetadataEntry {
+	name: string,
+	type: string,
+	localised_name?: FactorioLocalisedString,
+	icon?: {
+		x: number,
+		y: number,
+		size: number,
+		path?: string,
+	},
+}
+export type ExportMetadata = Record<string, ExportMetadataEntry[]>;
+export type ExportDefines = FactorioDefines;

packages/lib/src/external/FactorioVersions.ts

@@ -1,7 +1,7 @@
 import { Static, Type } from "@sinclair/typebox";
 import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data/version";
 
-const ARCHIVE_URL = "https://factorio.com/download/archive";
+const ARCHIVE_URL = "https://factorio.com/download/archive/";
 
 /**
  * Represents the parsed result of a factorio version
@@ -60,7 +60,7 @@ function parseFactorioVersions(html: string): ExternalFactorioVersion[] {
 			versions.set(version, {
 				version: version,
 				stable: kind === "stable",
-				headlessUrl: `www.factorio.com/get-download/${version}/headless/linux64`,
+				headlessUrl: `https://www.factorio.com/get-download/${version}/headless/linux64`,
 			});
 		}
 	}

packages/lib/src/file_ops.ts

@@ -1,9 +1,12 @@
 /**
  * @module lib/file_ops
  */
-import fs from "fs-extra";
+import crypto from "crypto";
+import events from "events";
+import fs from "node:fs/promises";
 import path from "path";
-import crypto from "crypto"; // needed for getTempFile
+import stream from "stream";
+
 
 /**
  * Returns the newest file in a directory
@@ -43,7 +46,7 @@ export async function getNewestFile(directory: string, filter = (_name: string)
  * @returns The size in bytes of all files in the directory.
  */
 export async function directorySize(directory: string) {
-	let dirEntries: fs.Dirent[];
+	let dirEntries;
 	try {
 		dirEntries = await fs.readdir(directory, { withFileTypes: true });
 	} catch (err: any) {
@@ -88,8 +91,12 @@ export async function findUnusedName(directory: string, name: string, extension
 	}
 
 	while (true) {
-		if (!await fs.pathExists(path.join(directory, `${name}${extension}`))) {
-			return `${name}${extension}`;
+		try {
+			await fs.access(path.join(directory, `${name}${extension}`));
+		} catch (err: any) {
+			if (err.code === "ENOENT") {
+				return `${name}${extension}`;
+			}
 		}
 
 		let match = /^(.*?)(-(\d+))?$/.exec(name)!;
@@ -122,12 +129,13 @@ export async function getTempFile(prefix = "tmp.", suffix = "", tmpdir = "./") {
 }
 
 /**
- * Safely write data to a file
+ * Safely overwrite file data
  *
- * Same as fs-extra.outputFile except the data is written to a temporary
- * file that's renamed over the target file.  The name of the temporary file
- * is the same as the target file with the suffix `.tmp` added before the
- * extension.
+ * Writes the passed data to a temporary file and rename it over the target
+ * file.  If the operation fails because the parent directory does not exist, it
+ * attempts to create the directory, including any ancestors and then retries
+ * the operation.  The name of the temporary file is the same as the target file
+ * with the suffix `.tmp` added before the extension.
  *
  * If the operation fails it may leave behind the temporary file.  This
  * should not be too much of an issue as the next time the same file is
@@ -135,26 +143,32 @@ export async function getTempFile(prefix = "tmp.", suffix = "", tmpdir = "./") {
  *
  * @param file - Path to file to write.
  * @param data - Content to write.
- * @param options - see fs.writeFile, `flag` must not be set.
+ * @param options - see fs.writeFile, `flag` and `flush` may not be used.
  */
-export async function safeOutputFile(file: string, data: string | Buffer, options: fs.WriteFileOptions ={}) {
+export async function safeOutputFile(
+	file: string,
+	data: string | Buffer,
+	options?: Omit<
+		Extract<Parameters<typeof fs.writeFile>[2], { flush?: boolean }>,
+		"flag" | "flush"
+	>,
+) {
 	let { dir, name, ext } = path.parse(file);
-	if (dir && !await fs.pathExists(dir)) {
-		await fs.mkdirs(dir);
-	}
 	let temporary = path.join(dir, `${name}.tmp${ext}`);
-	let fd = await fs.open(temporary, "w");
 	try {
-		await fs.writeFile(fd, data, options);
-		await fs.fsync(fd);
-	} finally {
-		await fs.close(fd);
+		await fs.writeFile(temporary, data, { ...options, flush: true });
+	} catch (err: any) {
+		if (err.code === "ENOENT") {
+			// Try creating the folder and then retry the operation.
+			await fs.mkdir(dir, { recursive: true });
+			await fs.writeFile(temporary, data, { ...options, flush: true });
+		} else {
+			throw err;
+		}
 	}
-
 	await fs.rename(temporary, file);
 }
 
-
 // Reserved names by almost all filesystems
 const badNames = [".", ".."];
 
@@ -235,3 +249,93 @@ export function cleanFilename(name: string) {
 
 	return name;
 }
+
+/**
+ * Creates a write stream to a guaranteed newly created temporary file in the
+ * same folder as the target `filePath`.
+ *
+ * @param filePath - The target file to create a temporary stream for.
+ * @returns The path to the temporary file that was opened and the write stream to it.
+ */
+export async function createTempWriteStream(filePath: string) {
+	const { dir, name: fileName, ext } = path.parse(filePath);
+	let increment = 1;
+	let writeStream;
+	let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext });
+	while (true) {
+		try {
+			writeStream = (await fs.open(tempFilePath, "wx")).createWriteStream({ flush: true });
+			break;
+		} catch (err: any) {
+			if (err.code === "EEXIST") {
+				increment += 1;
+				tempFilePath = path.format({ dir, name: `${fileName}-${increment}.tmp`, ext });
+			} else {
+				throw err;
+			}
+		}
+	}
+
+	return [tempFilePath, writeStream] as const;
+}
+
+/**
+ * Download a file by fetching the given url and streaming the body content to it.
+ *
+ * @param url - The resource to download
+ * @param filePath - Path to the file store the resource data in.
+ * @param overwriteMode - What to do if the target filePath already exists. Use
+ *     overwrite to clobber it, use rename to write to a differently named file,
+ *     and error to throw an EEXIST error.
+ * @returns file path that was written to. May be different from `filePath` if
+ * overwriteMode is set to rename.
+ */
+export async function downloadFile(
+	url: URL | string,
+	filePath: string,
+	overwriteMode: "overwrite" | "rename" | "error",
+) {
+	// Prepare file to stream content to.
+	let writeStream;
+	let tempFilePath;
+	if (overwriteMode === "error") {
+		tempFilePath = filePath;
+		writeStream = (await fs.open(tempFilePath, "wx")).createWriteStream({ flush: true });
+	} else {
+		[tempFilePath, writeStream] = await createTempWriteStream(filePath);
+	}
+
+	// Fetch stream bytes
+	const response = await fetch(url);
+	if (!response.ok || !response.body) {
+		writeStream.end();
+		await events.once(writeStream, "close");
+		await fs.unlink(tempFilePath);
+		throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
+	}
+	const writer = stream.Writable.toWeb(writeStream);
+	await response.body.pipeTo(writer);
+
+	let targetFilePath = filePath;
+	if (overwriteMode === "overwrite") {
+		await fs.rename(tempFilePath, targetFilePath);
+	} else if (overwriteMode === "rename") {
+		let increment = 1;
+		let { dir, name, ext } = path.parse(targetFilePath);
+		while (true) {
+			try {
+				await fs.writeFile(targetFilePath, Buffer.alloc(0), { flag: "wx" });
+				break;
+			} catch (err: any) {
+				if (err.code === "EEXIST") {
+					increment += 1;
+					targetFilePath = path.format({ dir, name: `${name}-${increment}`, ext });
+				} else {
+					throw err;
+				}
+			}
+		}
+		await fs.rename(tempFilePath, targetFilePath);
+	}
+	return targetFilePath;
+}

packages/lib/src/hash.ts

@@ -3,7 +3,7 @@
  * @module lib/hash
  */
 import crypto from "crypto";
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 
 
 /**
@@ -35,6 +35,6 @@ export function hashStream(stream: NodeJS.ReadableStream): Promise<string> {
  * @param path - Path to the file to hash.
  * @returns hash of the file given.
  */
-export function hashFile(path: string) {
-	return hashStream(fs.createReadStream(path));
+export async function hashFile(path: string) {
+	return hashStream((await fs.open(path)).createReadStream());
 }

packages/lib/src/link/connectors.ts

@@ -431,7 +431,6 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
 	constructor(
 		protected _url: string,
 		maxReconnectDelay: number,
-		protected _tlsCa: string | undefined,
 	) {
 		super(undefined as any, new libData.Address(libData.Address.controller, 0));
 		this._backoff = new ExponentialBackoff({ max: maxReconnectDelay });
@@ -543,17 +542,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
 
 		// Open WebSocket to controller
 		logger.verbose(`Connector | connecting to ${url}`);
-
-
-		if (process.env.APP_ENV === "browser") {
-			this._socket = new WebSocket(url) as WebSocketClusterio;
-
-		} else {
-			let options: { ca?: string } = {};
-			if (this._tlsCa) { options.ca = this._tlsCa; }
-			this._socket = new WebSocket(url, options) as WebSocketClusterio;
-		}
-
+		this._socket = new WebSocket(url) as WebSocketClusterio;
 		this._attachSocketHandlers();
 	}
 

packages/lib/src/link/link.ts

@@ -8,7 +8,7 @@ import { BaseConnector, WebSocketBaseConnector } from "./connectors";
 import { strict as assert } from "assert";
 import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin";
 import type {
-	AddressType, JSONDeserialisable, MessageRoutable, MessageRequest, MessageEvent, IControllerUser,
+	AddressType, JSONDeserialisable, MessageRoutable, MessageRequest, MessageEvent, IUser,
 } from "../data";
 
 export interface Request<Req, Res> {
@@ -18,7 +18,7 @@ export interface Request<Req, Res> {
 		type: "request";
 		src: AddressType | readonly AddressType[];
 		dst: AddressType | readonly AddressType[];
-		permission?: null | string | ((user: IControllerUser, message: MessageRequest) => void);
+		permission?: null | string | ((user: IUser, message: MessageRequest) => void);
 		plugin?: string;
 		Response?: JSONDeserialisable<Res>,
 	}
@@ -32,7 +32,7 @@ export interface Event<T> {
 		type: "event";
 		src: AddressType | readonly AddressType[];
 		dst: AddressType | readonly AddressType[];
-		permission?: null | string | ((user: IControllerUser, message: MessageEvent) => void);
+		permission?: null | string | ((user: IUser, message: MessageEvent) => void);
 		plugin?: string;
 	}
 }

packages/lib/src/link/messages.ts

@@ -23,6 +23,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	controller.ControllerRestartRequest,
 	controller.ControllerUpdateRequest,
 	controller.ControllerConfigGetRequest,
+	controller.ControllerConfigSetRequest,
 	controller.ControllerConfigSetFieldRequest,
 	controller.ControllerConfigSetPropRequest,
 	controller.HostGenerateTokenRequest,
@@ -43,6 +44,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	host.HostRestartRequest,
 	host.HostUpdateRequest,
 	host.HostConfigGetRequest,
+	host.HostConfigSetRequest,
 	host.HostConfigSetFieldRequest,
 	host.HostConfigSetPropRequest,
 	host.HostListRequest,
@@ -59,6 +61,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	instance.InstanceDetailsUpdatesEvent,
 	instance.InstanceCreateRequest,
 	instance.InstanceConfigGetRequest,
+	instance.InstanceConfigSetRequest,
 	instance.InstanceConfigSetFieldRequest,
 	instance.InstanceConfigSetPropRequest,
 	instance.InstanceAssignRequest,

packages/lib/src/load_plugin_list.ts

@@ -1,4 +1,4 @@
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import path from "path";
 import { logger } from "./logging";
 import * as libFileOps from "./file_ops";
@@ -174,12 +174,15 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
  */
 async function checkPackageJson(pluginPath: string): Promise<boolean> {
 	const packageJsonPath = path.join(pluginPath, "package.json");
-	if (!await fs.exists(packageJsonPath)) {
-		return false;
-	}
 
 	// Read and parse package.json
-	const packageJson = JSON.parse(await fs.readFile(packageJsonPath, { encoding: "utf8" }));
+	let content;
+	try {
+		content = await fs.readFile(packageJsonPath, { encoding: "utf8" });
+	} catch (err: any) {
+		return false;
+	}
+	const packageJson = JSON.parse(content);
 
 	// Check if package has the clusterio-plugin keyword
 	return packageJson.keywords?.includes("clusterio-plugin");

packages/lib/src/logging_utils.ts

@@ -4,7 +4,8 @@
  * @author Hornwitser
  * @module lib/logging_utils
  */
-import fs from "fs-extra";
+import fs from "node:fs/promises";
+import type { ReadStream } from "node:fs";
 import path from "path";
 import stream from "stream";
 import util from "util";
@@ -320,7 +321,7 @@ export class LogIndex {
 	async indexFile(file: string) {
 		let filePath = path.join(this.logDirectory, file);
 		let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
-		let fileStream = fs.createReadStream(filePath);
+		let fileStream = (await fs.open(filePath)).createReadStream();
 		fileStream.pipe(lineStream);
 		let entry = {
 			levels: new Set() as Set<keyof typeof levels>,
@@ -478,11 +479,11 @@ export async function queryLog(logDirectory: string, filter: QueryLogFilter, ind
 			continue;
 		}
 		let filePath = path.join(logDirectory, file);
-		let fileStream: fs.ReadStream;
+		let fileStream: ReadStream;
 		let lineStream: libStream.LineSplitter;
 		if (filter.order === "asc") {
 			lineStream = new libStream.LineSplitter({ readableObjectMode: true });
-			fileStream = fs.createReadStream(filePath);
+			fileStream = (await fs.open(filePath)).createReadStream();
 		} else {
 			lineStream = new libStream.ReverseLineSplitter({ readableObjectMode: true });
 			fileStream = await libStream.createReverseReadStream(filePath);

packages/lib/src/permissions.ts

@@ -6,9 +6,7 @@
  * @author Hornwitser
  * @module lib/users
  */
-import { Permission, Role } from "./data";
-import { SubscribableDatastore } from "./datastore";
-
+import { Permission } from "./data";
 
 export const permissions = new Map<string, Permission>();
 
@@ -58,54 +56,6 @@ export function definePermission({
 	permissions.set(name, new Permission(name, title, description, grantByDefault));
 }
 
-
-/**
- * Ensure the default admin role has full access
- *
- * Ensures the role with id 0 exisits and has the core.admin permission.
- *
- * @param roles - role storage to modify.
- * @returns the default admin role
- */
-export function ensureDefaultAdminRole(roles: SubscribableDatastore<Role>) {
-	let admin = roles.get(0);
-	if (!admin) {
-		admin = Role.fromJSON({
-			id: 0,
-			name: "Cluster Admin",
-			description: "Cluster wide administrator.",
-			permissions: [],
-		});
-	}
-	admin.permissions.add("core.admin");
-	roles.set(admin);
-	return admin;
-}
-
-/**
- * Ensure the default player role has default accesses
- *
- * Ensures the role with id 1 exists and has all the permissions that are
- * assigned by default.  Note that this may not be the same role as the one
- * that is assigned to new users by default.
- *
- * @param roles - role storage to modify.
- */
-export function ensureDefaultPlayerRole(roles: SubscribableDatastore<Role>) {
-	let player = roles.get(1);
-	if (!player) {
-		player = Role.fromJSON({
-			id: 1,
-			name: "Player",
-			description: "Default player role.",
-			permissions: [],
-		});
-	}
-	player.grantDefaultPermissions();
-	roles.set(player);
-}
-
-
 // Definitions for the built in permissions used in Clusterio.
 // description should answer "this permission allows you to ___"
 definePermission({

packages/lib/src/plugin.ts

@@ -7,6 +7,14 @@ import type { Logger } from "./logging";
 import type { FieldDefinition } from "./config";
 import type { PlayerStats } from "./data";
 
+
+export const PluginFeatureFlags = [
+	/** The plugin requires module code to be patched into the save */
+	"SavePatching",
+	/** The plugin requires access to script commands over rcon */
+	"ScriptCommands",
+] as const;
+
 /** Used to define the plugin export in plugins */
 export type PluginDeclaration = {
 	name: string;
@@ -26,6 +34,8 @@ export type PluginDeclaration = {
 	webEntrypoint?: string;
 	controlConfigFields?: Record<string, FieldDefinition>;
 
+	features?: (typeof PluginFeatureFlags)[number][];
+
 	messages?: any[];
 	routes?: string[];
 }

packages/lib/src/plugin_loader.ts

@@ -4,7 +4,7 @@
  * @module lib/plugin_loader
  */
 import path from "path";
-import fs from "fs/promises";
+import fs from "node:fs/promises";
 import * as libErrors from "./errors";
 import * as libPlugin from "./plugin";
 import { logger } from "./logging";
@@ -45,6 +45,9 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
 			pluginInfo = require(pluginPath).plugin;
 			pluginPackage = require(path.posix.join(pluginPath, "package.json"));
 		} catch (err: any) {
+			if (err.code === "InstallationError") {
+				throw err;
+			}
 			throw new libErrors.PluginError(pluginName, err);
 		}
 

packages/lib/src/prometheus.ts

@@ -272,18 +272,31 @@ function labelsToKey(labels: LabelValues, metricLabels: string[]) {
 function keyToLabels(key: string) {
 	let labels = new Map<string, string>();
 
-	if (key !== "") {
-		for (let pair of key.split(",")) {
-			let [name, value] = pair.split("=", 2);
-			if (!name || value === undefined) {
-				continue;
+	let pos = 0;
+	let matchLabel = /([^=]+)="(([^\\"]|\\["n\\])+)"/y;
+	while (pos < key.length) {
+		if (pos !== 0) {
+			/* istanbul ignore if */
+			if (key[pos] !== ",") {
+				// Should be unreachable
+				throw Error(`Expected , at pos ${pos} in "${key}"`);
 			}
+			pos += 1;
+		}
+		matchLabel.lastIndex = pos;
+		const match = matchLabel.exec(key);
+		/* istanbul ignore else */
+		if (match) {
+			let [, name, value] = match;
 			labels.set(name, value
-				.slice(1, -1)
 				.replace(/\\"/g, "\"")
 				.replace(/\\n/g, "\n")
 				.replace(/\\\\/g, "\\")
 			);
+			pos += match[0].length;
+		} else {
+			// Should be unreachable
+			throw new Error(`Failed to parse label at pos ${pos} in "${key}"`);
 		}
 	}
 

packages/lib/src/stream.ts

@@ -1,6 +1,10 @@
 import assert from "assert";
-import fs from "fs-extra";
+import fs from "node:fs"; // Need the non-promise api for reverse stream
 import stream from "stream";
+import util from "node:util";
+
+const fsOpen = util.promisify(fs.open);
+const fsFstat = util.promisify(fs.fstat);
 
 
 /**
@@ -13,7 +17,7 @@ import stream from "stream";
  * ```ts
  * let lineStream = new lib.LineSplitter({ readableObjectMode: true });
  * lineStream.on("data", line => { console.log(line.toString("utf8")); });
- * let fileStream = fs.createReadStream(path);
+ * let fileStream = (await fs.open(path)).createReadStream();
  * fileStream.pipe(lineStream);
  * ```
  */
@@ -141,8 +145,8 @@ export async function createReverseReadStream(path: string, options?: ReadStream
 	if (typeof options === "string") {
 		options = { encoding: options };
 	}
-	const fileFd = await fs.open(path, "r");
-	const fileSize = (await fs.fstat(fileFd)).size;
+	const fileFd = await fsOpen(path, "r");
+	const fileSize = (await fsFstat(fileFd)).size;
 	let filePosition = fileSize;
 	const reverseFs = {
 		read(
@@ -162,8 +166,5 @@ export async function createReverseReadStream(path: string, options?: ReadStream
 		open() { assert(false); },
 		close(fd: number, callback: () => void) { return fs.close(fd, callback); },
 	};
-	return fs.createReadStream(
-		"",
-		{ ...options, fd: fileFd, fs: reverseFs } as ReadStreamOptions, // type specification is missing fs prop.
-	);
+	return fs.createReadStream("", { ...options, fd: fileFd, fs: reverseFs });
 }

packages/lib/src/subscriptions.ts

@@ -1,6 +1,6 @@
 import { Type, Static } from "@sinclair/typebox";
 import { Link, Event, EventClass, RequestHandler, WebSocketBaseConnector } from "./link";
-import { Address, MessageRequest, IControllerUser, plainJson, JsonBoolean } from "./data";
+import { Address, MessageRequest, IUser, plainJson, JsonBoolean } from "./data";
 import isDeepStrictEqual from "./is_deep_strict_equal";
 import { RequestError, SessionLost } from "./errors";
 import { logger } from "./logging";
@@ -18,7 +18,7 @@ export class SubscriptionRequest {
 	static type = "request" as const;
 	static src = ["control", "host", "instance"] as const;
 	static dst = "controller" as const;
-	static permission(user: IControllerUser, message: MessageRequest) {
+	static permission(user: IUser, message: MessageRequest) {
 		if (typeof message.data === "object" && message.data !== null) {
 			const data = message.data as Static<typeof SubscriptionRequest.jsonSchema>;
 			const entry = Link._eventsByName.get(data[0]);
@@ -195,6 +195,7 @@ export class SubscriptionController {
 			throw new Error(`Event ${event.eventName} is not a registered as subscribable`);
 		}
 		if (event.subscribe === false) {
+			// Unsubscribe logic
 			const subscriber = eventData.subscriptions.find(sub => sub.dst.addressedTo(src));
 			if (!subscriber) {
 				return false;

packages/lib/src/system_collectors.ts

@@ -1,4 +1,4 @@
-import fs from "fs-extra";
+import fs from "node:fs/promises";
 import os from "os";
 import util from "util";
 import { Gauge } from "./prometheus";
@@ -92,14 +92,7 @@ export const systemDiskCapacityBytes = new Gauge(
 	"Size of the filesystem of the current working directory Node.js runs on",
 	{
 		callback: async function(collector) {
-			// statfs was added in Node.js v18.15.0 and may not be present.
-			// TODO: remove this check once minimum supported Node.js >= v18.15.0.
-			if (!fs.statfs) {
-				collector.set(0);
-				return;
-			}
-			const statFsAsync = util.promisify(fs.statfs);
-			const stats = await statFsAsync(".");
+			const stats = await fs.statfs(".");
 			collector.set(stats.blocks * stats.bsize);
 		},
 	},
@@ -110,13 +103,7 @@ export const systemDiskAvailableBytes = new Gauge(
 	"Available space on the filesystem of the current working directory Node.js runs on",
 	{
 		callback: async function(collector) {
-			// TODO: remove this check once minimum supported Node.js >= v18.15.0.
-			if (!fs.statfs) {
-				collector.set(0);
-				return;
-			}
-			const statFsAsync = util.promisify(fs.statfs);
-			const stats = await statFsAsync(".");
+			const stats = await fs.statfs(".");
 			collector.set(stats.bavail * stats.bsize);
 		},
 	},
@@ -134,7 +121,12 @@ function minZip<T>(a: T[], b: T[]): [T, T][] {
 
 let previousTotalCpuMs: number[] = [];
 let previousIdleCpuMs: number[] = [];
-export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean, restartRequired: boolean) {
+export async function gatherSystemInfo(
+	id: SystemInfo["id"],
+	canRestart: boolean,
+	restartRequired: boolean,
+	startedAtMs: number
+) {
 	const cpus = os.cpus();
 	const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
 	const currentIdleCpuMs = cpus.map(({ times }) => times.idle);
@@ -143,15 +135,11 @@ export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean
 	const cpuUsage = minZip(deltaTotalCpuMs, deltaIdleCpuMs).map(([total, idle]) => (total - idle) / total);
 	previousTotalCpuMs = currentTotalCpuMs;
 	previousIdleCpuMs = currentIdleCpuMs;
-	let diskCapacity = 0;
-	let diskAvailable = 0;
-	if (fs.statfs) { // TODO: remove this check once minimum supported Node.js >= v18.15.0.
-		const statFsAsync = util.promisify(fs.statfs);
-		const stats = await statFsAsync(".");
-		diskCapacity = stats.blocks * stats.bsize;
-		diskAvailable = stats.bavail * stats.bsize;
-	}
+	const stats = await fs.statfs(".");
+	let diskCapacity = stats.blocks * stats.bsize;
+	let diskAvailable = stats.bavail * stats.bsize;
 
+	const now = Date.now();
 	return new SystemInfo(
 		id,
 		os.hostname(),
@@ -166,7 +154,9 @@ export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean
 		diskAvailable,
 		canRestart,
 		restartRequired,
-		Date.now(),
+		now - (os.uptime() * 1000),
+		startedAtMs,
+		now,
 		false,
 	);
 }