v2.0.0-alpha.19

* Add bidirectional sync to admin and ban list

* Add tests

* Response to comments

* Skip broken tests from change of default value

.devcontainer/devcontainer.json

@@ -0,0 +1,4 @@
+{
+	"image":"mcr.microsoft.com/devcontainers/universal:2",
+	"postCreateCommand": "wget https://factorio.com/get-download/1.1.107/headless/linux64 && tar -xf linux64 && pnpm install",
+}

.eslintrc.js

@@ -23,7 +23,9 @@ module.exports = {
 				"@typescript-eslint/no-useless-constructor": "error",
 				"@typescript-eslint/no-unused-vars": "off",
 				"@typescript-eslint/triple-slash-reference": "off",
+				"@typescript-eslint/no-shadow": ["error", { "allow": ["Event", "Request", "yargs"] }],
 				"prefer-const": "off",
+				"no-shadow": "off",
 			},
 		},
 		{

.github/pull_request_template.md

@@ -0,0 +1,17 @@
+Describe why and what has been changed by this pullrequest, feel free to also ask for input on decisions you are unsure about etc.
+The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferrentially link to the associated issues. If there are no relevant issues, link to the pull request itself.
+
+## Changelog
+```
+### Features
+- Added doodad to web interface. [#999](https://github.com/clusterio/clusterio/issues/999)
+
+### Changes
+- ...
+
+### Fixes
+- ...
+
+### Breaking changes
+- ...
+```

.github/workflows/node.js.yml

@@ -12,41 +12,46 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x, 20.x]
+        node-version: [18.x, 20.x, 22.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+        factorio-version: [1.1.110]
+        # Supported versions are latest stable/experimental and one minor version before stable (even if from a previous major) 
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: pnpm/action-setup@v2.4.0
+    - uses: actions/checkout@v4
+    - uses: pnpm/action-setup@v4
       with:
         version: 8
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
-    - run: pnpm i --no-optional --no-frozen-lockfile
-    - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
+    - name: Prepare and install
+      run: pnpm i --no-frozen-lockfile
+    - name: Use Factorio ${{ matrix.factorio-version }}
+      run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/${{ matrix.factorio-version }}/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
     - name: Run tests
-      if: ${{ matrix.node-version != '18.x' }}
+      if: ${{ matrix.node-version != '18.x' || matrix.factorio-version != '1.1.110' }}
       run: pnpm test
     - name: Run coverage
-      if: ${{ matrix.node-version == '18.x' }}
+      if: ${{ matrix.node-version == '18.x' && matrix.factorio-version == '1.1.110' }}
       run: pnpm run ci-cover
     - name : Upload coverage to Codecov
-      if: ${{ matrix.node-version == '18.x' }}
-      uses: codecov/codecov-action@v2
+      if: ${{ matrix.node-version == '18.x' && matrix.factorio-version == '1.1.110' }}
+      uses: codecov/codecov-action@v4
       with:
         files: ./coverage/lcov.info
+        token: ${{ secrets.CODECOV_TOKEN }}
 
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: pnpm/action-setup@v2.4.0
+    - uses: actions/checkout@v4
+    - uses: pnpm/action-setup@v4
       with:
         version: 8
-    - uses: actions/setup-node@v3
+    - uses: actions/setup-node@v4
       with:
         node-version: 18.x
-    - run: pnpm i --no-optional --no-frozen-lockfile
+    - run: pnpm i --no-frozen-lockfile
     - run: pnpm run lint

.gitignore

@@ -31,6 +31,7 @@ node_modules
 
 Factorio*
 /factorio
+!/test/file/factorio
 /instances
 /mods
 /database

CHANGELOG.md

@@ -23,6 +23,103 @@ Many thanks to the following for contributing to this release:
 [@username](https://github.com/username)
 -->
 
+## Version 2.0.0-alpha.19
+
+### Major Features
+
+- Added support for 2.0 builtin mods during mod pack creation. [#684](https://github.com/clusterio/clusterio/pull/684)
+- Added basic 2.0 support to clusterio lua api and 1st party plugins. [#685](https://github.com/clusterio/clusterio/pull/685)
+
+### Features
+
+- Added Datastore and Datastore Provider classes to the library to support savable Maps. [#629](https://github.com/clusterio/clusterio/pull/629)
+- Added clean script to speed up publishing and debugging of build steps. [#640](https://github.com/clusterio/clusterio/pull/640)
+- Added sub-tile option on plugin page headers. [#642](https://github.com/clusterio/clusterio/pull/642)
+- Greatly improved plugin and module template generation. [#641](https://github.com/clusterio/clusterio/pull/641)
+- Duplicate host connections from the same address will kill the older host process. [#647](https://github.com/clusterio/clusterio/pull/647)
+- Controller will refuse duplicate host connections from different addresses. [#647](https://github.com/clusterio/clusterio/pull/647)
+- Monorepos are now supported for external plugins. [#654](https://github.com/clusterio/clusterio/pull/654)
+- Changed default role selection to be a dropdown menu rather than id. [#664](https://github.com/clusterio/clusterio/pull/664)
+- Added credential option to config entries to allow writing sensitive data that can't be read back remotely.
+- Allow hidden and readonly graphical representations of config values. [#665](https://github.com/clusterio/clusterio/pull/665)
+- Added locale exporting for all builtin mods, not just for the base mod. [#684](https://github.com/clusterio/clusterio/pull/684)
+- Added option for silent testing which only shows the results, can be ran using `pnpm silent-test` [#680](https://github.com/clusterio/clusterio/pull/680)
+- Added prompt for http port on installation of Clusterio. [#564](https://github.com/clusterio/clusterio/issues/564)
+- Added drag and drop overlay to upload button on mods page and save table on instance page. [#591](https://github.com/clusterio/clusterio/issues/591)
+- Automatically add plugins from ./plugins, ./external_plugins and installed npm modules. [#692](https://github.com/clusterio/clusterio/pull/692)
+- `compat.table_to_json` and `compat.json_to_table` supported in all stages in pre 2.0 [#687](https://github.com/clusterio/clusterio/issues/687)
+- Add "connect to server" button to web interface, launches with steam api. [#608](https://github.com/clusterio/clusterio/issues/608)
+- Add space platform name mapping to space platform statistics. [#706](https://github.com/clusterio/clusterio/pull/706)
+- Track space platform speed metric clusterio_statistics_exporter_platform_speed. [#706](https://github.com/clusterio/clusterio/pull/706)
+- Track space platform weight metric clusterio_statistics_exporter_platform_weight. [#706](https://github.com/clusterio/clusterio/pull/706)
+- Added bidirectional sync to admin and ban list. [#709](https://github.com/clusterio/clusterio/pull/709)
+
+### Fixes
+
+- Fixed IPC errors reporting undefined when no stack trace was present. [#624](https://github.com/clusterio/clusterio/pull/624)
+- Fixed loopback routing for instances reporting the wrong error message. [#625](https://github.com/clusterio/clusterio/pull/625)
+- Fixed docs linking to non-existing js files after ts migration. [#633](https://github.com/clusterio/clusterio/pull/633)
+- Fixed CI workflows using deprecated version of nodejs. [#637](https://github.com/clusterio/clusterio/pull/637)
+- Fixed linux only tests to run on windows. [#661](https://github.com/clusterio/clusterio/pull/661)
+- Fixed log events being sent from a host over an invalid websocket. [#639](https://github.com/clusterio/clusterio/pull/639)
+- Fixed spaces passed in arguments to the installer causing it to break. [#620](https://github.com/clusterio/clusterio/issues/620)
+- Fixed numeric admin name breaking login. [#536](https://github.com/clusterio/clusterio/issues/536)
+- Fixed issues with subscriptions which prevented them from working on instances or generic events. [#655](https://github.com/clusterio/clusterio/pull/655).
+- Fixed unreliable datastore tests. [#662](https://github.com/clusterio/clusterio/pull/662).
+- Fixed 2.0 version extraction from linux headless. downloaded during installation of clusterio. [#671](https://github.com/clusterio/clusterio/pull/671)
+- Fixed invalid transient state during server start. [#676](https://github.com/clusterio/clusterio/issues/676)
+- Fixed host user sync to instances after reconnecting to the. controller. [#678](https://github.com/clusterio/clusterio/pull/678)
+- Fixed WebUI adding and displaying the wrong base mod version. [#684](https://github.com/clusterio/clusterio/pull/684)
+- Fixed WebUI inconsistent pattern matching for factorio version. [#684](https://github.com/clusterio/clusterio/pull/684)
+- Fixed `safeOutputFile` not being atomic because of a missing path separator. [#659](https://github.com/clusterio/clusterio/pull/659)
+- Fixed built-in mods showing as missing in the WebUI. [#581](https://github.com/clusterio/clusterio/issues/581)
+- Fixed dirty flag not being set when only the config of an instance is updated. [#675](https://github.com/clusterio/clusterio/pull/675)
+- Fixed incorrect command being used to remove players from the whitelist. [#679](https://github.com/clusterio/clusterio/pull/679)
+- Fixed CI following undocumented breaking change from downstream dependency. [#683](https://github.com/clusterio/clusterio/pull/683)
+- Fixed instance crash when missing saves folder. [#698](https://github.com/clusterio/clusterio/issues/698)
+- Fixed statistics exporter not working on newer versions of windows 11 due to wmic deprecation. [#707](https://github.com/clusterio/clusterio/issues/707)
+
+### Changes
+
+- Added prompt for the HTTP port to host the controller to the installer. [#619](https://github.com/clusterio/clusterio/pull/619)
+- Added tests for sendEvent and sendRequest throwing the correct error messages. [#638](https://github.com/clusterio/clusterio/pull/638)
+- Updated all web ui pages to use correct page header component. [#643](https://github.com/clusterio/clusterio/pull/643)
+- Bumped Typescript version to 5.5 and implemented `${configDir}` in base configs for "outDir". [#648](https://github.com/clusterio/clusterio/pull/648).
+- Added factorio.executable_path option which allows overriding the default path to the Factorio executable run.
+- Added instance version display. [#573](https://github.com/clusterio/clusterio/pull/573)
+- Added `factorio.shutdown_timeout` config to set the time the host will wait for a Factorio server to shut down befor killing it.
+- Changed shutdown logic to prefer sending a /quit command via RCON instead of using diverging logic on Windows and Linux.
+- Added `controller.factorio_username` and `controller.factorio_token` config to set Factorio credentials used for the whole cluster.
+- Added `controller.share_factorio_credential_with_hosts` config to optionally require hosts to provide their own credentials.
+- Added `host.factorio_username` and `host.factorio_token` config to set Factorio credentials used on a given host.
+- Changed `instance.rcon_password` config to be a credential field that can not be read by the control interface.
+- Changed `host.id` `instance.id` and `instance.assign_host` config to be hidden fields that can not be seen on the web ui.
+- Updated display name and description of `controller.external_address` to avoid confusion. [#674](https://github.com/clusterio/clusterio/pull/674)
+- Renamed config value `controller.external_address` to `controller.public_url`, migrations are applied automatically.
+- String config values with whitelisted values (enum) now display as a selection dropdown. [#701](https://github.com/clusterio/clusterio/pull/701)
+- Existing selection dropdowns now support search filtering. [#701](https://github.com/clusterio/clusterio/pull/701)
+- Added surface label to clusterio_statistics_exporter_instance_force_flow_statistics. [#691](https://github.com/clusterio/clusterio/issues/691)
+- Compat lib now supports versions from 0.17 to 2.0 [#711](https://github.com/clusterio/clusterio/pull/711)
+- Add bidirectional sync to admin and ban list. [#709](https://github.com/clusterio/clusterio/pull/709)
+
+### Breaking Changes
+
+- Previously silent errors for controller.sendEvent now throw exceptions [#625](https://github.com/clusterio/clusterio/pull/625).
+- @clusterio/controller export InstanceInfo has added factorioVersion parameter.
+- Argument order for `Config.canAccess` changed to require an access mode passed as the second argument.
+- Instances can now have `factorio.enable_script_commands` disabled which will throw an error when any script command is used over rcon. [#681](https://github.com/clusterio/clusterio/pull/681).
+- User IDs are now case insensitive, duplicate users will be automatically merged with a backup created. [#682](https://github.com/clusterio/clusterio/pull/682)
+- Plugins now require the keyword "clusterio-plugin" in package.json to be automatically loaded [#692](https://github.com/clusterio/clusterio/pull/692)
+
+Many thanks to the following for contributing to this release:  
+[@CCpersonguy](https://github.com/CCpersonguy)
+[@Cooldude2606](https://github.com/Cooldude2606)
+[@Danielv123](https://github.com/Danielv123)
+[@Hornwitser](https://github.com/Hornwitser)
+[@Laar](https://github.com/Laar)
+[@psihius](https://github.com/psihius)
+
+
 ## Version 2.0.0-alpha.18
 
 ### Features

README.md

@@ -198,7 +198,7 @@ For the standalone installation mode you can use
 If you plan to make your cluster available externally set the address
 that it will be accessible under with, for example
 
-    npx clusteriocontroller config set controller.external_address http://203.0.113.4:1234/
+    npx clusteriocontroller config set controller.public_url http://203.0.113.4:1234/
 
 Change the url to reflect the IP, protocol, and port the controller is accessible under, dns names are also supported.
 If you're planning on making the controller accessible on the internet it's recommended to set up TLS, see the [Setting Up TLS](/docs/setting-up-tls.md) document for more details.
@@ -209,12 +209,17 @@ See the [readme for @clusterio/controller](/packages/controller/README.md) for m
 
 ## Managing Factorio mods
 
-Clusterio currently only provides a manual method for running mods on servers by copying them either to each instance's individual `mods` folder, or by copying them into a host installation's `sharedMods` folder which will link/copy them to the `mods` folder of each instance on that host if the file doesn't already exist in the `mods` folder for that instance.
-Mods are copied from `sharedMods` on Windows and symlinked on Linux, this means that when mods are removed from `sharedMods` the mods will also be removed from the instance's `mods` folder on Linux if it got there by being linked by Clusterio, but not on Windows.
+Mods in clusterio are managed as parts of modpacks. There is currently no mod portal integration, so you first need to navigate to the mods section of the web UI and upload your mods. You can then create a modpack from the uploaded mods by navigating to the mods page, hitting `create`. Once you have a modpack, you can assign it to an instance under Instance config -> Factorio -> Mod pack. You can also set a default modpack for new instances in the controller settings.
 
-For managing the mod settings, the `mod-settings.dat` and `mod-list.json` files will also be linked/copied over if present in `sharedMods`, but since Factorio creates these files automatically it might be necessary to delete them from each instance's `mods` folder for changes to them in `sharedMods` to take effect.
+![Select mod pack for instance](docs/assets/config_select_mod_pack.png)
 
-Better remote management of mods is a planned feature.
+Mod settings can be managed under the modpack configuration page in the web UI. For mod settings to show, you first need to run an export. This is done by creating an instance with the modpack assigned, opening the instance page and pressing More -> Export data.
+
+![Export mod configuration data](docs/assets/export_mod_config_data.png)
+
+Once this is done, the mod settings will show up under the modpack configuration page.
+
+![Mod settings configuration in web interface](docs/assets/mod_settings_configuration.png)
 
 
 ## Setting up shared storage
@@ -231,7 +236,7 @@ After completing the installation start up the controller and at least one host
 The installer provides the `run-controller` and `run-host` scripts to make this simple.
 Once the controller process is running you can log into the web interface which is hosted by default on http://localhost:8080/ (adjust the port number if you changed it), use the admin authentication token provided from the installation to log in.
 
-The basics of setting up a Factorio server from the web interface is to create an instance, assign it to a host and then click start.
+The basics of setting up a Factorio server from the web interface is to create an instance, assign it to a host and then click start. You can also use the "Connect via Steam" button to launch Factorio directly and connect to the server using Steam's protocol handler.
 
 ### Running via Systemd
 

clean.js

@@ -0,0 +1,116 @@
+/**
+ * Clean the repository of build and execution artifacts.
+ *
+ * clean: A quick clean that removes all build artifacts making the output safe to publish
+ * clean-tests: Removes all build and test artifacts should not be required if tests are setup correctly.
+ * clean-all: Removes all build, test, and execution artifacts. Including cluster config and logs!
+ */
+
+"use strict";
+const path = require("path");
+const fs = require("fs-extra");
+const yargs = require("yargs");
+
+const npmPackage = require("./package.json");
+const workspaces = npmPackage.workspaces
+	.map(p => p.slice(0, -2))
+	.filter(p => !p.includes("external_plugins"));
+
+let DRY = false;
+
+/**
+ * Attempt to remove a file / directory if it exists.
+ *
+ * @param {string} filePath The path to remove.
+ */
+async function tryRemove(filePath) {
+	if (await fs.exists(filePath)) {
+		// eslint-disable-next-line no-console
+		console.log(filePath);
+	}
+
+	if (!DRY) {
+		await fs.remove(filePath);
+	}
+}
+
+/**
+ * Removes all build artifacts
+ */
+async function removeBuildArtifacts() {
+	const tasks = [];
+
+	for (const workspace of workspaces) {
+		const basePath = path.resolve(workspace);
+		const packages = await fs.readdir(basePath);
+		for (const packageName of packages) {
+			tasks.push(
+				tryRemove(path.join(basePath, packageName, "dist"))
+			);
+		}
+	}
+
+	await Promise.all(tasks);
+}
+
+/**
+ * Removes all test artifacts
+ */
+async function removeTestArtifacts() {
+	const artifacts = [
+		".nyc_output", "coverage", "temp", "static",
+	];
+
+	await Promise.all(artifacts.map(artifact => tryRemove(path.resolve(artifact))));
+}
+
+/**
+ * Removes all execution artifacts
+ */
+async function removeExecutionArtifacts() {
+	const artifacts = [
+		"database", "instances", "logs", "mods",
+		"config-control.json", "config-controller.json", "config-host.json",
+		"FactorioAdminToken.txt", "plugin-list.json",
+	];
+
+	await Promise.all(artifacts.map(artifact => tryRemove(path.resolve(artifact))));
+}
+
+/**
+ * Main function for this script
+ */
+async function main() {
+	await yargs
+		.scriptName("clean")
+		.usage("$0 <command> [options]")
+		.option("dry", {
+			nargs: 0,
+			describe: "Will not delete any files",
+			default: false,
+			type: "boolean",
+		})
+		.command("all", "Remove all artifacts", () => {}, argv => {
+			DRY = argv.dry;
+			removeBuildArtifacts();
+			removeTestArtifacts();
+			removeExecutionArtifacts();
+		})
+		.command("tests", "Remove all build and test artifacts", () => {}, argv => {
+			DRY = argv.dry;
+			removeBuildArtifacts();
+			removeTestArtifacts();
+		})
+		.command("fast", "Remove all build artifacts", () => {}, argv => {
+			DRY = argv.dry;
+			removeBuildArtifacts();
+		})
+		.demandCommand(1, "You need to specify a command to run")
+		.strict()
+		.parse();
+}
+
+// Run main if started from command line
+if (module === require.main) {
+	main();
+}

docs/communication.md

@@ -30,7 +30,7 @@ There's also the virtual connector which lets two links in the same program be c
 ## Message
 
 There's two types of messages currently implemented apart from the Link builtin close message and that's Request and Event.
-The builtin messages are defined in [packages/lib/link/messages.js](/packages/lib/link/messages.js) and are attached to links via the attachAllMessages function, which is usally called in the Link subclass constructor.
+The builtin messages are defined in [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) and are attached to links via the attachAllMessages function, which is usally called in the Link subclass constructor.
 
 The messages define which links they are valid on and only attaches handlers for those links.
 There's also a forwarding mechanism for when a request or event is to be forwarded another hop.

docs/config-system.md

@@ -6,7 +6,7 @@ Clusterio uses a field based configuration system to manage settings for the con
 Plugins can add their own config fields for the controller and instances configs.
 The fields are defined early in the startup, and fields for disabled plugins will still be created.
 
-The built-in config fields are defined in [packages/lib/config/definitions.js](/packages/lib/config/definitions.js) while plugins can add their own config field definitions to the `controllerConfigFields` and `instanceConfigFields` properties of the `plugin` export.
+The built-in config fields are defined in [packages/lib/src/config/definitions.ts](/packages/lib/src/config/definitions.ts) while plugins can add their own config field definitions to the `controllerConfigFields` and `instanceConfigFields` properties of the `plugin` export.
 A `<plugin_name>.load_plugin` field is automatically created for each plugin and it's used to enable/disable the loading plugins on startup.
 
 

docs/configuration.md

@@ -51,9 +51,9 @@ interfaces.
 Defaults to null.
 
 
-### controller.external_address
+### controller.public_url
 
-External address the controller is accessible on.
+Public address the controller is accessible on.
 Currently only used for `clusteriocontroller bootstrap create-ctl-config` in order to give the right url to connect to.
 This should be a full URL ending with a /.
 

docs/contributing.md

@@ -300,14 +300,16 @@ You will need commit access to the repository as well publish access to the @clu
 
     Check and make sure there are no untracked files with `git status`.
 
-2.  Update the version in package files.
+2.  Check all PRs merged since the last version and copy their change logs into the main changelog.
+
+3.  Update the version in package files.
     Make sure to update the command for the right version jump.
 
     ```sh
     sed -i 's/2\.0\.0-alpha\.12/2\.0\.0-alpha\.13/' {packages,plugins}/*/package.json
     ```
 
-3.  Inspect, commit and tag the changes.
+4.  Inspect, commit and tag the changes.
     Again, make sure to be tagging the right version.
 
     ```sh
@@ -316,9 +318,9 @@ You will need commit access to the repository as well publish access to the @clu
     git tag v2.0.0-alpha.13
     ```
 
-4.  Publish the packages.
+5.  Publish the packages.
 
     ```sh
-    rm -r ./{plugins,packages}/*/dist
+    pnpm clean
     pnpm -r --filter='./{plugins,packages}/*' publish
     ```

docs/devs/protocol.md

@@ -1,7 +1,7 @@
 # Protocol
 
 Clusterio communicates using a simple homebrew message passing protocol based on JSON over WebSocket with limited support for reliable transport of messages.
-The protocol is implemented in [packages/lib/link](/packages/lib/link) and uses the `ws` library to handle the WebSocket connections on the Node.js side and the native WebSocket API on the browser side.
+The protocol is implemented in [packages/lib/src/link](/packages/lib/src/link) and uses the `ws` library to handle the WebSocket connections on the Node.js side and the native WebSocket API on the browser side.
 
 Two general kinds of messages are supported: events and requests.
 Events are messages sent by one party and received by the other and offer no notification for the sender on whether it was actually received.
@@ -166,7 +166,7 @@ Events are messages send in one direction over the link, that the receiver is ex
 By convention message types ending in `*_event` are events.
 The data payload of an event consists solely of event specific properties.
 
-See [packages/lib/link/messages.js](/packages/lib/link/messages.js) for the recognized core events and their contents.
+See [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) for the recognized core events and their contents.
 Plugins may defined their own events as well.
 
 #### `*_event`
@@ -181,7 +181,7 @@ They function a lot like HTTP requests except that both parties of a connection
 By convention message types ending in `*_request` are requests, and is expected to be replied to with a corresponding `*_response` message.
 If an error occured while processing the request an error response containing an error property in the data payload is sent instead.
 
-See [packages/lib/link/messages.js](/packages/lib/link/messages.js) for the recognized core requests and their contents.
+See [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) for the recognized core requests and their contents.
 Plugins may define their own requests as well.
 
 #### `*_request`

package.json

@@ -1,16 +1,21 @@
 {
 	"name": "root",
 	"private": true,
+	"files": [],
 	"scripts": {
 		"build": "tsc --build",
 		"watch": "tsc --build --watch",
-		"test": "pnpm run build && mocha --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
+		"test": "pnpm run build && mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
+		"silent-test": "SILENT_TEST=y pnpm test",
 		"fast-test": "FAST_TEST=y pnpm test",
 		"cover": "nyc pnpm test",
 		"fast-cover": "FAST_TEST=y nyc pnpm test",
 		"ci-cover": "nyc --reporter=lcovonly pnpm run-script test",
 		"lint": "eslint packages plugins test",
 		"lint-fix": "eslint --fix packages plugins test",
+		"clean": "node ./clean.js fast",
+		"clean-tests": "node ./clean.js tests",
+		"clean-all": "node ./clean.js all",
 		"docs": "typedoc",
 		"bundle-dependencies": "bundle-dependencies"
 	},
@@ -23,11 +28,18 @@
 	"engines": {
 		"node": ">=18"
 	},
+	"workspaces": [
+		"packages/*",
+		"plugins/*",
+		"external_plugins/*",
+		"external_plugins/*/*"
+	],
 	"devDependencies": {
 		"@clusterio/controller": "workspace:*",
 		"@clusterio/ctl": "workspace:*",
 		"@clusterio/host": "workspace:*",
 		"@clusterio/lib": "workspace:*",
+		"@clusterio/web_ui": "workspace:*",
 		"@swc/core": "^1.4.0",
 		"@typescript-eslint/eslint-plugin": "^6.4.1",
 		"@typescript-eslint/parser": "^6.4.1",
@@ -41,7 +53,8 @@
 		"mocha": "^10.2.0",
 		"nyc": "^15.1.0",
 		"phin": "^3.7.0",
-		"typedoc": "^0.24.8",
-		"typescript": "^5.1.6"
+		"typedoc": "^0.26.3",
+		"typescript": "^5.5.3",
+		"yargs": "^17.7.2"
 	}
 }

packages/controller/controller.ts

@@ -66,9 +66,9 @@ void new lib.Gauge(
 			if (!controller || !controller.hosts) {
 				return;
 			}
-			for (let [id, host] of controller.hosts) {
+			for (const host of controller.hosts.values()) {
 				gauge.labels({
-					host_id: String(id),
+					host_id: String(host.id),
 					host_name: host.name,
 				}).set(1);
 			};
@@ -86,9 +86,9 @@ void new lib.Gauge(
 			if (!controller || !controller.instances) {
 				return;
 			}
-			for (let [id, instance] of controller.instances) {
+			for (const instance of controller.instances.values()) {
 				gauge.labels({
-					instance_id: String(id),
+					instance_id: String(instance.id),
 					instance_name: String(instance.config.get("instance.name")),
 					host_id: String(instance.config.get("instance.assigned_host")),
 				}).set(1);
@@ -139,7 +139,7 @@ async function handleBootstrapCommand(
 			return;
 		}
 
-		let admin = userManager.users.get(args.name);
+		let admin = userManager.getByName(args.name);
 		if (!admin) {
 			admin = userManager.createUser(args.name);
 		}
@@ -150,14 +150,14 @@ async function handleBootstrapCommand(
 		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
 
 	} else if (subCommand === "generate-user-token") {
-		let user = userManager.users.get(args.name);
+		let user = userManager.getByName(args.name);
 		if (!user) {
 			logger.error(`No user named '${args.name}'`);
 			process.exitCode = 1;
 			return;
 		}
 		// eslint-disable-next-line no-console
-		console.log(userManager.signUserToken(user.name));
+		console.log(userManager.signUserToken(user));
 
 	} else if (subCommand === "generate-host-token") {
 		// eslint-disable-next-line no-console
@@ -167,7 +167,7 @@ async function handleBootstrapCommand(
 		));
 
 	} else if (subCommand === "create-ctl-config") {
-		let admin = userManager.users.get(args.name);
+		let admin = userManager.getByName(args.name);
 		if (!admin) {
 			logger.error(`No user named '${args.name}'`);
 			process.exitCode = 1;
@@ -178,7 +178,7 @@ async function handleBootstrapCommand(
 		controlConfig.set("control.controller_url", Controller.calculateControllerUrl(controllerConfig));
 		controlConfig.set(
 			"control.controller_token",
-			userManager.signUserToken(admin.name),
+			userManager.signUserToken(admin),
 		);
 
 		let content = JSON.stringify(controlConfig, null, "\t");
@@ -248,8 +248,12 @@ async function initialize(): Promise<InitializeParameters> {
 		.command("config", "Manage Controller config", lib.configCommand)
 		.command("bootstrap", "Bootstrap access to cluster", yargs => {
 			yargs
-				.command("create-admin <name>", "Create a cluster admin")
-				.command("generate-user-token <name>", "Generate authentication token for the given user")
+				.command("create-admin <name>", "Create a cluster admin", yargs => {
+					yargs.positional("name", { describe: "Name of the admin user", type: "string" });
+				})
+				.command("generate-user-token <name>", "Generate authentication token for the given user", yargs => {
+					yargs.positional("name", { describe: "Name of the user", type: "string" });
+				})
 				.command("generate-host-token <id>", "Generate authentication token for the given host", yargs => {
 					yargs.positional("id", { describe: "ID of the host", type: "number" });
 				})
@@ -313,14 +317,7 @@ async function initialize(): Promise<InitializeParameters> {
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = new Map();
-	try {
-		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, { encoding: "utf8" })));
-	} catch (err: any) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
+	let pluginList = await lib.loadPluginList(args.pluginList);
 
 	// If the command is plugin management we don't try to load plugins
 	if (command === "plugin") {

packages/controller/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio controller",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",
@@ -13,13 +13,14 @@
 		"clusteriocontroller": "./dist/node/controller.js"
 	},
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {
 		"node": ">=18"
 	},
 	"dependencies": {
-		"@clusterio/lib": "^2.0.0-alpha.18",
+		"@clusterio/lib": "workspace:*",
 		"@sinclair/typebox": "^0.30.4",
 		"busboy": "^1.6.0",
 		"compression": "^1.7.4",
@@ -29,14 +30,13 @@
 		"jsonwebtoken": "^9.0.1",
 		"jszip": "^3.10.1",
 		"set-blocking": "^2.0.0",
-		"webpack-cli": "^5.1.4",
-		"winston": "^3.10.0",
+		"winston": "^3.17.0",
 		"winston-daily-rotate-file": "^4.7.1",
 		"ws": "^8.13.0",
 		"yargs": "^17.7.2"
 	},
 	"devDependencies": {
-		"@clusterio/web_ui": "^2.0.0-alpha.18",
+		"@clusterio/web_ui": "workspace:*",
 		"@swc/core": "^1.4.0",
 		"@types/busboy": "^1.5.0",
 		"@types/compression": "^1.7.2",
@@ -51,8 +51,9 @@
 		"react": "^18.2.0",
 		"react-dom": "^18.2.0",
 		"react-router-dom": "^6.4.14",
-		"typescript": "^5.1.6",
+		"typescript": "^5.5.3",
 		"webpack": "^5.88.2",
+		"webpack-cli": "^5.1.4",
 		"webpack-dev-middleware": "^6.1.1",
 		"webpack-merge": "^5.9.0"
 	},

packages/controller/src/BaseConnection.ts

@@ -35,13 +35,13 @@ export default class BaseConnection extends lib.Link {
 
 		this.handle(lib.SubscriptionRequest, this.handleSubscriptionRequest.bind(this));
 		this.connector.on("close", () => {
-			this._controller.subscriptions.unsubscribe(this);
+			this._controller.subscriptions.unsubscribeLink(this);
 		});
 	}
 
 	async disconnect(code: number, reason: string) {
 		try {
-			await this.connector.disconnect();
+			await this.connector.disconnect(code, reason);
 		} catch (err: any) {
 			if (!(err instanceof lib.SessionLost)) {
 				logger.error(`"Unexpected error preparing disconnect:\n${err.stack}`);

packages/controller/src/ControlConnection.ts

@@ -134,7 +134,7 @@ export default class ControlConnection extends BaseConnection {
 			this.checkPermission(message, entry);
 		} catch (err: any) {
 			this.connector.sendResponseError(new lib.ResponseError(err.message, err.code), message.src);
-			logger.audit(`Permission denied for ${message.name} by ${this.user.name} from ${this.connector.dst}`);
+			logger.audit(`Permission denied for ${message.name} by ${this.user.id} from ${this.connector.dst}`);
 			throw err;
 		}
 	}
@@ -195,7 +195,7 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleHostRevokeTokensRequest(request: lib.HostRevokeTokensRequest) {
-		const host = this._controller.hosts.get(request.hostId);
+		const host = this._controller.hosts.getMutable(request.hostId);
 		if (!host) {
 			throw new Error(`Unknown host id (${request.hostId})`);
 		}
@@ -207,7 +207,7 @@ export default class ControlConnection extends BaseConnection {
 			hostConnection.connector.terminate();
 		}
 
-		this._controller.hostsUpdated([host]);
+		this._controller.hosts.set(host);
 	}
 
 	async handleHostListRequest(): Promise<lib.HostDetails[]> {
@@ -397,8 +397,7 @@ export default class ControlConnection extends BaseConnection {
 		if (this._controller.modPacks.has(modPack.id)) {
 			throw new lib.RequestError(`Mod pack with ID ${modPack.id} already exist`);
 		}
-		this._controller.modPacks.set(modPack.id, modPack);
-		this._controller.modPacksUpdated([modPack]);
+		this._controller.modPacks.set(modPack);
 	}
 
 	async handleModPackUpdateRequest(request: lib.ModPackUpdateRequest) {
@@ -406,19 +405,16 @@ export default class ControlConnection extends BaseConnection {
 		if (modPack.id === undefined || !this._controller.modPacks.has(modPack.id)) {
 			throw new lib.RequestError(`Mod pack with ID ${modPack.id} does not exist`);
 		}
-		this._controller.modPacks.set(modPack.id, modPack);
-		this._controller.modPacksUpdated([modPack]);
+		this._controller.modPacks.set(modPack);
 	}
 
 	async handleModPackDeleteRequest(request: lib.ModPackDeleteRequest) {
 		let { id } = request;
-		let modPack = this._controller.modPacks.get(id);
+		let modPack = this._controller.modPacks.getMutable(id);
 		if (!modPack) {
 			throw new lib.RequestError(`Mod pack with ID ${id} does not exist`);
 		}
-		modPack.isDeleted = true;
-		this._controller.modPacks.delete(id);
-		this._controller.modPacksUpdated([modPack]);
+		this._controller.modPacks.delete(modPack);
 	}
 
 	async handleModGetRequest(request: lib.ModGetRequest) {
@@ -630,7 +626,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.User> {
 		let name = request.name;
-		let user = this._controller.userManager.users.get(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User ${name} does not exist`);
 		}
@@ -648,17 +644,17 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleUserRevokeTokenRequest(request: lib.UserRevokeTokenRequest) {
-		let user = this._controller.userManager.users.get(request.name);
+		let user = this._controller.userManager.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
-		if (user.name !== this.user.name) {
+		if (user.id !== this.user.id) {
 			this.user.checkPermission("core.user.revoke_other_token");
 		}
 
 		user.invalidateToken();
 		for (let controlConnection of this._controller.wsServer.controlConnections.values()) {
-			if (controlConnection.user.name === user.name) {
+			if (controlConnection.user.id === user.id) {
 				controlConnection.connector.terminate();
 			}
 		}
@@ -666,7 +662,7 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleUserUpdateRolesRequest(request: lib.UserUpdateRolesRequest) {
-		let user = this._controller.userManager.users.get(request.name);
+		let user = this._controller.userManager.getByName(request.name);
 		if (!user) {
 			throw new lib.RequestError(`User '${request.name}' does not exist`);
 		}
@@ -684,7 +680,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleUserSetAdminRequest(request: lib.UserSetAdminRequest) {
 		let { name, create, admin } = request;
-		let user = this._controller.userManager.users.get(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
@@ -701,7 +697,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleUserSetBannedRequest(request: lib.UserSetBannedRequest) {
 		let { name, create, banned, reason } = request;
-		let user = this._controller.userManager.users.get(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
@@ -719,7 +715,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleUserSetWhitelistedRequest(request: lib.UserSetWhitelistedRequest) {
 		let { name, create, whitelisted } = request;
-		let user = this._controller.userManager.users.get(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			if (create) {
 				this.user.checkPermission("core.user.create");
@@ -736,7 +732,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleUserDeleteRequest(request: lib.UserDeleteRequest) {
 		let name = request.name;
-		let user = this._controller.userManager.users.get(name);
+		let user = this._controller.userManager.getByName(name);
 		if (!user) {
 			throw new lib.RequestError(`User '${name}' does not exist`);
 		}

packages/controller/src/Controller.ts

@@ -55,17 +55,6 @@ export default class Controller {
 	config: lib.ControllerConfig;
 	app: Application;
 
-	/** True if (@link Controller.systems} has changed since last time it was saved */
-	systemsDirty = false;
-	/** True if {@link Controller.hosts} has changed since last time it was saved */
-	hostsDirty = false;
-	/** True if {@link Controller.instances} has changed since last time it was saved */
-	instancesDirty = false;
-	/** True if {@link Controller.saves} has changed since last time it was saved */
-	savesDirty = false;
-	/** True if {@link Controller.modPacks} has changed since last time it was saved */
-	modPacksDirty = false;
-
 	httpServer: http.Server | null = null;
 	httpServerCloser: HttpCloser | null = null;
 	httpsServer: https.Server | null = null;
@@ -95,7 +84,7 @@ export default class Controller {
 	public shouldRestart: boolean = false;
 	_fallbackedRequests: Map<lib.RequestClass<unknown, unknown>, lib.RequestHandler<unknown, unknown>> = new Map();
 	_registeredRequests: Map<lib.RequestClass<unknown, unknown>, lib.RequestHandler<unknown, unknown>> = new Map();
-	_registeredEvents = new Map();
+	_registeredEvents = new Map<lib.EventClass<unknown>, lib.EventHandler<unknown>>();
 	_snoopedEvents = new Map();
 
 	devMiddleware: any | null = null;
@@ -111,11 +100,33 @@ export default class Controller {
 		let databaseDirectory = config.get("controller.database_directory");
 		await fs.ensureDir(databaseDirectory);
 
-		const systems = await Controller.loadSystems(path.join(databaseDirectory, "systems.json"));
-		const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json"));
-		const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json"));
-		const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json"));
-		const modPacks = await Controller.loadModPacks(path.join(databaseDirectory, "mod-packs.json"));
+		const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "systems.json"),
+			lib.SystemInfo.fromJSON.bind(lib.SystemInfo)
+		).bootstrap());
+
+		const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "hosts.json"),
+			HostInfo.fromJSON.bind(HostInfo),
+			this.migrateHosts, this.finaliseHosts,
+		).bootstrap());
+
+		const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "instances.json"),
+			json => InstanceInfo.fromJSON(json, "controller"),
+			this.migrateInstances, this.finaliseInstances,
+		).bootstrap());
+
+		const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "saves.json"),
+			lib.SaveDetails.fromJSON.bind(lib.SaveDetails)
+		).bootstrap());
+
+		const modPacks = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "mod-packs.json"),
+			lib.ModPack.fromJSON.bind(lib.ModPack)
+		).bootstrap());
+
 		const userManager = new UserManager(config);
 		await userManager.load(path.join(databaseDirectory, "users.json"));
 
@@ -145,15 +156,15 @@ export default class Controller {
 		 * will restart the controller on non-zero exit codes.
 		 */
 		public canRestart: boolean = false,
-		public systems = new Map<lib.SystemInfo["id"], lib.SystemInfo>(),
+		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
 		/** Mapping of host id to host info */
-		public hosts = new Map<number, HostInfo>(),
+		public hosts = new lib.SubscribableDatastore<HostInfo>(),
 		/** Mapping of instance id to instance info */
-		public instances = new Map<number, InstanceInfo>(),
+		public instances = new lib.SubscribableDatastore<InstanceInfo>(),
 		/** Mapping of save id to save details */
-		public saves = new Map<string, lib.SaveDetails>(),
+		public saves = new lib.SubscribableDatastore<lib.SaveDetails>(),
 		/** Mapping of mod pack id to mod pack */
-		public modPacks = new Map<number, lib.ModPack>(),
+		public modPacks = new lib.SubscribableDatastore<lib.ModPack>(),
 		/** Mods stored on the controller */
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
 		/** User and roles manager for the cluster */
@@ -185,6 +196,13 @@ export default class Controller {
 		this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this));
 		this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this));
 		this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this));
+
+		// Handle updates for datastores
+		this.systems.on("update", this.systemsUpdated.bind(this));
+		this.hosts.on("update", this.hostsUpdated.bind(this));
+		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
+		this.saves.on("update", this.savesUpdated.bind(this));
+		this.modPacks.on("update", this.modPacksUpdated.bind(this));
 	}
 
 	async start(args: ControllerArgs) {
@@ -501,10 +519,8 @@ export default class Controller {
 			requests.push(lib.gatherSystemInfo("controller", this.canRestart));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
-				this.systems.set(metric.id, metric);
+				this.systems.set(metric);
 			}
-			this.systemsDirty = true;
-			this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(newMetrics));
 		} catch (err: any) {
 			logger.error(`Unexpected error updating system infos:\n${err.stack ?? err.message}`);
 		}
@@ -545,160 +561,57 @@ export default class Controller {
 			await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t"));
 		}
 
-		let databaseDirectory = this.config.get("controller.database_directory");
-		if (this.systemsDirty) {
-			this.systemsDirty = false;
-			await Controller.saveSystems(path.join(databaseDirectory, "systems.json"), this.systems);
-		}
-
-		if (this.hostsDirty) {
-			this.hostsDirty = false;
-			await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
-		}
-
-		if (this.instancesDirty) {
-			this.instancesDirty = false;
-			await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances);
-		}
-
-		if (this.savesDirty) {
-			this.savesDirty = false;
-			await Controller.saveSaves(path.join(databaseDirectory, "saves.json"), this.saves);
-		}
-
-		if (this.modPacksDirty) {
-			this.modPacksDirty = false;
-			await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks);
-		}
+		await Promise.all([
+			this.systems.save(),
+			this.hosts.save(),
+			this.instances.save(),
+			this.saves.save(),
+			this.modPacks.save(),
+		]);
 
 		if (this.userManager.dirty) {
-			await this.userManager.save(path.join(databaseDirectory, "users.json"));
+			await this.userManager.save(path.join(this.config.get("controller.database_directory"), "users.json"));
 		}
 
 		await lib.invokeHook(this.plugins, "onSaveData");
 	}
 
-	static async loadSystems(filePath: string): Promise<Map<lib.SystemInfo["id"], lib.SystemInfo>> {
-		let json: Static<typeof lib.SystemInfo.jsonSchema>[];
-		try {
-			json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
+	static migrateHosts(rawJson: unknown[]): Static<typeof HostInfo.jsonSchema>[] {
+		let serialized = rawJson as any;
 
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
+		// migrate from pre alpha.19 format
+		if (serialized.length && serialized[0] instanceof Array) {
+			serialized = serialized.map((e: any) => e[1]);
 		}
 
-		return new Map(json.map(m => lib.SystemInfo.fromJSON(m)).map(m => [m.id, m]));
+		return serialized;
 	}
 
-	static async saveSystems(filePath: string, systems: Map<lib.SystemInfo["id"], lib.SystemInfo>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...systems.values()], null, "\t"));
-	}
-
-	static async loadHosts(filePath: string): Promise<Map<number, HostInfo>> {
-		let serialized: [number, Static<typeof HostInfo.jsonSchema>][];
-		try {
-			serialized = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
+	static finaliseHosts(host: HostInfo): HostInfo {
+		if (host.connected) {
+			host.connected = false;
+			host.updatedAtMs = Date.now();
 		}
+		return host;
+	}
 
-		// TODO: migrate remove after release.
-		if (serialized.length && !(serialized[0] instanceof Array)) {
-			return new Map(); // Discard old format.
-		}
-
-		const hosts = new Map();
-		for (let [id, json] of serialized) {
-			const host = HostInfo.fromJSON(json);
-			if (host.connected) {
-				host.connected = false;
-				host.updatedAtMs = Date.now();
+	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceInfo.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof InstanceInfo.jsonSchema>[];
+		return serialized.map(json => {
+			if (!json.config) { // migrate: from pre Alpha 14 format.
+				return { config: json as any, status: "running" }; // Use running to force updatedAtMs
 			}
-			hosts.set(id, host);
+			return json;
+		});
+	}
+
+	static finaliseInstances(instance: InstanceInfo): InstanceInfo {
+		const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown";
+		if (instance.status !== status) {
+			instance.status = status;
+			instance.updatedAtMs = Date.now();
 		}
-		return hosts;
-	}
-
-	static async saveHosts(filePath: string, hosts: Map<number, HostInfo>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...hosts.entries()], null, "\t"));
-	}
-
-	static async loadInstances(filePath: string): Promise<Map<number, InstanceInfo>> {
-		logger.info(`Loading ${filePath}`);
-
-		let instances = new Map();
-		try {
-			let serialized = JSON.parse(
-				await fs.readFile(filePath, { encoding: "utf8" })
-			) as Static<typeof InstanceInfo.jsonSchema>[];
-			for (let json of serialized) {
-				if (!json.config) { // migrate: from pre Alpha 14 format.
-					json = { config: json as any, status: "running" }; // Use running to force updatedAtMs
-				}
-				const instance = InstanceInfo.fromJSON(json, "controller");
-				const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown";
-				if (instance.status !== status) {
-					instance.status = status;
-					instance.updatedAtMs = Date.now();
-				}
-				instances.set(instance.id, instance);
-			}
-
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
-		}
-
-		return instances;
-	}
-
-	static async saveInstances(filePath: string, instances: Map<number, InstanceInfo>) {
-		let serialized = [...instances.values()];
-		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
-	}
-
-	static async loadSaves(filePath: string): Promise<Map<string, lib.SaveDetails>> {
-		let json: Static<typeof lib.SaveDetails.jsonSchema>[];
-		try {
-			json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
-		}
-		return new Map(json.map(s => lib.SaveDetails.fromJSON(s)).map(s => [s.id, s]));
-	}
-
-	static async saveSaves(filePath: string, saves: Map<string, lib.SaveDetails>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...saves.values()], null, "\t"));
-	}
-
-
-	static async loadModPacks(filePath: string): Promise<Map<number, lib.ModPack>> {
-		let json;
-		try {
-			json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
-		}
-		return new Map(json.map((e: any) => [e.id, lib.ModPack.fromJSON(e)]));
-	}
-
-	static async saveModPacks(filePath: string, modPacks: Map<number, lib.ModPack>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t"));
+		return instance;
 	}
 
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
@@ -795,6 +708,10 @@ export default class Controller {
 		}
 	}
 
+	systemsUpdated(systems: lib.SystemInfo[]) {
+		this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(systems));
+	}
+
 	async handleSystemInfoSubscription(request: lib.SubscriptionRequest) {
 		const systems = [...this.systems.values()].filter(
 			metric => metric.updatedAtMs > request.lastRequestTimeMs,
@@ -816,11 +733,6 @@ export default class Controller {
 	}
 
 	hostsUpdated(hosts: HostInfo[]) {
-		const now = Date.now();
-		for (const host of hosts) {
-			host.updatedAtMs = now;
-		}
-		this.hostsDirty = true;
 		let updates = hosts.map(host => host.toHostDetails());
 		this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates));
 	}
@@ -876,8 +788,6 @@ export default class Controller {
 			"tags": ["clusterio"],
 			"max_players": 0,
 			"visibility": { "public": true, "lan": true },
-			"username": "",
-			"token": "",
 			"game_password": "",
 			"require_user_verification": true,
 			"max_upload_in_kilobytes_per_second": 0,
@@ -896,12 +806,10 @@ export default class Controller {
 		};
 		instanceConfig.set("factorio.settings", settings);
 
-		let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, Date.now());
-		this.instances.set(instanceId, instance);
-		this.instancesDirty = true;
+		let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, undefined, Date.now());
+		this.instances.set(instance);
 		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance);
 		this.addInstanceHooks(instance);
-		this.instanceDetailsUpdated([instance]);
 		return instance;
 	}
 
@@ -911,19 +819,7 @@ export default class Controller {
 	 * @internal
 	 */
 	clearSavesOfInstance(instanceId: number) {
-		const updates = [];
-		const now = Date.now();
-		for (const [id, save] of this.saves) {
-			if (save.instanceId === instanceId) {
-				save.isDeleted = true;
-				save.updatedAtMs = now;
-				updates.push(save);
-				this.saves.delete(id);
-			}
-		}
-		if (updates.length) {
-			this.savesUpdated(updates);
-		}
+		this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId));
 	}
 
 	/**
@@ -970,7 +866,6 @@ export default class Controller {
 
 		// Assign to target
 		instance.config.set("instance.assigned_host", hostId ?? null);
-		// "fieldChanged" event handler will set this.instancesDirty
 		if (hostId !== undefined && newHostConnection) {
 			await newHostConnection.send(
 				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host"))
@@ -978,8 +873,7 @@ export default class Controller {
 		} else {
 			instance.status = "unassigned";
 		}
-		instance.updatedAtMs = Date.now();
-		this.instanceDetailsUpdated([instance]);
+		this.instances.set(instance);
 	}
 
 	/**
@@ -996,16 +890,13 @@ export default class Controller {
 		if (hostId !== null) {
 			await this.sendTo({ hostId }, new lib.InstanceDeleteInternalRequest(instanceId));
 		}
-		this.instances.delete(instanceId);
-		this.instancesDirty = true;
+
+		let prev = instance.status;
+		this.instances.delete(instance);
 
 		this.clearSavesOfInstance(instanceId);
 		this.userManager.clearStatsOfInstance(instanceId);
 
-		let prev = instance.status;
-		instance.status = "deleted";
-		instance.updatedAtMs = Date.now();
-		this.instanceDetailsUpdated([instance]);
 		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance, prev);
 	}
 
@@ -1033,16 +924,18 @@ export default class Controller {
 	addInstanceHooks(instance: InstanceInfo) {
 		instance.config.on("fieldChanged", (field: string, curr: any, prev: any) => {
 			instance.updatedAtMs = Date.now();
-			if (field === "instance.name") {
-				this.instanceDetailsUpdated([instance]);
-			}
+			this.instances.set(instance); // Trigger update logic
 
-			this.instancesDirty = true;
 			lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, field, curr, prev);
 		});
 	}
 
 	instanceDetailsUpdated(instances: InstanceInfo[]) {
+		for (const instance of instances) {
+			if (instance.status === "stopped") {
+				this.subscriptions.unsubscribeAddress(lib.Address.fromShorthand({instanceId: instance.id}));
+			}
+		}
 		const updates = instances.map(instance => instance.toInstanceDetails());
 		this.subscriptions.broadcast(new lib.InstanceDetailsUpdatesEvent(updates));
 	}
@@ -1055,7 +948,6 @@ export default class Controller {
 	}
 
 	savesUpdated(saves: lib.SaveDetails[]) {
-		this.savesDirty = true;
 		this.subscriptions.broadcast(new lib.InstanceSaveDetailsUpdatesEvent(saves));
 	}
 
@@ -1066,11 +958,6 @@ export default class Controller {
 
 
 	modPacksUpdated(modPacks: lib.ModPack[]) {
-		const now = Date.now();
-		for (const modPack of modPacks) {
-			modPack.updatedAtMs = now;
-		}
-		this.modPacksDirty = true;
 		this.subscriptions.broadcast(new lib.ModPackUpdatesEvent(modPacks));
 		lib.invokeHook(this.plugins, "onModPacksUpdated", modPacks);
 	}
@@ -1228,7 +1115,7 @@ export default class Controller {
 	}
 
 	static calculateControllerUrl(config: lib.ControllerConfig) {
-		let url = config.get("controller.external_address");
+		let url = config.get("controller.public_url");
 		if (!url) {
 			if (config.get("controller.https_port")) {
 				url = `https://localhost:${config.get("controller.https_port")}/`;
@@ -1351,10 +1238,10 @@ export default class Controller {
 		throw Error(`Unknown type ${requestOrEvent.constructor.type}.`);
 	}
 
-	async sendRequest(request: any, dst: lib.Address) {
+	sendRequest(request: any, dst: lib.Address) {
 		let connection;
 		if (dst.type === lib.Address.controller) {
-			throw new Error("controller as dst is not supported.");
+			throw new Error(`Message would return back to sender ${dst}.`);
 
 		} else if (dst.type === lib.Address.control) {
 			connection = this.wsServer.controlConnections.get(dst.id);
@@ -1389,24 +1276,33 @@ export default class Controller {
 	sendEvent<T>(event: lib.Event<T>, dst: lib.Address) {
 		let connection;
 		if (dst.type === lib.Address.controller) {
-			throw new Error("controller as dst is not supported.");
+			throw new Error(`Message would return back to sender ${dst}.`);
 
 		} else if (dst.type === lib.Address.control) {
 			connection = this.wsServer.controlConnections.get(dst.id);
+			if (!connection) {
+				throw new Error("Target control connection does not exist.");
+			}
 
 		} else if (dst.type === lib.Address.instance) {
 			let instance = this.instances.get(dst.id);
 			if (!instance) {
-				return;
+				throw new Error(`Instance with ID ${dst.id} does not exist`);
 			}
 			let hostId = instance.config.get("instance.assigned_host");
 			if (hostId === null) {
-				return;
+				throw new Error("Instance is not assigned to a host");
 			}
 			connection = this.wsServer.hostConnections.get(hostId);
+			if (!connection) {
+				throw new Error("Host containing instance is not connected");
+			}
 
 		} else if (dst.type === lib.Address.host) {
 			connection = this.wsServer.hostConnections.get(dst.id);
+			if (!connection) {
+				throw new Error("Host is not connected");
+			}
 
 		} else if (dst.type === lib.Address.broadcast) {
 			if (dst.id === lib.Address.control) {
@@ -1424,7 +1320,7 @@ export default class Controller {
 				}
 
 			} else {
-				throw new Error(`Unexpected broacdast target ${dst.id}`);
+				throw new Error(`Unexpected broadcast target ${dst.id}`);
 			}
 			return;
 
@@ -1432,9 +1328,7 @@ export default class Controller {
 			throw new Error(`Unknown address type ${dst.type}`);
 		}
 
-		if (connection) {
-			connection.sendEvent(event, dst);
-		}
+		connection.sendEvent(event, dst);
 	}
 
 	async sendToHostByInstanceId<Req extends InstanceId, Res>(request: Req & lib.Request<Req, Res>): Promise<Res>;

packages/controller/src/HostConnection.ts

@@ -42,8 +42,7 @@ export default class HostConnection extends BaseConnection {
 			0,
 			false,
 		);
-		this._controller.hosts.set(this.id, this.info);
-		this._controller.hostsUpdated([this.info]);
+		this._controller.hosts.set(this.info);
 
 		this._checkPluginVersions();
 
@@ -57,10 +56,9 @@ export default class HostConnection extends BaseConnection {
 		}
 
 		this.connector.on("close", () => {
-			const now = Date.now();
 			// Update status to unknown for instances on this host.
 			const instances: InstanceInfo[] = [];
-			for (let instance of this._controller.instances.values()) {
+			for (let instance of this._controller.instances.valuesMutable()) {
 				if (instance.config.get("instance.assigned_host") !== this.id) {
 					continue;
 				}
@@ -68,21 +66,24 @@ export default class HostConnection extends BaseConnection {
 				instances.push(instance);
 				let prev = instance.status;
 				instance.status = "unknown";
-				instance.updatedAtMs = now;
 				lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 			}
-			this._controller.instanceDetailsUpdated(instances);
+			this._controller.instances.setMany(instances);
 
 			this.info.connected = false;
-			this._controller.hostsUpdated([this.info]);
+			this._controller.hosts.set(this.info);
 		});
 
+		this.handle(lib.GetFactorioCredentialsRequest, this.handleGetFactorioCredentialsRequest.bind(this));
 		this.handle(lib.HostInfoUpdateEvent, this.handleHostInfoUpdateEvent.bind(this));
 		this.handle(lib.InstanceStatusChangedEvent, this.handleInstanceStatusChangedEvent.bind(this));
 		this.handle(lib.InstancesUpdateRequest, this.handleInstancesUpdateRequest.bind(this));
 		this.handle(lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsUpdatesEvent.bind(this));
 		this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this));
 		this.handle(lib.InstancePlayerUpdateEvent, this.handleInstancePlayerUpdateEvent.bind(this));
+		this.snoopEvent(lib.InstanceAdminlistUpdateEvent, this.snoopInstanceAdminlistUpdateEvent.bind(this));
+		this.snoopEvent(lib.InstanceBanlistUpdateEvent, this.snoopInstanceBanlistUpdateEvent.bind(this));
+		this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.snoopInstanceWhitelistUpdateEvent.bind(this));
 	}
 
 	validateIngress(message: lib.MessageRoutable) {
@@ -155,14 +156,33 @@ export default class HostConnection extends BaseConnection {
 		return this.info.id;
 	}
 
+	/**
+	 * Remote address of the host this connection is connected to
+	 *
+	 * @returns {string} host remote address.
+	 */
+	get remoteAddress() {
+		return this.info.remoteAddress;
+	}
+
+	async handleGetFactorioCredentialsRequest() {
+		if (this._controller.config.get("controller.share_factorio_credentials_with_hosts")) {
+			return {
+				username: this._controller.config.get("controller.factorio_username") ?? undefined,
+				token: this._controller.config.get("controller.factorio_token") ?? undefined,
+			};
+		}
+		return {};
+	}
+
 	async handleHostInfoUpdateEvent(event: lib.HostInfoUpdateEvent) {
 		this.info.name = event.update.name;
 		this.info.publicAddress = event.update.publicAddress;
-		this._controller.hostsUpdated([this.info]);
+		this._controller.hosts.set(this.info);
 	}
 
 	async handleInstanceStatusChangedEvent(request: lib.InstanceStatusChangedEvent) {
-		let instance = this._controller.instances.get(request.instanceId);
+		let instance = this._controller.instances.getMutable(request.instanceId);
 
 		// It's possible to get updates from an instance that does not exist
 		// or is not assigned to the host it originated from if it was
@@ -188,9 +208,9 @@ export default class HostConnection extends BaseConnection {
 		let prev = instance.status;
 		instance.status = request.status;
 		instance.gamePort = request.gamePort;
-		instance.updatedAtMs = Date.now();
+		instance.factorioVersion = request.factorioVersion;
+		this._controller.instances.set(instance);
 		logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
-		this._controller.instanceDetailsUpdated([instance]);
 		await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 	}
 
@@ -210,7 +230,7 @@ export default class HostConnection extends BaseConnection {
 			const instanceConfig = new lib.InstanceConfig("controller");
 			instanceConfig.update(instanceData.config, false, "host");
 
-			let controllerInstance = this._controller.instances.get(instanceConfig.get("instance.id"));
+			let controllerInstance = this._controller.instances.getMutable(instanceConfig.get("instance.id"));
 			if (controllerInstance) {
 				// Check if this instance is assigned somewhere else.
 				if (controllerInstance.config.get("instance.assigned_host") !== this.id) {
@@ -225,9 +245,9 @@ export default class HostConnection extends BaseConnection {
 					let prev = controllerInstance.status;
 					controllerInstance.status = instanceData.status;
 					controllerInstance.gamePort = instanceData.gamePort;
-					controllerInstance.updatedAtMs = Date.now();
-					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
+					controllerInstance.factorioVersion = instanceData.factorioVersion;
 					instanceUpdates.push(controllerInstance);
+					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
 					await lib.invokeHook(
 						this._controller.plugins, "onInstanceStatusChanged", controllerInstance, prev
 					);
@@ -240,23 +260,20 @@ export default class HostConnection extends BaseConnection {
 				instanceConfig,
 				instanceData.status,
 				instanceData.gamePort,
+				instanceData.factorioVersion,
 				Date.now(),
 			);
-			this._controller.instances.set(instanceConfig.get("instance.id"), newInstance);
-			this._controller.instancesDirty = true;
+			instanceUpdates.push(newInstance);
 			this._controller.addInstanceHooks(newInstance);
 			await this.send(
 				new lib.InstanceAssignInternalRequest(
 					instanceConfig.get("instance.id"), instanceConfig.toRemote("host")
 				)
 			);
-			instanceUpdates.push(newInstance);
 			await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance);
 		}
 
-		if (instanceUpdates.length) {
-			this._controller.instanceDetailsUpdated(instanceUpdates);
-		}
+		this._controller.instances.setMany(instanceUpdates);
 
 		// Push lists to make sure they are in sync.
 		let adminlist: Set<string> = new Set();
@@ -265,13 +282,13 @@ export default class HostConnection extends BaseConnection {
 
 		for (let user of this._controller.userManager.users.values()) {
 			if (user.isAdmin) {
-				adminlist.add(user.name);
+				adminlist.add(user.id);
 			}
 			if (user.isBanned) {
-				banlist.set(user.name, user.banReason);
+				banlist.set(user.id, user.banReason);
 			}
 			if (user.isWhitelisted) {
-				whitelist.add(user.name);
+				whitelist.add(user.id);
 			}
 		}
 
@@ -279,33 +296,27 @@ export default class HostConnection extends BaseConnection {
 	}
 
 	async handleInstanceSaveDetailsUpdatesEvent(event: lib.InstanceSaveDetailsUpdatesEvent) {
-		const now = Date.now();
 		const updates: lib.SaveDetails[] = [];
+		const deletes: lib.SaveDetails[] = [];
 		for (const save of event.updates) {
 			const existingSave = this._controller.saves.get(save.id);
 			if (existingSave && save.equals(existingSave)) {
 				continue;
 			}
-			save.updatedAtMs = now;
-			this._controller.saves.set(save.id, save);
 			updates.push(save);
 		}
 		if (event.instanceId !== undefined) {
 			const updatedSaves = new Set(event.updates.map(s => s.id));
-			for (const [id, save] of this._controller.saves) {
-				if (save.instanceId === event.instanceId && !updatedSaves.has(id)) {
-					save.isDeleted = true;
-					save.updatedAtMs = now;
-					updates.push(save);
-					this._controller.saves.delete(id);
+			for (const save of this._controller.saves.values()) {
+				if (save.instanceId === event.instanceId && !updatedSaves.has(save.id)) {
+					deletes.push(save);
 				}
 			}
 		}
 		// Hosts eagerly send updates, which means we may get an update
-		// where nothing actualy changed.
-		if (updates.length) {
-			this._controller.savesUpdated(updates);
-		}
+		// where nothing actually changed.
+		this._controller.saves.setMany(updates);
+		this._controller.saves.deleteMany(deletes);
 	}
 
 	async handleLogMessageEvent(event: lib.LogMessageEvent) {
@@ -318,7 +329,7 @@ export default class HostConnection extends BaseConnection {
 
 	async handleInstancePlayerUpdateEvent(event: lib.InstancePlayerUpdateEvent, src: lib.Address) {
 		let instanceId = src.id;
-		let user = this._controller.userManager.users.get(event.name);
+		let user = this._controller.userManager.getByName(event.name);
 		if (!user) {
 			user = this._controller.userManager.createUser(event.name);
 		}
@@ -342,4 +353,66 @@ export default class HostConnection extends BaseConnection {
 			stats: event.stats,
 		});
 	}
+
+	async snoopInstanceAdminlistUpdateEvent(event: lib.InstanceAdminlistUpdateEvent, src: lib.Address) {
+		if (src.type !== lib.Address.instance) {
+			return;
+		}
+
+		let user = this._controller.userManager.getByName(event.name);
+		if (!user) {
+			user = this._controller.userManager.createUser(event.name);
+		}
+
+		user.isAdmin = event.admin;
+		this._controller.usersUpdated([user]);
+		const instance = this._controller.instances.get(src.id)!;
+		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
+			type: event.admin ? "promote" : "demote" as lib.PlayerEvent["type"],
+			name: event.name,
+			stats: user.instanceStats.get(src.id)!,
+		});
+	}
+
+	async snoopInstanceBanlistUpdateEvent(event: lib.InstanceBanlistUpdateEvent, src: lib.Address) {
+		if (src.type !== lib.Address.instance) {
+			return;
+		}
+
+		let user = this._controller.userManager.getByName(event.name);
+		if (!user) {
+			user = this._controller.userManager.createUser(event.name);
+		}
+
+		user.isBanned = event.banned;
+		user.banReason = event.reason;
+		this._controller.usersUpdated([user]);
+		const instance = this._controller.instances.get(src.id)!;
+		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
+			type: event.banned ? "ban" : "unban" as lib.PlayerEvent["type"],
+			name: event.name,
+			stats: user.instanceStats.get(src.id)!,
+			resaon: event.reason,
+		});
+	}
+
+	async snoopInstanceWhitelistUpdateEvent(event: lib.InstanceWhitelistUpdateEvent, src: lib.Address) {
+		if (src.type !== lib.Address.instance) {
+			return;
+		}
+
+		let user = this._controller.userManager.getByName(event.name);
+		if (!user) {
+			user = this._controller.userManager.createUser(event.name);
+		}
+
+		user.isWhitelisted = event.whitelisted;
+		this._controller.usersUpdated([user]);
+		const instance = this._controller.instances.get(src.id)!;
+		await lib.invokeHook(this._controller.plugins, "onPlayerEvent", instance, {
+			type: event.whitelisted ? "whitelisted" : "unwhitelisted" as lib.PlayerEvent["type"],
+			name: event.name,
+			stats: user.instanceStats.get(src.id)!,
+		});
+	}
 }

packages/controller/src/InstanceInfo.ts

@@ -11,6 +11,7 @@ export default class InstanceInfo {
 		public config: lib.InstanceConfig,
 		public status: lib.InstanceStatus,
 		public gamePort?: number,
+		public factorioVersion?: string,
 		public updatedAtMs = 0,
 	) {
 		this.config = config;
@@ -22,6 +23,7 @@ export default class InstanceInfo {
 		"status": lib.InstanceStatus,
 		"gamePort": Type.Optional(Type.Number()),
 		"updatedAtMs": Type.Optional(Type.Number()),
+		"factorioVersion": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>, location: lib.ConfigLocation) {
@@ -29,6 +31,7 @@ export default class InstanceInfo {
 			lib.InstanceConfig.fromJSON(json.config, location),
 			json.status,
 			json.gamePort,
+			json.factorioVersion,
 			json.updatedAtMs,
 		);
 	}
@@ -40,6 +43,7 @@ export default class InstanceInfo {
 			this.config.get("instance.assigned_host") ?? undefined,
 			this.gamePort,
 			this.status,
+			this.factorioVersion,
 			this.updatedAtMs,
 		);
 	}
@@ -48,6 +52,13 @@ export default class InstanceInfo {
 		return this.status === "deleted";
 	}
 
+	set isDeleted(value: boolean) {
+		if (value === false) {
+			throw new Error("Setting isDeleted to false is not supported, use status instead.");
+		}
+		this.status = "deleted";
+	}
+
 	/** Shorthand for `instance.config.get("instance.id")` */
 	get id():number {
 		return this.config.get("instance.id");

packages/controller/src/UserManager.ts

@@ -10,8 +10,8 @@ import ControllerUser from "./ControllerUser";
  * @alias module:controller/src/UserManager
  */
 export default class UserManager {
-	roles: Map<number, lib.Role> = new Map();
-	users: Map<string, ControllerUser> = new Map();
+	roles: Map<lib.Role["id"], lib.Role> = new Map();
+	users: Map<ControllerUser["id"], ControllerUser> = new Map();
 	dirty = false;
 
 	/**
@@ -24,6 +24,10 @@ export default class UserManager {
 	) {
 	}
 
+	getByName(name: string) {
+		return this.users.get(name.toLowerCase());
+	}
+
 	async load(filePath: string): Promise<void> {
 		try {
 			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
@@ -32,9 +36,29 @@ export default class UserManager {
 				this.roles.set(role.id, role);
 			}
 
+			let duplicates = 0;
 			for (let serializedUser of content.users) {
 				let user = ControllerUser.fromJSON(serializedUser, this);
-				this.users.set(user.name, user);
+				const existingUser = this.users.get(user.id);
+				if (existingUser) {
+					// Required migration to all lowercase ids in alpha 19
+					duplicates += 1;
+					// We assume the user with the lower online time is the duplicate
+					if (user.playerStats.onlineTimeMs <= existingUser.playerStats.onlineTimeMs) {
+						existingUser.merge(user);
+						continue; // Skip users.set
+					}
+					user.merge(existingUser);
+				}
+				this.users.set(user.id, user);
+			}
+
+			if (duplicates) {
+				const backupPath = `${filePath}.${Date.now()}.bak`;
+				lib.logger.warn(
+					`A total of ${duplicates} users were merged, a backup was written to: ${backupPath}`
+				);
+				await lib.safeOutputFile(backupPath, JSON.stringify(content, null, "\t"));
 			}
 
 		} catch (err: any) {
@@ -48,7 +72,7 @@ export default class UserManager {
 		}
 	}
 
-	async save(filePath:string): Promise<void> {
+	async save(filePath: string): Promise<void> {
 		let serializedRoles = [];
 		for (let role of this.roles.values()) {
 			serializedRoles.push(role.toJSON());
@@ -72,8 +96,8 @@ export default class UserManager {
 	 * @param name - Name of the user to create.
 	 * @returns The created user.
 	 */
-	createUser(name:string): ControllerUser {
-		if (this.users.has(name)) {
+	createUser(name: string): ControllerUser {
+		if (this.getByName(name)) {
 			throw new Error(`User '${name}' already exists`);
 		}
 
@@ -84,20 +108,20 @@ export default class UserManager {
 		}
 
 		let user = new ControllerUser(this, 0, name, roles);
-		this.users.set(name, user);
+		this.users.set(user.id, user);
 		this.dirty = true;
 		return user;
 	}
 
 	/**
-	 * Sign access token for the given user name
+	 * Sign access token for the given user
 	 *
-	 * @param name - user name to sign token for
+	 * @param user - user to sign token for
 	 * @returns JWT access token for the user.
 	 */
-	signUserToken(name: string): string {
+	signUserToken(user: ControllerUser): string {
 		return jwt.sign(
-			{ aud: "user", user: name },
+			{ aud: "user", user: user.id },
 			Buffer.from(this._config.get("controller.auth_secret"), "base64")
 		);
 	}

packages/controller/src/WsServer.ts

@@ -78,11 +78,11 @@ export default class WsServer {
 
 		let disconnectTasks = [];
 		for (let controlConnection of this.controlConnections.values()) {
-			disconnectTasks.push(controlConnection.disconnect(1001, "Server Quit"));
+			disconnectTasks.push(controlConnection.disconnect(lib.ConnectionClosed.GoingAway, "Server Quit"));
 		}
 
 		for (let hostConnection of this.hostConnections.values()) {
-			disconnectTasks.push(hostConnection.disconnect(1001, "Server Quit"));
+			disconnectTasks.push(hostConnection.disconnect(lib.ConnectionClosed.GoingAway, "Server Quit"));
 		}
 
 		logger.info(`WsServer | Waiting for ${disconnectTasks.length} connectors to close`);
@@ -97,7 +97,7 @@ export default class WsServer {
 		}
 
 		for (let socket of this.pendingSockets) {
-			socket.close(1001, "Server Quit");
+			socket.close(lib.ConnectionClosed.GoingAway, "Server Quit");
 		}
 	}
 
@@ -170,7 +170,7 @@ export default class WsServer {
 ${err.stack}`
 				);
 				wsRejectedConnectionsCounter.inc();
-				socket.close(1011, "Unexpected error");
+				socket.close(lib.ConnectionClosed.InternalError, "Unexpected error");
 			});
 		});
 	}
@@ -184,7 +184,7 @@ ${err.stack}`
 		} catch (err: any) {
 			logger.verbose(`WsServer | closing ${this.remoteAddr(req)} after receiving invalid message`);
 			wsRejectedConnectionsCounter.inc();
-			socket.close(1002, `Invalid message: ${err.message}`);
+			socket.close(lib.ConnectionClosed.ProtocolError, `Invalid message: ${err.message}`);
 			return;
 		}
 
@@ -196,14 +196,14 @@ ${err.stack}`
 		if (message.type !== "registerHost" && message.type !== "registerControl") {
 			logger.verbose(`WsServer | closing ${this.remoteAddr(req)} after receiving invalid handshake`);
 			wsRejectedConnectionsCounter.inc();
-			socket.close(1002, "Bad handshake");
+			socket.close(lib.ConnectionClosed.ProtocolError, "Bad handshake");
 			return;
 		}
 
 		if (this.stopAcceptingNewSessions) {
 			logger.verbose(`WsServer | closing ${this.remoteAddr(req)}, server is shutting down`);
 			wsRejectedConnectionsCounter.inc();
-			socket.close(1001, "Shutting down");
+			socket.close(lib.ConnectionClosed.GoingAway, "Shutting down");
 			return;
 		}
 
@@ -263,15 +263,26 @@ ${err.stack}`
 		} catch (err: any) {
 			logger.verbose(`WsServer | authentication failed for ${this.remoteAddr(req)}`);
 			wsRejectedConnectionsCounter.inc();
-			socket.close(4003, `Authentication failed: ${err.message}`);
+			socket.close(lib.ConnectionClosed.Unauthorized, `Authentication failed: ${err.message}`);
 			return;
 		}
 
-		let [connector, sessionToken] = this.createSession(new lib.Address(lib.Address.host, data.id));
 		let connection = this.hostConnections.get(data.id);
 		if (connection) {
-			logger.verbose(`WsServer | disconnecting existing connection for host ${data.id}`);
-			await connection.disconnect(1008, "Registered from another connection");
+			if (connection.remoteAddress !== this.remoteAddr(req)) {
+				logger.verbose(
+					`WsServer | disallowed duplicate connection from ${this.remoteAddr(req)} for host ${data.id}`
+				);
+				wsRejectedConnectionsCounter.inc();
+				socket.close(lib.ConnectionClosed.PolicyViolation, "Host already connected from another address");
+				return;
+			}
+			logger.verbose(
+				`WsServer | disconnecting duplicate connection from host ${data.id}`
+			);
+			await connection.disconnect(
+				lib.ConnectionClosed.PolicyViolation, "Newer connection for host from same address"
+			);
 		}
 
 		logger.info(
@@ -284,6 +295,8 @@ ${err.stack}`
 			);
 		}
 
+		let [connector, sessionToken] = this.createSession(new lib.Address(lib.Address.host, data.id));
+
 		// Handle close before HostConnection does. This avoids a host
 		// connection event happening between the connection breaking and
 		// it being removed from the host connections list.
@@ -314,7 +327,7 @@ ${err.stack}`
 				throw new Error("unexpected JsonWebToken type");
 			}
 
-			user = this.controller.userManager.users.get(tokenPayload.user);
+			user = this.controller.userManager.getByName(tokenPayload.user);
 			if (!user) {
 				throw new Error("invalid user");
 			}
@@ -326,7 +339,7 @@ ${err.stack}`
 		} catch (err: any) {
 			logger.verbose(`WsServer | authentication failed for ${this.remoteAddr(req)}`);
 			wsRejectedConnectionsCounter.inc();
-			socket.close(4003, `Authentication failed: ${err.message}`);
+			socket.close(lib.ConnectionClosed.Unauthorized, `Authentication failed: ${err.message}`);
 			return;
 		}
 

packages/controller/src/routes.ts

@@ -185,7 +185,7 @@ function validateUserToken(req: Request, res: Response, next: any) {
 		if (typeof tokenPayload === "string") {
 			throw new Error("unexpected JsonWebToken type");
 		}
-		let user = req.app.locals.controller.userManager.users.get(tokenPayload.user);
+		let user = req.app.locals.controller.userManager.getByName(tokenPayload.user);
 		if (!user) {
 			throw new Error("invalid user");
 		}
@@ -218,7 +218,7 @@ async function uploadExport(req: Request, res: Response) {
 		res.sendStatus(400);
 		return;
 	}
-	let modPack = res.app.locals.controller.modPacks.get(modPackId);
+	let modPack = res.app.locals.controller.modPacks.getMutable(modPackId);
 	if (!modPack) {
 		res.sendStatus(400);
 		return;
@@ -261,7 +261,7 @@ async function uploadExport(req: Request, res: Response) {
 
 	modPack.exportManifest = new lib.ExportManifest(assets);
 	modPack.fillDefaultSettings(settingPrototypes, logger);
-	res.app.locals.controller.modPacksUpdated([modPack]);
+	res.app.locals.controller.modPacks.set(modPack);
 
 	res.sendStatus(200);
 }

packages/controller/tsconfig.browser.json

@@ -7,6 +7,5 @@
 	"include": [ "web/**/*.tsx", "web/**/*.ts", "package.json" ],
 	"compilerOptions": {
 		"rootDir": "..",
-		"outDir": "dist/browser",
 	}
 }

packages/controller/tsconfig.node.json

@@ -4,7 +4,4 @@
 		{ "path": "../lib/tsconfig.node.json" },
 	],
 	"include": ["index.ts", "controller.ts", "package.json", "src/**/*.ts"],
-	"compilerOptions": {
-		"outDir": "dist/node",
-	},
 }

packages/create/create.js

@@ -16,6 +16,7 @@ const { copyPluginTemplates } = require("./template");
 let dev = false;
 const scriptExt = process.platform === "win32" ? ".cmd" : "";
 const finished = util.promisify(stream.finished);
+const { escapeArg } = require("./escape_arg");
 
 // I hate this, there is a bug with rl.close on windows that was meant to have been fixed in Node14.4
 // See nodejs/node#21771 and nodejs/node#30701 for the apparent fix that was applied
@@ -106,15 +107,25 @@ async function safeOutputFile(file, data, options={}) {
 }
 
 async function execFile(cmd, args) {
-	logger.verbose(`executing ${cmd} ${args.join(" ")}`);
+	const escaped = args.map(escapeArg);
+	logger.verbose(`executing ${cmd} ${escaped.join(" ")}`);
 	return new Promise((resolve, reject) => {
-		let child = child_process.execFile(cmd, args, { shell: true }, (err, stdout, stderr) => {
-			if (err) {
-				reject(err);
-			} else {
-				resolve({ stdout, stderr });
+		let child = child_process.execFile(
+			cmd,
+			escaped,
+			{
+				shell: true,
+				// eslint-disable-next-line node/no-process-env
+				env: process.platform === "win32" ? { ...process.env, pct: "%" } : undefined,
+			},
+			(err, stdout, stderr) => {
+				if (err) {
+					reject(err);
+				} else {
+					resolve({ stdout, stderr });
+				}
 			}
-		});
+		);
 		let stdout = new LineSplitter({ readableObjectMode: true });
 		stdout.on("data", line => { logger.verbose(line.toString()); });
 		child.stdout.pipe(stdout);
@@ -551,6 +562,25 @@ async function inquirerMissingArgs(args) {
 				},
 			},
 		], answers);
+
+		if (args.httpPort) { answers.httpPort = args.httpPort; }
+		answers = await inquirer.prompt([
+			{
+				type: "input",
+				name: "httpPort",
+				message: "HTTP port to listen on",
+				default: 8080,
+				validate: input => {
+					if (!input) {
+						return "May not be empty";
+					}
+					if (!Number.isInteger(Number(input))) {
+						return "Must be a number";
+					}
+					return true;
+				},
+			},
+		], answers);
 	}
 
 	if (answers.mode === "host") {
@@ -680,6 +710,7 @@ async function inquirerMissingArgs(args) {
 					{ name: "Command Line", value: "ctl" },
 					{ name: "Web UI", value: "web" },
 					{ name: "No Typescript", value: "js" },
+					{ name: "No Config", value: "no_config" },
 				],
 			},
 			{
@@ -715,7 +746,11 @@ async function downloadLinuxServer() {
 	const url = new URL(res.headers.location);
 	// get the filename of the latest factorio archive from redirected url
 	const filename = path.posix.basename(url.pathname);
-	const version = filename.match(/(?<=factorio_headless_x64_).*(?=\.tar\.xz)/)[0];
+	const match = filename.match(/(?<=factorio_headless_x64_|factorio-headless_linux_)\d+\.\d+\.\d+(?=\.tar\.xz)/);
+	if (!match || !match.length) {
+		throw Error(`Unable to extract version from filename: ${filename}`);
+	}
+	const version = match[0];
 
 	const tmpDir = "temp/create-temp/";
 	const archivePath = tmpDir + filename;
@@ -775,6 +810,9 @@ async function main() {
 		.option("admin", {
 			nargs: 1, describe: "Admin account name [standalone/controller]", type: "string",
 		})
+		.option("http-port", {
+			nargs: 1, describe: "HTTP port to listen on [standalone/controller]", type: "number",
+		})
 		.option("host-name", {
 			nargs: 1, describe: "Host name [host]", type: "string",
 		})
@@ -854,6 +892,7 @@ async function main() {
 	if (["standalone", "controller"].includes(answers.mode)) {
 		logger.info("Setting up controller");
 		await execController(["bootstrap", "create-admin", answers.admin]);
+		await execController(["config", "set", "controller.http_port", answers.httpPort]);
 		let result = await execController(["bootstrap", "generate-user-token", answers.admin]);
 		adminToken = result.stdout.split("\n").slice(-2)[0];
 	}
@@ -868,6 +907,8 @@ async function main() {
 		result = await execController(["bootstrap", "generate-host-token", hostId]);
 		let hostToken = result.stdout.split("\n").slice(-2)[0];
 
+		// Default to localhost on correct port for host in standalone mode
+		await execHost(["config", "set", "host.controller_url", `http://localhost:${answers.httpPort}/`]);
 		await execHost(["config", "set", "host.controller_token", hostToken]);
 		await execHost(["config", "set", "host.public_address", answers.publicAddress]);
 		await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]);

packages/create/escape_arg.js

@@ -0,0 +1,62 @@
+"use strict";
+
+function escapeArg(arg) {
+	// When shell: true is used execFile does not escaping of arguments, they are
+	// simply joined by space and then passed to the shell.
+
+	// For Windows the command is cmd.exe /d /s /c "command arg arg ..."
+	// The parsing and escaping on windows is cursed. The following references are not for mortals:
+	// https://daviddeley.com/autohotkey/parameters/parameters.htm#WIN
+	// https://stackoverflow.com/a/4095133
+	// https://ss64.com/nt/syntax-esc.html
+	if (process.platform === "win32") {
+		if (!/[\t "%&<>\\^|]/.test(arg)) {
+			return arg;
+		}
+
+		const escaped = [];
+		let pos = 0;
+		while (pos < arg.length) {
+			if (arg[pos] === "\\") {
+				// If a backslash is followed by a quote or the end of the string, then
+				// double the number of backslashes because why not!
+				let peek = pos + 1;
+				while (peek !== arg.length && arg[peek] === "\\") {
+					peek += 1;
+				}
+				if (peek === arg.length || arg[peek] === '"') {
+					const count = peek - pos;
+					escaped.push("\\".repeat(count * 2));
+					pos = peek;
+					continue;
+				} else {
+					escaped.push("\\");
+				}
+			} else if (arg[pos] === '"') {
+				escaped.push('""');
+			} else if (arg[pos] === "%") {
+				// Expect a pct environment variable to contain %, because cmd does variable substitutions
+				// before parsing string quotes, and there's no mechanism for escaping % symbols, and this
+				// is a half sane workaround. The less sane workaround is to replace % with %%cd:~,%.
+				escaped.push("%pct%");
+			} else {
+				escaped.push(arg[pos]);
+			}
+			pos += 1;
+		}
+		return `"${escaped.join("")}"`;
+	}
+
+	// For all other platforms the command is /bin/sh -c <command arg arg ...>
+	// The command and its arguments are passed as a single argument to the shell.
+	// See https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02
+	// eslint-disable-next-line no-lonely-if
+	if (!/[|&;<>()$`\\"' \t*?\[#~=%]/.test(arg)) {
+		return arg;
+	}
+	return `"${arg.replace(/[$`"\\]/g, "\\$&")}"`;
+}
+
+module.exports = {
+	escapeArg,
+};

packages/create/package.json

@@ -3,14 +3,14 @@
 	"description": "Installer for clusterio",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"main": "create.js",
 	"bin": {
 		"clusteriocreate": "./create.js"
 	},
 	"keywords": [
-		"factorio",
-		"clusterio"
+		"clusterio",
+		"factorio"
 	],
 	"engines": {
 		"node": ">=18"

packages/create/template.js

@@ -76,8 +76,15 @@ async function copyPluginTemplates(pluginName, templates) {
 	const files = new Map();
 	const prepare = [];
 
+	// Some basic flags for selecting files to include
 	const javascriptOnly = templates.includes("js");
-	const webpack = templates.includes("controller") || templates.includes("web");
+	const config = !templates.includes("no_config") && templates.some(value => (
+		["controller", "host", "instance", "ctl"].includes(value)
+	));
+	const webpack = config || templates.includes("controller") || templates.includes("web");
+
+	// A count of the number of isolated contexts the plugin runs under
+	let pluginContexts = 0;
 
 	// Get the file extension and path to the templates
 	const ext = javascriptOnly ? "js" : "ts";
@@ -88,7 +95,6 @@ async function copyPluginTemplates(pluginName, templates) {
 	files.set(".gitignore", path.join(commonPath, ".gitignore"));
 	files.set(".npmignore", path.join(commonPath, ".npmignore"));
 	files.set("package.json", path.join(commonPath, "package.json"));
-	files.set(`messages.${ext}`, path.join(templatePath, `messages.${ext}`));
 	files.set(`index.${ext}`, path.join(templatePath, `index.${ext}`));
 
 	// Files and dependencies to support typescript
@@ -105,24 +111,30 @@ async function copyPluginTemplates(pluginName, templates) {
 	if (webpack) {
 		prepare.push("webpack-cli --env production");
 		files.set("webpack.config.js", path.join(commonPath, "webpack.config.js"));
-		files.set(`web/index.${ext}x`, path.join(templatePath,
-			templates.includes("web") ? `web/plugin.${ext}x` : `web/no_plugin.${ext}x`
-		));
+		if (templates.includes("web")) {
+			files.set(`web/index.${ext}x`, path.join(templatePath, `web/plugin.${ext}x`));
+			pluginContexts += 1;
+		} else {
+			files.set(`web/index.${ext}x`, path.join(templatePath, `web/no_plugin.${ext}x`));
+		}
 	}
 
 	// Files for the controller
 	if (templates.includes("controller")) {
 		files.set(`controller.${ext}`, path.join(templatePath, `controller.${ext}`));
+		pluginContexts += 1;
 	}
 
 	// Files for hosts
 	if (templates.includes("host")) {
 		files.set(`host.${ext}`, path.join(templatePath, `host.${ext}`));
+		pluginContexts += 1;
 	}
 
 	// Files for instances
 	if (templates.includes("instance")) {
 		files.set(`instance.${ext}`, path.join(templatePath, `instance.${ext}`));
+		pluginContexts += 1;
 	}
 
 	// Files for lua modules
@@ -133,16 +145,26 @@ async function copyPluginTemplates(pluginName, templates) {
 		files.set("module/module_exports.lua", path.join(commonPath, "module/module_exports.lua"));
 		if (templates.includes("instance")) {
 			files.set("module/globals.lua", path.join(commonPath, "module/globals.lua"));
+		} else {
+			files.set(`instance.${ext}`, path.join(templatePath, `instance_empty.${ext}`));
+			pluginContexts += 1;
 		}
 	}
 
 	// Files for the control tool
 	if (templates.includes("ctl")) {
 		files.set(`ctl.${ext}`, path.join(templatePath, `ctl.${ext}`));
+		pluginContexts += 1;
+	}
+
+	// If there are more than one contexts then include the message file
+	if (pluginContexts > 1) {
+		files.set(`messages.${ext}`, path.join(templatePath, `messages.${ext}`));
 	}
 
 	// Properties that will control the replacements in the templates
 	const properties = {
+		config: config,
 		typescript: !javascriptOnly,
 		controller: templates.includes("controller"),
 		host: templates.includes("host"),
@@ -150,6 +172,7 @@ async function copyPluginTemplates(pluginName, templates) {
 		module: templates.includes("module"),
 		ctl: templates.includes("ctl"),
 		web: templates.includes("web"),
+		multi_context: pluginContexts > 1,
 		plugin_name: pluginName,
 		prepare: prepare.join(" && "),
 		ext: ext,

packages/create/templates/common/module/control.lua

@@ -1,21 +1,29 @@
 local clusterio_api = require("modules/clusterio/api")
+local compat = require("modules/clusterio/compat")
 
 --- Top level module table, contains event handlers and public methods
+--- @class __plugin_name__
 local MyModule = {
-	events = {},
-	on_nth_tick = {},
 }
 
---- global is 'synced' between players, you should use your plugin name to avoid conflicts
--- setupGlobalData should either be removed or called during clusterio_api.events.on_server_startup
-local globalData = {}
-local function setupGlobalData()
-	if global["__plugin_name__"] == nil then
-		global["__plugin_name__"] = {
+--- script_data is 'synced' between players, you should use your plugin name to avoid name conflicts
+-- The following helper function is desync safe and avoids name conflicts after being called during on_server_startup
+-- There are edge cases in the following events: pre player left, player left, console command, and config changed
+-- If you use any of these events and access globalData then make sure to call this function first
+local script_data = {}
+local function setup_script_data()
+	if compat.script_data["__plugin_name__"] == nil then
+		compat.script_data["__plugin_name__"] = {
 			-- starting values go here
 		}
 	end
-	globalData = global["__plugin_name__"]
+	MyModule.on_load()
+end
+
+--- The on_load function is called independently for each client when they first load the map
+-- It should be used to restore global aliases and metatables not registered with script.register_metatable
+function MyModule.on_load()
+	script_data = compat.script_data["__plugin_name__"]
 end
 
 --- Public methods should be available though your top level module table
@@ -28,27 +36,44 @@ local function bar()
 	game.print("bar")
 end
 
---- Clusterio provides a few custom events, on_server_startup is the most useful and should be used in place of on_load
-MyModule.events[clusterio_api.events.on_server_startup] = function(event)
-	setupGlobalData()
-	game.print(game.table_to_json(event))
+--- Clusterio provides a few custom events, on_server_startup is the most useful and should be used in place of on_init
+--- @param event EventData.on_server_startup
+local function on_server_startup(event)
+	setup_script_data()
+	game.print(compat.table_to_json(event))
 end
 
---- Factorio events are accessible through defines.events, you can have one handler per event per module
-MyModule.events[defines.events.on_player_crafted_item] = function(event)
-	game.print(game.table_to_json(event))
+--- Triggered every time a player crafts an item, see below for how handlers are registered
+--- @param event EventData.on_player_crafted_item
+local function on_player_crafted_item(event)
+	game.print(compat.table_to_json(event))
 //%if instance
+	local player = assert(game.get_player(event.player_index))
 	clusterio_api.send_json("__plugin_name__-plugin_example_ipc", {
-		tick = game.tick, player_name = game.get_player(event.player_index).name
+		tick = game.tick, player_name = player.name
 	})
 //%endif
 end
 
---- Nth tick is a special case that requires its own table, the index represents the time period between calls in ticks
-MyModule.on_nth_tick[300] = function()
+--- Run bar every 5 seconds, see below for how handlers are registered
+local function on_nth_tick_300()
 	game.print(game.tick)
 	bar()
 end
 
+--- Factorio events are accessible through defines.events, you can have one handler per event per module
+local e = defines.events
+local events = {
+	[clusterio_api.events.on_server_startup] = on_server_startup,
+	[e.on_player_crafted_item] = on_player_crafted_item,
+}
+
+--- Nth tick is a special case that requires its own table, the index represents the time period between calls in ticks
+local on_nth_tick = {
+	[300] = on_nth_tick_300,
+}
+
 --- Always return the top level module table from control, this is how clusterio will access your event handlers
+MyModule.events = events --- @package
+MyModule.on_nth_tick = on_nth_tick --- @package
 return MyModule

packages/create/templates/common/module/module_exports.lua

@@ -1,3 +1,4 @@
 --- Can contain anything you want to allow other plugins to have access to, this example exposes the control file
+--- If you only export a single file then it may be better to rename that file to module_exports and save the redirection
 -- Access the exports from other modules using require("modules/__plugin_name__")
 return require("modules/__plugin_name__/control")

packages/create/templates/common/package.json

@@ -15,12 +15,12 @@
 		"node": ">=18"
 	},
 	"peerDependencies": {
-		"@clusterio/lib": "^2.0.0-alpha.17"
+		"@clusterio/lib": "workspace:*"
 	},
 	"devDependencies": {
 //%// Typescript and type declarations
 //%if typescript
-		"typescript": "^5.1.6",
+		"typescript": "^5.5.3",
 		"@types/node": "^20.4.5",
 //%endif
 //%if typescript & web
@@ -36,9 +36,9 @@
 		"webpack": "^5.88.2",
 		"webpack-cli": "^5.1.4",
 		"webpack-merge": "^5.9.0",
-		"@clusterio/web_ui": "^2.0.0-alpha.17",
+		"@clusterio/web_ui": "workspace:*",
 //%endif
-		"@clusterio/lib": "^2.0.0-alpha.17"
+		"@clusterio/lib": "workspace:*"
 	},
 	"dependencies": {
 		"@sinclair/typebox": "^0.30.4"
@@ -48,6 +48,7 @@
 	},
 	"keywords": [
 		"clusterio",
+		"clusterio-plugin",
 		"factorio"
 	]
 }

packages/create/templates/plugin-js/controller.js

@@ -1,13 +1,17 @@
 "use strict";
 const lib = require("@clusterio/lib");
 const { BaseControllerPlugin } = require("@clusterio/controller");
+//%if multi_context // Subscribing requires multi context
 
 const {
 	PluginExampleEvent, PluginExampleRequest,
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 	ExampleSubscribableUpdate, ExampleSubscribableValue,
 //%endif
+//%if multi_context // Subscribing requires multi context
 } = require("./messages");
+//%endif
 
 class ControllerPlugin extends BaseControllerPlugin {
 //%if controller & web // Subscribing requires web content and the controller
@@ -15,14 +19,18 @@ class ControllerPlugin extends BaseControllerPlugin {
 	storageDirty = false;
 
 //%endif
+//%if multi_context // Subscribing requires multi context
 	async init() {
 		this.controller.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.controller.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 		this.controller.subscriptions.handle(ExampleSubscribableUpdate, this.handleExampleSubscription.bind(this));
 		this.exampleDatabase = new Map(); // If needed, replace with loading from database file
 //%endif
+//%if multi_context // Subscribing requires multi context
 	}
+//%endif
 
 	async onControllerConfigFieldChanged(field, curr, prev) {
 		this.logger.info(`controller::onControllerConfigFieldChanged ${field}`);
@@ -45,6 +53,7 @@ class ControllerPlugin extends BaseControllerPlugin {
 	async onPlayerEvent(instance, event) {
 		this.logger.info(`controller::onPlayerEvent ${instance.id} ${JSON.stringify(event)}`);
 	}
+//%if multi_context
 
 	async handlePluginExampleEvent(event) {
 		this.logger.info(JSON.stringify(event));
@@ -57,6 +66,7 @@ class ControllerPlugin extends BaseControllerPlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 
 	async handleExampleSubscription(request) {

packages/create/templates/plugin-js/ctl.js

@@ -1,6 +1,7 @@
 "use strict";
 const { BaseCtlPlugin } = require("@clusterio/ctl");
 const { CommandTree, Command } = require("@clusterio/lib");
+//%// We do not check for multi context here because it doesn't make sense to have a ctl without messages
 const { PluginExampleEvent, PluginExampleRequest } = require("./messages");
 /* eslint-disable no-console */
 

packages/create/templates/plugin-js/host.js

@@ -1,13 +1,17 @@
 "use strict";
 const lib = require("@clusterio/lib");
 const { BaseHostPlugin } = require("@clusterio/host");
+//%if multi_context
 const { PluginExampleEvent, PluginExampleRequest } = require("./messages");
+//%endif
 
 class HostPlugin extends BaseHostPlugin {
+//%if multi_context
 	async init() {
 		this.host.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.host.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
 	}
+//%endif
 
 	async onHostConfigFieldChanged(field, curr, prev) {
 		this.logger.info(`host::onInstanceConfigFieldChanged ${field}`);
@@ -16,6 +20,7 @@ class HostPlugin extends BaseHostPlugin {
 	async onShutdown() {
 		this.logger.info("host::onShutdown");
 	}
+//%if multi_context
 
 	async handlePluginExampleEvent(event) {
 		this.logger.info(JSON.stringify(event));
@@ -28,6 +33,7 @@ class HostPlugin extends BaseHostPlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 }
 
 module.exports = {

packages/create/templates/plugin-js/index.js

@@ -1,5 +1,6 @@
 "use strict";
 const lib = require("@clusterio/lib");
+//%if multi_context
 const Messages = require("./messages");
 
 lib.definePermission({
@@ -13,6 +14,7 @@ lib.definePermission({
 	title: "Example permission request",
 	description: "Example Description. Request. Change me in index.js",
 });
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 
 lib.definePermission({
@@ -34,9 +36,13 @@ const plugin = {
 	name: "__plugin_name__",
 	title: "__plugin_name__",
 	description: "Example Description. Plugin. Change me in index.js",
-//%if controller
+//%if controller | host & !config | instance & !config | ctl & !config // Blank line for formatting
 
-	controllerEntrypoint: "./controller",
+//%endif
+//%if controller
+controllerEntrypoint: "./dist/node/controller",
+//%endif
+//%if controller & config
 	controllerConfigFields: {
 		"__plugin_name__.myControllerField": {
 			title: "My Controller Field",
@@ -46,9 +52,13 @@ const plugin = {
 		},
 	},
 //%endif
-//%if host
+//%if host & config // Blank line for formatting
 
-	hostEntrypoint: "./host",
+//%endif
+//%if host
+	hostEntrypoint: "./dist/node/host",
+//%endif
+//%if host & config
 	hostConfigFields: {
 		"__plugin_name__.myHostField": {
 			title: "My Host Field",
@@ -58,9 +68,13 @@ const plugin = {
 		},
 	},
 //%endif
-//%if instance
+//%if instance & config | module & config // Blank line for formatting
 
-	instanceEntrypoint: "./instance",
+//%endif
+//%if instance | module // Modules load an empty instance plugin
+	instanceEntrypoint: "./dist/node/instance",
+//%endif
+//%if instance & config
 	instanceConfigFields: {
 		"__plugin_name__.myInstanceField": {
 			title: "My Instance Field",
@@ -70,9 +84,13 @@ const plugin = {
 		},
 	},
 //%endif
-//%if ctl
+//%if ctl & config // Blank line for formatting
 
-	ctlEntrypoint: "./ctl",
+//%endif
+//%if ctl
+	ctlEntrypoint: "./dist/node/ctl",
+//%endif
+//%if ctl & config
 	controlConfigFields: {
 		"__plugin_name__.myControlField": {
 			title: "My Control Field",
@@ -82,14 +100,18 @@ const plugin = {
 		},
 	},
 //%endif
+//%if multi_context // Subscribing requires multi context
 
 	messages: [
 		Messages.PluginExampleEvent,
 		Messages.PluginExampleRequest,
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 		Messages.ExampleSubscribableUpdate,
 //%endif
+//%if multi_context // Subscribing requires multi context
 	],
+//%endif
 //%if web // Web content template has an example route which is the plugin name
 
 	webEntrypoint: "./web",
@@ -97,11 +119,6 @@ const plugin = {
 		"/__plugin_name__",
 	],
 //%endif
-//%if controller & !web // The controller always includes web entry even if there is no content
-
-	webEntrypoint: "./web",
-	routes: [],
-//%endif
 };
 
 module.exports = {

packages/create/templates/plugin-js/instance.js

@@ -1,16 +1,23 @@
 "use strict";
 const lib = require("@clusterio/lib");
 const { BaseInstancePlugin } = require("@clusterio/host");
+//%if multi_context
 const { PluginExampleEvent, PluginExampleRequest } = require("./messages");
+//%endif
 
 class InstancePlugin extends BaseInstancePlugin {
+//%if multi_context | module
 	async init() {
+//%if multi_context
 		this.instance.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.instance.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
+//%endif
 //%if module
 		this.instance.server.handle("__plugin_name__-plugin_example_ipc", this.handlePluginExampleIPC.bind(this));
 //%endif
+//%if multi_context | module
 	}
+//%endif
 
 	async onInstanceConfigFieldChanged(field, curr, prev) {
 		this.logger.info(`instance::onInstanceConfigFieldChanged ${field}`);
@@ -26,13 +33,16 @@ class InstancePlugin extends BaseInstancePlugin {
 
 	async onPlayerEvent(event) {
 		this.logger.info(`onPlayerEvent::onPlayerEvent ${JSON.stringify(event)}`);
+//%if module
+		if (this.instance.status === "running") {
+			this.sendRcon("/sc __plugin_name__.foo()");
+		}
+//%endif
 	}
+//%if multi_context
 
 	async handlePluginExampleEvent(event) {
 		this.logger.info(JSON.stringify(event));
-//%if module
-		this.sendRcon("/sc __plugin_name__.foo()");
-//%endif
 	}
 
 	async handlePluginExampleRequest(request) {
@@ -42,6 +52,7 @@ class InstancePlugin extends BaseInstancePlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 //%if module
 
 	async handlePluginExampleIPC(event) {

packages/create/templates/plugin-js/instance_empty.js

@@ -0,0 +1,11 @@
+"use strict";
+const { BaseInstancePlugin } = require("@clusterio/host");
+
+class InstancePlugin extends BaseInstancePlugin {
+	// This class is empty because an instance plugin must be defined for a module to be injected
+	// This requirement may change in the future to allow for standalone modules
+}
+
+module.exports = {
+	InstancePlugin,
+};

packages/create/templates/plugin-js/web/plugin.jsx

@@ -1,6 +1,6 @@
 import React, {
 	useContext, useEffect, useState,
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller // Subscribing requires web content and the controller
 	useCallback, useSyncExternalStore,
 //%endif
 } from "react";
@@ -10,35 +10,39 @@ import React, {
 // } from "antd";
 
 import {
-	BaseWebPlugin, PageLayout, Control, ControlContext, notifyErrorHandler,
+	BaseWebPlugin, PageLayout, PageHeader, Control, ControlContext, notifyErrorHandler,
 } from "@clusterio/web_ui";
+//%if multi_context // Subscribing requires multi context
 
 import {
 	PluginExampleEvent, PluginExampleRequest,
-//%if controller & web // Subscribing requires web content and the controller
-	ExampleSubscribableUpdate,
 //%endif
+//%if controller // Subscribing requires web content and the controller
+	ExampleSubscribableUpdate, ExampleSubscribableValue,
+//%endif
+//%if multi_context // Subscribing requires multi context
 } from "../messages";
+//%endif
 
 import * as lib from "@clusterio/lib";
 
 function MyTemplatePage() {
 	let control = useContext(ControlContext);
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller // Subscribing requires web content and the controller
 	const plugin = control.plugins.get("__plugin_name__");
 	const [subscribableData, synced] = plugin.useSubscribableData();
 //%endif
 
 	return <PageLayout nav={[{ name: "__plugin_name__" }]}>
-		<h2>__plugin_name__</h2>
-//%if controller & web // Subscribing requires web content and the controller
+		<PageHeader title="__plugin_name__" />
+//%if controller // Subscribing requires web content and the controller
 		Synced: {String(synced)} Data: {JSON.stringify([...subscribableData.values()])}
 //%endif
 	</PageLayout>;
 }
 
 export class WebPlugin extends BaseWebPlugin {
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller // Subscribing requires web content and the controller
 	subscribableData = new lib.EventSubscriber(ExampleSubscribableUpdate, this.control);
 
 //%endif
@@ -47,15 +51,24 @@ export class WebPlugin extends BaseWebPlugin {
 			{
 				path: "/__plugin_name__",
 				sidebarName: "__plugin_name__",
-				permission: "__plugin_name__.page.view",
+				// This permission is client side only, so it must match the permission string of a resource request to be secure
+				// An undefined value means that the page will always be visible
+//%if controller // Subscribing requires web content and the controller
+				permission: "__plugin_name__.example.permission.subscribe",
+//%endif
+//%if !controller
+				permission: undefined, // "__plugin_name__.example.permission.request",
+//%endif
 				content: <MyTemplatePage/>,
 			},
 		];
+//%if multi_context
 
 		this.control.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.control.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
+//%endif
 	}
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller// Subscribing requires web content and the controller
 
 	useSubscribableData() {
 		const control = useContext(ControlContext);
@@ -63,6 +76,7 @@ export class WebPlugin extends BaseWebPlugin {
 		return useSyncExternalStore(subscribe, () => this.subscribableData.getSnapshot());
 	}
 //%endif
+//%if multi_context
 
 	async handlePluginExampleEvent(event) {
 		this.logger.info(JSON.stringify(event));
@@ -75,4 +89,5 @@ export class WebPlugin extends BaseWebPlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 }

packages/create/templates/plugin-ts/controller.ts

@@ -1,12 +1,16 @@
 import * as lib from "@clusterio/lib";
 import { BaseControllerPlugin, InstanceInfo } from "@clusterio/controller";
+//%if multi_context // Subscribing requires multi context
 
 import {
 	PluginExampleEvent, PluginExampleRequest,
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 	ExampleSubscribableUpdate, ExampleSubscribableValue,
 //%endif
+//%if multi_context // Subscribing requires multi context
 } from "./messages";
+//%endif
 
 export class ControllerPlugin extends BaseControllerPlugin {
 //%if controller & web // Subscribing requires web content and the controller
@@ -14,14 +18,18 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	storageDirty = false;
 
 //%endif
+//%if multi_context // Subscribing requires multi context
 	async init() {
 		this.controller.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.controller.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 		this.controller.subscriptions.handle(ExampleSubscribableUpdate, this.handleExampleSubscription.bind(this));
 		this.exampleDatabase = new Map(); // If needed, replace with loading from database file
 //%endif
+//%if multi_context // Subscribing requires multi context
 	}
+//%endif
 
 	async onControllerConfigFieldChanged(field: string, curr: unknown, prev: unknown) {
 		this.logger.info(`controller::onControllerConfigFieldChanged ${field}`);
@@ -44,6 +52,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	async onPlayerEvent(instance: InstanceInfo, event: lib.PlayerEvent) {
 		this.logger.info(`controller::onPlayerEvent ${instance.id} ${JSON.stringify(event)}`);
 	}
+//%if multi_context
 
 	async handlePluginExampleEvent(event: PluginExampleEvent) {
 		this.logger.info(JSON.stringify(event));
@@ -56,6 +65,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 
 	async handleExampleSubscription(request: lib.SubscriptionRequest) {

packages/create/templates/plugin-ts/ctl.ts

@@ -1,6 +1,6 @@
-type Control = any; // import type { Control } from "@clusterio/ctl";
-import { BaseCtlPlugin } from "@clusterio/ctl";
+import { BaseCtlPlugin, type Control } from "@clusterio/ctl";
 import { CommandTree, Command } from "@clusterio/lib";
+//%// We do not check for multi context here because it doesn't make sense to have a ctl without messages
 import { PluginExampleEvent, PluginExampleRequest } from "./messages";
 /* eslint-disable no-console */
 

packages/create/templates/plugin-ts/host.ts

@@ -1,12 +1,16 @@
 import * as lib from "@clusterio/lib";
 import { BaseHostPlugin } from "@clusterio/host";
+//%if multi_context
 import { PluginExampleEvent, PluginExampleRequest } from "./messages";
+//%endif
 
 export class HostPlugin extends BaseHostPlugin {
+//%if multi_context
 	async init() {
 		this.host.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.host.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
 	}
+//%endif
 
 	async onHostConfigFieldChanged(field: string, curr: unknown, prev: unknown) {
 		this.logger.info(`host::onInstanceConfigFieldChanged ${field}`);
@@ -15,6 +19,7 @@ export class HostPlugin extends BaseHostPlugin {
 	async onShutdown() {
 		this.logger.info("host::onShutdown");
 	}
+//%if multi_context
 
 	async handlePluginExampleEvent(event: PluginExampleEvent) {
 		this.logger.info(JSON.stringify(event));
@@ -27,4 +32,5 @@ export class HostPlugin extends BaseHostPlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 }

packages/create/templates/plugin-ts/index.ts

@@ -1,4 +1,5 @@
 import * as lib from "@clusterio/lib";
+//%if multi_context
 import * as Messages from "./messages";
 
 lib.definePermission({
@@ -12,6 +13,7 @@ lib.definePermission({
 	title: "Example permission request",
 	description: "Example Description. Request. Change me in index.ts",
 });
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 
 lib.definePermission({
@@ -28,37 +30,46 @@ lib.definePermission({
 	description: "Example Description. View. Change me in index.ts",
 });
 //%endif
+//%if config
 
 declare module "@clusterio/lib" {
-//%if controller
+//%endif
+//%if controller & config
 	export interface ControllerConfigFields {
 		"__plugin_name__.myControllerField": string;
 	}
 //%endif
-//%if host
+//%if host & config
 	export interface HostConfigFields {
 		"__plugin_name__.myHostField": string;
 	}
 //%endif
-//%if instance
+//%if instance & config
 	export interface InstanceConfigFields {
 		"__plugin_name__.myInstanceField": string;
 	}
 //%endif
-//%if ctl
+//%if ctl & config
 	export interface ControlConfigFields {
 		"__plugin_name__.myControlField": string;
 	}
 //%endif
+//%if config
 }
+//%endif
 
 export const plugin: lib.PluginDeclaration = {
 	name: "__plugin_name__",
 	title: "__plugin_name__",
 	description: "Example Description. Plugin. Change me in index.ts",
-//%if controller
+//%// There are a lot of statements here to make the formatting look good with and without 'config'
+//%if controller | host & !config | instance & !config | ctl & !config // Blank line for formatting
 
+//%endif
+//%if controller
 	controllerEntrypoint: "./dist/node/controller",
+//%endif
+//%if controller & config
 	controllerConfigFields: {
 		"__plugin_name__.myControllerField": {
 			title: "My Controller Field",
@@ -68,9 +79,13 @@ export const plugin: lib.PluginDeclaration = {
 		},
 	},
 //%endif
-//%if host
+//%if host & config // Blank line for formatting
 
+//%endif
+//%if host
 	hostEntrypoint: "./dist/node/host",
+//%endif
+//%if host & config
 	hostConfigFields: {
 		"__plugin_name__.myHostField": {
 			title: "My Host Field",
@@ -80,9 +95,13 @@ export const plugin: lib.PluginDeclaration = {
 		},
 	},
 //%endif
-//%if instance
+//%if instance & config | module & config // Blank line for formatting
 
+//%endif
+//%if instance | module // Modules load an empty instance plugin
 	instanceEntrypoint: "./dist/node/instance",
+//%endif
+//%if instance & config
 	instanceConfigFields: {
 		"__plugin_name__.myInstanceField": {
 			title: "My Instance Field",
@@ -92,9 +111,13 @@ export const plugin: lib.PluginDeclaration = {
 		},
 	},
 //%endif
-//%if ctl
+//%if ctl & config // Blank line for formatting
 
+//%endif
+//%if ctl
 	ctlEntrypoint: "./dist/node/ctl",
+//%endif
+//%if ctl & config
 	controlConfigFields: {
 		"__plugin_name__.myControlField": {
 			title: "My Control Field",
@@ -104,14 +127,18 @@ export const plugin: lib.PluginDeclaration = {
 		},
 	},
 //%endif
+//%if multi_context // Subscribing requires multi context
 
 	messages: [
 		Messages.PluginExampleEvent,
 		Messages.PluginExampleRequest,
+//%endif
 //%if controller & web // Subscribing requires web content and the controller
 		Messages.ExampleSubscribableUpdate,
 //%endif
+//%if multi_context // Subscribing requires multi context
 	],
+//%endif
 //%if web // Web content template has an example route which is the plugin name
 
 	webEntrypoint: "./web",
@@ -119,9 +146,4 @@ export const plugin: lib.PluginDeclaration = {
 		"/__plugin_name__",
 	],
 //%endif
-//%if controller & !web // The controller always includes web entry even if there is no content
-
-	webEntrypoint: "./web",
-	routes: [],
-//%endif
 };

packages/create/templates/plugin-ts/instance.ts

@@ -1,6 +1,8 @@
 import * as lib from "@clusterio/lib";
 import { BaseInstancePlugin } from "@clusterio/host";
+//%if multi_context
 import { PluginExampleEvent, PluginExampleRequest } from "./messages";
+//%endif
 //%if module
 
 type PuginExampleIPC = {
@@ -10,13 +12,19 @@ type PuginExampleIPC = {
 //%endif
 
 export class InstancePlugin extends BaseInstancePlugin {
+//%if multi_context | module
 	async init() {
+//%endif
+//%if multi_context
 		this.instance.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.instance.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
+//%endif
 //%if module
 		this.instance.server.handle("__plugin_name__-plugin_example_ipc", this.handlePluginExampleIPC.bind(this));
 //%endif
+//%if multi_context | module
 	}
+//%endif
 
 	async onInstanceConfigFieldChanged(field: string, curr: unknown, prev: unknown) {
 		this.logger.info(`instance::onInstanceConfigFieldChanged ${field}`);
@@ -33,9 +41,12 @@ export class InstancePlugin extends BaseInstancePlugin {
 	async onPlayerEvent(event: lib.PlayerEvent) {
 		this.logger.info(`onPlayerEvent::onPlayerEvent ${JSON.stringify(event)}`);
 //%if module
-		this.sendRcon("/sc __plugin_name__.foo()");
+		if (this.instance.status === "running") {
+			this.sendRcon("/sc __plugin_name__.foo()");
+		}
 //%endif
 	}
+//%if multi_context
 
 	async handlePluginExampleEvent(event: PluginExampleEvent) {
 		this.logger.info(JSON.stringify(event));
@@ -48,6 +59,7 @@ export class InstancePlugin extends BaseInstancePlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 //%if module
 
 	async handlePluginExampleIPC(event: PuginExampleIPC) {

packages/create/templates/plugin-ts/instance_empty.ts

@@ -0,0 +1,7 @@
+import * as lib from "@clusterio/lib";
+import { BaseInstancePlugin } from "@clusterio/host";
+
+export class InstancePlugin extends BaseInstancePlugin {
+	// This class is empty because an instance plugin must be defined for a module to be injected
+	// This requirement may change in the future to allow for standalone modules
+}

packages/create/templates/plugin-ts/tsconfig.browser.json

@@ -5,7 +5,4 @@
 		{ "path": "../../packages/web_ui/tsconfig.browser.json" },
 	],
 	"include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ],
-	"compilerOptions": {
-		"outDir": "dist/browser",
-	}
 }

packages/create/templates/plugin-ts/tsconfig.node.json

@@ -5,7 +5,4 @@
 	],
 	"include": ["./**/*.ts"],
 	"exclude": ["test/*", "./dist/*"],
-	"compilerOptions": {
-		"outDir": "dist/node",
-	},
 }

packages/create/templates/plugin-ts/web/plugin.tsx

@@ -10,35 +10,39 @@ import React, {
 // } from "antd";
 
 import {
-	BaseWebPlugin, PageLayout, Control, ControlContext, notifyErrorHandler,
+	BaseWebPlugin, PageLayout, PageHeader, Control, ControlContext, notifyErrorHandler,
 } from "@clusterio/web_ui";
+//%if multi_context // Subscribing requires multi context
 
 import {
 	PluginExampleEvent, PluginExampleRequest,
-//%if controller & web // Subscribing requires web content and the controller
-	ExampleSubscribableUpdate,
 //%endif
+//%if controller & web // Subscribing requires web content and the controller
+	ExampleSubscribableUpdate, ExampleSubscribableValue,
+//%endif
+//%if multi_context // Subscribing requires multi context
 } from "../messages";
+//%endif
 
 import * as lib from "@clusterio/lib";
 
 function MyTemplatePage() {
 	const control = useContext(ControlContext);
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller // Subscribing requires web content and the controller
 	const plugin = control.plugins.get("__plugin_name__") as WebPlugin;
 	const [subscribableData, synced] = plugin.useSubscribableData();
 //%endif
 
 	return <PageLayout nav={[{ name: "__plugin_name__" }]}>
-		<h2>__plugin_name__</h2>
-//%if controller & web // Subscribing requires web content and the controller
+		<PageHeader title="__plugin_name__" />
+//%if controller // Subscribing requires web content and the controller
 		Synced: {String(synced)} Data: {JSON.stringify([...subscribableData.values()])}
 //%endif
 	</PageLayout>;
 }
 
 export class WebPlugin extends BaseWebPlugin {
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller // Subscribing requires web content and the controller
 	subscribableData = new lib.EventSubscriber(ExampleSubscribableUpdate, this.control);
 
 //%endif
@@ -49,20 +53,22 @@ export class WebPlugin extends BaseWebPlugin {
 				sidebarName: "__plugin_name__",
 				// This permission is client side only, so it must match the permission string of a resource request to be secure
 				// An undefined value means that the page will always be visible
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller // Subscribing requires web content and the controller
 				permission: "__plugin_name__.example.permission.subscribe",
 //%endif
-//%if !controller | !web
+//%if !controller
 				permission: undefined, // "__plugin_name__.example.permission.request",
 //%endif
 				content: <MyTemplatePage/>,
 			},
 		];
+//%if multi_context
 
 		this.control.handle(PluginExampleEvent, this.handlePluginExampleEvent.bind(this));
 		this.control.handle(PluginExampleRequest, this.handlePluginExampleRequest.bind(this));
+//%endif
 	}
-//%if controller & web // Subscribing requires web content and the controller
+//%if controller // Subscribing requires web content and the controller
 
 	useSubscribableData() {
 		const control = useContext(ControlContext);
@@ -70,6 +76,7 @@ export class WebPlugin extends BaseWebPlugin {
 		return useSyncExternalStore(subscribe, () => this.subscribableData.getSnapshot());
 	}
 //%endif
+//%if multi_context
 
 	async handlePluginExampleEvent(event: PluginExampleEvent) {
 		this.logger.info(JSON.stringify(event));
@@ -82,4 +89,5 @@ export class WebPlugin extends BaseWebPlugin {
 			myResponseNumbers: request.myNumberArray,
 		};
 	}
+//%endif
 }

packages/ctl/ctl.ts

@@ -188,14 +188,7 @@ async function startControl() {
 	lib.handleUnhandledErrors();
 
 	logger.verbose(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = new Map();
-	try {
-		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, "utf8")));
-	} catch (err: any) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
+	let pluginList = await lib.loadPluginList(args.pluginList);
 
 	// If the command is plugin management we don't try to load plugins
 	if (args._[0] === "plugin") {

packages/ctl/package.json

@@ -3,7 +3,7 @@
 	"description": "Clusterio command line management tool",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -13,18 +13,19 @@
 		"clusterioctl": "./dist/node/ctl.js"
 	},
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {
 		"node": ">=18"
 	},
 	"dependencies": {
-		"@clusterio/lib": "^2.0.0-alpha.18",
+		"@clusterio/lib": "workspace:*",
 		"as-table": "^1.0.55",
 		"fs-extra": "^11.1.1",
 		"phin": "^3.7.0",
 		"set-blocking": "^2.0.0",
-		"winston": "^3.10.0",
+		"winston": "^3.17.0",
 		"yargs": "^17.7.2"
 	},
 	"publishConfig": {
@@ -35,6 +36,6 @@
 		"@types/node": "^20.4.5",
 		"@types/set-blocking": "^2.0.0",
 		"@types/yargs": "^17.0.24",
-		"typescript": "^5.1.6"
+		"typescript": "^5.5.3"
 	}
 }

packages/ctl/tsconfig.node.json

@@ -4,7 +4,4 @@
 		{ "path": "../lib/tsconfig.node.json" },
 	],
 	"include": ["index.ts", "package.json", "ctl.ts", "src/**/*.ts"],
-	"compilerOptions": {
-		"outDir": "dist/node",
-	},
 }

packages/host/host.ts

@@ -147,14 +147,7 @@ async function startHost() {
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = new Map();
-	try {
-		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, "utf8")));
-	} catch (err: any) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
+	let pluginList = await lib.loadPluginList(args.pluginList);
 
 	// If the command is plugin management we don't try to load plugins
 	if (command === "plugin") {
@@ -293,8 +286,8 @@ I           version of clusterio.  Expect things to break. I
 `
 	);
 	startHost().catch(err => {
-		if (err instanceof lib.AuthenticationFailed) {
-			logger.fatal(err.message);
+		if (err instanceof lib.WebSocketError) {
+			// Already handled by connector.on("error") within Host
 
 		} else if (err instanceof lib.StartupError) {
 			logger.fatal(`

packages/host/lua/clusterio_lib/api.lua

@@ -1,4 +1,5 @@
 -- Important: Keep this API in sync with modules/clusterio/api.lua
+local compat = require("compat")
 local api = {}
 
 
@@ -48,7 +49,7 @@ function api.send_json(channel, data)
 		return "\\x" .. string.format("%02x", match:byte())
 	end)
 
-	data = game.table_to_json(data)
+	data = compat.table_to_json(data)
 
 	-- If there's more than about 4kB of data users running with the Windows
 	-- console open will start to experience stuttering, otherwise we could
@@ -58,7 +59,7 @@ function api.send_json(channel, data)
 	else
 		local file_no = remote.call("clusterio_api", "get_file_no")
 		local file_name = "clst_" .. file_no .. ".json"
-		game.write_file(file_name, data, false, 0)
+		compat.write_file(file_name, data, false, 0)
 		print("\f$ipc:" .. channel .. "?f" .. file_name)
 	end
 end

packages/host/lua/clusterio_lib/info.json

@@ -12,6 +12,10 @@
 		{
 			"version": "0.1.2",
 			"factorio_version": "1.1"
+		},
+		{
+			"version": "0.1.3",
+			"factorio_version": "2.0"
 		}
 	],
 	"title": "Clusterio Library (Alpha)",
@@ -19,6 +23,8 @@
 	"homepage": "https://github.com/clusterio/clusterio",
 	"description": "Library for interfacing with Clusterio.  Warning: work in progress alpha.",
 	"additional_files": {
-		"serialize.lua": ["..", "..", "modules", "clusterio", "serialize.lua"]
+		"compat.lua": ["..", "..", "modules", "clusterio", "compat.lua"],
+		"serialize.lua": ["..", "..", "modules", "clusterio", "serialize.lua"],
+		"json.lua": ["..", "..", "modules", "clusterio", "json.lua"]
 	}
 }

packages/host/lua/export/control.lua

@@ -1,11 +1,12 @@
 local function send_json(channel, data)
-	data = game.table_to_json(data)
+	data = helpers and helpers.table_to_json(data) or game.table_to_json(data)
 	print("\f$ipc:" .. channel .. "?j" .. data)
 end
 
 script.on_init(function()
-	send_json("mod_list", game.active_mods)
-	for name, mod_setting in pairs(game.mod_setting_prototypes) do
+	send_json("mod_list", script.active_mods)
+	local mod_settings = prototypes and prototypes.mod_setting or game.mod_setting_prototypes
+	for name, mod_setting in pairs(mod_settings) do
 		send_json("mod_setting_mod", { name = name, mod = mod_setting.mod })
 	end
 end)

packages/host/modules/clusterio/api.lua

@@ -1,6 +1,8 @@
 -- Important: Keep this API in sync with mod/api.lua
+local compat = require("modules/clusterio/compat")
 local api = {}
 
+clusterio_patch_number = clusterio_patch_number or 0 --- @type number
 
 api.events = {
 	on_instance_updated = script.generate_event_name(),
@@ -8,11 +10,11 @@ api.events = {
 }
 
 function api.get_instance_name()
-	return global.clusterio.instance_name
+	return compat.script_data.clusterio.instance_name
 end
 
 function api.get_instance_id()
-	return global.clusterio.instance_id
+	return compat.script_data.clusterio.instance_id
 end
 
 function api.send_json(channel, data)
@@ -23,7 +25,7 @@ function api.send_json(channel, data)
 		return "\\x" .. string.format("%02x", match:byte())
 	end)
 
-	data = game.table_to_json(data)
+	data = compat.table_to_json(data)
 
 	-- If there's more than about 4kB of data users running with the Windows
 	-- console open will start to experience stuttering, otherwise we could
@@ -31,9 +33,10 @@ function api.send_json(channel, data)
 	if #data < 4000 then
 		print("\f$ipc:" .. channel .. "?j" .. data)
 	else
-		global.clusterio_file_no = (global.clusterio_file_no or 0) + 1
-		local file_name = "clst_" .. global.clusterio_file_no .. ".json"
-		game.write_file(file_name, data, false, 0)
+		local script_data = compat.script_data
+		script_data.clusterio_file_no = (script_data.clusterio_file_no or 0) + 1
+		local file_name = "clst_" .. script_data.clusterio_file_no .. ".json"
+		compat.write_file(file_name, data, false, 0)
 		print("\f$ipc:" .. channel .. "?f" .. file_name)
 	end
 end

packages/host/modules/clusterio/compat.lua

@@ -0,0 +1,221 @@
+--[[
+Adds various functions used to ensure compatibly between version including some polyfills.
+
+Implementation Guidelines
+- If a function is only available at runtime in one version, then all versions must be made to error.
+- Where possible, directly assign the function / table to the member of compat.
+- When not possible, then it should replace itself with a direct assignment asap.
+- Versions are handled in ascending order covering all supported versions and ending with an error.
+]]
+
+local lib_json = require("json")
+
+--- @diagnostic disable: deprecated
+
+--- @class LibCompat
+--- @field script_data table
+--- @field prototypes LuaPrototypes
+--- @field active_mods table<string, string>
+--- @field write_file fun(filename: string, data: LocalisedString, append: boolean?, for_player: uint?)
+--- @field table_to_json fun(data: table): string
+--- @field json_to_table fun(json: string): AnyBasic?
+local compat = {}
+
+--- @class LibCompatMt
+--- @package
+local compat_mt = {}
+
+--- @alias VersionTable { [1]: number, [2]: number, [3]: number }
+
+--- Concert a version string into a version table
+--- @param version string
+--- @return VersionTable
+local function version_to_table(version)
+	local index = 0
+	local rtn = { 0, 0, 0 }
+	for part in version:gmatch("%d+") do
+		index = index + 1
+		rtn[index] = tonumber(part)
+	end
+	return rtn
+end
+
+--- The current version of factorio, the earliest supported is 0.17.69
+local current_version_str = "0.17.69"
+local current_version = version_to_table(current_version_str)
+do
+	local success, base_version = pcall(function()
+		return script.active_mods.base
+	end)
+	if success then
+		current_version_str = base_version
+		current_version = version_to_table(current_version_str)
+	end
+end
+
+--- Returns true if the current version is equal to the provided version
+--- @param version string Version to compare against
+--- @return boolean # True if versions are equal
+function compat.version_eq(version)
+	local version_tbl = version_to_table(version)
+	for i = 1, 3 do
+		if version_tbl[i] ~= current_version[i] then
+			return false
+		end
+	end
+	return true
+end
+
+--- Returns true if the current version is greater than or equal to the the provided version
+--- @param version string Version to compare against
+--- @return boolean # True if the version is greater or equal to provided
+function compat.version_ge(version)
+	local version_tbl = version_to_table(version)
+	for i = 1, 3 do
+		if version_tbl[i] > current_version[i] then
+			return false
+		elseif version_tbl[i] < current_version[i] then
+			return true
+		end
+	end
+	return true
+end
+
+--- Returns true if the current version is less than or equal to the the provided version
+--- @param version string Version to compare against
+--- @return boolean # True if the version is less or equal to provided
+function compat.version_le(version)
+	local version_tbl = version_to_table(version)
+	for i = 1, 3 do
+		if version_tbl[i] < current_version[i] then
+			return false
+		elseif version_tbl[i] > current_version[i] then
+			return true
+		end
+	end
+	return true
+end
+
+--- The major versions of factorio we support
+local v2 = compat.version_ge("2.0")
+local v1 = compat.version_ge("1.0")  and not compat.version_ge("2.0")
+local v0 = compat.version_ge("0.17") and not compat.version_ge("1.0")
+
+--- The minor versions of factorio we support
+local v2_0  = compat.version_ge("2.0")
+local v1_1  = compat.version_ge("1.1")  and not compat.version_ge("2.0")
+local v1_0  = compat.version_ge("1.0")  and not compat.version_ge("1.1")
+local v0_18 = compat.version_ge("0.18") and not compat.version_ge("1.0")
+local v0_17 = compat.version_ge("0.17") and not compat.version_ge("0.18")
+
+--- Raise an error because the current version is unsupported
+local function unsupported_version()
+	error("Unsupported version: " .. current_version_str, 3)
+end
+
+--- Raise an error because the function is only available at runtime
+local function runtime_only()
+	error("To maintain compatibly, this value can only be used during runtime.", 3)
+end
+
+--- Keys of this table will raise an unsupported error when accessed
+local unsupported_properties = {}
+
+--- Keys of this table can only be accessed at runtime
+--- The value is a function which returns the real value
+local runtime_properties = {}
+
+--- Handles the indexing of the tables above
+function compat_mt:__index(key)
+	if unsupported_properties[key] then
+		return unsupported_version()
+	elseif runtime_properties[key] then
+		if game == nil then
+			return runtime_only()
+		end
+		local value = runtime_properties[key]()
+		if value == nil then
+			return unsupported_version()
+		end
+		rawset(self, key, value)
+		return value
+	end
+end
+
+--- Prototype data contains all the definitions for the game, it is only available during runtime
+function runtime_properties.prototypes()
+	if v0 or v1 then
+		--- Lazy loading of prototype data
+		--- Iteration is UB, we can fix this once someone actually needs it
+		return setmetatable({}, {
+			__index = function(self, key)
+				local value = game[key .. "_prototypes"]
+				rawset(self, key, value)
+				return value
+			end
+		})
+	elseif v2 then
+		return prototypes
+	end
+end
+
+--- List of all active mods
+function runtime_properties.active_mods()
+	if v0_17 then
+		return game.active_mods
+	elseif v0 or v1 or v2 then
+		return script.active_mods
+	end
+end
+
+--- Writes a string to file
+function runtime_properties.write_file()
+	if v0 or v1 then
+		return game.write_file
+	elseif v2 then
+		return helpers.write_file
+	end
+end
+
+--- Convert tables to and from json, we have backported this to work before runtime
+if v0 or v1 then
+	compat.table_to_json = function(tbl)
+		if game then
+			compat.table_to_json = game.table_to_json
+			return game.table_to_json(tbl)
+		end
+		return lib_json.encode(tbl)
+	end
+	compat.json_to_table = function(json)
+		if game then
+			compat.json_to_table = game.json_to_table
+			return game.json_to_table(json)
+		end
+		return lib_json.decode(json)
+	end
+elseif v2 then
+	compat.table_to_json = helpers.table_to_json
+	compat.json_to_table = helpers.json_to_table
+else
+	unsupported_properties.table_to_json = true
+	unsupported_properties.json_to_table = true
+end
+
+--- Script data refers to lua data which is persisted between loads
+--- This is called immediately, during init and during load
+local function set_script_data()
+	if v0 or v1 then
+		--- @diagnostic disable-next-line
+		compat.script_data = global
+	elseif v2 then
+		compat.script_data = storage
+	else
+		unsupported_properties.script_data = true
+	end
+end
+
+set_script_data()
+compat.on_init = set_script_data --- @package
+compat.on_load = set_script_data --- @package
+
+return setmetatable(compat, compat_mt)

packages/host/modules/clusterio/impl.lua

@@ -1,10 +1,14 @@
 local api = require('modules/clusterio/api')
+local compat = require("modules/clusterio/compat")
 
+--- @class (exact) EventData.on_server_startup:EventData
 
 local function check_patch()
-	if global.clusterio_patch_number ~= clusterio_patch_number then
-		global.clusterio_patch_number = clusterio_patch_number
-		script.raise_event(api.events.on_server_startup, {})
+	if compat.script_data.clusterio_patch_number ~= clusterio_patch_number then
+		compat.script_data.clusterio_patch_number = clusterio_patch_number
+		script.raise_event(api.events.on_server_startup, {
+			name = api.events.on_server_startup, tick = game.tick
+		})
 	end
 end
 
@@ -14,8 +18,8 @@ impl.events = {}
 impl.events[defines.events.on_tick] = check_patch
 
 impl.events[api.events.on_server_startup] = function()
-	if not global.clusterio then
-		global.clusterio = {
+	if not compat.script_data.clusterio then
+		compat.script_data.clusterio = {
 			instance_id = nil,
 			instance_name = nil,
 		}
@@ -46,12 +50,35 @@ impl.events[defines.events.on_player_left_game] = function(event)
 	})
 end
 
+impl.events[defines.events.on_player_banned] = function(event)
+	api.send_json("player_event", {
+		type = "BAN",
+		name = event.player_name,
+		reason = event.reason,
+	})
+end
+
+impl.events[defines.events.on_player_unbanned] = function(event)
+	api.send_json("player_event", { type = "UNBANNED", name = event.player_name })
+end
+
+impl.events[defines.events.on_player_promoted] = function(event)
+	local player = game.players[event.player_index]
+	api.send_json("player_event", { type = "PROMOTE", name = player.name })
+end
+
+impl.events[defines.events.on_player_demoted] = function(event)
+	local player = game.players[event.player_index]
+	api.send_json("player_event", { type = "DEMOTE", name = player.name })
+end
+
 -- Internal API
 clusterio_private = {}
 function clusterio_private.update_instance(new_id, new_name)
 	check_patch()
-	global.clusterio.instance_id = new_id
-	global.clusterio.instance_name = new_name
+	local script_data = compat.script_data
+	script_data.clusterio.instance_id = new_id
+	script_data.clusterio.instance_name = new_name
 	script.raise_event(api.events.on_instance_updated, {
 		instance_id = new_id,
 		instance_name = new_name,
@@ -71,16 +98,17 @@ remote.add_interface('clusterio_api', {
 	end,
 
 	get_instance_id = function()
-		return global.clusterio.instance_id
+		return compat.script_data.clusterio.instance_id
 	end,
 
 	get_instance_name = function()
-		return global.clusterio.instance_name
+		return compat.script_data.clusterio.instance_name
 	end,
 
 	get_file_no = function()
-		global.clusterio_file_no = (global.clusterio_file_no or 0) + 1
-		return global.clusterio_file_no
+		local script_data = compat.script_data
+		script_data.clusterio_file_no = (script_data.clusterio_file_no or 0) + 1
+		return script_data.clusterio_file_no
 	end,
 })
 

packages/host/modules/clusterio/json.lua

@@ -0,0 +1,388 @@
+--
+-- json.lua / https://github.com/rxi/json.lua
+--
+-- Copyright (c) 2020 rxi
+--
+-- Permission is hereby granted, free of charge, to any person obtaining a copy of
+-- this software and associated documentation files (the "Software"), to deal in
+-- the Software without restriction, including without limitation the rights to
+-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+-- of the Software, and to permit persons to whom the Software is furnished to do
+-- so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in all
+-- copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+-- SOFTWARE.
+--
+
+local json = { _version = "0.1.2" }
+
+-------------------------------------------------------------------------------
+-- Encode
+-------------------------------------------------------------------------------
+
+local encode
+
+local escape_char_map = {
+	[ "\\" ] = "\\",
+	[ "\"" ] = "\"",
+	[ "\b" ] = "b",
+	[ "\f" ] = "f",
+	[ "\n" ] = "n",
+	[ "\r" ] = "r",
+	[ "\t" ] = "t",
+}
+
+local escape_char_map_inv = { [ "/" ] = "/" }
+for k, v in pairs(escape_char_map) do
+	escape_char_map_inv[v] = k
+end
+
+
+local function escape_char(c)
+	return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte()))
+end
+
+
+local function encode_nil(val)
+	return "null"
+end
+
+
+local function encode_table(val, stack)
+	local res = {}
+	stack = stack or {}
+
+	-- Circular reference?
+	if stack[val] then error("circular reference") end
+
+	stack[val] = true
+
+	if rawget(val, 1) ~= nil or next(val) == nil then
+		-- Treat as array -- check keys are valid and it is not sparse
+		local n = 0
+		for k in pairs(val) do
+			if type(k) ~= "number" then
+				error("invalid table: mixed or invalid key types")
+			end
+			n = n + 1
+		end
+		if n ~= #val then
+			error("invalid table: sparse array")
+		end
+		-- Encode
+		for i, v in ipairs(val) do
+			table.insert(res, encode(v, stack))
+		end
+		stack[val] = nil
+		return "[" .. table.concat(res, ",") .. "]"
+
+	else
+		-- Treat as an object
+		for k, v in pairs(val) do
+			if type(k) ~= "string" then
+				error("invalid table: mixed or invalid key types")
+			end
+			table.insert(res, encode(k, stack) .. ":" .. encode(v, stack))
+		end
+		stack[val] = nil
+		return "{" .. table.concat(res, ",") .. "}"
+	end
+end
+
+
+local function encode_string(val)
+	return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"'
+end
+
+
+local function encode_number(val)
+	-- Check for NaN, -inf and inf
+	if val ~= val or val <= -math.huge or val >= math.huge then
+		error("unexpected number value '" .. tostring(val) .. "'")
+	end
+	return string.format("%.14g", val)
+end
+
+
+local type_func_map = {
+	[ "nil"		 ] = encode_nil,
+	[ "table"	 ] = encode_table,
+	[ "string"	] = encode_string,
+	[ "number"	] = encode_number,
+	[ "boolean" ] = tostring,
+}
+
+
+encode = function(val, stack)
+	local t = type(val)
+	local f = type_func_map[t]
+	if f then
+		return f(val, stack)
+	end
+	error("unexpected type '" .. t .. "'")
+end
+
+
+function json.encode(val)
+	return ( encode(val) )
+end
+
+
+-------------------------------------------------------------------------------
+-- Decode
+-------------------------------------------------------------------------------
+
+local parse
+
+local function create_set(...)
+	local res = {}
+	for i = 1, select("#", ...) do
+		res[ select(i, ...) ] = true
+	end
+	return res
+end
+
+local space_chars	 = create_set(" ", "\t", "\r", "\n")
+local delim_chars	 = create_set(" ", "\t", "\r", "\n", "]", "}", ",")
+local escape_chars	= create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u")
+local literals			= create_set("true", "false", "null")
+
+local literal_map = {
+	[ "true"	] = true,
+	[ "false" ] = false,
+	[ "null"	] = nil,
+}
+
+
+local function next_char(str, idx, set, negate)
+	for i = idx, #str do
+		if set[str:sub(i, i)] ~= negate then
+			return i
+		end
+	end
+	return #str + 1
+end
+
+
+local function decode_error(str, idx, msg)
+	local line_count = 1
+	local col_count = 1
+	for i = 1, idx - 1 do
+		col_count = col_count + 1
+		if str:sub(i, i) == "\n" then
+			line_count = line_count + 1
+			col_count = 1
+		end
+	end
+	error( string.format("%s at line %d col %d", msg, line_count, col_count) )
+end
+
+
+local function codepoint_to_utf8(n)
+	-- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa
+	local f = math.floor
+	if n <= 0x7f then
+		return string.char(n)
+	elseif n <= 0x7ff then
+		return string.char(f(n / 64) + 192, n % 64 + 128)
+	elseif n <= 0xffff then
+		return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128)
+	elseif n <= 0x10ffff then
+		return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128,
+											 f(n % 4096 / 64) + 128, n % 64 + 128)
+	end
+	error( string.format("invalid unicode codepoint '%x'", n) )
+end
+
+
+local function parse_unicode_escape(s)
+	local n1 = tonumber( s:sub(1, 4),	16 )
+	local n2 = tonumber( s:sub(7, 10), 16 )
+	 -- Surrogate pair?
+	if n2 then
+		return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000)
+	else
+		return codepoint_to_utf8(n1)
+	end
+end
+
+
+local function parse_string(str, i)
+	local res = ""
+	local j = i + 1
+	local k = j
+
+	while j <= #str do
+		local x = str:byte(j)
+
+		if x < 32 then
+			decode_error(str, j, "control character in string")
+
+		elseif x == 92 then -- `\`: Escape
+			res = res .. str:sub(k, j - 1)
+			j = j + 1
+			local c = str:sub(j, j)
+			if c == "u" then
+				local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1)
+								 or str:match("^%x%x%x%x", j + 1)
+								 or decode_error(str, j - 1, "invalid unicode escape in string")
+				res = res .. parse_unicode_escape(hex)
+				j = j + #hex
+			else
+				if not escape_chars[c] then
+					decode_error(str, j - 1, "invalid escape char '" .. c .. "' in string")
+				end
+				res = res .. escape_char_map_inv[c]
+			end
+			k = j + 1
+
+		elseif x == 34 then -- `"`: End of string
+			res = res .. str:sub(k, j - 1)
+			return res, j + 1
+		end
+
+		j = j + 1
+	end
+
+	decode_error(str, i, "expected closing quote for string")
+end
+
+
+local function parse_number(str, i)
+	local x = next_char(str, i, delim_chars)
+	local s = str:sub(i, x - 1)
+	local n = tonumber(s)
+	if not n then
+		decode_error(str, i, "invalid number '" .. s .. "'")
+	end
+	return n, x
+end
+
+
+local function parse_literal(str, i)
+	local x = next_char(str, i, delim_chars)
+	local word = str:sub(i, x - 1)
+	if not literals[word] then
+		decode_error(str, i, "invalid literal '" .. word .. "'")
+	end
+	return literal_map[word], x
+end
+
+
+local function parse_array(str, i)
+	local res = {}
+	local n = 1
+	i = i + 1
+	while 1 do
+		local x
+		i = next_char(str, i, space_chars, true)
+		-- Empty / end of array?
+		if str:sub(i, i) == "]" then
+			i = i + 1
+			break
+		end
+		-- Read token
+		x, i = parse(str, i)
+		res[n] = x
+		n = n + 1
+		-- Next token
+		i = next_char(str, i, space_chars, true)
+		local chr = str:sub(i, i)
+		i = i + 1
+		if chr == "]" then break end
+		if chr ~= "," then decode_error(str, i, "expected ']' or ','") end
+	end
+	return res, i
+end
+
+
+local function parse_object(str, i)
+	local res = {}
+	i = i + 1
+	while 1 do
+		local key, val
+		i = next_char(str, i, space_chars, true)
+		-- Empty / end of object?
+		if str:sub(i, i) == "}" then
+			i = i + 1
+			break
+		end
+		-- Read key
+		if str:sub(i, i) ~= '"' then
+			decode_error(str, i, "expected string for key")
+		end
+		key, i = parse(str, i)
+		-- Read ':' delimiter
+		i = next_char(str, i, space_chars, true)
+		if str:sub(i, i) ~= ":" then
+			decode_error(str, i, "expected ':' after key")
+		end
+		i = next_char(str, i + 1, space_chars, true)
+		-- Read value
+		val, i = parse(str, i)
+		-- Set
+		res[key] = val
+		-- Next token
+		i = next_char(str, i, space_chars, true)
+		local chr = str:sub(i, i)
+		i = i + 1
+		if chr == "}" then break end
+		if chr ~= "," then decode_error(str, i, "expected '}' or ','") end
+	end
+	return res, i
+end
+
+
+local char_func_map = {
+	[ '"' ] = parse_string,
+	[ "0" ] = parse_number,
+	[ "1" ] = parse_number,
+	[ "2" ] = parse_number,
+	[ "3" ] = parse_number,
+	[ "4" ] = parse_number,
+	[ "5" ] = parse_number,
+	[ "6" ] = parse_number,
+	[ "7" ] = parse_number,
+	[ "8" ] = parse_number,
+	[ "9" ] = parse_number,
+	[ "-" ] = parse_number,
+	[ "t" ] = parse_literal,
+	[ "f" ] = parse_literal,
+	[ "n" ] = parse_literal,
+	[ "[" ] = parse_array,
+	[ "{" ] = parse_object,
+}
+
+
+parse = function(str, idx)
+	local chr = str:sub(idx, idx)
+	local f = char_func_map[chr]
+	if f then
+		return f(str, idx)
+	end
+	decode_error(str, idx, "unexpected character '" .. chr .. "'")
+end
+
+
+function json.decode(str)
+	if type(str) ~= "string" then
+		error("expected argument of type string, got " .. type(str))
+	end
+	local res, idx = parse(str, next_char(str, 1, space_chars, true))
+	idx = next_char(str, idx, space_chars, true)
+	if idx <= #str then
+		decode_error(str, idx, "trailing garbage")
+	end
+	return res
+end
+
+
+return json

packages/host/modules/clusterio/module.json

@@ -2,5 +2,8 @@
 	"name": "clusterio",
 	"version": "0.0.0",
 	"dependencies": {},
-	"load": ["impl.lua"]
+	"load": [
+		"impl.lua",
+		"compat.lua"
+	]
 }

packages/host/modules/clusterio/serialize.lua

@@ -1,44 +1,22 @@
 -- Library for serializing items in Factorio
 -- Based on code from playerManager and trainTeleports
+local compat = require("modules/clusterio/compat")
 local serialize = {}
 
-local function version_to_table(version)
-	local t = {}
-	for p in string.gmatch(version, "%d+") do
-		t[#t + 1] = tonumber(p)
-	end
-	return t
-end
-
 -- 0.17 compatibility
-local supports_bar, get_bar, set_bar, version
-if (pcall(function() local mods = script.active_mods end)) then
+local supports_bar, get_bar, set_bar
+if compat.version_ge("1.0.0") then
 	supports_bar = "supports_bar"
 	get_bar = "get_bar"
 	set_bar = "set_bar"
-	version = version_to_table(script.active_mods.base)
 else
 	supports_bar = "hasbar"
 	get_bar = "getbar"
 	set_bar = "setbar"
-	version = version_to_table("0.17.69")
 end
 
--- returns true if the game version is greater than or equal to the given version
-local function version_ge(comp)
-	comp = version_to_table(comp)
-	for i=1, 3 do
-		if comp[i] > version[i] then
-			return false
-		elseif comp[i] < version[i] then
-			return true
-		end
-	end
-	return true
-end
-
-local has_create_grid = version_ge("1.1.7")
-
+local has_create_grid = compat.version_ge("1.1.7")
+local has_quality = compat.version_ge("2.0.0")
 
 -- Equipment Grids are serialized into an array of equipment entries
 -- where ench entry is a table with the following fields:
@@ -109,6 +87,7 @@ end
 -- Item stacks are serialized into a table with the following fields:
 --	 n: name
 --	 c: count
+--	 q: quality (optional)
 --	 h: health (optional)
 --	 d: durability (optional)
 --	 a: ammo count (optional)
@@ -141,8 +120,9 @@ function serialize.serialize_item_stack(slot, entry)
 
 	entry.n = slot.name
 	entry.c = slot.count
+	if has_quality and slot.quality.level ~= 0 then entry.q = slot.quality.name end
 	if slot.health < 1 then entry.h = slot.health end
-	if slot.durability then entry.d = slot.durability end
+	if slot.type == "tool" and slot.durability then entry.d = slot.durability end
 	if slot.type == "ammo" then entry.a = slot.ammo end
 	if slot.is_item_with_label then
 		local label = {}
@@ -178,6 +158,7 @@ function serialize.deserialize_item_stack(slot, entry)
 		name = entry.n,
 		count = entry.c,
 	}
+	if entry.q then item_stack.quality = entry.q end
 	if entry.h then item_stack.health = entry.h end
 	if entry.d then item_stack.durability = entry.d end
 	if entry.a then item_stack.ammo = entry.a end
@@ -244,7 +225,7 @@ function serialize.serialize_inventory(inventory)
 		end
 
 		if item.n or item.f or item.e then
-			local item_serialized = game.table_to_json(item)
+			local item_serialized = compat.table_to_json(item)
 			if item_serialized == previous_serialized then
 				local previous_item = serialized.i[#serialized.i]
 				previous_item.r = (previous_item.r or 0) + 1
@@ -305,5 +286,4 @@ function serialize.deserialize_inventory(inventory, serialized)
 	end
 end
 
-
 return serialize

packages/host/package.json

@@ -3,7 +3,7 @@
 	"description": "Implementation of Clusterio host server",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"main": "dist/node/index.js",
 	"scripts": {
 		"prepare": "tsc --build",
@@ -13,13 +13,14 @@
 		"clusteriohost": "./dist/node/host.js"
 	},
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {
 		"node": ">=18"
 	},
 	"dependencies": {
-		"@clusterio/lib": "^2.0.0-alpha.18",
+		"@clusterio/lib": "workspace:*",
 		"@sinclair/typebox": "^0.30.4",
 		"fs-extra": "^11.1.1",
 		"jimp": "^0.22.8",
@@ -29,7 +30,7 @@
 		"rcon-client": "^4.2.3",
 		"semver": "^7.5.4",
 		"set-blocking": "^2.0.0",
-		"winston": "^3.10.0",
+		"winston": "^3.17.0",
 		"winston-daily-rotate-file": "^4.7.1",
 		"yargs": "^17.7.2"
 	},
@@ -43,6 +44,6 @@
 		"@types/semver": "^7.5.0",
 		"@types/set-blocking": "^2.0.0",
 		"@types/yargs": "^17.0.24",
-		"typescript": "^5.1.6"
+		"typescript": "^5.5.3"
 	}
 }

packages/host/src/Host.ts

@@ -140,7 +140,7 @@ export class HostRouter {
 						return false;
 					}
 					origin.connector.sendResponseError(
-						new lib.ResponseError("Instance is not running."), message.src
+						new lib.ResponseError(msg || "Instance is not running."), message.src
 					);
 				}
 				return true;
@@ -346,13 +346,17 @@ export default class Host extends lib.Link {
 					logger.fatal(`Unexpected error reconnecting to controller:\n${err.stack}`);
 					return this.shutdown();
 				});
-
 			} else {
 				logger.fatal("Controller connection was unexpectedly closed");
 				this.shutdown();
 			}
 		});
 
+		this.connector.on("error", err => {
+			logger.fatal(err.message);
+			this.shutdown();
+		});
+
 		for (let event of ["connect", "drop", "resume", "close"] as const) {
 			this.connector.on(event, () => {
 				let message = new lib.ControllerConnectionEvent(event);
@@ -614,11 +618,12 @@ export default class Host extends lib.Link {
 	}
 
 	async fetchMods(mods: Iterable<lib.ModRecord>) {
+		// This is better than the previous hard coded names
+		// But it really shouldn't be a hard coded version either
+		const builtinModNames = lib.ModPack.getBuiltinModNames("2.0");
 		const modInfos: Promise<lib.ModInfo>[] = [];
 		for (const mod of mods) {
-			// XXX These mods ship with Factorio, but the list of mods may
-			// change and should not be hardcoded here.
-			if (["core", "base"].includes(mod.name)) {
+			if (builtinModNames.includes(mod.name)) {
 				continue;
 			}
 			modInfos.push(this.fetchMod(mod));
@@ -636,6 +641,7 @@ export default class Host extends lib.Link {
 		} else {
 			instanceInfo = this.discoveredInstanceInfos.get(instanceId);
 			if (instanceInfo) {
+				await Instance.populate_folders(instanceInfo.path);
 				instanceInfo.config.update(config, true, "controller");
 
 			} else {
@@ -645,7 +651,7 @@ export default class Host extends lib.Link {
 				let instanceDir = await this._createNewInstanceDir(instanceConfig.get("instance.name"));
 
 				logger.info(`Creating ${instanceDir}`);
-				await Instance.create(instanceDir, this.config.get("host.factorio_directory"));
+				await Instance.populate_folders(instanceDir);
 				instanceInfo = {
 					path: instanceDir,
 					config: instanceConfig,
@@ -667,6 +673,9 @@ export default class Host extends lib.Link {
 				instanceId,
 				instanceConnection ? instanceConnection.instance.status : "stopped",
 				instanceConnection ? instanceConnection.instance.server.gamePort : this.gamePort(instanceId),
+				instanceConnection
+					? instanceConnection.instance.server.version
+					: instanceInfo.config.get("factorio.version"),
 			)
 		);
 
@@ -736,7 +745,7 @@ export default class Host extends lib.Link {
 	}
 
 	assignGamePort(instanceId: number) {
-		const availablePorts = lib.parseRanges(this.config.get("host.factorio_port_range"), 1, 2**16 - 1);
+		const availablePorts = lib.parseRanges(this.config.get("host.factorio_port_range"), 1, 2 ** 16 - 1);
 		for (const [id, instance] of this.instanceInfos) {
 			if (id === instanceId) {
 				continue;
@@ -1060,6 +1069,7 @@ export default class Host extends lib.Link {
 				instanceInfo.config.toRemote("controller"),
 				instanceConnection ? instanceConnection.instance.status : "stopped",
 				instanceConnection ? instanceConnection.instance.server.gamePort : this.gamePort(instanceId),
+				instanceConnection ? instanceConnection.instance.server.version : undefined,
 			));
 		}
 		await this.send(new lib.InstancesUpdateRequest(list));

packages/host/src/Instance.ts

@@ -4,6 +4,8 @@ import pidusage from "pidusage";
 import phin from "phin";
 import util from "util";
 import type { Static } from "@sinclair/typebox";
+import { exec } from "child_process";
+const execAsync = util.promisify(exec);
 
 // internal libraries
 import * as lib from "@clusterio/lib";
@@ -14,6 +16,12 @@ import { exportData } from "./export";
 import type Host from "./Host";
 import BaseInstancePlugin from "./BaseInstancePlugin";
 
+const scriptCommands = [
+	"/cheat", "/editor",
+	"/command", "/c",
+	"/measured-command", "/mc",
+	"/silent-command", "/sc",
+];
 
 const instanceRconCommandDuration = new lib.Histogram(
 	"clusterio_instance_rcon_command_duration_seconds",
@@ -91,6 +99,38 @@ const serverSettingsActions = {
 	},
 };
 
+/**
+ * Get process stats for a Windows process
+ * Due to wmic being removed in later versions of Windows, pidusage does not work
+ * until this PR is merged: https://github.com/soyuka/pidusage/pull/143
+ * @param pid - Process ID
+ * @param logger - Logger instance
+ * @returns Process stats
+ */
+async function getWindowsProcessStats(pid: number, logger: lib.Logger) {
+	try {
+		const { stdout } = await execAsync(
+			`powershell "Get-Process -Id ${pid} | Select-Object TotalProcessorTime,WorkingSet | ConvertTo-Json"`
+		);
+		const stats = JSON.parse(stdout);
+		// Convert ticks (100-nanosecond intervals) to milliseconds
+		const totalTimeMs = stats.TotalProcessorTime.Ticks / 10000;
+		return {
+			ctime: totalTimeMs,
+			memory: stats.WorkingSet,
+		};
+	} catch (err) {
+		logger.warn("Failed to get process stats:", err);
+		return { ctime: 0, memory: 0 };
+	}
+}
+
+async function getProcessStats(pid: number, logger: lib.Logger) {
+	if (process.platform === "win32") {
+		return getWindowsProcessStats(pid, logger);
+	}
+	return pidusage(pid);
+}
 
 /**
  * Keeps track of the runtime parameters of an instance
@@ -127,7 +167,7 @@ export default class Instance extends lib.Link {
 	_playerCheckInterval: ReturnType<typeof setInterval> | undefined;
 	_hadPlayersOnline = false;
 	_playerAutosaveSlot = 1;
-
+	_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
 
 	constructor(
 		host: Host,
@@ -153,7 +193,9 @@ export default class Instance extends lib.Link {
 		this._configFieldChanged = (field: string, curr: unknown, prev: unknown) => {
 			let hook = () => lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", field, curr, prev);
 
-			if (field === "factorio.settings") {
+			if (field === "factorio.shutdown_timeout") {
+				this.server.shutdownTimeoutMs = curr as number * 1000;
+			} else if (field === "factorio.settings") {
 				this.updateFactorioSettings(curr as any, prev as any).finally(hook);
 			} else if (field === "factorio.enable_whitelist") {
 				this.updateFactorioWhitelist(curr as any).finally(hook);
@@ -169,6 +211,7 @@ export default class Instance extends lib.Link {
 		let serverOptions = {
 			logger: this.logger,
 			version: this.config.get("factorio.version"),
+			executablePath: this.config.get("factorio.executable_path") ?? undefined,
 			gamePort: this.config.get("factorio.game_port") ?? host.assignGamePort(this.id),
 			rconPort: this.config.get("factorio.rcon_port") ?? undefined,
 			rconPassword: this.config.get("factorio.rcon_password") ?? undefined,
@@ -177,6 +220,7 @@ export default class Instance extends lib.Link {
 			verboseLogging: this.config.get("factorio.verbose_logging"),
 			stripPaths: this.config.get("factorio.strip_paths"),
 			maxConcurrentCommands: this.config.get("factorio.max_concurrent_commands"),
+			shutdownTimeoutMs: this.config.get("factorio.shutdown_timeout") * 1000,
 		};
 
 		this.server = new FactorioServer(
@@ -214,6 +258,10 @@ export default class Instance extends lib.Link {
 				this._recordPlayerJoin(event.name);
 			} else if (event.type === "leave") {
 				this._recordPlayerLeave(event.name, event.reason);
+			} else if (["BAN", "UNBANNED", "PROMOTE", "DEMOTE"].includes(event.type)) {
+				this._recordUserUpdate(event.type, event.name, event.reason);
+			} else {
+				this.logger.warn(`Unknown type from player event ipc: ${event.type}`);
 			}
 		});
 
@@ -236,13 +284,15 @@ export default class Instance extends lib.Link {
 		this.handle(lib.InstanceSendRconRequest, this.handleInstanceSendRconRequest.bind(this));
 	}
 
-	_watchPlayerJoinsByChat() {
+	_watchServerLogActions() {
 		this.server.on("output", (parsed: lib.ParsedFactorioOutput, line: string) => {
 			if (parsed.type !== "action") {
 				return;
 			}
 
 			let name = /^([^ ]+)/.exec(parsed.message)![1];
+
+			// Detect player leave and join
 			if (parsed.action === "JOIN") {
 				this._recordPlayerJoin(name);
 			} else if (["LEAVE", "KICK", "BAN"].includes(parsed.action)) {
@@ -253,6 +303,14 @@ export default class Instance extends lib.Link {
 				}[parsed.action]!;
 				this._recordPlayerLeave(name, reason);
 			}
+
+			// Detect banned and admin state change, whitelist changes are not logged
+			if (parsed.action === "BAN") {
+				const reason = /\. Reason: (.+)\.$/.exec(parsed.message)![1];
+				this._recordUserUpdate(parsed.action, name, reason !== "unspecified" ? reason : "");
+			} else if (["UNBANNED", "PROMOTE", "DEMOTE"].includes(parsed.action)) {
+				this._recordUserUpdate(parsed.action as "UNBANNED" | "PROMOTE" | "DEMOTE", name);
+			}
 		});
 
 		// Leave log entries are unreliable and sometimes don't show up.
@@ -263,6 +321,22 @@ export default class Instance extends lib.Link {
 		}, 60e3);
 	}
 
+	// Needed to cover promote/demote when player is not on map, not required when _watchServerLogActions is used
+	_watchPlayerPromote() {
+		this.server.on("output", (parsed: lib.ParsedFactorioOutput, line: string) => {
+			if (
+				parsed.type !== "action"
+				|| !["PROMOTE", "DEMOTE"].includes(parsed.action)
+				|| !parsed.message.includes("be promoted upon joining the game.")
+			) {
+				return;
+			}
+
+			const name = /^([^ ]+)/.exec(parsed.message)![1];
+			this._recordUserUpdate(parsed.action as "PROMOTE" | "DEMOTE", name);
+		});
+	}
+
 	async _checkOnlinePlayers() {
 		if (this.playersOnline.size) {
 			let actualPlayers = (await this.sendRcon("/players online"))
@@ -367,6 +441,17 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	}
 
 	async sendRcon(message: string, expectEmpty = false, plugin = "") {
+		const trimmedMessage = message.trim();
+		if (
+			!this.config.get("factorio.enable_script_commands")
+			&& scriptCommands.find(cmd => trimmedMessage.startsWith(cmd))
+		) {
+			throw new Error(
+				"Attempted to use script command while disabled. See config factorio.enable_script_commands.\n" +
+				`Command: ${message}`
+			);
+		}
+
 		let instanceId = String(this.id);
 		let observeDuration = instanceRconCommandDuration.labels(instanceId).startTimer();
 		try {
@@ -453,6 +538,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 				this.id,
 				status,
 				this.server.gamePort,
+				status === "running"? this.server.version : this.config.get("factorio.version"),
 			),
 		);
 	}
@@ -469,6 +555,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	notifyExit() {
 		this._loadedSave = null;
 		this.notifyStatus("stopped");
+		this.connector.emit("close");
 
 		this.config.off("fieldChanged", this._configFieldChanged);
 		clearTimeout(this._playerCheckInterval);
@@ -577,11 +664,32 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	 * entries from the given settings object.
 	 *
 	 * @param overrides - Server settings to override.
+	 * @param includeCredentials - Include Factorio username and token from host/controller config.
 	 * @returns
 	 *     server example settings with the given settings applied over it.
 	 */
-	async resolveServerSettings(overrides: Record<string, unknown>) {
+	async resolveServerSettings(overrides: Record<string, unknown>, includeCredentials: boolean) {
 		let serverSettings = await this.server.exampleSettings();
+		if (includeCredentials && overrides.username === undefined && overrides.token === undefined) {
+			let credentials = {
+				username: this._host.config.get("host.factorio_username") ?? undefined,
+				token: this._host.config.get("host.factorio_token") ?? undefined,
+			};
+			if (!credentials.username && !credentials.token) {
+				Object.assign(credentials, await this.sendTo("controller", new lib.GetFactorioCredentialsRequest()));
+				if (credentials.username || credentials.token) {
+					this.logger.info("Using Factorio credentials from controller config");
+				} else {
+					this.logger.warn("No Factorio credentials found");
+				}
+			} else {
+				this.logger.info("Using Factorio credentials from host config");
+			}
+			if (credentials.username) { serverSettings.username = credentials.username; }
+			if (credentials.token) { serverSettings.token = credentials.token; }
+		} else if (includeCredentials) {
+			this.logger.info("Using Factorio credentials from instance config");
+		}
 
 		for (let [key, value] of Object.entries(overrides)) {
 			if (!Object.hasOwnProperty.call(serverSettings, key)) {
@@ -599,12 +707,14 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	 * Generate the server-settings.json file from the example file in the
 	 * data directory and override any settings configured in the instance's
 	 * factorio_settings config entry.
+	 *
+	 * @param includeCredentials - Include Factorio username and token from host/controller config.
 	 */
-	async writeServerSettings() {
+	async writeServerSettings(includeCredentials: boolean) {
 		const warning = "Changes to this file will be overwitten by the factorio.settings config on the instance.";
 		const serverSettings = {
 			"_comment_warning": warning,
-			...await this.resolveServerSettings(this.config.get("factorio.settings")),
+			...await this.resolveServerSettings(this.config.get("factorio.settings"), includeCredentials),
 		};
 		await lib.safeOutputFile(
 			this.server.writePath("server-settings.json"),
@@ -615,14 +725,12 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	/**
 	 * Creates a new empty instance directory
 	 *
-	 * Creates the neccessary files for starting up a new instance into the
-	 * provided instance directory.
+	 * Ensures the neccessary files for starting up a new instance into the
+	 * provided instance directory are present.
 	 *
 	 * @param instanceDir -
-	 *     Directory to create the new instance into.
-	 * @param factorioDir - Path to factorio installation.
 	 */
-	static async create(instanceDir: string, factorioDir: string) {
+	static async populate_folders(instanceDir: string) {
 		await fs.ensureDir(path.join(instanceDir, "script-output"));
 		await fs.ensureDir(path.join(instanceDir, "saves"));
 	}
@@ -708,9 +816,10 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	 */
 	async prepare() {
 		this.logger.verbose("Writing server-settings.json");
-		await this.writeServerSettings();
+		await this.writeServerSettings(true);
+		this._expectedUserUpdates = [];
 
-		if (this.config.get("factorio.sync_adminlist")) {
+		if (this.config.get("factorio.sync_adminlist") !== "disabled") {
 			this.logger.verbose("Writing server-adminlist.json");
 			lib.safeOutputFile(
 				this.server.writePath("server-adminlist.json"),
@@ -718,7 +827,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			);
 		}
 
-		if (this.config.get("factorio.sync_banlist")) {
+		if (this.config.get("factorio.sync_banlist") !== "disabled") {
 			this.logger.verbose("Writing server-banlist.json");
 			lib.safeOutputFile(
 				this.server.writePath("server-banlist.json"),
@@ -728,7 +837,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			);
 		}
 
-		if (this.config.get("factorio.sync_whitelist")) {
+		if (this.config.get("factorio.sync_whitelist") !== "disabled") {
 			this.logger.verbose("Writing server-whitelist.json");
 			lib.safeOutputFile(
 				this.server.writePath("server-whitelist.json"),
@@ -831,11 +940,12 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 		this._loadedSave = saveName;
 		await this.server.start(saveName);
 
-		if (this.config.get("factorio.enable_save_patching")) {
+		if (this.config.get("factorio.enable_save_patching") && this.config.get("factorio.enable_script_commands")) {
 			await this.server.disableAchievements();
 			await this.updateInstanceData();
+			this._watchPlayerPromote();
 		} else {
-			this._watchPlayerJoinsByChat();
+			this._watchServerLogActions();
 		}
 
 		await this.sendSaveListUpdate();
@@ -862,7 +972,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 
 		this.server.on("exit", () => this.notifyExit());
 		await this.server.startScenario(scenario, seed, mapGenSettings, mapSettings);
-		this._watchPlayerJoinsByChat();
+		this._watchServerLogActions();
 
 		await lib.invokeHook(this.plugins, "onStart");
 
@@ -878,8 +988,8 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	}
 
 	async updateFactorioSettings(current: Record<string, unknown>, previous: Record<string, unknown>) {
-		current = await this.resolveServerSettings(current);
-		previous = await this.resolveServerSettings(previous);
+		current = await this.resolveServerSettings(current, false);
+		previous = await this.resolveServerSettings(previous, false);
 
 		for (let [key, action] of Object.entries(serverSettingsActions)) {
 			if (current[key] !== undefined && !util.isDeepStrictEqual(current[key], previous[key])) {
@@ -899,7 +1009,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			await this.sendRcon("/whitelist disable");
 		}
 
-		if (this.config.get("factorio.sync_whitelist")) {
+		if (this.config.get("factorio.sync_whitelist") !== "disabled") {
 			await this.sendRcon("/whitelist clear");
 			for (let player of this._host.whitelist) {
 				await this.sendRcon(`/whitelist ${player}`);
@@ -911,33 +1021,82 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 		}
 	}
 
+	_recordUserUpdate(
+		action: "BAN" | "UNBANNED" | "PROMOTE" | "DEMOTE" | "WHITELISTED" | "UNWHITELISTED",
+		name: string,
+		reason: string = ""
+	) {
+		// TODO: Implement bidirectional whitelist sync, likely by watching the json file
+		const addr = lib.Address.fromShorthand("allInstances");
+		const expectedIndex = this._expectedUserUpdates.findIndex(
+			expected => expected.name === name && expected.action === action && expected.reason === reason
+		);
+		if (expectedIndex >= 0) {
+			this._expectedUserUpdates = this._expectedUserUpdates.splice(expectedIndex, 1);
+		} else if (action === "BAN") {
+			if (this.config.get("factorio.sync_banlist") !== "bidirectional") { return; }
+			this.sendTo(addr, new lib.InstanceBanlistUpdateEvent(name, true, reason ?? ""));
+		} else if (action === "UNBANNED") {
+			if (this.config.get("factorio.sync_banlist") !== "bidirectional") { return; }
+			this.sendTo(addr, new lib.InstanceBanlistUpdateEvent(name, false, ""));
+		} else if (action === "PROMOTE") {
+			if (this.config.get("factorio.sync_adminlist") !== "bidirectional") { return; }
+			this.sendTo(addr, new lib.InstanceAdminlistUpdateEvent(name, true));
+		} else if (action === "DEMOTE") {
+			if (this.config.get("factorio.sync_adminlist") !== "bidirectional") { return; }
+			this.sendTo(addr, new lib.InstanceAdminlistUpdateEvent(name, false));
+		} else if (action === "WHITELISTED") {
+			if (this.config.get("factorio.sync_whitelist") !== "bidirectional") { return; }
+			this.sendTo(addr, new lib.InstanceWhitelistUpdateEvent(name, true));
+		} else if (action === "UNWHITELISTED") {
+			if (this.config.get("factorio.sync_whitelist") !== "bidirectional") { return; }
+			this.sendTo(addr, new lib.InstanceWhitelistUpdateEvent(name, false));
+		} else {
+			throw new Error(`Unexpected Action: ${action} for ${name}`);
+		}
+	}
+
 	async handleInstanceAdminlistUpdateEvent(request: lib.InstanceAdminlistUpdateEvent) {
-		if (!this.config.get("factorio.sync_adminlist")) {
+		const { name, admin } = request;
+		const sync = this.config.get("factorio.sync_adminlist");
+		if (sync === "disabled") {
 			return;
+		} else if (sync === "bidirectional") {
+			this._expectedUserUpdates.push({action: admin ? "PROMOTE" : "DEMOTE", name: name, reason: "" });
 		}
 
-		let { name, admin } = request;
 		let command = admin ? `/promote ${name}` : `/demote ${name}`;
 		await this.sendRcon(command);
 	}
 
 	async handleInstanceBanlistUpdateEvent(request: lib.InstanceBanlistUpdateEvent) {
-		if (!this.config.get("factorio.sync_banlist")) {
+		const { name, banned, reason } = request;
+		const sync = this.config.get("factorio.sync_banlist");
+		if (sync === "disabled") {
 			return;
+		} else if (sync === "bidirectional") {
+			this._expectedUserUpdates.push({action: banned ? "BAN" : "UNBANNED", name: name, reason: reason});
 		}
 
-		let { name, banned, reason } = request;
 		let command = banned ? `/ban ${name} ${reason}` : `/unban ${name}`;
 		await this.sendRcon(command);
 	}
 
 	async handleInstanceWhitelistUpdateEvent(request: lib.InstanceWhitelistUpdateEvent) {
-		if (!this.config.get("factorio.sync_whitelist")) {
+		const { name, whitelisted } = request;
+		const sync = this.config.get("factorio.sync_whitelist");
+		if (this.config.get("factorio.sync_whitelist") === "disabled") {
 			return;
+		} else if (sync === "bidirectional") {
+			// Factorio does not contain log events for whitelist so these values currently have no special meaning
+			this._expectedUserUpdates.push({
+				action: whitelisted ? "WHITELISTED" : "UNWHITELISTED",
+				name: name,
+				reason: "",
+			});
 		}
 
-		let { name, whitelisted } = request;
-		let command = whitelisted ? `/whitelist add ${name}` : `/whiteliste remove ${name}`;
+		let command = whitelisted ? `/whitelist add ${name}` : `/whitelist remove ${name}`;
 		await this.sendRcon(command);
 	}
 
@@ -991,9 +1150,13 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 
 		let pid = this.server.pid;
 		if (pid) {
-			let stats = await pidusage(pid);
-			instanceFactorioCpuTime.labels(String(this.id)).set(stats.ctime / 1000);
-			instanceFactorioMemoryUsage.labels(String(this.id)).set(stats.memory);
+			try {
+				let stats = await getProcessStats(pid, this.logger);
+				instanceFactorioCpuTime.labels(String(this.id)).set(stats.ctime / 1000);
+				instanceFactorioMemoryUsage.labels(String(this.id)).set(stats.memory);
+			} catch (err) {
+				this.logger.warn("Failed to get process stats:", err);
+			}
 		}
 
 		return new lib.InstanceMetricsRequest.Response(results);
@@ -1002,21 +1165,26 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 	async handleInstanceStartRequest(request: lib.InstanceStartRequest) {
 		let saveName = request.save;
 		try {
-			await this.prepare();
-			saveName = await this.prepareSave(saveName);
-		} catch (err: any) {
-			this.logger.error(`Error preparing instance: ${err.message}`);
-			this.notifyExit();
-			await this.sendSaveListUpdate();
-			throw err;
-		}
+			try {
+				await this.prepare();
+				saveName = await this.prepareSave(saveName);
+			} catch (err: any) {
+				this.logger.error(`Error preparing instance: ${err.message}`);
+				this.notifyExit();
+				await this.sendSaveListUpdate();
+				throw err;
+			}
 
-		try {
-			await this.start(saveName);
-		} catch (err: any) {
-			this.logger.error(`Error starting ${saveName}: ${err.message}`);
-			await this.stop();
-			throw err;
+			try {
+				await this.start(saveName);
+			} catch (err: any) {
+				this.logger.error(`Error starting ${saveName}: ${err.message}`);
+				await this.stop();
+				throw err;
+			}
+		} finally {
+			this.logger.verbose("Wiping credentials from server-settings.json");
+			await this.writeServerSettings(false);
 		}
 	}
 
@@ -1027,21 +1195,26 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 		}
 
 		try {
-			await this.prepare();
-		} catch (err: any) {
-			this.logger.error(`Error preparing instance: ${err.message}`);
-			this.notifyExit();
-			await this.sendSaveListUpdate();
-			throw err;
-		}
+			try {
+				await this.prepare();
+			} catch (err: any) {
+				this.logger.error(`Error preparing instance: ${err.message}`);
+				this.notifyExit();
+				await this.sendSaveListUpdate();
+				throw err;
+			}
 
-		let { scenario, seed, mapGenSettings, mapSettings } = request;
-		try {
-			await this.startScenario(scenario, seed, mapGenSettings, mapSettings);
-		} catch (err: any) {
-			this.logger.error(`Error starting scenario ${scenario}: ${err.message}`);
-			await this.stop();
-			throw err;
+			let { scenario, seed, mapGenSettings, mapSettings } = request;
+			try {
+				await this.startScenario(scenario, seed, mapGenSettings, mapSettings);
+			} catch (err: any) {
+				this.logger.error(`Error starting scenario ${scenario}: ${err.message}`);
+				await this.stop();
+				throw err;
+			}
+		} finally {
+			this.logger.verbose("Wiping credentials from server-settings.json");
+			await this.writeServerSettings(false);
 		}
 	}
 
@@ -1053,7 +1226,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 		this.notifyStatus("creating_save");
 		try {
 			this.logger.verbose("Writing server-settings.json");
-			await this.writeServerSettings();
+			await this.writeServerSettings(false);
 			await this.syncMods();
 
 		} catch (err: any) {
@@ -1081,7 +1254,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 		this.notifyStatus("exporting_data");
 		try {
 			this.logger.verbose("Writing server-settings.json");
-			await this.writeServerSettings();
+			await this.writeServerSettings(false);
 
 			this.logger.info("Exporting data .....");
 			await this.syncMods();

packages/host/src/export.ts

@@ -15,12 +15,12 @@ interface Prototype {
 }
 interface SimpleIconSpecification {
 	icon: string;
-	icon_size: number;
+	icon_size?: number;
 	icon_mipmaps?: number;
 }
 interface IconLayer {
 	icon: string;
-	icon_size: number;
+	icon_size?: number;
 	tint?: lib.FactorioColor,
 	shift?: [number, number],
 	scale?: number,
@@ -28,7 +28,7 @@ interface IconLayer {
 }
 interface LayeredIconSpecification {
 	icons: IconLayer[];
-	icon_size: number;
+	icon_size?: number;
 	icon_mipmaps?: number;
 }
 type IconSpecification = LayeredIconSpecification | SimpleIconSpecification;
@@ -64,7 +64,7 @@ async function generateExportMod(server: FactorioServer) {
 		sourceDir: path.join(__dirname, "..", "..", "..", "lua", "export"),
 		outputDir: server.writePath("mods"),
 		bumpPatch: false,
-		factorioVersion: server.version.replace(/\.\d+$/, ""),
+		factorioVersion: server.version?.replace(/\.\d+$/, ""),
 		dependencies,
 	});
 }
@@ -103,7 +103,8 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
 
 	let [, mod, filePath] = match;
 
-	if (["core", "base"].includes(mod)) {
+	const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)];
+	if (builtinModNames.includes(mod)) {
 		try {
 			return await fs.readFile(server.dataPath(mod, filePath));
 		} catch (err: any) {
@@ -164,9 +165,9 @@ async function loadSimpleIcon(
 	size: number,
 	iconCache: IconCache,
 ) {
-	let icon = await loadIcon(server, modVersions, item.icon, item.icon_size, 0, iconCache);
+	let icon = await loadIcon(server, modVersions, item.icon, item.icon_size ?? 64, 0, iconCache);
 	if (icon) {
-		let iconScale = size / item.icon_size;
+		let iconScale = size / (item.icon_size ?? 64);
 		if (iconScale !== 1) {
 			icon = icon.clone();
 			icon.scale(iconScale);
@@ -182,7 +183,7 @@ async function loadLayeredIcon(
 	size: number,
 	iconCache: IconCache,
 ) {
-	let baseLayerSize = item.icons[0].icon_size || item.icon_size;
+	let baseLayerSize = (item.icons[0].icon_size || item.icon_size) ?? 64;
 	let icon = await Jimp.create(size, size);
 
 	// The scaling factor of the base layer
@@ -192,7 +193,7 @@ async function loadLayeredIcon(
 	let baseUnit = size / (baseLayerSize * baseLayerScale);
 
 	for (let layer of item.icons) {
-		let layerSize = layer.icon_size || item.icon_size;
+		let layerSize = (layer.icon_size || item.icon_size) ?? 64;
 		let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
 
 		if (!iconLayer) {
@@ -396,11 +397,18 @@ async function exportLocale(
 		}
 	}
 
-	let baseLocaleFilePath = server.dataPath("base", "locale", languageCode, "base.cfg");
-	mergeLocale(lib.parse(await fs.readFile(baseLocaleFilePath, "utf8")));
+	const builtinModNames = [
+		// Filter if on builtin mods so "core" does not need to be specified in modOrder
+		"core", ...lib.ModPack.getBuiltinModNames(server.version!).filter(mod => modOrder.includes(mod)),
+	];
+	for (const builtinModName of builtinModNames) {
+		const localeFilePath = server.dataPath(builtinModName, "locale", languageCode, `${builtinModName}.cfg`);
+		mergeLocale(lib.parse(await fs.readFile(localeFilePath, "utf8")));
+	}
 
+	const builtinModNamesExport = ["export", ...builtinModNames];
 	for (let mod of modOrder) {
-		if (["core", "base", "export"].includes(mod)) {
+		if (builtinModNamesExport.includes(mod)) {
 			continue;
 		}
 

packages/host/src/patch.ts

@@ -202,12 +202,6 @@ export class PatchInfo {
 	}
 }
 
-
-const knownScenarios: Record<string, lib.ModuleInfo> = {
-	// First seen in 0.17.63
-	"4e866186ebe297f1038fd325b09df1a1f5e2fdd1": new lib.ModuleInfo("freeplay", "0.17.63", ["freeplay", "silo-script"]),
-};
-
 /**
  * Generates control.lua code for loading the Clusterio modules
  *
@@ -349,6 +343,13 @@ function reorderDependencies(modules: SaveModule[]) {
 	}
 }
 
+const knownScenarios: Record<string, lib.ModuleInfo> = {
+	// First seen in 0.17.63
+	"4e866186ebe297f1038fd325b09df1a1f5e2fdd1": new lib.ModuleInfo("freeplay", "0.17.63", [], ["scenario"]),
+	// First seen in 2.0
+	"bcbdde18ce4ec16ebfd93bd694cd9b12ef969c9a": new lib.ModuleInfo("freeplay", "2.0.0", [], ["scenario"]),
+};
+
 /**
  * Patch a save with the given modules
  *
@@ -387,6 +388,7 @@ export async function patch(savePath: string, modules: SaveModule[]) {
 
 		if (controlHash in knownScenarios) {
 			patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
+			root.file("scenario.lua", controlFile.nodeStream("nodebuffer"));
 		} else {
 			throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
 		}

packages/host/src/server.ts

@@ -351,20 +351,37 @@ const outputHeuristics: Heuristic[] = [
 		},
 	},
 
-	// Message indicating the server is shutting down
+	// Last messages before the server hangs during shutdown waiting for console input on Windows.
 	{
 		filter: {
 			type: "generic",
-			message: /^Quitting: /,
+			message: "Quitting: multiplayer error.",
 		},
 		action: function(parsed) {
-			this.emit("_quitting");
-
 			// On windows the Factorio server will wait for input on the console input buffer
-			// when an error occurs.  Since we can't send input there we terminate the server
+			// before shutting down.  Since we can't send input there we terminate the server
 			// process when this happens.
-			if (process.platform === "win32" && parsed.message === "Quitting: multiplayer error.") {
+			if (process.platform === "win32") {
 				this._server!.kill();
+				// Notify that the server is being killed after the last message
+				// is logged so that it doesn't look like it was killed unnecessarily.
+				process.nextTick(() => {
+					this._logger.info("Killing Factorio process to avoid it hanging forever.");
+				});
+			}
+		},
+	},
+	{
+		filter: {
+			type: "log",
+			message: /^updateTick\(\d+\) changing state from\(Disconnected\) to\(Closed\)$/,
+		},
+		action: function() {
+			if (process.platform === "win32") {
+				this._server!.kill();
+				process.nextTick(() => {
+					this._logger.info("Killing Factorio process to avoid it hanging forever.");
+				});
 			}
 		},
 	},
@@ -407,6 +424,8 @@ export interface FactorioServerOptions {
 	 * the latest version found in `factorioDir`.
 	 */
 	version?: string,
+	/** Path to executable to invoke when starting the server */
+	executablePath?: string;
 	/** UDP port to host game on. */
 	gamePort?: number,
 	/** TCP port to use for RCON. */
@@ -426,8 +445,11 @@ export interface FactorioServerOptions {
 	 * connection.
 	 */
 	maxConcurrentCommands?: number
-	/** Timeout in ms to wait for a response to a shutdown command. */
-	hangTimeoutMs?: number,
+	/**
+	 * Timeout in ms to wait after a shutdown is requested before killing the
+	 * process. Defaults to 0 meaning no timeout
+	 */
+	shutdownTimeoutMs?: number,
 }
 
 /**
@@ -450,6 +472,8 @@ export interface FactorioServerOptions {
  * @extends events.EventEmitter
  */
 export class FactorioServer extends events.EventEmitter {
+	/** Path to executable to invoke when starting the server */
+	executablePath?: string;
 	/** UDP port used for hosting the Factorio game server on */
 	gamePort: number;
 	/** TCP port used for RCON on the Factorio game server */
@@ -462,8 +486,8 @@ export class FactorioServer extends events.EventEmitter {
 	enableAuthserverBans: boolean;
 	/** Enable verbose logging */
 	verboseLogging: boolean;
-	/** Timeout in ms to wait for a response to a shutdown command */
-	hangTimeoutMs = 5000;
+	_shutdownTimeoutMs = 0;
+	_stopTimeoutId?: ReturnType<typeof setTimeout>;
 
 	_factorioDir: string;
 	_writeDir: string;
@@ -504,6 +528,7 @@ export class FactorioServer extends events.EventEmitter {
 
 		this._logger = options.logger || lib.logger;
 		this._targetVersion = options.version || "latest";
+		this.executablePath = options.executablePath;
 		/** UDP port used for hosting the Factorio game server on */
 		this.gamePort = options.gamePort || randomDynamicPort();
 		/** TCP port used for RCON on the Factorio game server */
@@ -518,8 +543,9 @@ export class FactorioServer extends events.EventEmitter {
 		this.verboseLogging = options.verboseLogging || false;
 		/** Maximum number of RCON commands transmitted in parallel on the RCON connection  */
 		this.maxConcurrentCommands = options.maxConcurrentCommands || 5;
-		/** Timeout in ms to wait for a response to a shutdown command */
-		this.hangTimeoutMs = options.hangTimeoutMs ?? 5000;
+		if (options.shutdownTimeoutMs !== undefined) {
+			this.shutdownTimeoutMs = options.shutdownTimeoutMs;
+		}
 
 		if (options.stripPaths) {
 			let charSet = new Set(path.resolve(this.writePath("temp")));
@@ -564,8 +590,8 @@ export class FactorioServer extends events.EventEmitter {
 
 	// TODO at the moment this is just a helper function, it has the potential for schema checking
 	handle<Event>(eventName: string, handler: (event: Event) => Promise<void>) {
-		this.on(`ipc-${eventName}`, (event) => handler(event).catch(err => {
-			this._logger.error(`Error handling ipc event:\n${err.stack}`);
+		this.on(`ipc-${eventName}`, (event) => handler(event).catch((err: Error) => {
+			this._logger.error(`Error handling ipc event:\n${err.stack ?? err.message}`);
 		}));
 	}
 
@@ -661,10 +687,6 @@ export class FactorioServer extends events.EventEmitter {
 	}
 
 	async _startRcon() {
-		if (this._state === "stopping") {
-			return;
-		}
-
 		if (this._rconClient !== null) {
 			throw Error("RCON client is already started");
 		}
@@ -678,16 +700,39 @@ export class FactorioServer extends events.EventEmitter {
 		};
 
 		this._rconClient = new Rcon(config);
-		this._rconClient.on("error", () => { /* Ignore */ });
-		this._rconClient.on("authenticated", () => { this._rconReady = true; this.emit("rcon-ready"); });
+		this._rconClient.on("error", () => {
+			// Errors before being authenticated might not emit end
+			this._rconClient = null;
+			if (!this._rconReady) {
+				this._rconReady = true;
+				this.emit("rcon-ready");
+			}
+		});
+		this._rconClient.on("authenticated", () => {
+			this._rconReady = true;
+			this.emit("rcon-ready");
+		 });
 		this._rconClient.on("end", () => {
 			this._rconClient = null;
+			if (!this._rconReady) {
+				this._rconReady = true;
+				this.emit("rcon-ready");
+			}
 		});
 		// XXX: Workaround to suppress bogus event listener warning
 		if (this._rconClient.emitter instanceof events.EventEmitter) {
 			this._rconClient.emitter.setMaxListeners(this.maxConcurrentCommands + 5);
 		}
-		await this._rconClient.connect();
+		try {
+			await this._rconClient.connect();
+		} catch (err: any) {
+			this._logger.error(`Failed to start RCON connection:\n${err?.stack ?? err?.message ?? err}`);
+			this._rconClient = null;
+			if (!this._rconReady) {
+				this._rconReady = true;
+				this.emit("rcon-ready");
+			}
+		}
 	}
 
 	/** Maximum number of RCON commands transmitted in parallel on the RCON connection  */
@@ -725,12 +770,12 @@ export class FactorioServer extends events.EventEmitter {
 	}
 
 	/**
-	 * The version of Factorio in use.  This will be the actual version if
-	 * "latest" (the default) was specified as `factorioVersion` to the
-	 * constructor.
+	 * The version of Factorio in use. This will be the actual version if
+	 * "latest" (the default) was specified as `factorioVersion` to the constructor.
+	 * This will be undefined before the server is initialized, or if init failed.
 	 */
-	get version() {
-		return this._version!;
+	get version(): string | undefined {
+		return this._version ?? undefined;
 	}
 
 	/**
@@ -759,6 +804,7 @@ export class FactorioServer extends events.EventEmitter {
 		this._rconReady = false;
 		this._gameReady = false;
 		this._unexpected = [];
+		this._killed = false;
 		this._runningAutosave = null;
 	}
 
@@ -793,6 +839,9 @@ export class FactorioServer extends events.EventEmitter {
 						`\n- ${this._unexpected.join("\n- ")}`
 					));
 				}
+
+			} else { // if state === "stopping"
+				this._clearStopTimeout();
 			}
 
 			if (this._rconClient) {
@@ -847,28 +896,36 @@ export class FactorioServer extends events.EventEmitter {
 		this._check(["init"]);
 		this._state = "create";
 
-		await this._writeConfigIni();
-		await this._writeMapSettings(mapGenSettings, mapSettings);
-		this._server = child_process.spawn(
-			this.binaryPath(),
-			[
-				"--config", this.writePath("config.ini"),
-				"--create", this.writePath("saves", name),
-				...(seed !== undefined ? ["--map-gen-seed", String(seed)] : []),
-				...(mapGenSettings !== undefined
-					? ["--map-gen-settings", this.writePath("map-gen-settings.json")] : []
-				),
-				...(mapSettings !== undefined ? ["--map-settings", this.writePath("map-settings.json")] : []),
-				...(this.verboseLogging ? ["--verbose"] : []),
-			],
-			{
-				detached: true,
-				stdio: "pipe",
-			}
-		);
-
-		this._attachStdio();
-		this._server.once("exit", () => this.emit("exit"));
+		try {
+			// Everything between _state and once exit must be sync and call _resetState on error
+			// If this this thread yields then the server will be left in an invalid transient state
+			/* eslint-disable node/no-sync */
+			this._writeConfigIniSync();
+			this._writeMapSettingsSync(mapGenSettings, mapSettings);
+			this._server = child_process.spawn(
+				this.binaryPath(),
+				[
+					"--config", this.writePath("config.ini"),
+					"--create", this.writePath("saves", name),
+					...(seed !== undefined ? ["--map-gen-seed", String(seed)] : []),
+					...(mapGenSettings !== undefined
+						? ["--map-gen-settings", this.writePath("map-gen-settings.json")] : []
+					),
+					...(mapSettings !== undefined ? ["--map-settings", this.writePath("map-settings.json")] : []),
+					...(this.verboseLogging ? ["--verbose"] : []),
+				],
+				{
+					detached: true,
+					stdio: "pipe",
+				}
+			);
+			this._attachStdio();
+			this._server.once("exit", () => this.emit("exit"));
+			/* eslint-enable node/no-sync */
+		} catch (err: any) {
+			this._resetState();
+			throw new lib.EnvironmentError(`Unexpected error:\n${err.stack}`);
+		}
 
 		try {
 			let [code, signal] = await events.once(this._server, "exit");
@@ -900,27 +957,36 @@ export class FactorioServer extends events.EventEmitter {
 		this._check(["init"]);
 		this._state = "running";
 
-		await this._writeConfigIni();
-		this._server = child_process.spawn(
-			this.binaryPath(),
-			[
-				"--config", this.writePath("config.ini"),
-				"--start-server", this.writePath("saves", save),
-				"--port", String(this.gamePort),
-				"--rcon-port", String(this.rconPort),
-				"--rcon-password", this.rconPassword,
-				...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
-				...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
-				...(this.verboseLogging ? ["--verbose"] : []),
-			],
-			{
-				detached: true,
-				stdio: "pipe",
-			}
-		);
+		try {
+			// Everything between _state and _watchExist must be sync and call _resetState on error
+			// If this this thread yields then the server will be left in an invalid transient state
+			/* eslint-disable node/no-sync */
+			this._writeConfigIniSync();
+			this._server = child_process.spawn(
+				this.binaryPath(),
+				[
+					"--config", this.writePath("config.ini"),
+					"--start-server", this.writePath("saves", save),
+					"--port", String(this.gamePort),
+					"--rcon-port", String(this.rconPort),
+					"--rcon-password", this.rconPassword,
+					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
+					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
+					...(this.verboseLogging ? ["--verbose"] : []),
+				],
+				{
+					detached: true,
+					stdio: "pipe",
+				}
+			);
+			this._attachStdio();
+			this._watchExit();
+			/* eslint-enable node/no-sync */
+		} catch (err: any) {
+			this.emit("error", new lib.EnvironmentError(`Unexpected error:\n${err.stack}`));
+			this._resetState();
+		}
 
-		this._attachStdio();
-		this._watchExit();
 		await this._waitForReady();
 	}
 
@@ -941,33 +1007,42 @@ export class FactorioServer extends events.EventEmitter {
 		this._check(["init"]);
 		this._state = "running";
 
-		await this._writeConfigIni();
-		await this._writeMapSettings(mapGenSettings, mapSettings);
-		this._server = child_process.spawn(
-			this.binaryPath(),
-			[
-				"--config", this.writePath("config.ini"),
-				"--start-server-load-scenario", scenario,
-				...(seed !== undefined ? ["--map-gen-seed", String(seed)] : []),
-				...(mapGenSettings !== undefined
-					? ["--map-gen-settings", this.writePath("map-gen-settings.json")] : []
-				),
-				...(mapSettings !== undefined ? ["--map-settings", this.writePath("map-settings.json")] : []),
-				"--port", String(this.gamePort),
-				"--rcon-port", String(this.rconPort),
-				"--rcon-password", this.rconPassword,
-				...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
-				...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
-				...(this.verboseLogging ? ["--verbose"] : []),
-			],
-			{
-				detached: true,
-				stdio: "pipe",
-			}
-		);
+		try {
+			// Everything between _state and _watchExist must be sync and call _resetState on error
+			// If this this thread yields then the server will be left in an invalid transient state
+			/* eslint-disable node/no-sync */
+			this._writeConfigIniSync();
+			this._writeMapSettingsSync(mapGenSettings, mapSettings);
+			this._server = child_process.spawn(
+				this.binaryPath(),
+				[
+					"--config", this.writePath("config.ini"),
+					"--start-server-load-scenario", scenario,
+					...(seed !== undefined ? ["--map-gen-seed", String(seed)] : []),
+					...(mapGenSettings !== undefined
+						? ["--map-gen-settings", this.writePath("map-gen-settings.json")] : []
+					),
+					...(mapSettings !== undefined ? ["--map-settings", this.writePath("map-settings.json")] : []),
+					"--port", String(this.gamePort),
+					"--rcon-port", String(this.rconPort),
+					"--rcon-password", this.rconPassword,
+					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
+					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
+					...(this.verboseLogging ? ["--verbose"] : []),
+				],
+				{
+					detached: true,
+					stdio: "pipe",
+				}
+			);
+			this._attachStdio();
+			this._watchExit();
+			/* eslint-enable node/no-sync */
+		} catch (err: any) {
+			this.emit("error", new lib.EnvironmentError(`Unexpected error:\n${err.stack}`));
+			this._resetState();
+		}
 
-		this._attachStdio();
-		this._watchExit();
 		await this._waitForReady();
 	}
 
@@ -1000,6 +1075,44 @@ export class FactorioServer extends events.EventEmitter {
 		return response;
 	}
 
+	/**
+	 * Timeout in ms to wait after a shutdown is requested before killing the
+	 * process. Defaults to 0 meaning no timeout
+	 */
+	get shutdownTimeoutMs() {
+		return this._shutdownTimeoutMs;
+	}
+
+	set shutdownTimeoutMs(newTimeoutMs: number) {
+		this._shutdownTimeoutMs = newTimeoutMs;
+		if (this._state === "stopping") {
+			this._clearStopTimeout();
+			this._setStopTimeout();
+		}
+	}
+
+	_setStopTimeout() {
+		if (this._shutdownTimeoutMs !== 0) {
+			this._stopTimeoutId = setTimeout(
+				this.onStopTimeout.bind(this),
+				this.shutdownTimeoutMs
+			);
+		}
+	}
+
+	_clearStopTimeout() {
+		clearTimeout(this._stopTimeoutId);
+		this._stopTimeoutId = undefined;
+	}
+
+	onStopTimeout() {
+		this._logger.error("Factorio appears to have hanged, killing process");
+		if (this._rconClient) {
+			this._rconClient.end().catch(() => {});
+		}
+		this._server!.kill("SIGKILL");
+	}
+
 	/**
 	 * Stop the server
 	 *
@@ -1008,88 +1121,54 @@ export class FactorioServer extends events.EventEmitter {
 	 */
 	async stop() {
 		this._check(["running"]);
-
-		let hanged = true;
-		let stopped = false;
-		function setAlive() { hanged = false; }
-
-		let timeoutId = setTimeout(() => {
-			if (hanged) {
-				this._logger.error("Factorio appears to have hanged, sending SIGKILL");
-				if (this._rconClient) {
-					this._rconClient.end().catch(() => {});
-				}
-				this._server!.kill("SIGKILL");
-			}
-		}, this.hangTimeoutMs);
-
-		// It's possible the server decides to exit on its own, in which
-		// case the stop operation has to be cancelled.
-		this._server!.once("exit", () => {
-			stopped = true;
-			clearTimeout(timeoutId);
-		});
-
 		this._state = "stopping";
-		// On Windows we need to save the map as there's no graceful shutdown.
-		if (this._rconClient && process.platform === "win32") {
-			// Not using events.once here as that will reject on error.
-			let saved = new Promise(resolve => this.once("_saved", resolve));
-			this.on("_saving", setAlive);
 
-			try {
-				await this.sendRcon("/server-save");
-				await saved;
+		// It's possible the server doesn't stop if it's stuck in an infinite
+		// loop or overloaded, kill it if the shutdown takes too long.
+		this._setStopTimeout();
 
-			} catch (err) {
-				// Ignore
-			}
-
-			clearTimeout(timeoutId);
-			this.off("_saving", setAlive);
-		}
-
-		if (this._rconClient) {
-
-			// If RCON is not yet fully connected that operation needs to
-			// complete before the RCON connection can be closed cleanly.
-			if (!this._rconReady) {
-				await events.once(this, "rcon-ready");
-			}
-
-			await this._rconClient.end();
+		// If RCON is not yet fully connected that operation needs to
+		// complete before the RCON connection can be used
+		if (!this._rconReady) {
+			// Not using events.once here to avoid throwing on error events.
+			await new Promise(resolve => this.once("rcon-ready", resolve));
 		}
 
 		// The Factorio server may have decided to get ahead of us and
 		// stop by itself before we had the chance to signal it.
-		if (stopped) {
+		if (this._state !== "stopping") {
 			return;
 		}
 
-		// On linux this sends SIGTERM to the process.  On windows this
-		// will terminate the process with no cleanup.  You can send your
-		// complaint for the lack of graceful shutdown to the Factorio
-		// developers.  Rcon does not recognize /quit, stdin is not
-		// recognized, and there's no "send CTRL+C to process" on Windows.
-		this._server!.kill();
+		const waitForExit = events.once(this._server!, "exit");
+		waitForExit.catch(() => {}); // Prevent unhandled promise rejection
 
-		// There appears to be an race condition where sending SIGTERM
-		// immediatly before the RCON interface comes online causes the
-		// Factorio server to hang.  It's also possible the server doesn't
-		// stop if it's stuck with an infinite lua code loop.  In either
-		// case there's no recovering from it.
-		if (process.platform !== "win32") {
-			this.on("_quitting", setAlive);
-			await events.once(this._server!, "exit");
-
-			clearTimeout(timeoutId);
-			this.off("_quitting", setAlive);
-
-		// On windows the process is terminated immediately, but to keep
-		// ordering wait until after the exit event here.
+		// Stop the server
+		if (this._rconClient) {
+			// Use RCON by default to stop the the server as it's available on
+			// all platforms and simplifies the code.
+			try {
+				await this.sendRcon("/quit");
+			} catch (err: any) {
+				// Ignore RCON connection being closed while sending the quit command.
+				if (err.message !== "Connection closed") {
+					throw err;
+				}
+			}
 		} else {
-			await events.once(this._server!, "exit");
+			// No rcon connection is available, fallback to signal.  On linux
+			// this sends SIGTERM to the process.  On windows this
+			// will terminate the process without saving.  You can send your
+			// complaint for the lack of graceful shutdown options to the
+			// Factorio developers.  I haven't found any way to make stdin work
+			// and there's no "send CTRL+C to process" API on Windows.
+			if (process.platform === "win32") {
+				this._logger.warn("No RCON connection, falling back to killing the server.");
+			}
+			this._server!.kill();
 		}
+
+		await waitForExit;
 	}
 
 	/**
@@ -1155,11 +1234,15 @@ export class FactorioServer extends events.EventEmitter {
 	/**
 	 * Get Factorio binary path
 	 *
-	 * Get the path to the factorio binary depending on the platform (MacOS support)
+	 * Get the path to the factorio binary depending on the configuration
+	 * and or the platform (MacOS support)
 	 *
 	 * @returns Path to factorio binary
 	 */
 	binaryPath() {
+		if (this.executablePath) {
+			return this.dataPath("..", this.executablePath);
+		}
 		if (process.platform === "darwin") {
 			return this.dataPath("..", "MacOS", "factorio");
 		}
@@ -1190,24 +1273,29 @@ export class FactorioServer extends events.EventEmitter {
 		return JSON.parse(await fs.readFile(this.dataPath("server-settings.example.json"), "utf-8"));
 	}
 
-	async _writeConfigIni() {
+	_writeConfigIniSync() {
 		let content = lib.stringify({
 			path: {
 				"read-data": this.dataPath(),
 				"write-data": this.writePath(),
 			},
 		});
-		await lib.safeOutputFile(this.writePath("config.ini"), content);
+		// Must be sync to allow process spawn without awaiting
+		// eslint-disable-next-line node/no-sync
+		fs.writeFileSync(this.writePath("config.ini"), content);
 	}
 
-	async _writeMapSettings(mapGenSettings?: object, mapSettings?: object) {
+	_writeMapSettingsSync(mapGenSettings?: object, mapSettings?: object) {
+		// Must be sync to allow process spawn without awaiting
 		if (mapGenSettings) {
-			await lib.safeOutputFile(
+			// eslint-disable-next-line node/no-sync
+			fs.writeFileSync(
 				this.writePath("map-gen-settings.json"), JSON.stringify(mapGenSettings, null, "\t")
 			);
 		}
 		if (mapSettings) {
-			await lib.safeOutputFile(
+			// eslint-disable-next-line node/no-sync
+			fs.writeFileSync(
 				this.writePath("map-settings.json"), JSON.stringify(mapSettings, null, "\t")
 			);
 		}

packages/host/tsconfig.node.json

@@ -4,7 +4,4 @@
 		{ "path": "../lib/tsconfig.node.json" },
 	],
 	"include": ["index.ts", "host.ts", "package.json", "src/**/*.ts", "src/**/*.js"],
-	"compilerOptions": {
-		"outDir": "dist/node",
-	},
 }

packages/lib/index.ts

@@ -28,6 +28,8 @@ export * from "./src/stream";
 export * from "./src/system_collectors";
 export * from "./src/zip_ops";
 export * from "./src/subscriptions";
+export * from "./src/datastore";
+export * from "./src/load_plugin_list";
 
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@clusterio/lib",
 	"description": "Shared library for Clusterio",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"repository": "https://github.com/clusterio/clusterio",
 	"license": "MIT",
 	"scripts": {
@@ -11,6 +11,7 @@
 	"main": "dist/node/index.js",
 	"browser": "browser.ts",
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {
@@ -26,8 +27,8 @@
 		"klaw": "^4.1.0",
 		"set-blocking": "^2.0.0",
 		"triple-beam": "^1.4.1",
-		"winston": "^3.10.0",
-		"winston-transport": "^4.5.0",
+		"winston": "^3.17.0",
+		"winston-transport": "^4.9.0",
 		"ws": "^8.13.0",
 		"yargs": "^17.7.2"
 	},
@@ -41,6 +42,6 @@
 		"@types/triple-beam": "^1.3.2",
 		"@types/ws": "^8.5.5",
 		"@types/yargs": "^17.0.24",
-		"typescript": "^5.1.6"
+		"typescript": "^5.5.3"
 	}
 }

packages/lib/src/config/classes.ts

@@ -7,7 +7,7 @@ import { basicType } from "../helpers";
 import * as libSchema from "../schema";
 import { StringEnum } from "../data/composites";
 
-const ConfigLocation = StringEnum(["controller", "host", "instance", "control"]);
+const ConfigLocation = StringEnum(["controller", "host", "control"]);
 export type ConfigLocation = Static<typeof ConfigLocation>;
 
 /**
@@ -77,6 +77,12 @@ export interface FieldDefinition {
 	 * Text used to describe the config field in user interfaces.
 	 */
 	description?: string;
+	/**
+	 * Value to use for the autocomplete attribute in the Web UI's input element.
+	 * Useful for making browsers behave properly with credential inputs.
+	 * Only used for string and number fields.
+	 */
+	autoComplete?: string;
 	/**
 	 * Web UI component to use for rending this input. Can be extended by
 	 * @{link "@clusterio/web_ui".BaseWebPlugin.inputComponents}
@@ -111,6 +117,22 @@ export interface FieldDefinition {
 	 * modified from.
 	 */
 	access?: ConfigLocation[];
+	/**
+	 * If present treat the config value as a credential.
+	 * Only the locations specified will be able to read the current value.
+	 * Access can be used to specify which locations are able to overwrite the value.
+	 */
+	credential?: ConfigLocation[];
+	/**
+	 * If present treat the config value as readonly.
+	 * Only the locations specified will be able to update the config value.
+	 * Access can be used to specify which locations are able to read the value.
+	 */
+	readonly?: ConfigLocation[];
+	/**
+	 * True if graphical interfaces should hide the config value
+	 */
+	hidden?: boolean;
 }
 
 export type ConfigDefs<Fields> = {
@@ -131,6 +153,12 @@ export type ConfigEvents = {
 	fieldChanged: (name: string, curr: FieldValue, prev: FieldValue) => void;
 }
 
+export enum ConfigAccess {
+	read = 0x1,
+	write = 0x2,
+	readWrite = 0x3,
+}
+
 /**
  * Collection of config entries
  *
@@ -159,6 +187,16 @@ export class Config<
 	declare static fieldDefinitions: ConfigDefs<any>;
 	declare ["constructor"]: typeof Config;
 
+	/**
+	 * Handle migration between clusterio versions
+	 *
+	 * @param config - Input config read and validated from file
+	 * @returns Output config to be loaded into the class
+	 * @protected
+	 */
+	static migrations(config: Static<typeof this.jsonSchema>) {
+		return config;
+	}
 
 	fields: Fields;
 	_unknownFields: Record<string, FieldValue> = {};
@@ -188,14 +226,14 @@ export class Config<
 			Object.entries(
 				this.constructor.fieldDefinitions
 			).filter(
-				([name, def]) => this._checkAccess(name, def, location, false)
+				([name, def]) => this._checkAccess(name, def, location, ConfigAccess.read, false)
 			).map(
 				([name, def]) => [name, defaultValue(def)]
 			)
 		) as Fields;
 
 		if (fields) {
-			this.update(fields, false, location);
+			this.update(this.constructor.migrations(fields), false, location);
 		}
 	}
 
@@ -204,21 +242,31 @@ export class Config<
 	 *
 	 * @param name - Name of the field to check.
 	 * @param def - definition for for the field to check.
-	 * @param remote - location of remote access to check for.
+	 * @param location - location to check access for.
+	 * @param mode - type of access to check for.
 	 * @param error - throw on failure to pass check.
 	 * @throws {InvalidAccess}
 	 *     if access is not granted and error is true.
 	 * @returns true if access is granted
 	 * @private
 	 */
-	_checkAccess(name: string, def: FieldDefinition, remote: ConfigLocation, error: boolean) {
-		for (let loc of [remote, this.location]) {
-			if (!(def.access ?? this.defaultAccess).includes(loc)) {
-				if (error) {
-					throw new InvalidAccess(`Field '${name}' is not accessible from ${loc}`);
-				} else {
-					return false;
-				}
+	_checkAccess(name: string, def: FieldDefinition, location: ConfigLocation, mode: ConfigAccess, error: boolean) {
+		// If credential array is present it acts as read permission and the access array as write permission,
+		// If readonly array is present it acts as write permission and the access array as the read permission.
+		// If neither is present access array acts as a combined read/write permission check.
+		if (
+			def.credential && mode & ConfigAccess.read && !(def.credential).includes(location)
+			|| def.readonly && mode & ConfigAccess.write && !(def.readonly).includes(location)
+			|| (
+				(!def.credential || mode & ConfigAccess.write)
+				&& (!def.readonly || mode & ConfigAccess.read)
+				&& !(def.access ?? this.defaultAccess).includes(location)
+			)
+		) {
+			if (error) {
+				throw new InvalidAccess(`Field '${name}' is not accessible from ${location}`);
+			} else {
+				return false;
 			}
 		}
 		return true;
@@ -263,14 +311,17 @@ export class Config<
 	/**
 	 * Serialize the config to a plain JavaScript object
 	 *
-	 * @param location - Location used for access control.
-	 * @returns JSON serializable representation of the group.
+	 * @param remote - Location this serialised representation is for
+	 * @returns JSON serializable representation of the config.
 	 */
-	toRemote(location: ConfigLocation): Static<typeof Config.jsonSchema> {
+	toRemote(remote: ConfigLocation): Static<typeof Config.jsonSchema> {
 		let fields: Record<string, FieldValue> = {};
 		for (let [name, value] of Object.entries(this.fields)) {
 			let def = this.constructor.fieldDefinitions[name];
-			if (!this._checkAccess(name, def, location, false)) {
+			if (
+				this.location !== "control" && !this._checkAccess(name, def, this.location, ConfigAccess.write, false)
+				|| this.location !== remote && !this._checkAccess(name, def, remote, ConfigAccess.read, false)
+			) {
 				continue;
 			}
 
@@ -293,12 +344,12 @@ export class Config<
 	 * @param json -
 	 *     Result from a previous call to .toRemote().
 	 * @param notify - Invoke fieldChanged events if true.
-	 * @param location - Location used for access control.
+	 * @param remote - Location this serialised representation was received from.
 	 */
 	update(
 		json: Static<typeof Config.jsonSchema>,
 		notify: boolean,
-		location = this.location
+		remote = this.location
 	) {
 		const valid = Config.validate(json);
 		if (!valid) {
@@ -316,7 +367,9 @@ export class Config<
 				continue;
 			}
 
-			if (!this._checkAccess(name, def, location, false)) {
+			if (
+				this.location !== "control" && !this._checkAccess(name, def, remote, ConfigAccess.write, false)
+			) {
 				continue;
 			}
 
@@ -346,30 +399,45 @@ export class Config<
 	 * Check if field can be accessed from the given location
 	 *
 	 * @param name - Name of field to check.
+	 * @param mode - Mode to access the field as.
 	 * @param location - Location used for access control.
 	 * @returns true if field is accessible
 	 */
-	canAccess(name: string, location = this.location) {
+	canAccess(name: string, mode: ConfigAccess, location = this.location) {
+		if (typeof mode !== "number") {
+			throw new TypeError("mode argument is required to canAccess");
+		}
 		let def = this.constructor.fieldDefinitions[name];
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
-		return this._checkAccess(name, def, location, false);
+		return this._checkAccess(name, def, location, mode, false);
 	}
 
 	/**
 	 * Get value for field
 	 *
 	 * @param name - Name of field to get.
-	 * @param location - Location used for access control.
+	 * @param remote - Location this value is being read from.
 	 * @returns Value stored for the field.
 	 */
-	get<Field extends keyof Fields & string>(name: Field, location = this.location) {
+	get<Field extends keyof Fields & string>(name: Field, remote = this.location) {
 		let def = this.constructor.fieldDefinitions[name];
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
-		this._checkAccess(name, def, location, true);
+		// If remote access is requested require it to be able to read the field
+		if (remote !== this.location) {
+			this._checkAccess(name, def, remote, ConfigAccess.read, true);
+		}
+		// Require that the field is either read or writable locally so that you can get the value
+		// back of write only fields recently written to.
+		if (
+			!this._checkAccess(name, def, this.location, ConfigAccess.write, false)
+			&& !this._checkAccess(name, def, this.location, ConfigAccess.read, false)
+		) {
+			throw new InvalidAccess(`Field '${name}' is not accessible from ${this.location}`);
+		}
 
 		return this.fields[name];
 	}
@@ -377,22 +445,25 @@ export class Config<
 	/**
 	 * Set value of field
 	 *
-	 * The field must be defined for the config group and the value is
+	 * The field must be defined for the config and the value is
 	 * type checked against the field definition.
 	 *
 	 * @param name - Name of field to set.
 	 * @param newValue - Value to set for field.
-	 * @param location - Location used for access control.
+	 * @param remote - Location this field is being set from.
 	 * @throws {InvalidField} if field is not defined.
 	 * @throws {InvalidValue} if value is not allowed for the field.
 	 */
-	set<Field extends keyof Fields & string>(name: Field, newValue: Fields[Field], location = this.location) {
+	set<Field extends keyof Fields & string>(name: Field, newValue: Fields[Field], remote = this.location) {
 		let value: FieldValue = newValue;
 		let def = this.constructor.fieldDefinitions[name];
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
-		this._checkAccess(name, def, location, true);
+		this._checkAccess(name, def, remote, ConfigAccess.write, true);
+		if (remote !== this.location) {
+			this._checkAccess(name, def, this.location, ConfigAccess.write, true);
+		}
 
 		// Empty strings are treates as null
 		if (value === "") {
@@ -447,12 +518,12 @@ export class Config<
 	 * Set property of object field
 	 *
 	 * Update value of stored object field by setting the specified property
-	 * on it.  The field must be defined as an object for the config group.
+	 * on it.  The field must be defined as an object for the config.
 	 *
 	 * @param name - Name of field to set property on.
 	 * @param prop - Name of property to set on field.
 	 * @param value - the value to set the property to.
-	 * @param location - Location used for access control.
+	 * @param remote - Location this property is being set from
 	 * @throws {InvalidField} if field is not defined.
 	 * @throws {InvalidValue} if field is not an object.
 	 */
@@ -460,13 +531,16 @@ export class Config<
 		name: Field,
 		prop: string,
 		value?: unknown,
-		location = this.location,
+		remote = this.location,
 	) {
 		let def = this.constructor.fieldDefinitions[name];
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
-		this._checkAccess(name, def, location, true);
+		this._checkAccess(name, def, remote, ConfigAccess.write, true);
+		if (remote !== this.location) {
+			this._checkAccess(name, def, this.location, ConfigAccess.write, true);
+		}
 
 		if (def.type !== "object") {
 			throw new InvalidValue(`Cannot set property on non-object field '${name}'`);

packages/lib/src/config/definitions.ts

@@ -1,7 +1,7 @@
 // Core definitions for the configuration system
 import * as classes from "./classes";
 import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin";
-
+import { Static } from "@sinclair/typebox";
 
 export interface ControllerConfigFields {
 	"controller.name": string;
@@ -11,7 +11,7 @@ export interface ControllerConfigFields {
 	"controller.https_port": number | null;
 	"controller.bind_address": string | null;
 	"controller.trusted_proxies": string | null;
-	"controller.external_address": string | null;
+	"controller.public_url": string | null;
 	"controller.tls_certificate": string | null;
 	"controller.tls_private_key": string | null;
 	"controller.auth_secret": string;
@@ -20,6 +20,9 @@ export interface ControllerConfigFields {
 	"controller.metrics_timeout": number;
 	"controller.system_metrics_interval": number;
 	"controller.proxy_stream_timeout": number;
+	"controller.factorio_username": string | null;
+	"controller.factorio_token": string | null;
+	"controller.share_factorio_credentials_with_hosts": boolean;
 	"controller.default_mod_pack_id": number | null;
 	"controller.default_role_id": number | null;
 	"controller.autosave_interval": number;
@@ -31,6 +34,15 @@ export interface ControllerConfigFields {
  */
 export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 	declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => ControllerConfig;
+	static migrations(config: Static<typeof this.jsonSchema>) {
+		if (config.hasOwnProperty("controller.external_address")) {
+			config["controller.public_url"] = config["controller.external_address"];
+			delete config["controller.external_address"];
+		}
+
+		return config;
+	}
+
 	static fieldDefinitions: classes.ConfigDefs<ControllerConfigFields> = {
 		"controller.name": {
 			title: "Name",
@@ -86,9 +98,9 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			type: "string",
 			optional: true,
 		},
-		"controller.external_address": {
-			title: "External Address",
-			description: "Public facing address the controller is hosted on.",
+		"controller.public_url": {
+			title: "Public URL",
+			description: "Public facing URL the controller is hosted on, including the protocol.",
 			type: "string",
 			optional: true,
 		},
@@ -147,6 +159,29 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			type: "number",
 			initialValue: 15,
 		},
+		"controller.factorio_username": {
+			title: "Factorio Username",
+			description: "Username to authenticate with Factorio API with.",
+			autoComplete: "section-factorio username",
+			type: "string",
+			optional: true,
+		},
+		"controller.factorio_token": {
+			title: "Factorio Token",
+			description: "Token to authenticate with Factorio API with.",
+			autoComplete: "section-factorio new-password",
+			type: "string",
+			credential: ["controller"],
+			optional: true,
+		},
+		"controller.share_factorio_credentials_with_hosts": {
+			title: "Share Factorio credentials with Hosts",
+			description:
+				"If enabled, the Factorio Username and Token will be shared with hosts and " +
+				"used in the server settings of instances in this cluster.",
+			type: "boolean",
+			initialValue: true,
+		},
 		"controller.default_mod_pack_id": {
 			title: "Default Mod Pack",
 			description: "Mod pack used by default for instances.",
@@ -157,6 +192,7 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 		"controller.default_role_id": {
 			title: "Default role",
 			description: "ID of role assigned by default to new users.",
+			inputComponent: "role",
 			type: "number",
 			optional: true,
 			initialValue: 1,
@@ -175,6 +211,8 @@ export interface HostConfigFields {
 	"host.tls_ca": string | null;
 	"host.public_address": string;
 	"host.factorio_port_range": string;
+	"host.factorio_username": string | null,
+	"host.factorio_token": string | null,
 	"host.max_reconnect_delay": number;
 }
 
@@ -194,6 +232,7 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			description: "ID of the host",
 			type: "number",
 			initialValue: () => Math.random() * 2**31 | 0,
+			hidden: true,
 		},
 		"host.factorio_directory": {
 			description: "Path to directory to look for factorio installs",
@@ -245,6 +284,25 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			type: "string",
 			initialValue: "34100-34199",
 		},
+		"host.factorio_username": {
+			title: "Factorio Username",
+			description:
+				"Username to authenticate with Factorio API with. If set this will be used in the server settings " +
+				"of all instances on this host.",
+			autoComplete: "section-factorio username",
+			type: "string",
+			optional: true,
+		},
+		"host.factorio_token": {
+			title: "Factorio Token",
+			description:
+				"Token to authenticate with Factorio API with. If set this will be used in the server settings " +
+				"of all instances on this host.",
+			autoComplete: "section-factorio new-password",
+			type: "string",
+			credential: ["host"],
+			optional: true,
+		},
 		"host.max_reconnect_delay": {
 			title: "Max Reconnect Delay",
 			description: "Maximum delay to wait before attempting to reconnect WebSocket",
@@ -261,6 +319,8 @@ export interface InstanceConfigFields {
 	"instance.auto_start": boolean;
 
 	"factorio.version": string;
+	"factorio.executable_path": string | null;
+	"factorio.shutdown_timeout": number;
 	"factorio.game_port": number | null;
 	"factorio.host_assigned_game_port": number | null;
 	"factorio.rcon_port": number | null;
@@ -268,14 +328,15 @@ export interface InstanceConfigFields {
 	"factorio.player_online_autosave_slots": number;
 	"factorio.mod_pack_id": number | null;
 	"factorio.enable_save_patching": boolean;
+	"factorio.enable_script_commands": boolean;
 	"factorio.enable_whitelist": boolean;
 	"factorio.enable_authserver_bans": boolean;
 	"factorio.settings": Record<string, unknown>;
 	"factorio.verbose_logging": boolean;
 	"factorio.strip_paths": boolean;
-	"factorio.sync_adminlist": boolean;
-	"factorio.sync_whitelist": boolean;
-	"factorio.sync_banlist": boolean;
+	"factorio.sync_adminlist": "enabled" | "disabled" | "bidirectional";
+	"factorio.sync_whitelist": "enabled" | "disabled" | "bidirectional";
+	"factorio.sync_banlist": "enabled" | "disabled" | "bidirectional";
 	"factorio.max_concurrent_commands": number;
 }
 
@@ -286,6 +347,20 @@ export interface InstanceConfigFields {
  */
 export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 	declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => InstanceConfig;
+	static migrations(config: Static<typeof this.jsonSchema>) {
+		function boolToEnableDisable(name: string) {
+			if (config.hasOwnProperty(name) && typeof config[name] === "boolean") {
+				config[name] = config[name] ? "enabled" : "disabled";
+			}
+		}
+
+		boolToEnableDisable("factorio.sync_adminlist");
+		boolToEnableDisable("factorio.sync_whitelist");
+		boolToEnableDisable("factorio.sync_banlist");
+
+		return config;
+	}
+
 	static fieldDefinitions: classes.ConfigDefs<InstanceConfigFields> = {
 		"instance.name": {
 			type: "string",
@@ -295,10 +370,12 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			description: "ID of the instance",
 			type: "number",
 			initialValue: () => Math.random() * 2**31 | 0,
+			hidden: true,
 		},
 		"instance.assigned_host": {
 			type: "number",
 			optional: true,
+			hidden: true,
 		},
 		"instance.auto_start": {
 			description: "Automatically start this instance when the host hosting it is started up",
@@ -311,6 +388,22 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			type: "string",
 			initialValue: "latest",
 		},
+		"factorio.executable_path": {
+			description:
+				"Relative path from the Factorio installation directory to the executable to run. " +
+				"Defaults to auto detect the path, only needed in special setups.",
+			restartRequired: true,
+			type: "string",
+			optional: true,
+		},
+		"factorio.shutdown_timeout": {
+			description:
+				"Timeout in seconds to wait after requesting the server to stop before killing " +
+				"the process. Set to 0 to disable.",
+			type: "number",
+			initialValue: 300,
+			optional: true,
+		},
 		"factorio.game_port": {
 			description: "UDP port to run game on, uses a port in host.factorio_port_range if null",
 			restartRequired: true,
@@ -329,7 +422,8 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			optional: true,
 		},
 		"factorio.rcon_password": {
-			description: "Password for RCON, randomly generated if null",
+			credential: ["host", "controller"],
+			description: "Password for RCON, randomly generated if null.",
 			restartRequired: true,
 			type: "string",
 			optional: true,
@@ -357,6 +451,14 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			type: "boolean",
 			initialValue: true,
 		},
+		"factorio.enable_script_commands": {
+			description:
+				"Allows achievement breaking commands to be executed over rcon. " +
+				"Required for Clusterio integrations and most plugins. " +
+				"This does not prevent players using script commands.",
+			type: "boolean",
+			initialValue: true,
+		},
 		"factorio.enable_whitelist": {
 			description: "Turn on whitelist for joining the server.",
 			type: "boolean",
@@ -393,19 +495,22 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			initialValue: true,
 		},
 		"factorio.sync_adminlist": {
-			description: "Synchronize adminlist with controller",
-			type: "boolean",
-			initialValue: true,
+			description: "Synchronize adminlist with the controller",
+			type: "string",
+			enum: ["disabled", "enabled", "bidirectional"],
+			initialValue: "bidirectional",
 		},
 		"factorio.sync_whitelist": {
-			description: "Synchronize whitelist with controller",
-			type: "boolean",
-			initialValue: true,
+			description: "Synchronize whitelist with the controller",
+			type: "string",
+			enum: ["disabled", "enabled"], // TODO: Implement bidirectional
+			initialValue: "enabled",
 		},
 		"factorio.sync_banlist": {
-			description: "Synchronize banlist with controller",
-			type: "boolean",
-			initialValue: true,
+			description: "Synchronize banlist with the controller",
+			type: "string",
+			enum: ["disabled", "enabled", "bidirectional"],
+			initialValue: "bidirectional",
 		},
 		"factorio.max_concurrent_commands": {
 			description: "Maximum number of RCON commands trasmitted in parallel",

packages/lib/src/data/ModPack.ts

@@ -173,8 +173,11 @@ export default class ModPack {
 		if (json.updated_at_ms) { modPack.updatedAtMs = json.updated_at_ms; }
 		if (json.is_deleted) { modPack.isDeleted = json.is_deleted; }
 
-		if (!modPack.mods.has("base")) {
-			modPack.mods.set("base", { name: "base", enabled: true, version: modPack.factorioVersion });
+		const builtinMods = ModPack.getBuiltinMods(modPack.factorioVersion);
+		for (const builtinMod of builtinMods) {
+			if (!modPack.mods.has(builtinMod.name)) {
+				modPack.mods.set(builtinMod.name, builtinMod);
+			}
 		}
 
 		return modPack;
@@ -374,12 +377,37 @@ export default class ModPack {
 				value = typeof value === "object" ? prototype.default_value : String(value ?? "");
 			}
 			if (type === "color-setting") {
-				if (typeof value !== "object") {
-					value = prototype.default_value;
-				}
 				value = libLuaTools.normalizeColor(value as object);
 			}
 			this.settings[settingType].set(prototype.name, { value });
 		}
 	}
+
+	/**
+	 * Returns an array of ModInfo containing all mods included in factorio
+	 *
+	 * @param factorioVersion - Factorio version to get the default mods for
+	 * @return Built in mods for the given version
+	 */
+	static getBuiltinMods(factorioVersion: string) {
+		let defaultMods: ModRecord[] = [
+			// "core" not included because core cannot be disabled
+			{ name: "base", enabled: true, version: factorioVersion },
+		];
+
+		const integerVersion = integerFactorioVersion(factorioVersion);
+		if (integerVersion >= integerFactorioVersion("1.2")) {
+			defaultMods = defaultMods.concat([
+				{ name: "elevated-rails", enabled: false, version: factorioVersion },
+				{ name: "quality", enabled: false, version: factorioVersion },
+				{ name: "space-age", enabled: false, version: factorioVersion },
+			]);
+		}
+
+		return defaultMods;
+	}
+
+	static getBuiltinModNames(factorioVersion: string) {
+		return this.getBuiltinMods(factorioVersion).map(builtinMod => builtinMod.name);
+	}
 }

packages/lib/src/data/PlayerStats.ts

@@ -92,4 +92,19 @@ export default class PlayerStats {
 		}
 		return json;
 	}
+
+	merge(other: PlayerStats) {
+		this.joinCount += other.joinCount;
+		this.onlineTimeMs += other.onlineTimeMs;
+		if (!this.firstJoinAt || (other.firstJoinAt && other.firstJoinAt < this.firstJoinAt)) {
+			this.firstJoinAt = other.firstJoinAt;
+		}
+		if (!this.lastJoinAt || (other.lastJoinAt && other.lastJoinAt > this.lastJoinAt)) {
+			this.lastJoinAt = other.lastJoinAt;
+		}
+		if (!this.lastLeaveAt || (other.lastLeaveAt && other.lastLeaveAt > this.lastLeaveAt)) {
+			this.lastLeaveAt = other.lastLeaveAt;
+			this.lastLeaveReason = other.lastLeaveReason;
+		}
+	}
 }

packages/lib/src/data/User.ts

@@ -150,9 +150,39 @@ export default class User {
 		return playerStats;
 	}
 
-	/** Name of the user, used by event subscriptions */
+	/**
+	 * Merge another user into this one, must have matching ids, user is not deleted
+	 * @param otherUser - User who's details are merged from
+	*/
+	merge(otherUser: User) {
+		if (this.id !== otherUser.id) {
+			throw new Error("Cannot merge users with different ids");
+		}
+
+		// Merge properties
+		this.roleIds = new Set([...this.roleIds, ...otherUser.roleIds]);
+		this.instances = new Set([...this.instances, ...otherUser.instances]);
+		this.isAdmin = this.isAdmin && otherUser.isAdmin; // More secure to use && rather ||
+		this.isBanned = this.isBanned || otherUser.isBanned;
+		this.isBanned = this.isWhitelisted || otherUser.isWhitelisted;
+		this.banReason = this.updatedAtMs > otherUser.updatedAtMs ? this.banReason : otherUser.banReason;
+		this.updatedAtMs = Date.now();
+
+		// Merge instance stats
+		const thisInstanceStats = this.instanceStats;
+		for (const [instanceId, instanceStats] of otherUser.instanceStats.entries()) {
+			if (thisInstanceStats.has(instanceId)) {
+				thisInstanceStats.get(instanceId)!.merge(instanceStats);
+			} else {
+				thisInstanceStats.set(instanceId, instanceStats);
+			}
+		}
+		this.recalculatePlayerStats();
+	}
+
+	/** Name of the user, used by event subscriptions and to ensure case insensitivity */
 	get id() {
-		return this.name;
+		return this.name.toLowerCase();
 	}
 }
 

packages/lib/src/data/messages_controller.ts

@@ -122,6 +122,18 @@ export class HostConfigCreateRequest {
 	static Response = plainJson(HostConfig.jsonSchema);
 }
 
+export class GetFactorioCredentialsRequest {
+	declare ["constructor"]: typeof GetFactorioCredentialsRequest;
+	static type = "request" as const;
+	static src = ["host", "instance"] as const;
+	static dst = "controller" as const;
+
+	static Response = plainJson(Type.Object({
+		"username": Type.Optional(Type.String()),
+		"token": Type.Optional(Type.String()),
+	}));
+}
+
 export class LogSetSubscriptionsRequest {
 	declare ["constructor"]: typeof LogSetSubscriptionsRequest;
 	static type = "request" as const;

packages/lib/src/data/messages_instance.ts

@@ -42,6 +42,7 @@ export class InstanceDetails {
 		public assignedHost: number | undefined,
 		public gamePort: number | undefined,
 		public status: InstanceStatus,
+		public factorioVersion: string | undefined,
 		/** Millisecond Unix timestamp this entry was last updated at */
 		public updatedAtMs = 0,
 	) { }
@@ -55,6 +56,7 @@ export class InstanceDetails {
 			"unknown", "unassigned", "stopped", "starting", "running", "stopping",
 			"creating_save", "exporting_data", "deleted",
 		]),
+		"factorioVersion": Type.Optional(Type.String()),
 		"updatedAtMs": Type.Optional(Type.Number()),
 	});
 
@@ -65,6 +67,7 @@ export class InstanceDetails {
 			json.assignedHost,
 			json.gamePort,
 			json.status,
+			json.factorioVersion,
 			json.updatedAtMs,
 		);
 	}
@@ -717,6 +720,7 @@ export class HostInstanceUpdate {
 		public config: Static<typeof InstanceConfig.jsonSchema>,
 		public status: InstanceStatus,
 		public gamePort: undefined | number,
+		public factorioVersion: undefined | string,
 	) { }
 
 	static jsonSchema = Type.Object({
@@ -725,10 +729,11 @@ export class HostInstanceUpdate {
 			"stopped", "starting", "running", "stopping", "creating_save", "exporting_data",
 		]),
 		"gamePort": Type.Optional(Type.Number()),
+		"factorioVersion": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.config, json.status, json.gamePort);
+		return new this(json.config, json.status, json.gamePort, json.factorioVersion);
 	}
 }
 
@@ -822,6 +827,7 @@ export class InstanceStatusChangedEvent {
 		public instanceId: number,
 		public status: InstanceStatus,
 		public gamePort?: number,
+		public factorioVersion?: string,
 	) { }
 
 	static jsonSchema = Type.Object({
@@ -830,10 +836,11 @@ export class InstanceStatusChangedEvent {
 			"stopped", "starting", "running", "stopping", "creating_save", "exporting_data",
 		]),
 		"gamePort": Type.Optional(Type.Integer()),
+		"factorioVersion": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.instanceId, json.status, json.gamePort);
+		return new this(json.instanceId, json.status, json.gamePort, json.factorioVersion);
 	}
 }
 
@@ -857,7 +864,7 @@ export class InstanceDetailsChangedEvent {
 export class InstanceBanlistUpdateEvent {
 	declare ["constructor"]: typeof InstanceBanlistUpdateEvent;
 	static type = "event" as const;
-	static src = "controller" as const;
+	static src = ["controller", "host", "instance"] as const;
 	static dst = "instance" as const;
 
 	constructor(
@@ -880,7 +887,7 @@ export class InstanceBanlistUpdateEvent {
 export class InstanceAdminlistUpdateEvent {
 	declare ["constructor"]: typeof InstanceAdminlistUpdateEvent;
 	static type = "event" as const;
-	static src = "controller" as const;
+	static src = ["controller", "host", "instance"] as const;
 	static dst = "instance" as const;
 
 	constructor(
@@ -901,7 +908,7 @@ export class InstanceAdminlistUpdateEvent {
 export class InstanceWhitelistUpdateEvent {
 	declare ["constructor"]: typeof InstanceWhitelistUpdateEvent;
 	static type = "event" as const;
-	static src = "controller" as const;
+	static src = ["controller", "host", "instance"] as const;
 	static dst = "instance" as const;
 
 	constructor(

packages/lib/src/datastore.ts

@@ -0,0 +1,285 @@
+import fs from "fs-extra";
+
+import { safeOutputFile } from "./file_ops";
+import { SubscribableValue } from "./subscriptions";
+import { logger } from "./logging";
+import { EventEmitter } from "stream";
+import { ControllerConfig } from "./config";
+import path from "path";
+
+type DatastoreKey = string | number;
+type DatastoreValue = string | number | boolean | object;
+
+// Abstract class which can provide saving and loading capabilities to a datastore
+export abstract class DatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> {
+	abstract save(data: Map<K, V>): Promise<void>
+	abstract load(): Promise<Map<K, V>>
+
+	// Load data from the source and return it, this should be passed to the constructor of a datastore
+	async bootstrap() {
+		return [this, await this.load()] as const;
+	}
+}
+
+// Store all data in memory, used as a default and should be avoided
+export class MemoryDatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> extends DatastoreProvider<K, V> {
+	constructor(
+		private value: Map<K, V> = new Map()
+	) {
+		super();
+	}
+
+	async save(data: Map<K, V>) {
+		this.value = new Map(data);
+	}
+
+	async load() {
+		return new Map(this.value);
+	}
+}
+
+// Store all data within a json object file and validate the result against a schema
+export class JsonDatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+	J, // Intermediate type returned from migrations
+> extends DatastoreProvider<K, V> {
+	constructor(
+		private filePath: string,
+		private fromJson: (json: J) => V = v => v as any,
+		private migrations: (rawJson: Record<DatastoreKey, unknown>) => Record<DatastoreKey, J> = v => v as any,
+		private finalise: (obj: V) => V = v => v,
+	) {
+		super();
+	}
+
+	// Save the data to the json file
+	async save(data: Map<K, V>) {
+		logger.verbose(`Saving ${this.filePath}`);
+		await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t"));
+	}
+
+	// Load the data from the json file
+	async load() {
+		// Read the raw json from the file
+		let rawJson;
+		try {
+			logger.verbose(`Loading ${this.filePath}`);
+			rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" }));
+		} catch (err: any) {
+			if (err.code !== "ENOENT") {
+				throw err;
+			}
+			return new Map();
+		}
+
+		// Apply migrations
+		const serialized = this.migrations(rawJson);
+
+		// Convert to data class objects
+		return new Map(Object.entries(serialized).map(([k, v]) => [k, this.finalise(this.fromJson(v))]));
+	}
+}
+
+// Store all data within a json array file where each element has a unique id
+export class JsonIdDatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue & { id: K },
+	J, // Intermediate type returned from migrations
+> extends DatastoreProvider<K, V> {
+	constructor(
+		private filePath: string,
+		private fromJson: (json: J) => V = v => v as any,
+		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as any,
+		private finalise: (obj: V) => V = v => v,
+	) {
+		super();
+	}
+
+	// Save the data to the json file
+	async save(data: Map<K, V>) {
+		logger.verbose(`Saving ${this.filePath}`);
+		await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t"));
+	}
+
+	// Load the data from the json file
+	async load() {
+		// Read the raw json from the file
+		let rawJson;
+		try {
+			logger.verbose(`Loading ${this.filePath}`);
+			rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" }));
+		} catch (err: any) {
+			if (err.code !== "ENOENT") {
+				throw err;
+			}
+			return new Map();
+		}
+
+		// Apply migrations
+		const serialized = this.migrations(rawJson);
+
+		// Convert to data class objects
+		return new Map(serialized.map((e) => {
+			const v = this.finalise(this.fromJson(e));
+			return [v.id, v];
+		}));
+	}
+}
+
+// Implements a similar interface to a map
+export abstract class Datastore<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> extends EventEmitter {
+	private dirty = false;
+
+	constructor(
+		private provider: DatastoreProvider<K, V> = new MemoryDatastoreProvider(),
+		protected data = new Map<K, V>(),
+	) {
+		super();
+	}
+
+	// Get the file path based on the controller config
+	static getFilePath(config: ControllerConfig, file: string) {
+		return path.resolve(config.get("controller.database_directory"), file);
+	}
+
+	// Save the datastore to the provider
+	async save() {
+		if (this.dirty) {
+			await this.provider.save(this.data);
+			this.dirty = false;
+		}
+	}
+
+	// Load the datastore from the provider
+	async load() {
+		this.data = await this.provider.load();
+		this.dirty = false;
+	}
+
+	// Set the dirty flag and call update handlers
+	protected touch(updates: any[]) {
+		if (updates.length) {
+			this.dirty = true;
+			this.emit("update", updates);
+		}
+	}
+
+	// Returns true if the datastore has the value, false otherwise
+	has(key: K) {
+		return this.data.has(key);
+	}
+
+	// Get a value from from the datastore
+	get(key: K) {
+		return this.data.get(key) as Readonly<V> ?? undefined;
+	}
+
+	// Get a mutable reference to a value in the datastore, call set after use
+	getMutable(key: K) {
+		return this.data.get(key);
+	}
+
+	// Allow iterating through the datastore like a Map.
+	[Symbol.iterator](): IterableIterator<[K, Readonly<V>]> {
+		return this.data.entries();
+	}
+
+	// Returns all values in the datastore
+	values() {
+		return this.data.values() as IterableIterator<Readonly<V>>;
+	}
+
+	// Returns all values in the datastore as mutable references, call setMany after use
+	valuesMutable() {
+		return this.data.values();
+	}
+}
+
+// General key value mapping which can be saved and loaded with the file system
+export class KeyValueDatastore<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> extends Datastore<K, V> {
+	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
+	set(key: K, value: V) {
+		this.data.set(key, value);
+		this.touch([[key, value]]);
+	}
+
+	// Set many values at once from an array of key value pairs
+	setMany(pairs: [K, V][]) {
+		for (const [k, v] of pairs) {
+			this.data.set(k, v);
+		}
+		this.touch(pairs);
+	}
+
+	// Delete a value in the datastore
+	delete(key: K) {
+		const value = this.data.get(key);
+		this.data.delete(key);
+		this.touch([[key, value!, true]]);
+	}
+
+	// Delete many values at once from an array of keys
+	deleteMany(keys: K[]) {
+		const updates = [] as [K, V, true][];
+		for (const key of keys) {
+			const value = this.data.get(key);
+			updates.push([key, value!, true]);
+			this.data.delete(key);
+		}
+		this.touch(updates);
+	}
+}
+
+// A special implementation of the datastore to work directly with subscribable values
+export class SubscribableDatastore<
+	V extends SubscribableValue,
+> extends Datastore<V["id"], V> {
+	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
+	set(value: V) {
+		this.data.set(value.id, value);
+		value.updatedAtMs = Date.now();
+		this.touch([value]);
+	}
+
+	// Set many values at once from an array of key value pairs
+	setMany(values: V[]) {
+		const nowMs = Date.now();
+		for (const value of values) {
+			value.updatedAtMs = nowMs;
+			this.data.set(value.id, value);
+		}
+		this.touch(values);
+	}
+
+	// Delete a value in the datastore
+	delete(value: V) {
+		this.data.delete(value.id);
+		value.updatedAtMs = Date.now();
+		value.isDeleted = true;
+		this.touch([value]);
+	}
+
+	// Delete many values at once from an array of values
+	deleteMany(values: V[]) {
+		const nowMs = Date.now();
+		for (const value of values) {
+			value.isDeleted = true;
+			value.updatedAtMs = nowMs;
+			this.data.delete(value.id);
+		}
+		this.touch(values);
+	}
+}

packages/lib/src/errors.ts

@@ -61,13 +61,34 @@ export class SessionLost extends Error {
 	code = "SessionLost";
 }
 
+/**
+ * Super class for all custom websocket errors
+ */
+export class WebSocketError extends Error {
+	code = "WebSocketError";
+}
+
 /**
  * Thrown when WebSocket authentication failed
  */
-export class AuthenticationFailed extends Error {
+export class AuthenticationFailed extends WebSocketError {
 	code = "AuthenticationFailed";
 }
 
+/**
+ * Thrown when WebSocket protocol is violated
+ */
+export class ProtocolError extends WebSocketError {
+	code = "ProtocolError";
+}
+
+/**
+ * Thrown when WebSocket policy is violated
+ */
+export class PolicyViolation extends WebSocketError {
+	code = "PolicyViolation";
+}
+
 /**
  * Errror class for known errors occuring during startup
  */

packages/lib/src/file_ops.ts

@@ -142,7 +142,7 @@ export async function safeOutputFile(file: string, data: string | Buffer, option
 	if (dir && !await fs.pathExists(dir)) {
 		await fs.mkdirs(dir);
 	}
-	let temporary = `${dir}${name}.tmp${ext}`;
+	let temporary = path.join(dir, `${name}.tmp${ext}`);
 	let fd = await fs.open(temporary, "w");
 	try {
 		await fs.writeFile(fd, data, options);

packages/lib/src/link/connectors.ts

@@ -9,6 +9,30 @@ import { logger } from "../logging";
 import ExponentialBackoff from "../ExponentialBackoff";
 import type { Request, Event } from "./link";
 
+/**
+ * Numbered codes describing why a connection was closed
+ * See: https://github.com/Luka967/websocket-close-codes
+ * See: https://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number
+ * See: https://www.rfc-editor.org/rfc/rfc6455.html#section-7.4
+ */
+export enum ConnectionClosed {
+	// Codes between 1000 and 1015 are pre-assigned by IANA
+	// A browser client can only (on demand) send 1000 from this range
+	NormalClosure = 1000, // Purpose fulfilled
+	GoingAway = 1001, // Server quit or browser navigation
+	ProtocolError = 1002, // Protocol not followed
+	PolicyViolation = 1008, // Generic code for any endpoint policy
+	InternalError = 1011, // Endpoint failed to fulfil request
+
+	// Codes after 3000 are available for frameworks
+	// However they should be registered to IANA
+	Unauthorized = 3000,
+	Forbidden = 3003,
+	Timeout = 3008,
+
+	// Codes after 4000 are available for applications
+	MalformedMessage = 4000,
+};
 
 /**
  * Base connector for links
@@ -34,6 +58,7 @@ export abstract class BaseConnector extends events.EventEmitter {
 	}
 
 	protected abstract send(message: libData.MessageRoutable): void;
+	abstract get valid(): boolean;
 
 	forward(message: libData.MessageRoutable) {
 		const seq = this._seq;
@@ -177,13 +202,13 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 		try {
 			json = JSON.parse(text);
 		} catch (err) {
-			this.close(4000, "Malformed JSON");
+			this.close(ConnectionClosed.MalformedMessage, "Malformed JSON");
 			return undefined;
 		}
 		if (!libData.Message.validate(json)) {
 			logger.error(`Received malformed message: ${text}`);
 			// logger.error(JSON.stringify(libData.Message.validate.errors, null, "\t"));
-			this.close(4000, "Malformed message");
+			this.close(ConnectionClosed.MalformedMessage, "Malformed message");
 			return undefined;
 		}
 		return libData.Message.fromJSON(json);
@@ -219,7 +244,7 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 			logger.verbose("Connector | closing after heartbeat timed out");
 			// eslint-disable-next-line node/no-process-env
 			if (process.env.APP_ENV === "browser") {
-				this._socket!.close(4008, "Heartbeat timeout");
+				this._socket!.close(ConnectionClosed.Timeout, "Heartbeat timeout");
 			} else {
 				this._socket!.terminate();
 			}
@@ -293,10 +318,12 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 	 * Gracefully disconnect
 	 *
 	 * Sets closing flag an initiates the disconnect sequence.
+	 * @param [code=1000] WebSocket close code.
+	 * @param [reason="Disconnect"] WebSocket close reason.
 	 */
-	async disconnect() {
+	async disconnect(code: number = ConnectionClosed.NormalClosure, reason: string = "Disconnect") {
 		if (this._state !== "connected") {
-			await this.close(1000, "Disconnect");
+			await this.close(code, reason);
 			return;
 		}
 
@@ -315,7 +342,7 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 			clearTimeout(timeout);
 		}
 
-		await this.close(1000, "Disconnect");
+		await this.close(code, reason);
 	}
 
 	/**
@@ -346,6 +373,14 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 	get hasSession() {
 		return ["connected", "resuming"].includes(this._state);
 	}
+
+	/**
+	 * True if the connector is valid and can accept new messages,
+	 * only needs to be checked on fringe cases during setup and tear down.
+	 */
+	get valid() {
+		return this.hasSession;
+	}
 }
 
 /**
@@ -449,7 +484,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		}
 
 		if (this._socket) {
-			this._socket!.close(1000, "Connector closing");
+			this._socket!.close(ConnectionClosed.NormalClosure, "Connector closing");
 
 		} else {
 			this._reset();
@@ -531,10 +566,23 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		this._socket!.onclose = (event) => {
 			const previousState = this._state;
 
-			// Authentication failed
-			if (event.code === 4003) {
-				this.emit("error", new libErrors.AuthenticationFailed(event.reason));
-				this._closing = true;
+			switch (event.code) {
+				case ConnectionClosed.Unauthorized:
+					// Authentication failed
+					this.emit("error", new libErrors.AuthenticationFailed(event.reason));
+					this._closing = true;
+					break;
+				case ConnectionClosed.ProtocolError:
+					// Connection was closed because invalid data was sent
+					this.emit("error", new libErrors.ProtocolError(event.reason));
+					this._closing = true;
+					break;
+				case ConnectionClosed.PolicyViolation:
+					// Connection was closed to prevent the server entering an invalid state
+					this.emit("error", new libErrors.PolicyViolation(event.reason));
+					this._closing = true;
+					break;
+				default:
 			}
 
 			this._socket = null;
@@ -564,7 +612,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 
 			// Log must be sent after state change is complete
 			let message = `Connector | Close (code: ${event.code}, reason: ${event.reason})`;
-			if (previousState === "connected" && event.code !== 1000) {
+			if (previousState === "connected" && event.code !== ConnectionClosed.NormalClosure) {
 				logger.info(message);
 			} else {
 				logger.verbose(message);
@@ -723,4 +771,14 @@ export class VirtualConnector extends BaseConnector {
 	send(message: libData.Message) {
 		this.other.emit("message", message);
 	}
+
+	/**
+	 * True if the connector is valid and can accept new messages,
+	 * only needs to be checked on fringe cases during setup and tear down.
+	 *
+	 * For a virtual connector this is always true.
+	 */
+	get valid() {
+		return true;
+	}
 }

packages/lib/src/link/link.ts

@@ -538,6 +538,9 @@ export class Link {
 		if (!entry) {
 			throw new Error(`Attempt to send unregistered Request ${request.constructor.name}`);
 		}
+		if (this.connector.src.addressedTo(dst)) {
+			throw new Error(`Message would return back to sender ${dst}.`);
+		}
 		if (this.validateSent) {
 			entry.requestFromJSON(JSON.parse(JSON.stringify(request)));
 		}
@@ -572,6 +575,9 @@ export class Link {
 		if (!entry) {
 			throw new Error(`Attempt to send unregistered Event ${event.constructor.name}`);
 		}
+		if (this.connector.src.addressedTo(dst)) {
+			throw new Error(`Message would return back to sender ${dst}.`);
+		}
 		if (this.validateSent) {
 			entry.eventFromJSON(JSON.parse(JSON.stringify(event)));
 		}

packages/lib/src/link/messages.ts

@@ -24,6 +24,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	controller.ControllerConfigSetPropRequest,
 	controller.HostGenerateTokenRequest,
 	controller.HostConfigCreateRequest,
+	controller.GetFactorioCredentialsRequest,
 	controller.LogSetSubscriptionsRequest,
 	controller.LogQueryRequest,
 	controller.LogMessageEvent,

packages/lib/src/load_plugin_list.ts

@@ -0,0 +1,186 @@
+import fs from "fs-extra";
+import path from "path";
+import { logger } from "./logging";
+import * as libFileOps from "./file_ops";
+import vm from "vm";
+
+/**
+ * Searches for local plugins in the plugins/ and external_plugins/ directories
+ * and adds them to the plugin list if they contain a package.json file.
+ *
+ * @param pluginList - Map of plugin names to their file paths
+ * @returns Promise that resolves to true if any plugins were added to the list, false otherwise
+ */
+async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolean> {
+	// Check folders for plugins
+	const pluginFolders = [
+		"plugins",
+		"external_plugins",
+	];
+
+	let hasChanged = 0;
+	const ignoredFolders = new Set(["node_modules", ".git", "dist", "build", "disabled"]);
+	const maxDepth = 3;
+
+	for (const pluginFolder of pluginFolders) {
+		let pluginNames;
+		try {
+			pluginNames = await fs.readdir(pluginFolder);
+		} catch (err: any) {
+			if (err.code === "ENOENT") {
+				// Skip if folder doesn't exist
+				continue;
+			}
+			throw err;
+		}
+		for (const pluginName of pluginNames) {
+			const pluginPath = path.join(pluginFolder, pluginName);
+			if (pluginList.has(pluginName)) {
+				// If the one in the list is not the same as this one, warn
+				if (pluginList.get(pluginName) !== path.resolve(pluginPath)) {
+					logger.warn(
+						`${pluginName} is already in the plugin list, but with a different path - ` +
+						`using ${pluginList.get(pluginName)}`
+					);
+				}
+				continue;
+			}
+			const stats = await fs.stat(pluginPath);
+			if (stats.isDirectory()) {
+				// Check for package.json and plugin keyword
+				if (await checkPackageJson(pluginPath)) {
+					pluginList.set(pluginName, path.resolve(pluginPath));
+					logger.info(`Added ${pluginName} from ${pluginFolder}`);
+					hasChanged += 1;
+					continue;
+				}
+
+				if (pluginPath.split(path.sep).length < maxDepth && !ignoredFolders.has(pluginName)) {
+					logger.warn(`Possible monorepo detected, scanning ${pluginPath} for clusterio-plugins`);
+					pluginFolders.push(pluginPath);
+				}
+			}
+		}
+	}
+
+	return hasChanged > 0;
+}
+
+function getPluginName(requireSpec: string) {
+	const context = vm.createContext({ require: require, pluginInfo: null });
+	const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
+	vm.runInContext(code, context);
+	return context.pluginInfo.name;
+}
+
+/**
+ * Find NPM packages in the root package.json that satisfies the requirements for plugins.
+ * Adds discovered plugins to the plugin list.
+ *
+ * @param pluginList - Map of plugin names to their file paths
+ * @returns Promise that resolves to true if any plugins were added to the list, false otherwise
+ */
+async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean> {
+	let dependencies;
+	try {
+		const rootPackageJson = JSON.parse(await fs.readFile(path.resolve("package.json"), { encoding: "utf8" }));
+		dependencies = rootPackageJson.dependencies;
+	} catch (err: any) {
+		if (err.code === "ENOENT") {
+			// Skip if package.json doesn't exist
+			return false;
+		}
+		throw err;
+	}
+
+	if (!dependencies) {
+		return false;
+	}
+
+	let hasChanged = 0;
+	for (const [packageName, packageVersion] of Object.entries(dependencies)) {
+		if ([...pluginList.values()].includes(packageName)) {
+			continue; // This npm module is already in the plugin list
+		}
+		// Find the package in node_modules and read the package.json to check if it's a clusterio-plugin
+		if (await checkPackageJson(path.resolve("node_modules", packageName))) {
+			const pluginName = getPluginName(path.resolve("node_modules", packageName));
+			if (pluginList.has(pluginName)) {
+				logger.warn(
+					`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
+					`but with a different path - using ${pluginList.get(pluginName)}`
+				);
+			} else {
+				pluginList.set(pluginName, packageName);
+				logger.info(`Added ${pluginName} from NPM`);
+				hasChanged += 1;
+			}
+		} else if (![
+			"@clusterio/controller",
+			"@clusterio/ctl",
+			"@clusterio/host",
+			"@clusterio/lib",
+			"@clusterio/web_ui",
+		].includes(packageName)) {
+			logger.warn(`${packageName}@${packageVersion} is not a clusterio-plugin`);
+		}
+	}
+	return hasChanged > 0;
+}
+
+/**
+ * Loads the plugin list from the specified file and optionally discovers local plugins
+ *
+ * @param pluginListPath - Path to the plugin list JSON file
+ * @param findLocal - Whether to look for local plugins in plugins/ and external_plugins/ directories
+ * @returns Map of plugin name to plugin path
+ */
+export async function loadPluginList(pluginListPath: string, findLocal = true): Promise<Map<string, string>> {
+	let pluginList = new Map<string, string>();
+
+	// Try to load existing plugin list
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(pluginListPath, { encoding: "utf8" })));
+		logger.info(`Loaded ${pluginList.size} plugins from ${pluginListPath}`);
+	} catch (err: any) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+		logger.info(`No existing plugin list found at ${pluginListPath}`);
+	}
+
+	// Track if any changes were made
+	let hasChanges = false;
+
+	// Optionally discover local plugins
+	if (findLocal) {
+		hasChanges = await findLocalPlugins(pluginList) || hasChanges;
+	}
+
+	// Discover NPM plugins
+	hasChanges = await findNpmPlugins(pluginList) || hasChanges;
+
+	if (hasChanges) {
+		await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t"));
+	}
+
+	return pluginList;
+}
+
+/**
+ * Helper function to check if a package.json file exists and contains the clusterio-plugin keyword
+ * @param pluginPath - Path to the plugin directory
+ * @returns Promise that resolves to true if package.json exists and has clusterio-plugin keyword
+ */
+async function checkPackageJson(pluginPath: string): Promise<boolean> {
+	const packageJsonPath = path.join(pluginPath, "package.json");
+	if (!await fs.exists(packageJsonPath)) {
+		return false;
+	}
+
+	// Read and parse package.json
+	const packageJson = JSON.parse(await fs.readFile(packageJsonPath, { encoding: "utf8" }));
+
+	// Check if package has the clusterio-plugin keyword
+	return packageJson.keywords?.includes("clusterio-plugin");
+}

packages/lib/src/logging.ts

@@ -72,7 +72,7 @@ export class WebConsoleFormat {
 		public options = {}
 	) { }
 
-	transform(info: any, _options: object) {
+	transform(info: any, _options: unknown) {
 		let src = " ";
 		if (info.host_id !== undefined) {
 			src += `h:${info.host_name} - `;

packages/lib/src/logging_utils.ts

@@ -139,6 +139,10 @@ function formatServerOutput(parsed: ParsedFactorioOutput) {
 // These are defined here to avoid circular dependencies and pulling them
 // into the web interface code.
 
+export interface TerminalFormatOptions extends winston.Logform.ColorizeOptions {
+	showTimestamp?: boolean
+}
+
 /**
  * Formats winston log messages for a character terminal.
  */
@@ -146,12 +150,13 @@ export class TerminalFormat {
 	colorize: winston.Logform.Colorizer;
 
 	constructor(
-		public options: object = {}
+		public options: TerminalFormatOptions = {}
 	) {
 		this.colorize = winston.format.colorize(options);
 	}
 
-	transform(info: any, options: { showTimestamp: boolean }) {
+	transform(info: any, opts: unknown) {
+		const options = opts as TerminalFormatOptions;
 		info = this.colorize.transform(info, this.colorize.options);
 		let ts = "";
 		if (options.showTimestamp && info.timestamp) {
@@ -207,7 +212,9 @@ export class LinkTransport extends Transport {
 		}
 
 		try {
-			this.link.send(new libData.LogMessageEvent(info));
+			if (this.link.connector.valid) {
+				this.link.send(new libData.LogMessageEvent(info));
+			}
 		} catch (err) {
 			// Ignore session lost errors.
 			if (!(err instanceof libErrors.SessionLost)) {

packages/lib/src/plugin.ts

@@ -53,13 +53,14 @@ export type PluginWebpackEnvInfo = PluginDeclaration & {
  * Information about the event.
  */
 export interface PlayerEvent {
-	type: "join" | "leave" | "import";
-	/** Name of the player that joined/left */
+	type: "join" | "leave" | "import" | "promote" | "demote" | "ban" | "unban" | "whitelisted" | "unwhitelisted";
+	/** Name of the player that caused the event */
 	name: string,
 	/**
-	 * Only present for type "leave". Reason for player leaving the
+	 * Only present for type "leave" and "ban". Reason for player leaving the
 	 * game, one of the possible reasons in defines.disconnect_reason
 	 * or "server_quit" if the server exits while the player is online.
+	 * When type is "ban" this is the reason given for the ban.
 	 */
 	reason?: string;
 	/**

packages/lib/src/plugin_loader.ts

@@ -26,6 +26,15 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
 		let pluginInfo: libPlugin.PluginNodeEnvInfo;
 		let pluginPackage: { name?: string, version: string, main?: string, private?: boolean };
 
+		// Check if plugin path exists, otherwise remove it
+		try {
+			require.resolve(pluginPath);
+		} catch (err) {
+			logger.error(`Plugin path ${pluginPath} does not exist, not loading ${pluginName}`);
+			pluginList.delete(pluginName);
+			continue;
+		}
+
 		try {
 			pluginInfo = require(pluginPath).plugin;
 			pluginPackage = require(path.posix.join(pluginPath, "package.json"));

packages/lib/src/subscriptions.ts

@@ -1,11 +1,12 @@
 import { Type, Static } from "@sinclair/typebox";
-import { Link, Event, EventClass, RequestHandler, WebSocketClientConnector, WebSocketBaseConnector } from "./link";
-import { logger } from "./logging";
-import { Address, MessageRequest, IControllerUser } from "./data";
+import { Link, Event, EventClass, RequestHandler, WebSocketBaseConnector } from "./link";
+import { Address, MessageRequest, IControllerUser, plainJson, JsonBoolean } from "./data";
 import isDeepStrictEqual from "./is_deep_strict_equal";
+import { RequestError, SessionLost } from "./errors";
+import { logger } from "./logging";
 
 export type SubscriptionRequestHandler<T> = RequestHandler<SubscriptionRequest, Event<T> | null>;
-export type EventSubscriberCallback<T> = (updates: T[], synced: boolean) => void
+export type EventSubscriberCallback<T> = (event: T | null, synced: boolean) => void
 
 /**
  * A subscription request sent by a subscriber, this updates what events the subscriber will be sent
@@ -15,7 +16,7 @@ export type EventSubscriberCallback<T> = (updates: T[], synced: boolean) => void
 export class SubscriptionRequest {
 	declare ["constructor"]: typeof SubscriptionRequest;
 	static type = "request" as const;
-	static src = ["control", "instance"] as const;
+	static src = ["control", "host", "instance"] as const;
 	static dst = "controller" as const;
 	static permission(user: IControllerUser, message: MessageRequest) {
 		if (typeof message.data === "object" && message.data !== null) {
@@ -31,6 +32,9 @@ export class SubscriptionRequest {
 		}
 	}
 
+	/** Indicates if updates have been sent */
+	static Response = JsonBoolean;
+
 	constructor(
 		public eventName: string,
 		public subscribe: boolean,
@@ -56,9 +60,14 @@ export class SubscriptionRequest {
 	}
 }
 
+type Subscriber = {
+	link: Link,
+	dst: Address,
+}
+
 type EventData = {
 	subscriptionUpdate?: SubscriptionRequestHandler<unknown>,
-	subscriptions: Set<Link>,
+	subscriptions: Array<Subscriber>,
 };
 
 /**
@@ -85,7 +94,7 @@ export class SubscriptionController {
 		}
 		this._events.set(entry.name, {
 			subscriptionUpdate: subscriptionUpdate,
-			subscriptions: new Set(),
+			subscriptions: [],
 		});
 	}
 
@@ -102,13 +111,28 @@ export class SubscriptionController {
 		if (!eventData) {
 			throw new Error(`Event ${entry.name} is not a registered as subscribable`);
 		}
-		for (let link of eventData.subscriptions) {
-			if ((link.connector as WebSocketBaseConnector).closing) {
-				eventData.subscriptions.delete(link);
+		const toRemove: Subscriber[] = [];
+		for (const subscriber of eventData.subscriptions) {
+			if ((subscriber.link.connector as WebSocketBaseConnector).closing) {
+				toRemove.push(subscriber);
 			} else {
-				link.send(event);
+				subscriber.link.sendTo(subscriber.dst, event);
 			}
 		}
+		eventData.subscriptions = eventData.subscriptions.filter(subscriber => !toRemove.includes(subscriber));
+	}
+
+	/**
+	 * Unsubscribe from all events of a given address.
+	 * Used when an instance is stopped to stop all active subscriptions.
+	 * @param address - Address to stop sending events to.
+	 */
+	unsubscribeAddress(address: Address) {
+		for (let eventData of this._events.values()) {
+			eventData.subscriptions = eventData.subscriptions.filter(
+				subscriber => !subscriber.dst.addressedTo(address)
+			);
+		}
 	}
 
 	/**
@@ -116,9 +140,11 @@ export class SubscriptionController {
 	 * Used when a link is closed to stop all active subscriptions.
 	 * @param link - Link to stop sending events to.
 	 */
-	unsubscribe(link: Link) {
+	unsubscribeLink(link: Link) {
 		for (let eventData of this._events.values()) {
-			eventData.subscriptions.delete(link);
+			eventData.subscriptions = eventData.subscriptions.filter(
+				subscriber => subscriber.link !== link
+			);
 		}
 	}
 
@@ -138,16 +164,23 @@ export class SubscriptionController {
 			throw new Error(`Event ${event.eventName} is not a registered as subscribable`);
 		}
 		if (event.subscribe === false) {
-			eventData.subscriptions.delete(link);
+			const index = eventData.subscriptions.findLastIndex(subscriber => subscriber.dst.addressedTo(src));
+			if (index >= 0) {
+				eventData.subscriptions.splice(index, 1);
+			}
 		} else {
-			eventData.subscriptions.add(link);
+			if (!eventData.subscriptions.some(subscriber => subscriber.dst.addressedTo(src))) {
+				eventData.subscriptions.push({ link: link, dst: src });
+			}
 			if (eventData.subscriptionUpdate) {
 				const eventReplay = await eventData.subscriptionUpdate(event, src, dst);
 				if (eventReplay) {
-					link.send(eventReplay);
+					link.sendTo(src, eventReplay);
+					return true;
 				}
 			}
 		}
+		return false;
 	}
 }
 
@@ -157,9 +190,9 @@ export interface SubscribableValue {
 	isDeleted: boolean,
 }
 
-export interface EventSubscribable<T, V extends SubscribableValue> extends Event<T> {
+export type EventSubscribable<T, V extends SubscribableValue> = Event<T> & Partial<{
 	updates: V[],
-}
+}>
 
 /**
  * Component for subscribing to and tracking updates of a remote resource
@@ -167,43 +200,61 @@ export interface EventSubscribable<T, V extends SubscribableValue> extends Event
  */
 export class EventSubscriber<
 	T extends EventSubscribable<T, V>,
-	K extends string | number = T["updates"][number]["id"],
-	V extends SubscribableValue = T["updates"][number],
+	K extends string | number = NonNullable<T["updates"]>[number]["id"],
+	V extends SubscribableValue = NonNullable<T["updates"]>[number],
 > {
-	_callbacks = new Array<EventSubscriberCallback<V>>();
+	/** The time at which an event was last received, is less than 0 when there have been no events */
+	lastResponseTimeMs = -1;
 	/** Values of the subscribed resource */
 	values = new Map<K, V>();
 	/** True if this subscriber is currently synced with the source */
 	synced = false;
-	_snapshot?: readonly [ReadonlyMap<K, Readonly<V>>, boolean];
-	_snapshotLastUpdatedMs = 0;
-	lastResponseTimeMs = -1;
+	/** True if this subscriber is expecting to receive updates from the source */
+	_syncing = false;
+	/** Repeat calls to getSnapshot will return the same readonly copy unless values has updated */
+	_snapshot: readonly [ReadonlyMap<K, Readonly<V>>, boolean] = [new Map<K, V>(), false];
+	_snapshotLastUpdatedMs = -1;
+	/** Callbacks will be called when an event is received or the synced state changes */
+	_callbacks = new Array<EventSubscriberCallback<T>>();
 
 	constructor(
 		private Event: EventClass<T>,
 		public control: Link,
 	) {
-		control.handle(Event, this._handle.bind(this));
+		control.handle(Event, this._handleEvent.bind(this));
 		control.connector.on("connect", () => {
-			this._updateSubscription();
+			this.handleConnectionEvent("connect");
 		});
 		control.connector.on("close", () => {
-			if (this.synced) {
-				this.synced = false;
-				for (let callback of this._callbacks) {
-					callback([], this.synced);
-				}
-			}
+			this.handleConnectionEvent("close");
 		});
 	}
 
+	/**
+	 * Handle connection events from this connector or another in the chain to the controller
+	 * This should be called by an instance plugin during onControllerConnectionEvent
+	 * @param event - event type that occurred
+	 */
+	handleConnectionEvent(event: "connect" | "drop" | "resume" | "close") {
+		if (event === "connect" || event === "resume") {
+			this._updateSubscription();
+		} else if (this.synced) {
+			this.synced = false;
+			for (let callback of this._callbacks) {
+				callback(null, false);
+			}
+		}
+	}
+
 	/**
 	 * Handle incoming events and distribute it to the correct callbacks
 	 * @param event - event from subscribed resource
 	 * @internal
 	 */
-	async _handle(event: EventSubscribable<T, V>) {
-		for (const value of event.updates) {
+	async _handleEvent(event: T) {
+		// We support the automatic maintaining of a map for events with the updates property
+		const updates = event.updates ?? [];
+		for (const value of updates) {
 			this.lastResponseTimeMs = Math.max(this.lastResponseTimeMs, value.updatedAtMs);
 			// Warn about updates which changes content but don't update updatedAtMs
 			const existing = this.values.get(value.id as K);
@@ -219,8 +270,13 @@ export class EventSubscriber<
 				this.values.set(value.id as K, value);
 			}
 		}
-		for (let callback of this._callbacks) {
-			callback(event.updates, this.synced);
+		// Event handling logic
+		if (this._syncing && !this.synced) {
+			this._snapshotLastUpdatedMs = -1;
+			this.synced = true;
+		}
+		for (const callback of this._callbacks) {
+			callback(event, this.synced);
 		}
 	}
 
@@ -228,10 +284,10 @@ export class EventSubscriber<
 	 * Subscribe to receive all event notifications
 	 * @param handler -
 	 *     callback invoked whenever the subscribed resource changes or the
-	 *     synced property changes, in which case the updates will be empty.
+	 *     synced property changes, in which case the event will be null.
 	 * @returns function that will unsubscribe from notifications
 	 */
-	subscribe(handler: EventSubscriberCallback<V>) {
+	subscribe(handler: EventSubscriberCallback<T>) {
 		this._callbacks.push(handler);
 		if (this._callbacks.length === 1) {
 			this._updateSubscription();
@@ -277,26 +333,30 @@ export class EventSubscriber<
 	 * Update the subscription with the controller based on current handler counts
 	 */
 	async _updateSubscription() {
-		if (!(this.control.connector as WebSocketClientConnector).connected) {
+		if (!this.control.connector.valid) {
 			return;
 		}
 		const entry = Link._eventsByClass.get(this.Event)!;
 
 		try {
-			await this.control.send(new SubscriptionRequest(
+			this._syncing = this._callbacks.length > 0;
+			const updatesSent = await this.control.sendTo("controller", new SubscriptionRequest(
 				entry.name,
-				this._callbacks.length > 0,
+				this._syncing,
 				this.lastResponseTimeMs
 			));
-			this.synced = this._callbacks.length > 0;
-			this._snapshotLastUpdatedMs = 0;
-			for (let callback of this._callbacks) {
-				callback([], this.synced);
+			if (!updatesSent) {
+				this.synced = this._syncing;
+				for (const callback of this._callbacks) {
+					callback(null, this.synced);
+				}
 			}
 		} catch (err: any) {
-			logger.error(
-				`Unexpected error updating ${entry.name} subscription:\n${err.stack}`
-			);
+			if (!(err instanceof RequestError) || err.code !== "SessionLost") {
+				logger.error(
+					`Unexpected error updating ${entry.name} subscription:\n${err.stack}`
+				);
+			}
 		}
 	}
 }

packages/lib/tsconfig.browser.json

@@ -26,6 +26,5 @@
 	],
 	"compilerOptions": {
 		"rootDir": ".",
-		"outDir": "dist/browser",
 	}
 }

packages/lib/tsconfig.node.json

@@ -2,7 +2,6 @@
 	"extends": "../../tsconfig.node.json",
 	"include": ["index.ts", "browser.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"],
 	"compilerOptions": {
-		"outDir": "dist/node",
 		"allowJs": true,
 	},
 }

packages/web_ui/package.json

@@ -3,21 +3,23 @@
 	"description": "Clusterio web interface implementation",
 	"author": "Hornwitser",
 	"license": "MIT",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"browser": "src/index.ts",
 	"main": "dist/browser/src/index.js",
 	"scripts": {
 		"test": "echo \"Error: no test specified\" && exit 1"
 	},
 	"keywords": [
-		"clusterio"
+		"clusterio",
+		"factorio"
 	],
 	"engines": {
 		"node": ">=12"
 	},
 	"dependencies": {
 		"@ant-design/icons": "^5.1.4",
-		"@clusterio/lib": "^2.0.0-alpha.18",
+		"@clusterio/lib": "workspace:*",
+		"@sinclair/typebox": "^0.30.4",
 		"@swc/core": "^1.4.0",
 		"antd": "^5.13.0",
 		"assert": "^2.0.0",
@@ -39,7 +41,7 @@
 		"style-loader": "^3.3.3",
 		"swc-loader": "^0.2.6",
 		"terser-webpack-plugin": "^5.3.9",
-		"typescript": "^5.1.6",
+		"typescript": "^5.5.3",
 		"util": "^0.12.5",
 		"webpack": "^5.88.2",
 		"webpack-cli": "^5.1.4",

packages/web_ui/src/BaseWebPlugin.ts

@@ -24,6 +24,7 @@ export interface PluginLoginForm {
 }
 
 export interface InputComponentProps {
+	disabled?: boolean,
 	fieldDefinition: FieldDefinition,
 	value: null | boolean | number | string,
 	onChange: (value: null | boolean | number | string) => void,

packages/web_ui/src/bootstrap.tsx

@@ -7,6 +7,7 @@ import * as lib from "@clusterio/lib";
 
 import App from "./components/App";
 import BaseWebPlugin, { InputComponent } from "./BaseWebPlugin";
+import InputRole from "./components/InputRole";
 import InputModPack from "./components/InputModPack";
 import { Control, ControlConnector } from "./util/websocket";
 
@@ -109,6 +110,7 @@ async function loadPlugins(pluginInfos: lib.PluginWebpackEnvInfo[], control: Con
 function inputComponentsFromPlugins(plugins: Map<string, BaseWebPlugin>) {
 	const inputComponents: Record<string, InputComponent> = {
 		"mod_pack": InputModPack,
+		"role": InputRole,
 	};
 	for (let [pluginName, plugin] of plugins) {
 		for (const [name, Component] of Object.entries(plugin.inputComponents)) {

packages/web_ui/src/components/AssignInstanceModal.tsx

@@ -77,13 +77,14 @@ export default function AssignInstanceModal(props: AssignInstanceModalProps) {
 			</Paragraph>
 			<Form form={form} initialValues={{ host: props.hostId ?? "null" }}>
 				<Form.Item name="host" label="Host">
-					<Select>
+					<Select showSearch optionFilterProp="name">
 						<Select.Option value={"null"}>
 							<Typography.Text italic>Unassigned</Typography.Text>
 						</Select.Option>
 						{[...hosts.values()].sort((a, b) => strcmp(a.name, b.name)).map((host) => <Select.Option
 							key={host["id"]}
 							value={host["id"]}
+							name={host["name"]}
 							disabled={!host["connected"]}
 						>
 							{host["name"]}
@@ -95,4 +96,3 @@ export default function AssignInstanceModal(props: AssignInstanceModalProps) {
 		</Modal>
 	</>;
 }
-

packages/web_ui/src/components/BaseConfigTree.tsx

@@ -1,6 +1,6 @@
 import React, { useContext, useEffect, useState } from "react";
 import {
-	Button, Card, Checkbox, Form, FormInstance, Input, InputNumber, Space, Spin, Tooltip, Tree, Typography,
+	Button, Card, Checkbox, Form, FormInstance, Input, InputNumber, Select, Space, Spin, Tooltip, Tree, Typography,
 } from "antd";
 import ReloadOutlined from "@ant-design/icons/ReloadOutlined";
 
@@ -16,13 +16,13 @@ const { Title } = Typography;
 function getInitialValues(config: lib.Config<any>, props: BaseConfigTreeProps) {
 	let initialValues: any = {};
 	for (let [name, def] of Object.entries(config.constructor.fieldDefinitions)) {
-		if (!config.canAccess(name)) {
+		if (!config.canAccess(name, lib.ConfigAccess.read)) {
 			continue;
 		}
 
 		let value = config.get(name) as any;
 		if (def.type === "object") {
-			for (let prop of Object.keys(value)) {
+			for (let prop of Object.keys(value ?? {})) {
 				initialValues[`${name}.${prop}`] = JSON.stringify(value[prop]);
 			}
 		} else {
@@ -32,21 +32,33 @@ function getInitialValues(config: lib.Config<any>, props: BaseConfigTreeProps) {
 	return initialValues;
 }
 
-function renderInput(inputComponents: Record<string, InputComponent>, def: lib.FieldDefinition) {
+function renderInput(inputComponents: Record<string, InputComponent>, def: lib.FieldDefinition, readonly: boolean) {
 	if (def.inputComponent && Object.prototype.hasOwnProperty.call(inputComponents, def.inputComponent)) {
 		// Field.Item will provide the value and onChange props to the component.
 		type InputPartial = React.ComponentClass<Omit<InputComponentProps, "value" | "onChange">>;
 		const CustomInput = inputComponents[def.inputComponent] as unknown as InputPartial;
-		return <CustomInput fieldDefinition={def} />;
+		return <CustomInput fieldDefinition={def} disabled={readonly} />;
 	}
 	if (def.type === "boolean") {
-		return <Checkbox/>;
+		return <Checkbox disabled={readonly} />;
 	}
 	if (def.type === "string") {
-		return <Input/>;
+		if (def.credential) {
+			return <Input.Password autoComplete={def.autoComplete ?? "new-password"} disabled={readonly} />;
+		}
+		if (def.enum) {
+			return <Select
+				showSearch
+				style={{ minWidth: 175 }}
+				options={def.enum.map(value => ({ label: value, value: value }))}
+				allowClear={def.optional}
+				disabled={readonly}
+			/>;
+		}
+		return <Input autoComplete={def.autoComplete} disabled={readonly} />;
 	}
 	if (def.type === "number") {
-		return <InputNumber/>;
+		return <InputNumber autoComplete={def.autoComplete} disabled={readonly} />;
 	}
 
 	return `Unknown type ${def.type}`;
@@ -110,7 +122,8 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) {
 			if (key.endsWith(":add.name") || key.endsWith(":add.value")) {
 				continue;
 			}
-			if (initialValues[key] === value) {
+			const def = config?.constructor.fieldDefinitions[key];
+			if (!def?.credential && initialValues[key] === value) {
 				changed = newChangedFields.delete(key) || changed;
 			} else {
 				changed = !newChangedFields.has(key) || changed;
@@ -213,6 +226,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) {
 	</>;
 }
 
+// eslint-disable-next-line complexity
 function computeTreeData(
 	control: Control,
 	form: FormInstance<any>,
@@ -259,18 +273,24 @@ function computeTreeData(
 		};
 
 		for (let [field, def] of Object.entries(groupDefs)) {
-			if (!config.canAccess(`${groupName}.${field}`)) {
+			if (def.hidden === true) {
+				continue;
+			}
+
+			const fieldName = `${groupName}.${field}`;
+			const canWrite = config.canAccess(fieldName, lib.ConfigAccess.write);
+			const canRead = config.canAccess(fieldName, lib.ConfigAccess.read);
+			if (!canWrite && !canRead) {
 				continue;
 			}
 
-			let fieldName = `${groupName}.${field}`;
 			let value = config.get(fieldName) as any;
 			let childNode: ChildNode = {
 				key: fieldName,
 				children: [],
 			};
 
-			if (def.type === "object") {
+			if (def.type === "object" && !Object.keys(control.inputComponents).includes(def.inputComponent!)) {
 				let restartRequiredProps = new Set(def.restartRequiredProps || []);
 				childNode.title = <Form.Item
 					label={<>
@@ -279,7 +299,7 @@ function computeTreeData(
 					</>}
 					tooltip={def.description}
 				/>;
-				for (let prop of Object.keys(value)) {
+				for (let prop of Object.keys(value ?? {})) {
 					let propPath = `${groupName}.${field}.${prop}`;
 					let restart = Boolean(
 						def.restartRequiredProps && Number(def.restartRequired) ^ Number(restartRequiredProps.has(prop))
@@ -304,7 +324,7 @@ function computeTreeData(
 								return Promise.resolve();
 							}}]}
 						>
-							<Input className="json-input" />
+							<Input className="json-input" disabled={!canWrite} />
 						</Form.Item>,
 					});
 
@@ -312,47 +332,53 @@ function computeTreeData(
 					propsMap.set(propPath, [fieldName, prop]);
 				}
 
-				let newPropPath = `${groupName}.${field}:add`;
-				childNode.children.push({
-					key: newPropPath,
-					title: <Space>
-						<Form.Item
-							name={`${newPropPath}.name`}
-							noStyle={true}
-						>
-							<Input />
-						</Form.Item>:
-						<Form.Item
-							name={`${newPropPath}.value`}
-							noStyle={true}
-						>
-							<Input className="json-input" />
-						</Form.Item>
-						<Button
-							size="small"
-							onClick={() => {
-								let propName = form.getFieldValue(`${newPropPath}.name`);
-								let propValue = form.getFieldValue(`${newPropPath}.value`);
-								if (!Object.prototype.hasOwnProperty.call(value, propName)) {
-									let newConfig = ConfigClass.fromJSON(config!.toJSON(), "control");
-									(newConfig as lib.Config<any>).setProp(fieldName, propName, null);
-									setConfig(newConfig);
-								}
+				if (canWrite) {
+					let newPropPath = `${groupName}.${field}:add`;
+					childNode.children.push({
+						key: newPropPath,
+						title: <Space>
+							<Form.Item
+								name={`${newPropPath}.name`}
+								noStyle={true}
+							>
+								<Input />
+							</Form.Item>:
+							<Form.Item
+								name={`${newPropPath}.value`}
+								noStyle={true}
+							>
+								<Input className="json-input" />
+							</Form.Item>
+							<Button
+								size="small"
+								onClick={() => {
+									let propName = form.getFieldValue(`${newPropPath}.name`);
+									let propValue = form.getFieldValue(`${newPropPath}.value`);
+									if (!value || !Object.prototype.hasOwnProperty.call(value, propName)) {
+										let newConfig = ConfigClass.fromJSON(config!.toJSON(), "control");
+										(newConfig as lib.Config<any>).setProp(fieldName, propName, null);
+										setConfig(newConfig);
+									}
 
-								let propPath = `${groupName}.${field}.${propName}`;
-								form.setFieldsValue({
-									[`${newPropPath}.name`]: "",
-									[`${newPropPath}.value`]: "",
-									[propPath]: propValue,
-								});
-								form.validateFields([propPath]);
-								onValuesChange({ [propPath]: propValue });
-							}}
-						>
-							Add
-						</Button>
-					</Space>,
-				});
+									let propPath = `${groupName}.${field}.${propName}`;
+									form.setFieldsValue({
+										[`${newPropPath}.name`]: "",
+										[`${newPropPath}.value`]: "",
+										[propPath]: propValue,
+									});
+									form.validateFields([propPath]);
+									onValuesChange({ [propPath]: propValue });
+								}}
+							>
+								Add
+							</Button>
+						</Space>,
+					});
+				}
+
+				if (!childNode.children.length) {
+					continue;
+				}
 
 			} else {
 				childNode.title = <Form.Item
@@ -366,7 +392,7 @@ function computeTreeData(
 					tooltip={def.description}
 					valuePropName={def.type === "boolean" ? "checked" : "value"}
 				>
-					{renderInput(control.inputComponents, def)}
+					{renderInput(control.inputComponents, def, !canWrite)}
 				</Form.Item>;
 				initialValues[fieldName] = value;
 			}