Create the character at spawn when it can't be made in place

create_character needs a free spot within 200 tiles and does not
generate chunks to find one, so it failed for a player teleported into
a chunk that does not exist yet, and where there is only water or void.
The download then failed and left the player as a spectator.

Resolves #1187

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Default owners for all files in the repository.
+* @psihius @Danielv123 @Hornwitser @Cooldude2606 @ILLISIS

.github/actions/pnpm-install/action.yml

@@ -1,5 +1,5 @@
 name: Install dependencies
-description: Set up pnpm and Node.js, restore caches and run pnpm install
+description: Set up pnpm and Node.js and run pnpm install
 
 inputs:
   node-version:
@@ -12,42 +12,15 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: pnpm/action-setup@v6
     - uses: actions/setup-node@v7
       with:
         node-version: ${{ inputs.node-version }}
+        package-manager-cache: false
 
-    # The lockfile is not committed, so the usual lockfile hash key does not
-    # apply. Instead a new store snapshot is saved whenever a package.json
-    # changes or a week passes, and every run restores the most recent one.
-    - id: store
-      shell: bash
-      run: |
-        echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
-        echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
-    - uses: actions/cache@v6
-      with:
-        path: ${{ steps.store.outputs.path }}
-        key: pnpm-store-${{ runner.os }}-${{ steps.store.outputs.week }}-${{ hashFiles('**/package.json', 'pnpm-workspace.yaml') }}
-        restore-keys: pnpm-store-${{ runner.os }}-
-
-    # The prepare scripts run tsc --build and webpack, both incremental when
-    # their previous output is present. Every run saves its output and the
-    # next run starts from the most recent one.
-    - if: inputs.ignore-scripts != 'true'
-      uses: actions/cache@v6
-      with:
-        path: |
-          packages/*/dist
-          plugins/*/dist
-          packages/*/node_modules/.cache
-          plugins/*/node_modules/.cache
-        key: build-${{ runner.os }}-${{ github.sha }}
-        restore-keys: build-${{ runner.os }}-
+    # pnpm/action-setup installs a bootstrap pnpm and then switches to the
+    # packageManager version, which takes about 10 seconds.
+    - shell: bash
+      run: npm install --global --no-audit --no-fund "$(node -p 'require("./package.json").packageManager.split("+")[0]')"
 
     - shell: bash
       run: pnpm i --no-frozen-lockfile ${{ inputs.ignore-scripts == 'true' && '--ignore-scripts' || '' }}
-    # Drop packages the restored store had but this install did not use, so
-    # the snapshot does not grow with every version bump.
-    - shell: bash
-      run: pnpm store prune

.github/workflows/ci-changes.yml

@@ -0,0 +1,126 @@
+name: Classify CI changes
+
+on:
+  workflow_call:
+    outputs:
+      build_and_test:
+        description: Run the test and coverage matrix, compatibility tests, and mod build.
+        value: ${{ jobs.changes.outputs.build_and_test }}
+      js_lint:
+        description: Run the JavaScript/TypeScript linter.
+        value: ${{ jobs.changes.outputs.js_lint }}
+      lua_lint:
+        description: Run the Lua linter.
+        value: ${{ jobs.changes.outputs.lua_lint }}
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      build_and_test: ${{ steps.syntax.outputs.build_and_test || steps.paths.outputs.code }}
+      js_lint: ${{ steps.lint.outputs.js_lint }}
+      lua_lint: ${{ steps.lint.outputs.lua_lint }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: dorny/paths-filter@v4
+        id: paths
+        with:
+          token: ''
+          base: ${{ github.event.pull_request.base.sha || github.event.before }}
+          ref: ${{ github.sha }}
+          list-files: json
+          predicate-quantifier: every
+          filters: |
+            # Files requiring validation beyond documentation checks.
+            code:
+              - '**'
+              - '!**/*.md'
+              - '!docs/**/*.{png,jpg,svg,gif}'
+            # Sources and configuration owned by each linter.
+            # Workflow scripts and shared build configuration run both linters.
+            js:
+              - '**/*.{js,cjs,mjs,jsx,ts,cts,mts,tsx}'
+              - '!.github/**'
+              - '!**/{webpack,babel,rollup,vite,jest}.config.*'
+            lua:
+              - '{*.lua,**/*.lua,.emmyrc.json}'
+              - '!.github/**'
+            # Modified JS/TS source eligible for the comment-only shortcut.
+            # TypeScript type changes still count as substantive changes.
+            syntax:
+              - modified: '**/*.{js,cjs,mjs,jsx,ts,cts,mts,tsx}'
+              # Source fixtures and templates always receive full validation.
+              - '!test/file/**'
+              - '!packages/create/templates/**'
+              - '!.github/**'
+              - '!**/{eslint,webpack,babel,rollup,vite,jest}.config.*'
+      - name: Select linters
+        id: lint
+        env:
+          CODE_FILES: ${{ steps.paths.outputs.code_files }}
+          JS_FILES: ${{ steps.paths.outputs.js_files }}
+          LUA_FILES: ${{ steps.paths.outputs.lua_files }}
+        run: |
+          # Unclassified non-documentation files can affect either language.
+          jq -nr --argjson code "$CODE_FILES" --argjson js "$JS_FILES" --argjson lua "$LUA_FILES" '
+            ($code - $js - $lua | length > 0) as $shared |
+            "js_lint=\($shared or ($js | length > 0))",
+            "lua_lint=\($shared or ($lua | length > 0))"
+          ' >> "$GITHUB_OUTPUT"
+      - name: Install Difftastic
+        if: steps.paths.outputs.syntax == 'true'
+        env:
+          DIFFT_VERSION: 0.71.0
+          DIFFT_SHA256: 61aea5394a53c56f144637cf39b3a0c0dafa27769731745e5851ff276e6478da
+        run: |
+          mkdir -p "$RUNNER_TEMP/difftastic"
+          curl -fsSL "https://github.com/Wilfred/difftastic/releases/download/$DIFFT_VERSION/difft-$DIFFT_VERSION-x86_64-unknown-linux-gnu.tar.gz" -o "$RUNNER_TEMP/difftastic.tar.gz"
+          echo "$DIFFT_SHA256  $RUNNER_TEMP/difftastic.tar.gz" | sha256sum -c
+          tar -xzf "$RUNNER_TEMP/difftastic.tar.gz" -C "$RUNNER_TEMP/difftastic"
+          echo "$RUNNER_TEMP/difftastic" >> "$GITHUB_PATH"
+      - name: Check for substantive changes
+        id: syntax
+        if: steps.paths.outputs.code == 'true'
+        env:
+          CODE_FILES: ${{ steps.paths.outputs.code_files }}
+          SYNTAX_FILES: ${{ steps.paths.outputs.syntax_files }}
+          BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
+          HEAD_SHA: ${{ github.sha }}
+        run: |
+          # Skip only when every non-documentation file is an ordinary source edit.
+          echo 'build_and_test=true' >> "$GITHUB_OUTPUT"
+          # A new branch has no previous commit; keep full validation on its first push.
+          if [[ "$BASE" == 0000000000000000000000000000000000000000 ]]; then
+            exit 0
+          fi
+          if ! jq -en --argjson code "$CODE_FILES" --argjson syntax "$SYNTAX_FILES" \
+            '$code - $syntax | length == 0' > /dev/null; then
+            exit 0
+          fi
+          git cat-file -e "$BASE^{commit}"
+          jq -j '.[] | ., "\u0000"' <<< "$SYNTAX_FILES" > "$RUNNER_TEMP/syntax-files"
+          while IFS= read -r -d '' file; do
+            # Mode changes and symlinks are substantive even when text is identical.
+            [[ -z "$(git diff --summary --no-renames "$BASE" "$HEAD_SHA" -- "$file")" ]] || exit 0
+            git ls-tree "$HEAD_SHA" -- "$file" | grep -qE '^100(644|755) blob ' || exit 0
+            before="$RUNNER_TEMP/before.${file##*.}"
+            after="$RUNNER_TEMP/after.${file##*.}"
+            git show "$BASE:$file" > "$before"
+            git show "$HEAD_SHA:$file" > "$after"
+            # Difftastic ignores ALL comments; retain compiler/bundler/coverage directives.
+            if grep -Eq '@ts-|@jsx|@internal|<reference\b|<amd-|sourceMappingURL|sourceURL|webpack|vite|[#@]__|/\*!|(istanbul|c8|v8)[[:space:]]+ignore' "$before" "$after"; then
+              exit 0
+            fi
+            case "$file" in
+              *.js|*.cjs|*.mjs|*.jsx)
+                # JavaScript JSDoc can contain type information.
+                if grep -Fq '/**' "$before" "$after"; then exit 0; fi ;;
+            esac
+            if ! difft --ignore-comments --check-only --exit-code "$before" "$after"; then
+              exit 0
+            fi
+          done < "$RUNNER_TEMP/syntax-files"
+          echo 'build_and_test=false' >> "$GITHUB_OUTPUT"

.github/workflows/lua-lint.yml

@@ -4,11 +4,20 @@ on:
   push:
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 env:
   EMMYLUA_CHECK_VERSION: "0.25.1"
 
 jobs:
+  changes:
+    uses: ./.github/workflows/ci-changes.yml
+
   lint:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.lua_lint != 'false' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7

.github/workflows/node.js.yml

@@ -7,26 +7,54 @@ on:
   push:
   pull_request:
 
+# Superseded runs of a branch or pull request are cancelled. Runs on master
+# each get their own group so every commit there is tested.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 env:
   NODE_VERSION: 24.x
 
 jobs:
+  changes:
+    uses: ./.github/workflows/ci-changes.yml
+
   test:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.build_and_test != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
-        # Supported node versions are active lts and maintenance see https://nodejs.org/en/about/releases/
-        node-version: [24.x]
-
-        # Supported factorio versions are latest stable and one minor version before stable (even if from a previous major) 
+        # To stay within the concurrent job limit, every Factorio version is
+        # tested on the latest Node.js, and every other supported Node.js
+        # version only with the latest Factorio. Coverage is performed on the
+        # oldest supported Node.js.
+        # Node.js release schedule: https://nodejs.org/en/about/releases/
+        # Supported factorio versions are latest stable and one minor version before stable (even if from a previous major)
+        node-version: [26.x]
         factorio-version: [1.1.110, 2.0.77, 2.1.17]
-    
-        # Coverage is performed on the older of the supported node versions
+
+        # The tests run in two halves of similar length on separate runners,
+        # the clusterioctl, plugin system and instance integration tests and the rest.
+        tests: [ctl, other]
+
         include:
+          - node-version: 24.x
+            factorio-version: 2.1.17
+            tests: ctl
+          - node-version: 24.x
+            factorio-version: 2.1.17
+            tests: other
           - node-version: 22.x
             factorio-version: 2.1.17
+            tests: ctl
+            coverage: true
+          - node-version: 22.x
+            factorio-version: 2.1.17
+            tests: other
             coverage: true
 
     steps:
@@ -39,24 +67,28 @@ jobs:
       run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/${{ matrix.factorio-version }}/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
     - name: Run tests
       if: ${{ !matrix.coverage }}
-      run: pnpm test
+      run: pnpm test --grep "^(Integration of Clusterio (clusterioctl|plugin system)|Clusterio Instance) " ${{ matrix.tests == 'other' && '--invert' || '' }}
     - name: Run coverage
       if: ${{ matrix.coverage }}
-      run: pnpm run ci-cover
+      run: pnpm run ci-cover --grep "^(Integration of Clusterio (clusterioctl|plugin system)|Clusterio Instance) " ${{ matrix.tests == 'other' && '--invert' || '' }}
     - name: Upload coverage to Codecov
       if: ${{ matrix.coverage }}
       uses: codecov/codecov-action@v7
       with:
         files: ./coverage/lcov.info
+        # Otherwise the raw V8 coverage in coverage/tmp is uploaded as well
+        disable_search: true
         token: ${{ secrets.CODECOV_TOKEN }}
     - name: Upload test logs for debugging
       if: ${{ !cancelled() }}
       uses: actions/upload-artifact@v7
       with:
-        name: logs-${{ matrix.node-version }}-${{ matrix.factorio-version }}
+        name: logs-${{ matrix.node-version }}-${{ matrix.factorio-version }}-${{ matrix.tests }}
         path: temp/test/logs
 
   version-compat:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.build_and_test != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     env:
@@ -79,16 +111,16 @@ jobs:
       uses: ./.github/actions/pnpm-install
       with:
         node-version: ${{ env.NODE_VERSION }}
-    - name: Build
-      run: pnpm run build
     - name: Test ${{ matrix.controller }} controller with ${{ matrix.host }} host
       run: >
         node scripts/version-compat-test.js
         --controller ${{ matrix.controller }}
         --host ${{ matrix.host }}
-        --published-version $PUBLISHED_VERSION
+        --published-version "$PUBLISHED_VERSION"
 
   lint:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.js_lint != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -100,6 +132,8 @@ jobs:
     - run: pnpm run lint
 
   build:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.build_and_test != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:

.github/workflows/publish.yml

@@ -23,6 +23,8 @@ jobs:
           registry-url: https://registry.npmjs.org
       - run: pnpm install --no-frozen-lockfile
       - run: pnpm -r --filter='./packages/*' --filter='./plugins/*' --fail-if-no-match publish --no-git-checks
+        env:
+          NODE_ENV: production
       - uses: softprops/action-gh-release@v3
         with:
           files: pnpm-lock.yaml

.gitignore

@@ -23,9 +23,6 @@ coverage
 build/Release
 dist/
 
-# Autogenerated docs output
-/docs/api
-
 node_modules
 
 /factorio*
@@ -35,6 +32,7 @@ node_modules
 /plugin-list.json
 /config-*.json
 /config-*.json.lock
+/npm-operation.lock*
 pnpm-lock.yaml
 mod_cache.json
 .luarc.json

.oxlintrc.json

@@ -0,0 +1,458 @@
+// Formatting rules come from @stylistic as Oxlint does not implement them.
+{
+	"$schema": "./node_modules/oxlint/configuration_schema.json",
+	"plugins": [
+		"node"
+	],
+	"jsPlugins": [
+		"@stylistic/eslint-plugin"
+	],
+	"categories": {
+		"correctness": "off"
+	},
+	"env": {
+		"builtin": true,
+		"node": true
+	},
+	"ignorePatterns": [
+		"**/dist/",
+		"dist/"
+	],
+	"rules": {
+		"@stylistic/indent": [
+			"error",
+			"tab",
+			{
+				"ignoredNodes": [
+					"JSXFragment"
+				],
+				"ignoreComments": true,
+				"SwitchCase": 1
+			}
+		],
+		"accessor-pairs": "error",
+		"array-callback-return": "error",
+		"arrow-body-style": "error",
+		"block-scoped-var": "error",
+		"complexity": [
+			"error",
+			{
+				"max": 25
+			}
+		],
+		"curly": "error",
+		"default-case": "error",
+		"default-case-last": "error",
+		"eqeqeq": "error",
+		"func-name-matching": "error",
+		"func-style": [
+			"error",
+			"declaration",
+			{
+				"allowArrowFunctions": true
+			}
+		],
+		"grouped-accessor-pairs": "error",
+		"guard-for-in": "error",
+		"id-match": "error",
+		"max-depth": "error",
+		"max-nested-callbacks": "error",
+		"new-cap": [
+			"error",
+			{
+				"capIsNewExceptions": [
+					"StringEnum",
+					"StringKey"
+				],
+				"capIsNewExceptionPattern": "^Type\\.",
+				"newIsCapExceptions": [
+					"constructor"
+				]
+			}
+		],
+		"no-alert": "error",
+		"no-array-constructor": "error",
+		"no-caller": "error",
+		"no-console": "error",
+		"no-constant-condition": [
+			"error",
+			{
+				"checkLoops": false
+			}
+		],
+		"no-constructor-return": "error",
+		"no-div-regex": "error",
+		"no-duplicate-imports": "error",
+		"no-else-return": "error",
+		"no-eq-null": "error",
+		"no-eval": "error",
+		"no-extend-native": "error",
+		"no-extra-bind": "error",
+		"no-extra-label": "error",
+		"no-implicit-coercion": "error",
+		"no-implicit-globals": "error",
+		"no-implied-eval": "error",
+		"no-iterator": "error",
+		"no-label-var": "error",
+		"no-lone-blocks": "error",
+		"no-lonely-if": "error",
+		"no-loop-func": "error",
+		"no-multi-assign": "error",
+		"no-multi-str": "error",
+		"no-nested-ternary": "error",
+		"no-new": "error",
+		"no-new-func": "error",
+		"no-new-wrappers": "error",
+		"no-plusplus": [
+			"error",
+			{
+				"allowForLoopAfterthoughts": true
+			}
+		],
+		"no-proto": "error",
+		"no-restricted-globals": "error",
+		"no-restricted-imports": "error",
+		"no-restricted-properties": "error",
+		"no-return-assign": "error",
+		"no-script-url": "error",
+		"no-self-compare": "error",
+		"no-sequences": "error",
+		"no-shadow": [
+			"error",
+			{
+				"allow": [
+					"Event",
+					"Request",
+					"yargs"
+				]
+			}
+		],
+		"no-template-curly-in-string": "error",
+		"no-throw-literal": "error",
+		"no-unmodified-loop-condition": "error",
+		"no-unneeded-ternary": "error",
+		"no-unused-expressions": "error",
+		"no-useless-backreference": "error",
+		"no-useless-call": "error",
+		"no-useless-computed-key": "error",
+		"no-useless-concat": "error",
+		"no-useless-rename": "error",
+		"no-useless-return": "error",
+		"no-var": "error",
+		"no-void": [
+			"error",
+			{
+				"allowAsStatement": true
+			}
+		],
+		"prefer-arrow-callback": "error",
+		"prefer-exponentiation-operator": "error",
+		"prefer-numeric-literals": "error",
+		"prefer-object-spread": "error",
+		"prefer-promise-reject-errors": "error",
+		"prefer-regex-literals": "error",
+		"prefer-rest-params": "error",
+		"prefer-spread": "error",
+		"prefer-template": "error",
+		"radix": [
+			"error",
+			"always"
+		],
+		"symbol-description": "error",
+		"unicode-bom": [
+			"error",
+			"never"
+		],
+		"vars-on-top": "error",
+		"yoda": [
+			"error",
+			"never",
+			{
+				"exceptRange": true
+			}
+		],
+		"node/no-sync": [
+			"error",
+			{
+				"ignores": [
+					"TechnologySync",
+					"techsToSync"
+				]
+			}
+		],
+		"@stylistic/array-bracket-newline": "off",
+		"@stylistic/array-bracket-spacing": [
+			"error",
+			"never"
+		],
+		"@stylistic/array-element-newline": "off",
+		"@stylistic/arrow-parens": "off",
+		"@stylistic/arrow-spacing": [
+			"error",
+			{
+				"after": true,
+				"before": true
+			}
+		],
+		"@stylistic/block-spacing": "error",
+		"@stylistic/brace-style": [
+			"error",
+			"1tbs",
+			{
+				"allowSingleLine": true
+			}
+		],
+		"@stylistic/comma-dangle": [
+			"error",
+			{
+				"arrays": "always-multiline",
+				"objects": "always-multiline",
+				"imports": "always-multiline",
+				"exports": "always-multiline",
+				"functions": "only-multiline",
+				"generics": "only-multiline",
+				"enums": "always-multiline",
+				"tuples": "always-multiline"
+			}
+		],
+		"@stylistic/comma-spacing": "error",
+		"@stylistic/comma-style": [
+			"error",
+			"last"
+		],
+		"@stylistic/computed-property-spacing": [
+			"error",
+			"never"
+		],
+		"@stylistic/dot-location": [
+			"error",
+			"property"
+		],
+		"@stylistic/eol-last": "error",
+		"@stylistic/function-call-argument-newline": "off",
+		"@stylistic/function-paren-newline": "off",
+		"@stylistic/generator-star-spacing": [
+			"error",
+			{
+				"before": false,
+				"after": true
+			}
+		],
+		"@stylistic/implicit-arrow-linebreak": "error",
+		"@stylistic/jsx-quotes": "error",
+		"@stylistic/key-spacing": "error",
+		"@stylistic/keyword-spacing": "error",
+		"@stylistic/line-comment-position": "off",
+		"@stylistic/linebreak-style": [
+			"error",
+			"unix"
+		],
+		"@stylistic/lines-around-comment": "off",
+		"@stylistic/lines-between-class-members": [
+			"error",
+			"always",
+			{
+				"exceptAfterSingleLine": true
+			}
+		],
+		"@stylistic/max-len": [
+			"error",
+			{
+				"code": 120
+			}
+		],
+		"@stylistic/max-statements-per-line": "off",
+		"@stylistic/multiline-comment-style": [
+			"error",
+			"separate-lines"
+		],
+		"@stylistic/multiline-ternary": "off",
+		"@stylistic/new-parens": "error",
+		"@stylistic/newline-per-chained-call": "off",
+		"@stylistic/no-confusing-arrow": "error",
+		"@stylistic/no-extra-parens": "off",
+		"@stylistic/no-floating-decimal": "error",
+		"@stylistic/no-mixed-operators": [
+			"error",
+			{
+				"groups": [
+					[
+						"&",
+						"|",
+						"^",
+						"~",
+						"<<",
+						">>",
+						">>>"
+					],
+					[
+						"==",
+						"!=",
+						"===",
+						"!==",
+						">",
+						">=",
+						"<",
+						"<="
+					],
+					[
+						"in",
+						"instanceof"
+					]
+				]
+			}
+		],
+		"@stylistic/no-multi-spaces": "error",
+		"@stylistic/no-multiple-empty-lines": "error",
+		"@stylistic/no-tabs": "off",
+		"@stylistic/no-trailing-spaces": "error",
+		"@stylistic/no-whitespace-before-property": "error",
+		"@stylistic/nonblock-statement-body-position": "error",
+		"@stylistic/object-curly-newline": "error",
+		"@stylistic/object-curly-spacing": "off",
+		"@stylistic/one-var-declaration-per-line": "error",
+		"@stylistic/operator-linebreak": [
+			"error",
+			"after",
+			{
+				"overrides": {
+					"?": "before",
+					":": "before",
+					"||": "before",
+					"&&": "before",
+					"??": "before",
+					"|": "before",
+					"&": "before"
+				}
+			}
+		],
+		"@stylistic/padded-blocks": "off",
+		"@stylistic/padding-line-between-statements": "error",
+		"@stylistic/quote-props": "off",
+		"@stylistic/quotes": [
+			"error",
+			"double",
+			{
+				"avoidEscape": true
+			}
+		],
+		"@stylistic/rest-spread-spacing": [
+			"error",
+			"never"
+		],
+		"@stylistic/semi": "error",
+		"@stylistic/semi-spacing": "error",
+		"@stylistic/semi-style": "off",
+		"@stylistic/space-before-blocks": "error",
+		"@stylistic/space-before-function-paren": "off",
+		"@stylistic/space-in-parens": [
+			"error",
+			"never"
+		],
+		"@stylistic/space-infix-ops": "off",
+		"@stylistic/space-unary-ops": "error",
+		"@stylistic/spaced-comment": [
+			"error",
+			"always",
+			{
+				"line": {
+					"markers": [
+						"/"
+					]
+				}
+			}
+		],
+		"@stylistic/switch-colon-spacing": "error",
+		"@stylistic/template-curly-spacing": [
+			"error",
+			"never"
+		],
+		"@stylistic/template-tag-spacing": "error",
+		"@stylistic/wrap-iife": "error",
+		"@stylistic/wrap-regex": "off",
+		"@stylistic/yield-star-spacing": "error"
+	},
+	"overrides": [
+		{
+			"files": [
+				"{packages,plugins}/**/*.{ts,tsx}"
+			],
+			"rules": {
+				"no-unused-vars": "off",
+				"typescript/ban-ts-comment": "error",
+				"typescript/no-duplicate-enum-values": "error",
+				"typescript/no-empty-object-type": "error",
+				"typescript/no-explicit-any": "off",
+				"typescript/no-extra-non-null-assertion": "error",
+				"typescript/no-misused-new": "error",
+				"typescript/no-namespace": "error",
+				"typescript/no-non-null-asserted-optional-chain": "error",
+				"typescript/no-require-imports": "off",
+				"typescript/no-this-alias": "error",
+				"typescript/no-unnecessary-type-constraint": "error",
+				"typescript/no-unsafe-declaration-merging": "error",
+				"typescript/no-unsafe-function-type": "error",
+				"typescript/no-wrapper-object-types": "error",
+				"typescript/prefer-as-const": "error",
+				"typescript/prefer-namespace-keyword": "error",
+				"typescript/triple-slash-reference": "off",
+				"no-shadow": [
+					"error",
+					{
+						"allow": [
+							"Event",
+							"Request",
+							"yargs"
+						]
+					}
+				],
+				"no-useless-constructor": "error"
+			},
+			"plugins": [
+				"typescript"
+			]
+		},
+		{
+			"files": [
+				"test/**/*.js",
+				"plugins/*/test/**/*.js"
+			],
+			"rules": {
+				"prefer-arrow-callback": "off"
+			},
+			"env": {
+				"mocha": true
+			}
+		},
+		{
+			"files": [
+				"packages/web_ui/src/**/*.{ts,tsx}",
+				"packages/lib/browser.ts",
+				"packages/create/templates/*/web/**/*.{ts,js,tsx,jsx}",
+				"{packages,plugins}/*/web/**/*.{ts,tsx}"
+			],
+			"globals": {
+				"SpeechRecognitionAlternative": "readonly",
+				"SpeechRecognitionResult": "readonly",
+				"SpeechRecognitionResultList": "readonly"
+			},
+			"env": {
+				"browser": true
+			}
+		},
+		{
+			"files": [
+				"packages/create/templates/**/*.{js,cjs,ts,cts,jsx,tsx}"
+			],
+			"rules": {
+				"@stylistic/indent": "off",
+				"@stylistic/spaced-comment": "off",
+				"@stylistic/comma-spacing": "off",
+				"@stylistic/max-len": "off"
+			},
+			"jsPlugins": [
+				"@stylistic/eslint-plugin"
+			]
+		}
+	]
+}

AGENTS.md

@@ -0,0 +1,36 @@
+# Working on Clusterio
+
+Clusterio manages distributed Factorio servers through a controller, hosts, and plugins.
+
+Read [contributing](docs/contributing.md) for the project map, setup, commands, debugging, tests, and code style.
+For PR reviews, follow the [review guide](docs/pr-review-guide.md), including its candidate ledger and final evidence audit.
+Use the detailed guides for the affected area:
+
+- [Communication and routing](docs/communication.md), [wire protocol](docs/devs/protocol.md), and [configuration](docs/config-system.md).
+- [Plugins](docs/writing-plugins.md), [plugin migration](docs/devs/plugin-migration-to-2.0.md), and [Factorio/Lua](docs/developing-for-clusterio.md).
+- [Compatibility policies](docs/decisions.md) and [releases](docs/release-workflow.md).
+
+
+## Working rules
+
+- Inspect `git status` before editing.
+  Preserve unrelated work and external plugin repositories; continue an existing task branch.
+- Read the relevant source and tests before changing behavior.
+  If documentation and implementation disagree, report the discrepancy and distinguish observed behavior from intended policy.
+  Update affected documentation in the same change.
+- Follow nearby code and `.oxlintrc.json`: tabs, double quotes, semicolons, and `.js` extensions in relative TypeScript imports.
+  Import shared public APIs through `@clusterio/lib` and check Node/browser exports when changing them.
+- Run commands from the root using the pinned pnpm version and the [documented Node requirements](docs/contributing.md#supported-nodejs-and-factorio-versions).
+  Install with `pnpm install --no-frozen-lockfile`; the lockfile is intentionally ignored.
+- `pnpm build` compiles TypeScript.
+  It does not regenerate validators or production web bundles; use the appropriate package's `prepare` script.
+  Edit sources, not generated outputs.
+- Start validation with focused tests, then run `pnpm test` and `pnpm lint` for code changes when possible.
+  Root tests include integration setup even with fast/external exclusions.
+  The harness needs Factorio, recreates `temp/test/`, and shares ports 8880/4443; do not run competing suites.
+- Report checks, failures, skips, and unverified behavior precisely.
+  Documentation-only changes need path, command, and formatting checks.
+- Preview cleanup with `node clean.js --dry-run`; see the [cleanup scopes](docs/contributing.md#cleaning-generated-and-runtime-files).
+  Runtime-data deletion and release publication require explicit authorization.
+  Keep local configs, credentials, runtime data, and generated output out of commits.
+- Follow the [PR template](.github/pull_request_template.md), including `### Changelog` (`None.` when appropriate).

CHANGELOG.md

@@ -31,6 +31,74 @@ Many thanks to the following for contributing to this release:
 [@username](https://github.com/username)
 -->
 
+## Version 2.0.3
+
+### Features
+
+- Added per instance options to choose which parts of the player inventory sync covers. [#968](<https://github.com/clusterio/clusterio/issues/968>)
+- Added opt-in UDP messaging between instances and the Factorio server via `factorio.enable_lua_udp`, with a `sendUdp` plugin API, `clusterio_api.send_udp` in-game and a `LuaUdp` plugin feature flag. [[#969](<https://github.com/clusterio/clusterio/issues/969>)](<https://github.com/clusterio/clusterio/issues/969>)
+- Added notes that can be attached to hosts, instances, users, roles, mod packs, mods and saves in the web interface and through ctl. [#903](<https://github.com/clusterio/clusterio/issues/903>)
+- Added tools to inspect, replace, delete and unlock player data stored by inventory sync, in the web interface and as `clusterioctl inventory-sync` commands. [#1003](<https://github.com/clusterio/clusterio/issues/1003>)
+- Notes in the web interface render as Markdown and link `@<type>/<id>` mentions to the resource. [#1000](<https://github.com/clusterio/clusterio/pull/1000>)
+- Added timestamps to log lines in the web interface console. [#1137](<https://github.com/clusterio/clusterio/pull/1137>)
+- Added syncing of quick bar slots that hold a specific blueprint, blueprint book or planner, including ones from the player's blueprint library, on Factorio 2.1. [#1092](<https://github.com/clusterio/clusterio/issues/1092>)
+- Added a `set_destination` remote interface to Player Sync. A plugin that moves a player to another instance calls it before `connect_to_server`, and the player's data is sent to that instance while the client reconnects, so the join applies it without a download. [#1099](<https://github.com/clusterio/clusterio/issues/1099>)
+- Added a cluster research queue to research_sync: servers can follow one research queue, set from the research screen on any following server, with a checkbox on the lab window. [#1115](<https://github.com/clusterio/clusterio/pull/1115>)
+
+### Fixes
+
+- Fixed instances that start at the same time each downloading a new Factorio version into the same files and leaving a corrupt install. [[#1105](<https://github.com/clusterio/clusterio/issues/1105>)](<https://github.com/clusterio/clusterio/issues/1105>)
+- Fixed `controller.max_send_buffer_size` counting buffered messages in UTF-16 code units instead of UTF-8 bytes, which let non-ASCII messages go over the limit. [[#1108](<https://github.com/clusterio/clusterio/issues/1108>)](<https://github.com/clusterio/clusterio/issues/1108>)
+- Removed incompatible Server Select and Discord Bridge releases from the installer's plugin choices so selecting all offered plugins no longer installs them. [#1102](<https://github.com/clusterio/clusterio/issues/1102>)
+- Fixed reading map exchange strings from Factorio 2.1.18 and later, which gave wrong enemy evolution and expansion settings and made `instance load-scenario --map-exchange-string` fail. [[#1131](<https://github.com/clusterio/clusterio/issues/1131>)](<https://github.com/clusterio/clusterio/issues/1131>)
+- Fixed Player Sync erroring when the controller returned no data for a downloading player, which left the player waiting for the timeout. [#1094](<https://github.com/clusterio/clusterio/issues/1094>)
+- Fixed spoilable items arriving fresh after being serialized, the spoil percent is now kept. [#1093](<https://github.com/clusterio/clusterio/issues/1093>)
+- Fixed named personal logistic groups losing their requests and trash rules when joining a server where the group was missing or empty. [#1091](<https://github.com/clusterio/clusterio/issues/1091>)
+- Fixed Player Sync leaving quick bar slots from an earlier visit in place, which made moved or cleared slots come back after changing server. [#1092](<https://github.com/clusterio/clusterio/issues/1092>)
+- Fixed research_sync sending research it received from the controller back to it, so every instance got the research again each time an instance started. [#1110](<https://github.com/clusterio/clusterio/issues/1110>)
+- Fixed items in main inventory slots from armor equipment such as toolbelts not fitting in the inventory when joining another server. [#1100](<https://github.com/clusterio/clusterio/issues/1100>)
+
+### Changes
+
+- Replaced jszip with yauzl and yazl so saves and mods are no longer loaded fully into memory when patched, read or exported. [#1002](<https://github.com/clusterio/clusterio/issues/1002>)
+- Bumped dependency versions, including typebox 0.34 and react-router 7. Plugins using `@sinclair/typebox` should update to ^0.34. [#1113](<https://github.com/clusterio/clusterio/pull/1113>)
+
+### Contributors
+
+Many thanks to the following for contributing to this release:
+[@psihius](<https://github.com/psihius>), [@ILLISIS](<https://github.com/ILLISIS>), [@bbassie](<https://github.com/bbassie>), [@ColumbusLabs](<https://github.com/ColumbusLabs>)
+
+### Statistics
+
+Commits: 101  Pull Requests: 29  Files Changed: 135  Additions: +8,830  Deletions: -994
+
+## Version 2.0.2
+
+### Fixes
+
+- Fixed installation with selected plugins by relying on automatic plugin discovery, and ensured installer failures return a nonzero exit status. [#1082](<https://github.com/clusterio/clusterio/pull/1082>), [#1084](<https://github.com/clusterio/clusterio/issues/1084>)
+- Fixed web plugin `controllerConnectionEvent` handlers receiving the connector's event data instead of the event name. [#1083](<https://github.com/clusterio/clusterio/issues/1083>)
+- Updated installation instructions to recommend latest Node.js LTS and corrected outdated mod management, CLI, plugin, and release documentation. [#1086](<https://github.com/clusterio/clusterio/pull/1086>)
+
+### Meta
+
+- Made player-auth login tests independent of wall-clock timing and added verification-token expiry coverage. [#1087](<https://github.com/clusterio/clusterio/pull/1087>)
+- Added Node.js 26 to the CI test matrix across all tested Factorio versions. [#1088](<https://github.com/clusterio/clusterio/pull/1088>)
+
+Many thanks to the following for contributing to this release:
+[@psihius](<https://github.com/psihius>), [@bbassie](<https://github.com/bbassie>)
+
+## Version 2.0.1
+
+### Fixes
+
+- Removed obsolete development-branch warnings from controller, host and ctl startup. [#1078](<https://github.com/clusterio/clusterio/pull/1078>)
+- Removed the obsolete alpha label and development warning from the Clusterio Library mod title and description. [#1079](<https://github.com/clusterio/clusterio/pull/1079>)
+- Fixed Factorio mod release uploads to use valid multipart requests and files from the requested output directory, uploading variants in version order. [#1080](<https://github.com/clusterio/clusterio/pull/1080>)
+
+Many thanks to the following for contributing to this release:
+[@psihius](<https://github.com/psihius>)
+
 ## Version 2.0.0
 
 ### Features
@@ -662,7 +730,7 @@ Many thanks to the following for contributing to this release:
 ## Version 2.0.0-alpha.14
 
 This release does not retain backwards compatibility with 2.0.0-alpha.13 meaning all parts of the cluster needs to updated at the same time.
-See the [migration guide](/docs/migrating-from-2.0.0-alpha.13.md) for how to update.
+See the [migration guide](https://github.com/clusterio/clusterio/blob/e0755cb4ee7bce0068c844ab6271da481a843e62/docs/migrating-from-2.0.0-alpha.13.md) for how to update.
 
 ### Major Features
 

README.md

@@ -11,6 +11,7 @@ Discord for development/support/play: https://discord.gg/5XuDkje
 ## Important notice
 
 Clusterio 2.0 is the current stable release and requires Node.js 22 or newer.
+For new installations, use the latest Node.js LTS release (see [Installation](#installation)).
 The previous stable series is no longer supported.
 If you are upgrading from a 2.0 alpha, review the [changelog](CHANGELOG.md) before upgrading.
 Plugin authors should also follow the [2.0 migration guide](docs/devs/plugin-migration-to-2.0.md).
@@ -41,7 +42,7 @@ Plugin authors should also follow the [2.0 migration guide](docs/devs/plugin-mig
 * [Setting up remote hosts](#setting-up-remote-hosts)
 * [Setting up clusterioctl](#setting-up-clusterioctl)
 * [Known issues](#known-issues)
-* [Common problems](#Common-problems)
+* [Common problems](#common-problems)
 
 
 ## Introduction
@@ -72,7 +73,7 @@ Plugins add functionality to Factorio servers, Clusterio itself or both.
 These are the plugins supported and maintained by the Clusterio developers:
 
 - [Global Chat](/plugins/global_chat/README.md): share the in-game chat between servers.
-- [Research Sync](/plugins/research_sync/README.md): synchronize research progress and technologies unlocked between servers.
+- [Research Sync](/plugins/research_sync/README.md): synchronize research progress and technologies unlocked and the research queue between servers.
 - [Statistics Exporter](/plugins/statistics_exporter/README.md): collect in-game statistics from all the servers and makes it available to the Prometheus endpoint on the controller.
 - [Subspace Storage](https://github.com/clusterio/subspace_storage): Provide shared storage that can transport items between servers via teleport chests.
 - [Player Auth](/plugins/player_auth/README.md): Provides authentication to the cluster via logging into a Factorio server.
@@ -88,6 +89,9 @@ There's also plugins developed and maintained by the community:
   See repo readme for plugin breakdown. Maintained by Cooldude2606.
 - [Chat Relay](https://github.com/PHIDIAS0303/chat_relay) (@PHIDIAS0303/chat_relay): Chat relay of Discord and Factorio, seperate channel.
 
+The installer offers the six plugins maintained in this repository.
+Community plugins are versioned and released independently of Clusterio; check their compatibility before installing them manually.
+
 Want to make your own plugin?
 Check out the documentation on [Writing Plugins](/docs/writing-plugins.md) for where to start.
 If you have a plugin written for an earlier 2.0 alpha, see [Migrating Plugins to 2.0](/docs/devs/plugin-migration-to-2.0.md).
@@ -96,8 +100,9 @@ Note that from 2.0 onwards minor versions of Clusterio may contain breaking chan
 
 ## Installation
 
-Clusterio runs on all LTS version of NodeJS, it is distributed via npm and comes with a guided install script.
-If you already have a recent Node.js installed, you can set it up in a new directory with:
+Install the latest [Node.js LTS release](https://nodejs.org/en/download), which is recommended for new Clusterio installations.
+Clusterio is distributed via npm and comes with a guided install script.
+If you already have a supported Node.js version installed, you can set it up in a new directory with:
 
     npm init "@clusterio"
 
@@ -106,12 +111,14 @@ Check out all our of decisions for [node, factorio, and operation support](/docs
 
 ### Ubuntu setup
 
-1.  Install Node.js v18 or higher.
-    Note the version of Node.js provided by the Ubuntu/debian repos are too old and you will have to use the nodesource PPA
+1.  Install the latest Node.js LTS release using the NodeSource repository for Ubuntu/Debian.
+    The distribution's default repository may provide an older version.
 
-        curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - &&\
+        curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - &&\
         sudo apt-get install -y nodejs
 
+    Verify the installation with `node --version` and `npm --version`.
+
 2.  Log in as the user you wish to install Clusterio as (do not use root), and then create a new directory and run the Clusterio installer:
 
         mkdir clusterio
@@ -143,7 +150,7 @@ Your instances (save files etc) will be stored there.
 
 ### Windows setup
 
-1.  Download and install the latest LTS release from http://nodejs.org.
+1.  Download and install the latest [Node.js LTS release](https://nodejs.org/en/download).
 
 2.  Create a new empty directory for the installation and navigate into it.
     Open a PowerShell window in this new directory by Shift+Right clicking inside it and choosing "Open PowerShell window here" and then run the following command.
@@ -157,7 +164,8 @@ Your instances (save files etc) will be stored there.
 
 ### MacOS Setup
 
-1.  Install the latest Node.js LTS release from http://nodejs.org or use brew (`brew install node`).
+1.  Install the latest [Node.js LTS release](https://nodejs.org/en/download).
+    If you use [nvm](https://github.com/nvm-sh/nvm#installing-and-updating), run `nvm install --lts`.
 
 2.  Open Terminal or Command prompt in the directory you want to install to and run the following commands.
 
@@ -211,7 +219,11 @@ See the [readme for @clusterio/controller](/packages/controller/README.md) for m
 
 ## Managing Factorio mods
 
-Mods in clusterio are managed as parts of modpacks. There is currently no mod portal integration, so you first need to navigate to the mods section of the web UI and upload your mods. You can then create a modpack from the uploaded mods by navigating to the mods page, hitting `create`. Once you have a modpack, you can assign it to an instance under Instance config -> Factorio -> Mod pack. You can also set a default modpack for new instances in the controller settings.
+Mods in Clusterio are managed as parts of mod packs.
+On the Mods page of the web interface, use **Search** to browse the Factorio Mod Portal and download mods to the controller, or upload mod zip files directly.
+Mod Portal downloads require the controller's [Factorio account credentials](docs/configuration.md#controllerfactorio_username) to be configured.
+Use **Create** in the Mod Packs section to create a mod pack, then add the mods you want to use.
+Assign the mod pack to an instance under Instance config -> Factorio -> Mod pack, or set a default mod pack for new instances in the controller settings.
 
 ![Select mod pack for instance](docs/assets/config_select_mod_pack.png)
 
@@ -227,7 +239,9 @@ Once this is done, the mod settings will show up under the modpack configuration
 ## Setting up shared storage
 
 The chests teleporting items to and from the shared storage on the controller has been moved into the Subspace Storage mod and plugin.
-To get this to work you will need to install the Subspace Storage plugin and copy the [Subspace Storage mod](https://mods.factorio.com/mod/subspace_storage) and the [Clusterio Lib mod](https://mods.factorio.com/mod/clusterio_lib) into the `sharedMods` folder on each host installation.
+To get this to work, install the Subspace Storage plugin on the controller and hosts using the [plugin installation instructions](#installing-plugins).
+Download or upload the [Subspace Storage mod](https://mods.factorio.com/mod/subspace_storage) and the [Clusterio Lib mod](https://mods.factorio.com/mod/clusterio_lib) to the controller, add them to a mod pack, and assign it to each participating instance as described in [Managing Factorio mods](#managing-factorio-mods).
+Hosts fetch the mods from the controller and synchronize each instance's mods folder with its assigned mod pack when the instance starts.
 
 Do not install the [Clusterio mod](https://mods.factorio.com/mod/clusterio), this was for the old 1.2.x version of Clusterio and is not compatible with Clusterio 2.0.
 
@@ -237,6 +251,7 @@ Do not install the [Clusterio mod](https://mods.factorio.com/mod/clusterio), thi
 After completing the installation and creating your admin user, start up the controller and at least one host separately.
 The installer provides the `run-controller` and `run-host` scripts to make this simple.
 Once the controller process is running you can log into the web interface which is hosted by default on http://localhost:8080/ (adjust the port number if you changed it), use the admin authentication token provided from the installation to log in.
+The first time a host starts it prints a link for approving it, open the link and click Approve.
 
 The basics of setting up a Factorio server from the web interface is to create an instance, assign it to a host and then click start. You can also use the "Connect via Steam" button to launch Factorio directly and connect to the server using Steam's protocol handler.
 
@@ -258,9 +273,23 @@ If you have copied these files over to `/etc/systemd/system/` then you can start
 
 ## Setting up remote hosts
 
-Run the installer as described in the installation section and choose "Host only" as the operating mode to install, it'll ask for a controller URL and an authentication token.
-The URL is the same as what is needed to connect to the web interface, and the authentication token can be generated on the Hosts page in the web interface.
-Once you start up the host it should show up in the Hosts list and be available for assigning and running instances on.
+Run the installer as described in the installation section and choose "Host only" as the operating mode to install, it'll ask for a name for the host and a controller URL.
+The URL is the same as what is needed to connect to the web interface.
+Then start the host, the first time it starts it prints a code and waits for the host to be approved:
+
+    Waiting for approval of "my-host" with code ABC-DEF, open this link to approve it:
+    https://example.com/hosts?approve=ABC-DEF
+    Or run: npx clusterioctl host approve ABC-DEF
+
+Open the link to approve it in the web interface, where you can also change its name, or click Add host on the Hosts page and approve the request with the same code, or run `npx clusterioctl host approve <code>`.
+The host then saves its connection details and starts up, it should show up in the Hosts list and be available for assigning and running instances on.
+Only approve a request if its code matches the one shown by the host, approving gives the host full access to the cluster.
+Approving requires the `core.host.approve` permission, which the Cluster Admin role has.
+If you have that permission you can skip the wait by starting the host with `npx clusteriohost run --user-token <token>` and your own authentication token, it is only used to approve the host and is not saved.
+
+To remove a host from the cluster, click Revoke host access on the page for the host in the web interface, or run `npx clusterioctl host revoke <host>`.
+This disconnects the host, its instances become unassigned and its saved connection details stop working.
+To add the computer back to the cluster start the host again, it asks for approval again.
 
 
 ## Setting up clusterioctl
@@ -275,20 +304,13 @@ If you want to use a different user for the command line interface, you can gene
 
 ## Known Issues
 
-- When the `server_select` plugin is installed you may see `[error] Unhandled event server_select:UpdateInstancesEvent` logged when starting an instance.
-  This should be ignored as it does not indicate an error occured.
+- `@hornwitser/server_select@2.0.1` fails to load with Clusterio 2.0 because its entrypoints omit the required `.js` extension.
+  `@hornwitser/discord_bridge@0.2.5` is not automatically discovered because it lacks the `clusterio-plugin` keyword.
+  The Clusterio 2.0.2 installer still offers these versions; leave them unchecked until compatible plugin releases are available.
+  If already installed, stop the controller and host, run `npm uninstall @hornwitser/server_select @hornwitser/discord_bridge` in each affected installation directory, and restart them to continue with the core plugins.
+  See [#1102](https://github.com/clusterio/clusterio/issues/1102).
 - After updating Factorio you may get the error `Error: Expected empty response but got "Cannot execute command. Error: [string "clusterio_private.update_instance(..."]:1: attempt to index global 'clusterio_private' (a nil value)?` on instance start.
   This is due to Factorio applying a migration replacing the patched content of the save with the original scenario, attempting to start the instance again should resolve the issue.
-- In the mod pack view of the web interface the "base" mod is displayed with an error complaining about the mod being missing from the controller storage.
-  This is due to the "base" mod currently not being connectly handled as a built-in mod to Factorio in the web interface, this error message should be ignored.
-- After migrating from Alpha 13 to Alpha 14 running an instance data export fails with the error `Upload failed: 401 Unauthorized` due to the host token being issued to a slave.
-  To workaround this find the id of the host in the host config and generate a new host token with the same id.
-  Stop the host and replace the value of the `host.controller_token` field in `config-host.json` with the new host token before starting it up again.
-- After migrating from Alpha 13 to Alpha 14 you get kicked out from the web interface due to the master to controller rename.
-  If you don't have the authentication token readily available you can manually migrate the token by opening the Inspect/Developer Tools on the page and running the following piece of JavaScript in the Console:
-  ```
-  localStorage.controller_token = localStorage.master_token
-  ```
 
 
 ## Common problems

clean.js

@@ -49,7 +49,7 @@ async function removeBuildArtifacts() {
 				tryRemove(path.join(basePath, packageName, "dist"))
 			);
 			tasks.push(
-				tryRemove(path.join(basePath, packageName, "node_modules", ".cache"))
+				tryRemove(path.join(basePath, packageName, "node_modules", ".vite"))
 			);
 		}
 	}

docs/assets/clusterio_2.0_block_diagram.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2023-07-15T21:19:41.336Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/20.8.16 Chrome/106.0.5249.199 Electron/21.4.0 Safari/537.36" etag="WQippoIg5xK5LnRPeLWb" version="20.8.16" type="device"><diagram id="wKlty5uROKaaHdmDaOKw" name="Page-1">5Vtdk6I4FP01Ps4UEEB9bO12Pmp3qmvdqnlOQ4R0R+KE2Or++k0kgBBEelTQoR9suFwgOeee5N4YB2C63H5hcBX+TX1EBpbhbwfgcWBZznAsPqVhlxhsABJDwLCfmMzcMMf/IWU0lHWNfRQXHDmlhONV0ejRKEIeL9ggY3RTdFtQUnzrCgZIM8w9SHTrT+zzUHXLMHL7V4SDkJcuLGHqqwxxCH26OTCBpwGYMkp5crTcThGR0KWwJPfNjlzN2sVQxJvc4Lz9M3Hs7zvzZR7gHwEh3/59/OQodt4hWasOq9byXYoAo+vIR/IpxgBMNiHmaL6Cnry6EZQLW8iXRJyZ4jDrpfQNCIxjdRy/Ie6F6mSBCZlSQpk4j2gkHjWBBAeROCVowaU7Z/QNlXx8zATFmEq/mK4lwBMdBoXMO2IcbQ9MCpYviC4RZzvhsqkgNKwiE6ogCrJ7c6DFgcL6A7hnjWyEu3ka90NEBxbwIRotPAkZjMPsGUVQhZvrjdDLojFtK8Sw6D9ish04CpQZMk/p1j2TEHU1lYwaMobq9ICuzOWQLmBdiy5XY2tKIwEmIYgJ+zOjHhKQnaWcEoOG4RhoUUWaYZiPk6m8Q7ThwL7Y/12GAbvIwFhnwDSrGLgWAeZYAxf5YpxWp5TxkAY0guQpt06K8Oc+f1G6UqC/Is53KnjhmtNaSswXaCKrmhL36WFWB70YrpiHanqoOsghCxA/HYqy97VEMkQgx+/FWayKl/2tD4zB3YHDiuKIxwdPfpaGPD7cYnyYdmnu+ZC7OEjen8dH1pEzhlj3BkJmAd2RYVSFzHQInP0VFPkPMk/JpzlhmWHZ2X0jXtfLVdpo5fD7UWaBhmE2vokwGxXjxho6LcQN0Ab7GfQ4ZZgK6xyxdznmX3CkbxAjVxzps3E8xdjteqy3bmGsv4Bwz1CpfVcqHRYjCJhtqNTutUqB0blKR7ek0lNaPE7aGSp17kqlpRwMOG2o1Om3Su2uVQp0Amrw7t06Q7kuGTVcachWJy9OmG3fwLB61UIXuA2HzduodLOZNo0Rt77UHTt17tcZZsHwBoKm04wZjBoGFbiRqLI6qGxTkHoyG4s3FDHuvLIFva9s04TobnRazujaqG1TkHqq0+5rW9vU4O2bTq0706nRQXWbgtRXnXZe3dq38BXPVYslu+ka08VleB4x+rLDT/QiDN8iUdUvZLjLBHgmPj2yjoUNU4+Ti4qlAS9XFEv5q6uqb9DdVrVStdPHJVzBUMDd/bWm6YVP8V4HD8JBQLLNL4qjQP6fpgRKj8+i+caEUO9N/H/EMGBwmb5GNDt5U3KfRrZAlhcZrdwEdHrvkGQJe5A8KPMS+/5e+lUBVAyxZutTlxhMS7VJxcLTuM3wcP7MDUnWhegyh26BLxvohDkVfF1tR5Kj5x7P4S6WYX889+izwqoYq8xWrkeZ/pWlnr7kGb0nscFeraoabBI7iuTJVOFEaKe2jyX2WuZum6MiT3YJ/yRHUnfVLLza7okHJTmU9qBLVQNOZc4zF1Mhkq1U+5Pl5tVey9IaWqd1CVrVpb67s2bqu7ednNrMVbEK2u7MNdTg/kF99PlVlPdHd9L2SiLlmauCsbHbImNAF8hXGvMauu5XLtqWw84XI/W67Y8F324Pe3Ga//QmSQHyny+Bp/8B</diagram></mxfile>

docs/assets/clusterio_2.0_block_diagram.svg

@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg
+  PUBLIC '-//W3C//DTD SVG 1.1//EN'
+  'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd'>
+<!-- Exported from clusterio_2.0_block_diagram.drawio with draw.io Desktop 32.4.1. Export settings: SVG, light theme, white background, width 1000. Native SVG labels: disable Formatted Text and Word Wrap in an export copy, preserving the title at font size 20. -->
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="background: #ffffff; background-color: #ffffff; color-scheme: light;" version="1.1" width="1001px" height="1001px" viewBox="0 0 1001 1001" role="img" aria-labelledby="diagram-title diagram-description">
+  <desc id="diagram-description">A controller and two hosts run across two physical servers. Each host manages three Factorio servers. The hosts and web or command-line clients connect to the controller over WebSocket.</desc>
+  <title id="diagram-title">Clusterio 2.0 system architecture</title>
+  <defs/>
+  <rect fill="#ffffff" width="100%" height="100%" x="0" y="0" style="fill: rgb(255, 255, 255);"/>
+  <g>
+    <g data-cell-id="0">
+      <g data-cell-id="1">
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-59">
+          <g>
+            <rect x="0" y="0" width="1000" height="1000" fill="none" stroke="none" transform="rotate(90,500,500)" pointer-events="all"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-21">
+          <g>
+            <rect x="40" y="140" width="440" height="640" rx="26.4" ry="26.4" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-dasharray="6 6" pointer-events="all" style="fill: rgb(218, 232, 252); stroke: rgb(108, 142, 191);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-6">
+          <g>
+            <rect x="80" y="180" width="220" height="60" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="190" y="220">Controller Process</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-19">
+          <g>
+            <path d="M 120 400 L 120 280 L 120 252.74" fill="none" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 110, 175);"/>
+            <path d="M 120 242.24 L 127 256.24 L 120 252.74 L 113 256.24 Z" fill="#006eaf" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 110, 175); stroke: rgb(0, 110, 175);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-26">
+          <g>
+            <path d="M 220 550 L 160 550 L 160 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-23">
+          <g>
+            <rect x="220" y="520" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="330" y="560">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-29">
+          <g>
+            <path d="M 220 630 L 140 630 L 140 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-24">
+          <g>
+            <rect x="220" y="600" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="330" y="640">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-28">
+          <g>
+            <path d="M 220 710 L 120 710 L 120 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-25">
+          <g>
+            <rect x="220" y="680" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="330" y="720">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-35">
+          <g>
+            <rect x="520" y="360" width="440" height="420" rx="25.2" ry="25.2" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-dasharray="6 6" pointer-events="all" style="fill: rgb(218, 232, 252); stroke: rgb(108, 142, 191);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-44">
+          <g>
+            <path d="M 600 400 L 600 320 L 190 320 L 190 252.74" fill="none" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 110, 175);"/>
+            <path d="M 190 242.24 L 197 256.24 L 190 252.74 L 183 256.24 Z" fill="#006eaf" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 110, 175); stroke: rgb(0, 110, 175);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-37">
+          <g>
+            <path d="M 700 550 L 640 550 L 640 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-38">
+          <g>
+            <rect x="700" y="520" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="810" y="560">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-39">
+          <g>
+            <path d="M 700 630 L 620 630 L 620 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-40">
+          <g>
+            <rect x="700" y="600" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="810" y="640">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-41">
+          <g>
+            <path d="M 700 710 L 600 710 L 600 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-42">
+          <g>
+            <rect x="700" y="680" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="810" y="720">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-46">
+          <g>
+            <path d="M 560 210 L 312.74 210" fill="none" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 110, 175);"/>
+            <path d="M 302.24 210 L 316.24 203 L 312.74 210 L 316.24 217 Z" fill="#006eaf" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 110, 175); stroke: rgb(0, 110, 175);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-45">
+          <g>
+            <rect x="560" y="180" width="320" height="60" fill="#1ba1e2" stroke="#006eaf" stroke-width="2" pointer-events="all" style="fill: rgb(27, 161, 226); stroke: rgb(0, 110, 175);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="720" y="220">Web Interface / clusterioctl</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-49">
+          <g>
+            <rect x="60" y="40" width="580" height="60" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="40px" style="fill: rgb(0, 0, 0);">
+              <text x="64" y="88">Clusterio 2.0 Block Diagram</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-51">
+          <g>
+            <rect x="352" y="860" width="100" height="40" rx="8" ry="8" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-dasharray="6 6" pointer-events="all" style="fill: rgb(218, 232, 252); stroke: rgb(108, 142, 191);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-52">
+          <g>
+            <rect x="60" y="860" width="220" height="40" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="24px" style="fill: rgb(0, 0, 0);">
+              <text x="64" y="890">Physical Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-54">
+          <g>
+            <path d="M 836 880 L 923.26 880" fill="none" stroke="#001dbc" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 29, 188);"/>
+            <path d="M 933.76 880 L 919.76 887 L 923.26 880 L 919.76 873 Z" fill="#001dbc" stroke="#001dbc" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 29, 188); stroke: rgb(0, 29, 188);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-55">
+          <g>
+            <rect x="544" y="860" width="260" height="40" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="24px" style="fill: rgb(0, 0, 0);">
+              <text x="548" y="890">WebSocket connection</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-56">
+          <g>
+            <rect x="352" y="920" width="100" height="40" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-57">
+          <g>
+            <rect x="60" y="920" width="192" height="40" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="24px" style="fill: rgb(0, 0, 0);">
+              <text x="64" y="950">Node.js Process</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-36">
+          <g>
+            <rect x="560" y="400" width="220" height="60" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="670" y="440">Host Process</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-9">
+          <g>
+            <rect x="80" y="400" width="220" height="60" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="190" y="440">Host Process</text>
+            </g>
+          </g>
+        </g>
+      </g>
+    </g>
+  </g>
+</svg>

docs/communication.md

@@ -1,36 +1,58 @@
 # Communication Layer
 
-<sub>This document describes the communication layer in Clusterio and unless you're developing Clusterio it's probably not going to be very useful for you.</sub>
-
-Communication is facilitated by the lib/link library and is based on three core abstractions: links, connectors and messages.
-Links represents one side of a two way communication pipe, connectors deal with connecting and establishing communication, and messages are the content sent over the link.
+Clusterio uses links, connectors, and message classes to communicate between controllers, hosts, instances, and control clients.
+A link sends and receives messages through a connector, while message classes define the requests and events that can be exchanged.
+See the [system overview](how-it-works.md#system-overview) for a diagram of the processes and their connections, and the [protocol reference](devs/protocol.md) for wire formats and session flow.
 
 
-## Link
+## Links and routing
 
-The Link class represents one side of a two way communication channel where the actual communication is done by a connector associated with the link.
-The link also knows what the endpoint types it's connected to are via the source and target attributes.
+[Link](../packages/lib/src/link/link.ts) handles request/response matching, event dispatch, message registration, and routing through an attached router.
+Its connector identifies the local and remote endpoints through `connector.src` and `connector.dst`, both `Address` objects.
 
-The endpoint types currently implemented are controller, host, instance, and control.
-Although host and instances run in the same Node.js program they use virtual links to communicate between them in order to simplify the communication architecture.
+The controller's `WsServer` stores one link for each host or control connection.
+On a host, the `Host` object is a link to the controller, and each `Instance` is linked to a corresponding `InstanceConnection` stored on the host.
+The command-line client's `Ctl` object and the web interface's `Control` object each act as a link to the controller.
 
-Upon creation the link will register with the connector in order to receive and process messages from it.
-The messages received are validated using the validator registered for the type of the message.
-A message for which there is no validator for is considered an error.
+Requests can pass through several links before reaching their destination.
+Each receiving link checks the message name, allowed endpoint types, and source address before routing it.
+The controller checks the user's permissions before validating the message data; permission callbacks therefore receive untrusted data.
+See [message permissions](writing-plugins.md#static-permission).
+
+At the destination, the registered schema validates the payload and `fromJSON` restores the message before its handler runs.
+
+Routing can perform work before the destination handler runs.
+In particular, the [host router](../packages/host/src/Host.ts) can start or wake an instance for messages marked as instance-starting.
 
 
-## Connector
+## Connectors and sessions
 
-The connector used with a link is responsible for establishing and/or maintain the connection that messages are sent over.
-Reconnect logic is expected to be implemented in the connector, though this has not yet been done.
+[Connectors](../packages/lib/src/link/connectors.ts) transport messages, assign sequence numbers, and deliver incoming messages to links.
+WebSocket connectors connect hosts and control clients to the controller and validate incoming message structures, including their type and addresses.
+Hosts and instances share a Node.js process and communicate in memory through paired `VirtualConnector` objects.
 
-There's also the virtual connector which lets two links in the same program be connected to each other without having to establish a network connection.
+The protocol reference describes [connection and recovery behavior](devs/protocol.md#connection-flow), [connector states and events](devs/protocol.md#connector-flow), and [graceful disconnects](devs/protocol.md#graceful-disconnect).
+For handling these events in a plugin, including queueing work and preparing for disconnection, see [connection handling](writing-plugins.md#handling-connection-events).
 
 
-## Message
+## Message definitions and registration
 
-There's two types of messages currently implemented apart from the Link builtin close message and that's Request and Event.
-The builtin messages are defined in [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) and are attached to links via the attachAllMessages function, which is usally called in the Link subclass constructor.
+Core request and event classes live under [packages/lib/src/data/](../packages/lib/src/data/).
+Their static `type`, `src`, and `dst` properties specify the message type and allowed endpoint types, while their schema and permission fields define the payload and access rules.
+Plugin classes also supply a `plugin` name.
 
-The messages define which links they are valid on and only attaches handlers for those links.
-There's also a forwarding mechanism for when a request or event is to be forwarded another hop.
+`Link.register` registers request and event classes and compiles their payload validators.
+Core classes are registered by the link module; `registerPluginMessages` registers each plugin's `messages` array.
+It also registers the plugin's `errors` array with `Link.registerError`, so an error thrown by a request handler is rejected on the requester as the same class.
+Incoming messages invoke handlers attached with `link.handle(MessageClass, handler)`.
+Plugins attach these handlers from their entrypoint functions rather than relying on method-name conventions.
+
+Send a message with `link.sendTo(destination, new MessageClass(...))`.
+Requests return a promise for the response; events do not return a handler result.
+Use broadcast destinations only for events.
+When using a request's source as a later event destination, remove its request ID with `Address.withoutRequestId()`.
+See [writing plugins](writing-plugins.md#defining-link-messages) for a complete example.
+
+When changing a wire message structure in [messages_core.ts](../packages/lib/src/data/messages_core.ts), rebuild the packages with `pnpm -r run prepare` to regenerate the standalone validator.
+When changing a request or event class, keep serialization, allowed endpoints, permission checks, registration, and handlers consistent.
+The [contributor guide](contributing.md#checking-your-code-for-errors) explains how to run focused tests.

docs/config-system.md

@@ -3,18 +3,18 @@
 <sub>This document describes the implementation and inner working of the configuration system in Clusterio, and unless you're developing Clusterio it's probably not going to be very useful for you.</sub>
 
 Clusterio uses a field based configuration system to manage settings for the controller, instances, hosts, and the clusterioctl utility.
-Plugins can add their own config fields for the controller, host and instance configs.
+Plugins can add their own config fields for the controller, host, instance, and ctl configs.
 The fields are defined early in the startup, and fields for disabled plugins will still be created.
 
-The built-in config fields are defined in [packages/lib/src/config/definitions.ts](/packages/lib/src/config/definitions.ts) while plugins can add their own config field definitions to the `controllerConfigFields`, `hostConfigFields` and `instanceConfigFields` properties of the `plugin` export.
-A `<plugin_name>.load_plugin` field is automatically created for each plugin and it's used to enable/disable the loading plugins on startup.
+The built-in config fields are defined in [packages/lib/src/config/definitions.ts](/packages/lib/src/config/definitions.ts) while plugins add fields through `controllerConfigFields`, `hostConfigFields`, `instanceConfigFields`, and `ctlConfigFields` on the `plugin` export.
+Controller, host, and instance configs also receive `<plugin_name>.load_plugin` to enable/disable loading on startup and a hidden `<plugin_name>.version` for migrations.
 
 
-## Defing Fields
+## Defining Fields
 
 Fields are defined as POJS object properties where the name of the property correspond to the name of the configuration field.
 The property name is the primary way the field is accessed both in code and on the command line, as such it should follow the lower\_case\_underscore style and consist of a group name and a field name separated by a dot.
-For plugins the group name is the name of the plugin, e.g., the `level` field of the `foo_frobber` plugin would be named `foo_frobber.lavel`.
+For plugins the group name is the name of the plugin, e.g., the `level` field of the `foo_frobber` plugin would be named `foo_frobber.level`.
 
 **access**:
     The locations this config field can be read and modified from.
@@ -44,7 +44,8 @@ For plugins the group name is the name of the plugin, e.g., the `level` field of
 
 **initialValue** (optional if optional is true):
     Value this config field should take in a newly initialized config.
-    This can also be an async function returning the value to use.
+    This can also be a synchronous function returning the value to use.
+    The function is called with the config after its stored fields are loaded, so it can base the value on other fields.
 
 **validator** (optional):
     Function called with the value being set and the config it belongs to.
@@ -83,7 +84,22 @@ For plugins the group name is the name of the plugin, e.g., the `level` field of
 
 **inputComponent** (optional):
     Name of the input component the web interface should render for this field instead of the default for its type.
-    Plugins can add their own components through `inputComponents` on their web plugin class.
+    Plugins register components with the `control.hooks.inputComponents` hook in their web entrypoint.
+    The hook returns an object whose keys match the field's `inputComponent` value.
+
+For example, with `inputComponent: "foo_frobber_level"` on a field:
+
+```js
+import LevelInput from "./LevelInput.js";
+
+export default async function({ control, plugin }) {
+    control.hooks.inputComponents.attach(plugin.name, () => ({
+        foo_frobber_level: LevelInput,
+    }));
+}
+```
+
+`LevelInput` must implement the `InputComponent` contract exported by `@clusterio/web_ui`.
 
 If the value for a field stored in on disk somehow ends up being invalid the field will take on the initial\_value instead when the config is loaded.
 
@@ -101,7 +117,7 @@ E.g.:
     let value = configInstance.fields["group.field"]
 
 The set method also takes an extra argument, which is the value to set the field to.
-If the value is not valid this will throw an InvalidValue exception, if the field doesn't exist it will throw an InvalidField exception.
+If the value is not valid this will throw an InvalidValueError exception, if the field doesn't exist it will throw an InvalidFieldError exception.
 
 
 ## Config Lifecycle

docs/configuration.md

@@ -48,6 +48,23 @@ Must be greater than 0.
 Defaults to 60.
 
 
+### controller.host_pairing_timeout
+
+Time in seconds a host can wait for approval before its request expires, after which the host requests again with a new code.
+Set to 0 for no limit.
+
+Defaults to 3600.
+
+
+### controller.host_pairing_max_pending
+
+Maximum number of hosts that can wait for approval at the same time.
+Hosts over the limit keep trying until there is room.
+Set to 0 for no limit.
+
+Defaults to 50.
+
+
 ### controller.http_port
 
 Port to host HTTP server on.
@@ -93,7 +110,7 @@ Defaults to null meaning the header is never trusted.
 ### controller.public_url
 
 Public address the controller is accessible on.
-Used as the URL in configs generated with `clusteriocontroller bootstrap create-ctl-config` and `clusterioctl host create-config`, and by plugins like Player Auth that need to show the address of the web interface in-game.
+Used as the URL in configs generated with `clusteriocontroller bootstrap create-ctl-config`, and by plugins like Player Auth that need to show the address of the web interface in-game.
 This should be a full URL ending with a /.
 
 Defaults to null meaning assume localhost.
@@ -103,7 +120,7 @@ Defaults to null meaning assume localhost.
 
 Used to override where static assets such as the web bundle is loaded from.
 This can be used if you want to put the static assets on a separate cdn url like `https://cdn.example.com/static/` while keeping the API hosted directly by clusterio.
-Due to a limitation on how the webpack assets are built the pathname component of the url have to be `/static/`, values with any other path are rejected.
+Due to a limitation on how the web assets are built the pathname component of the url have to be `/static/`, values with any other path are rejected.
 
 See also the `clusteriocontroller copy-static` command for how to obtain the static asset files that need to be hosted.
 
@@ -277,6 +294,7 @@ Defaults to 1000.
 
 Whether the `controller update` command in clusterioctl and the update button in the web interface are allowed to update the `@clusterio/controller` package with npm.
 For security reasons this can only be changed locally with `clusteriocontroller config set` while the controller is stopped, and it is hidden in the web interface.
+The installer sets this, controller.allow_plugin_updates and controller.allow_plugin_install from one question about remote updates and plugin installs.
 See [Managing a Cluster](/docs/managing-a-cluster.md) for details on remote updates.
 
 Defaults to true.
@@ -326,6 +344,7 @@ Defaults to "New Host".
 ### host.id
 
 Immutable numeric id of the host which uniquely identifies this host in the cluster.
+The controller picks the id when the host is approved and the host writes it here, the controller does not accept hosts with an id it has not approved.
 Hidden in the web interface.
 
 Defaults to a random 31 bit number.
@@ -335,6 +354,7 @@ Defaults to a random 31 bit number.
 
 Directory to look for the Factorio server(s) in.
 This can either point to a Factorio server directory, or to a directory containing multiple versions of the Factorio server.
+The headless build has no graphics, so when exporting data from an instance running it the base game's icons are read from a full install of the same major and minor version in this directory instead, if there is one.
 
 Defaults to "factorio".
 
@@ -365,7 +385,9 @@ Defaults to "http://localhost:8080/".
 ### host.controller_token
 
 Access token used for authenticating with the controller.
-You can generate an access token with `clusterioctl host generate-token --id <host-id>`, or use the `clusterioctl host create-config` to create a new host config with the correct url and token in it.
+The host saves it together with host.id when it is approved, see [Setting up remote hosts](/README.md#setting-up-remote-hosts).
+The token only works while the host exists on the controller, if the host is revoked it asks to be approved again the next time it starts.
+Set it to `enter token here` to make the host ask to join again.
 The value can only be read and changed from the host itself.
 
 Defaults to "enter token here".
@@ -418,6 +440,7 @@ Defaults to 60.
 
 Whether the `host update` command in clusterioctl and the update button in the web interface are allowed to update the `@clusterio/host` package with npm.
 For security reasons this can only be changed locally with `clusteriohost config set` while the host is stopped, and it is hidden in the web interface.
+The installer sets this, host.allow_plugin_updates and host.allow_plugin_install from one question about remote updates and plugin installs.
 See [Managing a Cluster](/docs/managing-a-cluster.md) for details on remote updates.
 
 Defaults to true.
@@ -498,6 +521,31 @@ This allows you to prevent certain instances from being started when using the b
 Defaults to false.
 
 
+### instance.is_template
+
+If enabled this instance is a template other instances can inherit config values from.
+Templates cannot be assigned to a host and therefore never start.
+Hidden from `instance list` in clusterioctl unless `--templates` is passed, and listed separately in the web interface.
+An instance cannot stop being a template while other instances inherit from it.
+Hidden in the web interface, set when creating the template.
+
+Defaults to false.
+
+
+### instance.template_id
+
+ID of the template to inherit config values from.
+Inherited fields take their value from the template and follow it when it changes, setting a field on this instance stops inheriting it.
+Object fields like `factorio.settings` are inherited per property, properties set on this instance override the ones from the template.
+When set on an existing instance fields with the same value as the template or the default value become inherited, except the server name in `factorio.settings`.
+`instance.name`, `instance.is_template`, `instance.template_id`, the port fields, `factorio.rcon_password` and hidden fields are never inherited.
+A template may itself inherit from another template, but not from itself through a chain of templates.
+The fields currently inherited are stored in the hidden `instance.inherited_fields` field.
+In the web interface inherited values are linked and read only until they are unlinked.
+
+Defaults to null meaning no template.
+
+
 ### factorio.version
 
 Version of Factorio to use for this instance.
@@ -591,6 +639,8 @@ Most plugins require the use of save patching to function.
 Turning it off makes it possible to use Clusterio to run and manage regular vanilla games and scenarios not compatible with Clusterio.
 Plugins that need save patching can't be loaded on the instance while this is disabled.
 Disabling it while such a plugin is loaded is rejected, and so is loading such a plugin while it is disabled.
+Saves patched before are restored to their original control.lua and have the modules removed when started with this disabled.
+Data the modules stored in the game is left in place.
 
 Defaults to true.
 
@@ -606,6 +656,23 @@ Like factorio.enable_save_patching, plugins that need script commands can't be l
 Defaults to true.
 
 
+### factorio.enable_lua_udp
+
+Enable UDP messaging between the host and the Factorio server by passing `--enable-lua-udp` to Factorio.
+This lets plugins send UDP packets to the server and receive data sent with `clusterio_api.send_udp` in-game.
+Requires Factorio 2.1.10 or later, earlier versions crash when a packet is received on a headless server.
+
+Defaults to false.
+
+
+### factorio.lua_udp_port
+
+UDP port on localhost the Factorio server receives Lua UDP messages on when `factorio.enable_lua_udp` is enabled.
+If null a random port in the dynamic range is assigned each time the instance starts.
+
+Defaults to null.
+
+
 ### factorio.enable_whitelist
 
 Turn on the whitelist on the server.
@@ -627,6 +694,8 @@ The settings in `data/server-settings.example.json` of the Factorio installation
 
 Changes to the following properties will be applied live if the instance is running while it is changed: `afk_autokick_interval`, `allow_commands`, `autosave_interval`, `autosave_only_on_server`, `description`, `ignore_player_limit_for_returning_players`, `max_players`, `max_upload_slots`, `max_upload_in_kilobytes_per_second`, `name`, `only_admins_can_pause_the_game`, `game_password`, `require_user_verification`, `tags`, `visibility`.
 
+With `auto_pause` on, the plugins included with Clusterio stop their periodic commands while no players are online, so the game stays paused.
+
 Defaults to a set of common settings filled in when the instance is created, such as `"tags": ["clusterio"]`, `"auto_pause": false` and a `name` and `description` based on the cluster and instance names.
 
 
@@ -722,9 +791,9 @@ Defaults to 5.
 
 Whether to load the given plugin on the instance.
 Note that plugins need to be loaded on the controller and the host in order for them to be loaded on instances.
-Loading a plugin that requires save patching or script commands is rejected while factorio.enable_save_patching or factorio.enable_script_commands respectively is disabled on the instance.
+Loading a plugin that requires save patching, script commands or Lua UDP is rejected while factorio.enable_save_patching, factorio.enable_script_commands or factorio.enable_lua_udp respectively is disabled on the instance.
 
-Defaults to true.
+Defaults to true, or false if the instance does not have the features the plugin requires.
 
 
 ## Ctl Configuration

docs/contributing.md

@@ -3,23 +3,28 @@
 You are probably reading this because you want to contribute code to Clusterio.
 Great!  We need people like you.
 Though we have some conventions and workflows that we kindly ask you to follow.
-We have included all the pratical considerations in this document.
-Although, we recomend checking out out our [past decisions](/docs/decisions.md) if you have the time.
+This document covers the development workflow, commands, and repository layout.
+Read our [past decisions](/docs/decisions.md) for compatibility policies and design rationale.
 
 
 ## Contents
 
+- [Project map](#project-map)
 - [Workflow](#workflow)
     - [Starting a Feature Branch](#starting-a-feature-branch)
     - [Setting up the Development Environment](#setting-up-the-development-environment)
+    - [Build outputs](#build-outputs)
+    - [Debugging](#debugging)
     - [Checking your code for errors](#checking-your-code-for-errors)
+    - [Cleaning generated and runtime files](#cleaning-generated-and-runtime-files)
     - [Submitting a Pull Request](#submitting-a-pull-request)
+    - [Breaking Compatibility with the Last Release](#breaking-compatibility-with-the-last-release)
     - [Editing your Feature Branch](#editing-your-feature-branch)
     - [Resolving Merge Conflicts](#resolving-merge-conflicts)
     - [Updating Your GitHub Fork](#updating-your-github-fork)
     - [Publishing releases to npm](#publishing-releases-to-npm)
 - [Changelog](#changelog)
-- [Supported Node.js and Factorio Version](#supported-node.js-and-factorio-version)
+- [Supported Node.js and Factorio Versions](#supported-nodejs-and-factorio-versions)
 - [Code Style](#code-style)
     - [Naming Style](#naming-style)
     - [Units](#units)
@@ -28,6 +33,28 @@ Although, we recomend checking out out our [past decisions](/docs/decisions.md)
     - [Markdown](#markdown)
 
 
+## Project map
+
+Clusterio manages distributed Factorio servers through a controller, hosts, and plugins.
+
+| Location | Responsibility |
+| --- | --- |
+| `packages/controller/` | Cluster state, users and roles, connections, HTTP API, and serving the web UI. Start with `src/Controller.ts` and `src/ControlConnection.ts`. |
+| `packages/host/` | Instance lifecycle, Factorio processes, save patching, and exports. Start with `src/Host.ts`, `src/Instance.ts`, `src/server.ts`, and `src/patch.ts`. |
+| `packages/ctl/` | Command-line administration and plugin commands. |
+| `packages/lib/` | Shared messages, routing, config, permissions, plugin loading, logging, and utilities for Node and browser consumers. |
+| `packages/web_ui/` | React/Ant Design interface and browser control connection; bundled through the controller's Vite build. |
+| `packages/create/` | Installer and JavaScript/TypeScript plugin templates. |
+| `plugins/` | Bundled plugins, including their Node, web, Lua, and test code. |
+| `external_plugins/` | Plugins developed in independent repositories and included in the local workspace. |
+| `packages/host/modules/` | Lua modules injected into saves. |
+| `packages/host/lua/` | Factorio mod sources, including `clusterio_lib`. |
+| `test/` | Root test suite, fixtures, helpers, integration harness, and manual tests. Plugin tests also live under `plugins/*/test/`. |
+
+Start with [how it works](how-it-works.md), [communication](communication.md), and the [config system](config-system.md) for architecture.
+See [writing plugins](writing-plugins.md) and [developing for Clusterio](developing-for-clusterio.md) for plugin and Factorio contracts.
+
+
 ## Workflow
 
 While we don't require you to follow this workflow, it makes things much simpler for both you and us if you choose to do so.
@@ -52,16 +79,22 @@ Do not run `git pull upstream` it will mess thing up!
 
 ### Setting up the Development Environment
 
-The project is organized into several packages that are all tied together using pnpm's workspace feature.
-In order to get the development environment up and running you will need to run the following command:
+Run commands from the repository root unless stated otherwise.
+Use the latest Node.js LTS release and the pnpm version pinned by `packageManager` in root `package.json`.
+Install dependencies with:
 
-    pnpm install
+    pnpm install --no-frozen-lockfile
 
-This installs dependencies needed by tests and links the packages up so they work from the git work tree.
-To start a cluster from the repo you will first need to initialize it, this can be done using the installer:
+This installs dependencies and builds the packages.
+`pnpm-lock.yaml` is intentionally ignored; do not force-add it or assume a frozen-lockfile install is supported.
+For a fresh development cluster, run the installer and select standalone mode to create `config-controller.json` and `config-host.json`:
 
     node packages/create --dev
 
+The first time the host starts it prints a link for approving it in the web interface.
+
+You can skip this step if you already have a working controller and host configuration.
+
 To add plugins to use in the development environment use
 
     node packages/ctl plugin add ./path/to/plugin
@@ -79,29 +112,61 @@ By default node does not enable source maps, to get stack traces pointing to the
 
 Note that to use `clusterioctl` you will have to create a ctl config first:
 
-    node packages/controller bootstrap create-ctl-config [admin-user]
+    node packages/controller bootstrap create-ctl-config <admin-user>
 
-Once you've set up the cluster you can use `node packages/controller run` to run the controller and `node packages/host run` to run the host which connects to the controller.
+Use the name of an existing admin user.
+Run `node --enable-source-maps packages/controller run` and `node --enable-source-maps packages/host run` in separate terminals.
 
 When editing TypeScript files ran by Node.js you need to run `pnpm build` to compile the files to JavaScript before restarting the controller/host/ctl for changes take effect.
 To have TypeScript watch for changes and rebuild immediadly you may run `pnpm watch` instead.
 
 For web development on the controller there are also the following flags:
 
-`--dev` - Recompile the `web_ui` package on the fly providing live reloading of it.
+`--dev` - Serve the `web_ui` package from source through Vite's development server with hot module replacement.
 
-`--dev-plugin [name]` - Recompile the web module for the given plugin on the fly proving live reloading of it.
+`--dev-plugin [name]` - Serve the web module for the given plugin from source the same way.
+This also serves the `web_ui` package from source, as the plugin has to share its modules with it.
 
 These options can be combined and multiple plugins can be actived for dev mode.
 Keep in mind if you're working on external plugins the `--dev-plugin` option is likely to only work if the plugin is placed inside the `external_plugins` directory and its dependencies are installed with pnpm as part of the clusterio repository workspace.
-Note that webpack is configured to delete the web interface build when using these options, so you will have to run `pnpm run -r --if-present prepare` before you start the controller without these flags again.
+
+
+### Build outputs
+
+| Command | Output |
+| --- | --- |
+| `pnpm build` | Compiled JavaScript and type declarations in each package's `dist/node/` and, where applicable, `dist/browser/`. |
+| `pnpm watch` | TypeScript compilation as files change; restart affected Node processes afterward. |
+| `pnpm -r run prepare` | Compiled packages, generated validators, and production web bundles. |
+| `pnpm build-mod` | Factorio mod archives in root `dist/`. |
+
+`pnpm build` alone does not regenerate the standalone validator or web bundles.
+After changing `packages/lib/src/data/messages_core.ts`, run `pnpm -r run prepare` to compile the schema and regenerate its validator.
+
+
+### Debugging
+
+Establish the controller, host, instance, Factorio version, enabled plugins, and exact reproduction first.
+Enable source maps and confirm the affected process is running rebuilt code.
+Use the contextual logger so host, instance, and plugin metadata remain available.
+Development logs are in `logs/`; integration logs are in `temp/test/logs/` and are uploaded by CI.
+
+For a failing request or event, start with its handler and the input that triggers the failure.
+Handlers are registered with `handle(MessageClass, handler)`; searching for the message class will help you find its handler.
+Inspect the input and the state used by the failing operation, then try to reproduce the failure in a [focused test](#checking-your-code-for-errors).
+If the handler is never reached, investigate message registration, routing, and permission checks using the [communication guide](communication.md).
+For errors reported by Factorio, start with the Lua stack trace and the module or mod it identifies; the [developing-for-Clusterio guide](developing-for-clusterio.md) explains how that code communicates with the host.
+
+For connection and recovery bugs, test disconnect, reconnect, timeout, and shutdown as applicable.
+A cached controller status does not prove the current state of a host or Factorio process.
+Recovery mode disables normal plugin loading; account for it when diagnosing missing plugin behavior and [Lua state migrations](developing-for-clusterio.md#initialisation-and-recovery).
+Verify UI behavior in the browser: successful compilation does not verify rendering, permissions, or subscriptions.
 
 
 ### Starting a Feature Branch
 
 Development is done exclusively in feature branches.
-This means the first thing to do before starting any development at all is to make a new branch starting from the master branch of the main repository.
-To do this check out that branch from the main repository.
+To start a new feature branch, first check out master from the main repository.
 
     git fetch upstream
     git checkout upstream/master
@@ -117,18 +182,61 @@ But it's nice to have its name be somewhat descriptive of what it implements.
 
 ### Checking your code for errors
 
-There's automated tests and linting setup for the project.
-Please do add tests for any code you add and run both:
+Tests use Mocha and Node assertions, mostly as JavaScript ESM importing compiled packages.
+Run the test suite and linter from the repository root:
 
     pnpm run test
     pnpm run lint
 
-To check that your changes pass the integration tests as well as the ESLint rules set up for the project.
+`pnpm test` builds TypeScript and runs root and bundled-plugin tests, excluding `test/manual/`.
+Run tests from the root: individual package `test` scripts are often placeholders that deliberately fail.
+Use `pnpm cover` for the full suite with c8 coverage.
 
 The full test suite can take a while to run.
 Use `pnpm run fast-test` to skip tests marked as slow, or `pnpm run no-external-test` to skip tests that depend on external APIs such as the Factorio mod portal or GitHub.
 These correspond to setting the `FAST_TEST` and `NO_EXTERNAL_TEST` environment variables and can be combined.
-When writing tests that take a lot of time call `slowTest(this)` at the start of the test, and for tests that call out to external APIs call `externalTest(this)`, both are exported from `test/integration/index.js`.
+For example, `FAST_TEST=y NO_EXTERNAL_TEST=y pnpm test` combines both exclusions.
+These flags still run the remaining integration setup and tests; neither produces an isolated unit-test run.
+When writing slow tests call `slowTest(this)`, and for external API tests call `externalTest(this)`; both are exported from `test/common.js`.
+For tests that need a Factorio installation, call `requiresFactorio(this)` in the test or pass the suite to `requiresFactorio`, imported from `test/integration/index.js`.
+`test/common.js` also provides JSON round-trip/schema helpers, and `test/mock.js` provides shared mocks.
+
+For a focused test, build first and select the file explicitly instead of appending it to the full-suite script:
+
+```sh
+pnpm build
+NODE_EXTRA_CA_CERTS="$PWD/test/file/tls/cert.pem" pnpm exec mocha \
+  --enable-source-maps --check-leaks test/lib/helpers.js
+```
+
+Replace the example file with the relevant test; use `--grep` to narrow it further.
+Preserve the test TLS CA configuration for HTTPS integration tests.
+Use `--require ./test/setup_logging.js` when an isolated test needs the shared logger configured.
+Some tests import `test/integration/index.js`, which installs root hooks and starts a controller and host even for a targeted invocation.
+
+The integration harness recreates its configs, database, mods, and instances under `temp/test/` and uses HTTP port 8880 and HTTPS port 4443.
+Do not run competing integration suites in the same checkout or on those ports.
+It locates Factorio through `host.factorio_directory` in root `config-host.json`, defaulting to `factorio/`.
+The full harness currently requires a usable Factorio installation: a missing directory fails discovery, and an empty directory fails setup while selecting a version.
+Some individual tests have conditional skips, but these do not make a full run without Factorio succeed.
+
+Use the relevant production bundle, mod build, or [manual scenario](../test/manual/README.md) for changes affecting those surfaces.
+For documentation changes, check that links, paths, and example commands are correct.
+
+
+### Cleaning generated and runtime files
+
+Inspect `node clean.js --help` and preview the selected scope with `--dry-run` before cleaning.
+
+| Command | Removal scope |
+| --- | --- |
+| `node clean.js --dry-run` | Preview workspace package/plugin build output and Vite dependency caches; omit `--dry-run` to clean, or use `pnpm clean`. |
+| `node clean.js tests --dry-run` | Also preview root `.nyc_output/`, `coverage/`, the whole `temp/` directory, and `static/`. |
+| `node clean.js all --dry-run` | Also preview runtime databases, instances, logs, mods, local configs, `FactorioAdminToken.txt`, and `plugin-list.json`. |
+
+`all` deletes cluster runtime data, including configurations and saves; back up anything you want to keep before using it.
+The cleaner excludes `external_plugins/` and does not remove mod archives in root `dist/`.
+Do not commit local configs, tokens, databases, saves, logs, or generated build output.
 
 
 ### Submitting a Pull Request
@@ -145,6 +253,38 @@ If there are more changes you want to have merged or mistakes you need to fix yo
 Pull request descriptions should contain a `### Changelog` section with additions for the changelog contained inside a code block, see the [CHANGELOG.md](/CHANGELOG.md) file for the formatting of changelog entries and possible sections to use.
 If there are no changelog entries to add, remove the code block and start the changelog section with the text `None.`, followed by an optional reason for why it is not is needed.
 
+### Breaking Compatibility with the Last Release
+
+The version-compat job in CI runs the local controller with the latest published host, and the published controller with the local host.
+It fails if a link message changed its src, dst, data or response schema since that release.
+Control is left out of src and dst, and messages that only go between control and the controller are not checked, as the web UI is served by the controller it talks to.
+If the break is intended, add an entry to [scripts/version-compat-breaks.json](../scripts/version-compat-breaks.json) in the same pull request.
+Breaking changes are only accepted if the next release is intended to include them, see our decision on [breaking changes](/docs/decisions.md#breaking-changes).
+The entry is keyed by the published version directly before the breaking change was introduced, which is the latest release at the time.
+Every entry needs the pull request number, a reason saying what breaks, and an impact saying why end users are not or only minimally affected.
+
+```json
+{
+	"2.0.3": {
+		"messages": {
+			"request FactorioVersionsRequest": {
+				"pr": 1129,
+				"reason": "What breaks between the two versions.",
+				"impact": "Why end users are not or only minimally affected."
+			}
+		},
+		"connect": { "pr": 1234, "reason": "Why a host cannot connect at all.", "impact": "..." }
+	}
+}
+```
+
+Listed messages are reported as warnings with the reason and impact instead of failing the job.
+Only add `connect` when the published and local versions cannot connect at all, it skips starting the cluster.
+Listing a message that did not change fails the job.
+Entries only apply while their version is the latest release, after the next release they are ignored and can be removed.
+Describe the break in the Breaking Changes section of the changelog.
+
+
 ### Editing your Feature Branch
 
 Until your feature branch is merged you can edit it as you see fit by using the rebase tool.
@@ -200,12 +340,15 @@ The format of this log should be self explanatory, please follow it carefully. P
 
 ## Supported Node.js and Factorio Versions
 
-Clusterio run on Node.js v18 and up, make sure to check that the Node builtin API's and npm packages you use are supported on these versions.
-The GitHub Actions tests are run on whatever version of Node.js v18 and v20 is the default.
+For development, use the latest Node.js LTS release, following the [installation instructions](../README.md#installation).
+Make sure the Node.js built-in APIs and npm packages you use remain compatible with the supported versions.
+The GitHub Actions tests run every Factorio version on the latest Node.js, and every other supported Node.js version with the latest Factorio only, which keeps the number of jobs within GitHub's concurrent job limit.
+Coverage is collected on the oldest supported Node.js; see the [CI workflow](../.github/workflows/node.js.yml) for the version matrix.
 
-For Factorio Clusterio 2.0 aims to support version 0.17.69 and up, including the latest experimental release.
-It's recommended that you use the lua API reference for 0.17.69, as there's no information on what version Factorio API's were introduced in.
-The GitHub Actions tests are run against the latest experimental release.
+For Factorio, active support covers the latest stable version and the previous major version, using version lines such as 2.1, 2.0 and 1.1 as described in the [Factorio support policy](decisions.md#factorio).
+Older versions from 0.17 onwards receive passive support where it does not restrict actively supported versions.
+Check the Lua API documentation for the Factorio versions your changes target and preserve compatibility with the actively supported versions.
+The GitHub Actions tests use pinned Factorio releases; see the [CI workflow](../.github/workflows/node.js.yml) for the current version matrix.
 
 Check out all our of decisions for [node, factorio, and operation support](/docs/decisions.md#supportability).
 
@@ -226,13 +369,10 @@ Fields in configs uses lowercase_underscore.
 JavaScript files are named using lowercase_underscore for files that export multiple items.
 Files containing and exporting a single class should be named the same as the class in PascalCase.
 
-Relative imports in lib package are prefixed with lib to make them easier to distinguish, as the names tend to be very generic.
-E.g.:
-
-    import libLink from "./link";
-    import libLuaTools from "./lua_tools";
-
-Note that no sub-modules of the lib package is imported directly from the outside of it with the exception of `build_mod`.
+Use ESM imports, including `.js` extensions in TypeScript relative imports.
+For namespace imports within lib, use the `lib` prefix to distinguish generic module names, for example `import * as libFileOps from "./file_ops.js";`.
+Production consumers should import shared public APIs through `@clusterio/lib` rather than reaching into its internals.
+When adding public shared exports, check both `packages/lib/index.ts` and `packages/lib/browser.ts` and keep Node-only dependencies out of browser code.
 
 For lua code lowercase_underscore is used for everything.
 
@@ -241,7 +381,7 @@ For lua code lowercase_underscore is used for everything.
 When naming variables whos unit is not a base SI unit like seconds or metres the unit used should be suffixed in the variable name:
 E.g.:
 
-    const startTimeMs = date.now();
+    const startTimeMs = Date.now();
     const endNs = process.hrtime();
 
 This makes it easier to spot different unit bases are being mixed in the code.
@@ -282,7 +422,7 @@ For example:
 ### Markdown
 
 Markdown files are formatted using one line per sentence to improve diff reports when making changes.
-Headings have two spaces before them and on space after them to make them stand out, the heading at the top of the file does not have any spaces before it, and heading immediadly followed by another heading have one space between them.
+Headings have two blank lines before them and one after them; the top heading has no blank lines before it, and consecutive headings have one blank line between them.
 For example:
 
 ```md

docs/decisions.md

@@ -18,7 +18,7 @@ If there is a particular issue or concern you would like the team to comment on
     - [Naming Times and Durations](#naming-times-and-durations)
     - [Strings](#strings)
 3. Code Style (TS / JS / Web)
-    - [ESLint](#eslint)
+    - [Oxlint](#oxlint)
     - [Imports Within `lib` Package](#imports-within-lib-package)
     - [Name Style](#name-style-ts)
 4. Code Style (Lua)
@@ -38,7 +38,7 @@ If there is a particular issue or concern you would like the team to comment on
     - [React](#react)
     - [Separation of Host and Controller](#separation-of-host-and-controller)
     - [Typescript](#typescript)
-    - [Webpack](#webpack)
+    - [Vite](#vite)
 8. Supportability
     - [Breaking Changes](#breaking-changes)
     - [Distribution](#distribution)
@@ -91,6 +91,11 @@ Therefore, we decided to use TSDoc comments for this purpose as it is well inter
 Our specific implementation is TypeDoc because it is lightweight it and can output TSDoc comments as html.
 API Extractor was also considered but not selected.
 
+🧾 29th October 2026 | [Github PR #1147](https://github.com/clusterio/clusterio/pull/1147)
+
+We dropped TypeDoc as the generated docs were never published and it does not support TypeScript 7.
+TSDoc comments are still used for editor hover text.
+
 
 ## Code Style (General)
 
@@ -157,14 +162,15 @@ Therefore, we selected double quotes as our standard.
 
 ## Code Style (TS / JS / Web)
 
-### ESLint
+### Oxlint
 
-🧾 Time Immemorial  
+🧾 20th October 2026
 
-> Use ESLint for style checking.
+> Use Oxlint for style checking, with formatting rules from `@stylistic/eslint-plugin`.
 
-We needed to maintain a consistent style across all our files.
-Therefore, we utilise the industry-standard ESLint, as it is familiar and highly configurable.
+We needed to maintain a consistent style across all our files and used ESLint for it from the start.
+Oxlint runs the same rules about three times faster, reads ESLint disable comments, and does not depend on the TypeScript compiler API.
+The formatting rules ESLint deprecated run through `@stylistic/eslint-plugin` loaded as an Oxlint JS plugin, so the code style did not change.
 
 
 ### Imports Within `lib` Package
@@ -347,14 +353,15 @@ Transitioning to any other langauge would be too significant an undertaking.
 Initially, we supported multiple languages for plugins, including Rust and Go; however, this approach was quickly abandoned due to its complexity.
 
 
-### Webpack
+### Vite
 
-🧾 Time Immemorial
+🧾 20th October 2026
 
-> Our web stack uses Webpack and associated plugins.
+> Our web stack uses Vite. Plugin web bundles are ES modules that import shared dependencies through an import map.
 
-We required a bundler for our web build, and at the time, Webpack was the standard.
-While we have considered alternatives since, transitioning to another solution would be a significant undertaking.
+We used Webpack with Module Federation for loading plugins from the start, as it was the standard at the time.
+Vite builds the web interface several times faster, its development server updates modules without reloading the page, and plugin bundles no longer depend on one bundler's runtime.
+See [#1127](https://github.com/clusterio/clusterio/issues/1127) for the discussion.
 
 
 ## Supportability
@@ -370,7 +377,7 @@ We aim to be highly transparent with breaking changes and maintain compatibility
 During the alpha stage every alpha was allowed to contain breaking changes, as long as they were documented and provided migration actions.
 From 2.0 onwards changes which require action from people running a cluster, such as changes to configs, stored data or the command line, are held back for the next major version.
 Changes to the plugin API may still be made in minor versions.
-Plugin authors should check the changelog when a new minor version is released, non-trivial changes come with a migration guide such as [Migrating Plugins to 2.0](/docs/devs/plugin-migration-to-2.0.md).
+Plugin authors should check the changelog when a new minor version is released, non-trivial changes come with a migration guide such as [Migrating Plugins to 2.1](/docs/devs/plugin-migration-to-2.1.md).
 
 
 ### Distribution
@@ -452,11 +459,13 @@ While we aim to increase the number of useful checks performed by CI in the futu
 
 🧾 Time Immemorial
 
-> We do not use any CD actions, and have no plans of implementing them.
+> We publish packages and mods when a maintainer publishes a stable GitHub release.
 
-Unlike CI, we do not find continuous deployment useful at our current stage of the project.
-This is due to the frequent breaking changes merged into the main branch, which we prefer to bundle together as a single version.
-In the future, we may automate the deployment process, but it will always require manual activation.
+Changes merged into the main branch are bundled into releases before publication.
+Publishing a stable GitHub release triggers the [Publish workflow](../.github/workflows/publish.yml), which publishes npm packages and Factorio mods.
+Publishing a prerelease or pushing a tag alone does not trigger publication.
+Running clusters are updated by their operators.
+See the [release workflow](release-workflow.md) for the publication procedure.
 
 
 ### Mocha
@@ -475,5 +484,5 @@ However, we are always looking for a better framework which is worth the effort
 
 > We perform all tests through GitHub actions using Ubuntu latest, using all supported versions of node and all supported versions of factorio.
 
-We perform tests on all versions of NodeJS and Factorio that we support.
+The [current CI workflow](../.github/workflows/node.js.yml) tests selected Node.js major versions and pinned Factorio releases.
 However, we do not conduct tests on all supported operating systems, as the added demand on our testing resources would be significant, and the value it would provide for a cross-platform engine like NodeJS would be limited.

docs/developing-for-clusterio.md

@@ -10,7 +10,9 @@ This document describes the various different types of content that can be creat
     - [event_handler interface](#event_handler-interface)
 - [Communicating with Clusterio](#communicating-with-clusterio)
 - [Clusterio Modules](#clusterio-modules)
+    - [Initialisation and recovery](#initialisation-and-recovery)
 - [Clusterio Lua API](#clusterio-lua-api)
+    - [Compatibility library](#compatibility-library)
 - [Clusterio Plugins](#clusterio-plugins)
 
 
@@ -73,7 +75,7 @@ The following entries are recognized by `add_lib`, all of which are optional:
     Callback called when the callback of `script.on_configuration_changed` is invoked.
 - `events`:
     Table mapping event ids as defined in `defines.events` to callbacks.
-    For example the table `{ [defines.events.on_player_died] = foo }` will call bar with the usual `event` table as argument every time a player dies.
+    For example the table `{ [defines.events.on_player_died] = foo }` will call foo with the usual `event` table as argument every time a player dies.
 - `on_nth_tick`:
     Table mapping the nth tick number to a callback.
     For example the table `{ [30] = bar }` will call bar every 30th tick.
@@ -92,7 +94,7 @@ For example a module could be defined as
 ```lua
 -- example.lua
 local function my_handler(event)
-    local name = game.player[event.player_index].name
+    local name = game.get_player(event.player_index).name
     game.print("It appears that " .. name .. " has died")
 end
 
@@ -176,12 +178,27 @@ This means always declearing your top level variables and functions local and ex
 You can require other files in your module by prefixing your require paths with `modules/your_module_name/`.
 It's also possible to require files from other modules this way too.
 
-The global variables and data stored in the global table is shared by all Clusterio modules as well as the scenario, so it's important that you use unique names.
-It's recommended to prefix the global variables and data in the global table that your module has with the name of your module.
+Persistent state is shared by all save-patched modules and the scenario.
+Use `compat.script_data` to access the version-appropriate persistent table (`global` or `storage`), and keep your state under a unique module name.
+See the [compatibility library](#compatibility-library) below.
+
+
+### Initialisation and recovery
 
 Because modules can be patched into an existing game you cannot rely on the `on_init` callback to be called in Clusterio Modules.
 Nor can you rely on the `on_configuration_changed` callback, as this is not called when level code changes.
-The Clusterio Lua API provides the custom `on_server_startup` event that can be used as a substitute, see the next section.
+The Clusterio Lua API provides `on_server_startup` for setup and migrations, but some Factorio events can run before it.
+In particular, player-leaving, console-command, and configuration-change handlers must not assume startup has initialized their state.
+
+Use an idempotent state accessor in handlers that can run early.
+It should migrate any legacy namespace before creating fresh state, fill missing fields, and return the same state on repeated calls.
+A guard that returns early because the new namespace is absent can silently discard recovery work on an older save.
+See [player_sync's state accessor](../plugins/player_sync/module/get_script_data.lua) for an implementation that handles a renamed namespace and [the module template](../packages/create/templates/common/module/control.lua) for lifecycle setup.
+
+Use `on_load` only to restore local references and other operations permitted by Factorio's load lifecycle; do not create or migrate persistent state there.
+Perform state changes in startup or other events that permit them.
+Recovery mode skips normal Node plugin loading, so save data may still use an older plugin namespace when Lua handlers run.
+Test old and current saves, early events, repeated access, and save/load behavior on the relevant supported Factorio versions.
 
 
 ## Clusterio Lua API
@@ -190,6 +207,29 @@ Clusterio provides a save patched module as well as a regular Factorio mod for i
 This includes tools sending data to plugins, script events to listen for, and an item serialization module.
 
 
+### Compatibility library
+
+Use the compatibility helpers for APIs that differ between supported Factorio versions:
+
+```lua
+-- In a save-patched module:
+local compat = require("modules/clusterio/compat")
+-- In a mod, use require("__clusterio_lib__/compat") instead.
+
+local function encode_example()
+    local script_data = compat.script_data
+    local encoded = compat.table_to_json({ example = true })
+    local decoded = compat.json_to_table(encoded)
+end
+```
+
+`script_data` selects the persistent state table, while the JSON helpers select the available runtime API.
+Access persistent state from the appropriate lifecycle callbacks, after initialisation or loading has made it available.
+The JSON helpers also work before runtime; other compatibility helpers may require `game` to be available.
+Legacy examples using `global` or `game.table_to_json` directly are version-specific.
+See [compat.lua](../packages/host/modules/clusterio/compat.lua) and the [Factorio support policy](decisions.md#factorio) before introducing version checks.
+
+
 ### clusterio_api library
 
 **require path**  
@@ -228,7 +268,8 @@ Event data:
 
 Raised when Clusterio is starting up the server.
 It can be used to initialize clusterio related features in a mod, and as a stand-in for the `on_init` and `on_configuration_changed` event for modules.
-It is invoked on the first tick the server runs after the save has been patched, before most other events.
+It is invoked on the first tick the server runs after the save has been patched, before most other events but not all of them.
+See [initialisation and recovery](#initialisation-and-recovery) for handlers that can run earlier.
 
 Use this event to initialize and/or migrate the data structures you need.
 Keep in mind that both mods and Clusterio modules can switch from any version to any version so it should be able to handle both forwards and backwards migrations gracefully.
@@ -236,7 +277,7 @@ Keep in mind that both mods and Clusterio modules can switch from any version to
 #### clusterio_api.send_json(channel, data)
 
 Send JSON data to Clusterio over the given channel.
-The `data` argument should be table that can be serialized with `game.table_to_json`.
+The `data` argument should be a table that can be serialized with `compat.table_to_json`.
 Clusterio plugins can listen to channels and will receive an event with the data sent here.
 See the [Communicating with Factorio section](writing-plugins.md#communicating-with-factorio) in the Writing Plugins document for more information.
 
@@ -249,7 +290,20 @@ Strings embedded into the tables sent must be valid UTF-8 text.
 
 Parameters:
 - `channel`: string identifying which channel to send it on.
-- `data`: table that can be converted to JSON with game.table_to_json
+- `data`: table that can be converted to JSON with `compat.table_to_json`
+
+#### clusterio_api.send_udp(channel, data)
+
+Send data to Clusterio over the given channel using UDP instead of stdout.
+Clusterio plugins receive it as `udp-channel_name` events, see the [Communicating over UDP section](writing-plugins.md#communicating-over-udp) in the Writing Plugins document.
+Requires `factorio.enable_lua_udp` to be enabled on the instance and Factorio 2.1.10 or later, otherwise the data is silently dropped.
+The port to send to is passed to the module with a script command on startup, so `factorio.enable_script_commands` must be enabled as well.
+
+**Note**: Delivery is not guaranteed and payloads should be kept well below 32 kB.
+
+Parameters:
+- `channel`: string identifying which channel to send it on.
+- `data`: string with the payload to send, any bytes are allowed.
 
 
 ### serialize library
@@ -260,7 +314,7 @@ From a module
 From a mod  
 `local serialize = require("__clusterio_lib__/serialize")`
 
-Serializes inventories and item stacks into a table that in turn can be converted into JSON with `game.table_to_json`.
+Serializes inventories and item stacks into a table that in turn can be converted into JSON with `compat.table_to_json`.
 Properly handles the extra data for most items.
 
 #### serialize.serialize_equipment_grid(LuaEquipmentGrid)

docs/devs/plugin-migration-to-2.0.md

@@ -18,6 +18,12 @@ See [Breaking Changes](/docs/decisions.md#breaking-changes) in the decision log.
 Clusterio's packages are now ES modules and plugins should be too.
 Add `"type": "module"` to the plugin's `package.json` and replace `require()` and `module.exports` with `import` and `export`.
 
+Keep `"clusterio-plugin"` in the package's `"keywords"` for automatic discovery, and keep `"main"` pointing to the plugin info module (or use the default `index.js`).
+An `"exports"` map is not required for ES modules.
+If you add one, expose `"./package.json": "./package.json"`; otherwise Clusterio cannot discover or load the plugin by its package name.
+Clusterio currently reads the implementation path from `"main"` rather than conditional exports.
+See [Plugin Structure](../writing-plugins.md#plugin-structure) for a compatible package example.
+
 Relative imports must include the full filename with the `.js` extension.
 This also applies in TypeScript, where you import `./messages.js` even though the source file is `messages.ts`.
 Importing a directory no longer resolves to its `index.js`.

docs/devs/plugin-migration-to-2.1.md

@@ -0,0 +1,105 @@
+# Migrating Plugins to 2.1
+
+This guide covers the changes needed to bring a plugin written for 2.0 up to 2.1.
+Plugins still on an alpha should start with the [2.0 guide](plugin-migration-to-2.0.md).
+
+
+## Request errors
+
+`RequestError` is gone.
+Errors thrown by request handlers are now received by the requester as the same class, see [Request errors](/docs/writing-plugins.md#request-errors).
+Replace `RequestError` with the class that fits:
+
+- Invalid input: `ValidationError`.
+- Operation disabled by config: `ForbiddenError`.
+- Missing instance, user, save etc.: `NotFoundError`.
+- Instance running, name taken etc.: `ConflictError`.
+- Host not connected, instance not running: `UnavailableError`.
+- Timeouts: `TimeoutError`.
+
+`catch` blocks that check `err instanceof RequestError` should check `ClientError` instead, or the specific class.
+Checks of `err.code` for remote errors can use `instanceof`, e.g. `err instanceof lib.SessionLost`.
+
+The config errors `lib.InvalidValue`, `lib.InvalidField` and `lib.InvalidAccess` were renamed to `lib.InvalidValueError`, `lib.InvalidFieldError` and `lib.InvalidAccessError`.
+
+Errors of classes that are not registered, like a `TypeError` from a bug, are received as `InternalServerError`.
+Plugins that define their own error classes need to list them in `errors` of the plugin declaration.
+
+
+## Build the web module with Vite
+
+The web interface is now built with Vite instead of webpack, and plugin web modules are loaded as ES modules instead of through Module Federation.
+A plugin built with webpack fails to load with the error "Incompatible dist/web/manifest.json, the plugin must be rebuilt with Vite".
+
+Delete `webpack.config.cjs` and add a `vite.config.js` next to `package.json`:
+
+```js
+import { defineConfig } from "vite";
+import { clusterioPlugin } from "@clusterio/web_ui/vite.mjs";
+
+export default defineConfig({
+	plugins: [clusterioPlugin()],
+});
+```
+
+In `package.json` replace `webpack-cli --mode production` in the `prepare` script with `vite build`, and replace the `webpack`, `webpack-cli` and `webpack-merge` dev dependencies with `vite`.
+Add `vite.config.js` to `.npmignore` if you had `webpack.config.js` there.
+
+`clusterioPlugin` exposes `index.ts` (or `index.js`) as `"./"`, `package.json` as `"./package.json"` and `web/index.tsx` (or `.ts`, `.jsx`, `.js`) as `"./web"`.
+These are the same keys the plugin template used with webpack, so `webEntrypoint: "./web"` stays as it is.
+If your webpack config exposed other paths, move the code so it is reachable from `web/index.tsx`.
+
+The web interface provides `@clusterio/lib`, `@clusterio/web_ui`, `@sinclair/typebox`, `antd`, `react`, `react-dom`, `react-router` and `react-router-dom` to plugins.
+These are left as imports in the plugin bundle and resolved by the browser, there is no need to list them as shared.
+Everything else is bundled into the plugin, including `@ant-design/icons`.
+Import `react-dom/client` and `react/jsx-runtime` by those names, other subpaths of the shared packages are bundled as a separate copy.
+
+Imports of `assert`, `buffer`, `events`, `os`, `path`, `process`, `stream`, `util` and `zlib` resolve to the same browser packages as before, provided by `@clusterio/web_ui`.
+
+The `container` property of `WebPluginContext` is now the plugin's ES module namespace, `await container.get("./web")` returns the module instead of a factory function.
+
+`lib.PluginWebpackEnvInfo` is renamed to `lib.PluginWebEnvInfo`.
+
+
+## Renamed lib functions
+
+`lib.safeOutputFile` is renamed to `lib.atomicOverwriteFile`.
+It takes the same arguments.
+
+
+## Host pairing
+
+Host tokens are no longer generated by users.
+A host joins the cluster by having the code it shows when it first starts approved, see [Host pairing](protocol.md#host-pairing) for the HTTP endpoints it uses.
+
+- `HostGenerateTokenRequest` and `HostConfigCreateRequest` are removed.
+  Pending requests are listed with `HostPairingListRequest`, approved with `HostPairingApproveRequest` and rejected with `HostPairingRejectRequest`, and changes to them are sent to subscribers of `HostPairingUpdatesEvent`.
+- `HostRevokeTokensRequest` is replaced by `HostRevokeRequest`, which deletes the host instead of only invalidating its tokens.
+  Instances assigned to the host become unassigned.
+- The permissions `core.host.generate_token`, `core.host.revoke_token` and `core.host.create_config` are removed.
+  Check `core.host.approve` and `core.host.revoke` instead.
+- `clusteriocontroller bootstrap generate-host-token` is removed, scripts that set up hosts can start the host with `--user-token` to approve it with a user token.
+
+Host records are no longer created when a host connects, only by approving it, and the controller rejects hosts it has no record of.
+Revoking a host deletes its record and note.
+Plugins that keep data per host should remove it when the host is deleted.
+On the controller the `update` event of `controller.hosts.records` gives the record with `isDeleted` set to true, and subscribers of `HostUpdatesEvent` receive the host with `isDeleted` set to true.
+A host that is revoked and approved again gets a new id.
+
+
+## Host manager
+
+`controller.hosts` is now a `HostManager`, like `controller.users` is a `UserManager`.
+`get`, `getMutable`, `has` and `values` work as before.
+Everything else on the datastore, like `set`, `delete`, `size` and the `update` event, is on `controller.hosts.records`.
+
+The host manager also signs and checks host tokens and creates and deletes hosts:
+
+- `controller.generateHostToken(hostId)` is now `controller.hosts.signHostToken(hostId)`.
+- `controller.verifyHostToken(token)` is now `controller.hosts.verifyHostToken(token)`.
+  It returns the `HostRecord` instead of the host id.
+- `controller.hostPairing.createHost(name)` is now `controller.hosts.createHost(name)`.
+  It returns the new `HostRecord` instead of `{ hostId, token, name }`, sign a token for it with `signHostToken`.
+- `controller.hosts.revokeHost(hostId, revokedBy)` deletes a host the same way `HostRevokeRequest` does.
+
+User tokens are checked with `controller.users.verifyUserToken(token)`, which returns the user or throws if the token is invalid.

docs/devs/protocol.md

@@ -2,75 +2,132 @@
 
 Clusterio communicates using a simple homebrew message passing protocol based on JSON over WebSocket with limited support for reliable transport of messages.
 The protocol is implemented in [packages/lib/src/link](/packages/lib/src/link) and uses the `ws` library to handle the WebSocket connections on the Node.js side and the native WebSocket API on the browser side.
+The wire schemas are defined in [messages_core.ts](/packages/lib/src/data/messages_core.ts); see the [communication overview](../communication.md) for how links, connectors, and routing fit together.
 
 Two general kinds of messages are supported: events and requests.
 Events are messages sent by one party and received by the other and offer no notification for the sender on whether it was actually received.
 Requests are pairs of request-response messages where one side sends the requests and the other side replies with a response.
 Apart from the handshake used to establish connections the protocol is directionless, and either party can send or receive both requests and events.
 
-Events and requests are defined as instances of the `Request` and `Event` classes, and specify a unique type string, who can send and receive the message, optional forwarding of the message and the format of the data payload as a JSON schema.
-All messages sent and received are validated with JSON schemas to catch potential errors in sending messages and avoid having to write lots of validation code in message handlers.
-
-Messages are sent by calling the `.send()` method of the message instance and passing a `Link` instance to send it over and the data to send.
-For the controller there's a `Link` instance for each host and control connection stored in the `WsServer` class.
-For hosts the `Host` instance is a link to the controller, and each `Instance` instance is an in-memory link to the `Host` instance and has a corresponding `InstanceConnection` link stored on the `Host` instance.
-For ctl there's a `Ctl` instance and web_ui there's a `Control` instance that acts as the link to the controller.
-
-Messages received invokes message handlers that were registered when the link was brought up.
-These handlers are named after the name of the property the message is stored under with `EventHandler` appended for events and `RequestHandler` appended for requests.
-The handlers are defined on the link class itself for core messages, and on the plugin class in the case of plugin defined messages.
-
 
 ## Connection flow
 
 Clients connect via WebSocket to the `/api/socket` endpoint and upon establishing connection to the server the connection is in the handshake phase and the server will send a `hello` message to the client.
-If the client has no previous session with the server it sends a `register_host` or `register_control` message depending on what kind of client it is to the server.
-If the register is valid then the server will respond with a `ready` message with a session token and the connection has left the handshake phase, otherwise the server will close the connection with a WebSocket close code indicating the kind of failure.
+If the client has no previous session with the server it sends a `registerHost` or `registerControl` message depending on what kind of client it is to the server.
+If the registration is valid, the server responds with a `ready` message containing the client's address, a session token, and session parameters.
+The connection has then left the handshake phase; otherwise the server closes it with a WebSocket close code indicating the kind of failure.
 
 Upon leaving the handshake phase both parties should start sending heartbeat messages with the sequence number of the last received message.
-Messages sent over the link should be stored in a send buffer and only cleared from the send buffer when a heartbeat with a `last_seq` equal or or higher is received or the session is invalidated.
+Routable messages are stored in a send buffer until a heartbeat acknowledges an equal or higher sequence number, or the session ends.
+The heartbeat's top-level `seq` acknowledges receipt by the transport, not successful completion of a message handler.
 
-If the connection goes stale or is unexpectedly closed the client can reastablish the same session by connecting to the server again and sending a `resume` message after received the `hello` from the server.
+If the connection goes stale or is unexpectedly closed, the client can re-establish the same session by connecting again and sending a `resume` message after receiving `hello`.
 If the session resume is successful then the server will respond with a `continue` message and the connection is active again.
-The client should then re-send all messages it has in its send buffer that is greater than the `last_seq` number in the `continue` message.
-Otherwise the server will respond with `invalidate` and the client is expected to drop its send buffer start a new session with `register_host` or `register_control`.
+The client then re-sends buffered messages newer than the `lastSeq` in `continue`.
+The server likewise re-sends messages newer than the `lastSeq` supplied by the client in `resume`.
+Otherwise the server responds with `invalidate`, and the client discards the old send buffer and registers a new session.
+Session expiry, buffer overflow, or process failure can lose pending work; plugins must not treat transport recovery as an application-level exactly-once guarantee.
+The [send-buffer setting](../configuration.md#controllermax_send_buffer_size) controls how much unacknowledged message data the controller retains for a connection.
 
 ![Connection flow](../assets/connection_flow.svg)
 
 
+## Host pairing
+
+The host id and token sent in `registerHost` are requested by the host over HTTP the first time it starts, before it connects.
+Hosts are only added to the controller by approving a pairing request, and `registerHost` is rejected with close code 3000 (Unauthorized) for a host id the controller has no record of.
+Revoking a host deletes its record, so its token stops working and the host has to be approved again the next time it starts, which gives it a new id.
+See [Setting up remote hosts](/README.md#setting-up-remote-hosts) for how this looks to the user.
+
+The endpoints below take and return JSON bodies and are sent with `Cache-Control: no-store`.
+The `POST` endpoints return errors as `{"error": "<message>"}` and respond with 503 while the controller is in recovery mode.
+A request body that is not valid JSON or is larger than 4 kB is rejected with 400 or 413 before it reaches the endpoint, without a JSON body.
+`GET /api/host-pairing/check` also works in recovery mode, and responds to a token that is not valid with a plain 401 without a JSON body.
+A user or host token is sent in either an `Authorization: Bearer <token>` header or an `X-Access-Token` header.
+
+### `POST /api/host-pairing`
+
+Creates a request for a host to join the cluster.
+
+- `name: string` - Name of the host, at most 100 characters after trimming whitespace, without control characters.
+- `version: string` (optional) - Version of the host.
+
+Responds with 202 and `{"requestId": string, "code": string}`.
+The `code` is shown to the user to approve the request with, formatted as six characters like `ABC-DEF`.
+Codes are matched ignoring case and anything that is not a letter or digit, so `abcdef` matches `ABC-DEF`.
+The `requestId` is a secret used to poll for the result, it is never shown to users.
+
+If a user token is sent and the user has the `core.host.approve` permission the host is created right away and the response is 200 with the same body as an approved poll.
+An invalid token gives 401 and a user without the permission gives 403, a token is never ignored to create a pending request instead.
+
+A missing or too long name gives 400.
+Too many pending requests from the same address, or in total as set by [controller.host_pairing_max_pending](/docs/configuration.md#controllerhost_pairing_max_pending), gives 429 with a `Retry-After` header in seconds.
+
+### `POST /api/host-pairing/poll`
+
+- `requestId: string` - Id returned when the request was created.
+
+Responds with 202 and `{"status": "pending"}` while the request is waiting for approval.
+Once approved it responds with 200 and `{"hostId": integer, "token": string, "name": string}`, where `name` is the name the host was approved with, which can differ from the one asked for.
+Rejected requests give 403, and unknown or expired requests give 404.
+The result of an approved or rejected request is only returned once, after which the request is gone.
+
+A pending request expires 30 seconds after it was last polled, and after waiting for the time set by [controller.host_pairing_timeout](/docs/configuration.md#controllerhost_pairing_timeout).
+The host polls every 2 seconds, and makes a new request when the previous one has expired.
+
+### `POST /api/host-pairing/cancel`
+
+- `requestId: string` - Id returned when the request was created.
+
+Removes a pending request, sent by the host when it stops waiting.
+Responds with 204 whether or not the request existed.
+
+### `GET /api/host-pairing/check`
+
+Checks a host token.
+Responds with 200 and `{"valid": true}` if the token is valid and the host it was issued to still exists, and with 401 otherwise.
+A host with a saved token checks it with this when it starts, and requests to join again if the token is not accepted.
+
+
 ## Connector flow
 
 To manage the complexity of connecting, reconnecting, resuming or invalidating dropped sessions the WebSocket connection for a `Link` class is managed by a connector.
-There's three kinds of connectors defined: `WebSocketClientConnector`, `WebSocketServerConnector` and `VirtualConnector`.
+There's three kinds of connectors defined: `WebSocketClientConnector`, `WsServerConnector` and `VirtualConnector`.
 The virtual connector is for in-memory connections and doesn't have any connection state logic.
 
 For the WebSocket connectors there's four states they can exist in:
+
 - `closed` - The connector has just been created or has been explicitly closed.
 - `connecting` (client only) - The connector is in the process of trying to connect to the server.
 - `connected` - The connector is successfully connected.
 - `resuming` - The connection dropped and the connector is in the process of attempting to resume it.
 
 Along with the following events emitted when transition between states:
-- `connect` From `connecting` (client) or `closed` (server) to `connected`
-- `drop` - From `connected` to `resuming`
-- `resume` - From `resuming` to `connected`
-- `invalidate` - From `resuming` to `connecting` or `closed`
-- `close` - From any state to `closed`
+
+- `connect` - A new session is established, moving from `connecting` (client) or `closed` (server) to `connected`.
+- `drop` - The transport was lost but the session is retained for resumption, moving from `connected` to `resuming`.
+- `resume` - The retained session is reconnected and unacknowledged buffered messages are replayed, moving from `resuming` to `connected`.
+- `invalidate` - The previous session cannot be reused, moving from `resuming` to `connecting` (client only); server session expiry emits `close` instead.
+- `close` - The connector is closed, moving from any state to `closed`.
+
+Invalidation or closure rejects pending requests with `SessionLost`.
+Plugins must handle lost sessions, stale state, and application retries where required; see [connection handling](../writing-plugins.md#handling-connection-events) for the plugin hooks and queueing behavior.
 
 ![Conector States](../assets/connector_states.svg)
 
 
 ### WsClientConnector
 
-The client connector will keep trying to establish and mantain a connection after `.connect()` is called on it, stopping only in the case of authentication failure or `.close()`/`.setClosing()` being called.
-Other than that, the logic is pretty straight forward.
+The client connector, named `WebSocketClientConnector` in the source, keeps trying to establish and maintain a connection after `.connect()` is called, stopping on terminal failures such as authentication or policy rejection, or when `.close()`/`.setClosing()` is called.
+For example, policy violation (1008) stops retries, while try-again-later (1013), used for send-buffer overflow, permits reconnecting.
+Connection retries use exponential backoff.
 
 ![WsClientConnector](../assets/WsClientConnector.svg)
 
 
 ### WsServerConnector
 
-The server connector doesn't do the work of connecting or validating the handshake.
+The [server connector](/packages/controller/src/WsServerConnector.ts) doesn't do the work of connecting or validating the handshake.
 Instead it's handed over a WebSocket connection from `WsServer` once there's only the step of sending the final message left to do.
 Because of this there's no `connecting` state, and once a session times out from the `resuming` state it goes directly to closed.
 
@@ -79,19 +136,26 @@ Because of this there's no `connecting` state, and once a session times out from
 
 ## Wire format
 
-All communication in the protocol is done via WebSocket text frames containing a JSON payload with the following properties:
+All communication uses WebSocket text frames containing a JSON object.
+Every frame has a `type` string; the other fields depend on that type.
+Requests, responses, and events carry an integer transport sequence number in `seq`, plus source and destination addresses in `src` and `dst`.
+Message-specific data is carried in `data` when present.
 
-- `seq: integer | null` - The sequence number of the message, can only be null for handshake and heartbeats.
-- `type: string` - The type of message
-- `data: object` - Message data
+An address is `[type, id]`, optionally followed by a request ID: `[type, id, requestId]`.
+Address types are 0 for controller, 1 for host, 2 for instance, 3 for control, and 4 for broadcast.
+The controller's ID is 0; a broadcast address uses its ID to identify the target endpoint type.
+For example, `[2, 123]` addresses instance 123, while `[4, 2]` addresses all instances.
+Broadcast destinations are for events.
 
-Before a connection is established it's in the handshake phase and the `seq` property is `null`.
-The `type` property is used to distinguish which of handshake, heartbeat, events or requests this message is and determines the expected format of the `data` property.
+A request puts its request ID in `src`; the response returns that ID in `dst` so the sender can match it to the pending request.
+This ID is separate from transport `seq`, which is used for acknowledgements and buffering.
 
 
 ### Handshake Messages
 
-The following messages are the only valid messages that can be sent during the handshake phase.
+The following messages are used during the handshake phase.
+Their listed fields belong inside the frame's `data` object, except for `invalidate`, which has no data.
+Property names are case-sensitive.
 
 
 #### `hello`
@@ -102,7 +166,7 @@ Sent by the controller when the WebSocket connection has been opened.
 - `plugins: Object<string, string>` -
     Object mapping plugin names to plugin version for plugins that are loaded on the controller.
 
-#### `register_host`
+#### `registerHost`
 
 Host handshake for establishing a new connection session.
 
@@ -112,7 +176,12 @@ Host handshake for establishing a new connection session.
 - `plugins: Object<string, string>` -
     Object mapping plugin names to plugin version for plugins that are available on the host.
 
-#### `register_control`
+The controller closes the connection with code 3000 (Unauthorized) if the token is invalid, was issued for another id, or if there is no host with this id, see [Host pairing](#host-pairing).
+The host checks its token before connecting and requests to join again if it is not accepted, so this normally only happens to a host that is already running.
+A host that has been rejected exits with code 8, and the run scripts created by the installer do not restart it.
+A host that is revoked while connected is disconnected, is rejected when it tries to connect again, and shuts down.
+
+#### `registerControl`
 
 Control handshake for enstablishing a new connection session.
 
@@ -123,29 +192,32 @@ Control handshake for enstablishing a new connection session.
 
 Handshake for resuming an existing connection session where the connection had dropped unexpectedly.
 
-- `session_token: string` - Session token to resume session with.
-- `last_seq: integer | null` - Last message seq received by the client.
+- `sessionToken: string` - Session token to resume session with.
+- `lastSeq: integer` (optional) - Last transport sequence number received by the client; omitted if none has been received.
 
 #### `invalidate`
 
 Sent by the controller in reply to a resume for a session that cannot be resumed.
-The client should start a new session with a register\_\* message in response.
+The frame is `{"type":"invalidate"}`, without a `data` object.
+The client should start a new session with `registerHost` or `registerControl` in response.
 
 #### `continue`
 
 Sent by the controller to signal resuming the session was successful.
 
-- `last_seq: integer | null` - Last message seq received by the server.
-- `session_timeout: integer` - Session timeout in seconds.
-- `heartbeat_interval: number` - Rate client should send heartbeats at.
+- `lastSeq: integer` (optional) - Last transport sequence number received by the server; omitted if none has been received.
+- `sessionTimeout: number` - Session timeout in seconds.
+- `heartbeatInterval: number` - Interval between heartbeats in seconds.
 
 #### `ready`
 
 Sent by the controller to signal the connection has been established with a new session.
 
-- `session_token: string` - Token to resume the session with.
-- `session_timeout: integer` - Session timeout in seconds.
-- `heartbeat_interval: number` - Rate client should send heartbeats at.
+- `src: Address` - The client's address, used as the source of messages it sends.
+- `sessionToken: string` - Token to resume the session with.
+- `sessionTimeout: number` - Session timeout in seconds.
+- `heartbeatInterval: number` - Interval between heartbeats in seconds.
+- `account` (optional) - Account details for a control client: a `name` and a `roles` array whose entries contain `name`, `id`, and `permissions`.
 
 
 ### Heartbeat message
@@ -158,64 +230,84 @@ The heartbeat should contain the `seq` of the last received message, and receivi
 
 Sent by both the controller and clients at the interval given in the continue/ready messages.
 
-- `last_seq: integer` - Last message seq received over the link.
+- `seq: integer` (optional, top-level) - Last transport sequence number received over the link.
+
+For example, `{"type":"heartbeat","seq":123}` acknowledges messages through sequence 123.
+Before receiving any messages, send `{"type":"heartbeat"}`; there is no `data.last_seq` field.
 
 ### Events
 
-Events are messages send in one direction over the link, that the receiver is expected to act upon, but not make any replies (for that there's [Requests](#requests).
-By convention message types ending in `*_event` are events.
-The data payload of an event consists solely of event specific properties.
+Events are messages sent in one direction that the receiver acts upon without replying; use [requests](#requests) when a response is needed.
+Their wire `type` is `"event"`, with a separate `name` identifying the registered event class.
+Plugin names are prefixed, for example `foo_frobber:SomeEvent`.
+The `data` payload, when present, contains event-specific properties.
 
-See [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) for the recognized core events and their contents.
-Plugins may defined their own events as well.
+See [packages/lib/src/data/](/packages/lib/src/data/) for core event definitions.
+Plugins may define their own events as well.
 
-#### `*_event`
+#### `event`
 
-- any properties defined by the event type.
+- `seq: integer` - Transport sequence number.
+- `src: Address` - Source address.
+- `dst: Address` - Destination address, including broadcast destinations.
+- `name: string` - Registered event name.
+- `data` (optional) - Payload defined by the event class.
 
 
 ### Requests
 
-Requests are messages which expects a response in return.
-They function a lot like HTTP requests except that both parties of a connection can inniate a request that is expected to be responded to.
-By convention message types ending in `*_request` are requests, and is expected to be replied to with a corresponding `*_response` message.
-If an error occured while processing the request an error response containing an error property in the data payload is sent instead.
+Requests expect a response in return.
+They function like HTTP requests except that either party of a connection can initiate one.
+The wire `type` is `"request"`; a successful reply uses `"response"`, and a failed request uses `"responseError"`.
+The request's `name` identifies the registered class, prefixed with the plugin name for plugin messages, such as `foo_frobber:Frobnicate`.
 
-See [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) for the recognized core requests and their contents.
+See [packages/lib/src/data/](/packages/lib/src/data/) for core request definitions.
 Plugins may define their own requests as well.
 
-#### `*_request`
+#### `request`
 
-- any properties defined by the request type.
+- `seq: integer` - Transport sequence number.
+- `src: Address` - Source address, including the request ID.
+- `dst: Address` - Destination address.
+- `name: string` - Registered request name.
+- `data` (optional) - Payload defined by the request class.
 
-#### `*_response` success
+#### `response` success
 
-- `seq: integer` - The seq number of the request message.
-- any additional properties defined by the request type.
+- `seq: integer` - This response's transport sequence number, not the request's sequence number.
+- `src: Address` - Responding endpoint.
+- `dst: Address` - Requester's address, including the original request ID.
+- `data` (optional) - Payload defined by the request class's `Response` definition.
 
-#### `*_response` error
+#### `responseError`
 
-- `seq: integer` - The seq number of the request message.
-- `error: string` - Human readable text message describing the error.
+- `seq: integer` - This error response's transport sequence number.
+- `src: Address` - Responding endpoint.
+- `dst: Address` - Requester's address, including the original request ID.
+- `data.message: string` - Human-readable error description.
+- `data.code: string` (optional) - Registered name of the error class, `<plugin>:<class>` for plugin errors.
+  Missing or unknown names are received as `InternalServerError`.
+- `data.stack: string` (optional) - Error stack.
 
 
 ### Graceful Disconnect
 
-To ensure no data is lost when closing the link it's important to send a prepare disconnect request and wait for it to complete before closing the WebSocket.
-Either party of the link can send this request and the party sending the request should be the one that initates the close of the WebSocket.
+To reduce data loss when closing a link, ask the peer to prepare and wait for it to finish before closing the WebSocket.
+Either party can initiate the disconnect and is then responsible for closing the socket.
+These are connector control frames, not registered request/response classes; forced closure and process crashes can still lose work.
 
-#### `prepare_disconnect_request`
+#### `disconnect` with `data: "prepare"`
 
-Request graceful disconnect of the connection.
+The frame `{"type":"disconnect","data":"prepare"}` requests a graceful disconnect.
 The receiving party should stop tasks that send events and requests over the link and wait for pending requests it has over the link to complete, but it should not stop responding to requests sent to it.
 Only after this process is done should the response be sent.
 
-#### `prepare_disconnect_response`
+#### `disconnect` with `data: "ready"`
 
-Signals the preparation for disconnect is complete and that the connection can be closed.
-The party that initiatied the prepare disconnect request may still send other requests/events over the link to finish its part of the disconnection proceedure.
-
-If the error response variant is received it should be logged if possible and treated as a success.
+The frame `{"type":"disconnect","data":"ready"}` signals that preparation is complete and the connection can be closed.
+When using `WebSocketBaseConnector.disconnect()`, the initiator must stop producing new work and await its own `Link.prepareDisconnect()` before calling `disconnect()`.
+The connector closes as soon as the peer sends `ready` or the timeout expires; it does not drain the initiator's pending requests itself.
+There is no error-response variant; the connector also has a timeout so an unresponsive peer cannot block shutdown indefinitely.
 
 
 ### Example handshake
@@ -224,10 +316,10 @@ Server hello after opening the WebSocket.
 
 ```json
 {
-    "seq": null,
     "type": "hello",
     "data": {
-        "version": "2.0.0"
+        "version": "2.0.0",
+        "plugins": {}
     }
 }
 ```
@@ -236,12 +328,12 @@ Host registering a new connection session.
 
 ```json
 {
-    "seq": null,
-    "type": "register_host",
+    "type": "registerHost",
     "data": {
         "token": "xyzabc123",
         "version": "2.0.0",
-        "id": 123
+        "id": 123,
+        "plugins": {}
     }
 }
 ```
@@ -250,12 +342,12 @@ Controller reply signaling the session is established and ready.
 
 ```json
 {
-    "seq": null,
     "type": "ready",
     "data": {
-        "session_token": "xyz.azy.bzy",
-        "session_timeout": 60,
-        "heartbeat_interval": 15
+        "src": [1, 123],
+        "sessionToken": "xyz.azy.bzy",
+        "sessionTimeout": 60,
+        "heartbeatInterval": 15
     }
 }
 ```

docs/how-it-works.md

@@ -1,3 +1,25 @@
+# How Clusterio works
+
+Clusterio runs a group of Factorio servers as a cluster, managed through a web interface or the `clusterioctl` command-line client.
+The controller is the central point of contact: clients connect to it to manage the cluster, and hosts connect to it to receive commands and exchange messages.
+Each host manages its local Factorio server processes, with each server belonging to an instance.
+
+
+## System overview
+
+The diagram shows one possible layout with two physical servers, each running a host and three Factorio servers.
+The controller shares a machine with one host in this example, but it can run on a separate machine.
+The number of hosts and instances can vary with the needs of the cluster.
+
+![Clusterio system diagram showing a controller, web and command-line clients, and two hosts managing three Factorio servers each](assets/clusterio_2.0_block_diagram.svg)
+
+The blue arrows point from the hosts and clients to the controller they connect to; messages can travel in both directions over those WebSocket connections.
+The orange lines connect each host to the Factorio processes it manages.
+Open the [full-size diagram](assets/clusterio_2.0_block_diagram.svg) to zoom in, or use the [Draw.io source](assets/clusterio_2.0_block_diagram.drawio) to edit it.
+
+For how messages are handled and routed between these components, see the [communication guide](communication.md).
+
+
 ## Save Patching
 
 In order to efficiently get data in and out of running Factorio servers and to provide custom interfaces Clusterio uses save patching<sup>[1]</sup> to add code to the game.
@@ -5,6 +27,7 @@ The save patching is based on the built in [event_handler](https://github.com/wu
 Mods on the other hand work without modifications.
 
 This can be disabled with the `factorio.enable_save_patching` config on a per instance basis, however disabling it means that modules are not loaded into the game and most plugins will not function.
+A save that was patched before has its modules removed again when it's started with save patching disabled.
 To use the modules outside of Clusterio the `npx clusteriohost create-scenario` command creates a scenario with the modules patched into it.
 Note that the IPC between the game and the host is not available outside of Clusterio, so modules that depend on it will not work as expected in such a scenario.
 

docs/managing-a-cluster.md

@@ -16,23 +16,47 @@ The easiest way to do this is to run `npx clusteriocontroller bootstrap create-c
 
 Lists all hosts.
 
-### Generate token
+Hosts are added to the cluster by running the installer on the computer the host runs on and approving the code it shows, see [Setting up remote hosts](/README.md#setting-up-remote-hosts).
+Listing, approving and rejecting hosts waiting for approval needs the `core.host.approve` permission.
 
-    ctl> host generate-token [--id <host id>]
+### List pending hosts
 
-Generate a token for a host. `--id` is the host id.
+    ctl> host pending
 
-### Create config
+Lists hosts waiting for approval with the code they show, the name they asked for, the address they connected from and how long they have been waiting.
+A request goes away when its host stops waiting, and after the time set by `controller.host_pairing_timeout`, an hour by default, after which the host asks again with a new code.
 
-    ctl> host create-config [--id <host id>] [--name <host name>] [--generate-token <bool>] [--output <path to output>]
+### Approve host
 
-Create a host config. `--id` id the host id. `--name` is the host name. `--generate-token` decides whether to also generate a token, defaulting to `0`. `--output` is the filename for the output of the config; Default is "config-host.json", "-" for stdout.
+    ctl> host approve <code> [--name <host name>]
+
+Approves the host showing the code and prints the id of the new host.
+The code is not case sensitive and the dash can be left out.
+`--name` gives the host a different name than the one it asked for.
+Approving gives the host full access to the cluster, check with `host pending` that the name and address match the computer you are setting up.
+
+### Reject host
+
+    ctl> host reject <code>
+
+Rejects the host showing the code, the host then exits with an error.
+
+### Revoke host
+
+    ctl> host revoke <host>
+
+Specify the host by name or ID.
+Removes the host from the cluster, disconnects it and makes its token stop working.
+Instances assigned to the host become unassigned, their files stay on the computer.
+Requires the `core.host.revoke` permission.
+To add the computer back to the cluster, start the host again and approve it.
 
 ### Update Clusterio
 
-    ctl> host update [--id <host id>] [--name <host name>] [--restart]
+    ctl> host update <host> [--all] [--restart]
 
-If remote updates are enabled on the host, this will attempt to update `@clusterio/host` on the machine. Changes will not be applied until the host is restarted. If the `--restart` flag is provided, the host will automatically restart once the update is complete.
+Specify the host by name or ID.
+If remote updates are enabled on the host, this will attempt to update `@clusterio/host` on the machine. Add `--all` to also update all npm-installed plugins. Changes will not be applied until the host is restarted. If the `--restart` flag is provided, the host will automatically restart once the update is complete.
 
 Remote updates are enabled by default but can be disabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the following command locally while the host is offline:
 
@@ -40,7 +64,7 @@ Remote updates are enabled by default but can be disabled on a per-machine basis
 
 The controller also supports remote updates, following the same logic. The corresponding commands are:
 
-    ctl> controller update [--restart]
+    ctl> controller update [--all] [--restart]
     controller> config set controller.allow_remote_updates true/false
 
 ### Mark restart required
@@ -55,15 +79,16 @@ Marks the host as requiring a restart, the same way a plugin update does. Use it
 
 ### List Instances
 
-    ctl> instance list
+    ctl> instance list [--templates]
 
 List instances.
+Templates are only listed if `--templates` is passed.
 
 ### Create instance
 
-    ctl> instance create <name> [--id <instance id>]
+    ctl> instance create <name> [--id <instance id>] [--from <instance id>] [--template <instance>] [--as-template]
 
-Create a new instance. `name` is the instance name. `--id` is the optional id.
+Create a new instance. `name` is the instance name. `--id` is the optional id. `--from` copies the config of an existing instance. `--template` makes the new instance inherit all config values from the given template. `--as-template` creates the instance as a template, see `instance.is_template` in the [configuration docs](configuration.md). Use `--as-template --from <instance>` to make a template out of an existing instance.
 
 ### Kill instance
 
@@ -96,7 +121,7 @@ Most notably Clusterio records wall clock time while the save time is recorded i
 
     ctl> instance config list <instance>
 
-List config values for an instance. `instance` is the instance whose config values to list.
+List config values for an instance. `instance` is the instance whose config values to list. Values inherited from a template are marked with `(inherited)`, object fields also list the properties set on the instance.
 
 #### Set config value
 
@@ -108,15 +133,21 @@ Set field in instance config. `instance` is instance to set. `field` is field to
 
 To be written.
 
+#### Inherit config value
+
+    ctl> instance config inherit <instance> <field> [prop]
+
+Make a field in the instance config take its value from the template again, undoing a previous `set`. `instance` is the instance to update, `field` is the field to inherit, `prop` limits it to one property of an object field.
+
 #### Edit instance config
 
     ctl> instance config edit <instance> [editor]
 
-Edit instance config in a text editor. Once file is saved and editor has been exited, config values will be read back in. `instance` is the instance to edit, `editor` is the editor to use. If `editor` is not given, clusterioctl will read from the `EDITOR` or `VISUAL` enviroment variables.
+Edit instance config in a text editor. Once file is saved and editor has been exited, changed config values will be read back in. `instance` is the instance to edit, `editor` is the editor to use. If `editor` is not given, clusterioctl will read from the `EDITOR` or `VISUAL` enviroment variables.
 
 ### Assign instance to host
 
-    ctl> instace assign <instance> [host]
+    ctl> instance assign <instance> [host]
 
 Assigns instance to host. `instance` is instance to assign, `host` is host to assign to. If `host` is not provided, instance will be unassigned.
 
@@ -230,6 +261,38 @@ Note: If the player joins the cluster again a new account will be made for them
 To be written.
 
 
+## Notes
+
+Free text notes can be attached to hosts, instances, users, roles, mod packs, mods and saves.
+They are stored on the controller and shown to everyone with access to the resource in the web interface, and can for example be used to record who runs a host or what an instance is meant for.
+
+The web interface renders notes as Markdown.
+A mention of the form `@<type>/<id>` becomes a link to that resource showing its name, for example `@instance/778528005` or `@user/cooldude2606`.
+This works for hosts, instances, users, roles and mod packs, and the id is shown if the name is not available.
+The note itself stores the mention as written.
+
+### List notes
+
+    ctl> note list [--type <type>]
+
+Lists all notes, optionally limited to one resource type.
+
+### Set note
+
+    ctl> note set <type> <id> <content>
+
+Sets the note of a resource.
+The type is one of host, instance, user, role, mod_pack, mod or save.
+The id is the numeric id for hosts, instances, roles and mod packs, the lower case name for users, `<name>_<version>` for mods and `<instance id>/<file name>` for saves.
+
+### Delete note
+
+    ctl> note delete <type> <id>
+
+Removes the note of a resource.
+Notes are also removed when the resource they are attached to is deleted.
+
+
 ## Plugins (Local)
 
 To be written.
@@ -237,9 +300,11 @@ To be written.
 
 ## Plugins (Remote)
 
-By default, remote updates are enabled unless you selected "no" during installation.
-Remote updates allow ctl and the web UI to update plugins on machines within the cluster.
-However, remote installations are disabled by default to prevent the risk of arbitrary code execution through the installation of malicious npm packages onto the machine. See below if you wish to enable this feature.
+The installer asks "Allow remote updates and plugin installs from the web interface?" and the answer sets `allow_remote_updates`, `allow_plugin_updates` and `allow_plugin_install` together on that machine, pass `--remote-npm` or `--no-remote-npm` to answer it from the command line.
+Remote updates allow ctl and the web UI to update Clusterio and plugins on machines within the cluster.
+Remote installs allow them to install any package from npm onto the machine, which is a risk of arbitrary code execution through malicious npm packages, so only allow them if you trust everyone with access to install plugins.
+In a config created without the installer remote updates are enabled and remote installs are disabled by default.
+See below for changing these settings after installing.
 
 ### List
 
@@ -250,10 +315,15 @@ This command lists all plugins currently known to the remote. Newly installed pl
 
 ### Update
 
-    ctl> controller plugin update <name> [--restart]
-    ctl> host plugin update <host> <name> [--restart]
+    ctl> controller plugin update <name>[@<version>] [--restart]
+    ctl> host plugin update <host> <name>[@<version>] [--restart]
 
-If remote plugin updates are enabled on the remote, this will execute `npm update --save <name>`. Changes will not be applied until the remote is restarted. If the `--restart` flag is provided, the remote will automatically restart once the update is complete.
+If remote plugin updates are enabled on the remote, this will execute `npm update --save <name>`, which stays within the version range saved in the remote's `package.json`.
+To update to a specific version, including one outside that range, pass it as `<name>@<version>`, for example `@clusterio/plugin-example@2.1.0`.
+The version must be published on npm, and the remote then executes `npm install --save <name>@<version>`.
+The plugin must already be installed on the remote, and only remote plugin updates need to be enabled.
+Changes will not be applied until the remote is restarted.
+If the `--restart` flag is provided, the remote will automatically restart once the update is complete.
 
 ### Update All
 
@@ -264,10 +334,30 @@ Updates the Clusterio package and all plugins installed from npm in a single `np
 
 ### Install
 
-    ctl> controller plugin install <name> [--restart]
-    ctl> host plugin install <host> <name> [--restart]
+    ctl> controller plugin install <name>[@<version>] [--restart]
+    ctl> host plugin install <host> <name>[@<version>] [--restart]
+
+If remote plugin installs are enabled on the remote, this will execute `npm install --save <name>`.
+Pass `<name>@<version>` to install a specific version, for example to match the version used by the rest of the cluster.
+The package, and the version if given, must be published on npm.
+Changes will not be applied until the remote is restarted.
+If the `--restart` flag is provided, the remote will automatically restart once the update is complete.
+
+### Search
+
+    ctl> controller plugin search [query] [--page <n>] [--page-size <n>]
+
+Searches the npm registry for packages tagged with the clusterio-plugin keyword.
+The search is done by the controller and does not require remote installs to be enabled.
+The web UI has a plugin browser on the plugins page that uses the same search.
+
+### Concurrent Operations
+
+Only one npm operation runs at a time on a machine.
+A standalone controller and host share their working directory, so the operation holds the `npm-operation.lock` file there while npm runs.
+Other installs and updates on the same machine or installation are rejected with "Another package operation is already in progress on this machine" until it finishes.
+A lock file left behind by a process that is no longer running is removed automatically.
 
-If remote plugin installs are enabled on the remote, this will execute `npm install --save <name>`. Changes will not be applied until the remote is restarted. If the `--restart` flag is provided, the remote will automatically restart once the update is complete.
 
 ### Enable/Disable Updates
 

docs/migrating-from-2.0.0-alpha.13.md

@@ -1,46 +0,0 @@
-# Migrating from 2.0.0-alpha.13 to 2.0.0-alpha.14
-
-Alpha 14 comes with significant breaking changes for existing clusters, most notably the rename of master and slave to controller and host.
-Please study the entire migration guide carefully before attempting to update an existing cluster.
-
-## Before updating
-
-If you're using mods you need to create mod pack string for the mod configurations you're currently running.
-You can do this by running the follow RCON command on an instance that is currently running the mod configuration you want to use (you can also run this command in single player and find the string printed in Factorio's [Log file](https://wiki.factorio.com/Log_file)):
-
-    /sc local m={name="pack",description="",factorio_version=game.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}} for n,v in pairs(game.active_mods) do table.insert(m.mods,{name=n,enabled=true,version=v}) end local function set(s,c) for n,v in pairs(s) do m.settings[c][n]=v end end set(settings.startup,"startup") set(settings.global,"runtime-global") if game.player then set(game.player.mod_settings,"runtime-per-user") else set(settings.player,"runtime-per-user") end local e=game.encode_string(game.table_to_json(m)) log(e) rcon.print(e)
-
-If you for some reason want to run this function in 2.0+, use the following command:
-
-    /sc local a={name="pack",description="",factorio_version=script.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}}for b,c in pairs(script.active_mods)do table.insert(a.mods,{name=b,enabled=true,version=c})end;local function d(e,f)for b,c in pairs(e)do a.settings[f][b]=c end end;d(settings.startup,"startup")d(settings.global,"runtime-global")if game.player then d(game.player.mod_settings,"runtime-per-user")else d(settings.player,"runtime-per-user")end;local g=helpers.encode_string(helpers.table_to_json(a))log(g)rcon.print(g)
-
-This should produce a [Mod Pack String](https://forums.factorio.com/viewtopic.php?f=96&t=103578) which you need to copy and paste somewhere to keep it for later.
-Mod Pack strings looks something like this:
-
-    eNqFketuwyAMhd/Fv0OUVpW25lWmqSLESdAIRNh0l4h3n9mlitZt5QdCx/7so8MKXs8ILSzaPEEFPZKJdmEbvIgiDNpwiDaczhjpU93Vu/r+Tmpz6Anah/V7RqcJRUavO4c9tBwTVnAF5upCGJeIsYx3tvsPbQTdb0lKHYlnPJHY0+ONvcdjvW8gP1ZAyGz9KL5XINaR0yLPXEFMnu2ManSh0+6j/GOFitqXG4mjNSUidVm5wlm7JMYG7Qhl3BX8FjyqZ9vztOk+NM2fvRPaceLbzbN+UejQFFOWXzfArvk6v2HWD9ZLv5JfxA1UwsubPBaMKhHGklLO76vAwfQ=
-
-If you have multiple mod configuration create a mod pack string for each and note them down.
-
-Double check that you are on a supported version of Node.js by running `node --version`, the version should be v18 or later, if not you will need to update Node.js before continuing.
-
-
-## Migrate installations
-
-**Note:** Check that plugins you are using have updates avaialble for alpha 14 before attempting to update, if they do not either wait for the plugin to be updated or uninstall the plugin from all places it is installed on first.
-
-Shutdown all slaves and the master server and then migrate each installations by running the following command:
-
-    npm exec @clusterio/create -y -- --migrate-rename --log-level=verbose
-
-This will convert the existing installation's usage of the master and slave names in packages, configs, database and logs to the new controller and host terms and update the installed packages.
-
-If you're using systemd then copy the generated clusteriocontroller.service and/or clusteriohost.service files from the systemd folder to /etc/systemd/system/ and remove the old clusteriomaster.service and clusterioslave.service files.
-
-Once the installations have been migrated start up the controller and hosts again.
-
-
-## Post installation
-
-Import the Mod Pack strings that was created beforehand on the controller and upload the mods needed by those mod packs to the controller.
-You can assign a default mod pack in the controller config which instances use by default or set the mod pack on the per instance basis in the instance config.
-Once you have assigned a mod pack to an instance run a data export on the instance in order to properly populate the mod settings in the mod and get item icons and item names resolved in the web interface.

docs/pr-review-guide.md

@@ -0,0 +1,83 @@
+# Reviewing pull requests
+
+Review observable behavior across the whole PR, including callers, consumers, and lifecycle transitions.
+Reading every changed file and passing tests do not establish that those behaviors were covered.
+Use [contributing](contributing.md) for setup and validation commands.
+
+
+## Establish the scope
+
+- Record the PR's base and current head; preserve unrelated work with an isolated checkout when needed.
+- Read the requirements and relevant documentation at the reviewed revision.
+  Distinguish intended policy from observed behavior when they disagree.
+- Map each affected behavior from input through authorization, routing, state changes, and its consumers.
+  On a follow-up, verify the fixes and reassess the whole PR's affected behavior; explicitly label a deliberately narrower review.
+- Compare the PR base, previously reviewed head, and current head when attributing a follow-up finding.
+  Distinguish a new regression, an incomplete fix, and a previously missed defect.
+  Existing producer behavior can become unsafe through a new consumer; reject a finding as pre-existing only after comparing the complete failing path.
+- Trace the actual entrypoint and earlier guards before declaring a failure reachable.
+  Check supported configurations and compatibility policy; a merged revision or earlier clean review is not proof of correctness.
+
+
+## Surface overlapping responsibilities
+
+- For substantial new or replaced functionality, check whether existing code or test helpers already serve the same responsibility, including outside the changed files.
+  Search by behavior, then verify the implementations and their callers; nearby code is not necessarily the preferred shared implementation.
+- Compare contracts, callers, and constraints, not just names or similar code.
+  Distinguish shared behavior and general correctness requirements from differences that may justify separate implementations.
+- Surface significant overlap to the developer with source locations for both implementations, the duplicated responsibility, its consequences, and any legitimate differences.
+  Explain maintenance or consistency risks without presenting them as reproduced runtime defects.
+- Recommend an approach based on the evidence, including alternatives, tradeoffs, and affected callers or tests.
+  Leave the architectural decision to the developer unless already authorized; do not silently consolidate the implementations or dismiss the overlap as intentional.
+- Recheck after substantial fixes, especially when shared functionality is replaced, bypassed, or overridden.
+  Keep unresolved design questions in the candidate ledger and final report, and record the developer's decision when provided.
+
+
+## Exercise the boundaries
+
+- Write a small matrix of relevant states and transitions before probing.
+  Distinguish intended state, eligibility, and observed state, and vary the conditions that can make them differ.
+  Exercise transitions through the components that produce and consume the state; constructing the final state alone does not verify the transition.
+- Check adverse paths as well as success: failure, retry, removal, restart, reconnect, and overlapping operations where applicable.
+  Select relevant cases from these recurring review gaps:
+
+  | Boundary | Questions to check |
+  | --- | --- |
+  | Data contracts | Does real serializer output satisfy every consumer, including Lua restoration and inspection UI? Check omitted, empty, partial, malformed, reserved, and numeric-limit values before durable mutation. |
+  | State and identity | Do delayed reads and mutation completions still belong to the selected entity? Do disabled components retain their local state? Can an identifier or generation be reused after recovery? |
+  | Persistence and synchronization | Do save/reload, migration, deletion, and resubscription preserve versions and tombstones? Compare full snapshots, deltas, and optimized paths with ordinary transfers. |
+  | Ownership and errors | Can concurrent work remove another operation's lock, handle, or cache? Check synchronous throws, early promise rejection, partial mutation, and cleanup after each failure. |
+  | Protocol and compatibility | What does the receiver do with the message, close code, or acknowledgement? Check early startup, mixed versions, and supported older APIs. |
+  | Distribution | Do generated validators, browser bundles, and installed packages behave like the source checkout? Check actual export maps and plugin discovery. |
+
+- Use focused reproductions with concrete inputs and expected versus actual results.
+  Keep the logic under review real: stubbing permissions, effects, or state transitions can erase the defect being investigated.
+  Check that the harness reaches the intended path; compare a normal case and the base revision where practical.
+  Identify assumptions introduced by fixtures, caches, shared identities, or execution order, and vary those relevant to the change.
+- Control concurrency with explicit gates and assertions about ownership and completion, rather than timing alone.
+  Check both that invalid actions are prevented and that valid work eventually makes progress.
+  Vary delays and completion order, including changes to an action's preconditions between initiation and execution.
+
+
+## Account for every candidate
+
+- Keep a compact ledger of candidate issues and uncovered scenarios throughout the review.
+  Give each candidate a disposition: reported, rejected with evidence, or unresolved with the next check needed.
+- Before finishing, compare probe outputs with the final findings.
+  A reproduced anomaly must not disappear just because several other issues were already found.
+- Carry unresolved candidates and coverage gaps into handoffs and context summaries, alongside published findings.
+  For each finding, identify the trigger, consequence, source location, and supporting evidence.
+  Reconcile the previous ledger at every follow-up; an unconfirmed candidate remains unresolved until evidence rejects it or confirms it.
+  Name remaining material gaps in the final report even when other findings already block merging.
+
+
+## Report and follow up
+
+- Separate source inspection, focused probes, full suites, hosted CI, browser checks, and Factorio runtime evidence.
+  Record failures, skips, and unavailable validation; investigate external-service failures before attributing them to the PR.
+- Reproduce the original failure when checking a fix, including adjacent cases exposed by the change.
+  A fix for one malformed field calls for checking the remaining consumer requirements, not just that example.
+  Limit conclusions to the scenarios exercised, and state which relevant assumptions remain unverified.
+  Resolve a review thread only when its exact concern is addressed.
+- When authorized to publish, refresh the head and existing comments, avoid duplicates, and anchor feedback to the reviewed code.
+  Read back each posted inline comment to verify its body and location.

docs/release-workflow.md

@@ -11,16 +11,21 @@ If the script fails due to GitHub's 60/hour public access rate limit, you can se
 
 ## Packages
 
-The [Publish workflow](../.github/workflows/publish.yml) publishes npm packages and Lua mods when a GitHub release is published.
+The [Publish workflow](../.github/workflows/publish.yml) publishes npm packages and Lua mods on GitHub's `release: released` event for a stable release.
+Publishing a prerelease does not trigger it; see [GitHub's release event documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#release).
+The npm release includes the packages under `packages/` and the six core plugins under `plugins/`.
+Community plugins hosted in separate repositories have independent versions and releases.
 For this workflow, first merge the changelog and package version updates into `master`, then create and push the release tag on that merged commit.
 Create a draft GitHub release for the existing tag and use the GitHub Release Markdown from `scripts/pr_to_changelog.mts` as its notes.
-Publishing the draft triggers the workflow; pushing a tag alone does not.
+Publishing the draft as a stable release triggers the workflow; pushing a tag alone does not.
+Preparing a release PR, pushing a tag, and publishing a release or packages are separate actions; obtain authorization for the publication steps before executing them on someone else's behalf.
 The steps below describe publishing npm packages manually instead.
 
 > [!NOTE]
 > You will need commit access to the repository as well publish access to the @clusterio org on npm to do this.
 
-1. Ensure you are on the latest version of master branch with no untracked files
+1. Inspect `git status` and `git remote -v`, verify which remote is the main repository, and use a clean release worktree on its latest master.
+   The commands below assume that remote is named `upstream`; preserve unrelated work and external plugin checkouts.
 
     ```sh
     git switch master
@@ -28,31 +33,45 @@ The steps below describe publishing npm packages manually instead.
     git status
     ```
 
-2. Bump the version in the `package.json` files
+2. Create a release branch and bump the version in the `package.json` files. The commands below use 2.0.1 to 2.0.2 as an example; substitute the versions for the release you are preparing.
 
     ```sh
-    sed -i 's/2\.0\.0-alpha\.XX/2\.0\.0-alpha\.YY/' {packages,plugins}/*/package.json
-    ````
+    git switch -c release/2.0.2
+    sed -i 's/"version": "2\.0\.1"/"version": "2.0.2"/' {packages,plugins}/*/package.json
+    pnpm install --no-frozen-lockfile
+    ```
 
-3. Tag and push the release to github
+3. Review and commit the version and changelog changes, then open and merge a release pull request.
 
     ```sh
     git add -p
-    git commit -m v2.0.0-alpha.XX
-    git tag v2.0.0-alpha.XX
+    git commit -m "Prepare 2.0.2 release"
     ```
 
-4. Perform a clean install and publish the packages.
+4. After the release pull request is merged, tag the merged commit and push the tag.
 
     ```sh
+    git switch master
+    git pull --ff-only upstream master
+    git tag v2.0.2
+    git push upstream v2.0.2
+    ```
+
+5. For manual publication, perform a clean install and publish the packages. For automated publication, publish the GitHub release as described above instead.
+
+    The repository does not track `pnpm-lock.yaml`, so allow pnpm to generate or update it during installation.
+
+    ```sh
+    node clean.js --dry-run
     pnpm clean
+    pnpm install --no-frozen-lockfile
     pnpm -r --filter='./packages/*' --filter='./plugins/*' --fail-if-no-match publish
     ```
 
 ## Plugins
 
 1. Bump the version in the `package.json` file
-2. Run `pnpm install` in the repository root
+2. Run `pnpm install --no-frozen-lockfile` in the repository root
 3. Run `pnpm adduser` to authenticate
 4. Run `pnpm publish` in the plugin/package directory
 

docs/setting-up-tls.md

@@ -63,7 +63,7 @@ This creates a private key file named `key.pem` and a public certificate file na
     npx clusteriocontroller config set controller.tls_certificate cert.pem
     npx clusteriocontroller config set controller.tls_private_key key.pem
 
-Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure node to trust it by setting the `NODE_EXTRA_CA_CERTS` environment variable to a path to this file before the host is started. You can do this by passing `--self-signed-cert path/to/cert` to the installer when setting up a host, or manually by editing run-host.cmd, run-host.sh and/or clusteriohost.service by adding the following line next to the one setting `NODE_OPTIONS`: 
+Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure node to trust it by setting the `NODE_EXTRA_CA_CERTS` environment variable to a path to this file before the host is started. You can do this by passing `--self-signed-cert path/to/cert` to the installer when setting up a host, which also makes the installer trust the certificate when it asks the controller to approve the host, or manually by editing run-host.cmd, run-host.sh and/or clusteriohost.service by adding the following line next to the one setting `NODE_OPTIONS`:
 
 **For run-host.cmd**
 

docs/writing-plugins.md

@@ -16,6 +16,7 @@ Each entrypoint of a plugin exports a function that attaches handlers to hooks c
 - [Defining Link Messages](#defining-link-messages)
 - [Sending Link Messages](#sending-link-messages)
     - [Handling connection events](#handling-connection-events)
+- [Web Pages for a Single Resource](#web-pages-for-a-single-resource)
 - [Collecting Statistics](#collecting-statistics)
 - [Adding Custom Commands to clusterioctl](#adding-custom-commands-to-clusterioctl)
 
@@ -39,6 +40,11 @@ The basic file structure of a plugin is the following.
 Clusterio plugins are Node.js packages that can be published on npm for ease of installation and distribution.
 The usual guides for creating such packages apply.
 At minimum the `package.json` file must contain a version entry.
+For automatic discovery, include `"clusterio-plugin"` in the package's `"keywords"` array.
+Installed npm plugins are discovered from the installation's dependencies when controller, host or ctl loads its configuration, and added to `plugin-list.json`.
+Discovery starts with the root package's `dependencies`, not `devDependencies`, and follows dependencies of recognized plugins.
+It does not search arbitrary nested dependency trees for plugins.
+For tooling that needs the same discovery behavior, reuse [loadPluginList](../packages/lib/src/load_plugin_list.ts) rather than maintaining another package scanner.
 
 A possible workflow for developing plugins is to place the plugin in a sub-directory of where clusterio has been installed, and rely on Node.js searching up the folder heirarchy for it to find `@clusterio/lib`.
 After creating a directory for your plugin, the boilerplate can be generated automaticlly from a set of tempaltes using `npm exec @clusterio/create -y -- --plugin-template` after which you will be asked which areas your plugin will interact with and your plugin's name.
@@ -47,26 +53,50 @@ Note that it's important that the path starts with ./ or ../ (use .\ or ..\ on W
 
 For a plugin to be recognised by Clusterio it needs to export an entry named `plugin` from its main entrypoint.
 By default the main entrypoint is the `index.js` file, but this may be changed by setting the `"main"` entry to a different file in `package.json`.
+Keep this `"main"` entry when using a package `"exports"` map: Clusterio currently loads the plugin implementation from `"main"` (or `index.js`), rather than selecting a conditional export.
+
+If a plugin published on npm defines an `"exports"` map, it must also expose `"./package.json"`.
+Both automatic discovery and runtime loading resolve `<package-name>/package.json` to read the plugin's metadata.
+Packages that hide this file are currently skipped by discovery, and manually adding their package names to `plugin-list.json` does not make them loadable.
+The generated plugin templates omit `"exports"`, which leaves `package.json` accessible.
+For example, a plugin that needs an export map can use:
+
+```json
+{
+    "name": "example-clusterio-plugin",
+    "version": "1.0.0",
+    "type": "module",
+    "main": "./index.js",
+    "keywords": ["clusterio-plugin"],
+    "exports": {
+        ".": "./index.js",
+        "./package.json": "./package.json"
+    }
+}
+```
+
 Here's an example of `index.js`:
 
 ```js
+import * as messages from "./messages.js";
+
 export const plugin = {
     name: "foo_frobber",
     title: "Foo Frobber",
     description: "Does advanced frobnication",
     instanceEntrypoint: "instance.js",
     controllerEntrypoint: "controller.js",
-    messages: {
-        /* See below */
-    },
+    messages: [messages.Frobnicate],
 };
 ```
 
 The following properties are recognized:
 
 **name**:
-    Internal name of the plugin.
-    Must be the same as the plugin's directory name
+    Stable internal identifier of the plugin, used as its key in the plugin list.
+    For local plugins automatically discovered under `plugins/` or `external_plugins/`, it must match the plugin's directory name.
+    It does not have to match the npm package name.
+    For example, the package `@clusterio/plugin-subspace_storage` declares the name `subspace_storage`.
 
 **title**:
     Name of the plugin as shown to users.
@@ -90,28 +120,45 @@ The following properties are recognized:
     This is an optional parameter.
     A plugin can be made that only runs on the controller.
 
-**ControllerConfigGroup**:
-    Object mapping controller configuration fields to `FieldDefinitions`for this plugin.
+**controllerConfigFields**:
+    Object mapping controller configuration fields to `FieldDefinition` objects for this plugin.
     See [Plugin Configuration](#plugin-configuration)
 
+**hostEntrypoint**:
+    Optional path to the Node.js module exporting the host entrypoint function.
+
+**hostConfigFields**:
+    Object mapping host configuration fields to `FieldDefinition` objects.
+
 **ctlEntrypoint**:
     Path to a Node.js module relative to the plugin directory which exports the clusterioctl entrypoint function for this plugin.
     This is an optional paramater.
     A plugin can be made that only runs on the clusterioctl side.
 
+**ctlConfigFields**:
+    Object mapping ctl configuration fields to `FieldDefinition` objects.
+
+**webEntrypoint**:
+    Optional key of the web entrypoint in the plugin's web bundle, this should be `"./web"`.
+    The `clusterioPlugin` Vite plugin exposes `web/index.tsx` (or `.ts`, `.jsx`, `.js`) under this key; it is not a Node.js file path.
+
 **migrationEntrypoint**:
     Path to a Node.js module relative to the plugin directory which exports the migration entrypoint function for this plugin.
     This is an optional parameter.
     See [Migrating Stored Data](#migrating-stored-data)
 
 **features**:
-    Array of features the plugin needs from the instances it is loaded on, `"SavePatching"` and/or `"ScriptCommands"`.
+    Array of features the plugin needs from the instances it is loaded on, any of `"SavePatching"`, `"ScriptCommands"` and `"LuaUdp"`.
     See [Handling Invalid Configuration](#handling-invalid-configuration)
 
 **messages**:
-    Object with link messages definitions for this plugin.
+    Array of request and event classes for this plugin.
     See guide for [defining link messages](#defining-link-messages) below.
 
+**errors**:
+    Array of error classes for this plugin.
+    See [Request errors](#request-errors).
+
 **permissions**:
     Array of permission definitions for this plugin.
     See [Plugin Permissions](#plugin-permissions)
@@ -123,7 +170,8 @@ The only restriction imposed on modules embedded into plugins is that they must
 While there is no standard for how to organize a plugin it's recommended to put the controller entrypoint into controller.js and the instance entrypoint into instance.js.
 You can put them into whatever file you want (even the same one for both).
 
-The entrypoint paths must include the .js extension and use forward slashes for directory separators if any.
+Node.js entrypoint paths must include the `.js` extension and use forward slashes for directory separators.
+This rule does not apply to the `webEntrypoint` exposure key.
 
 
 ## Defining the plugin entrypoint
@@ -210,7 +258,7 @@ try {
     // Operation that should not throw but may end up throwing
 } catch (err) {
     logger.error(`Operation failed:\n${err.stack}`);
-)
+}
 ```
 
 For plugin hooks expections thrown are automatically catched and logged, but for event handlers registered on EventEmitters it's critical that exceptions are catched and handled appropriately.
@@ -219,7 +267,7 @@ For plugin hooks expections thrown are automatically catched and logged, but for
 ## Plugin Configuration
 
 Clusterio provides a configuration system that handles storing, distributing, editing and validating config fields for you.
-You can take advantage of it by declaring config fields under either `controllerConfigFields` or `instanceConfigFields` in the `plugin` export.
+Declare fields under `controllerConfigFields`, `hostConfigFields`, `instanceConfigFields`, or `ctlConfigFields` in the `plugin` export.
 For example in index.js:
 
 ```ts
@@ -260,7 +308,7 @@ See [Configuration System](config-system.md) for more details on how this system
 
 ### Handling Invalid Configuration
 
-If the plugin requires save patching or script commands to function, which can be disabled to run vanilla or scenarios not compatible with Clusterio, declare it under `features` in the `plugin` export:
+If the plugin requires save patching, script commands or Lua UDP to function, which can be disabled to run vanilla or scenarios not compatible with Clusterio, declare it under `features` in the `plugin` export:
 
 ```js
 export const plugin = {
@@ -271,6 +319,7 @@ export const plugin = {
 ```
 
 Enabling the plugin on an instance that has the feature turned off then fails config validation, so the mistake is caught when the config is edited instead of when the instance starts.
+Instances that have the feature turned off get the plugin disabled by default when it is added.
 
 Other constraints on config fields are expressed with a `validator` on the field definition, which throws when the value is not acceptable.
 Use `dependsOn` when the check involves other fields so that the validator runs when those change too.
@@ -314,6 +363,8 @@ In JavaScript the `declare module` block is not needed.
 
 Permissions declared this way are registered when the plugin is loaded.
 The older `lib.definePermission()` function still works, but a plugin should not use both for the same permission name.
+Controller-side permission checks are authoritative; hiding a web UI control does not authorize or protect its underlying request.
+Declare the [permission on control-origin messages](#static-permission) and test denied requests as well as allowed ones.
 
 
 ## Migrating Stored Data
@@ -341,7 +392,7 @@ They run synchronously, a handler that throws stops the controller or host from
 `lib.renameConfigField`, `lib.renameConfigPrefix`, `lib.renamePermission` and `lib.renamePermissionPrefix` cover the common case of renaming.
 
 Handlers run every time the data is loaded and must leave data that is already migrated unchanged.
-For changes that can't be repeated, each config has a hidden `<plugin>.version` field with the version of the plugin it was last loaded with, which is updated after the migrations have run:
+For changes that can't be repeated, controller, host, and instance configs have a hidden `<plugin>.version` field with the version of the plugin they were last loaded with, which is updated after the migrations have run:
 
 ```js
 hooks.instanceConfig.attach(plugin.name, config => {
@@ -448,7 +499,7 @@ async function handleFoo(content) {
 **Note:** This only handles IPC events, all other events must use `on`. You should not include the `ipc-` prefix when using this method. Error logging is handled for you, if you want custom error handling use `on`.
 
 It's recommended to either use the plugin name as the channel name or to prefix the channel name with the name of the plugin if you need multiple channels.
-It's also important to catch any errors that might occur as they will otherwise be propogated to the instance code and kill the server.
+With raw `server.on(...)` listeners, catch errors in the handler; `server.handle(...)` already handles error logging for IPC handlers.
 
 Data out from Factorio does not have the same limits as data into Factorio, RCON responses can be in 100kB range without causing issues, and payloads to the `send_json` API can be in the 4MB range provided the server has a fast enough storage system.
 
@@ -456,6 +507,62 @@ Data out from Factorio does not have the same limits as data into Factorio, RCON
 For `send_json` it's possible that payloads greater than 4kB are received after payloads that were sent at a later point in time.
 For RCON, commands longer than 50 characters may end up being executed after shorter commands sent after it.
 
+### Auto pause
+
+Each RCON command makes a paused game run one tick, so commands sent on a timer keep a server with `auto_pause` on running while nobody is online.
+`instance.autoPaused` is true while the instance was started with `auto_pause` on and no players are online, and the `autoPauseChanged` hook is invoked with the new value when it changes.
+Skip periodic commands while it is true:
+
+```js
+instance.hooks.start.attach(plugin.name, async () => {
+    timer = setInterval(() => {
+        if (!instance.autoPaused) {
+            instance.sendRcon("/sc my_plugin.update()").catch(handleError);
+        }
+    }, 1000);
+});
+```
+
+Commands sent in response to events from elsewhere in the cluster are fine, they cost one tick each.
+A game paused with `game.tick_paused` does not run ticks for commands.
+
+### Communicating over UDP
+
+Instances can optionally exchange UDP packets with the Factorio server for low latency fire-and-forget communication, using the `--enable-lua-udp` option of Factorio.
+This needs Factorio 2.1.10 or later, earlier versions crash when a packet is received on a headless server.
+This is disabled by default and enabled by setting `factorio.enable_lua_udp` on the instance.
+Sending data out of Factorio also needs `factorio.enable_script_commands`, as the port to send to is passed to the module with a script command on startup.
+Plugins requiring it should declare the `LuaUdp` feature.
+Note that UDP does not guarantee delivery or ordering, and packets should be kept well below 32 kB.
+
+Data out from Factorio is sent with the `send_udp` API of the Clusterio module.
+Unlike `send_json` the payload is an arbitrary string, and it is delivered to plugins as a `udp-channel_name` event with the payload as a Buffer:
+
+```lua
+clusterio_api.send_udp("my_plugin_foo", "123")
+```
+
+```js
+export default async function(context) {
+    const { instance, logger } = context;
+    instance.server.handleUdp("my_plugin_foo", async data => {
+        logger.info(`got ${data.toString()}`);
+    });
+}
+```
+
+Like `handle` for `send_json`, `handleUdp` logs errors thrown by the handler.
+Use `instance.server.on("udp-my_plugin_foo", ...)` for custom error handling.
+
+Data into Factorio is sent with the `sendUdp` method on the instance and delivered in-game as `defines.events.on_udp_packet_received` events with the packet in `event.payload`.
+The Clusterio module polls for received packets every tick, all modules and mods receive every packet so make sure the payload identifies your plugin.
+
+```js
+instance.hooks.start.attach(plugin.name, async () => {
+    await instance.sendUdp("my_plugin_foo:123");
+});
+```
+
 
 ## Defining Link Messages
 
@@ -470,59 +577,59 @@ Events are simple one-way notifications that invoke a handler on the target it's
 Requests are pairs of request and response messages where the request is sent to the target and the response is the reply back from the target.
 The requests are similar to HTTP requests, except both parties of a link may initiate one.
 
-Messages are defined as items of the `plugin.messages` array exported by `index.js`.
-For example, the following could be defined in `plugin`:
+Export message classes from `messages.js` and list them in the `plugin.messages` array in `index.js`, as in the [plugin structure example](#plugin-structure).
+For example, `messages.js` can contain:
 
 ```js
-messages: [
-    class Frobnicate {
-        static plugin = "foo_frobber"; // Plugin name
-        static type = "request";
-        static src = "controller";
-        static dst = "instance";
-        static jsonSchema = {
+export class Frobnicate {
+    static plugin = "foo_frobber";
+    static type = "request";
+    static src = "instance";
+    static dst = "controller";
+    static jsonSchema = {
+        type: "object",
+        properties: {
+            verbosity: { type: "integer" },
+            special: { type: "boolean" },
+        },
+        required: ["verbosity"],
+        additionalProperties: false,
+    };
+    constructor(json) {
+        this.verbosity = json.verbosity;
+        this.special = json.special;
+    }
+    static fromJSON(json) {
+        return new Frobnicate(json);
+    }
+    toJSON() {
+        return {
+            verbosity: this.verbosity,
+            special: this.special,
+        };
+    }
+    static Response = {
+        jsonSchema: {
             type: "object",
             properties: {
-                "verbosity": { type: "integer" },
-                "special": { type: "boolean" },
-            },
-            required: ["verbosity"],
-            additionalProperties: false,
-        };
-        constructor(json) {
-            this.verbosity = json.verbosity;
-            this.special = json.special;
-        }
-        static fromJSON(json) {
-            return new Frobnicate(json);
-        }
-        toJSON() {
-            return {
-                verbosity: this.verbosity,
-                special: this.special,
-            };
-        }
-        static Response = {
-            jsonSchema: {
-                type: "object",
-                properties: {
-                    "report": {
-                        type: "array",
-                        items: { type: "string" },
-                    },
+                report: {
+                    type: "array",
+                    items: { type: "string" },
                 },
             },
-            fromJSON(json) {
-                return json
-            }
-        }
-    }
-]
+            required: ["report"],
+            additionalProperties: false,
+        },
+        fromJSON(json) {
+            return json;
+        },
+    };
+}
 ```
 
-This specifies a request that can be sent from the controller to an instance, being routed through the correct host behind the scenes.
+This specifies a request sent from an instance to the controller through its host.
 The request data must contain the property `verbosity` with an integer number as the value, and it may also contain a boolean `special` property.
-It also defines that the response sent may contain a `report` property mapping to an array of strings.
+The response must contain a `report` property containing an array of strings.
 
 The following properties are recognized as part of a message class:
 
@@ -551,6 +658,34 @@ If the target is multiple hops away it the message will be forwarded towards its
 For example a message sent from a control connection an instance will be forwarded by the controller to the relevant host and from the host to the correct instance.
 Messages sent to a different destination will be rejected.
 
+#### static permission
+
+Required for request and event classes whose `src` includes `"control"`.
+Use a permission name, a synchronous permission-checking function, or `null` when every authenticated control client is deliberately allowed to send the message.
+Registration rejects control-origin messages with no permission declaration.
+
+A callback receives `(user, message)`, where `message` is the raw wire envelope.
+It runs on the controller before destination payload validation and deserialization, so `message.data` is untrusted and may be missing or malformed.
+Check any required payload shape before inspecting fields, and use `user.checkPermission(...)` or throw a `ClientError` such as `PermissionError` for a normal denial.
+For example, this checks the payload shape needed to choose a permission; the destination schema still validates the full payload:
+
+```js
+import { ValidationError } from "@clusterio/lib";
+
+function checkFrobnicatePermission(user, message) {
+    const data = message.data;
+    if (!data || typeof data !== "object" || typeof data.special !== "boolean") {
+        throw new ValidationError("Expected a boolean special field");
+    }
+    user.checkPermission(data.special ? "foo_frobber.special" : "foo_frobber.frobnicate");
+}
+```
+
+To use this callback, declare both permissions, require `special` in the message schema, and set `static permission = checkFrobnicatePermission` on a class that accepts `"control"` sources.
+The `Frobnicate` example above only accepts instances and therefore does not need a permission declaration.
+Do not make permission callbacks async: their result is not awaited, and returning `false` does not deny access.
+Unexpected exceptions are contained and logged as errors; they are not a substitute for intentional permission denials.
+
 #### static jsonSchema
 
 A JSON schema that specifies what's valid to send in the message.
@@ -562,7 +697,8 @@ This should return an object that's valid according to `jsonSchema` and accepted
 
 #### static fromJSON
 
-Required function to restore the class from a JSON object.
+Function to restore the class from a JSON object; it must be provided together with `jsonSchema` for messages carrying data.
+Messages without data can omit both and are constructed without arguments.
 This function is called when the message reaches its destination and is expected to return an identical instance of the class.
 I.e. the result of `fromJSON(JSON.parse(JSON.stringify(instance)))` should be a complete copy of the message.
 
@@ -571,56 +707,112 @@ I.e. the result of `fromJSON(JSON.parse(JSON.stringify(instance)))` should be a
 Optional [JSON sterilisable class](/docs/devs/json-serialisable-classes.md) that specifies the response to the request.
 Required properties are `jsonSchema` and `fromJSON`.
 
+Test JSON round trips, invalid payloads, allowed source/destination types, and authorization failures when changing messages.
+Keep registration, schemas, serialization, permissions, and handlers consistent.
+
+
 ## Sending Link Messages
 
 Link messages are sent by calling the `.sendTo()` method on a connection object with the destination and the data you want to send, where the data is an instance of one of the defined message classes in plugin.messages.
 For example:
 
 ```js
+import * as messages from "./messages.js";
+
 // In an instance entrypoint
 async function frobnicate(instance) {
-    const response = await instance.sendTo("controller", new messages.Frobnicate({ 
+    const response = await instance.sendTo("controller", new messages.Frobnicate({
         verbosity: 2,
         special: false,
     }));
     console.log(response); // { report: [...] }
 }
 ```
-Because the send and handle methods rely on this, you should always call them directly on the connection object.
-
-If you extract them into a variable, destructure them, or pass them as unbound callbacks, they lose their this context. The method will eventually crash with an error like Cannot read properties of undefined (reading 'sendRequest').
-
-While it's fine to type cast a direct method invocation, casting and then storing the result causes that same dangerous loss of context.
-
-Here is what that looks like in practice:
+Register the corresponding handler on the controller:
 
 ```js
-// ❌ BAD: The extracted method loses its `this` binding and crashes when called.
-const send = instance.sendTo;
-send("controller", new messages.Frobnicate({ foo: "bar" }));
+import * as messages from "./messages.js";
 
-// ✅ GOOD: Call it directly on the connection object. (Cast args/results here if needed).
-instance.sendTo("controller", new messages.Frobnicate({ foo: "bar" }));
-
-// ✅ GOOD: Explicitly bind the receiver if you need to pass a method as a value (like a handler).
-instance.handle(messages.Frobnicate, plugin.handleFrobnicate.bind(plugin));
+export default async function({ controller }) {
+    controller.handle(messages.Frobnicate, async request => ({
+        report: [`Frobnication verbosity: ${request.verbosity}`],
+    }));
+}
 ```
 
+The send and handle methods depend on `this`.
+Call them on their connection object, or explicitly bind them when passing a method as a value:
+
+```js
+const send = instance.sendTo.bind(instance);
+const response = await send("controller", new messages.Frobnicate({ verbosity: 2, special: false }));
+```
+
+Extracting an unbound method loses its receiver; storing a returned response does not.
 For classes with `static type = "request"` the send method is async and returns a promise that resolves to the response data received from the target it was sent to, or rejects with an error if the request failed.
-The destination specification can either be the ID of a particular control, host or instance, or one of the keywords used to send to multiple targets at once.
+The destination can identify a particular control, host, or instance.
+The `allHosts`, `allControls`, and `allInstances` broadcast destinations are for events, not requests.
 
 Example destinations:
 
 ```js
-sendTo("controller", message);
-sendTo("allHosts", message);
-sendTo("allControls", message);
-sendTo("allInstances", message);
-sendTo({ hostId: 123 }, message);
-sendTo({ instanceId: 123 }, message);
-sendTo({ controlId: 123 }, message);
+connection.sendTo("controller", message);
+connection.sendTo("allHosts", event);
+connection.sendTo("allControls", event);
+connection.sendTo("allInstances", event);
+connection.sendTo({ hostId: 123 }, message);
+connection.sendTo({ instanceId: 123 }, message);
+connection.sendTo({ controlId: 123 }, message);
 ```
 
+### Request errors
+
+An error thrown by a request handler is sent back to the requester, and the promise returned by `.sendTo()` rejects with an instance of the same class.
+The error classes follow the HTTP status codes:
+
+```
+Error
+├── ClientError (4xx)
+│   ├── ValidationError (400)
+│   │   ├── InvalidValueError
+│   │   └── InvalidFieldError
+│   ├── ForbiddenError (403)
+│   │   ├── PermissionError
+│   │   └── InvalidAccessError
+│   ├── NotFoundError (404)
+│   ├── TimeoutError (408)
+│   └── ConflictError (409)
+└── ServerError (5xx)
+    ├── InternalServerError (500)
+    └── UnavailableError (503)
+```
+
+Throw a `ClientError` subclass when the request can't be done as asked, and `UnavailableError` when the host or instance that should handle it is not connected or not running, for example:
+
+```js
+controller.handle(messages.Frobnicate, async request => {
+    if (!frobnicators.has(request.id)) {
+        throw new lib.NotFoundError(`Frobnicator ${request.id} does not exist`);
+    }
+    // ...
+});
+```
+
+Errors of a class that is not registered are logged as unexpected by the side that threw them and are received as `InternalServerError`.
+Registered errors are not logged.
+The message and stack trace of the original error are kept in both cases.
+
+Plugins can define their own error classes by listing them in `errors` of the plugin declaration.
+They need a static `plugin` property set to the plugin name and must take the message as the first constructor argument:
+
+```js
+export class FrobnicatorJammed extends lib.ConflictError {
+    static plugin = "foo_frobber";
+}
+```
+
+A subclass that is not registered is received as its closest registered parent class.
+
 ### Handling connection events
 
 There are a few connection related events that plugins neeed to repsond to in order to avoid data loss and connection problems.
@@ -630,25 +822,56 @@ This is signaled to controller plugins via the `prepareHostDisconnect` hook and
 After the prepare disconnect the connection will be closed, which will result in pending requests and events being dropped.
 Plugins must respond to the prepare disconnect by stopping any processess it does that send events or requests over the link in question.
 This can be accomplished either through listening for the prepare disconnect hook, or by checking the `connected` property of the `HostConnection` class and `Host` class on the controller and host respectively.
-For example the sending of an event from an instance plugin can be stopped while the connection is not connected, not in the dropped state, and not in the process of discunnecting by using the following code:
+For example, an instance plugin can avoid starting a request while the host is disconnected or preparing to disconnect:
 
 ```js
 if (host.connected) {
-    instance.sendTo("controller", new messages.Frobnicate({ foo: "bar" }));
+    await instance.sendTo("controller", new messages.Frobnicate({ verbosity: 2, special: false }));
 }
 ```
 
 If the event or request needs to be sent to the controller it can be put into a queue and sent out when the connection is established again.
-The re-establishement of the connection is  notified to plugins via the `connect` event to the `controllerConnectionEvent` and `hostConnectionEvent` hooks.
+The connection hooks report `connect` for a new session and `resume` for a retained session.
+Keep application queues separate from messages already accepted into the connector buffer; blindly sending both again can duplicate an operation.
 
 The second connection event which is of lesser importance to respond to is the `drop` connection event served through `controllerConnectionEvent` for instance plugins and through `hostConnectionEvent` for controller plugins.
 This is raised when the connection between the controller and host in question is lost, most likely due to networking issues.
-When in the dropped state the host will keep trying to reconnect to the controller in order to re-establish it, and if successful no events or requests will be lost.
+When in the dropped state the host retries the connection and attempts to resume the retained session.
+Successful resumption replays unacknowledged buffered messages; expiry, invalidation, or buffer overflow can instead end the session.
+This is not an application-level exactly-once guarantee; stateful plugins need their own recovery and duplicate-handling strategy.
 However while in the dropped state any requests and events sent gets queued up in memory until the connection is either re-established or the session times out.
 This means that if your plugin sends a lot of events or requests, they can end up being queued up in a buffer and sent out all at once the connection is re-estabished.
 To avoid this you should be throtteling and/or stopping your requests/events after `drop` has been raised, and continue back as normal when `resume` is raised.
 
 
+## Web Pages for a Single Resource
+
+A web page showing one resource from a subscription has more to handle than the resource itself.
+It can still be loading, it might not exist, it can be deleted while the page is open, and the subscription is rejected if the user lacks the permission for it.
+`useResourceState` and `ResourceState` from `@clusterio/web_ui` show these the same way as the pages for hosts, instances and the other built in resources.
+
+```tsx
+function FrobberPage() {
+    const id = Number(useParams().id);
+    const [frobbers, synced, error] = useSyncExternalStore(subscribe, () => frobberSubscriber.getSnapshot());
+    const frobber = frobbers.get(id);
+    const state = useResourceState(frobber, synced, error, id);
+
+    const nav = [{ name: "Frobbers", path: "/frobbers" }, { name: frobber?.name ?? String(id) }];
+    if (!frobber) {
+        return <ResourceState state={state} type="Frobber" id={id} nav={nav} listPath="/frobbers" error={error} />;
+    }
+
+    return <PageLayout nav={nav}>{frobber.name}</PageLayout>;
+}
+```
+
+`useResourceState` returns `"ok"` when there is a value and otherwise `"loading"`, `"missing"`, `"deleted"` or `"error"`.
+It tells deleted from missing by remembering if it has seen a value for that id.
+`ResourceState` renders the whole page for the states without a value, with a link back to the page at `listPath`.
+The `useHost`, `useInstance`, `useUser`, `useRole` and `useModPack` hooks return `[value, synced, error]` so they can be used the same way.
+
+
 ## Collecting Statistics
 
 Clusterio comes with its own Prometheus client implementation, one part due to Not Invented Here and another part due to collectors in prom-client being difficult to get to work nicely with collecting data from plugins optionally loaded at runtime on different computers.
@@ -673,7 +896,7 @@ It's recommended that plugin metrics follow `clusterio_<plugin_name>_<metric_nam
 For metrics that are per-instance, you must define an `instance_id` label and set it accordingly, for example:
 
 ```js
-import { Counter } from "@clusterio/lib";
+import { Gauge } from "@clusterio/lib";
 
 const barMetric = new Gauge(
     "clusterio_foo_frobber_bar_metric", "Bar instance level",

eslint.config.mjs

@@ -1,504 +0,0 @@
-import {
-	defineConfig,
-	globalIgnores,
-} from "eslint/config";
-
-import typescriptEslint from "@typescript-eslint/eslint-plugin";
-import node from "eslint-plugin-node";
-import globals from "globals";
-import tsParser from "@typescript-eslint/parser";
-import js from "@eslint/js";
-import stylistic from "@stylistic/eslint-plugin";
-
-import {
-	FlatCompat,
-} from "@eslint/eslintrc";
-
-const compat = new FlatCompat({
-	baseDirectory: import.meta.dirname,
-	recommendedConfig: js.configs.recommended,
-	allConfig: js.configs.all,
-});
-
-export default defineConfig([{
-	plugins: {
-		"@typescript-eslint": typescriptEslint,
-		"@stylistic": stylistic,
-		node,
-	},
-
-	languageOptions: {
-		globals: {
-			...globals.node,
-			...globals.commonjs,
-		},
-	},
-
-	"rules": {
-		"@stylistic/indent": ["error", "tab", {
-			"ignoredNodes": [
-				"JSXFragment", // TODO: Linter crashes if this is removed
-			],
-			"ignoreComments": true,
-			"SwitchCase": 1, // Indent case statements one level from switch
-		}],
-		"accessor-pairs": "error",
-		"array-bracket-newline": "off",
-		"array-bracket-spacing": ["error", "never"],
-		"array-callback-return": "error",
-		"array-element-newline": "off",
-		"arrow-body-style": "error",
-		"arrow-parens": "off",
-
-		"arrow-spacing": ["error", {
-			"after": true,
-			"before": true,
-		}],
-
-		"block-scoped-var": "error",
-		"block-spacing": "error",
-
-		"brace-style": ["error", "1tbs", {
-			"allowSingleLine": true,
-		}],
-
-		"camelcase": "off",
-		"capitalized-comments": "off",
-		"class-methods-use-this": "off",
-
-		"comma-dangle": ["error", {
-			"arrays": "always-multiline",
-			"objects": "always-multiline",
-			"imports": "always-multiline",
-			"exports": "always-multiline",
-			"functions": "only-multiline",
-		}],
-
-		"comma-spacing": "error",
-		"comma-style": ["error", "last"],
-		"complexity": ["error", { "max": 25 }],
-		"computed-property-spacing": ["error", "never"],
-		"consistent-return": "error",
-		"consistent-this": "error",
-		"curly": "error",
-		"default-case": "error",
-		"default-case-last": "error",
-		"default-param-last": "off",
-		"dot-location": ["error", "property"],
-
-		"dot-notation": ["error", {
-			"allowPattern": "^[a-z]+(_[a-z]+)*$",
-		}],
-
-		"eol-last": "error",
-		"eqeqeq": "error",
-		"func-call-spacing": "error",
-		"func-name-matching": "error",
-		"func-names": "off",
-
-		"func-style": ["error", "declaration", {
-			"allowArrowFunctions": true,
-		}],
-
-		"function-call-argument-newline": "off",
-		"function-paren-newline": "off",
-
-		"generator-star-spacing": ["error", {
-			"before": false,
-			"after": true,
-		}],
-
-		"grouped-accessor-pairs": "error",
-		"guard-for-in": "error",
-		"id-blacklist": "error",
-		"id-length": "off",
-		"id-match": "error",
-		"implicit-arrow-linebreak": "error",
-
-		"indent-legacy": "off",
-		"init-declarations": "off",
-		"jsx-quotes": "error",
-		"key-spacing": "error",
-		"keyword-spacing": "error",
-		"line-comment-position": "off",
-		"linebreak-style": ["error", "unix"],
-		"lines-around-comment": "off",
-		"lines-around-directive": "off",
-
-		"lines-between-class-members": ["error", "always", {
-			"exceptAfterSingleLine": true,
-		}],
-
-		"max-classes-per-file": "off",
-		"max-depth": "error",
-
-		"max-len": ["error", {
-			"code": 120,
-		}],
-
-		"max-lines": "off",
-		"max-lines-per-function": "off",
-		"max-nested-callbacks": "error",
-		"max-params": "off",
-		"max-statements": "off",
-		"max-statements-per-line": "off",
-		"multiline-comment-style": ["error", "separate-lines"],
-		"multiline-ternary": "off",
-
-		"new-cap": ["error", {
-			"capIsNewExceptions": ["StringEnum", "StringKey"],
-			"capIsNewExceptionPattern": "^Type\\.",
-			"newIsCapExceptions": ["constructor"],
-		}],
-
-		"new-parens": "error",
-		"newline-after-var": "off",
-		"newline-before-return": "off",
-		"newline-per-chained-call": "off",
-		"no-alert": "error",
-		"no-array-constructor": "error",
-		"no-await-in-loop": "off",
-		"no-bitwise": "off",
-		"no-buffer-constructor": "error",
-		"no-caller": "error",
-		"no-catch-shadow": "off",
-		"no-confusing-arrow": "error",
-		"no-console": "error",
-
-		"no-constant-condition": ["error", {
-			"checkLoops": false,
-		}],
-
-		"no-constructor-return": "error",
-		"no-continue": "off",
-		"no-div-regex": "error",
-		"no-duplicate-imports": "error",
-		"no-else-return": "error",
-		"no-empty-function": "off",
-		"no-eq-null": "error",
-		"no-eval": "error",
-		"no-extend-native": "error",
-		"no-extra-bind": "error",
-		"no-extra-label": "error",
-		"no-extra-parens": "off",
-		"no-floating-decimal": "error",
-		"no-implicit-coercion": "error",
-		"no-implicit-globals": "error",
-		"no-implied-eval": "error",
-		"no-inline-comments": "off",
-		"no-invalid-this": "error",
-		"no-iterator": "error",
-		"no-label-var": "error",
-		"no-lone-blocks": "error",
-		"no-lonely-if": "error",
-		"no-loop-func": "error",
-		"no-magic-numbers": "off",
-
-		"no-mixed-operators": ["error", {
-			"groups": [
-				["&", "|", "^", "~", "<<", ">>", ">>>"],
-				["==", "!=", "===", "!==", ">", ">=", "<", "<="],
-				["in", "instanceof"],
-			],
-		}],
-
-		"no-multi-assign": "error",
-		"no-multi-spaces": "error",
-		"no-multi-str": "error",
-		"no-multiple-empty-lines": "error",
-		"no-native-reassign": "error",
-		"no-negated-condition": "off",
-		"no-negated-in-lhs": "error",
-		"no-nested-ternary": "error",
-		"no-new": "error",
-		"no-new-func": "error",
-		"no-new-object": "error",
-		"no-new-wrappers": "error",
-		"no-octal-escape": "error",
-		"no-param-reassign": "off",
-
-		"no-plusplus": ["error", {
-			"allowForLoopAfterthoughts": true,
-		}],
-
-		"no-proto": "error",
-		"no-restricted-exports": "error",
-		"no-restricted-globals": "error",
-		"no-restricted-imports": "error",
-		"no-restricted-properties": "error",
-		"no-restricted-syntax": "error",
-		"no-return-assign": "error",
-		"no-return-await": "off",
-		"no-script-url": "error",
-		"no-self-compare": "error",
-		"no-sequences": "error",
-
-		"no-shadow": ["error", {
-			"allow": ["Event", "Request", "yargs"],
-		}],
-
-		"no-spaced-func": "error",
-
-		"no-tabs": "off",
-
-		"no-template-curly-in-string": "error",
-		"no-ternary": "off",
-		"no-throw-literal": "error",
-		"no-trailing-spaces": "error",
-		"no-undef-init": "error",
-		"no-undefined": "off",
-		"no-unmodified-loop-condition": "error",
-		"no-unneeded-ternary": "error",
-		"no-unused-expressions": "error",
-		"no-use-before-define": "off",
-		"no-useless-backreference": "error",
-		"no-useless-call": "error",
-		"no-useless-computed-key": "error",
-		"no-useless-concat": "error",
-		"no-useless-constructor": "off",
-		"no-useless-rename": "error",
-		"no-useless-return": "error",
-		"no-var": "error",
-
-		"no-void": ["error", {
-			"allowAsStatement": true,
-		}],
-
-		"no-warning-comments": "off",
-		"no-whitespace-before-property": "error",
-		"nonblock-statement-body-position": "error",
-		"object-curly-newline": "error",
-		"object-curly-spacing": "off",
-		"object-shorthand": "off",
-		"one-var": "off",
-		"one-var-declaration-per-line": "error",
-		"operator-assignment": "off",
-
-		"operator-linebreak": ["error", "after", {
-			"overrides": {
-				"?": "before",
-				":": "before",
-				"||": "before",
-				"&&": "before",
-				"??": "before",
-			},
-		}],
-
-		"padded-blocks": "off",
-		"padding-line-between-statements": "error",
-		"prefer-arrow-callback": "error",
-		"prefer-const": "off",
-		"prefer-destructuring": "off",
-		"prefer-exponentiation-operator": "error",
-		"prefer-named-capture-group": "off",
-		"prefer-numeric-literals": "error",
-		"prefer-object-spread": "error",
-		"prefer-promise-reject-errors": "error",
-		"prefer-reflect": "off",
-		"prefer-regex-literals": "error",
-		"prefer-rest-params": "error",
-		"prefer-spread": "error",
-		"prefer-template": "error",
-		"quote-props": "off",
-
-		"quotes": ["error", "double", {
-			"avoidEscape": true,
-		}],
-
-		"radix": ["error", "always"],
-		"require-atomic-updates": "off",
-		"require-await": "off",
-		"require-jsdoc": "off",
-		"require-unicode-regexp": "off",
-		"rest-spread-spacing": ["error", "never"],
-		"semi": "error",
-		"semi-spacing": "error",
-		"semi-style": "off",
-		"sort-imports": "off",
-		"sort-keys": "off",
-		"sort-vars": "off",
-		"space-before-blocks": "error",
-		"space-before-function-paren": "off",
-		"space-in-parens": ["error", "never"],
-		"space-infix-ops": "off",
-		"space-unary-ops": "error",
-
-		"spaced-comment": ["error", "always", {
-			"line": {
-				"markers": ["/"],
-			},
-		}],
-
-		"switch-colon-spacing": "error",
-		"symbol-description": "error",
-		"template-curly-spacing": ["error", "never"],
-		"template-tag-spacing": "error",
-		"unicode-bom": ["error", "never"],
-
-		"vars-on-top": "error",
-		"wrap-iife": "error",
-		"wrap-regex": "off",
-		"yield-star-spacing": "error",
-
-		"yoda": ["error", "never", {
-			"exceptRange": true,
-		}],
-
-		"node/callback-return": "off",
-		"node/exports-style": "off",
-		"node/file-extension-in-import": "off",
-		"node/global-require": "off",
-		"node/handle-callback-err": "off",
-		"node/no-callback-literal": "off",
-		"node/no-deprecated-api": "off",
-		"node/no-exports-assign": "off",
-		"node/no-extraneous-import": "off",
-		"node/no-extraneous-require": "off",
-
-		"node/no-missing-import": "off",
-
-		"node/no-missing-require": "off",
-		"node/no-mixed-requires": "off",
-		"node/no-new-require": "off",
-		"node/no-path-concat": "off",
-		"node/no-process-env": "off",
-		"node/no-process-exit": "off",
-		"node/no-restricted-require": "off",
-		"node/no-sync": "error",
-		"node/no-unpublished-bin": "off",
-		"node/no-unpublished-import": "off",
-
-		"node/no-unpublished-require": "off",
-
-		"node/no-unsupported-features/es-builtins": "off",
-		"node/no-unsupported-features/es-syntax": "off",
-		"node/no-unsupported-features/node-builtins": "off",
-		"node/prefer-global/buffer": "off",
-		"node/prefer-global/console": "off",
-		"node/prefer-global/process": "off",
-		"node/prefer-global/text-decoder": "off",
-		"node/prefer-global/text-encoder": "off",
-		"node/prefer-global/url-search-params": "off",
-		"node/prefer-global/url": "off",
-		"node/prefer-promises/dns": "off",
-		"node/prefer-promises/fs": "off",
-		"node/process-exit-as-throw": "off",
-
-		"node/shebang": "off",
-	},
-}, {
-	files: ["{packages,plugins}/**/*.ts?(x)"],
-
-	languageOptions: {
-		parser: tsParser,
-	},
-
-	extends: compat.extends("plugin:@typescript-eslint/recommended"),
-
-	"rules": {
-		"@typescript-eslint/no-explicit-any": "off",
-		"@typescript-eslint/no-useless-constructor": "error",
-		"@typescript-eslint/no-unused-vars": "off",
-		"@typescript-eslint/triple-slash-reference": "off",
-
-		"@typescript-eslint/no-shadow": ["error", {
-			"allow": ["Event", "Request", "yargs"],
-		}],
-
-		// Allow require imports where necessary (dynamic loading, webpack configs, etc.)
-		"@typescript-eslint/no-require-imports": "off",
-
-		"prefer-const": "off",
-		"no-shadow": "off",
-	},
-}, {
-	files: ["test/**/*.js", "plugins/*/test/**/*.js"],
-
-	languageOptions: {
-		globals: {
-			...globals.mocha,
-		},
-
-		sourceType: "module",
-		parserOptions: {},
-	},
-
-	"rules": {
-		"prefer-arrow-callback": "off",
-		"no-invalid-this": "off",
-		"node/no-unpublished-import": "off",
-		"node/no-unpublished-require": "off",
-	},
-}, {
-	files: [
-		"packages/web_ui/src/**/*.ts?(x)",
-		"packages/lib/browser.ts",
-		"packages/create/templates/*/web/**/*.{t,j}s?(x)",
-		"{packages,plugins}/*/web/**/*.ts?(x)",
-	],
-
-	languageOptions: {
-		globals: {
-			...globals.browser,
-		},
-
-		"sourceType": "module",
-
-		parserOptions: {
-			"ecmaFeatures": {
-				"jsx": true,
-			},
-		},
-	},
-
-	"rules": {
-		"node/callback-return": "off",
-		"node/exports-style": "off",
-		"node/file-extension-in-import": "off",
-		"node/global-require": "off",
-		"node/handle-callback-err": "off",
-		"node/no-callback-literal": "off",
-		"node/no-deprecated-api": "off",
-		"node/no-exports-assign": "off",
-		"node/no-extraneous-import": "off",
-		"node/no-extraneous-require": "off",
-		"node/no-missing-import": "off",
-		"node/no-missing-require": "off",
-		"node/no-mixed-requires": "off",
-		"node/no-new-require": "off",
-		"node/no-path-concat": "off",
-		"node/no-process-env": "off",
-		"node/no-process-exit": "off",
-		"node/no-restricted-require": "off",
-		"node/no-sync": "error",
-		"node/no-unpublished-bin": "off",
-		"node/no-unpublished-import": "off",
-		"node/no-unpublished-require": "off",
-		"node/no-unsupported-features/es-builtins": "off",
-		"node/no-unsupported-features/es-syntax": "off",
-		"node/no-unsupported-features/node-builtins": "off",
-		"node/prefer-global/buffer": "off",
-		"node/prefer-global/console": "off",
-		"node/prefer-global/process": "off",
-		"node/prefer-global/text-decoder": "off",
-		"node/prefer-global/text-encoder": "off",
-		"node/prefer-global/url-search-params": "off",
-		"node/prefer-global/url": "off",
-		"node/prefer-promises/dns": "off",
-		"node/prefer-promises/fs": "off",
-		"node/process-exit-as-throw": "off",
-		"node/shebang": "off",
-	},
-}, {
-	files: ["packages/create/templates/**/*.?(c){t,j}s?(x)"],
-
-	"rules": {
-		"node/no-missing-require": "off",
-		"node/no-missing-import": "off",
-		"@stylistic/indent": "off",
-		"spaced-comment": "off",
-		"comma-spacing": "off",
-		"max-len": "off",
-		"indent": "off",
-	},
-}, globalIgnores(["**/dist/", "dist/"])]);

logo-square.svg

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg width="100%" height="100%" viewBox="0 0 90 90" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:1.41421;"><g transform="translate(0.84,24.491)"><path d="M48.48,19.309l-9.917,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l2.718,0l0.002,-8.867l0.96,-0.96l3.117,0l1.68,1.68l0,10.293l-2.64,2.64ZM32.828,19.309l-9.918,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l8.478,0.009l0,2.137l-2.64,2.64ZM17.116,19.309l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.942l0,0.018l-7.596,0l0,5.04l8.556,0l0,2.146l-2.64,2.64ZM64.301,19.309l-9.997,0l-1.68,-1.68l0,-2.142l0.96,-0.96l7.597,-0.002l0,-0.993l-8.557,0l0,-6.196l1.68,-1.68l0,0l0.96,-0.96c2.029,0 7.282,0 9.997,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.96l-7.597,0.002l0,0.957l8.557,0l0,6.227l-0.96,0.96l0,0l-1.68,1.68ZM76.745,19.309l-3.12,0l-1.68,-1.68l0,-6.216l0.96,-0.96l6.48,0l0,6.216l-2.64,2.64ZM78.185,11.653l-6,0l0,5.877l1.54,1.539l2.921,0l1.539,-1.539l0,-5.877ZM17.017,5.896l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-6.48l8.556,0l0,-1.807l-1.539,-1.54ZM25.788,5.896l-2.778,0l-1.54,1.54l0,10.094l1.54,1.539l9.719,0l1.539,-1.539l0,-1.799l-8.478,-0.008l-0.002,-9.827ZM41.441,5.896l-2.779,0l-1.539,1.54l0,10.094l1.539,1.539l9.719,0l1.539,-1.539l0,-10.094l-1.539,-1.54l-2.778,0l-0.003,9.827l-4.157,0l-0.002,-9.827ZM64.201,5.896l-9.797,0l-0.82,0.82l-0.72,0.72l0,5.856l8.557,0l0,2.432l-8.557,0.003l0,1.803l1.54,1.539l9.797,0l1.54,-1.539l0,-5.888l-8.557,0l0,-2.396l8.557,-0.003l0,-1.807l-1.54,-1.54ZM81.921,9.497l-13.471,0l0,-2.161l2.64,-2.64l10.111,0l1.68,1.68c0,1.034 0,2.161 0,2.161l-0.96,0.96ZM80.141,5.896l-9.912,0l-1.539,1.54l0,1.821l12.991,0l0,-1.821l-1.54,-1.54Z" style="fill:#141414;"/><path d="M73.625,19.309l3.12,0l-3.12,0ZM76.646,19.069l-2.921,0l-1.54,-1.539l0,-5.877l6,0l0,5.877l-1.539,1.539ZM17.017,19.069l-9.798,0l-1.539,-1.539l0,-10.094l1.539,-1.54l9.798,0l1.539,1.54l0,1.807l-8.556,0l0,6.48l8.556,0l0,1.807l-1.539,1.539ZM32.729,19.069l-9.719,0l-1.54,-1.539l0,-10.094l1.54,-1.54l2.778,0l0.002,9.827l8.478,0.008l0,1.799l-1.539,1.539ZM48.381,19.069l-9.719,0l-1.539,-1.539l0,-10.094l1.539,-1.54l2.779,0l0.002,9.827l4.157,0l0.003,-9.827l2.778,0l1.539,1.54l0,10.094l-1.539,1.539ZM64.201,19.069l-9.797,0l-1.54,-1.539l0,-1.803l8.557,-0.003l0,-2.432l-8.557,0l0,-5.856l1.54,-1.54l9.797,0l1.54,1.54l0,1.807l-8.557,0.003l0,2.396l8.557,0l0,5.888l-1.54,1.539ZM78.425,11.413l0,6.216l0,-6.216ZM68.45,7.336l1.68,-1.68l-1.68,1.68c0,1.034 0,2.161 0,2.161l0,-2.161ZM81.681,9.257l-12.991,0l0,-1.821l1.539,-1.54l9.912,0l1.54,1.54l0,1.821Z" style="fill:#d58b36;"/></g><g transform="translate(-71.277,41.509)"><path d="M105.009,19.309l-3.12,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68l0,6.2l-0.96,0.96l-7.596,-0.003l0,4.816l-0.96,0.953l0,0.007ZM111.885,19.309l-3.12,0l0,-3.944l0.96,-0.96l4.8,0l0,2.212l-2.64,2.692ZM96.07,19.309l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-1.086,0.96l-7.47,0l0,0.959l5.841,0l0,2.13l-0.96,0.96l-4.881,0l0,0.991l8.556,0l0,2.146l-2.64,2.64ZM116.135,11.413l0.96,-0.96l6.481,0l0,5.953l-2.64,2.903l-3.121,0l-1.68,-1.943l0,-5.953ZM137.546,19.309l-2.872,0l0,-1.178c-1.382,-0.123 -2.749,-0.431 -4.028,-0.977l-0.056,-0.024l-1.307,0.804l-2.03,-1.25l1.277,-0.786c-0.81,-0.666 -1.487,-1.522 -1.761,-2.534l-1.749,0l0,-1.768l0.96,-0.96l1.141,0c0.332,-0.616 0.809,-1.156 1.35,-1.61l-1.198,-0.78l0.94,-0.93l2.03,-1.25l1.227,0.755c0.949,-0.424 2.033,-0.741 3.204,-0.922l0,-0.248l0.013,0l0.947,-0.96l2.872,0l0,1.116c1.505,0.156 2.889,0.535 4.059,1.079l1.332,-0.82l2.03,1.25l-1.12,1.059c0.701,0.671 1.181,1.437 1.377,2.261l1.976,0l0,1.768l-0.96,0.962l-0.002,-0.002l-1.327,0c-0.228,0.624 -0.547,1.236 -1.053,1.681l1.109,0.679l-0.968,0.978l-0.015,-0.004l-2.007,1.236l-1.411,-0.868c-0.958,0.432 -1.978,0.728 -3.02,0.908l0,0.375l-0.96,0.93l0,0.03ZM122.376,11.653l-6.001,0l0,5.624l1.55,1.792l2.901,0l1.55,-1.792l0,-5.624ZM113.325,15.605l-4.32,0l0,3.464l2.778,0l1.542,-1.59l0,-1.874ZM95.971,5.896l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-2.431l5.841,0l0,-1.65l-5.841,0l0,-2.399l8.556,0l0,-1.807l-1.539,-1.54ZM111.786,5.896l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l2.781,0l0,-5.777l8.556,0.004l0,-5.86l-1.539,-1.54ZM140.11,13.349c0.327,-0.363 0.421,-0.95 0.212,-1.53c-0.459,-1.279 -2.339,-2.021 -4.278,-2.122c-0.058,0 -0.058,0 -0.115,0c-2.792,0.201 -3.386,1.31 -3.433,1.781c-0.151,1.505 1.994,2.74 4.45,2.74c1.269,0 2.7,-0.357 3.164,-0.869ZM109.009,11.642l0,-2.398l-4.24,0.002l0,2.396l4.24,0ZM105.969,9.485l0,0.957l2.8,0l0,-0.958l-2.8,0.001ZM122.527,9.449l-6.392,0l0,-2.113Line truncated

package.json

@@ -15,10 +15,9 @@
 		"cover": "c8 pnpm test",
 		"fast-cover": "FAST_TEST=y c8 pnpm test",
 		"ci-cover": "c8 --reporter=lcovonly pnpm test",
-		"lint": "eslint packages plugins test",
-		"lint-fix": "eslint --fix packages plugins test",
+		"lint": "oxlint packages plugins test",
+		"lint-fix": "oxlint --fix packages plugins test",
 		"clean": "node ./clean.js",
-		"docs": "typedoc",
 		"bundle-dependencies": "bundle-dependencies"
 	},
 	"c8": {
@@ -29,7 +28,7 @@
 		]
 	},
 	"engines": {
-		"node": ">=22"
+		"node": "^22.18.0 || >=24.2.0"
 	},
 	"workspaces": [
 		"packages/*",
@@ -43,24 +42,16 @@
 		"@clusterio/host": "workspace:^",
 		"@clusterio/lib": "workspace:^",
 		"@clusterio/web_ui": "workspace:^",
-		"@eslint/eslintrc": "^3.3.7",
-		"@eslint/js": "^10.0.1",
 		"@stylistic/eslint-plugin": "^5.10.0",
-		"@swc/core": "catalog:",
 		"@types/node": "catalog:",
 		"c8": "^12.0.0",
-		"@typescript-eslint/eslint-plugin": "^8.70.0",
-		"@typescript-eslint/parser": "^8.70.0",
-		"eslint": "^10.10.0",
-		"eslint-plugin-node": "^11.1.0",
 		"express": "catalog:",
-		"form-data": "^4.0.6",
-		"globals": "^17.12.0",
 		"jsonwebtoken": "catalog:",
-		"jszip": "catalog:",
 		"mocha": "^11.8.0",
-		"typedoc": "^0.28.20",
+		"oxlint": "^1.88.0",
 		"typescript": "catalog:",
-		"yargs": "catalog:"
+		"yargs": "catalog:",
+		"yauzl": "catalog:",
+		"yazl": "catalog:"
 	}
 }

packages/controller/.npmignore

@@ -1,2 +1,2 @@
-webpack.config.js
+vite.config.js
 dist/browser

packages/controller/controller.ts

@@ -23,13 +23,14 @@ import util from "util";
 import winston from "winston";
 import DailyRotateFile from "winston-daily-rotate-file";
 
-import jwt from "jsonwebtoken";
 
 // homebrew modules
+// Reduce startup time by lazy compiling schemas.
+import "./src/set_lazy_schema_compilation.js";
 import * as lib from "@clusterio/lib";
 import { ConsoleTransport, levels, logger } from "@clusterio/lib";
 
-import Controller from "./src/Controller.js";
+import type Controller from "./src/Controller.js";
 import UserManager from "./src/UserManager.js";
 import UserRecord from "./src/UserRecord.js";
 import packageConfig from "@clusterio/controller/package.json" with { type: "json" };
@@ -147,15 +148,15 @@ async function handleBootstrapCommand(
 		path.join(databaseDirectory, "roles.json"),
 	);
 
-	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+	const roles = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 		path.join(databaseDirectory, "roles.json"),
 		lib.Role.fromJSON.bind(lib.Role),
-	).bootstrap());
+	));
 
-	const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+	const users = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 		path.join(databaseDirectory, "users.json"),
 		UserRecord.fromJSON.bind(UserRecord),
-	).bootstrap());
+	));
 
 	const userManager = new UserManager(users, roles, controllerConfig);
 
@@ -186,14 +187,6 @@ async function handleBootstrapCommand(
 		// eslint-disable-next-line no-console
 		console.log(userManager.signUserToken(user));
 
-	} else if (subCommand === "generate-host-token") {
-		// eslint-disable-next-line no-console
-		console.log(jwt.sign(
-			{ aud: "host", host: args.id },
-			// Auth secret is never null after startup
-			Buffer.from(controllerConfig.get("controller.auth_secret")!, "base64")
-		));
-
 	} else if (subCommand === "create-ctl-config") {
 		let admin = userManager.getByName(args.name);
 		if (!admin) {
@@ -203,6 +196,7 @@ async function handleBootstrapCommand(
 		}
 		let ctlConfig = new lib.CtlConfig("control");
 
+		const { default: Controller } = await import("./src/Controller.js");
 		ctlConfig.set("ctl.controller_url", Controller.calculateControllerUrl(controllerConfig));
 		ctlConfig.set(
 			"ctl.controller_token",
@@ -215,7 +209,7 @@ async function handleBootstrapCommand(
 			console.log(content);
 		} else {
 			logger.info(`Writing ${args.output}`);
-			await lib.safeOutputFile(args.output, content, { mode: 0o600 });
+			await lib.atomicOverwriteFile(args.output, content, { mode: 0o600 });
 		}
 	}
 }
@@ -322,9 +316,6 @@ async function initialize(): Promise<InitializeParameters> {
 				.command("generate-user-token <name>", "Generate authentication token for the given user", yargs => {
 					yargs.positional("name", { describe: "Name of the user", type: "string" });
 				})
-				.command("generate-host-token <id>", "Generate authentication token for the given host", yargs => {
-					yargs.positional("id", { describe: "ID of the host", type: "number" });
-				})
 				.command("create-ctl-config <name>", "Create clusterioctl config for the given user", yargs => {
 					yargs.option("output", {
 						describe: "Path to output config (- for stdout)", type: "string",
@@ -400,7 +391,8 @@ async function initialize(): Promise<InitializeParameters> {
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = await lib.loadPluginList(args.pluginList);
+	const skippedNpmPlugins = new Map<string, string>();
+	let pluginList = await lib.loadPluginList(args.pluginList, true, skippedNpmPlugins);
 
 	// If the command is plugin management we don't try to load plugins
 	if (command === "plugin") {
@@ -415,7 +407,7 @@ async function initialize(): Promise<InitializeParameters> {
 	}
 
 	logger.info("Loading Plugin info");
-	const pluginInfos = await lib.loadPluginInfos(pluginList);
+	const pluginInfos = await lib.loadPluginInfos(pluginList, skippedNpmPlugins);
 	lib.registerPluginMessages(pluginInfos);
 	lib.registerPluginPermissions(pluginInfos);
 	lib.addPluginConfigFields(pluginInfos);
@@ -474,13 +466,14 @@ async function startup() {
 		return;
 	}
 
-	controller = new Controller(
-		clusterLogger,
-		pluginInfos,
+	// Loaded here as the server is not needed by the other commands
+	const { default: Controller } = await import("./src/Controller.js");
+	controller = await Controller.fromConfig(
 		controllerConfig,
+		pluginInfos,
+		clusterLogger,
 		Boolean(args.canRestart),
 		Boolean(args.recovery),
-		...await Controller.bootstrap(controllerConfig)
 	);
 
 	// Refuse to start if there are no users loaded
@@ -503,14 +496,6 @@ async function startup() {
 }
 
 export function bootstrap() {
-	// eslint-disable-next-line no-console
-	console.warn(`
-+==========================================================+
-I WARNING:  This is the development branch for the 2.0     I
-I           version of clusterio.  Expect things to break. I
-+==========================================================+
-`
-	);
 	startup().catch(err => {
 		if (err instanceof lib.StartupError || err.code === "InstallationError") {
 			logger.fatal(`

packages/controller/index.ts

@@ -5,7 +5,9 @@ export { default as ControlConnection } from "./src/ControlConnection.js";
 export { default as HostConnection } from "./src/HostConnection.js";
 export { default as InstanceRecord } from "./src/InstanceRecord.js";
 export { default as InstanceManager } from "./src/InstanceManager.js";
+export { default as HostManager } from "./src/HostManager.js";
 export { default as HostRecord } from "./src/HostRecord.js";
+export { default as HostPairingManager, HostPairingError } from "./src/HostPairingManager.js";
 export { BaseControllerPlugin, ControllerHooks, type ControllerPluginContext } from "./src/BaseControllerPlugin.js";
 export { default as UserManager } from "./src/UserManager.js";
 export { default as UserRecord } from "./src/UserRecord.js";

packages/controller/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/controller",
-	"version": "2.0.0",
+	"version": "2.0.3",
 	"description": "Implementation of Clusterio controller",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -25,14 +25,14 @@
 	"type": "module",
 	"main": "dist/node/index.js",
 	"scripts": {
-		"prepare": "tsc --build && webpack-cli --mode production",
+		"prepare": "tsc --build && vite build",
 		"test": "echo \"Error: no test specified\" && exit 1"
 	},
 	"bin": {
 		"clusteriocontroller": "./dist/node/controller.js"
 	},
 	"engines": {
-		"node": ">=22"
+		"node": "^22.18.0 || >=24.2.0"
 	},
 
 	"dependencies": {
@@ -43,7 +43,6 @@
 		"express": "catalog:",
 		"finalhandler": "^2.1.1",
 		"jsonwebtoken": "catalog:",
-		"jszip": "catalog:",
 		"semver": "catalog:",
 		"winston": "catalog:",
 		"winston-daily-rotate-file": "catalog:",
@@ -52,8 +51,7 @@
 	},
 	"devDependencies": {
 		"@clusterio/web_ui": "workspace:^",
-		"@swc/core": "catalog:",
-		"@types/busboy": "^1.5.4",
+		"@types/busboy": "^1.5.5",
 		"@types/compression": "^1.8.1",
 		"@types/express": "catalog:",
 		"@types/finalhandler": "catalog:",
@@ -62,13 +60,12 @@
 		"@types/semver": "catalog:",
 		"@types/ws": "catalog:",
 		"@types/yargs": "catalog:",
+		"antd": "catalog:",
 		"react": "catalog:",
 		"react-dom": "catalog:",
+		"react-router": "catalog:",
 		"react-router-dom": "catalog:",
 		"typescript": "catalog:",
-		"webpack": "catalog:",
-		"webpack-cli": "catalog:",
-		"webpack-dev-middleware": "^6.1.3",
-		"webpack-merge": "catalog:"
+		"vite": "catalog:"
 	}
 }

packages/controller/src/BaseConnection.ts

@@ -63,7 +63,7 @@ export default class BaseConnection extends lib.Link {
 		let { id } = request;
 		let modPack = this._controller.modPacks.get(id);
 		if (!modPack) {
-			throw new lib.RequestError(`Mod pack with ID ${id} does not exist`);
+			throw new lib.NotFoundError(`Mod pack with ID ${id} does not exist`);
 		}
 		return modPack;
 	}
@@ -71,11 +71,11 @@ export default class BaseConnection extends lib.Link {
 	async handleModPackGetDefaultRequest(): Promise<lib.ModPack> {
 		let id = this._controller.config.get("controller.default_mod_pack_id");
 		if (id === null) {
-			throw new lib.RequestError("Default mod pack not set on controller");
+			throw new lib.ConflictError("Default mod pack not set on controller");
 		}
 		let modPack = this._controller.modPacks.get(id);
 		if (!modPack) {
-			throw new lib.RequestError(`Default mod pack configured (${id}) does not exist`);
+			throw new lib.NotFoundError(`Default mod pack configured (${id}) does not exist`);
 		}
 		return modPack;
 	}
@@ -84,10 +84,10 @@ export default class BaseConnection extends lib.Link {
 		let filename = lib.ModInfo.filename(mod.name, mod.version);
 		let modInfo = this._controller.modStore.files.get(filename);
 		if (!modInfo) {
-			throw new lib.RequestError(`Mod ${filename} does not exist on controller`);
+			throw new lib.NotFoundError(`Mod ${filename} does not exist on controller`);
 		}
 		if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
-			throw new lib.RequestError(`Mod ${filename} checksum does not match controller's checksum`);
+			throw new lib.ConflictError(`Mod ${filename} checksum does not match controller's checksum`);
 		}
 		return modInfo;
 	}

packages/controller/src/BaseControllerPlugin.ts

@@ -107,7 +107,7 @@ export class ControllerHooks extends lib.AsyncHookCollection {
 	 * @param curr - The current value of the field.
 	 * @param prev - The previous value of the field.
 	 */
-	// eslint-disable-next-line max-len
+	// eslint-disable-next-line @stylistic/max-len
 	readonly instanceConfigFieldChanged: lib.AsyncHook<[instance: InstanceRecord, field: string, curr: unknown, prev: unknown]>;
 
 	/**
@@ -165,7 +165,7 @@ export class ControllerHooks extends lib.AsyncHookCollection {
 	 *     The connection the event occured on.
 	 * @param event - one of connect, drop, resume and close
 	 */
-	// eslint-disable-next-line max-len
+	// eslint-disable-next-line @stylistic/max-len
 	readonly hostConnectionEvent: lib.AsyncHook<[connection: HostConnection, event: "connect" | "drop" | "resume" | "close"]>;
 
 	/**
@@ -202,7 +202,7 @@ export class ControllerHooks extends lib.AsyncHookCollection {
 	 *     The connection the event occured on.
 	 * @param event - one of connect, drop, resume, and close.
 	 */
-	// eslint-disable-next-line max-len
+	// eslint-disable-next-line @stylistic/max-len
 	readonly controlConnectionEvent: lib.AsyncHook<[connection: ControlConnection, event: "connect" | "drop" | "resume" | "close"]>;
 
 	/**

packages/controller/src/ControlConnection.ts

@@ -8,6 +8,7 @@ import { logFilter, logger } from "@clusterio/lib";
 import BaseConnection from "./BaseConnection.js";
 import User from "./User.js";
 import UserRecord from "./UserRecord.js";
+import type InstanceRecord from "./InstanceRecord.js";
 import * as routes from "./routes.js";
 import Controller from "./Controller.js";
 
@@ -81,10 +82,11 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.ControllerConfigSetRequest, this.handleControllerConfigSetRequest.bind(this));
 		this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this));
 		this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this));
-		this.handle(lib.HostRevokeTokensRequest, this.handleHostRevokeTokensRequest.bind(this));
+		this.handle(lib.HostRevokeRequest, this.handleHostRevokeRequest.bind(this));
 		this.handle(lib.HostListRequest, this.handleHostListRequest.bind(this));
-		this.handle(lib.HostGenerateTokenRequest, this.handleHostGenerateTokenRequest.bind(this));
-		this.handle(lib.HostConfigCreateRequest, this.handleHostConfigCreateRequest.bind(this));
+		this.handle(lib.HostPairingListRequest, this.handleHostPairingListRequest.bind(this));
+		this.handle(lib.HostPairingApproveRequest, this.handleHostPairingApproveRequest.bind(this));
+		this.handle(lib.HostPairingRejectRequest, this.handleHostPairingRejectRequest.bind(this));
 		this.handle(lib.InstanceDetailsGetRequest, this.handleInstanceDetailsGetRequest.bind(this));
 		this.handle(lib.InstanceDetailsListRequest, this.handleInstanceDetailsListRequest.bind(this));
 		this.handle(lib.InstanceCreateRequest, this.handleInstanceCreateRequest.bind(this));
@@ -93,6 +95,7 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.InstanceConfigSetRequest, this.handleInstanceConfigSetRequest.bind(this));
 		this.handle(lib.InstanceConfigSetFieldRequest, this.handleInstanceConfigSetFieldRequest.bind(this));
 		this.handle(lib.InstanceConfigSetPropRequest, this.handleInstanceConfigSetPropRequest.bind(this));
+		this.handle(lib.InstanceConfigInheritFieldRequest, this.handleInstanceConfigInheritFieldRequest.bind(this));
 		this.handle(lib.InstanceAssignRequest, this.handleInstanceAssignRequest.bind(this));
 		this.handle(lib.InstanceSaveDetailsListRequest, this.handleInstanceSaveDetailsListRequest.bind(this));
 		this.handle(lib.InstanceRenameSaveRequest, controller.sendRequestToHostByInstanceId.bind(controller));
@@ -119,6 +122,8 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.RoleUpdateRequest, this.handleRoleUpdateRequest.bind(this));
 		this.handle(lib.RoleGrantDefaultPermissionsRequest, this.handleRoleGrantDefaultPermissionsRequest.bind(this));
 		this.handle(lib.RoleDeleteRequest, this.handleRoleDeleteRequest.bind(this));
+		this.handle(lib.NoteListRequest, this.handleNoteListRequest.bind(this));
+		this.handle(lib.NoteSetRequest, this.handleNoteSetRequest.bind(this));
 		this.handle(lib.UserGetRequest, this.handleUserGetRequest.bind(this));
 		this.handle(lib.UserListRequest, this.handleUserListRequest.bind(this));
 		this.handle(lib.UserCreateRequest, this.handleUserCreateRequest.bind(this));
@@ -133,6 +138,8 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.PluginListRequest, this.handlePluginListRequest.bind(this));
 		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
 		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
+		this.handle(lib.PluginSearchRequest, this.handlePluginSearchRequest.bind(this));
+		this.handle(lib.PluginLatestVersionsRequest, this.handlePluginLatestVersionsRequest.bind(this));
 		this.handle(lib.UpdateAllRequest, this.handleUpdateAllRequest.bind(this));
 		this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this));
 	}
@@ -151,7 +158,7 @@ export default class ControlConnection extends BaseConnection {
 		try {
 			this.checkPermission(message, entry);
 		} catch (err: any) {
-			this.connector.sendResponseError(new lib.ResponseError(err.message, err.code), message.src);
+			this.connector.sendResponseError(lib.Link.errorToResponse(err, false), message.src);
 			logger.audit(`Permission denied for ${message.name} by ${this.user.id} from ${this.connector.dst}`);
 			throw err;
 		}
@@ -193,12 +200,15 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleControllerRestartRequest() {
 		if (!this._controller.canRestart) {
-			throw new lib.RequestError("Cannot restart, controller does not have a process monitor to restart it.");
+			throw new lib.ConflictError("Cannot restart, controller does not have a process monitor to restart it.");
+		}
+		if (await lib.packageOperationRunning()) {
+			throw new lib.ConflictError("Cannot restart while a plugin install or update is running on this machine.");
 		}
 		const downgrade = await this._controller.checkRestartDowngrade();
 		if (downgrade) {
 			const { installedVersion, runningVersion } = downgrade;
-			throw new lib.RequestError(
+			throw new lib.ConflictError(
 				`Cannot restart controller with installed Clusterio version ${installedVersion} because it ` +
 				`is older than running version ${runningVersion}. ` +
 				"Stop the controller before starting the older version manually."
@@ -241,49 +251,25 @@ export default class ControlConnection extends BaseConnection {
 		this._controller.config.setProp(field as keyof lib.ControllerConfigFields, prop, value, "control");
 	}
 
-	async handleHostRevokeTokensRequest(request: lib.HostRevokeTokensRequest) {
-		const host = this._controller.hosts.getMutable(request.hostId);
-		if (!host) {
-			throw new Error(`Unknown host id (${request.hostId})`);
-		}
-
-		host.tokenValidAfter = Math.floor(Date.now() / 1000);
-
-		const hostConnection = this._controller.wsServer.hostConnections.get(request.hostId);
-		if (hostConnection) {
-			hostConnection.connector.terminate();
-		}
-
-		this._controller.hosts.set(host);
+	async handleHostRevokeRequest(request: lib.HostRevokeRequest) {
+		await this._controller.hosts.revokeHost(request.hostId, this.user.name);
 	}
 
 	async handleHostListRequest(): Promise<lib.HostDetails[]> {
 		return [...this._controller.hosts.values()].map(host => host.toHostDetails());
 	}
 
-	async handleHostGenerateTokenRequest(message: lib.HostGenerateTokenRequest) {
-		let hostId = message.hostId;
-		if (hostId === undefined) {
-			hostId = Math.random() * 2**31 | 0;
-		}
-		return this._controller.generateHostToken(hostId);
+	async handleHostPairingListRequest(): Promise<lib.HostPairingDetails[]> {
+		this._controller.hostPairing.prune();
+		return [...this._controller.hostPairing.records.values()];
 	}
 
-	async handleHostConfigCreateRequest(request: lib.HostConfigCreateRequest) {
-		const hostConfig = new lib.HostConfig("host");
+	async handleHostPairingApproveRequest(request: lib.HostPairingApproveRequest) {
+		return this._controller.hostPairing.approve(request.code, request.name, this.user.name);
+	}
 
-		hostConfig.set("host.controller_url", this._controller.getControllerUrl());
-		if (request.id !== undefined) {
-			hostConfig.set("host.id", request.id);
-		}
-		if (request.name !== undefined) {
-			hostConfig.set("host.name", request.name);
-		}
-		if (request.generateToken) {
-			this.user.checkPermission("core.host.generate_token");
-			hostConfig.set("host.controller_token", this._controller.generateHostToken(hostConfig.get("host.id")));
-		}
-		return hostConfig.toJSON();
+	async handleHostPairingRejectRequest(request: lib.HostPairingRejectRequest) {
+		this._controller.hostPairing.reject(request.code, this.user.name);
 	}
 
 	async handleInstanceDetailsGetRequest(request: lib.InstanceDetailsGetRequest) {
@@ -296,23 +282,30 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleInstanceCreateRequest(request: lib.InstanceCreateRequest) {
 		if ((request.config["instance.assigned_host"] ?? null) !== null) {
-			throw new lib.RequestError("instance.assigned_host must be set through the assign-host interface");
+			throw new lib.ValidationError("instance.assigned_host must be set through the assign-host interface");
 		}
 		const instanceConfig = new lib.InstanceConfig("controller");
 		if (request.cloneFromId) {
 			const baseInstance = this._controller.instances.get(request.cloneFromId);
 			if (!baseInstance) {
-				throw new lib.RequestError(`Instance with ID ${request.cloneFromId} does not exist`);
+				throw new lib.NotFoundError(`Instance with ID ${request.cloneFromId} does not exist`);
 			}
 			instanceConfig.update(baseInstance.config.toJSON(), false);
 			instanceConfig.set("instance.assigned_host", null); // New instances are unassigned
 		}
 		instanceConfig.update(request.config, false, "control");
+		this._controller.instances.checkInheritance(
+			instanceConfig, instanceConfig.get("instance.is_template"), instanceConfig.get("instance.template_id")
+		);
+		if (Object.hasOwn(request.config, "instance.template_id")) {
+			this._controller.instances.applyTemplate(instanceConfig, Object.keys(request.config));
+		}
 		await this._controller.instances.createInstance(instanceConfig);
 	}
 
 	async handleInstanceDeleteRequest(request: lib.InstanceDeleteRequest) {
 		await this._controller.instances.deleteInstance(request.instanceId);
+		this._controller.deleteNote("instance", request.instanceId);
 	}
 
 	async handleInstanceConfigGetRequest(request: lib.InstanceConfigGetRequest) {
@@ -322,12 +315,12 @@ export default class ControlConnection extends BaseConnection {
 
 	private static validateInstanceConfigSetField(fieldName: string) {
 		if (fieldName === "instance.assigned_host") {
-			throw new lib.RequestError("instance.assigned_host must be set through the assign-host interface");
+			throw new lib.ValidationError("instance.assigned_host must be set through the assign-host interface");
 		}
 
 		if (fieldName === "instance.id") {
 			// XXX is this worth implementing?  It's race condition galore.
-			throw new lib.RequestError("Setting instance.id is not supported");
+			throw new lib.ValidationError("Setting instance.id is not supported");
 		}
 	}
 
@@ -347,17 +340,41 @@ export default class ControlConnection extends BaseConnection {
 				}
 			}
 
+			this.checkStagedInheritance(instance);
 			instance.config.commitStaging();
+			for (const [fieldName, fieldValue] of Object.entries(request.fields)) {
+				const props = typeof fieldValue === "object"
+					? Object.keys(fieldValue).filter(prop => fieldValue[prop] !== undefined)
+					: undefined;
+				this._controller.instances.overrideField(instance, fieldName, props);
+			}
 			await this._controller.instanceConfigUpdated(instance);
 		} finally {
 			instance.config.revertStaging();
 		}
 	}
 
+	private checkStagedInheritance(instance: InstanceRecord) {
+		const staged = instance.config.staging;
+		this._controller.instances.checkInheritance(
+			instance.config, staged.get("instance.is_template"), staged.get("instance.template_id")
+		);
+	}
+
 	async handleInstanceConfigSetFieldRequest(request: lib.InstanceConfigSetFieldRequest) {
 		const instance = this._controller.instances.getForRequest(request.instanceId);
-		ControlConnection.validateInstanceConfigSetField(request.field);
-		instance.config.set(request.field as keyof lib.InstanceConfigFields, request.value, "control");
+		const field = request.field as keyof lib.InstanceConfigFields;
+		ControlConnection.validateInstanceConfigSetField(field);
+		if (["instance.is_template", "instance.template_id"].includes(field)) {
+			try {
+				instance.config.stage(field, request.value, "control");
+				this.checkStagedInheritance(instance);
+			} finally {
+				instance.config.revertStaging();
+			}
+		}
+		instance.config.set(field, request.value, "control");
+		this._controller.instances.overrideField(instance, field);
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
@@ -366,9 +383,18 @@ export default class ControlConnection extends BaseConnection {
 		let { field, prop, value } = request;
 		ControlConnection.validateInstanceConfigSetField(field);
 		instance.config.setProp(field as keyof lib.InstanceConfigFields, prop, value, "control");
+		if (value !== undefined) {
+			this._controller.instances.overrideField(instance, field, [prop]);
+		}
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
+	async handleInstanceConfigInheritFieldRequest(request: lib.InstanceConfigInheritFieldRequest) {
+		ControlConnection.validateInstanceConfigSetField(request.field);
+		this._controller.instances.inheritField(request.instanceId, request.field, request.prop);
+		await this._controller.instanceConfigUpdated(this._controller.instances.getForRequest(request.instanceId));
+	}
+
 	async handleInstanceAssignRequest(request: lib.InstanceAssignRequest) {
 		await this._controller.instances.assignInstance(request.instanceId, request.hostId ?? null);
 	}
@@ -385,7 +411,7 @@ export default class ControlConnection extends BaseConnection {
 		let ready = new Promise<void>((resolve, reject) => {
 			stream.events.on("source", resolve);
 			stream.events.on("timeout", () => reject(
-				new lib.RequestError("Timed out establishing stream from host")
+				new lib.TimeoutError("Timed out establishing stream from host")
 			));
 		});
 		ready.catch(() => {});
@@ -402,17 +428,17 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleInstanceTransferSaveRequest(request: lib.InstanceTransferSaveRequest) {
 		if (request.sourceInstanceId === request.targetInstanceId) {
-			throw new lib.RequestError("Source and target instance may not be the same");
+			throw new lib.ValidationError("Source and target instance may not be the same");
 		}
 		let sourceInstance = this._controller.instances.getForRequest(request.sourceInstanceId);
 		let targetInstance = this._controller.instances.getForRequest(request.targetInstanceId);
 		let sourceHostId = sourceInstance.config.get("instance.assigned_host");
 		let targetHostId = targetInstance.config.get("instance.assigned_host");
 		if (sourceHostId === null) {
-			throw new lib.RequestError("Source instance is not assigned a host");
+			throw new lib.ConflictError("Source instance is not assigned a host");
 		}
 		if (targetHostId === null) {
-			throw new lib.RequestError("Target instance is not assigned a host");
+			throw new lib.ConflictError("Target instance is not assigned a host");
 		}
 
 		// Let host handle request if source and target is on the same host.
@@ -423,12 +449,12 @@ export default class ControlConnection extends BaseConnection {
 		// Check connectivity
 		let sourceHostConnection = this._controller.wsServer.hostConnections.get(sourceHostId);
 		if (!sourceHostConnection || sourceHostConnection.connector.closing) {
-			throw new lib.RequestError("Source host is not connected to the controller");
+			throw new lib.UnavailableError("Source host is not connected to the controller");
 		}
 
 		let targetHostConnection = this._controller.wsServer.hostConnections.get(targetHostId);
 		if (!targetHostConnection || targetHostConnection.connector.closing) {
-			throw new lib.RequestError("Target host is not connected to the controller");
+			throw new lib.UnavailableError("Target host is not connected to the controller");
 		}
 
 		// Create stream to proxy from target to source
@@ -437,7 +463,7 @@ export default class ControlConnection extends BaseConnection {
 			if (stream.source) {
 				stream.source.destroy();
 			}
-			stream.events.emit("error", new lib.RequestError("Timed out establishing transfer stream"));
+			stream.events.emit("error", new lib.TimeoutError("Timed out establishing transfer stream"));
 		});
 
 		// Ignore errors if not listening for them to avoid crash.
@@ -477,10 +503,10 @@ export default class ControlConnection extends BaseConnection {
 	async handleModPackCreateRequest(request: lib.ModPackCreateRequest) {
 		let modPack = request.modPack;
 		if (modPack.id === undefined) {
-			throw new lib.RequestError("Mod pack need an ID to be created");
+			throw new lib.ValidationError("Mod pack need an ID to be created");
 		}
 		if (this._controller.modPacks.has(modPack.id)) {
-			throw new lib.RequestError(`Mod pack with ID ${modPack.id} already exist`);
+			throw new lib.ConflictError(`Mod pack with ID ${modPack.id} already exist`);
 		}
 		this._controller.modPacks.set(modPack);
 	}
@@ -488,7 +514,7 @@ export default class ControlConnection extends BaseConnection {
 	async handleModPackUpdateRequest(request: lib.ModPackUpdateRequest) {
 		let modPack = request.modPack;
 		if (modPack.id === undefined || !this._controller.modPacks.has(modPack.id)) {
-			throw new lib.RequestError(`Mod pack with ID ${modPack.id} does not exist`);
+			throw new lib.NotFoundError(`Mod pack with ID ${modPack.id} does not exist`);
 		}
 		this._controller.modPacks.set(modPack);
 	}
@@ -497,9 +523,10 @@ export default class ControlConnection extends BaseConnection {
 		let { id } = request;
 		let modPack = this._controller.modPacks.getMutable(id);
 		if (!modPack) {
-			throw new lib.RequestError(`Mod pack with ID ${id} does not exist`);
+			throw new lib.NotFoundError(`Mod pack with ID ${id} does not exist`);
 		}
 		this._controller.modPacks.delete(modPack);
+		this._controller.deleteNote("mod_pack", id);
 	}
 
 	async handleModGetRequest(request: lib.ModGetRequest) {
@@ -578,7 +605,7 @@ export default class ControlConnection extends BaseConnection {
 				author: (a:ModVersions, b:ModVersions) => strcmp(a.versions[0].author, b.versions[0].author),
 			};
 			if (!Object.prototype.hasOwnProperty.call(sorters, sort)) {
-				throw new lib.RequestError(`Invalid value for sort: ${sort}`);
+				throw new lib.ValidationError(`Invalid value for sort: ${sort}`);
 			}
 			resultList.sort(sorters[sort as keyof typeof sorters]);
 			let order = request.sortOrder;
@@ -625,7 +652,7 @@ export default class ControlConnection extends BaseConnection {
 		} catch (error: any) {
 			logger.error(`Error fetching all mods from portal (${request.factorioVersion}): ${error}`);
 			// Propagate a user-friendly error back to the client
-			throw new lib.RequestError(`Portal mod fetch failed: ${error}`);
+			throw new lib.InternalServerError(`Portal mod fetch failed: ${error}`);
 		}
 	}
 
@@ -643,7 +670,7 @@ export default class ControlConnection extends BaseConnection {
 			const token = this._controller.config.get("controller.factorio_token");
 
 			if (!username || username === "" || !token || token === "") {
-				throw new lib.RequestError("Factorio credentials (username, token) not configured on the controller.");
+				throw new lib.ConflictError("Factorio credentials (username, token) not configured on the controller.");
 			}
 
 			try {
@@ -652,18 +679,16 @@ export default class ControlConnection extends BaseConnection {
 
 			} catch (error: any) {
 				logger.error(`Error downloading mods from portal: ${error.message}`);
-				// Improve error message clarity
-				let errorMessage = "Mod portal download failed";
-				if (error instanceof lib.RequestError) {
-					errorMessage += `: ${error.message}`;
-				} else if (error.message.includes("401") || error.message.includes("Unauthorized")) {
-					errorMessage += ": Authentication failed. Check Factorio username/token in controller config.";
-				} else if (error.message.includes("404") || error.message.includes("Not Found")) {
-					errorMessage += ": Mod version not found on the portal.";
-				} else {
-					errorMessage += `: ${error.message}`;
+				const errorMessage = "Mod portal download failed";
+				if (error.message.includes("401") || error.message.includes("Unauthorized")) {
+					throw new lib.ConflictError(
+						`${errorMessage}: Authentication failed. Check Factorio username/token in controller config.`
+					);
 				}
-				throw new lib.RequestError(errorMessage);
+				if (error.message.includes("404") || error.message.includes("Not Found")) {
+					throw new lib.NotFoundError(`${errorMessage}: Mod version not found on the portal.`);
+				}
+				throw new lib.InternalServerError(`${errorMessage}: ${error.message}`);
 			}
 		}
 
@@ -673,7 +698,7 @@ export default class ControlConnection extends BaseConnection {
 		for (const mod of request.mods) {
 			const modInfo = installedMods.find(m => m.name === mod.name && mod.version.testVersion(m.version));
 			if (!modInfo) {
-				throw new lib.RequestError(`Failed to find mod ${mod.name} in store after download attempt.`);
+				throw new lib.InternalServerError(`Failed to find mod ${mod.name} in store after download attempt.`);
 			}
 			modInfos.push(modInfo);
 		}
@@ -707,7 +732,7 @@ export default class ControlConnection extends BaseConnection {
 				} catch (error: any) {
 					if (!error.message.includes("404") && !error.message.includes("Not Found")) {
 						logger.error(`Error fetching mod from portal: ${error.message}`);
-						throw new lib.RequestError(`Dependency resolution failed: ${error.message}`);
+						throw new lib.InternalServerError(`Dependency resolution failed: ${error.message}`);
 					} else {
 						releases = null;
 					}
@@ -863,6 +888,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleModDeleteRequest(request: lib.ModDeleteRequest) {
 		await this._controller.modStore.deleteMod(request.name, request.version);
+		this._controller.deleteNote("mod", `${request.name}_${request.version}`);
 	}
 
 	async handleLogSetSubscriptionsRequest(request: lib.LogSetSubscriptionsRequest) {
@@ -946,7 +972,7 @@ export default class ControlConnection extends BaseConnection {
 		let { id, name, description, permissions } = request;
 		let role = this._controller.roles.getMutable(id);
 		if (!role) {
-			throw new lib.RequestError(`Role with ID ${id} does not exist`);
+			throw new lib.NotFoundError(`Role with ID ${id} does not exist`);
 		}
 
 		role.name = name;
@@ -958,7 +984,7 @@ export default class ControlConnection extends BaseConnection {
 	async handleRoleGrantDefaultPermissionsRequest(request: lib.RoleGrantDefaultPermissionsRequest) {
 		let role = this._controller.roles.get(request.id);
 		if (!role) {
-			throw new lib.RequestError(`Role with ID ${request.id} does not exist`);
+			throw new lib.NotFoundError(`Role with ID ${request.id} does not exist`);
 		}
 
 		role.grantDefaultPermissions();
@@ -970,9 +996,10 @@ export default class ControlConnection extends BaseConnection {
 
 		const role = this._controller.roles.get(id);
 		if (!role) {
-			throw new lib.RequestError(`Role with ID ${id} does not exist`);
+			throw new lib.NotFoundError(`Role with ID ${id} does not exist`);
 		}
 		this._controller.roles.delete(role);
+		this._controller.deleteNote("role", id);
 
 		for (const user of this._controller.users.values()) {
 			if (user.removeRole(id)) {
@@ -981,11 +1008,23 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
+	async handleNoteListRequest() {
+		return [...this._controller.notes.values()].filter(note => !note.isDeleted);
+	}
+
+	async handleNoteSetRequest(request: lib.NoteSetRequest) {
+		const { resourceType, resourceId, content } = request;
+		if (!resourceType || !resourceId) {
+			throw new lib.ValidationError("resourceType and resourceId must not be empty");
+		}
+		this._controller.setNote(resourceType, resourceId, content, this.user.name);
+	}
+
 	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.UserDetails> {
 		let name = request.name;
 		let user = this._controller.users.getByName(name);
 		if (!user) {
-			throw new lib.RequestError(`User ${name} does not exist`);
+			throw new lib.NotFoundError(`User ${name} does not exist`);
 		}
 
 		return user.toUserDetails();
@@ -1002,7 +1041,7 @@ export default class ControlConnection extends BaseConnection {
 	async handleUserRevokeTokenRequest(request: lib.UserRevokeTokenRequest) {
 		let user = this._controller.users.getByName(request.name);
 		if (!user) {
-			throw new lib.RequestError(`User '${request.name}' does not exist`);
+			throw new lib.NotFoundError(`User '${request.name}' does not exist`);
 		}
 		if (user.id !== this.user.id) {
 			this.user.checkPermission("core.user.revoke_other_token");
@@ -1019,12 +1058,12 @@ export default class ControlConnection extends BaseConnection {
 	async handleUserUpdateRolesRequest(request: lib.UserUpdateRolesRequest) {
 		let user = this._controller.users.getByName(request.name);
 		if (!user) {
-			throw new lib.RequestError(`User '${request.name}' does not exist`);
+			throw new lib.NotFoundError(`User '${request.name}' does not exist`);
 		}
 
 		for (let roleId of request.roles) {
 			if (!this._controller.roles.has(roleId)) {
-				throw new lib.RequestError(`Role with ID ${roleId} does not exist`);
+				throw new lib.NotFoundError(`Role with ID ${roleId} does not exist`);
 			}
 		}
 
@@ -1040,7 +1079,7 @@ export default class ControlConnection extends BaseConnection {
 				this.user.checkPermission("core.user.create");
 				user = this._controller.users.createUser(name);
 			} else {
-				throw new lib.RequestError(`User '${name}' does not exist`);
+				throw new lib.NotFoundError(`User '${name}' does not exist`);
 			}
 		}
 
@@ -1056,7 +1095,7 @@ export default class ControlConnection extends BaseConnection {
 				this.user.checkPermission("core.user.create");
 				user = this._controller.users.createUser(name);
 			} else {
-				throw new lib.RequestError(`User '${name}' does not exist`);
+				throw new lib.NotFoundError(`User '${name}' does not exist`);
 			}
 		}
 
@@ -1074,7 +1113,7 @@ export default class ControlConnection extends BaseConnection {
 				this.user.checkPermission("core.user.create");
 				user = this._controller.users.createUser(name);
 			} else {
-				throw new lib.RequestError(`User '${name}' does not exist`);
+				throw new lib.NotFoundError(`User '${name}' does not exist`);
 			}
 		}
 
@@ -1086,7 +1125,7 @@ export default class ControlConnection extends BaseConnection {
 		let name = request.name;
 		let user = this._controller.users.getByName(name);
 		if (!user) {
-			throw new lib.RequestError(`User '${name}' does not exist`);
+			throw new lib.NotFoundError(`User '${name}' does not exist`);
 		}
 
 		this._controller.users.deleteUser(user);
@@ -1095,6 +1134,7 @@ export default class ControlConnection extends BaseConnection {
 				controlConnection.connector.terminate();
 			}
 		}
+		this._controller.deleteNote("user", user.id);
 
 		if (user.isAdmin) {
 			this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, false));
@@ -1316,47 +1356,54 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleControllerUpdateRequest(request: lib.ControllerUpdateRequest) {
 		if (!this._controller.config.get("controller.allow_remote_updates")) {
-			throw new lib.RequestError("Remote updates are disabled on this machine");
+			throw new lib.ForbiddenError("Remote updates are disabled on this machine");
 		}
 		return await lib.updatePackage("@clusterio/controller");
 	}
 
 	async handlePluginUpdateRequest(request: lib.PluginUpdateRequest) {
 		if (!this._controller.config.get("controller.allow_plugin_updates")) {
-			throw new lib.RequestError("Plugin updates are disabled on this machine");
+			throw new lib.ForbiddenError("Plugin updates are disabled on this machine");
 		}
 		return await lib.handlePluginUpdate(request.pluginPackage, this._controller.pluginInfos);
 	}
 
 	async handlePluginInstallRequest(request: lib.PluginInstallRequest) {
 		if (!this._controller.config.get("controller.allow_plugin_install")) {
-			throw new lib.RequestError("Plugin installs are disabled on this machine");
+			throw new lib.ForbiddenError("Plugin installs are disabled on this machine");
 		}
-		await lib.handlePluginInstall(request.pluginPackage);
-		// A new plugin is not in pluginInfos, so checkRestartRequired cannot notice it
-		if (!this._controller.pluginInfos.some(pluginInfo => pluginInfo.npmPackage === request.pluginPackage)) {
+		const npmPackage = await lib.handlePluginInstall(request.pluginPackage);
+		// A new plugin is not in pluginInfos, so checkRestartRequired cannot notice it. Version
+		// changes to loaded plugins are noticed there.
+		if (!this._controller.pluginInfos.some(pluginInfo => pluginInfo.npmPackage === npmPackage)) {
 			this._controller.config.restartRequired = true;
 		}
 	}
 
 	async handleUpdateAllRequest(request: lib.UpdateAllRequest) {
 		if (!this._controller.config.get("controller.allow_remote_updates")) {
-			throw new lib.RequestError("Remote updates are disabled on this machine");
+			throw new lib.ForbiddenError("Remote updates are disabled on this machine");
 		}
 		if (!this._controller.config.get("controller.allow_plugin_updates")) {
-			throw new lib.RequestError("Plugin updates are disabled on this machine");
+			throw new lib.ForbiddenError("Plugin updates are disabled on this machine");
 		}
 		return await lib.handleUpdateAll("@clusterio/controller", this._controller.pluginInfos);
 	}
 
-	async handlePluginListRequest(request: lib.PluginListRequest) {
-		return this._controller.pluginInfos.map(pluginInfo => lib.PluginDetails.fromNodeEnvInfo(
-			pluginInfo,
-			this._controller.loadedPlugins.has(pluginInfo),
-			this._controller.config.get(`${pluginInfo.name}.load_plugin`),
+	async handlePluginSearchRequest(request: lib.PluginSearchRequest) {
+		return await lib.searchPlugins(request.query, request.page, request.pageSize);
+	}
+
+	async handlePluginLatestVersionsRequest(request: lib.PluginLatestVersionsRequest) {
+		return new lib.PluginLatestVersionsRequest.Response(await lib.fetchLatestVersions(
+			request.packages, this._controller.pluginLatestVersionCache,
 		));
 	}
 
+	async handlePluginListRequest(request: lib.PluginListRequest) {
+		return await this._controller.listPlugins();
+	}
+
 	async handleDebugDumpWsRequest(request: lib.DebugDumpWsRequest) {
 		if (this.ws_dumper) {
 			return; // Already dumping

packages/controller/src/Controller.ts

@@ -11,7 +11,6 @@ import events from "events";
 import fs from "node:fs/promises";
 import http from "http";
 import https from "https";
-import jwt from "jsonwebtoken";
 import path from "path";
 import semver from "semver";
 import stream from "stream";
@@ -28,7 +27,9 @@ import User from "./User.js";
 import UserRecord from "./UserRecord.js";
 import UserManager from "./UserManager.js";
 import WsServer from "./WsServer.js";
+import HostManager from "./HostManager.js";
 import HostRecord from "./HostRecord.js";
+import HostPairingManager from "./HostPairingManager.js";
 import { BaseControllerPlugin, ControllerHooks } from "./BaseControllerPlugin.js";
 import ControllerRouter from "./ControllerRouter.js";
 import InstanceManager from "./InstanceManager.js";
@@ -52,7 +53,7 @@ type ControllerDebugEvents = {
 
 type ControllerEvents = {
 	"stop": [],
-}
+};
 
 /**
  * Manages all controller related operations
@@ -75,6 +76,8 @@ export default class Controller {
 	hooks = new ControllerHooks(logger);
 	/** Plugins which are currently loaded */
 	loadedPlugins: Set<lib.PluginNodeEnvInfo> = new Set();
+	/** Latest versions of plugins on npm, as shown on the plugins page */
+	pluginLatestVersionCache: lib.LatestVersionCache = new Map();
 
 	/** WebSocket server */
 	wsServer: WsServer;
@@ -85,10 +88,14 @@ export default class Controller {
 
 	/** Event subscription controller */
 	subscriptions: lib.SubscriptionController;
+	/** Host manager for the cluster, also contains access to the host records */
+	hosts: HostManager;
 	/** User manager for the cluster, also contains access to the user records */
 	users: UserManager;
 	/** Instance manager for the cluster, also contains access to the user records */
 	instances: InstanceManager;
+	/** Pending requests from hosts to join the cluster */
+	hostPairing: HostPairingManager;
 
 	// Possible states are new, starting, running, stopping, stopped
 	private _state: string = "new";
@@ -104,7 +111,7 @@ export default class Controller {
 	_registeredEvents = new Map<lib.EventClass<unknown>, lib.EventHandler<unknown>>();
 	_snoopedEvents = new Map();
 
-	devMiddleware: any | null = null;
+	devServer: import("vite").ViteDevServer | null = null;
 
 	startedAtMs = Date.now();
 	autosaveInterval?: ReturnType<typeof setInterval>;
@@ -133,7 +140,7 @@ export default class Controller {
 		const cachePath = path.join(this.config.get("controller.database_directory"), "latest-releases.json");
 		try {
 			const releases = await lib.fetchLatestReleases();
-			await lib.safeOutputFile(cachePath, JSON.stringify(releases, null, "\t"));
+			await lib.atomicOverwriteFile(cachePath, JSON.stringify(releases, null, "\t"));
 			return releases;
 		} catch (err: any) {
 			logger.warn(`Failed to retrieve Factorio releases ${err.message}`);
@@ -148,7 +155,14 @@ export default class Controller {
 		}
 	});
 
-	static async bootstrap(config: lib.ControllerConfig) {
+	/** Create a controller with its datastores and mod store loaded from disk */
+	static async fromConfig(
+		config: lib.ControllerConfig,
+		pluginInfos: lib.PluginNodeEnvInfo[],
+		clusterLogger: winston.Logger,
+		canRestart = false,
+		recoveryMode = false,
+	) {
 		let databaseDirectory = config.get("controller.database_directory");
 		await fs.mkdir(databaseDirectory, { recursive: true });
 
@@ -158,23 +172,23 @@ export default class Controller {
 			path.join(databaseDirectory, "roles.json"),
 		);
 
-		const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		const systems = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "systems.json"),
 			lib.SystemInfo.fromJSON.bind(lib.SystemInfo),
 			this.migrateSystems,
-		).bootstrap());
+		));
 
-		const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		const hosts = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "hosts.json"),
 			HostRecord.fromJSON.bind(HostRecord),
 			this.migrateHosts, this.finaliseHosts,
-		).bootstrap());
+		));
 
-		const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		const instances = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "instances.json"),
 			InstanceRecord.fromJSON.bind(InstanceRecord),
 			this.migrateInstances, this.finaliseInstances,
-		).bootstrap());
+		));
 		// Save plugin versions written while loading
 		for (const instance of instances.values()) {
 			if (instance.config.dirty) {
@@ -182,33 +196,40 @@ export default class Controller {
 			}
 		}
 
-		const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		const saves = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "saves.json"),
 			lib.SaveDetails.fromJSON.bind(lib.SaveDetails)
-		).bootstrap());
+		));
 
-		const modPacks = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		const modPacks = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "mod-packs.json"),
 			lib.ModPack.fromJSON.bind(lib.ModPack),
 			this.migrateModPacks,
-		).bootstrap());
+		));
 
-		const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		const roles = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "roles.json"),
 			lib.Role.fromJSON.bind(lib.Role),
 			this.migrateRoles,
-		).bootstrap());
+		));
 
-		const users = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		const users = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
 			path.join(databaseDirectory, "users.json"),
 			UserRecord.fromJSON.bind(UserRecord),
 			undefined, this.finaliseUsers,
-		).bootstrap());
+		));
+
+		const notes = await lib.SubscribableDatastore.fromProvider(new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "notes.json"),
+			lib.Note.fromJSON.bind(lib.Note),
+		));
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.mkdir(modsDirectory, { recursive: true });
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
+		await this.validateExports(modPacks, "static");
+
 		// Add default mod packs
 		if (modPacks.size === 0) {
 			modPacks.setMany(lib.ModPack.defaultModPacks);
@@ -218,7 +239,12 @@ export default class Controller {
 		lib.Role.ensureDefaultAdminRole(roles);
 		lib.Role.ensureDefaultPlayerRole(roles);
 
-		return [
+		return new this(
+			clusterLogger,
+			pluginInfos,
+			config,
+			canRestart,
+			recoveryMode,
 			systems,
 			hosts,
 			instances,
@@ -227,7 +253,8 @@ export default class Controller {
 			modStore,
 			roles,
 			users,
-		] as const;
+			notes,
+		);
 	}
 
 	constructor(
@@ -247,7 +274,7 @@ export default class Controller {
 		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
 		/** Mapping of host id to host record */
-		public hosts = new lib.SubscribableDatastore<HostRecord>(),
+		hosts = new lib.SubscribableDatastore<HostRecord>(),
 		/** Mapping of instance id to instance record */
 		instances = new lib.SubscribableDatastore<InstanceRecord>(),
 		/** Mapping of save id to save details */
@@ -260,6 +287,8 @@ export default class Controller {
 		public roles = new lib.SubscribableDatastore<lib.Role>(),
 		/** Mapping of user id to user record */
 		users = new lib.SubscribableDatastore<UserRecord>(),
+		/** Mapping of note id to note attached to a resource */
+		public notes = new lib.SubscribableDatastore<lib.Note>(),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -272,8 +301,10 @@ export default class Controller {
 		this.trustedProxies = this.parseTrustedProxies();
 		this.wsServer = new WsServer(this);
 
+		this.hosts = new HostManager(hosts, this);
 		this.users = new UserManager(users, roles, config);
 		this.instances = new InstanceManager(instances, this);
+		this.hostPairing = new HostPairingManager(this);
 
 		this.modStore.on("change", mod => {
 			this.modsUpdated([mod]);
@@ -283,6 +314,7 @@ export default class Controller {
 		this.subscriptions = new lib.SubscriptionController(config.get("controller.max_subscription_filters"));
 		this.subscriptions.handle(lib.SystemInfoUpdateEvent, this.handleSystemInfoSubscription.bind(this));
 		this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this));
+		this.subscriptions.handle(lib.HostPairingUpdatesEvent, this.handleHostPairingSubscription.bind(this));
 		this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this));
 		this.subscriptions.handle(
 			lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsSubscription.bind(this)
@@ -291,15 +323,18 @@ export default class Controller {
 		this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this));
 		this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this));
 		this.subscriptions.handle(lib.RoleUpdatesEvent, this.handleRoleSubscription.bind(this));
+		this.subscriptions.handle(lib.NoteUpdatesEvent, this.handleNoteSubscription.bind(this));
 
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
-		this.hosts.on("update", this.hostsUpdated.bind(this));
+		this.hosts.records.on("update", this.hostsUpdated.bind(this));
+		this.hostPairing.records.on("update", this.hostPairingRequestsUpdated.bind(this));
 		this.instances.records.on("update", this.instanceUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
 		this.roles.on("update", this.rolesUpdated.bind(this));
 		this.users.records.on("update", this.usersUpdated.bind(this));
+		this.notes.on("update", this.notesUpdated.bind(this));
 	}
 
 	get authSecret() {
@@ -354,7 +389,7 @@ export default class Controller {
 		}, 600e3);
 		logSizeGauge.callback = async () => { logSizeGauge.set(await this.logSize()); };
 
-		// Start webpack development server if enabled
+		// Start Vite development server if enabled
 		if (args.dev || args.devPlugin) {
 			await this._startDevServer(args);
 		}
@@ -392,15 +427,21 @@ export default class Controller {
 		}
 
 		Controller.addAppRoutes(this.app, this.pluginInfos);
+		if (this.devServer) {
+			this.app.use(this.devServer.middlewares);
+		}
 
-		if (!args.dev) {
+		if (!this.devServer) {
 			let manifestPath = path.join(import.meta.dirname, "..", "..", "web", "manifest.json");
 
 			let manifest = await Controller.loadJsonObject(manifestPath);
-			if (!manifest["main.js"]) {
-				logger.error("Missing main.js entry in dist/web/manifest.json");
+			if (!manifest.main) {
+				logger.error("Missing main entry in dist/web/manifest.json");
 			}
-			this.app.locals.mainBundle = manifest["main.js"] || "no_web_build";
+			this.app.locals.webBundle = {
+				main: manifest.main || "no_web_build",
+				imports: manifest.imports || {},
+			};
 		}
 		this.app.locals.staticRoot = this.config.get("controller.static_url");
 
@@ -459,6 +500,7 @@ export default class Controller {
 			logger.info(`Listening for HTTPS on port ${(this.httpsServer.address() as AddressInfo).port}`);
 		}
 
+		this.hostPairing.start();
 		this.onAutosaveIntervalChanged();
 		this.onSystemMetricsIntervalChanged();
 
@@ -468,37 +510,44 @@ export default class Controller {
 	}
 
 	async _startDevServer(args: ControllerArgs) {
-		logger.warn("Webpack development mode enabled");
+		logger.warn("Vite development mode enabled");
 
-		const webpack = (await import("webpack")).default;
-		const webpackDevMiddleware = (await import("webpack-dev-middleware")).default;
-		const webpackConfigs = [];
+		const vite = await import("vite");
+		const webUiVite = await import("@clusterio/web_ui/vite.mjs");
 
-		if (args.dev) {
-			// @ts-expect-error this file is run from a different folder
-			webpackConfigs.push((await import("../../../webpack.config.cjs")).default({})); // Path outside of build
-		}
-		if (args.devPlugin) {
-			let devPlugins = new Map();
-			for (let name of args.devPlugin) {
-				let info = this.pluginInfos.find(i => i.name === name);
-				if (!info) {
-					throw new lib.StartupError(`No plugin named ${name}`);
-				}
-				const webpackConfig = path.posix.join(info.requirePath, "webpack.config.cjs");
-				let config = (await import(
-					path.isAbsolute(webpackConfig) ? pathToFileURL(webpackConfig).href : webpackConfig
-				)).default({});
-				devPlugins.set(name, webpackConfigs.length);
-				webpackConfigs.push(config);
+		const devPlugins = new Map<string, string>();
+		const pluginDirs = new Map<string, string>();
+		for (let name of (args.devPlugin ?? []).map(String)) {
+			let info = this.pluginInfos.find(i => i.name === name);
+			if (!info) {
+				throw new lib.StartupError(`No plugin named ${name}`);
 			}
-			this.app.locals.devPlugins = devPlugins;
+			pluginDirs.set(name, path.dirname(info.packagePath));
+			devPlugins.set(name, webUiVite.devPluginUrl(name));
 		}
+		this.app.locals.devPlugins = devPlugins;
 
-
-		const compiler = webpack(webpackConfigs);
-		this.devMiddleware = webpackDevMiddleware(compiler, { serverSideRender: true });
-		this.app.use(this.devMiddleware);
+		const controllerDir = path.join(import.meta.dirname, "..", "..", "..");
+		this.devServer = await vite.createServer({
+			configFile: path.join(controllerDir, "vite.config.js"),
+			root: controllerDir,
+			base: "/static/",
+			appType: "custom",
+			server: {
+				middlewareMode: true,
+				preTransformRequests: false,
+			},
+			plugins: [webUiVite.clusterioDevPlugins(pluginDirs)],
+		});
+		// Plugins served from source import the shared modules from the development server, the web
+		// interface has to use the same copies, so it is served from source as well with --dev-plugin.
+		this.app.locals.webBundle = {
+			main: this.devServer.config.plugins.find(p => p.name === "clusterio:app")!.api.entry,
+			imports: Object.fromEntries(webUiVite.sharedModules.map(
+				name => [name, webUiVite.devSharedUrl(name)]
+			)),
+		};
+		this.app.locals.devServer = this.devServer;
 	}
 
 	/**
@@ -545,12 +594,14 @@ export default class Controller {
 			this.autosaveInterval = undefined;
 		}
 
+		this.hostPairing.stop();
+
 		if (this.clusterLogIndex) {
 			await this.clusterLogIndex.save();
 		}
 
-		if (this.devMiddleware) {
-			await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); });
+		if (this.devServer) {
+			await this.devServer.close();
 		}
 
 		await this.hooks.shutdown.invoke();
@@ -624,7 +675,9 @@ export default class Controller {
 				requests.push(hostConnection.send(new lib.SystemInfoRequest()));
 			}
 			const restartRequired = await this.checkRestartRequired();
-			requests.push(lib.gatherSystemInfo("controller", this.canRestart, restartRequired, this.startedAtMs));
+			requests.push(lib.gatherSystemInfo(
+				"controller", this.canRestart, restartRequired, this.startedAtMs, await this.listPlugins(),
+			));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
 				this.systems.set(metric);
@@ -634,6 +687,15 @@ export default class Controller {
 		}
 	}
 
+	/** Plugins on the controller, including those installed or updated since it started */
+	async listPlugins() {
+		return await lib.listPluginDetails(
+			this.pluginInfos,
+			pluginInfo => this.loadedPlugins.has(pluginInfo),
+			pluginInfo => this.config.get(`${pluginInfo.name}.load_plugin`),
+		);
+	}
+
 	async checkRestartRequired() {
 		if (this.config.restartRequired) {
 			return true;
@@ -737,13 +799,14 @@ export default class Controller {
 		await Promise.all([
 			this.config.save(),
 			this.systems.save(),
-			this.hosts.save(),
+			this.hosts.records.save(),
 			this.instances.records.save(),
 			this.saves.save(),
 			this.modPacks.save(),
 			this.roles.save(),
 			this.users.records.save(),
 			this.hooks.save.invoke(),
+			this.notes.save(),
 		]);
 	}
 
@@ -818,8 +881,43 @@ export default class Controller {
 		});
 	}
 
+	/** Remove export manifests that reference files missing from staticDirectory */
+	static async validateExports(modPacks: lib.SubscribableDatastore<lib.ModPack>, staticDirectory: string) {
+		const invalid: lib.ModPack[] = [];
+		for (const modPack of modPacks.valuesMutable()) {
+			if (!modPack.exportManifest) {
+				continue;
+			}
+			const missing: string[] = [];
+			for (const file of Object.values(modPack.exportManifest.assets)) {
+				const valid = path.basename(file) === file
+					&& await fs.stat(path.join(staticDirectory, file)).then(s => s.isFile(), () => false);
+				if (!valid) {
+					missing.push(file);
+				}
+			}
+			if (missing.length) {
+				Controller.dropExport(modPack, `missing ${missing.join(", ")}`);
+				invalid.push(modPack);
+			}
+		}
+		if (invalid.length) {
+			modPacks.setMany(invalid);
+		}
+	}
+
+	/** Clear the export manifest of a mutable mod pack, caller must save it */
+	static dropExport(modPack: lib.ModPack, reason: string) {
+		logger.warn(`Removing export data of mod pack ${modPack.name} (id ${modPack.id}): ${reason}`);
+		modPack.exportManifest = undefined;
+	}
+
 	static migrateRoles(rawJson: unknown[]): Static<typeof lib.Role.jsonSchema>[] {
 		const serialized = rawJson as Static<typeof lib.Role.jsonSchema>[];
+		// migrate: host token permissions were removed with host pairing in 2.1
+		const removedPermissions = new Set([
+			"core.host.generate_token", "core.host.revoke_token", "core.host.create_config",
+		]);
 		return serialized.map(json => {
 			json.permissions = json.permissions.map(permission => {
 				// migrate: core.instance.save.list.subscribe was renamed in alpha 17
@@ -827,7 +925,7 @@ export default class Controller {
 					return "core.instance.save.subscribe";
 				}
 				return permission;
-			});
+			}).filter(permission => !removedPermissions.has(permission));
 			return lib.Role.pluginMigrations.reduce(json);
 		});
 	}
@@ -922,6 +1020,7 @@ export default class Controller {
 			express.static(path.join(import.meta.dirname, "..", "..", "web", "static"), staticOptions)
 		);
 		app.use("/static", express.static("static", staticOptions)); // Used for data export files
+		app.use("/static", routes.missingExportAsset);
 
 		// Add API routes
 		routes.addRouteHandlers(app);
@@ -953,21 +1052,25 @@ export default class Controller {
 		return systems.length ? new lib.SystemInfoUpdateEvent(systems) : null;
 	}
 
-	/**
-	 * Generate access token for host
-	 *
-	 * @returns access token for host
-	 * @param hostId - ID of host to generate a token for.
-	 */
-	generateHostToken(hostId: number): string {
-		return jwt.sign({ aud: "host", host: hostId }, this.authSecret);
-	}
-
 	hostsUpdated(hosts: HostRecord[]) {
 		let updates = hosts.map(host => host.toHostDetails());
 		this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates));
 	}
 
+	hostPairingRequestsUpdated(requests: lib.HostPairingDetails[]) {
+		this.subscriptions.broadcast(new lib.HostPairingUpdatesEvent(requests));
+	}
+
+	async handleHostPairingSubscription(request: lib.SubscriptionRequest) {
+		if (request.action === "unsubscribe") {
+			return null;
+		}
+		const requests = [...this.hostPairing.records.values()].filter(
+			pairing => pairing.updatedAtMs > request.lastRequestTimeMs,
+		);
+		return requests.length ? new lib.HostPairingUpdatesEvent(requests) : null;
+	}
+
 	async handleHostSubscription(request: lib.SubscriptionRequest) {
 		if (request.action === "unsubscribe") {
 			return null;
@@ -1137,6 +1240,60 @@ export default class Controller {
 		return roles.length ? new lib.RoleUpdatesEvent(roles) : null;
 	}
 
+	notesUpdated(notes: lib.Note[]) {
+		this.subscriptions.broadcast(new lib.NoteUpdatesEvent(notes));
+	}
+
+	async handleNoteSubscription(request: lib.SubscriptionRequest) {
+		if (request.action === "unsubscribe") {
+			return null;
+		}
+		// Deleted notes are kept as tombstones so resubscribing clients learn of the deletion
+		const notes = [...this.notes.values()].filter(
+			note => note.updatedAtMs > request.lastRequestTimeMs && (request.lastRequestTimeMs > 0 || !note.isDeleted),
+		);
+		return notes.length ? new lib.NoteUpdatesEvent(notes) : null;
+	}
+
+	/**
+	 * Set or remove the note attached to a resource
+	 * @param resourceType - Type of resource the note belongs to.
+	 * @param resourceId - Id of the resource within its type.
+	 * @param content - New text of the note, empty removes the note.
+	 * @param updatedBy - Name of the user making the change.
+	 */
+	setNote(resourceType: string, resourceId: string, content: string, updatedBy = "") {
+		const existing = this.notes.getMutable(lib.Note.idFor(resourceType, resourceId));
+		if (content === "") {
+			this.deleteNote(resourceType, resourceId, updatedBy);
+			return;
+		}
+		if (existing) {
+			existing.content = content;
+			existing.updatedBy = updatedBy;
+			existing.isDeleted = false;
+			this.notes.set(existing);
+		} else {
+			this.notes.set(new lib.Note(resourceType, resourceId, content, updatedBy));
+		}
+	}
+
+	/**
+	 * Remove the note attached to a resource, if any
+	 * @param resourceType - Type of resource the note belongs to.
+	 * @param resourceId - Id of the resource within its type.
+	 * @param updatedBy - Name of the user making the change.
+	 */
+	deleteNote(resourceType: string, resourceId: string | number, updatedBy = "") {
+		const note = this.notes.getMutable(lib.Note.idFor(resourceType, resourceId));
+		if (note && !note.isDeleted) {
+			note.content = "";
+			note.updatedBy = updatedBy;
+			note.isDeleted = true;
+			this.notes.set(note);
+		}
+	}
+
 	async loadPlugins() {
 		const context = { metrics, logger, controller: this };
 		for (let pluginInfo of this.pluginInfos) {
@@ -1267,26 +1424,29 @@ export default class Controller {
 			let depth = routeDepth + Number(req.path.slice(-1) === "/");
 			let webRoot = "../".repeat(depth) || "./";
 			let staticRoot = res.app.locals.staticRoot ?? `${webRoot}static/`;
-			let mainBundle: string = "";
-			if (res.app.locals.mainBundle) {
-				mainBundle = res.app.locals.mainBundle;
-			} else {
-				let stats = res.locals.webpack.devMiddleware.stats.stats[0];
-				mainBundle = stats.toJson().assetsByChunkName["main"][0];
-			}
-			mainBundle = routes.stripStaticPrefix(mainBundle);
+			const { main, imports } = res.app.locals.webBundle;
+			const importMap = JSON.stringify({
+				imports: Object.fromEntries(Object.entries(imports).map(
+					([name, file]) => [name, `${staticRoot}${routes.stripStaticPrefix(file)}`]
+				)),
+			}).replace(/</g, "\\u003c");
 
 			fs.readFile(
 				path.join(import.meta.dirname, "..", "..", "..", "web", "index.html"),
 				"utf8",
-			).then((content) => {
-				res.type("text/html");
-				res.send(content
+			).then(async (content) => {
+				content = content
 					.replace(/__CLUSTER_NAME__/g, res.app.locals.controller.config.get("controller.name"))
 					.replace(/__WEB_ROOT__/g, webRoot)
 					.replace(/__STATIC_ROOT__/g, staticRoot)
-					.replace(/__MAIN_BUNDLE__/g, mainBundle)
-				);
+					.replace(/__IMPORT_MAP__/g, () => importMap)
+					.replace(/__MAIN_BUNDLE__/g, () => routes.stripStaticPrefix(main))
+				;
+				if (res.app.locals.devServer) {
+					content = await res.app.locals.devServer.transformIndexHtml(req.originalUrl, content);
+				}
+				res.type("text/html");
+				res.send(content);
 			}).catch(err => {
 				next(err);
 			});
@@ -1375,24 +1535,24 @@ export default class Controller {
 		} else if (dst.type === lib.Address.control) {
 			connection = this.wsServer.controlConnections.get(dst.id);
 			if (!connection) {
-				throw new lib.RequestError("Target control connection does not exist.");
+				throw new lib.NotFoundError("Target control connection does not exist.");
 			}
 
 		} else if (dst.type === lib.Address.instance) {
 			const instance = this.instances.getForRequest(dst.id);
 			let hostId = instance.config.get("instance.assigned_host");
 			if (hostId === null) {
-				throw new lib.RequestError("Instance is not assigned to a host");
+				throw new lib.ConflictError("Instance is not assigned to a host");
 			}
 			connection = this.wsServer.hostConnections.get(hostId);
 			if (!connection) {
-				throw new lib.RequestError("Host containing instance is not connected");
+				throw new lib.UnavailableError("Host containing instance is not connected");
 			}
 
 		} else if (dst.type === lib.Address.host) {
 			connection = this.wsServer.hostConnections.get(dst.id);
 			if (!connection) {
-				throw new lib.RequestError("Host is not connected");
+				throw new lib.UnavailableError("Host is not connected");
 			}
 
 		} else {
@@ -1476,11 +1636,11 @@ export default class Controller {
 		let instance = this.instances.getForRequest(request.instanceId);
 		let hostId = instance.config.get("instance.assigned_host");
 		if (hostId === null) {
-			throw new lib.RequestError("Instance is not assigned to a host");
+			throw new lib.ConflictError("Instance is not assigned to a host");
 		}
 		let connection = this.wsServer.hostConnections.get(hostId);
 		if (!connection) {
-			throw new lib.RequestError("Host containing instance is not connected");
+			throw new lib.UnavailableError("Host containing instance is not connected");
 		}
 		return await connection.send(request);
 	}

packages/controller/src/ControllerRouter.ts

@@ -74,7 +74,7 @@ export default class ControllerRouter {
 					return false;
 				}
 				origin.connector.sendResponseError(
-					new lib.ResponseError(msg || "Unroutable destination"), message.src
+					new lib.ResponseError(msg || "Unroutable destination", "UnavailableError"), message.src
 				);
 				return true;
 			}

packages/controller/src/HostConnection.ts

@@ -34,20 +34,17 @@ export default class HostConnection extends BaseConnection {
 
 		this.plugins = new Map(Object.entries(registerData.plugins));
 
-		const previousHostInfo = this._controller.hosts.get(registerData.id);
-		this.info = new HostRecord(
-			registerData.id,
-			previousHostInfo?.name ?? "",
-			registerData.version,
-			this.plugins,
-			true,
-			remoteAddress,
-			previousHostInfo?.publicAddress,
-			previousHostInfo?.tokenValidAfter,
-			0,
-			false,
-		);
-		this._controller.hosts.set(this.info);
+		// Hosts are only created by approving them, WsServer rejects hosts without a record.
+		const record = this._controller.hosts.getMutable(registerData.id);
+		if (!record) {
+			throw new Error(`Host ${registerData.id} is not registered`);
+		}
+		record.version = registerData.version;
+		record.plugins = this.plugins;
+		record.connected = true;
+		record.remoteAddress = remoteAddress;
+		this.info = record;
+		this._controller.hosts.records.set(this.info);
 
 		this._checkPluginVersions();
 
@@ -71,7 +68,7 @@ export default class HostConnection extends BaseConnection {
 			this._controller.instances.records.setMany(instances);
 
 			this.info.connected = false;
-			this._controller.hosts.set(this.info);
+			this.saveRecord();
 		});
 
 		this.handle(lib.GetFactorioCredentialsRequest, this.handleGetFactorioCredentialsRequest.bind(this));
@@ -175,10 +172,17 @@ export default class HostConnection extends BaseConnection {
 		return {};
 	}
 
+	/** Store changes to the host record, unless the host was revoked */
+	saveRecord() {
+		if (this._controller.hosts.getMutable(this.id) === this.info) {
+			this._controller.hosts.records.set(this.info);
+		}
+	}
+
 	async handleHostInfoUpdateEvent(event: lib.HostInfoUpdateEvent) {
 		this.info.name = event.update.name;
 		this.info.publicAddress = event.update.publicAddress;
-		this._controller.hosts.set(this.info);
+		this.saveRecord();
 	}
 
 	async handleInstanceStatusChangedEvent(request: lib.InstanceStatusChangedEvent) {

packages/controller/src/HostManager.ts

@@ -0,0 +1,149 @@
+import crypto from "node:crypto";
+import jwt from "jsonwebtoken";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+
+import type Controller from "./Controller.js";
+import HostRecord from "./HostRecord.js";
+
+/**
+ * Access layer for hosts on the controller.
+ *
+ * @alias module:controller/src/HostManager
+ */
+export default class HostManager {
+	constructor(
+		public readonly records: lib.SubscribableDatastore<HostRecord>,
+		private readonly _controller: Controller,
+	) { }
+
+	/* ------------------------------------------------------------------------- */
+	/* Basic Accessors                                                           */
+	/* ------------------------------------------------------------------------- */
+
+	has(id: HostRecord["id"]): boolean {
+		return this.records.has(id);
+	}
+
+	getMutable(id: HostRecord["id"]): HostRecord | undefined {
+		return this.records.getMutable(id);
+	}
+
+	get(id: HostRecord["id"]): Readonly<HostRecord> | undefined {
+		return this.records.get(id);
+	}
+
+	valuesMutable(): IterableIterator<HostRecord> {
+		return this.records.values();
+	}
+
+	values(): IterableIterator<Readonly<HostRecord>> {
+		return this.valuesMutable();
+	}
+
+	/**
+	 * Get host by ID for a request.
+	 *
+	 * @param hostId - ID of host to get.
+	 * @returns The host record.
+	 * @throws {module:lib.NotFoundError} if the host does not exist.
+	 */
+	getForRequest(hostId: number): HostRecord {
+		const host = this.records.getMutable(hostId);
+		if (!host) {
+			throw new lib.NotFoundError(`Unknown host id (${hostId})`);
+		}
+		return host;
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Lifecycle                                                                 */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Create a host record with a new random id
+	 *
+	 * @param name - Name of the host.
+	 * @returns the created host record.
+	 */
+	createHost(name: string): HostRecord {
+		let hostId;
+		do {
+			hostId = crypto.randomInt(1, 2 ** 31);
+		} while (this.records.has(hostId));
+
+		const host = new HostRecord(
+			hostId, name, "", new Map(), false, "", "", Math.floor(Date.now() / 1000),
+		);
+		this.records.set(host);
+		return host;
+	}
+
+	/**
+	 * Delete a host, disconnect it and unassign its instances
+	 *
+	 * @param hostId - ID of host to revoke.
+	 * @param revokedBy - Name of the user revoking it, for logging.
+	 */
+	async revokeHost(hostId: number, revokedBy: string) {
+		const host = this.getForRequest(hostId);
+		this.records.delete(host);
+		this._controller.deleteNote("host", hostId, revokedBy);
+		const system = this._controller.systems.getMutable(hostId);
+		if (system) {
+			this._controller.systems.delete(system);
+		}
+
+		// Disconnected first so unassigning the instances does not depend on the host responding.
+		const hostConnection = this._controller.wsServer.hostConnections.get(hostId);
+		if (hostConnection) {
+			hostConnection.connector.terminate();
+		}
+		const assigned = [...this._controller.instances.values()].filter(
+			instance => instance.config.get("instance.assigned_host") === hostId
+		);
+		for (const instance of assigned) {
+			await this._controller.instances.unassignInstance(instance.id);
+		}
+		logger.audit(`User ${revokedBy} revoked host ${host.name} (${host.id})`);
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Helpers                                                                   */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Sign access token for the given host
+	 *
+	 * @param hostId - ID of host to sign token for.
+	 * @returns JWT access token for the host.
+	 */
+	signHostToken(hostId: number): string {
+		return jwt.sign({ aud: "host", host: hostId }, this._controller.authSecret);
+	}
+
+	/**
+	 * Check a host token
+	 *
+	 * @param token - Host token to check.
+	 * @returns the host the token was issued to.
+	 * @throws if the token is invalid, its host is not registered or it was issued before tokenValidAfter.
+	 */
+	verifyHostToken(token: string): HostRecord {
+		// migrate: allow pre-rename tokens issued to hosts before alpha-14
+		const payload = jwt.verify(token, this._controller.authSecret, { audience: ["host", "slave"] });
+		if (typeof payload === "string") {
+			throw new Error("unexpected JsonWebToken type");
+		}
+		const hostId = payload.host !== undefined ? payload.host : payload.slave;
+		const host = this.records.getMutable(hostId);
+		if (!host) {
+			throw new Error("host is not registered, start the host again to request to join the cluster");
+		}
+		if (!payload.iat || payload.iat < host.tokenValidAfter) {
+			throw new Error("invalid token");
+		}
+		return host;
+	}
+}

packages/controller/src/HostPairingManager.ts

@@ -0,0 +1,299 @@
+import type Controller from "./Controller.js";
+
+import crypto from "crypto";
+import net from "net";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+
+/** Symbols used in pairing codes, avoids 0/O and 1/I/L */
+const codeAlphabet = "23456789ABCDEFGHJKMNPQRSTUVWXYZ";
+const codeLength = 6;
+
+/** Time a request is kept after the last poll from the host */
+export const pollTimeoutMs = 30e3;
+/** Maximum number of pending requests from one address */
+export const maxPendingPerAddress = 3;
+/** Maximum length of a host name */
+export const maxNameLength = 100;
+/** Control and bidirectional override characters refused in host names */
+const nameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u;
+
+/** Result of an approved request, sent to the host */
+export interface HostPairingResult {
+	hostId: number,
+	token: string,
+	name: string,
+}
+
+/** Error with an HTTP status code to respond to the host with */
+export class HostPairingError extends Error {
+	constructor(
+		message: string,
+		public status: number,
+		/** Seconds to wait before trying again, for status 429 */
+		public retryAfter?: number,
+	) {
+		super(message);
+	}
+}
+
+type PairingRequest = {
+	/** Secret the host polls with, never shown to users */
+	secret: string,
+	details: lib.HostPairingDetails,
+	/** Address with IPv6 addresses grouped by /64, used for limits */
+	addressKey: string,
+	lastPollMs: number,
+} & (
+	{ state: "pending" | "rejected" } | { state: "approved", result: HostPairingResult }
+);
+
+/**
+ * Strips the IPv4 mapped prefix and groups IPv6 addresses by their /64
+ *
+ * @param address - Address to make a key for
+ * @returns key used to count requests from an address.
+ */
+export function addressKey(address: string) {
+	if (address.startsWith("::ffff:") && net.isIPv4(address.slice(7))) {
+		return address.slice(7);
+	}
+	if (!net.isIPv6(address)) {
+		return address;
+	}
+	const [head, tail = ""] = address.split("::");
+	const headParts = head ? head.split(":") : [];
+	const tailParts = tail ? tail.split(":") : [];
+	const missing = 8 - headParts.length - tailParts.length;
+	const parts = address.includes("::")
+		? [...headParts, ...new Array(missing).fill("0"), ...tailParts]
+		: headParts;
+	return `${parts.slice(0, 4).map(part => part.toLowerCase().replace(/^0+(?=.)/, "")).join(":")}::/64`;
+}
+
+/**
+ * Normalise a pairing code typed by a user
+ *
+ * @param code - Code as typed, may contain dashes and lower case letters.
+ * @returns code formatted as shown by the host.
+ */
+export function normaliseCode(code: string) {
+	const plain = code.toUpperCase().replace(/[^0-9A-Z]/g, "");
+	return `${plain.slice(0, 3)}-${plain.slice(3)}`;
+}
+
+function generateCode() {
+	let code = "";
+	for (let i = 0; i < codeLength; i++) {
+		code += codeAlphabet[crypto.randomInt(codeAlphabet.length)];
+	}
+	return normaliseCode(code);
+}
+
+/**
+ * Keeps track of requests from hosts to join the cluster
+ *
+ * Requests only live in memory, and only while the host keeps polling
+ * for the result.
+ */
+export default class HostPairingManager {
+	/** Pending requests by code, broadcast to subscribed control links */
+	records = new lib.SubscribableDatastore<lib.HostPairingDetails>();
+	private _bySecret = new Map<string, PairingRequest>();
+	private _byCode = new Map<string, PairingRequest>();
+	private _pruneInterval?: ReturnType<typeof setInterval>;
+
+	constructor(
+		private _controller: Controller,
+	) { }
+
+	start() {
+		this._pruneInterval = setInterval(() => this.prune(), 5e3);
+		this._pruneInterval.unref();
+	}
+
+	stop() {
+		clearInterval(this._pruneInterval);
+		this._pruneInterval = undefined;
+	}
+
+	/**
+	 * Check and trim a host name given by a host or a user
+	 *
+	 * @param name - Name to check.
+	 * @returns the trimmed name.
+	 */
+	static validateName(name: unknown) {
+		if (typeof name !== "string" || !name.trim()) {
+			throw new HostPairingError("Host name is required", 400);
+		}
+		const trimmed = name.trim();
+		if (trimmed.length > maxNameLength) {
+			throw new HostPairingError(`Host name must be at most ${maxNameLength} characters`, 400);
+		}
+		// Names come from unauthenticated requests and are printed to terminals and logs.
+		if (nameControlCharacters.test(trimmed)) {
+			throw new HostPairingError("Host name may not contain control characters", 400);
+		}
+		return trimmed;
+	}
+
+	/**
+	 * Create a pending request
+	 *
+	 * @param name - Name the host asked for.
+	 * @param version - Version of the host.
+	 * @param remoteAddress - Address the request came from.
+	 * @returns the secret to poll with and the code to show the user.
+	 */
+	create(name: string, version: string, remoteAddress: string) {
+		this.prune();
+		const key = addressKey(remoteAddress);
+		const now = Date.now();
+		const pendingFromAddress = [...this._bySecret.values()].filter(
+			request => request.addressKey === key && request.state === "pending"
+		).length;
+		if (pendingFromAddress >= maxPendingPerAddress) {
+			throw new HostPairingError("Too many pending requests from this address", 429, 30);
+		}
+		const maxPending = this._controller.config.get("controller.host_pairing_max_pending");
+		if (maxPending && this.records.size >= maxPending) {
+			throw new HostPairingError("Too many pending requests", 429, 30);
+		}
+
+		let code;
+		do {
+			code = generateCode();
+		} while (this._byCode.has(code));
+
+		const request: PairingRequest = {
+			secret: crypto.randomBytes(32).toString("base64url"),
+			details: new lib.HostPairingDetails(code, name, remoteAddress, version, now),
+			addressKey: key,
+			lastPollMs: now,
+			state: "pending",
+		};
+		this._bySecret.set(request.secret, request);
+		this._byCode.set(code, request);
+		this.records.set(request.details);
+		logger.verbose(`Host pairing request ${code} for "${name}" from ${remoteAddress}`);
+		return { requestId: request.secret, code };
+	}
+
+	/**
+	 * Poll for the result of a request
+	 *
+	 * @param secret - Request id returned when the request was created.
+	 * @returns the result if the request was approved, or null if still pending.
+	 */
+	poll(secret: string): HostPairingResult | null {
+		this.prune();
+		const request = this._bySecret.get(secret);
+		if (!request) {
+			throw new HostPairingError("Request not found", 404);
+		}
+		request.lastPollMs = Date.now();
+		if (request.state === "approved") {
+			this._remove(request);
+			return request.result;
+		}
+		if (request.state === "rejected") {
+			this._remove(request);
+			throw new HostPairingError("Request was rejected", 403);
+		}
+		return null;
+	}
+
+	/**
+	 * Cancel a pending request
+	 *
+	 * @param secret - Request id returned when the request was created.
+	 */
+	cancel(secret: string) {
+		const request = this._bySecret.get(secret);
+		if (request && request.state === "pending") {
+			logger.verbose(`Host pairing request ${request.details.code} cancelled by the host`);
+			this._remove(request);
+		}
+	}
+
+	private _pending(code: string) {
+		this.prune();
+		const request = this._byCode.get(normaliseCode(code));
+		if (!request || request.state !== "pending") {
+			throw new lib.NotFoundError(`No pending host request with code ${code}`);
+		}
+		return request;
+	}
+
+	/**
+	 * Approve a pending request, creating the host
+	 *
+	 * @param code - Code of the request.
+	 * @param name - Name to give the host, defaults to the requested name.
+	 * @param approvedBy - Name of the user approving it, for logging.
+	 * @returns the id of the new host.
+	 */
+	approve(code: string, name: string | undefined, approvedBy: string) {
+		const request = this._pending(code);
+		let hostName;
+		try {
+			hostName = name === undefined ? request.details.name : HostPairingManager.validateName(name);
+		} catch (err: any) {
+			throw new lib.ValidationError(err.message);
+		}
+
+		const host = this._controller.hosts.createHost(hostName);
+		const result = { hostId: host.id, token: this._controller.hosts.signHostToken(host.id), name: host.name };
+		Object.assign(request, { state: "approved", result });
+		this.records.delete(request.details);
+		logger.audit(
+			`User ${approvedBy} approved host ${result.name} (${result.hostId}) with code ` +
+			`${request.details.code} from ${request.details.remoteAddress}`
+		);
+		return result.hostId;
+	}
+
+	/**
+	 * Reject a pending request
+	 *
+	 * @param code - Code of the request.
+	 * @param rejectedBy - Name of the user rejecting it, for logging.
+	 */
+	reject(code: string, rejectedBy: string) {
+		const request = this._pending(code);
+		request.state = "rejected";
+		this.records.delete(request.details);
+		logger.audit(
+			`User ${rejectedBy} rejected host ${request.details.name} with code ` +
+			`${request.details.code} from ${request.details.remoteAddress}`
+		);
+	}
+
+	private _remove(request: PairingRequest) {
+		this._bySecret.delete(request.secret);
+		this._byCode.delete(request.details.code);
+		this.records.delete(request.details);
+	}
+
+	/** Remove requests the host stopped polling for or that waited too long */
+	prune() {
+		const now = Date.now();
+		const timeoutMs = this._controller.config.get("controller.host_pairing_timeout") * 1000;
+		for (const request of this._bySecret.values()) {
+			const timedOut = request.state === "pending" && timeoutMs > 0
+				&& now - request.details.createdAtMs > timeoutMs;
+			if (!timedOut && now - request.lastPollMs <= pollTimeoutMs) {
+				continue;
+			}
+			if (request.state === "approved") {
+				logger.warn(
+					`Approved host ${request.result.name} (${request.result.hostId}) was never picked up ` +
+					"by the host, revoke it on the hosts page if it is not needed"
+				);
+			}
+			this._remove(request);
+		}
+	}
+}

packages/controller/src/InstanceManager.ts

@@ -1,9 +1,14 @@
+import { isDeepStrictEqual } from "node:util";
 import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
 
 import type Controller from "./Controller.js";
 import type HostConnection from "./HostConnection.js";
 import InstanceRecord from "./InstanceRecord.js";
 
+type InheritedFields = lib.InstanceConfigFields["instance.inherited_fields"];
+type InheritableField = keyof lib.InstanceConfigFields & string;
+
 export default class InstanceManager {
 	private static readonly DefaultFactorioSettings = {
 		tags: ["clusterio"],
@@ -24,6 +29,9 @@ export default class InstanceManager {
 		non_blocking_saving: false,
 	} as const;
 
+	/** Non-zero while template values are being copied into inheriting instances */
+	private _inheriting = 0;
+
 	constructor(
 		public readonly records: lib.SubscribableDatastore<InstanceRecord>,
 		private readonly _controller: Controller,
@@ -65,12 +73,12 @@ export default class InstanceManager {
 	 *
 	 * @param instanceId - ID of instance to get.
 	 * @returns The instance record.
-	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 * @throws {module:lib.NotFoundError} if the instance does not exist.
 	 */
 	getForRequest(instanceId: number): InstanceRecord {
 		const instance = this.records.get(instanceId);
 		if (!instance) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
+			throw new lib.NotFoundError(`Instance with ID ${instanceId} does not exist`);
 		}
 		return instance;
 	}
@@ -106,11 +114,271 @@ export default class InstanceManager {
 	 */
 	addInstanceHooks(instance: InstanceRecord): void {
 		instance.config.on("fieldChanged", (field: string, curr: unknown, prev: unknown) => {
+			if (field === "instance.template_id") {
+				this._linkMatchingFields(instance);
+			} else if (!this._inheriting && instance.config.isInherited(field)) {
+				this._unlinkEditedField(instance, field, curr, prev);
+			}
 			this.records.set(instance);
 			this._notifyConfigFieldChanged(instance, field, curr, prev);
+			this._updateInheritors(instance, field, curr);
 		});
 	}
 
+	/* ------------------------------------------------------------------------- */
+	/* Inheritance                                                               */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Get the instances that inherit directly from the given template.
+	 *
+	 * @param templateId - ID of the template.
+	 * @returns Instances with `instance.template_id` set to templateId.
+	 */
+	inheritorsOf(templateId: number): InstanceRecord[] {
+		return [...this.records.values()].filter(
+			instance => instance.config.get("instance.template_id") === templateId
+		);
+	}
+
+	/**
+	 * Check that new values for `instance.is_template` and `instance.template_id` are allowed.
+	 *
+	 * @param config - Current config of the instance.
+	 * @param isTemplate - New value for `instance.is_template`.
+	 * @param templateId - New value for `instance.template_id`.
+	 * @throws {module:lib.ValidationError} if the template is not a template or would create a cycle.
+	 * @throws {module:lib.ConflictError} if the instance is assigned or still has inheritors.
+	 */
+	checkInheritance(config: lib.InstanceConfig, isTemplate: boolean, templateId: number | null): void {
+		const instanceId = config.get("instance.id");
+		if (isTemplate && config.get("instance.assigned_host") !== null) {
+			throw new lib.ConflictError("Instance must be unassigned before it can be made a template");
+		}
+		if (!isTemplate && config.get("instance.is_template")) {
+			const count = this.inheritorsOf(instanceId).length;
+			if (count) {
+				throw new lib.ConflictError(`Instance is the template of ${count} other instance(s)`);
+			}
+		}
+		if (templateId === null) {
+			return;
+		}
+		const template = this.records.get(templateId);
+		if (!template) {
+			throw new lib.NotFoundError(`Instance with ID ${templateId} does not exist`);
+		}
+		if (!template.config.get("instance.is_template")) {
+			throw new lib.ValidationError(`Instance ${template.config.get("instance.name")} is not a template`);
+		}
+		const visited = new Set<number>();
+		for (let id: number | null = templateId; id !== null && !visited.has(id);) {
+			if (id === instanceId) {
+				throw new lib.ValidationError("Templates cannot inherit from themselves");
+			}
+			visited.add(id);
+			id = this.records.get(id)?.config.get("instance.template_id") ?? null;
+		}
+	}
+
+	/**
+	 * Copy the inheritable values from the template set in `instance.template_id`
+	 * into a config that is not yet part of the cluster and mark them inherited.
+	 *
+	 * @param config - Config of the instance to be created.
+	 * @param own - Fields set by the creator, object fields keep their properties as overrides.
+	 */
+	applyTemplate(config: lib.InstanceConfig, own: string[] = []): void {
+		const template = this._templateOf(config);
+		if (!template) {
+			return;
+		}
+		const inherited: InheritedFields = {};
+		for (const name of InstanceManager._inheritableFields()) {
+			const def = lib.InstanceConfig.fieldDefinitions[name];
+			if (def.type === "object") {
+				inherited[name] = own.includes(name) ? Object.keys(config.get(name) as object ?? {}) : [];
+			} else if (!own.includes(name)) {
+				inherited[name] = true;
+			}
+		}
+		config.set("instance.inherited_fields", inherited);
+		config.update(InstanceManager._inheritedValues(config, template.config, Object.keys(inherited)), false);
+	}
+
+	/**
+	 * Make a field, or a property of an object field, take its value from the template again.
+	 *
+	 * @param instanceId - ID of the instance to update.
+	 * @param field - Name of the field to inherit.
+	 * @param prop - Property of an object field to inherit, inherits all properties if not given.
+	 * @throws {module:lib.ValidationError} if the instance has no template or the field cannot be inherited.
+	 */
+	inheritField(instanceId: number, field: string, prop?: string): void {
+		const instance = this.getForRequest(instanceId);
+		if (!this._templateOf(instance.config)) {
+			throw new lib.ValidationError("Instance does not have a template");
+		}
+		if (!lib.InstanceConfig.isInheritable(field)) {
+			throw new lib.ValidationError(`Field ${field} cannot be inherited`);
+		}
+		const name = field as InheritableField;
+		const inherited = { ...instance.config.get("instance.inherited_fields") };
+		if (lib.InstanceConfig.fieldDefinitions[name].type !== "object") {
+			inherited[field] = true;
+		} else if (prop === undefined) {
+			inherited[field] = [];
+		} else {
+			const entry = inherited[field];
+			const overrides = entry instanceof Array ? entry : Object.keys(instance.config.get(name) as object ?? {});
+			inherited[field] = overrides.filter(p => p !== prop);
+		}
+		instance.config.set("instance.inherited_fields", inherited);
+		this._copyInheritedValues(instance, [field]);
+	}
+
+	/**
+	 * Stop inheriting the given field or properties, used when a value is explicitly set.
+	 *
+	 * @param instance - Instance the field was set on.
+	 * @param field - Name of the field set.
+	 * @param props - Properties set on an object field.
+	 */
+	overrideField(instance: InstanceRecord, field: string, props?: string[]): void {
+		const entry = instance.config.get("instance.inherited_fields")[field];
+		if (!instance.config.isInherited(field)) {
+			return;
+		}
+		const inherited = { ...instance.config.get("instance.inherited_fields") };
+		if (entry instanceof Array && props) {
+			inherited[field] = [...new Set([...entry, ...props])];
+		} else {
+			delete inherited[field];
+		}
+		instance.config.set("instance.inherited_fields", inherited);
+	}
+
+	private _templateOf(config: lib.InstanceConfig) {
+		const templateId = config.get("instance.template_id");
+		return templateId !== null ? this.records.get(templateId) : undefined;
+	}
+
+	private static _inheritableFields() {
+		return Object.keys(lib.InstanceConfig.fieldDefinitions).filter(
+			name => lib.InstanceConfig.isInheritable(name)
+		) as InheritableField[];
+	}
+
+	/** Default value of a field for the given config */
+	private static _defaultValue(name: InheritableField, config: lib.InstanceConfig): unknown {
+		const def = lib.InstanceConfig.fieldDefinitions[name];
+		if (name === "factorio.settings") {
+			return InstanceManager.DefaultFactorioSettings;
+		}
+		return typeof def.initialValue === "function" ? def.initialValue(config) : def.initialValue ?? null;
+	}
+
+	/** Values the given inherited fields take from the template, with object overrides applied */
+	private static _inheritedValues(config: lib.InstanceConfig, template: lib.InstanceConfig, fields: string[]) {
+		const inherited = config.get("instance.inherited_fields");
+		const values: Record<string, any> = {};
+		for (const field of fields) {
+			const name = field as InheritableField;
+			const entry = inherited[field];
+			if (!entry) {
+				continue;
+			}
+			const value = structuredClone(template.get(name)) as any;
+			if (entry instanceof Array) {
+				const own = config.get(name) as Record<string, unknown> ?? {};
+				for (const prop of entry) {
+					if (Object.hasOwn(own, prop)) {
+						value[prop] = structuredClone(own[prop]);
+					}
+				}
+			}
+			values[field] = value;
+		}
+		return values;
+	}
+
+	/** Update the given inherited fields of a live instance from its template */
+	private _copyInheritedValues(instance: InstanceRecord, fields: string[]): void {
+		const template = this._templateOf(instance.config);
+		if (!template) {
+			return;
+		}
+		const values = InstanceManager._inheritedValues(instance.config, template.config, fields);
+		this._inheriting += 1;
+		try {
+			instance.config.update(values, true);
+		} finally {
+			this._inheriting -= 1;
+		}
+	}
+
+	/** Unlink a field, or the changed properties of an object field, edited on the instance */
+	private _unlinkEditedField(instance: InstanceRecord, field: string, curr: unknown, prev: unknown): void {
+		const entry = instance.config.get("instance.inherited_fields")[field];
+		if (!(entry instanceof Array)) {
+			this.overrideField(instance, field);
+			return;
+		}
+		const currObj = curr as Record<string, unknown> ?? {};
+		const prevObj = prev as Record<string, unknown> ?? {};
+		const changed = Object.keys(currObj).filter(p => !isDeepStrictEqual(currObj[p], prevObj[p]));
+		const removed = Object.keys(prevObj).filter(p => !Object.hasOwn(currObj, p));
+		const overrides = [...new Set([...entry, ...changed])].filter(p => !removed.includes(p));
+		const inherited = { ...instance.config.get("instance.inherited_fields"), [field]: overrides };
+		instance.config.set("instance.inherited_fields", inherited);
+		if (removed.length) {
+			// Removed properties fall back to the template value
+			this._copyInheritedValues(instance, [field]);
+		}
+	}
+
+	/** Recompute inherited fields after template_id changed, values equal to the template or default are inherited */
+	private _linkMatchingFields(instance: InstanceRecord): void {
+		const template = this._templateOf(instance.config);
+		const inherited: InheritedFields = {};
+		if (template) {
+			for (const name of InstanceManager._inheritableFields()) {
+				const value = instance.config.get(name) as any;
+				const templateValue = template.config.get(name) as any;
+				const defaultValue = InstanceManager._defaultValue(name, instance.config) as any;
+				if (lib.InstanceConfig.fieldDefinitions[name].type === "object") {
+					inherited[name] = Object.keys(value ?? {}).filter(prop => !(
+						isDeepStrictEqual(value[prop], templateValue?.[prop])
+						|| defaultValue && isDeepStrictEqual(value[prop], defaultValue[prop])
+					) || name === "factorio.settings" && prop === "name");
+				} else if (
+					isDeepStrictEqual(value, templateValue)
+					|| defaultValue !== undefined && isDeepStrictEqual(value, defaultValue)
+				) {
+					inherited[name] = true;
+				}
+			}
+		}
+		instance.config.set("instance.inherited_fields", inherited);
+		this._copyInheritedValues(instance, Object.keys(inherited));
+	}
+
+	/** Copy a changed value from a template to the instances inheriting it */
+	private _updateInheritors(template: InstanceRecord, field: string, value: unknown): void {
+		if (!lib.InstanceConfig.isInheritable(field)) {
+			return;
+		}
+		for (const instance of this.inheritorsOf(template.id)) {
+			if (!instance.config.isInherited(field)) {
+				continue;
+			}
+			this._copyInheritedValues(instance, [field]);
+			this._controller.instanceConfigUpdated(instance).catch(err => {
+				logger.error(`Failed to send config of instance ${instance.id} to its host:\n${err.stack}`);
+			});
+		}
+	}
+
 	/* ------------------------------------------------------------------------- */
 	/* Lifecycle                                                                 */
 	/* ------------------------------------------------------------------------- */
@@ -125,7 +393,7 @@ export default class InstanceManager {
 	 * @param instanceConfig - Config to base the newly created instance on.
 	 * @param _suppressChanges - When true, side effects are suppressed, caller takes ownership.
 	 * @returns The created instance record.
-	 * @throws {module:lib.RequestError} if an instance with the same ID already exists.
+	 * @throws {module:lib.ConflictError} if an instance with the same ID already exists.
 	 */
 	async createInstance(
 		instanceConfig: lib.InstanceConfig,
@@ -133,16 +401,27 @@ export default class InstanceManager {
 	): Promise<InstanceRecord> {
 		const instanceId = instanceConfig.get("instance.id");
 		if (this.records.has(instanceId)) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} already exists`);
+			throw new lib.ConflictError(`Instance with ID ${instanceId} already exists`);
 		}
 
 		const controllerName = this._controller.config.get("controller.name");
-		instanceConfig.set("factorio.settings", {
-			name: `${controllerName} - ${instanceConfig.get("instance.name")}`,
-			description: `Clusterio instance for ${controllerName}`,
-			...InstanceManager.DefaultFactorioSettings,
-			...instanceConfig.get("factorio.settings"),
-		});
+		const serverName = `${controllerName} - ${instanceConfig.get("instance.name")}`;
+		const settings = instanceConfig.get("factorio.settings");
+		if (!instanceConfig.isInherited("factorio.settings")) {
+			instanceConfig.set("factorio.settings", {
+				name: serverName,
+				description: `Clusterio instance for ${controllerName}`,
+				...InstanceManager.DefaultFactorioSettings,
+				...settings,
+			});
+		} else if (instanceConfig.isInherited("factorio.settings", "name")) {
+			// Each instance gets its own server name
+			instanceConfig.setProp("factorio.settings", "name", serverName);
+			const inherited = instanceConfig.get("instance.inherited_fields");
+			instanceConfig.set("instance.inherited_fields", {
+				...inherited, "factorio.settings": [...inherited["factorio.settings"] as string[], "name"],
+			});
+		}
 
 		const instance = new InstanceRecord(instanceConfig, "unassigned");
 		this.addInstanceHooks(instance);
@@ -172,8 +451,8 @@ export default class InstanceManager {
 	 * @param hostId - ID of host to assign the instance to. If undefined,
 	 * the instance will be unassigned.
 	 *
-	 * @throws {module:lib.RequestError} if the instance does not exist.
-	 * @throws {module:lib.RequestError} if the target host is not connected.
+	 * @throws {module:lib.NotFoundError} if the instance does not exist.
+	 * @throws {module:lib.UnavailableError} if the target host is not connected.
 	 */
 	async assignInstance(
 		instanceId: number,
@@ -186,12 +465,15 @@ export default class InstanceManager {
 		if (currentAssignedHost === hostId) {
 			return;
 		}
+		if (hostId !== null && instance.config.get("instance.is_template")) {
+			throw new lib.ValidationError("Templates cannot be assigned to a host");
+		}
 
 		let newHostConnection: HostConnection | undefined;
 		if (hostId !== null) {
 			newHostConnection = hostConnections.get(hostId);
 			if (!newHostConnection) {
-				throw new lib.RequestError("Target host is not connected to the controller");
+				throw new lib.UnavailableError("Target host is not connected to the controller");
 			}
 		}
 
@@ -247,11 +529,15 @@ export default class InstanceManager {
 	 *
 	 * @param instanceId - ID of instance to delete.
 	 *
-	 * @throws {module:lib.RequestError} if the instance does not exist.
+	 * @throws {module:lib.NotFoundError} if the instance does not exist.
 	 */
 	async deleteInstance(instanceId: number): Promise<void> {
 		const instance = this.getForRequest(instanceId);
 		const hostId = instance.config.get("instance.assigned_host");
+		const inheritors = this.inheritorsOf(instanceId).length;
+		if (inheritors) {
+			throw new lib.ConflictError(`Instance is the template of ${inheritors} other instance(s)`);
+		}
 
 		if (hostId !== null) {
 			await this._controller.sendTo(

packages/controller/src/InstanceRecord.ts

@@ -83,6 +83,9 @@ export default class InstanceRecord {
 			this.startedAtMs,
 			this.updatedAtMs,
 			this.config.get("instance.exclude_from_start_all"),
+			this.config.get("instance.is_template"),
+			this.config.get("instance.template_id") ?? undefined,
+			this.config.get("factorio.mod_pack_id") ?? undefined,
 		);
 	}
 

packages/controller/src/UserManager.ts

@@ -40,8 +40,8 @@ export default class UserManager {
 			}
 
 			// We intentionally write the roles first because if it fails then the old user file will still contain them
-			await lib.safeOutputFile(rolesFilePath, JSON.stringify(combinedJson.roles));
-			await lib.safeOutputFile(usersFilePath, JSON.stringify(combinedJson.users));
+			await lib.atomicOverwriteFile(rolesFilePath, JSON.stringify(combinedJson.roles));
+			await lib.atomicOverwriteFile(usersFilePath, JSON.stringify(combinedJson.users));
 		} catch (err: any) {
 			if (err.code !== "ENOENT") {
 				throw err;
@@ -151,11 +151,34 @@ export default class UserManager {
 	 * @returns JWT access token for the user.
 	 */
 	signUserToken(user: UserRecord): string {
-		return jwt.sign(
-			{ aud: "user", user: user.id },
-			// auth secret is never null after startup
-			Buffer.from(this._controllerConfig.get("controller.auth_secret")!, "base64")
-		);
+		return jwt.sign({ aud: "user", user: user.id }, this._authSecret);
+	}
+
+	/**
+	 * Check a user token
+	 *
+	 * @param token - User token to check.
+	 * @returns the user the token was issued to.
+	 * @throws if the token is invalid, its user does not exist or it was issued before tokenValidAfter.
+	 */
+	verifyUserToken(token: string): User {
+		const payload = jwt.verify(token, this._authSecret, { audience: "user" });
+		if (typeof payload === "string") {
+			throw new Error("unexpected JsonWebToken type");
+		}
+		const user = this.getByNameMutable(payload.user);
+		if (!user) {
+			throw new Error("invalid user");
+		}
+		if (!payload.iat || payload.iat < user.tokenValidAfter) {
+			throw new Error("invalid token");
+		}
+		return user;
+	}
+
+	private get _authSecret() {
+		// auth secret is never null after startup
+		return Buffer.from(this._controllerConfig.get("controller.auth_secret")!, "base64");
 	}
 
 	/** Clear all player stats for specific instance */

packages/controller/src/WsServer.ts

@@ -252,21 +252,9 @@ ${err.stack}`
 		}
 
 		try {
-			// migrate: allow pre-rename tokens issued to hosts before alpha-14
-			const tokenPayload = jwt.verify(data.token, this.controller.authSecret, { audience: ["host", "slave"] });
-			if (typeof tokenPayload === "string") {
-				throw new Error("unexpected JsonWebToken type");
-			}
-
-			// migrate: allow pre-rename tokens issued to hosts before alpha-14
-			if ((tokenPayload.host !== undefined ? tokenPayload.host : tokenPayload.slave) !== data.id) {
+			if (this.controller.hosts.verifyHostToken(data.token).id !== data.id) {
 				throw new Error("missmatched host id");
 			}
-
-			let host = this.controller.hosts.get(data.id);
-			if (!tokenPayload.iat || tokenPayload.iat < (host?.tokenValidAfter ?? 0)) {
-				throw new Error("invalid token");
-			}
 		} catch (err: any) {
 			logger.verbose(`WsServer | authentication failed for ${this.remoteAddr(req)}`);
 			wsRejectedConnectionsCounter.inc();
@@ -303,6 +291,13 @@ ${err.stack}`
 			);
 		}
 
+		// The host may have been revoked while waiting for the old connection to close.
+		if (!this.controller.hosts.has(data.id)) {
+			wsRejectedConnectionsCounter.inc();
+			socket.close(lib.ConnectionClosed.Unauthorized, "Authentication failed: host was revoked");
+			return;
+		}
+
 		let [connector, sessionToken] = this.createSession(new lib.Address(lib.Address.host, data.id));
 
 		// Handle close before HostConnection does. This avoids a host
@@ -325,18 +320,7 @@ ${err.stack}`
 	async registerControl(data: lib.RegisterControlData, socket: WebSocket, req: IncomingMessage) {
 		let user;
 		try {
-			const tokenPayload = jwt.verify(data.token, this.controller.authSecret, { audience: "user" });
-			if (typeof tokenPayload === "string") {
-				throw new Error("unexpected JsonWebToken type");
-			}
-
-			user = this.controller.users.getByNameMutable(tokenPayload.user);
-			if (!user) {
-				throw new Error("invalid user");
-			}
-			if ((tokenPayload.iat || 0) < user.tokenValidAfter) {
-				throw new Error("invalid token");
-			}
+			user = this.controller.users.verifyUserToken(data.token);
 			user.checkPermission("core.control.connect");
 
 		} catch (err: any) {

packages/controller/src/routes.ts

@@ -1,12 +1,11 @@
-import type { Application, Request, Response } from "express";
+import express, { type Application, type Request, type Response } from "express";
 import Controller from "./Controller.js";
+import HostPairingManager, { HostPairingError } from "./HostPairingManager.js";
 
 import busboy from "busboy";
 import crypto from "crypto";
 import events from "events";
 import fs from "node:fs/promises";
-import JSZip from "jszip";
-import jwt from "jsonwebtoken";
 import path from "path";
 import nodeStream from "stream";
 import util from "util";
@@ -21,9 +20,10 @@ declare global {
 	namespace Express {
 		export interface Locals {
 			controller: Controller,
-			mainBundle: string,
+			webBundle: { main: string, imports: Record<string, string> },
+			devServer?: import("vite").ViteDevServer,
 			staticRoot: string | null,
-			devPlugins: Map<string, number>,
+			devPlugins?: Map<string, string>,
 			streams: Map<string, ProxyStream>,
 			pendingMetrics?: Promise<string>,
 		}
@@ -125,7 +125,7 @@ async function getMetrics(req: Request, res: Response) {
 
 
 /**
- * Removes the static/ prefix webpack adds to bundle paths so they can be
+ * Removes the static/ prefix from bundle paths in manifest.json so they can be
  * resolved relative to staticRoot.
  */
 export function stripStaticPrefix(bundlePath: string) {
@@ -141,22 +141,18 @@ function getPlugins(req: Request, res: Response) {
 		// Note: Cast through unknown is needed because load_plugin is
 		// defined at runtime and no other fields in the controller config
 		// currently have the boolean type.
-		let web: { main:any, error:string|undefined } = { main: undefined, error: undefined};
+		let web: { main: any, error: string|undefined } = { main: undefined, error: undefined};
 		let devPlugins = req.app.locals.devPlugins;
 		if (devPlugins && devPlugins.has(name)) {
-			let stats = res.locals.webpack.devMiddleware.stats.stats[devPlugins.get(name)!];
-			web.main = stats.toJson().assetsByChunkName[name][0];
+			web.main = devPlugins.get(name);
 		} else if (pluginInfo.manifest) {
-			web.main = pluginInfo.manifest[`${pluginInfo.name}.js`];
+			web.main = pluginInfo.manifest.main;
 			if (!web.main) {
-				web.error = `Missing ${pluginInfo.name}.js entry in manifest.json`;
+				web.error = "Incompatible dist/web/manifest.json, the plugin must be rebuilt with Vite";
 			}
 		} else if (lib.pluginNeedsWebBuild(pluginInfo)) {
 			web.error = "Missing dist/web/manifest.json";
 		}
-		if (web.main === "remoteEntry.js") {
-			web.error = "Incompatible old remoteEntry.js entrypoint.";
-		}
 		if (web.main) {
 			web.main = stripStaticPrefix(web.main);
 		}
@@ -192,17 +188,7 @@ function validateHostToken(req: Request, res: Response, next: any) {
 	}
 
 	try {
-		const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "host" });
-		if (typeof tokenPayload === "string") {
-			throw new Error("unexpected JsonWebToken type");
-		}
-		let host = req.app.locals.controller.hosts.get(tokenPayload.host);
-		if (!host) {
-			throw new Error("invalid host");
-		}
-		if (!tokenPayload.iat || tokenPayload.iat < (host.tokenValidAfter ?? 0)) {
-			throw new Error("invalid token");
-		}
+		req.app.locals.controller.hosts.verifyHostToken(token);
 	} catch (err) {
 		res.sendStatus(401);
 		return;
@@ -219,19 +205,7 @@ function validateUserToken(req: Request, res: Response, next: any) {
 	}
 
 	try {
-		const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "user" });
-		if (typeof tokenPayload === "string") {
-			throw new Error("unexpected JsonWebToken type");
-		}
-		let user = req.app.locals.controller.users.getByName(tokenPayload.user);
-		if (!user) {
-			throw new Error("invalid user");
-		}
-		if (!tokenPayload.iat || tokenPayload.iat < user.tokenValidAfter) {
-			throw new Error("invalid token");
-		}
-		res.locals.user = user;
-
+		res.locals.user = req.app.locals.controller.users.verifyUserToken(token);
 	} catch (err) {
 		res.sendStatus(401);
 		return;
@@ -240,6 +214,105 @@ function validateUserToken(req: Request, res: Response, next: any) {
 	next();
 }
 
+// Responds with the status of a pairing error, returns false for other errors.
+function sendHostPairingError(res: Response, err: unknown) {
+	if (!(err instanceof HostPairingError)) {
+		return false;
+	}
+	if (err.retryAfter !== undefined) {
+		res.set("Retry-After", String(err.retryAfter));
+	}
+	res.status(err.status).json({ error: err.message });
+	return true;
+}
+
+// Checks recovery mode and the request id for the pairing endpoints, returns false if responded to.
+function checkHostPairingRequest(req: Request, res: Response, needsRequestId: boolean) {
+	if (req.app.locals.controller.recoveryMode) {
+		res.status(503).json({ error: "Controller is in recovery mode" });
+		return false;
+	}
+	if (needsRequestId && typeof req.body?.requestId !== "string") {
+		res.status(400).json({ error: "requestId is required" });
+		return false;
+	}
+	return true;
+}
+
+// Request from a host for a host to join the cluster.
+function createHostPairing(req: Request, res: Response) {
+	if (!checkHostPairingRequest(req, res, false)) {
+		return;
+	}
+	const controller = req.app.locals.controller;
+	const remoteAddress = controller.wsServer.remoteAddr(req);
+	try {
+		const name = HostPairingManager.validateName(req.body?.name);
+		const version = typeof req.body?.version === "string" ? req.body.version.slice(0, 64) : "";
+
+		// A user allowed to approve hosts can skip the wait by passing their token.
+		const token = getAccessToken(req);
+		if (token !== undefined) {
+			let user;
+			try {
+				user = controller.users.verifyUserToken(token);
+			} catch (err) {
+				res.status(401).json({ error: "Invalid user token" });
+				return;
+			}
+			try {
+				user.checkPermission("core.host.approve");
+			} catch (err) {
+				logger.audit(
+					`User ${user.name} tried to approve host ${name} from ${remoteAddress} without permission`
+				);
+				res.status(403).json({ error: `User ${user.name} is not allowed to approve hosts` });
+				return;
+			}
+			const host = controller.hosts.createHost(name);
+			logger.audit(
+				`User ${user.name} approved host ${host.name} (${host.id}) from ${remoteAddress} with their token`
+			);
+			res.status(200).json({ hostId: host.id, token: controller.hosts.signHostToken(host.id), name: host.name });
+			return;
+		}
+
+		res.status(202).json(controller.hostPairing.create(name, version, remoteAddress));
+	} catch (err) {
+		if (!sendHostPairingError(res, err)) {
+			throw err;
+		}
+	}
+}
+
+// Poll from a host for the result of its request.
+function pollHostPairing(req: Request, res: Response) {
+	if (!checkHostPairingRequest(req, res, true)) {
+		return;
+	}
+	try {
+		const result = req.app.locals.controller.hostPairing.poll(req.body.requestId);
+		if (result) {
+			res.status(200).json(result);
+		} else {
+			res.status(202).json({ status: "pending" });
+		}
+	} catch (err) {
+		if (!sendHostPairingError(res, err)) {
+			throw err;
+		}
+	}
+}
+
+// Cancel from a host that stopped waiting.
+function cancelHostPairing(req: Request, res: Response) {
+	if (!checkHostPairingRequest(req, res, true)) {
+		return;
+	}
+	req.app.locals.controller.hostPairing.cancel(req.body.requestId);
+	res.sendStatus(204);
+}
+
 // Handle an uploaded export package.
 async function uploadExport(req: Request, res: Response) {
 	if (req.get("Content-Type") !== "application/zip") {
@@ -266,9 +339,7 @@ async function uploadExport(req: Request, res: Response) {
 	for await (let chunk of req) {
 		data.push(chunk);
 	}
-	let buffer: Buffer | null = Buffer.concat(data);
-	let zip = await JSZip.loadAsync(buffer);
-	buffer = null;
+	let zip = await lib.ZipArchive.fromBuffer(Buffer.concat(data));
 
 	// This is hardcoded to prevent path expansion attacks
 	let exportFiles = [
@@ -288,14 +359,15 @@ async function uploadExport(req: Request, res: Response) {
 			continue;
 		}
 
+		let content = await zip.readEntry(file);
 		if (filePath === "export/settings.json") {
-			settingPrototypes = JSON.parse(await file.async("text"));
+			settingPrototypes = JSON.parse(content.toString("utf8"));
 		}
 
 		let { name, ext } = path.posix.parse(filePath);
-		let hash = await lib.hashStream(file.nodeStream());
+		let hash = await lib.hashStream(await zip.openStream(file));
 		assets[name] = `${name}.${hash}${ext}`;
-		await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
+		await lib.atomicOverwriteFile(path.join("static", `${name}.${hash}${ext}`), content);
 	}
 
 	modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString());
@@ -305,12 +377,36 @@ async function uploadExport(req: Request, res: Response) {
 	res.sendStatus(200);
 }
 
+// Reached when no static file matched, drops exports whose files were removed
+export function missingExportAsset(req: Request, res: Response, next: any) {
+	const file = req.path.slice(1);
+	if (!["GET", "HEAD"].includes(req.method) || !file) {
+		next();
+		return;
+	}
+	const modPacks = res.app.locals.controller.modPacks;
+	const affected = [...modPacks.valuesMutable()].filter(
+		modPack => modPack.exportManifest && Object.values(modPack.exportManifest.assets).includes(file)
+	);
+	if (!affected.length) {
+		next();
+		return;
+	}
+	for (const modPack of affected) {
+		Controller.dropExport(modPack, `${file} was removed from disk`);
+	}
+	modPacks.setMany(affected);
+	res.status(404).type("text/plain").send(
+		`Export data file ${file} is missing, run export-data on an instance to regenerate it`
+	);
+}
+
 type ProxyStreamEvents = {
 	"error": [ err: any ],
 	"close": [],
 	"timeout": [],
 	"source": [],
-}
+};
 
 export interface ProxyStream {
 	id: string;
@@ -550,7 +646,7 @@ function checkModName(name: string) {
 	try {
 		lib.checkFilename(name);
 	} catch (err: any) {
-		throw new lib.RequestError(`Mod name ${err.message}`);
+		throw new lib.ValidationError(`Mod name ${err.message}`);
 	}
 }
 
@@ -691,6 +787,18 @@ export function addRouteHandlers(app: Application) {
 	);
 	app.get("/api/plugins", getPlugins);
 	app.get("/api/cluster-name", getClusterName);
+	// Set before parsing so errors from the body parser and token check are not cached either.
+	const noStore = (req: Request, res: Response, next: any) => {
+		res.set("Cache-Control", "no-store");
+		next();
+	};
+	const pairingJson = express.json({ limit: "4kb" });
+	app.post("/api/host-pairing", noStore, pairingJson, createHostPairing);
+	app.post("/api/host-pairing/poll", noStore, pairingJson, pollHostPairing);
+	app.post("/api/host-pairing/cancel", noStore, pairingJson, cancelHostPairing);
+	app.get("/api/host-pairing/check", noStore, validateHostToken, (req: Request, res: Response) => {
+		res.json({ valid: true });
+	});
 	app.put("/api/upload-export",
 		validateHostToken,
 		(req:Request, res:Response, next:any) => uploadExport(req, res).catch(next)

packages/controller/src/set_lazy_schema_compilation.ts

@@ -0,0 +1,3 @@
+// This needs to be a separate file due to ECMAScript Modules evaluating code in
+// imported modules before any code in the file that imports it.
+(global as any).lazySchemaCompilation = true;

packages/controller/typedoc.json

@@ -1,4 +0,0 @@
-{
-	"entryPoints": ["index.ts"],
-	"includeVersion": true
-}

packages/controller/vite.config.js

@@ -0,0 +1,6 @@
+import { defineConfig } from "vite";
+import { clusterioApp } from "@clusterio/web_ui/vite.mjs";
+
+export default defineConfig({
+	plugins: [clusterioApp({ entry: "web/index.tsx" })],
+});

packages/controller/web/index.html

@@ -5,6 +5,7 @@
 		<meta name="viewport" content="width=device-width, initial-scale=1" />
 		<meta name="theme-color" content="#2a2a2a" />
 		<title>__CLUSTER_NAME__</title>
+		<script type="importmap">__IMPORT_MAP__</script>
 		<style>
 			html {
 				color: #fff;
@@ -26,7 +27,6 @@
 		<div id="root">
 			<div id="bootstrap-msg">Loading web interface...</div>
 		</div>
-		<script id="script-bundle" src="__STATIC_ROOT____MAIN_BUNDLE__" defer></script>
 		<script>
 			webRoot = new URL("__WEB_ROOT__", document.location).pathname;
 			staticRoot = new URL("__STATIC_ROOT__", document.location).href;
@@ -41,9 +41,9 @@
 					bootstrapMsg.textContent = `${errorMsg}\nSee web console for details.`;
 				}
 			}
-			const bundleScript = document.getElementById("script-bundle");
-			bundleScript.addEventListener("error", () => scriptError("Main bundle failed to load."));
 			window.addEventListener("error", event => scriptError(event.error));
 		</script>
+		<script id="script-bundle" type="module" src="__STATIC_ROOT____MAIN_BUNDLE__"
+			onerror="scriptError('Main bundle failed to load.')"></script>
 	</body>
 </html>

packages/controller/web/index.tsx

@@ -1,3 +1,4 @@
+// eslint-disable-next-line no-implicit-globals
 declare function scriptError(err: any): void;
 import("@clusterio/web_ui")
 	.then(webUi => webUi.bootstrap())

packages/controller/webpack.config.cjs

@@ -1,31 +0,0 @@
-"use strict";
-const path = require("path");
-const webpack = require("webpack");
-const { merge } = require("webpack-merge");
-
-const common = require("@clusterio/web_ui/webpack.common");
-
-module.exports = (env = {}, argv = {}) => merge(common(env, argv), {
-	context: __dirname,
-	entry: "./web/index.tsx",
-	devServer: {
-		contentBase: "./dist/web",
-	},
-	output: {
-		path: path.resolve(__dirname, "dist", "web"),
-	},
-	plugins: [
-		new webpack.container.ModuleFederationPlugin({
-			name: "controller",
-			shared: {
-				"@clusterio/lib": { singleton: true },
-				"@clusterio/web_ui": { singleton: true },
-				"antd": { singleton: true },
-				"react": { singleton: true },
-				"react-dom": { singleton: true },
-				"react-router": { singleton: true },
-				"react-router-dom": { singleton: true },
-			},
-		}),
-	],
-});

packages/create/README.md

@@ -2,4 +2,115 @@
 
 Installer for Clusterio.
 
-See [the Clusterio README.md](https://github.com/clusterio/clusterio#readme) the for installation guide.
+Install the latest [Node.js LTS release](https://nodejs.org/en/download) before running the installer.
+See [the Clusterio installation guide](https://github.com/clusterio/clusterio#installation) for platform-specific instructions.
+
+Run the installer in a new empty directory:
+
+    npm init "@clusterio"
+
+It asks which components to install and how to set them up.
+Every question can be answered with an option instead, pass options to the installer after `--`:
+
+    npm init "@clusterio" -- --mode host --host-name my-host --controller-url https://example.com/
+
+The installer runs without prompts when all the questions for the mode are answered with options.
+
+
+## Modes
+
+- `standalone`: controller and host on this computer. The host is approved like any other host the first time it starts.
+- `controller`: controller only.
+- `host`: host only, joins the cluster of an existing controller, see [Setting up a host](#setting-up-a-host).
+- `ctl`: the clusterioctl command line interface only.
+- `plugins`: plugins only, for adding plugins to an existing installation.
+- `plugin-template`: files for writing a new plugin.
+
+
+## Options
+
+| Option | Modes | Description |
+| --- | --- | --- |
+| `--mode <mode>` | all | Operating mode to install. |
+| `--admin <name>` | controller | Name of the admin account to create. |
+| `--http-port <port>` | controller | Port the controller web interface listens on, defaults to 8080. |
+| `--host-name <name>` | host | Name to request for the host, defaults to the name of the computer. |
+| `--controller-url <url>` | host, ctl | URL of the controller, the same as for its web interface. |
+| `--controller-token <token>` | ctl | Authentication token saved for clusterioctl. |
+| `--public-address <address>` | host | DNS name or IP address players connect to this computer with. |
+| `--factorio-dir <path>` | host | Path to the Factorio installation. |
+| `--self-signed-cert <path>` | host | Certificate to trust for a controller using a self-signed certificate. It is added to the run scripts. |
+| `--remote-npm`, `--no-remote-npm` | controller, host | Allow or deny remote updates, plugin updates and plugin installs from the web interface. |
+| `--plugins <package...>` | all | Plugins to install, pass `--plugins` without packages to install none. |
+| `--plugin-template <template...>`, `--plugin-name <name>` | plugin-template | Templates to use and name of the new plugin. |
+| `--log-level <level>` | all | Log level to print, defaults to info. |
+| `--allow-install-as-root` | all | Linux only, allow installing as root (not recommended). |
+| `--migrate-rename` | | Migrate an installation from before the slave/master rename. |
+
+Options for the `controller` and `host` modes also work in the `standalone` mode, except for `--controller-url` as the host connects to the controller on the same computer.
+`--host-name` defaults to `local` in the `standalone` mode.
+
+
+## Setting up a host
+
+A new host has to be approved before it can join the cluster:
+
+1.  Run the installer with the host mode on the computer for the host.
+    It asks for a name for the host, the controller URL, the public address, the Factorio installation, whether to allow remote updates and which plugins to install.
+
+2.  Start the host with `run-host.cmd`, `run-host.sh` or the systemd service in the `systemd` folder.
+    The first time it starts it prints a code and waits for the host to be approved:
+
+        Waiting for approval of "my-host" with code ABC-DEF, open this link to approve it:
+        https://example.com/hosts?approve=ABC-DEF
+        Or run: npx clusterioctl host approve ABC-DEF
+
+3.  Open the link to approve the request in the web interface, where its name can be changed first.
+    It can also be approved under Add host on the Hosts page, or with `npx clusterioctl host approve ABC-DEF`.
+    Only approve a request if its code matches the one shown by the host.
+    Approving requires the `core.host.approve` permission.
+
+4.  The host saves `host.id`, `host.name` and `host.controller_token` to `config-host.json` and starts up.
+    Later starts connect to the cluster straight away.
+
+The host keeps waiting until the request is approved, also if the controller is restarted or can't be reached for a while.
+Press Ctrl+C to stop waiting.
+If the request is rejected the host exits with code 8, which the run scripts do not restart on.
+
+Start the host with `npx clusteriohost run --user-token <token>` and the token of a user that has the `core.host.approve` permission to approve the host without waiting.
+The token is only used for approving the host and is not saved.
+
+If the controller uses a self-signed certificate pass it with `--self-signed-cert <path>`, see [setting up TLS](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate).
+The installer adds it to the run scripts.
+
+When a host is revoked in the web interface or with `npx clusterioctl host revoke <host>` its token stops working and a running host shuts down.
+Starting it again requests to join the cluster again, and the host gets a new id once approved.
+
+To move a host to another controller, change `host.controller_url` with `npx clusteriohost config set host.controller_url <url>`.
+The new controller does not accept the token of the host, so the host requests to join it the next time it starts.
+
+
+## Docker
+
+To run a host in a container install it when building the image with all answers given as options.
+The host requests to join the cluster the first time the container starts.
+
+```dockerfile
+FROM node:22
+# Install the Factorio headless server to /opt/factorio here
+RUN mkdir /clusterio && chown node:node /clusterio
+USER node
+WORKDIR /clusterio
+RUN npm init "@clusterio" -- --mode host --host-name my-host --controller-url https://example.com/ \
+	--public-address factorio.example.com --factorio-dir /opt/factorio --no-remote-npm --plugins
+VOLUME /clusterio
+CMD exec ./node_modules/.bin/clusteriohost run ${USER_TOKEN:+--user-token "$USER_TOKEN"}
+```
+
+    docker run -d --name my-host -v my-host:/clusterio -p 34100-34199:34100-34199/udp my-host-image
+    docker logs my-host
+
+- The volume holds `config-host.json`, without it every new container requests to join as a new host.
+  Docker copies the installation from the image into a new empty volume, later image updates do not change the installation in an existing volume.
+- The code to approve is printed to the container log, or set `USER_TOKEN` to approve the host without waiting.
+- Stop the container after revoking its host, otherwise a restart requests to join again.

packages/create/create.js

@@ -1,10 +1,14 @@
 #!/usr/bin/env node
 
+import checkbox from "@inquirer/checkbox";
+import confirm from "@inquirer/confirm";
+import input from "@inquirer/input";
+import select from "@inquirer/select";
 import child_process from "node:child_process";
 import fs from "node:fs/promises";
-import inquirer from "inquirer";
 import os from "node:os";
 import path from "node:path";
+import readline from "node:readline";
 import stream from "node:stream";
 import util from "node:util";
 import yargs from "yargs";
@@ -16,15 +20,13 @@ const scriptExt = process.platform === "win32" ? ".cmd" : "";
 import { escapeArg } from "./escape_arg.js";
 
 // I hate this, there is a bug with rl.close on windows that was meant to have been fixed in Node14.4
-// See nodejs/node#21771 and nodejs/node#30701 for the apparent fix that was applied
-// One option is to not call rl.close: SBoudrias/Inquirer.js#767
-// But this comes with its own issues: SBoudrias/Inquirer.js#899
+// See nodejs/node#21771, nodejs/node#30701 and nodejs/node#31762
 // The solution below was adapted from aws-amplify/amplify-cli#12347
 if (process.platform === "win32") {
-	const _inquirer_close = inquirer.ui.Prompt.prototype.close;
-	inquirer.ui.Prompt.prototype.close = function() {
-		this.rl.terminal = false;
-		_inquirer_close.call(this);
+	const close = readline.Interface.prototype.close;
+	readline.Interface.prototype.close = function() {
+		this.terminal = false;
+		close.call(this);
 	};
 }
 
@@ -35,10 +37,6 @@ const availablePlugins = [
 	{ name: "Research Sync", value: "@clusterio/plugin-research_sync" },
 	{ name: "Statistics Exporter", value: "@clusterio/plugin-statistics_exporter" },
 	{ name: "Subspace Storage", value: "@clusterio/plugin-subspace_storage" },
-
-	// Comunity plugins
-	{ name: "Discord Bridge", value: "@hornwitser/discord_bridge" },
-	{ name: "Server Select", value: "@hornwitser/server_select" },
 ];
 
 const factorioLocations = {
@@ -106,7 +104,7 @@ async function pathExists(filePath) {
 	return true;
 }
 
-async function safeOutputFile(file, data, options={}) {
+async function atomicOverwriteFile(file, data, options={}) {
 	let { dir, name, ext } = path.parse(file);
 	let temporary = path.join(dir, `${name}.tmp${ext}`);
 	if (dir) {
@@ -165,21 +163,48 @@ async function execCtl(args) {
 	return await execFile(path.join("node_modules", ".bin", `clusterioctl${scriptExt}`), args);
 }
 
-function validateHostToken(token) {
-	let parts = token.split(".");
-	if (parts.length !== 3) {
-		throw new InstallError("Invalid token");
-	}
+// Same characters as the controller refuses in host names.
+const hostNameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u;
 
-	let parsed;
+function validateHostName(value) {
+	if (!value || !value.trim()) {
+		return "Host name may not be empty";
+	}
+	if (value.trim().length > 100) {
+		return "Host name may be at most 100 characters";
+	}
+	if (hostNameControlCharacters.test(value)) {
+		return "Host name may not contain control characters";
+	}
+	return true;
+}
+
+function validateControllerUrl(value) {
+	let url;
 	try {
-		parsed = JSON.parse(Buffer.from(parts[1], "base64"));
+		url = new URL(value);
 	} catch (err) {
-		throw new InstallError("Invalid token");
+		return "Controller URL must be a URL like https://example.com/";
 	}
+	if (!["http:", "https:"].includes(url.protocol)) {
+		return "Controller URL must start with http:// or https://";
+	}
+	return true;
+}
 
-	if (parsed.aud !== "host" || !Number.isInteger(parsed.host)) {
-		throw new InstallError("Invalid token");
+// The host requires the controller URL to end with a slash.
+function normaliseControllerUrl(value) {
+	const url = new URL(value);
+	if (!url.pathname.endsWith("/")) {
+		url.pathname += "/";
+	}
+	return url.href;
+}
+
+// Remote updates, plugin updates and plugin installs are allowed or denied together.
+async function setRemoteUpdates(exec, component, allow) {
+	for (let field of ["allow_remote_updates", "allow_plugin_updates", "allow_plugin_install"]) {
+		await exec(["config", "set", `${component}.${field}`, allow]);
 	}
 }
 
@@ -232,7 +257,7 @@ async function migrateRename(args) {
 
 	async function migrateConfig(source, destination) {
 		if (await pathExists(source) && !await pathExists(destination)) {
-			await safeOutputFile(destination, JSON.stringify(
+			await atomicOverwriteFile(destination, JSON.stringify(
 				renameConfig(JSON.parse(await fs.readFile(source))), null, "\t"
 			), { mode: 0o600 });
 			logger.info(`Migrated ${source} to ${destination}`);
@@ -244,7 +269,7 @@ async function migrateRename(args) {
 
 	let instancesFile = path.join("database", "instances.json");
 	if (await pathExists(instancesFile)) {
-		await safeOutputFile(instancesFile, JSON.stringify(
+		await atomicOverwriteFile(instancesFile, JSON.stringify(
 			JSON.parse(await fs.readFile(instancesFile)).map(renameConfig), null, "\t"
 		));
 		logger.info(`Migrated ${instancesFile}`);
@@ -256,7 +281,7 @@ async function migrateRename(args) {
 		for (let role of users["roles"]) {
 			role["permissions"] = role["permissions"].map(rename);
 		}
-		await safeOutputFile(usersFile, JSON.stringify(users, null, "\t"));
+		await atomicOverwriteFile(usersFile, JSON.stringify(users, null, "\t"));
 		logger.info(`Migrated ${usersFile}`);
 	}
 
@@ -284,7 +309,7 @@ async function migrateRename(args) {
 				lines.push(inputLine);
 			}
 		}
-		safeOutputFile(outputFile, Buffer.concat(lines));
+		atomicOverwriteFile(outputFile, Buffer.concat(lines));
 	}
 
 	async function migrateLogsDir(inputLogs, outputLogs) {
@@ -392,31 +417,7 @@ async function installClusterio(mode, plugins) {
 		throw new InstallError(`Failed to install: ${err.message}`);
 	}
 
-	if (plugins.length) {
-		logger.info("Setting up plugins");
-		let pluginList;
-		try {
-			pluginList = new Map(JSON.parse(await fs.readFile("plugin-list.json")));
-		} catch (err) {
-			if (err.code === "ENOENT") {
-				pluginList = new Map();
-			} else {
-				throw new InstallError(`Error loading plugin-list.json: ${err.message}`);
-			}
-		}
-		for (let plugin of plugins) {
-			if (!pluginList.has(plugin)) {
-
-				let pluginInfo = (await import(plugin)).plugin;
-				pluginList.set(pluginInfo.name, plugin);
-			}
-		}
-		try {
-			await safeOutputFile("plugin-list.json", JSON.stringify([...pluginList], null, "\t"));
-		} catch (err) {
-			throw new InstallError(`Error writing plugin-list.json: ${err.message}`);
-		}
-	}
+	// Controller, host and ctl discover installed plugins when loading their configuration.
 }
 
 async function groupIdToName(gid) {
@@ -433,7 +434,7 @@ async function groupIdToName(gid) {
 async function writeScripts(mode, selfSignedCert) {
 	if (["standalone", "controller"].includes(mode)) {
 		if (process.platform === "win32") {
-			await safeOutputFile(
+			await atomicOverwriteFile(
 				"run-controller.cmd",
 				`\
 @echo off
@@ -446,7 +447,7 @@ goto restart
 `
 			);
 		} else {
-			await safeOutputFile(
+			await atomicOverwriteFile(
 				"run-controller.sh",
 				`\
 #!/bin/bash
@@ -458,7 +459,7 @@ done
 `,
 				{ mode: 0o755 },
 			);
-			await safeOutputFile(
+			await atomicOverwriteFile(
 				"systemd/clusteriocontroller.service",
 				`\
 [Unit]
@@ -485,7 +486,7 @@ WantedBy=multi-user.target
 	if (["standalone", "host"].includes(mode)) {
 		if (process.platform === "win32") {
 			let extraCaCertsLine = selfSignedCert ? `set "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
-			await safeOutputFile(
+			await atomicOverwriteFile(
 				"run-host.cmd",
 				`\
 @echo off
@@ -499,7 +500,7 @@ goto restart
 			);
 		} else {
 			let extraCaCertsLine = selfSignedCert ? `export "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : "";
-			await safeOutputFile(
+			await atomicOverwriteFile(
 				"run-host.sh",
 				`\
 #!/bin/bash
@@ -512,7 +513,7 @@ done
 				{ mode: 0o755 },
 			);
 			extraCaCertsLine = selfSignedCert ? `Environment=NODE_EXTRA_CA_CERTS=${selfSignedCert}\n` : "";
-			await safeOutputFile(
+			await atomicOverwriteFile(
 				"systemd/clusteriohost.service",
 				`\
 [Unit]
@@ -538,96 +539,87 @@ WantedBy=multi-user.target
 }
 
 // eslint-disable-next-line complexity
-async function inquirerMissingArgs(args) {
+async function promptMissingArgs(args) {
 	let answers = {};
 	if (args.mode) { answers.mode = args.mode; }
 	if (args["plugin-template"]) { answers.mode = "plugin-template"; }
-	answers = await inquirer.prompt([
-		{
-			type: "list",
-			name: "mode",
-			message: "Operating mode to install",
-			default: "standalone",
-			choices: [
-				{ name: "Standalone (install both controller and host on this computer)", value: "standalone" },
-				{ name: "Controller only", value: "controller" },
-				{ name: "Host only", value: "host" },
-				{ name: "Ctl only", value: "ctl" },
-				{ name: "Plugins only", value: "plugins" },
-				{ name: "Plugin template", value: "plugin-template" },
-			],
-		},
-	], answers);
+	answers.mode ??= await select({
+		message: "Operating mode to install",
+		default: "standalone",
+		choices: [
+			{ name: "Standalone (install both controller and host on this computer)", value: "standalone" },
+			{ name: "Controller only", value: "controller" },
+			{ name: "Host only", value: "host" },
+			{ name: "Ctl only", value: "ctl" },
+			{ name: "Plugins only", value: "plugins" },
+			{ name: "Plugin template", value: "plugin-template" },
+		],
+	});
 
 	if (["standalone", "controller"].includes(answers.mode)) {
 		if (args.admin) { answers.admin = args.admin; }
-		answers = await inquirer.prompt([
-			{
-				type: "input",
-				name: "admin",
-				message: "Admin account name",
-				validate: input => {
-					if (!input) {
-						return "May not be empty";
-					}
-					return true;
-				},
+		answers.admin ??= await input({
+			message: "Admin account name",
+			validate: value => {
+				if (!value) {
+					return "May not be empty";
+				}
+				return true;
 			},
-		], answers);
+		});
 
 		if (args.httpPort) { answers.httpPort = args.httpPort; }
-		answers = await inquirer.prompt([
-			{
-				type: "input",
-				name: "httpPort",
-				message: "HTTP port to listen on",
-				default: 8080,
-				validate: input => {
-					if (!input) {
-						return "May not be empty";
-					}
-					if (!Number.isInteger(Number(input))) {
-						return "Must be a number";
-					}
-					return true;
-				},
+		answers.httpPort ??= await input({
+			message: "HTTP port to listen on",
+			default: "8080",
+			validate: value => {
+				if (!value) {
+					return "May not be empty";
+				}
+				if (!Number.isInteger(Number(value))) {
+					return "Must be a number";
+				}
+				return true;
 			},
-		], answers);
+		});
+	}
+
+	if (answers.mode === "standalone") {
+		answers.hostName = args.hostName ?? "local";
+		const result = validateHostName(answers.hostName);
+		if (result !== true) {
+			throw new InstallError(result);
+		}
 	}
 
 	if (answers.mode === "host") {
 		if (args.hostName) { answers.hostName = args.hostName; }
-		answers = await inquirer.prompt([
-			{
-				type: "input",
-				name: "hostName",
-				message: "Name of host",
-			},
-		], answers);
+		if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; }
+		answers.hostName ??= await input({
+			message: "Name of host",
+			default: os.hostname(),
+			validate: validateHostName,
+		});
+		answers.controllerUrl ??= await input({
+			message: "Controller URL",
+			validate: validateControllerUrl,
+		});
+
+		// The prompts do not validate answers given as arguments
+		for (let result of [validateHostName(answers.hostName), validateControllerUrl(answers.controllerUrl)]) {
+			if (result !== true) {
+				throw new InstallError(result);
+			}
+		}
+		answers.controllerUrl = normaliseControllerUrl(answers.controllerUrl);
 	}
 
-	if (["host", "ctl"].includes(answers.mode)) {
+	if (answers.mode === "ctl") {
 		if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; }
-		answers = await inquirer.prompt([
-			{
-				type: "input",
-				name: "controllerUrl",
-				message: "Controller URL",
-			},
-		], answers);
+		answers.controllerUrl ??= await input({ message: "Controller URL" });
 
 		if (args.controllerToken) { answers.controllerToken = args.controllerToken; }
-		answers = await inquirer.prompt([
-			{
-				type: "input",
-				name: "controllerToken",
-				message: "Controller authentication Token",
-			},
-		], answers);
-	}
-
-	if (answers.mode === "host") {
-		validateHostToken(answers.controllerToken);
+		answers.controllerToken ??= await input({ message: "Controller authentication Token" });
 	}
 
 	if (["standalone", "host"].includes(answers.mode)) {
@@ -640,14 +632,10 @@ async function inquirerMissingArgs(args) {
 				myIp = await result.text();
 			} catch (err) { /* ignore */ }
 		}
-		answers = await inquirer.prompt([
-			{
-				type: "input",
-				name: "publicAddress",
-				message: "Public DNS/IP address of this server",
-				default: myIp,
-			},
-		], answers);
+		answers.publicAddress ??= await input({
+			message: "Public DNS/IP address of this server",
+			default: myIp,
+		});
 
 		if (args.factorioDir) { answers.factorioDir = args.factorioDir; }
 		let locations = factorioLocations[process.platform] || [];
@@ -658,28 +646,17 @@ async function inquirerMissingArgs(args) {
 			}
 		}
 
-		answers = await inquirer.prompt([
-			{
-				type: "list",
-				name: "factorioDir",
-				message: "Path to Factorio installation",
-				choices: [
-					...foundLocations.map(location => ({ name: `${location} (auto detected)`, value: location })),
-					{ name: "Use local factorio directory, you must copy an installation to it", value: "factorio" },
-					{ name: "Provide path manually", value: null },
-				],
-			},
-		], answers);
+		answers.factorioDir ??= await select({
+			message: "Path to Factorio installation",
+			choices: [
+				...foundLocations.map(location => ({ name: `${location} (auto detected)`, value: location })),
+				{ name: "Use local factorio directory, you must copy an installation to it", value: "factorio" },
+				{ name: "Provide path manually", value: null },
+			],
+		});
 
 		if (answers.factorioDir === null) {
-			answers = await inquirer.prompt([
-				{
-					type: "input",
-					name: "factorioDir",
-					message: "Path to Factorio installation",
-					askAnswered: true,
-				},
-			], answers);
+			answers.factorioDir = await input({ message: "Path to Factorio installation" });
 		} else if (answers.factorioDir === "factorio") {
 			await fs.mkdir("factorio", { recursive: true });
 		}
@@ -687,14 +664,10 @@ async function inquirerMissingArgs(args) {
 
 	if (["standalone", "controller", "host"].includes(answers.mode)) {
 		if (args.hasOwnProperty("remoteNpm")) { answers.remoteNpm = args.remoteNpm; }
-		answers = await inquirer.prompt([
-			{
-				type: "confirm",
-				name: "remoteNpm",
-				message: "Allow remote updates via npm?",
-				default: true,
-			},
-		], answers);
+		answers.remoteNpm ??= await confirm({
+			message: "Allow remote updates and plugin installs from the web interface?",
+			default: true,
+		});
 	}
 
 	if (answers.mode === "plugin-template") {
@@ -705,29 +678,23 @@ async function inquirerMissingArgs(args) {
 			answers.pluginName = args["plugin-name"];
 		}
 		answers.plugins = [];
-		answers = await inquirer.prompt([
-			{
-				type: "checkbox",
-				name: "pluginTemplate",
-				message: "Plugin templates to use",
-				choices: [
-					{ name: "Controller", value: "controller" },
-					{ name: "Host", value: "host" },
-					{ name: "Instance", value: "instance" },
-					{ name: "Lua Module", value: "module" },
-					{ name: "Command Line", value: "ctl" },
-					{ name: "Web UI", value: "web" },
-					{ name: "No Typescript", value: "js" },
-					{ name: "No Config", value: "no_config" },
-				],
-			},
-			{
-				type: "input",
-				name: "pluginName",
-				message: "Name of your new plugin",
-				default: path.basename(path.resolve()),
-			},
-		], answers);
+		answers.pluginTemplate ??= await checkbox({
+			message: "Plugin templates to use",
+			choices: [
+				{ name: "Controller", value: "controller" },
+				{ name: "Host", value: "host" },
+				{ name: "Instance", value: "instance" },
+				{ name: "Lua Module", value: "module" },
+				{ name: "Command Line", value: "ctl" },
+				{ name: "Web UI", value: "web" },
+				{ name: "No Typescript", value: "js" },
+				{ name: "No Config", value: "no_config" },
+			],
+		});
+		answers.pluginName ??= await input({
+			message: "Name of your new plugin",
+			default: path.basename(path.resolve()),
+		});
 	}
 
 	if (dev) {
@@ -735,15 +702,11 @@ async function inquirerMissingArgs(args) {
 	} else if (args.plugins) {
 		answers.plugins = args.plugins;
 	}
-	answers = await inquirer.prompt([
-		{
-			type: "checkbox",
-			name: "plugins",
-			message: "Plugins to install",
-			choices: availablePlugins,
-			pageSize: 20,
-		},
-	], answers);
+	answers.plugins ??= await checkbox({
+		message: "Plugins to install",
+		choices: availablePlugins,
+		pageSize: 20,
+	});
 
 	return answers;
 }
@@ -771,13 +734,13 @@ async function main() {
 			nargs: 1, describe: "HTTP port to listen on [standalone/controller]", type: "number",
 		})
 		.option("host-name", {
-			nargs: 1, describe: "Host name [host]", type: "string",
+			nargs: 1, describe: "Host name [standalone/host]", type: "string",
 		})
 		.option("controller-url", {
 			nargs: 1, describe: "Controller URL [host/ctl]", type: "string",
 		})
 		.option("controller-token", {
-			nargs: 1, describe: "Controller authentication token [host/ctl]", type: "string",
+			nargs: 1, describe: "Controller authentication token [ctl]", type: "string",
 		})
 		.option("public-address", {
 			nargs: 1, describe: "DNS/IP Address to connect to this server [standalone/host]", type: "string",
@@ -789,7 +752,8 @@ async function main() {
 			nargs: 1, describe: "Path to self-signed certificate of the controller [standalone/host]", type: "string",
 		})
 		.option("remote-npm", {
-			nargs: 0, description: "Allow remote updates via npm [standalone/controller/host]", type: "boolean",
+			nargs: 0, type: "boolean", description:
+				"Allow remote updates and plugin installs from the web interface [standalone/controller/host]",
 		})
 		.option("plugins", {
 			array: true, describe: "Plugins to install", type: "string",
@@ -826,7 +790,7 @@ async function main() {
 		return;
 	}
 
-	let answers = await inquirerMissingArgs(args);
+	let answers = await promptMissingArgs(args);
 	logger.verbose(JSON.stringify(answers));
 
 	if (answers.pluginTemplate) {
@@ -844,44 +808,28 @@ async function main() {
 		await execController(["config", "create"]);
 		await execController(["bootstrap", "create-admin", answers.admin]);
 		await execController(["config", "set", "controller.http_port", answers.httpPort]);
-		await execController(["config", "set", "controller.allow_remote_updates", answers.remoteNpm]);
-		await execController(["config", "set", "controller.allow_plugin_updates", answers.remoteNpm]);
+		await setRemoteUpdates(execController, "controller", answers.remoteNpm);
 		let result = await execController(["bootstrap", "generate-user-token", answers.admin]);
 		adminToken = result.stdout.split("\n").slice(-2)[0];
 	}
 
-	if (answers.mode === "standalone") {
+	if (["standalone", "host"].includes(answers.mode)) {
 		logger.info("Setting up host");
 		await execHost(["config", "create"]);
-		await execHost(["config", "set", "host.name", "local"]);
-
-		let result = await execHost(["config", "show", "host.id"]);
-		let hostId = Number.parseInt(result.stdout.split("\n").slice(-2)[0], 10);
-
-		result = await execController(["bootstrap", "generate-host-token", hostId]);
-		let hostToken = result.stdout.split("\n").slice(-2)[0];
-
-		// Default to localhost on correct port for host in standalone mode
-		await execHost(["config", "set", "host.controller_url", `http://localhost:${answers.httpPort}/`]);
-		await execHost(["config", "set", "host.controller_token", hostToken]);
-		await execHost(["config", "set", "host.public_address", answers.publicAddress]);
-		await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]);
-		await execHost(["config", "set", "host.allow_remote_updates", answers.remoteNpm]);
-		await execHost(["config", "set", "host.allow_plugin_updates", answers.remoteNpm]);
 	}
 
-	if (answers.mode === "host") {
-		logger.info("Setting up host");
-		let hostId = JSON.parse(Buffer.from(answers.controllerToken.split(".")[1], "base64")).host;
-		await execHost(["config", "create"]);
-		await execHost(["config", "set", "host.id", hostId]);
+	if (answers.mode === "standalone") {
+		// Default to localhost on correct port for host in standalone mode
+		answers.controllerUrl = `http://localhost:${answers.httpPort}/`;
+	}
+
+	if (["standalone", "host"].includes(answers.mode)) {
+		// The host requests to join the cluster when it is started.
 		await execHost(["config", "set", "host.name", answers.hostName]);
 		await execHost(["config", "set", "host.controller_url", answers.controllerUrl]);
-		await execHost(["config", "set", "host.controller_token", answers.controllerToken]);
 		await execHost(["config", "set", "host.public_address", answers.publicAddress]);
 		await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]);
-		await execHost(["config", "set", "host.allow_remote_updates", answers.remoteNpm]);
-		await execHost(["config", "set", "host.allow_plugin_updates", answers.remoteNpm]);
+		await setRemoteUpdates(execHost, "host", answers.remoteNpm);
 	}
 
 	if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) {
@@ -905,7 +853,11 @@ async function main() {
 
 if (import.meta.main) {
 	main().catch(err => {
-		if (err instanceof InstallError) {
+		process.exitCode = 1;
+		if (err.name === "ExitPromptError") {
+			// Ctrl+C at a prompt
+			process.exitCode = 130;
+		} else if (err instanceof InstallError) {
 			logger.error(err.message);
 		} else {
 			logger.fatal(`

packages/create/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/create",
-	"version": "2.0.0",
+	"version": "2.0.3",
 	"description": "Installer for clusterio",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -28,12 +28,15 @@
 		"clusteriocreate": "./create.js"
 	},
 	"engines": {
-		"node": ">=22"
+		"node": "^22.18.0 || >=24.2.0"
 	},
 
 	"dependencies": {
+		"@inquirer/checkbox": "^5.2.8",
+		"@inquirer/confirm": "^6.3.5",
+		"@inquirer/input": "^5.1.9",
+		"@inquirer/select": "^5.2.8",
 		"chalk": "catalog:",
-		"inquirer": "^8.2.4",
 		"yargs": "catalog:"
 	}
 }

packages/create/template.js

@@ -88,7 +88,7 @@ export async function copyPluginTemplates(pluginName, templates) {
 	const config = !templates.includes("no_config") && templates.some(value => (
 		["controller", "host", "instance", "ctl"].includes(value)
 	));
-	const webpack = config || templates.includes("controller") || templates.includes("web");
+	const webBuild = config || templates.includes("controller") || templates.includes("web");
 
 	// A count of the number of isolated contexts the plugin runs under
 	let pluginContexts = 0;
@@ -113,15 +113,15 @@ export async function copyPluginTemplates(pluginName, templates) {
 		files.set("tsconfig.json", path.join(templatePath, "tsconfig.json"));
 		files.set("tsconfig.node.json", path.join(templatePath, "tsconfig.node.json"));
 		files.set("tsconfig.base.json", path.join(templatePath, "tsconfig.base.json"));
-		if (webpack) {
+		if (webBuild) {
 			files.set("tsconfig.browser.json", path.join(templatePath, "tsconfig.browser.json"));
 		}
 	}
 
-	// Files and dependences to support webpack
-	if (webpack) {
-		prepare.push("webpack-cli --mode production");
-		files.set("webpack.config.cjs", path.join(commonPath, "webpack.config.cjs"));
+	// Files and dependences to support the web build
+	if (webBuild) {
+		prepare.push("vite build");
+		files.set("vite.config.js", path.join(commonPath, "vite.config.js"));
 		if (templates.includes("web")) {
 			files.set(`web/index.${ext}x`, path.join(templatePath, `web/plugin.${ext}x`));
 			pluginContexts += 1;
@@ -185,7 +185,7 @@ export async function copyPluginTemplates(pluginName, templates) {
 		ctl: templates.includes("ctl"),
 		web: templates.includes("web"),
 		// Macro flags for context requirements
-		webpack: webpack,
+		web_build: webBuild,
 		config: config,
 		// String values for package json
 		clusterio_version: _package.version,

packages/create/templates/common/package.json

@@ -55,7 +55,7 @@
 	"devDependencies": {
 //%// Typescript and type declarations
 //%if typescript
-		"typescript": "^5.9.3",
+		"typescript": "^7.0.2",
 		"@types/node": "^24.13.4",
 //%endif
 //%if typescript & web
@@ -67,15 +67,13 @@
 		"react": "^18.2.0",
 		"react-dom": "^18.2.0",
 //%endif
-//%if webpack
-		"webpack": "^5.110.3",
-		"webpack-cli": "^7.2.3",
-		"webpack-merge": "^5.9.0",
+//%if web_build
+		"vite": "^8.3.4",
 //%endif
 //%if controller
 		"@clusterio/controller": "^__clusterio_version__",
 //%endif
-//%if webpack
+//%if web_build
 		"@clusterio/web_ui": "^__clusterio_version__",
 //%endif
 //%if host
@@ -87,7 +85,7 @@
 		"@clusterio/lib": "^__clusterio_version__"
 	},
 	"dependencies": {
-		"@sinclair/typebox": "^0.30.6"
+		"@sinclair/typebox": "^0.34.52"
 	},
 	"publishConfig": {
 		"access": "public"

packages/create/templates/common/template.npmignore

@@ -1,3 +1,3 @@
-webpack.config.js
+vite.config.js
 dist/browser
 dist/web

packages/create/templates/common/vite.config.js

@@ -0,0 +1,6 @@
+import { defineConfig } from "vite";
+import { clusterioPlugin } from "@clusterio/web_ui/vite.mjs";
+
+export default defineConfig({
+	plugins: [clusterioPlugin()],
+});

packages/create/templates/common/webpack.config.cjs

@@ -1,34 +0,0 @@
-"use strict";
-const path = require("path");
-const webpack = require("webpack");
-const { merge } = require("webpack-merge");
-
-const common = require("@clusterio/web_ui/webpack.common");
-
-module.exports = (env = {}, argv = {}) => merge(common(env, argv), {
-	context: __dirname,
-	entry: "./web/index.__ext__x",
-	output: {
-		path: path.resolve(__dirname, "dist", "web"),
-	},
-	plugins: [
-		new webpack.container.ModuleFederationPlugin({
-			name: "__plugin_name__",
-			library: { type: "window", name: "plugin___plugin_name__" },
-			exposes: {
-				"./": "./index.__ext__",
-				"./package.json": "./package.json",
-				"./web": "./web/index.__ext__x",
-			},
-			shared: {
-				"@clusterio/lib": { import: false },
-				"@clusterio/web_ui": { import: false },
-//%if web
-				"antd": { import: false },
-				"react": { import: false },
-				"react-dom": { import: false },
-//%endif
-			},
-		}),
-	],
-});

packages/create/templates/plugin-ts/tsconfig.node.json

@@ -2,6 +2,7 @@
 	"extends": "./tsconfig.base.json",
 	"compilerOptions": {
 		"lib": ["es2023"],
+		"types": ["node"],
 		"module": "node16",
 		"moduleResolution": "node16",
 		"target": "es2022",

packages/ctl/ctl.ts

@@ -307,12 +307,12 @@ async function startCtl() {
 			logger.error(`Error running command: ${err.message}`);
 			process.exitCode = 1;
 
-		} else if (err instanceof lib.RequestError) {
-			if (err.stack) {
-				logger.error(`Error sending request:\n${err.stack}`);
-			} else {
-				logger.error(`Error sending request: ${err.message}`);
-			}
+		} else if (err instanceof lib.ClientError || err instanceof lib.UnavailableError) {
+			logger.error(`Error sending request: ${err.message}`);
+			process.exitCode = 1;
+
+		} else if (err instanceof lib.ServerError) {
+			logger.error(`Error sending request:\n${err.stack ?? err.message}`);
 			process.exitCode = 1;
 
 		} else {
@@ -327,14 +327,6 @@ async function startCtl() {
 }
 
 export function bootstrap() {
-	// eslint-disable-next-line no-console
-	console.warn(`
-+==========================================================+
-I WARNING:  This is the development branch for the 2.0     I
-I           version of clusterio.  Expect things to break. I
-+==========================================================+
-`
-	);
 	startCtl().catch(err => {
 		if (err.errors) {
 			logger.fatal(JSON.stringify(err.errors, null, "\t"));

packages/ctl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/ctl",
-	"version": "2.0.0",
+	"version": "2.0.3",
 	"description": "Clusterio command line management tool",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -32,7 +32,7 @@
 		"clusterioctl": "./dist/node/ctl.js"
 	},
 	"engines": {
-		"node": ">=22"
+		"node": "^22.18.0 || >=24.2.0"
 	},
 
 	"dependencies": {

packages/ctl/src/commands.ts

@@ -8,6 +8,7 @@ import { modPackCommands } from "./commands_mod_pack.js";
 import { modCommands } from "./commands_mod.js";
 import { permissionCommands } from "./commands_permission.js";
 import { roleCommands } from "./commands_role.js";
+import { noteCommands } from "./commands_note.js";
 import { userCommands } from "./commands_user.js";
 import { logCommands } from "./commands_log.js";
 import { debugCommands } from "./commands_debug.js";
@@ -22,6 +23,7 @@ export async function registerCommands(ctlHooks: CtlHooks, yargs: Argv) {
 	rootCommands.add(modCommands);
 	rootCommands.add(permissionCommands);
 	rootCommands.add(roleCommands);
+	rootCommands.add(noteCommands);
 	rootCommands.add(userCommands);
 	rootCommands.add(logCommands);
 	rootCommands.add(debugCommands);

packages/ctl/src/commands_controller.ts

@@ -180,7 +180,7 @@ controllerPluginCommands.add(new lib.Command({
 }));
 controllerPluginCommands.add(new lib.Command({
 	definition: ["update <plugin>", "Update a plugin on the controller", (yargs) => {
-		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to update, optionally as name@version", type: "string" });
 		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
 	}],
 	handler: async function(args: { plugin: string, restart: boolean }, ctl: Ctl) {
@@ -192,9 +192,28 @@ controllerPluginCommands.add(new lib.Command({
 		}
 	},
 }));
+controllerPluginCommands.add(new lib.Command({
+	definition: ["search [query]", "Search npm for plugins", (yargs) => {
+		yargs.positional("query", { describe: "Text to search for", type: "string", default: "" });
+		yargs.option("page", { type: "number", description: "Page of results to show", default: 1 });
+		yargs.option("page-size", { type: "number", description: "Results per page", default: 20 });
+	}],
+	handler: async function(args: { query: string, page: number, pageSize: number }, ctl: Ctl) {
+		const response = await ctl.sendTo(
+			"controller", new lib.PluginSearchRequest(args.query, args.page, args.pageSize)
+		);
+		print(asTable(response.results.map(plugin => ({
+			name: plugin.name,
+			version: plugin.version,
+			publisher: plugin.publisher ?? "",
+			description: plugin.description ?? "",
+		}))));
+		print(`Showing ${response.results.length} of ${response.total} results`);
+	},
+}));
 controllerPluginCommands.add(new lib.Command({
 	definition: ["install <plugin>", "Install a plugin on the controller", (yargs) => {
-		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to install, optionally as name@version", type: "string" });
 		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
 	}],
 	handler: async function(args: { plugin: string, restart: boolean }, ctl: Ctl) {

packages/ctl/src/commands_host.ts

@@ -5,7 +5,6 @@ import os from "os";
 import child_process from "child_process";
 
 import * as lib from "@clusterio/lib";
-import { logger } from "@clusterio/lib";
 import type { Ctl } from "../ctl.js";
 import { print } from "./command_ops.js";
 import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops.js";
@@ -51,62 +50,78 @@ hostCommands.add(new lib.Command({
 	},
 }));
 
+/** Format a duration in milliseconds as a short age like "1m 5s" */
+function formatAge(ms: number) {
+	const seconds = Math.max(0, Math.floor(ms / 1000));
+	if (seconds < 60) {
+		return `${seconds}s`;
+	}
+	if (seconds < 3600) {
+		return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
+	}
+	return `${Math.floor(seconds / 3600)}h ${Math.floor(seconds / 60) % 60}m`;
+}
+
+/** Strip case and separators the same way the controller does when matching pairing codes */
+function plainCode(code: string) {
+	return code.toUpperCase().replace(/[^0-9A-Z]/g, "");
+}
+
 hostCommands.add(new lib.Command({
-	definition: ["generate-token", "Generate token for a host", (yargs) => {
-		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
-	}],
-	handler: async function(args: { id?: number }, ctl: Ctl) {
-		let hostId = typeof args.id === "number" ? args.id : undefined;
-		let response = await ctl.send(new lib.HostGenerateTokenRequest(hostId));
-		print(response);
+	definition: ["pending", "List host installers waiting for approval"],
+	handler: async function(args: object, ctl: Ctl) {
+		let requests = await ctl.send(new lib.HostPairingListRequest());
+		if (!requests.length) {
+			print("No pending host requests");
+			return;
+		}
+		let now = Date.now();
+		print(asTable(requests.map(request => ({
+			code: request.code,
+			name: request.name,
+			address: request.remoteAddress,
+			age: formatAge(now - request.createdAtMs),
+		}))));
 	},
 }));
 
 hostCommands.add(new lib.Command({
-	definition: ["revoke-token <host>", "Revoke all tokens for a host", (yargs) => {
-		yargs.positional("host", { describe: "Host to revoke tokens for", type: "string" });
+	definition: ["approve <code>", "Approve a host installer waiting for approval", (yargs) => {
+		yargs.positional("code", { describe: "Code shown by the installer", type: "string" });
+		yargs.option("name", {
+			type: "string", nargs: 1, describe: "Name to give the host instead of the one it asked for",
+		});
+	}],
+	handler: async function(args: { code: string, name?: string }, ctl: Ctl) {
+		let name = args.name?.trim();
+		if (name === undefined) {
+			let requests = await ctl.send(new lib.HostPairingListRequest());
+			name = requests.find(request => plainCode(request.code) === plainCode(args.code))?.name;
+		}
+		let hostId = await ctl.send(new lib.HostPairingApproveRequest(args.code, args.name));
+		print(name ? `Approved host ${name} with id ${hostId}` : `Approved host with id ${hostId}`);
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["reject <code>", "Reject a host installer waiting for approval", (yargs) => {
+		yargs.positional("code", { describe: "Code shown by the installer", type: "string" });
+	}],
+	handler: async function(args: { code: string }, ctl: Ctl) {
+		await ctl.send(new lib.HostPairingRejectRequest(args.code));
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["revoke <host>", "Remove a host from the cluster and disconnect it", (yargs) => {
+		yargs.positional("host", { describe: "Host to revoke", type: "string" });
 	}],
 	handler: async function(args: { host: string }, ctl: Ctl) {
 		let hostId = await lib.resolveHost(ctl, args.host);
-		await ctl.send(new lib.HostRevokeTokensRequest(hostId));
+		await ctl.send(new lib.HostRevokeRequest(hostId));
 	},
 }));
 
-hostCommands.add(new lib.Command({
-	definition: ["create-config", "Create host config", (yargs) => {
-		yargs.option("id", { type: "number", nargs: 1, describe: "Host id" });
-		yargs.option("name", { type: "string", nargs: 1, describe: "Host name" });
-		yargs.option("generate-token", {
-			type: "boolean", nargs: 0, describe: "Generate authentication token", default: false,
-		});
-		yargs.option("output", {
-			type: "string", nargs: 1, describe: "Path to output config (- for stdout)", default: "config-host.json",
-		});
-	}],
-	handler: async function(
-		args: { id?: number, name?: string, generateToken: boolean, output: string },
-		ctl: Ctl
-	) {
-		let config = await ctl.send(
-			new lib.HostConfigCreateRequest(args.id, args.name, args.generateToken)
-		);
-
-		let content = JSON.stringify(config, null, "\t");
-		if (args.output === "-") {
-			print(content);
-		} else {
-			logger.info(`Writing ${args.output}`);
-			try {
-				await fs.writeFile(args.output, content, { flag: "wx", mode: 0o600 });
-			} catch (err: any) {
-				if (err.code === "EEXIST") {
-					throw new lib.CommandError(`File ${args.output} already exists`);
-				}
-				throw err;
-			}
-		}
-	},
-}));
 hostCommands.add(new lib.Command({
 	definition: ["update <host>", "Update the host", (yargs) => {
 		yargs.positional("host", { describe: "Host to update", type: "string" });
@@ -271,14 +286,14 @@ hostPluginCommands.add(new lib.Command({
 		let hostId = await lib.resolveHost(ctl, args.host);
 		const plugins = await ctl.sendTo({ hostId }, new lib.PluginListRequest());
 		print(asTable(plugins.map(p => ({
-			title: p.title, version: p.version, loaded: p.loaded, enabled: p.enabled, npmPackage: p.npmPackage,
+			title: p.title, version: p.version, state: p.state, npmPackage: p.npmPackage,
 		}))));
 	},
 }));
 hostPluginCommands.add(new lib.Command({
 	definition: ["update <host> <plugin>", "Update a plugin on a host", (yargs) => {
 		yargs.positional("host", { describe: "Host to update on", type: "string" });
-		yargs.positional("plugin", { describe: "Plugin to update", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to update, optionally as name@version", type: "string" });
 		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
 	}],
 	handler: async function(args: { host: string, plugin: string, restart: boolean }, ctl: Ctl) {
@@ -294,7 +309,7 @@ hostPluginCommands.add(new lib.Command({
 hostPluginCommands.add(new lib.Command({
 	definition: ["install <host> <plugin>", "Install a plugin on a host", (yargs) => {
 		yargs.positional("host", { describe: "Host to install on", type: "string" });
-		yargs.positional("plugin", { describe: "Plugin to install", type: "string" });
+		yargs.positional("plugin", { describe: "Plugin to install, optionally as name@version", type: "string" });
 		yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" });
 	}],
 	handler: async function(args: { host: string, plugin: string, restart: boolean }, ctl: Ctl) {

packages/ctl/src/commands_instance.ts

@@ -20,10 +20,14 @@ export const instanceCommands = new lib.CommandTree({
 	name: "instance", alias: ["i"], description: "Instance management",
 });
 instanceCommands.add(new lib.Command({
-	definition: [["list", "l"], "List instances known to the controller"],
-	handler: async function(args: object, ctl: Ctl) {
+	definition: [["list", "l"], "List instances known to the controller", (yargs) => {
+		yargs.options({
+			"templates": { describe: "Include templates", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args: { templates: boolean }, ctl: Ctl) {
 		let list = await ctl.send(new lib.InstanceDetailsListRequest());
-		print(asTable(list));
+		print(asTable(list.filter(instance => args.templates || !instance.isTemplate)));
 	},
 }));
 
@@ -34,18 +38,30 @@ instanceCommands.add(new lib.Command({
 		yargs.options({
 			"id": { type: "number", nargs: 1, describe: "Instance id" },
 			"from": { type: "number", nargs: 1, describe: "Clone config from instance id" },
+			"template": { type: "string", nargs: 1, describe: "Template to inherit config from" },
+			"as-template": { type: "boolean", nargs: 0, describe: "Create the instance as a template" },
 		});
 	}],
-	handler: async function(args: { name: string, id?: number, from?: number, }, ctl: Ctl) {
+	handler: async function(
+		args: { name: string, id?: number, from?: number, template?: string, asTemplate?: boolean },
+		ctl: Ctl,
+	) {
 		let instanceConfig = new lib.InstanceConfig("control");
+		let fields: (keyof lib.InstanceConfigFields)[] = ["instance.id", "instance.name"];
 		if (args.id !== undefined) {
 			instanceConfig.set("instance.id", args.id);
 		}
 		instanceConfig.set("instance.name", args.name);
+		if (args.template !== undefined) {
+			instanceConfig.set("instance.template_id", await lib.resolveInstance(ctl, args.template));
+			fields.push("instance.template_id");
+		}
+		if (args.asTemplate) {
+			instanceConfig.set("instance.is_template", true);
+			fields.push("instance.is_template");
+		}
 		await ctl.send(new lib.InstanceCreateRequest(
-			instanceConfig.toRemote("controller", [
-				"instance.id", "instance.name",
-			]),
+			instanceConfig.toRemote("controller", fields),
 			args.from
 		));
 	},
@@ -61,13 +77,42 @@ instanceConfigCommands.add(new lib.Command({
 	handler: async function(args: { instance: string }, ctl: Ctl) {
 		let instanceId = await lib.resolveInstance(ctl, args.instance);
 		let config = await ctl.send(new lib.InstanceConfigGetRequest(instanceId));
+		const inherited = config["instance.template_id"] !== null
+			? config["instance.inherited_fields"] as Record<string, true | string[]> ?? {}
+			: {};
 
 		for (let [name, value] of Object.entries(config)) {
-			print(`${name} ${JSON.stringify(value)}`);
+			if (name === "instance.inherited_fields") {
+				continue;
+			}
+			const entry = inherited[name];
+			let note = "";
+			if (entry instanceof Array && entry.length) {
+				note = ` (inherited except ${entry.join(", ")})`;
+			} else if (entry) {
+				note = " (inherited)";
+			}
+			print(`${name} ${JSON.stringify(value)}${note}`);
 		}
 	},
 }));
 
+instanceConfigCommands.add(new lib.Command({
+	definition: [
+		"inherit <instance> <field> [prop]",
+		"Make field or property in instance config use the value of its template",
+		(yargs) => {
+			yargs.positional("instance", { describe: "Instance to update config on", type: "string" });
+			yargs.positional("field", { describe: "Field to inherit", type: "string" });
+			yargs.positional("prop", { describe: "Property of object field to inherit", type: "string" });
+		},
+	],
+	handler: async function(args: { instance: string, field: string, prop?: string }, ctl: Ctl) {
+		let instanceId = await lib.resolveInstance(ctl, args.instance);
+		await ctl.send(new lib.InstanceConfigInheritFieldRequest(instanceId, args.field, args.prop));
+	},
+}));
+
 instanceConfigCommands.add(new lib.Command({
 	definition: ["set <instance> <field> [value]", "Set field in instance config", (yargs) => {
 		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
@@ -170,8 +215,10 @@ instanceConfigCommands.add(new lib.Command({
 		await events.once(editorSpawn, "exit");
 		const data = await fs.readFile(tmpFile, "utf8");
 		const final = await configToKeyVal(data);
+		const original = await configToKeyVal(allConfigElements);
 		for (let index in final) {
-			if (index in final) {
+			// Unchanged fields are skipped so they keep inheriting from the template
+			if (index in final && final[index] !== original[index]) {
 				try {
 					await ctl.send(new lib.InstanceConfigSetFieldRequest(
 						instanceId,

packages/ctl/src/commands_note.ts

@@ -0,0 +1,72 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import type { Ctl } from "../ctl.js";
+import { print } from "./command_ops.js";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const noteCommands = new lib.CommandTree({
+	name: "note", description: "Notes attached to hosts, instances, users and other resources",
+});
+noteCommands.add(new lib.Command({
+	definition: [["list", "l"], "List notes in the cluster", (yargs) => {
+		yargs.options({
+			"type": { describe: "Only list notes for this resource type", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(args: { type?: string }, ctl: Ctl) {
+		let notes = await ctl.send(new lib.NoteListRequest());
+		if (args.type !== undefined) {
+			notes = notes.filter(note => note.resourceType === args.type);
+		}
+		print(asTable(notes.map(note => ({
+			type: note.resourceType,
+			id: note.resourceId,
+			updatedBy: note.updatedBy,
+			content: note.content,
+		}))));
+	},
+}));
+
+noteCommands.add(new lib.Command({
+	definition: ["get <type> <id>", "Show the note of a resource", (yargs) => {
+		yargs.positional("type", {
+			describe: "Resource type, one of host, instance, user, role, mod_pack, mod, save", type: "string",
+		});
+		yargs.positional("id", { describe: "Id of the resource", type: "string" });
+	}],
+	handler: async function(args: { type: string, id: string }, ctl: Ctl) {
+		const notes = await ctl.send(new lib.NoteListRequest());
+		const note = notes.find(n => n.resourceType === args.type && n.resourceId === args.id);
+		if (!note) {
+			throw new lib.CommandError(`No note for ${args.type} ${args.id}`);
+		}
+		print(note.content);
+	},
+}));
+
+noteCommands.add(new lib.Command({
+	definition: ["set <type> <id> <content>", "Set the note of a resource", (yargs) => {
+		yargs.positional("type", {
+			describe: "Resource type, one of host, instance, user, role, mod_pack, mod, save", type: "string",
+		});
+		yargs.positional("id", { describe: "Id of the resource", type: "string" });
+		yargs.positional("content", { describe: "Text of the note", type: "string" });
+	}],
+	handler: async function(args: { type: string, id: string, content: string }, ctl: Ctl) {
+		await ctl.send(new lib.NoteSetRequest(args.type, args.id, args.content));
+	},
+}));
+
+noteCommands.add(new lib.Command({
+	definition: ["delete <type> <id>", "Remove the note of a resource", (yargs) => {
+		yargs.positional("type", {
+			describe: "Resource type, one of host, instance, user, role, mod_pack, mod, save", type: "string",
+		});
+		yargs.positional("id", { describe: "Id of the resource", type: "string" });
+	}],
+	handler: async function(args: { type: string, id: string }, ctl: Ctl) {
+		await ctl.send(new lib.NoteSetRequest(args.type, args.id, ""));
+	},
+}));

packages/ctl/src/config_ops.ts

@@ -2,7 +2,7 @@ import * as lib from "@clusterio/lib";
 
 export async function getEditor(argsEditor: string) {
 	// eslint-disable-next-line
-	return argsEditor || process.env.EDITOR || process.env.VISUAL || undefined
+	return argsEditor || process.env.EDITOR || process.env.VISUAL || undefined;
 	// needed for the process.env statements to not be flagged by eslint
 	// priority for editors is CLI argument > env.EDITOR > env.VISUAL
 }

packages/ctl/typedoc.json

@@ -1,4 +0,0 @@
-{
-	"entryPoints": ["index.ts"],
-	"includeVersion": true
-}

packages/host/README.md

@@ -132,6 +132,10 @@ Lists up all configuration entries with their currently configured values.
 ### `run`
 
 Runs the host.
+A host that has not been approved prints a code and a link and waits for approval before it connects, see [Setting up a host](https://github.com/clusterio/clusterio/blob/master/packages/create/README.md#setting-up-a-host).
+If the controller no longer accepts the token of the host when it starts, for example because the host was revoked, it asks to be approved again.
+If its request is rejected it exits with code 8 and the run scripts created by the installer do not restart it.
+A host that is revoked while it runs shuts down.
 
 
 ## See Also

packages/host/host.ts

@@ -20,11 +20,14 @@ import winston from "winston";
 import "winston-daily-rotate-file";
 
 // internal libraries
+// Reduce startup time by lazy compiling schemas.
+import "./src/set_lazy_schema_compilation.js";
 import * as lib from "@clusterio/lib";
 import { ConsoleTransport, levels, logger } from "@clusterio/lib";
 
-import Host from "./src/Host.js";
+import type Host from "./src/Host.js";
 import { createScenarioCommand, handleCreateScenarioCommand } from "./src/create_scenario.js";
+import { PairingError, ensurePaired } from "./src/pairing.js";
 
 let host: Host | undefined;
 
@@ -134,6 +137,10 @@ async function startHost() {
 				type: "boolean", nargs: 0, default: false,
 				describe: "Start the host in recovery mode with all plugins disabled and controller disconnected",
 			});
+			yargs.option("user-token", {
+				type: "string", nargs: 1,
+				describe: "Token of a user allowed to approve hosts, approves this host without waiting. Not saved",
+			});
 		})
 		.demandCommand(1, "You need to specify a command to run")
 		.strict()
@@ -164,7 +171,8 @@ async function startHost() {
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = await lib.loadPluginList(args.pluginList);
+	const skippedNpmPlugins = new Map<string, string>();
+	let pluginList = await lib.loadPluginList(args.pluginList, true, skippedNpmPlugins);
 
 	// If the command is plugin management we don't try to load plugins
 	if (command === "plugin") {
@@ -173,7 +181,7 @@ async function startHost() {
 	}
 
 	logger.info("Loading Plugin info");
-	let pluginInfos = await lib.loadPluginInfos(pluginList);
+	let pluginInfos = await lib.loadPluginInfos(pluginList, skippedNpmPlugins);
 	lib.registerPluginMessages(pluginInfos);
 	lib.registerPluginPermissions(pluginInfos);
 	lib.addPluginConfigFields(pluginInfos);
@@ -211,19 +219,6 @@ async function startHost() {
 	// and as the process name in ps/top on linux.
 	process.title = "clusteriohost";
 
-	// make sure we have the controller access token
-	if (hostConfig.get("host.controller_token") === "enter token here") {
-		logger.fatal("ERROR invalid config!");
-		logger.fatal(
-			"Controller requires an access token for socket operations. As clusterio\n"+
-			"hosts depends upon this, please set your token using the command npx\n"+
-			"clusteriohost config set host.controller_token <token>.  You can generate an\n"+
-			"auth token using npx clusterioctl generate-host-token."
-		);
-		process.exitCode = 1;
-		return;
-	}
-
 	// make sure url ends with /
 	if (!hostConfig.get("host.controller_url").endsWith("/")) {
 		logger.fatal("ERROR invalid config!");
@@ -232,14 +227,46 @@ async function startHost() {
 		return;
 	}
 
+	if (!args.recovery) {
+		const abort = new AbortController();
+		const onSignal = () => { abort.abort(); };
+		process.on("SIGINT", onSignal);
+		process.on("SIGTERM", onSignal);
+		process.on("SIGHUP", onSignal);
+		try {
+			await ensurePaired(hostConfig, {
+				version: packageConfig.version,
+				userToken: args.userToken as string | undefined,
+				signal: abort.signal,
+			});
+		} catch (err: any) {
+			if (abort.signal.aborted) {
+				logger.info("Stopped waiting for approval");
+				return;
+			}
+			if (err instanceof PairingError) {
+				logger.fatal(`Unable to join the cluster: ${err.message}`);
+				// Exit code 8 stops the run scripts from restarting the host.
+				process.exitCode = err.status === 403 ? 8 : 1;
+				return;
+			}
+			throw err;
+		} finally {
+			process.off("SIGINT", onSignal);
+			process.off("SIGTERM", onSignal);
+			process.off("SIGHUP", onSignal);
+		}
+	}
+
+	// Loaded here as the server is not needed by the other commands
+	const { default: Host } = await import("./src/Host.js");
 	let hostConnector = new HostConnector(hostConfig, pluginInfos);
-	host = new Host(
-		hostConnector,
+	host = await Host.fromConfig(
 		hostConfig,
 		pluginInfos,
+		hostConnector,
 		Boolean(args.canRestart),
 		Boolean(args.recovery),
-		...await Host.bootstrap(hostConfig)
 	);
 
 	// Handle interrupts
@@ -267,14 +294,6 @@ async function startHost() {
 }
 
 export function bootstrap() {
-	// eslint-disable-next-line no-console
-	console.warn(`
-+==========================================================+
-I WARNING:  This is the development branch for the 2.0     I
-I           version of clusterio.  Expect things to break. I
-+==========================================================+
-`
-	);
 	startHost().catch(err => {
 		if (err instanceof lib.WebSocketError) {
 			// Already handled by connector.on("error") within Host

packages/host/index.ts

@@ -4,6 +4,7 @@ export { BaseInstancePlugin, InstanceHooks, type InstancePluginContext } from ".
 export { default as Host, HostRouter } from "./src/Host.js";
 export { default as Instance } from "./src/Instance.js";
 export { default as InstanceConnection } from "./src/InstanceConnection.js";
+export { PairingError, checkHostToken, pairHost } from "./src/pairing.js";
 
 if (import.meta.main) {
 	bootstrap();

packages/host/lua/clusterio_lib/api.lua

@@ -38,17 +38,20 @@ function api.get_instance_id()
 end
 
 
+local function escape_channel(channel)
+	-- Escape bad characters.  The question mark is used for separating the
+	-- channel name from the payload.
+	return channel:gsub("([\x00-\x1f?\\])", function(match)
+		return "\\x" .. string.format("%02x", match:byte())
+	end)
+end
+
 function api.send_json(channel, data)
 	if not remote.interfaces.clusterio_api then
 		return
 	end
 
-	-- Escape bad characters.  The question mark is used for separating the
-	-- channel name from the payload.
-	channel = channel:gsub("([\x00-\x1f?\\])", function(match)
-		return "\\x" .. string.format("%02x", match:byte())
-	end)
-
+	channel = escape_channel(channel)
 	data = compat.table_to_json(data)
 
 	-- If there's more than about 4kB of data users running with the Windows
@@ -64,5 +67,14 @@ function api.send_json(channel, data)
 	end
 end
 
+function api.send_udp(channel, data)
+	local port = call_api("get_host_udp_port")
+	if not port then
+		return -- Lua UDP is not enabled for this instance
+	end
+
+	helpers.send_udp(port, escape_channel(channel) .. "?" .. data, 0)
+end
+
 
 return api

packages/host/lua/clusterio_lib/changelog.txt

@@ -1,4 +1,29 @@
 ---------------------------------------------------------------------------------------------------
+Version: 2.1.21
+Date: 2026-10-14
+  Changes:
+    - Removed the obsolete alpha label and development warning from the mod title and description.
+---------------------------------------------------------------------------------------------------
+Version: 2.1.20
+Date: 2026-10-14
+  Changes:
+    - Removed the obsolete alpha label and development warning from the mod title and description.
+---------------------------------------------------------------------------------------------------
+Version: 2.1.18
+Date: 2026-10-14
+  Changes:
+    - Removed the obsolete alpha label and development warning from the mod title and description.
+---------------------------------------------------------------------------------------------------
+Version: 2.1.17
+Date: 2026-10-14
+  Changes:
+    - Removed the obsolete alpha label and development warning from the mod title and description.
+---------------------------------------------------------------------------------------------------
+Version: 2.1.11
+Date: 2026-10-14
+  Changes:
+    - Removed the obsolete alpha label and development warning from the mod title and description.
+---------------------------------------------------------------------------------------------------
 Version: 2.0.20
 Date: 25. 02. 2025
   Features:

packages/host/lua/clusterio_lib/info.json

@@ -2,30 +2,30 @@
 	"name": "clusterio_lib",
 	"variants": [
 		{
-			"version": "2.0.17",
+			"version": "2.1.17",
 			"factorio_version": "0.17"
 		},
 		{
-			"version": "2.0.18",
+			"version": "2.1.18",
 			"factorio_version": "0.18"
 		},
 		{
-			"version": "2.0.11",
+			"version": "2.1.11",
 			"factorio_version": "1.1"
 		},
 		{
-			"version": "2.0.20",
+			"version": "2.1.20",
 			"factorio_version": "2.0"
 		},
 		{
-			"version": "2.0.21",
+			"version": "2.1.21",
 			"factorio_version": "2.1"
 		}
 	],
-	"title": "Clusterio Library (Alpha)",
+	"title": "Clusterio Library",
 	"author": "Clusterio Team",
 	"homepage": "https://github.com/clusterio/clusterio",
-	"description": "Library for interfacing with Clusterio.  Warning: work in progress alpha.",
+	"description": "Library for interfacing with Clusterio.",
 	"additional_files": {
 		"compat.lua": ["..", "..", "modules", "clusterio", "compat.lua"],
 		"serialize.lua": ["..", "..", "modules", "clusterio", "serialize.lua"],

packages/host/modules/clusterio/api.lua

@@ -17,14 +17,16 @@ function api.get_instance_id()
 	return compat.script_data.clusterio.instance_id
 end
 
-function api.send_json(channel, data)
-
+local function escape_channel(channel)
 	-- Escape bad characters.  The question mark is used for separating the
 	-- channel name from the payload.
-	channel = channel:gsub("([\x00-\x1f?\\])", function(match)
+	return channel:gsub("([\x00-\x1f?\\])", function(match)
 		return "\\x" .. string.format("%02x", match:byte())
 	end)
+end
 
+function api.send_json(channel, data)
+	channel = escape_channel(channel)
 	data = compat.table_to_json(data)
 
 	-- If there's more than about 4kB of data users running with the Windows
@@ -41,5 +43,15 @@ function api.send_json(channel, data)
 	end
 end
 
+function api.send_udp(channel, data)
+	local clusterio = compat.script_data.clusterio
+	local port = clusterio and clusterio.host_udp_port
+	if not port then
+		return -- Lua UDP is not enabled for this instance
+	end
+
+	helpers.send_udp(port, escape_channel(channel) .. "?" .. data, 0)
+end
+
 
 return api

packages/host/modules/clusterio/impl.lua

@@ -12,16 +12,27 @@ local function check_patch()
 	end
 end
 
+-- Added in Factorio 2.0.59, indexing a missing key on a LuaObject errors
+local has_recv_udp = helpers ~= nil and pcall(function() return helpers.recv_udp end)
+
 local impl = {}
 impl.events = {}
 
-impl.events[defines.events.on_tick] = check_patch
+impl.events[defines.events.on_tick] = function()
+	check_patch()
+	-- Dispatch on_udp_packet_received for packets sent by the host, this is
+	-- a no-op when the server was started without --enable-lua-udp
+	if has_recv_udp then
+		helpers.recv_udp(0)
+	end
+end
 
 impl.events[api.events.on_server_startup] = function()
 	if not compat.script_data.clusterio then
 		compat.script_data.clusterio = {
 			instance_id = nil,
 			instance_name = nil,
+			host_udp_port = nil,
 		}
 	end
 
@@ -74,11 +85,12 @@ end
 
 -- Internal API
 clusterio_private = {}
-function clusterio_private.update_instance(new_id, new_name)
+function clusterio_private.update_instance(new_id, new_name, host_udp_port)
 	check_patch()
 	local script_data = compat.script_data
 	script_data.clusterio.instance_id = new_id
 	script_data.clusterio.instance_name = new_name
+	script_data.clusterio.host_udp_port = host_udp_port
 	script.raise_event(api.events.on_instance_updated, {
 		instance_id = new_id,
 		instance_name = new_name,
@@ -105,6 +117,12 @@ remote.add_interface('clusterio_api', {
 		return compat.script_data.clusterio.instance_name
 	end,
 
+	get_host_udp_port = function()
+		-- Not initialised until the first tick after the save is patched
+		local clusterio = compat.script_data.clusterio
+		return clusterio and clusterio.host_udp_port
+	end,
+
 	get_file_no = function()
 		local script_data = compat.script_data
 		script_data.clusterio_file_no = (script_data.clusterio_file_no or 0) + 1

packages/host/modules/clusterio/serialize.lua

@@ -17,6 +17,7 @@ end
 
 local has_create_grid = compat.version_ge("1.1.7")
 local has_quality = compat.version_ge("2.0.0")
+local has_spoilage = compat.version_ge("2.0.0")
 
 -- Equipment Grids are serialized into an array of equipment entries
 -- where ench entry is a table with the following fields:
@@ -106,6 +107,7 @@ end
 --	 c: count
 --	 q: quality (optional)
 --	 h: health (optional)
+--	 p: spoil percent (optional)
 --	 d: durability (optional)
 --	 a: ammo count (optional)
 --	 l: label (optional)
@@ -139,6 +141,7 @@ function serialize.serialize_item_stack(slot, entry)
 	entry.c = slot.count
 	if has_quality and slot.quality.level ~= 0 then entry.q = slot.quality.name end
 	if slot.health < 1 then entry.h = slot.health end
+	if has_spoilage and slot.spoil_percent > 0 then entry.p = slot.spoil_percent end
 	if slot.type == "tool" and slot.durability then entry.d = slot.durability end
 	if slot.type == "ammo" then entry.a = slot.ammo end
 	if slot.is_item_with_label then
@@ -177,6 +180,7 @@ function serialize.deserialize_item_stack(slot, entry)
 	}
 	if entry.q then item_stack.quality = entry.q end
 	if entry.h then item_stack.health = entry.h end
+	if entry.p then item_stack.spoil_percent = entry.p end
 	if entry.d then item_stack.durability = entry.d end
 	if entry.a then item_stack.ammo = entry.a end
 
@@ -288,10 +292,10 @@ function serialize.deserialize_inventory(inventory, serialized)
 			if entry.f then
 				local call_success, call_return = pcall(inventory.set_filter, index, entry.f)
 				if not call_success then
-					print("Error: '" .. call_return .. "' thrown setting filter " .. entry.f)
+					print("Error: '" .. call_return .. "' thrown setting filter " .. serpent.line(entry.f))
 
 				elseif not call_return then
-					print("Error: Failed to set filter " .. entry.f)
+					print("Error: Failed to set filter " .. serpent.line(entry.f))
 				end
 			end
 

packages/host/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/host",
-	"version": "2.0.0",
+	"version": "2.0.3",
 	"description": "Implementation of Clusterio host server",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -33,21 +33,25 @@
 		"clusteriohost": "./dist/node/host.js"
 	},
 	"engines": {
-		"node": ">=22"
+		"node": "^22.18.0 || >=24.2.0"
 	},
 
 	"dependencies": {
 		"@clusterio/lib": "workspace:^",
+		"@jimp/core": "^1.6.1",
+		"@jimp/js-png": "^1.6.1",
+		"@jimp/plugin-crop": "^1.6.1",
+		"@jimp/plugin-resize": "^1.6.1",
+		"@jimp/types": "^1.6.1",
 		"@sinclair/typebox": "catalog:",
-		"jimp": "^1.6.1",
-		"jszip": "catalog:",
-		"pidusage": "^4.0.1",
+		"pidusage": "^4.1.0",
 		"rcon-client": "^4.2.5",
 		"semver": "catalog:",
 		"set-blocking": "catalog:",
 		"winston": "catalog:",
 		"winston-daily-rotate-file": "catalog:",
-		"yargs": "catalog:"
+		"yargs": "catalog:",
+		"yazl": "catalog:"
 	},
 	"devDependencies": {
 		"@types/node": "catalog:",
@@ -55,6 +59,7 @@
 		"@types/semver": "catalog:",
 		"@types/set-blocking": "catalog:",
 		"@types/yargs": "catalog:",
+		"@types/yazl": "catalog:",
 		"typescript": "catalog:"
 	}
 }

packages/host/src/BaseInstancePlugin.ts

@@ -23,6 +23,7 @@ export class InstanceHooks extends lib.AsyncHookCollection {
 		this.controllerConnectionEvent = this.newHook();
 		this.prepareControllerDisconnect = this.newHook();
 		this.playerEvent = this.newHook();
+		this.autoPauseChanged = this.newHook();
 	}
 
 	/**
@@ -147,6 +148,19 @@ export class InstanceHooks extends lib.AsyncHookCollection {
 	 * @param event - Information about the event.
 	 */
 	readonly playerEvent: lib.AsyncHook<[event: lib.PlayerEvent]>;
+
+	/**
+	 * Called when Factorio auto pauses or unpauses the game
+	 *
+	 * Invoked when the last player leaves or the first player joins an
+	 * instance started with the auto_pause server setting on.  Every RCON
+	 * command makes a paused game run one tick, so periodic commands
+	 * should be skipped while paused.  See `Instance.autoPaused`.  It is
+	 * not invoked on start, read `autoPaused` in the start hook instead.
+	 *
+	 * @param paused - true if the game is now paused.
+	 */
+	readonly autoPauseChanged: lib.AsyncHook<[paused: boolean]>;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
@@ -160,6 +174,7 @@ export interface BaseInstancePlugin {
 	onControllerConnectionEvent?(event: "connect" | "drop" | "resume" | "close"): void;
 	onPrepareControllerDisconnect?(connection: Instance): Promise<void>;
 	onPlayerEvent?(event: lib.PlayerEvent): Promise<void>;
+	onAutoPauseChanged?(paused: boolean): Promise<void>;
 }
 
 /**
@@ -222,6 +237,7 @@ export class BaseInstancePlugin {
 		attach(instance.hooks.controllerConnectionEvent, this.onControllerConnectionEvent);
 		attach(instance.hooks.prepareControllerDisconnect, this.onPrepareControllerDisconnect);
 		attach(instance.hooks.playerEvent, this.onPlayerEvent);
+		attach(instance.hooks.autoPauseChanged, this.onAutoPauseChanged);
 	}
 
 	static fromContext(context: InstancePluginContext) {
@@ -259,6 +275,22 @@ export class BaseInstancePlugin {
 		return await this.instance.sendRcon(message, expectEmpty, this.info.name);
 	}
 
+	/**
+	 * Send UDP message to instance
+	 *
+	 * Send a UDP packet to the Factorio server, which is delivered in-game
+	 * as an on_udp_packet_received event.  Delivery is not guaranteed and
+	 * the payload should be kept well below 64 kB.  Requires the LuaUdp
+	 * plugin feature.
+	 *
+	 * This should not be called before onStart or after onStop.
+	 *
+	 * @param message - payload of the packet to send.
+	 */
+	async sendUdp(message: string | Buffer): Promise<void> {
+		await this.instance.sendUdp(message);
+	}
+
 	/**
 	 * Send serially ordered RCON message to instance
 	 *

packages/host/src/Host.ts

@@ -22,7 +22,38 @@ function checkRequestSaveName(name: string) {
 	try {
 		lib.checkFilename(name);
 	} catch (err: any) {
-		throw new lib.RequestError(`Save name ${err.message}`);
+		throw new lib.ValidationError(`Save name ${err.message}`);
+	}
+}
+
+async function copySaveExclusive(sourcePath: string, targetPath: string) {
+	await fs.copyFile(
+		sourcePath,
+		targetPath,
+		fs.constants.COPYFILE_EXCL | fs.constants.COPYFILE_FICLONE
+	);
+}
+
+/**
+ * Move a file without overwriting an existing target, fails with EEXIST
+ * if the target exists.
+ */
+async function moveSaveExclusive(sourcePath: string, targetPath: string) {
+	try {
+		await fs.link(sourcePath, targetPath);
+	} catch (err: any) {
+		// Hard links fail across filesystems and on some filesystems with
+		// varying codes, the copy is still exclusive.
+		if (err.code === "EEXIST") {
+			throw err;
+		}
+		await copySaveExclusive(sourcePath, targetPath);
+	}
+	try {
+		await fs.unlink(sourcePath);
+	} catch (err: any) {
+		await fs.unlink(targetPath).catch(() => {});
+		throw err;
 	}
 }
 
@@ -172,7 +203,7 @@ export class HostRouter {
 						return false;
 					}
 					origin.connector.sendResponseError(
-						new lib.ResponseError(msg || "Instance is not running."), message.src
+						new lib.ResponseError(msg || "Instance is not running.", "UnavailableError"), message.src
 					);
 				}
 				return true;
@@ -234,13 +265,13 @@ export class HostRouter {
 		let dst = message.dst;
 		if (nextHop) {
 			origin.connector.sendResponseError(
-				new lib.ResponseError("Instance is already running."), message.src
+				new lib.ResponseError("Instance is already running.", "ConflictError"), message.src
 			);
 			return;
 		}
 		if (this.host._shuttingDown) {
 			origin.connector.sendResponseError(
-				new lib.ResponseError("Host is shutting down."), message.src
+				new lib.ResponseError("Host is shutting down.", "UnavailableError"), message.src
 			);
 			return;
 		}
@@ -249,9 +280,7 @@ export class HostRouter {
 			instanceConnection.connector.forward(message);
 		}).catch(err => {
 			logger.error(`Error starting instance:\n${err.stack ?? err.message}`, this.host.instanceLogMeta(dst.id));
-			origin.connector.sendResponseError(
-				new lib.ResponseError(err.message, err.code, err.stack), message.src
-			);
+			origin.connector.sendResponseError(lib.Link.errorToResponse(err), message.src);
 		});
 	}
 
@@ -264,9 +293,7 @@ export class HostRouter {
 			}
 		} catch (err: any) {
 			if (message.type === "request") {
-				origin.connector.sendResponseError(
-					new lib.ResponseError(err.message, err.code, err.stack), message.src
-				);
+				origin.connector.sendResponseError(lib.Link.errorToResponse(err), message.src);
 			}
 			logger.warn(`Failed to deliver ${(message as any).name || "message"} (${message.type}): ${err.message}`);
 			if (err instanceof lib.InvalidMessage && err.errors) {
@@ -322,6 +349,8 @@ export default class Host extends lib.Link {
 	_startup = true;
 	_disconnecting = false;
 	_shuttingDown = false;
+	/** Instances waiting on {@link Instance.fromConfig} */
+	_startingInstances = new Set<number>();
 
 	startedAtMs = Date.now();
 
@@ -329,12 +358,18 @@ export default class Host extends lib.Link {
 		"_warning": "Changes to this file will be overwritten by the controller's copy.",
 	};
 
-	static async bootstrap(hostConfig: lib.HostConfig) {
+	/** Create a host with its mod store loaded from disk */
+	static async fromConfig(
+		hostConfig: lib.HostConfig,
+		pluginInfos: lib.PluginNodeEnvInfo[],
+		connector: HostConnector,
+		canRestart = false,
+		recoveryMode = false,
+	) {
 		const modsDirectory = hostConfig.get("host.mods_directory");
 		await fs.mkdir(modsDirectory, { recursive: true });
-		return [
-			await lib.ModStore.fromDirectory(modsDirectory),
-		] as const;
+		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
+		return new this(connector, hostConfig, pluginInfos, canRestart, recoveryMode, modStore);
 	}
 
 	constructor(
@@ -498,8 +533,8 @@ export default class Host extends lib.Link {
 		}
 
 		let instancesDir = this.config.get("host.instances_directory");
-		for (let i = 0; i < 10; i++) { // Limit attempts in case this is somehow an infinite loop
-			let candidateDir = path.join(instancesDir, await lib.findUnusedName(instancesDir, name));
+		for (let i = 1; ; i++) {
+			let candidateDir = path.join(instancesDir, i === 1 ? name : `${name}-${i}`);
 			try {
 				await fs.mkdir(candidateDir);
 			} catch (err: any) {
@@ -510,7 +545,6 @@ export default class Host extends lib.Link {
 			}
 			return candidateDir;
 		}
-		throw Error("Unable to create instance dir, retry threshold reached");
 	}
 
 	async broadcastEventToInstance<T>(event: lib.Event<T>) {
@@ -528,12 +562,15 @@ export default class Host extends lib.Link {
 
 	async handleHostRestartRequest() {
 		if (!this.canRestart) {
-			throw new lib.RequestError("Cannot restart, host does not have a process monitor to restart it.");
+			throw new lib.ConflictError("Cannot restart, host does not have a process monitor to restart it.");
+		}
+		if (await lib.packageOperationRunning()) {
+			throw new lib.ConflictError("Cannot restart while a plugin install or update is running on this machine.");
 		}
 		const downgrade = await this.checkRestartDowngrade();
 		if (downgrade) {
 			const { installedVersion, runningVersion } = downgrade;
-			throw new lib.RequestError(
+			throw new lib.ConflictError(
 				`Cannot restart host with installed Clusterio version ${installedVersion} because it is older than ` +
 				`running version ${runningVersion}. Stop the host before starting the older version manually.`
 			);
@@ -554,7 +591,7 @@ export default class Host extends lib.Link {
 		if (fieldName === "host.id") {
 			// Changing host.id at runtime is not worth the effort to
 			// support and will break a lot of things if it is allowed.
-			throw new lib.RequestError("Setting 'host.id' while host is running is not supported");
+			throw new lib.ConflictError("Setting 'host.id' while host is running is not supported");
 		}
 	}
 
@@ -758,7 +795,7 @@ export default class Host extends lib.Link {
 		this.send(
 			new lib.InstanceStatusChangedEvent(
 				instanceId,
-				instanceConnection ? instanceConnection.instance.status : "stopped",
+				instanceConnection ? instanceConnection.instance.status : this._unconnectedStatus(instanceId),
 				instanceConnection ? instanceConnection.instance.server.gamePort : this.gamePort(instanceId),
 				instanceConnection
 					? instanceConnection.instance.server.version
@@ -799,7 +836,7 @@ export default class Host extends lib.Link {
 	getRequestInstance(instanceId: number) {
 		let instance = this.assignedInstances.get(instanceId);
 		if (!instance) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
+			throw new lib.NotFoundError(`Instance with ID ${instanceId} does not exist`);
 		}
 		return instance;
 	}
@@ -845,7 +882,7 @@ export default class Host extends lib.Link {
 		}
 		if (!availablePorts.size) {
 			const others = inUse.size ? ` (${inUse.size} in use by other processes)` : "";
-			throw new lib.RequestError(`No available port left to assign from host.factorio_port_range${others}`);
+			throw new lib.ConflictError(`No available port left to assign from host.factorio_port_range${others}`);
 		}
 		const [newPort] = availablePorts;
 		instanceToAssign.config.set("factorio.host_assigned_game_port", newPort);
@@ -874,6 +911,11 @@ export default class Host extends lib.Link {
 		}
 	}
 
+	/** Status of an instance without a connection */
+	_unconnectedStatus(instanceId: number) {
+		return this._startingInstances.has(instanceId) ? "starting" as const : "stopped" as const;
+	}
+
 	/**
 	 * Initialize and connect an unloaded instance
 	 *
@@ -882,29 +924,36 @@ export default class Host extends lib.Link {
 	 */
 	async _connectInstance(instanceId: number) {
 		let unloadedInstance = this.getRequestInstance(instanceId);
-		if (this.instanceConnections.has(instanceId)) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
+		if (this.instanceConnections.has(instanceId) || this._startingInstances.has(instanceId)) {
+			throw new lib.ConflictError(`Instance with ID ${instanceId} is running`);
 		}
 
 		let hostAddress = new lib.Address(lib.Address.host, this.config.get("host.id"));
 		let instanceAddress = new lib.Address(lib.Address.instance, instanceId);
 		let [connectionClient, connectionServer] = lib.VirtualConnector.makePair(instanceAddress, hostAddress);
-		let instance = new Instance(
-			this, connectionClient,
-			unloadedInstance.path, this.config.get("host.factorio_directory"),
-			unloadedInstance.config
-		);
+		let instance: Instance;
+		this._startingInstances.add(instanceId);
+		try {
+			instance = await Instance.fromConfig(
+				unloadedInstance.config, this, connectionClient,
+				unloadedInstance.path, this.config.get("host.factorio_directory"),
+			);
+		} finally {
+			this._startingInstances.delete(instanceId);
+		}
 		let instanceConnection = new InstanceConnection(connectionServer, this, instance);
 
 		this.instanceConnections.set(instanceId, instanceConnection);
-		await instance.init(this.pluginInfos);
+		await instance.loadPlugins(this.pluginInfos);
 
 		return instanceConnection;
 	}
 
 	async handleSystemInfoRequest() {
 		const restartRequired = await this.checkRestartRequired();
-		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart, restartRequired, this.startedAtMs);
+		return lib.gatherSystemInfo(
+			this.config.get("host.id"), this.canRestart, restartRequired, this.startedAtMs, await this.listPlugins(),
+		);
 	}
 
 	async checkRestartRequired() {
@@ -1025,19 +1074,16 @@ export default class Host extends lib.Link {
 		checkRequestSaveName(newName);
 		let instance = this.getRequestInstance(instanceId);
 		try {
-			// Link save in the new location.
-			await fs.link(
+			await moveSaveExclusive(
 				path.join(instance.path, "saves", oldName),
 				path.join(instance.path, "saves", newName),
 			);
-			// Unlink save in old location.
-			await fs.unlink(path.join(instance.path, "saves", oldName));
 		} catch (err: any) {
 			if (err.code === "EEXIST") {
-				throw new lib.RequestError(`${newName} already exists`);
+				throw new lib.ConflictError(`${newName} already exists`);
 			}
 			if (err.code === "ENOENT") {
-				throw new lib.RequestError(`${oldName} does not exist`);
+				throw new lib.NotFoundError(`${oldName} does not exist`);
 			}
 			throw err;
 		}
@@ -1057,13 +1103,32 @@ export default class Host extends lib.Link {
 			);
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
-				throw new lib.RequestError(`${source} does not exist`);
+				throw new lib.NotFoundError(`${source} does not exist`);
 			}
 			throw err;
 		}
 		await this.sendSaveListUpdate(instanceId, path.join(instance.path, "saves"));
 	}
 
+	async _moveOrCopySave(sourcePath: string, targetPath: string, copy: boolean) {
+		const { dir, name, ext } = path.parse(targetPath);
+		for (let i = 1; ; i++) {
+			targetPath = path.format({ dir, name: i === 1 ? name : `${name}-${i}`, ext });
+			try {
+				if (copy) {
+					await copySaveExclusive(sourcePath, targetPath);
+				} else {
+					await moveSaveExclusive(sourcePath, targetPath);
+				}
+				return path.basename(targetPath);
+			} catch (err: any) {
+				if (err.code !== "EEXIST") {
+					throw err;
+				}
+			}
+		}
+	}
+
 	async handleInstanceTransferSaveRequest(request: lib.InstanceTransferSaveRequest) {
 		let { sourceName, targetName, copy, sourceInstanceId, targetInstanceId } = request;
 		checkRequestSaveName(sourceName);
@@ -1071,31 +1136,27 @@ export default class Host extends lib.Link {
 		let sourceInstance = this.getRequestInstance(sourceInstanceId);
 		let targetInstance = this.getRequestInstance(targetInstanceId);
 
+		if (!targetName.endsWith(".zip")) {
+			targetName += ".zip";
+		}
+
 		// For consistency with remote transfer initiated through pullSave the
 		// target is renamed if it already exists.
-		targetName = await lib.findUnusedName(
-			path.join(targetInstance.path, "saves"), targetName, ".zip"
-		);
-
+		const sourcePath = path.join(sourceInstance.path, "saves", sourceName);
 		try {
-			if (copy) {
-				await fs.copyFile(
-					path.join(sourceInstance.path, "saves", sourceName),
-					path.join(targetInstance.path, "saves", targetName),
-					fs.constants.COPYFILE_FICLONE
-				);
-			} else {
-				await fs.rename(
-					path.join(sourceInstance.path, "saves", sourceName),
-					path.join(targetInstance.path, "saves", targetName),
-				);
-			}
+			targetName = await this._moveOrCopySave(
+				sourcePath,
+				path.join(targetInstance.path, "saves", targetName),
+				copy,
+			);
 		} catch (err: any) {
-			if (err.code === "ENOENT") {
-				throw new lib.RequestError(`${sourceName} does not exist`);
+			// ENOENT may also come from a missing target saves folder.
+			if (err.code === "ENOENT" && !await fs.access(sourcePath).then(() => true, () => false)) {
+				throw new lib.NotFoundError(`${sourceName} does not exist`);
 			}
 			throw err;
 		}
+
 		await this.sendSaveListUpdate(sourceInstanceId, path.join(sourceInstance.path, "saves"));
 		await this.sendSaveListUpdate(targetInstanceId, path.join(targetInstance.path, "saves"));
 
@@ -1123,7 +1184,7 @@ export default class Host extends lib.Link {
 			await fs.unlink(path.join(instance.path, "saves", name));
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
-				throw new lib.RequestError(`${name} does not exist`);
+				throw new lib.NotFoundError(`${name} does not exist`);
 			}
 			throw err;
 		}
@@ -1154,7 +1215,7 @@ export default class Host extends lib.Link {
 			content = (await fs.open(path.join(instance.path, "saves", name))).createReadStream();
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
-				throw new lib.RequestError(`${name} does not exist`);
+				throw new lib.NotFoundError(`${name} does not exist`);
 			}
 			throw err;
 		}
@@ -1173,12 +1234,12 @@ export default class Host extends lib.Link {
 	async handleInstanceDeleteInternalRequest(request: lib.InstanceDeleteInternalRequest) {
 		let instanceId = request.instanceId;
 		if (this.instanceConnections.has(instanceId)) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} is running`);
+			throw new lib.ConflictError(`Instance with ID ${instanceId} is running`);
 		}
 
 		let instance = this.discoveredInstances.get(instanceId);
 		if (!instance) {
-			throw new lib.RequestError(`Instance with ID ${instanceId} does not exist`);
+			throw new lib.NotFoundError(`Instance with ID ${instanceId} does not exist`);
 		}
 
 		this.discoveredInstances.delete(instanceId);
@@ -1188,45 +1249,52 @@ export default class Host extends lib.Link {
 
 	async handleHostUpdateRequest(request: lib.HostUpdateRequest) {
 		if (!this.config.get("host.allow_remote_updates")) {
-			throw new lib.RequestError("Remote updates are disabled on this machine");
+			throw new lib.ForbiddenError("Remote updates are disabled on this machine");
 		}
 		return await lib.updatePackage("@clusterio/host");
 	}
 
 	async handlePluginUpdateRequest(request: lib.PluginUpdateRequest) {
 		if (!this.config.get("host.allow_plugin_updates")) {
-			throw new lib.RequestError("Plugin updates are disabled on this machine");
+			throw new lib.ForbiddenError("Plugin updates are disabled on this machine");
 		}
 		return await lib.handlePluginUpdate(request.pluginPackage, this.pluginInfos);
 	}
 
 	async handlePluginInstallRequest(request: lib.PluginInstallRequest) {
 		if (!this.config.get("host.allow_plugin_install")) {
-			throw new lib.RequestError("Plugin installs are disabled on this machine");
+			throw new lib.ForbiddenError("Plugin installs are disabled on this machine");
 		}
-		await lib.handlePluginInstall(request.pluginPackage);
-		// A new plugin is not in pluginInfos, so checkRestartRequired cannot notice it
-		if (!this.pluginInfos.some(pluginInfo => pluginInfo.npmPackage === request.pluginPackage)) {
+		const npmPackage = await lib.handlePluginInstall(request.pluginPackage);
+		// A new plugin is not in pluginInfos, so checkRestartRequired cannot notice it. Version
+		// changes to loaded plugins are noticed there.
+		if (!this.pluginInfos.some(pluginInfo => pluginInfo.npmPackage === npmPackage)) {
 			this.config.restartRequired = true;
 		}
 	}
 
 	async handleUpdateAllRequest(request: lib.UpdateAllRequest) {
 		if (!this.config.get("host.allow_remote_updates")) {
-			throw new lib.RequestError("Remote updates are disabled on this machine");
+			throw new lib.ForbiddenError("Remote updates are disabled on this machine");
 		}
 		if (!this.config.get("host.allow_plugin_updates")) {
-			throw new lib.RequestError("Plugin updates are disabled on this machine");
+			throw new lib.ForbiddenError("Plugin updates are disabled on this machine");
 		}
 		return await lib.handleUpdateAll("@clusterio/host", this.pluginInfos);
 	}
 
 	async handlePluginListRequest(request: lib.PluginListRequest) {
-		return this.pluginInfos.map(pluginInfo => lib.PluginDetails.fromNodeEnvInfo(
-			pluginInfo,
-			this.loadedPlugins.has(pluginInfo),
-			this.config.get(`${pluginInfo.name}.load_plugin`),
-		));
+		return await this.listPlugins();
+	}
+
+	/** Plugins on this host, including those installed or updated since it started */
+	async listPlugins() {
+		return await lib.listPluginDetails(
+			this.pluginInfos,
+			pluginInfo => this.loadedPlugins.has(pluginInfo),
+			pluginInfo => this.config.get(`${pluginInfo.name}.load_plugin`),
+			pluginInfo => Boolean(pluginInfo.hostEntrypoint || pluginInfo.instanceEntrypoint),
+		);
 	}
 
 	sendHostUpdate() {
@@ -1253,7 +1321,7 @@ export default class Host extends lib.Link {
 			let instanceConnection = this.instanceConnections.get(instanceId);
 			list.push(new lib.HostInstanceUpdate(
 				instance.config.toRemote("controller"),
-				instanceConnection ? instanceConnection.instance.status : "stopped",
+				instanceConnection ? instanceConnection.instance.status : this._unconnectedStatus(instanceId),
 				instanceConnection ? instanceConnection.instance.server.gamePort : this.gamePort(instanceId),
 				instanceConnection ? instanceConnection.instance.server.version : undefined,
 			));

packages/host/src/Instance.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs/promises";
 import path from "path";
 import pidusage from "pidusage";
+import stream from "node:stream";
 import util from "util";
 import type { Static } from "@sinclair/typebox";
 import { exec } from "child_process";
@@ -9,8 +10,8 @@ const execAsync = util.promisify(exec);
 // internal libraries
 import * as lib from "@clusterio/lib";
 
-import { FactorioServer } from "./server.js";
-import { loadModules, patch } from "./patch.js";
+import { FactorioServer, checkForUpdates } from "./server.js";
+import { loadModules, patch, unpatch } from "./patch.js";
 import { exportData } from "./export.js";
 import type Host from "./Host.js";
 import { BaseInstancePlugin, InstanceHooks } from "./BaseInstancePlugin.js";
@@ -20,7 +21,7 @@ function checkRequestSaveName(name: string) {
 	try {
 		lib.checkFilename(name);
 	} catch (err: any) {
-		throw new lib.RequestError(`Save name ${err.message}`);
+		throw new lib.ValidationError(`Save name ${err.message}`);
 	}
 }
 
@@ -31,6 +32,13 @@ const scriptCommands = [
 	"/silent-command", "/sc",
 ];
 
+function instanceLogger(config: lib.InstanceConfig) {
+	return lib.logger.child({
+		instance_id: config.get("instance.id"),
+		instance_name: config.get("instance.name"),
+	});
+}
+
 // Statuses the instance passes through on its way to running or stopped.
 const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
 	"starting", "stopping", "creating_save", "exporting_data",
@@ -182,6 +190,7 @@ export default class Instance extends lib.Link {
 	_loadedSave: string | null = null;
 	_playerCheckInterval: ReturnType<typeof setInterval> | undefined;
 	_hadPlayersOnline = false;
+	_autoPause = false;
 	_playerAutosaveSlot = 1;
 	_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
 	_settleWaiters: (() => void)[] = [];
@@ -190,8 +199,8 @@ export default class Instance extends lib.Link {
 		host: Host,
 		connector: lib.VirtualConnector,
 		dir: string,
-		factorioDir: string,
-		instanceConfig: lib.InstanceConfig
+		instanceConfig: lib.InstanceConfig,
+		server: FactorioServer,
 	) {
 		super(connector);
 		this._host = host;
@@ -201,10 +210,7 @@ export default class Instance extends lib.Link {
 
 		this.id = this.config.get("instance.id");
 
-		this.logger = lib.logger.child({
-			instance_id: this.id,
-			instance_name: this.name,
-		});
+		this.logger = instanceLogger(instanceConfig);
 
 		this.hooks = new InstanceHooks(this.logger);
 
@@ -224,26 +230,7 @@ export default class Instance extends lib.Link {
 		};
 		this.config.on("fieldChanged", this._configFieldChanged);
 
-		let serverOptions = {
-			logger: this.logger,
-			// TODO: factorio.version is not validated
-			version: this.config.get("factorio.version") as lib.TargetVersion,
-			executablePath: this.config.get("factorio.executable_path") ?? undefined,
-			gamePort: this.config.get("factorio.game_port") ?? host.assignGamePort(this.id),
-			rconPort: this.config.get("factorio.rcon_port") ?? undefined,
-			rconPassword: this.config.get("factorio.rcon_password") ?? undefined,
-			enableWhitelist: this.config.get("factorio.enable_whitelist"),
-			enableAuthserverBans: this.config.get("factorio.enable_authserver_bans"),
-			verboseLogging: this.config.get("factorio.verbose_logging"),
-			consoleLogging: this.config.get("factorio.console_logging"),
-			stripPaths: this.config.get("factorio.strip_paths"),
-			maxConcurrentCommands: this.config.get("factorio.max_concurrent_commands"),
-			shutdownTimeoutMs: this.config.get("factorio.shutdown_timeout") * 1000,
-		};
-
-		this.server = new FactorioServer(
-			factorioDir, this._dir, serverOptions
-		);
+		this.server = server;
 
 		this.server.on("output", (parsed, line) => {
 			this.logger.log("server", { message: line, instance_id: this.id, parsed });
@@ -402,10 +389,19 @@ export default class Instance extends lib.Link {
 		}
 	}
 
+	/**
+	 * True while Factorio has paused the game because auto_pause is on
+	 * and no players are online.
+	 */
+	get autoPaused() {
+		return this._autoPause && this.playersOnline.size === 0;
+	}
+
 	_recordPlayerJoin(name: string) {
 		if (this.playersOnline.has(name)) {
 			return;
 		}
+		const wasPaused = this.autoPaused;
 		this.playersOnline.add(name);
 
 		let stats = this.playerStats.get(name);
@@ -425,6 +421,9 @@ export default class Instance extends lib.Link {
 			stats,
 		};
 		this.sendTo("controller", new lib.InstancePlayerUpdateEvent("join", name, stats));
+		if (wasPaused) {
+			this.hooks.autoPauseChanged.invoke(false);
+		}
 		this.hooks.playerEvent.invoke(event);
 	}
 
@@ -446,6 +445,9 @@ export default class Instance extends lib.Link {
 			stats,
 		};
 		this.sendTo("controller", new lib.InstancePlayerUpdateEvent("leave", name, stats, reason));
+		if (this.autoPaused) {
+			this.hooks.autoPauseChanged.invoke(true);
+		}
 		this.hooks.playerEvent.invoke(event);
 	}
 
@@ -505,6 +507,23 @@ end`.replace(/\r?\n/g, " ");
 		}
 	}
 
+	/**
+	 * Send UDP message to the Factorio server
+	 *
+	 * Delivered in-game as an on_udp_packet_received event.  Requires
+	 * factorio.enable_lua_udp to be enabled on the instance.
+	 *
+	 * @param message - payload of the packet to send.
+	 */
+	async sendUdp(message: string | Buffer) {
+		if (!this.config.get("factorio.enable_lua_udp")) {
+			throw new Error(
+				"Attempted to send UDP message while Lua UDP is disabled. See config factorio.enable_lua_udp."
+			);
+		}
+		await this.server.sendUdp(message);
+	}
+
 	static async listSaves(instanceId: number, savesDir: string, loadedSave: string | null) {
 		let defaultSave = null;
 		if (loadedSave === null) {
@@ -625,6 +644,7 @@ end`.replace(/\r?\n/g, " ");
 
 	notifyExit() {
 		this._loadedSave = null;
+		this._autoPause = false;
 		this.notifyStatus("stopped");
 		this.connector.emit("close");
 
@@ -650,20 +670,22 @@ end`.replace(/\r?\n/g, " ");
 		this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
 	}
 
-	async _loadStats() {
+	static async _readStats(statsPath: string) {
 		let instanceStats;
 		try {
-			instanceStats = JSON.parse(await fs.readFile(this.path("instance-stats.json"), "utf8"));
+			instanceStats = JSON.parse(await fs.readFile(statsPath, "utf8"));
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
-				return;
+				return { playerStats: new Map<string, lib.PlayerStats>(), playerAutosaveSlot: 1 };
 			}
 			throw err;
 		}
-		this.playerStats = new Map(instanceStats["players"].map(
-			([id, stats]: [number, Static<typeof lib.PlayerStats.jsonSchema>]) => [id, new lib.PlayerStats(stats)])
-		);
-		this._playerAutosaveSlot = instanceStats["player_autosave_slot"] || 1;
+		return {
+			playerStats: new Map<string, lib.PlayerStats>(instanceStats["players"].map(
+				([id, stats]: [string, Static<typeof lib.PlayerStats.jsonSchema>]) => [id, new lib.PlayerStats(stats)]
+			)),
+			playerAutosaveSlot: (instanceStats["player_autosave_slot"] as number) || 1,
+		};
 	}
 
 	async _saveStats() {
@@ -671,24 +693,38 @@ end`.replace(/\r?\n/g, " ");
 			players: [...this.playerStats],
 			player_autosave_slot: this._playerAutosaveSlot,
 		}, null, "\t");
-		await lib.safeOutputFile(this.path("instance-stats.json"), content);
+		await lib.atomicOverwriteFile(this.path("instance-stats.json"), content);
 	}
 
-	async init(pluginInfos: lib.PluginNodeEnvInfo[]) {
-		// After changing the status this function is responsible for calling notifyExit if it throws.
-		this.notifyStatus("starting");
+	/**
+	 * Create an instance with a Factorio server, downloading Factorio if needed
+	 *
+	 * The host reports the starting status for the instance while this runs.
+	 */
+	static async fromConfig(
+		config: lib.InstanceConfig,
+		host: Host,
+		connector: lib.VirtualConnector,
+		dir: string,
+		factorioDir: string,
+	) {
+		const id = config.get("instance.id");
+		// TODO: factorio.version is not validated
+		const targetVersion = config.get("factorio.version") as lib.TargetVersion;
+		const gamePort = config.get("factorio.game_port") ?? await host.assignFreeGamePort(id);
+		host.send(new lib.InstanceStatusChangedEvent(id, "starting", gamePort, targetVersion));
+
+		let server: FactorioServer;
+		let stats: Awaited<ReturnType<typeof Instance._readStats>>;
 		try {
-			if (this.config.get("factorio.game_port") === null) {
-				this.server.gamePort = await this._host.assignFreeGamePort(this.id);
-			}
-			const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest());
-			await this._loadStats();
+			const factorioVersions = await host.sendTo("controller", new lib.FactorioVersionsRequest());
+			stats = await Instance._readStats(path.join(dir, "instance-stats.json"));
 			// Resolve a release channel target (e.g. "stable") to a concrete
 			// version using the controller's cached latest-releases data, so the
 			// existing download/lookup logic can fetch and run it.
-			const targetVersion = this.config.get("factorio.version") as string;
+			let version = targetVersion;
 			if (lib.isReleaseChannel(targetVersion)) {
-				const releases = await this.sendTo("controller", new lib.LatestReleasesRequest());
+				const releases = await host.sendTo("controller", new lib.LatestReleasesRequest());
 				const resolved = lib.resolveReleaseChannel(releases, targetVersion);
 				if (!resolved) {
 					throw new Error(
@@ -696,16 +732,46 @@ end`.replace(/\r?\n/g, " ");
 						"latest-releases unavailable and no cached copy on the controller."
 					);
 				}
-				this.server.setTargetVersion(resolved);
+				version = resolved;
 			}
-			await this.server.checkForUpdates(factorioVersions);
-			await this.server.init();
+			const logger = instanceLogger(config);
+			await checkForUpdates(factorioDir, version, factorioVersions, logger);
+			server = await FactorioServer.fromOptions(factorioDir, dir, {
+				logger,
+				version,
+				executablePath: config.get("factorio.executable_path") ?? undefined,
+				gamePort,
+				rconPort: config.get("factorio.rcon_port") ?? undefined,
+				rconPassword: config.get("factorio.rcon_password") ?? undefined,
+				enableLuaUdp: config.get("factorio.enable_lua_udp"),
+				luaUdpPort: config.get("factorio.lua_udp_port") ?? undefined,
+				enableWhitelist: config.get("factorio.enable_whitelist"),
+				enableAuthserverBans: config.get("factorio.enable_authserver_bans"),
+				verboseLogging: config.get("factorio.verbose_logging"),
+				consoleLogging: config.get("factorio.console_logging"),
+				stripPaths: config.get("factorio.strip_paths"),
+				maxConcurrentCommands: config.get("factorio.max_concurrent_commands"),
+				shutdownTimeoutMs: config.get("factorio.shutdown_timeout") * 1000,
+			});
 		} catch (err) {
-			this.notifyExit();
-			await this.sendSaveListUpdate();
+			host.send(new lib.InstanceStatusChangedEvent(id, "stopped", gamePort, targetVersion));
+			await host.sendSaveListUpdate(id, path.join(dir, "saves"));
 			throw err;
 		}
 
+		const instance = new this(host, connector, dir, config, server);
+		instance._status = "starting";
+		instance.playerStats = stats.playerStats;
+		instance._playerAutosaveSlot = stats.playerAutosaveSlot;
+		return instance;
+	}
+
+	/**
+	 * Load the instance plugins and report the instance as initialised
+	 *
+	 * Must be called once the instance is connected to the host.
+	 */
+	async loadPlugins(pluginInfos: lib.PluginNodeEnvInfo[]) {
 		// load plugins
 		const plugins: Record<string, string> = {};
 		for (let pluginInfo of pluginInfos) {
@@ -804,10 +870,11 @@ end`.replace(/\r?\n/g, " ");
 			"_comment_warning": warning,
 			...await this.resolveServerSettings(this.config.get("factorio.settings"), includeCredentials),
 		};
-		await lib.safeOutputFile(
+		await lib.atomicOverwriteFile(
 			this.server.writePath("server-settings.json"),
 			JSON.stringify(serverSettings, null, "\t")
 		);
+		return serverSettings;
 	}
 
 	/**
@@ -903,12 +970,12 @@ end`.replace(/\r?\n/g, " ");
 	 * Factorio refuses to read it if it is newer than the game.  It then
 	 * loads default settings instead, which can corrupt the save.
 	 *
-	 * @throws {lib.RequestError} if the mod pack version is higher than the server version.
+	 * @throws {lib.ConflictError} if the mod pack version is higher than the server version.
 	 */
 	checkModPackVersion(modPack: lib.ModPack) {
 		const serverVersion = this.server.version;
 		if (serverVersion && modPack.integerFactorioVersion > lib.integerFullVersion(serverVersion)) {
-			throw new lib.RequestError(
+			throw new lib.ConflictError(
 				`Mod pack ${modPack.name} is for Factorio ${modPack.factorioVersion} ` +
 				`which is newer than the Factorio ${serverVersion} this instance runs`
 			);
@@ -922,12 +989,14 @@ end`.replace(/\r?\n/g, " ");
 	 */
 	async prepare() {
 		this.logger.verbose("Writing server-settings.json");
-		await this.writeServerSettings(true);
+		// auto_pause is only read on startup
+		const serverSettings = await this.writeServerSettings(true);
+		this._autoPause = serverSettings.auto_pause === true;
 		this._expectedUserUpdates = [];
 
 		if (this.config.get("factorio.sync_adminlist") !== "disabled") {
 			this.logger.verbose("Writing server-adminlist.json");
-			await lib.safeOutputFile(
+			await lib.atomicOverwriteFile(
 				this.server.writePath("server-adminlist.json"),
 				JSON.stringify([...this._host.adminlist], null, "\t")
 			);
@@ -935,7 +1004,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_banlist") !== "disabled") {
 			this.logger.verbose("Writing server-banlist.json");
-			await lib.safeOutputFile(
+			await lib.atomicOverwriteFile(
 				this.server.writePath("server-banlist.json"),
 				JSON.stringify([...this._host.banlist].map(
 					([username, reason]) => ({ username, reason })
@@ -945,7 +1014,7 @@ end`.replace(/\r?\n/g, " ");
 
 		if (this.config.get("factorio.sync_whitelist") !== "disabled") {
 			this.logger.verbose("Writing server-whitelist.json");
-			await lib.safeOutputFile(
+			await lib.atomicOverwriteFile(
 				this.server.writePath("server-whitelist.json"),
 				JSON.stringify([...this._host.whitelist], null, "\t")
 			);
@@ -1013,6 +1082,9 @@ end`.replace(/\r?\n/g, " ");
 		}
 
 		if (!this.config.get("factorio.enable_save_patching")) {
+			if (await unpatch(this.path("saves", saveName))) {
+				this.logger.info("Removed Clusterio modules from save as save patching is disabled");
+			}
 			return saveName;
 		}
 
@@ -1089,7 +1161,8 @@ end`.replace(/\r?\n/g, " ");
 	 */
 	async updateInstanceData() {
 		let name = lib.escapeString(this.name);
-		await this.sendRcon(`/sc clusterio_private.update_instance(${this.id}, "${name}")`, true);
+		let hostUdpPort = this.server.hostUdpPort ?? "nil";
+		await this.sendRcon(`/sc clusterio_private.update_instance(${this.id}, "${name}", ${hostUdpPort})`, true);
 	}
 
 	async updateFactorioSettings(current: Record<string, unknown>, previous: Record<string, unknown>) {
@@ -1238,6 +1311,16 @@ end`.replace(/\r?\n/g, " ");
 		}
 	}
 
+	/** Stop the instance once a start, stop or save in progress has finished */
+	async _stopWhenSettled() {
+		// Stopping concurrently with these races them and asserts on the server state.
+		// Checked right before stop() as another waiter may have started stopping.
+		while (transitoryStatuses.has(this._status)) {
+			await this._waitForSettledStatus();
+		}
+		await this.stop();
+	}
+
 	async kill() {
 		if (this._status === "stopped") {
 			return;
@@ -1310,7 +1393,7 @@ end`.replace(/\r?\n/g, " ");
 	async handleInstanceLoadScenarioRequest(request: lib.InstanceLoadScenarioRequest) {
 		if (this.config.get("factorio.enable_save_patching")) {
 			this.notifyExit();
-			throw new lib.RequestError("Load scenario cannot be used with save patching enabled");
+			throw new lib.ConflictError("Load scenario cannot be used with save patching enabled");
 		}
 
 		try {
@@ -1372,7 +1455,7 @@ end`.replace(/\r?\n/g, " ");
 
 	async handleInstanceSaveGameRequest() {
 		if (this._status !== "running") {
-			throw new lib.RequestError("Instance is not running");
+			throw new lib.UnavailableError("Instance is not running");
 		}
 
 		const saved = new Promise<void>((resolve, reject) => {
@@ -1382,7 +1465,7 @@ end`.replace(/\r?\n/g, " ");
 			};
 			const onExit = () => {
 				this.server.off("save-finished", onSaved);
-				reject(new lib.RequestError("Instance stopped before the save finished"));
+				reject(new lib.UnavailableError("Instance stopped before the save finished"));
 			};
 			this.server.once("save-finished", onSaved);
 			this.server.once("exit", onExit);
@@ -1401,13 +1484,12 @@ end`.replace(/\r?\n/g, " ");
 			await this.syncMods();
 			let zip = await exportData(this.server);
 
-			let content = await zip.generateAsync({ type: "nodebuffer" });
 			let url = new URL(this._host.config.get("host.controller_url"));
 			url.pathname += "api/upload-export";
 			url.searchParams.set("mod_pack_id", String(this.activeModPack.id));
 			let response = await fetch(url, {
 				method: "PUT",
-				body: content,
+				body: stream.Readable.toWeb(lib.zipOutputStream(zip)),
 				duplex: "half",
 				headers: {
 					"Content-Type": "application/zip",
@@ -1425,7 +1507,7 @@ end`.replace(/\r?\n/g, " ");
 	}
 
 	async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) {
-		await this.stop();
+		await this._stopWhenSettled();
 		// Stopping invalidates this instance object, a new one must be created by the host
 		// To do this we simulate a waking an instance and sending a start request
 		const instanceConnection = await this._host._connectInstance(this.id);
@@ -1433,7 +1515,7 @@ end`.replace(/\r?\n/g, " ");
 	}
 
 	async handleInstanceStopRequest() {
-		await this.stop();
+		await this._stopWhenSettled();
 	}
 
 	async handleInstanceKillRequest() {

packages/host/src/create_scenario.ts

@@ -1,8 +1,9 @@
 // Implementation of the create-scenario command
 import fs from "node:fs/promises";
 import os from "node:os";
-import JSZip from "jszip";
 import path from "path";
+import { pipeline } from "node:stream/promises";
+import yazl from "yazl";
 
 import * as lib from "@clusterio/lib";
 import { logger } from "@clusterio/lib";
@@ -61,33 +62,50 @@ export async function createScenario(
 
 async function zipDirectory(dirPath: string, zipPath: string) {
 	const root = path.basename(path.resolve(dirPath));
-	const zip = new JSZip();
-	let dirs: [string, string][] = [[dirPath, root]];
-	while (dirs.length) {
-		let [dir, relativeDir] = dirs.pop()!;
-		for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
-			let fsPath = path.join(dir, entry.name);
-			let relativePath = path.posix.join(relativeDir, entry.name);
-			if (entry.isFile()) {
-				zip.file(relativePath, await fs.readFile(fsPath));
-			} else if (entry.isDirectory()) {
-				dirs.push([fsPath, relativePath]);
+	const zip = new yazl.ZipFile();
+	const output = lib.zipOutputStream(zip);
+	const fd = await fs.open(zipPath, "w");
+	// addFile() can fail before the traversal is done, so start writing first.
+	const writing = pipeline(output, fd.createWriteStream());
+	writing.catch(() => {});
+	try {
+		let dirs: [string, string][] = [[dirPath, root]];
+		while (dirs.length) {
+			let [dir, relativeDir] = dirs.pop()!;
+			for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
+				let fsPath = path.join(dir, entry.name);
+				let relativePath = path.posix.join(relativeDir, entry.name);
+				if (entry.isFile()) {
+					zip.addFile(fsPath, relativePath);
+				} else if (entry.isDirectory()) {
+					dirs.push([fsPath, relativePath]);
+				}
 			}
 		}
+		zip.end();
+	} catch (err) {
+		output.destroy();
+		await writing.catch(() => {});
+		throw err;
 	}
-	await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" }));
+	await writing;
 }
 
 async function extractZip(zipPath: string, dirPath: string) {
-	const zip = await JSZip.loadAsync(await fs.readFile(zipPath));
-	const root = lib.findRoot(zip);
-	for (let file of Object.values(zip.files)) {
-		if (file.dir || !file.name.startsWith(`${root}/`)) {
-			continue;
+	const zip = await lib.ZipArchive.fromFile(zipPath);
+	try {
+		const root = lib.findRoot(zip);
+		for (let entry of zip.entries.values()) {
+			if (entry.fileName.endsWith("/") || !entry.fileName.startsWith(`${root}/`)) {
+				continue;
+			}
+			const fsPath = path.join(dirPath, entry.fileName.slice(root.length + 1));
+			await fs.mkdir(path.dirname(fsPath), { recursive: true });
+			const fd = await fs.open(fsPath, "w");
+			await pipeline(await zip.openStream(entry), fd.createWriteStream());
 		}
-		const fsPath = path.join(dirPath, file.name.slice(root.length + 1));
-		await fs.mkdir(path.dirname(fsPath), { recursive: true });
-		await fs.writeFile(fsPath, await file.async("nodebuffer"));
+	} finally {
+		zip.close();
 	}
 }
 

packages/host/src/export.ts

@@ -1,12 +1,16 @@
 // Export of item icons and locale
 import fs from "node:fs/promises";
 import path from "path";
-import { Jimp, JimpMime, type JimpInstance } from "jimp";
-import JSZip from "jszip";
+import { createJimp } from "@jimp/core";
+import png from "@jimp/js-png";
+import * as crop from "@jimp/plugin-crop";
+import * as resize from "@jimp/plugin-resize";
+import type {} from "@jimp/types"; // needed to name jimp types in export.d.ts
+import yazl from "yazl";
 
 import * as lib from "@clusterio/lib";
 import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js";
-import type { FactorioServer } from "./server.js";
+import { findVersion, isHeadless, type FactorioServer } from "./server.js";
 import { definesPrototypesFallback } from "./export_fallback.js";
 
 interface SimpleIconSpecification {
@@ -64,21 +68,48 @@ async function generateExportMod(server: FactorioServer) {
 	});
 }
 
-let zipCache = new Map<string, JSZip>();
-async function loadZip(server: FactorioServer, modVersions: Map<string, string>, mod: string) {
+interface ModZip {
+	zip: lib.ZipArchive;
+	root: string;
+}
+type ZipCache = Map<string, ModZip>;
+async function loadZip(server: FactorioServer, modVersions: Map<string, string>, zipCache: ZipCache, mod: string) {
 	let modVersion = modVersions.get(mod);
 	if (!modVersion) {
 		throw new Error(`Got path for unknown mod ${mod}`);
 	}
 
 	let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`);
-	let zip = zipCache.get(zipPath);
-	if (!zip) {
-		zip = await JSZip.loadAsync(await fs.readFile(zipPath));
-		zipCache.set(zipPath, zip);
+	let modZip = zipCache.get(zipPath);
+	if (!modZip) {
+		const zip = await lib.ZipArchive.fromFile(zipPath);
+		try {
+			modZip = { zip, root: lib.findRoot(zip) };
+		} catch (err) {
+			zip.close();
+			throw err;
+		}
+		zipCache.set(zipPath, modZip);
 	}
 
-	return zip.folder(lib.findRoot(zip))!;
+	return modZip;
+}
+
+/** Where icon files are read from during an export. */
+interface ExportFiles {
+	/** Mod zips opened during the export, closed when it ends. */
+	zipCache: ZipCache;
+	/** Data dir of a full install to fall back to for builtin mod files. */
+	graphicsDataDir?: string;
+	/** Why no full install was found for a headless server. */
+	graphicsError?: string;
+}
+
+function closeZipCache(zipCache: ZipCache) {
+	for (const { zip } of zipCache.values()) {
+		zip.close();
+	}
+	zipCache.clear();
 }
 
 /**
@@ -86,11 +117,17 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
  *
  * @param server - The server to load the file from.
  * @param modVersions - Mapping of mod to version used.
+ * @param files - Where to read files from.
  * @param modPath - Factorio style path to the file to load.
  * @returns The content of the file or null if not found.
  * @internal
  */
-async function loadFile(server: FactorioServer, modVersions: Map<string, string>, modPath: string) {
+async function loadFile(
+	server: FactorioServer,
+	modVersions: Map<string, string>,
+	files: ExportFiles,
+	modPath: string,
+) {
 	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
 	if (!match) {
 		server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`);
@@ -101,19 +138,25 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
 
 	const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)];
 	if (builtinModNames.includes(mod)) {
-		try {
-			return await fs.readFile(server.dataPath(mod, filePath));
-		} catch (err: any) {
-			if (err.code === "ENOENT") {
-				return null;
-			}
-			throw err;
+		const dataDirs = [server.dataPath()];
+		if (files.graphicsDataDir) {
+			dataDirs.push(files.graphicsDataDir);
 		}
+		for (const dataDir of dataDirs) {
+			try {
+				return await fs.readFile(path.join(dataDir, mod, filePath));
+			} catch (err: any) {
+				if (err.code !== "ENOENT") {
+					throw err;
+				}
+			}
+		}
+		return null;
 	}
 
-	let zip;
+	let modZip;
 	try {
-		zip = await loadZip(server, modVersions, mod);
+		modZip = await loadZip(server, modVersions, files.zipCache, mod);
 	} catch (err: any) {
 		if (err.code === "ENOENT") {
 			return null;
@@ -121,19 +164,17 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
 		throw err;
 	}
 
-	let file = zip.file(filePath);
-	if (!file) {
-		return null;
-	}
-
-	return await file.async("nodebuffer");
+	return await modZip.zip.readFile(path.posix.join(modZip.root, filePath));
 }
 
-type Image = JimpInstance;
+// Only the parts of jimp the export uses
+const Jimp = createJimp({ formats: [png], plugins: [crop.methods, resize.methods] });
+type Image = InstanceType<typeof Jimp>;
 type IconCache = Map<string, Image | null>;
 async function loadIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	files: ExportFiles,
 	iconPath: string,
 	iconSize: number,
 	iconMipmaps: number,
@@ -141,7 +182,7 @@ async function loadIcon(
 ) {
 	let icon = iconCache.get(iconPath);
 	if (icon === undefined) {
-		let fileContent = await loadFile(server, modVersions, iconPath);
+		let fileContent = await loadFile(server, modVersions, files, iconPath);
 		if (fileContent) {
 			icon = await Jimp.fromBuffer(fileContent) as Image;
 			icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize });
@@ -157,11 +198,12 @@ async function loadIcon(
 async function loadSimpleIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	files: ExportFiles,
 	item: SimpleIconSpecification,
 	size: number,
 	iconCache: IconCache,
 ): Promise<Image | null> {
-	let icon = await loadIcon(server, modVersions, item.icon, item.icon_size ?? 64, 0, iconCache);
+	let icon = await loadIcon(server, modVersions, files, item.icon, item.icon_size ?? 64, 0, iconCache);
 	if (icon) {
 		let iconScale = size / (item.icon_size ?? 64);
 		if (iconScale !== 1) {
@@ -175,10 +217,11 @@ async function loadSimpleIcon(
 async function loadLayeredIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	files: ExportFiles,
 	item: LayeredIconSpecification,
 	size: number,
 	iconCache: IconCache,
-): Promise<Image> {
+): Promise<Image | null> {
 	let baseLayerSize = (item.icons[0].icon_size || item.icon_size) ?? 64;
 	let icon = new Jimp({ width: size, height: size });
 
@@ -190,10 +233,13 @@ async function loadLayeredIcon(
 
 	for (let layer of item.icons) {
 		let layerSize = (layer.icon_size || item.icon_size) ?? 64;
-		let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
+		let iconLayer = await loadIcon(
+			server, modVersions, files, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
+		);
 
+		// A partial icon would be wrong, skip it like a missing simple icon
 		if (!iconLayer) {
-			continue;
+			return null;
 		}
 
 		iconLayer = iconLayer.clone();
@@ -308,6 +354,7 @@ async function growSheet(state: SheetState) {
 async function exportIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	files: ExportFiles,
 	state: SheetState,
 	prototype: PrototypeWithIcon,
 ) {
@@ -318,9 +365,11 @@ async function exportIcon(
 
 	if (prototype.icons) {
 		icon = await loadLayeredIcon(
-			server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
+			server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache,
 		);
-		iconPos = state.pos;
+		if (icon) {
+			iconPos = state.pos;
+		}
 		iconPath = prototype.icons[0].icon;
 
 	} else if (prototype.icon) {
@@ -328,7 +377,7 @@ async function exportIcon(
 		iconPath = prototype.icon;
 		if (iconPos === undefined) {
 			icon = await loadSimpleIcon(
-				server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
+				server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache,
 			);
 			if (icon) {
 				iconPos = state.pos;
@@ -364,6 +413,7 @@ async function exportIcon(
  *
  * @param server - The server to load icons from.
  * @param modVersions - Mapping of mod name to version.
+ * @param files - Where to read icon files from.
  * @param prototypes - Array of prototypes with icon to export for this base type.
  * @param baseName - name of base type (used in metadata).
  * @param state - Shared spritesheet state (mutated in place).
@@ -373,6 +423,7 @@ async function exportIcon(
 async function exportIcons(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	files: ExportFiles,
 	prototypes: PrototypeWithIcon[],
 	baseName: string,
 	state: SheetState,
@@ -381,7 +432,13 @@ async function exportIcons(
 	state.metadata[baseName] ??= metadata;
 	let count = 0;
 	for (let prototype of prototypes) {
-		const icon = await exportIcon(server, modVersions, state, prototype);
+		let icon: lib.ExportMetadataEntry["icon"];
+		try {
+			icon = await exportIcon(server, modVersions, files, state, prototype);
+		} catch (err: any) {
+			// One broken icon should not stop the rest of the export
+			server._logger.warn(`Skipping icon of ${prototype.type} ${prototype.name}: ${err.message}`);
+		}
 		if (icon) {
 			count += 1;
 		}
@@ -404,6 +461,7 @@ async function exportIcons(
  *
  * @param server - The server to export the locale from.
  * @param modVersions - Mapping of mod name to version to export locale from.
+ * @param zipCache - Mod zips opened by this export.
  * @param modOrder - Load order of the mods.
  * @param languageCode - Language to export locale for.
  * @returns merged locale information
@@ -412,6 +470,7 @@ async function exportIcons(
 async function exportLocale(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	zipCache: ZipCache,
 	modOrder: string[],
 	languageCode: string
 ) {
@@ -445,18 +504,22 @@ async function exportLocale(
 			continue;
 		}
 
-		let zip;
+		let modZip;
 		try {
-			zip = await loadZip(server, modVersions, mod);
+			modZip = await loadZip(server, modVersions, zipCache, mod);
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
+				server._logger.warn(`Skipping locale of ${mod}, its mod zip was not found`);
 				continue;
 			}
 			throw err;
 		}
-		for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) {
-			let content = await file.async("nodebuffer");
-			mergeLocale(lib.parse(content.toString("utf8")));
+		const localePath = `${path.posix.join(modZip.root, "locale", languageCode)}/`;
+		for (const entry of modZip.zip.entries.values()) {
+			if (entry.fileName.startsWith(localePath) && entry.fileName.endsWith(".cfg")) {
+				let content = await modZip.zip.readEntry(entry);
+				mergeLocale(lib.parse(content.toString("utf8")));
+			}
 		}
 	}
 
@@ -466,10 +529,67 @@ async function exportLocale(
 /**
  * Export locale, icons and metadata for the given factorio server
  *
+ * If the server runs the headless build, builtin mod graphics are read
+ * from a full install of the same major and minor version found next to
+ * it, if there is one.
+ *
  * @param server - The server to export the data from.
  * @returns zip file with exported data.
  */
 export async function exportData(server: FactorioServer) {
+	const files: ExportFiles = { zipCache: new Map() };
+	if (await isHeadless(server.dataPath())) {
+		const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion;
+		try {
+			[files.graphicsDataDir] = await findVersion(server._factorioDir, majorMinor, false);
+		} catch (err: any) {
+			// Without it the export still has everything but the base game icons
+			files.graphicsError = err.message;
+		}
+	}
+	try {
+		return await buildExport(server, files);
+	} finally {
+		// Free up the zip files loaded during the export.
+		closeZipCache(files.zipCache);
+	}
+}
+
+/**
+ * Warn if any builtin mod icon files could not be found
+ *
+ * @param server - The server the export ran on.
+ * @param files - Where icon files were read from.
+ * @param iconCache - Icons loaded by the export, null for missing files.
+ * @internal
+ */
+function checkMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, iconCache: IconCache) {
+	const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)];
+	let missing = 0;
+	for (const [iconPath, icon] of iconCache) {
+		const mod = /^__([^\/]+)__\//.exec(iconPath)?.[1];
+		if (icon === null && mod && builtinModNames.includes(mod)) {
+			missing += 1;
+		}
+	}
+	if (!missing) {
+		return;
+	}
+
+	const dirs = [server.dataPath(), files.graphicsDataDir].filter(Boolean).join(" or ");
+	let message =
+		`${missing} icon files of the base game were not found in ${dirs}, icons using them were not exported.`;
+	if (files.graphicsError !== undefined) {
+		const majorMinor = server.version!.replace(/\.\d+$/, "");
+		message +=
+			` The headless build of Factorio has no graphics and no full install was found (${files.graphicsError}).` +
+			` Add a full install of Factorio ${majorMinor} to host.factorio_directory to read them from there,` +
+			" or export the data on a host with a full install.";
+	}
+	server._logger.warn(message);
+}
+
+async function buildExport(server: FactorioServer, files: ExportFiles) {
 	await generateExportMod(server);
 
 	let settings: lib.ExportSettings = {};
@@ -549,16 +669,14 @@ export async function exportData(server: FactorioServer) {
 		}
 	}
 
-	let locale = await exportLocale(server, modVersions, modOrder, "en");
+	let locale = await exportLocale(server, modVersions, files.zipCache, modOrder, "en");
 
-	// Free up the memory used by zip files loaded during the export.
-	zipCache.clear();
-
-	let zip = new JSZip();
-	zip.file("export/settings.json", JSON.stringify(settings satisfies lib.ExportSettings));
-	zip.file("export/prototypes.json", JSON.stringify(prototypes satisfies lib.ExportPrototypes));
-	zip.file("export/locale.json", JSON.stringify([...locale.entries()] satisfies lib.ExportLocale));
-	zip.file("export/defines.json", JSON.stringify(defines satisfies lib.ExportDefines));
+	let zip = new yazl.ZipFile();
+	const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name);
+	addJson("export/settings.json", settings satisfies lib.ExportSettings);
+	addJson("export/prototypes.json", prototypes satisfies lib.ExportPrototypes);
+	addJson("export/locale.json", [...locale.entries()] satisfies lib.ExportLocale);
+	addJson("export/defines.json", defines satisfies lib.ExportDefines);
 
 	// Build a single unified spritesheet across all prototypes with icons.
 	const state = await createSheetState();
@@ -572,15 +690,17 @@ export async function exportData(server: FactorioServer) {
 		if (prototypesWithIcons.length === 0) {
 			continue;
 		}
-		const count = await exportIcons(server, modVersions, prototypesWithIcons, baseName, state);
+		const count = await exportIcons(server, modVersions, files, prototypesWithIcons, baseName, state);
 		server._logger.info(`Exported ${count} ${baseName} icons`);
 	}
+	checkMissingBuiltinIcons(server, files, state.iconCache);
 
 	// Crop sheet to actual used height and write single spritesheet + metadata.
 	const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
 	state.sheet.crop({ x: 0, y: 0, w: SHEET_WIDTH, h: Math.max(usedRows * SHEET_ICON_SIZE, 1) });
-	zip.file("export/spritesheet.png", await state.sheet.getBuffer(JimpMime.png));
-	zip.file("export/metadata.json", JSON.stringify(state.metadata satisfies lib.ExportMetadata));
+	zip.addBuffer(await state.sheet.getBuffer("image/png"), "export/spritesheet.png");
+	addJson("export/metadata.json", state.metadata satisfies lib.ExportMetadata);
+	zip.end();
 
 	server._logger.info(`Export complete: ${state.pos} icons on ${usedRows} row(s)`);
 
@@ -589,6 +709,9 @@ export async function exportData(server: FactorioServer) {
 
 
 // For testing only
+export const _createSheetState = createSheetState;
+export const _exportIcons = exportIcons;
 export const _exportLocale = exportLocale;
 export const _loadSimpleIcon = loadSimpleIcon;
 export const _loadLayeredIcon = loadLayeredIcon;
+export const _checkMissingBuiltinIcons = checkMissingBuiltinIcons;

packages/host/src/pairing.ts

@@ -0,0 +1,305 @@
+import { setTimeout as sleep } from "node:timers/promises";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+
+/** Time to wait for a response from the controller */
+const requestTimeoutMs = 30e3;
+/** Placeholder token of a host that has not been paired */
+const unpairedToken = "enter token here";
+
+/** Error responded by the controller */
+export class PairingError extends Error {
+	constructor(
+		message: string,
+		/** HTTP status the controller responded with */
+		public status?: number,
+	) {
+		super(message);
+		this.name = "PairingError";
+	}
+}
+
+export interface PairingResult {
+	hostId: number;
+	token: string;
+	name: string;
+}
+
+export interface PairHostOptions {
+	/** URL of the controller, ending with a slash */
+	controllerUrl: string;
+	/** Name to request for the host */
+	name: string;
+	/** Version of the host, shown with the request */
+	version?: string;
+	/** Token of a user allowed to approve hosts, approves the host without waiting */
+	token?: string;
+	/** Interval to poll for the result at */
+	pollIntervalMs?: number;
+	/** Delay before trying again when the controller can't be reached or is busy */
+	retryDelayMs?: number;
+	/** Called with the code to approve the request with each time a request is made */
+	onPending?: (code: string) => void;
+	/** Called when the controller can't be reached or is busy and the request is retried */
+	onRetry?: (err: Error, delayMs: number) => void;
+	/** Signal to cancel pairing with */
+	signal?: AbortSignal;
+}
+
+interface Response {
+	status: number;
+	headers: Headers;
+	json: any;
+}
+
+async function request(
+	controllerUrl: string,
+	route: string,
+	options: { body?: object, token?: string, signal?: AbortSignal, timeoutMs?: number },
+): Promise<Response> {
+	const headers: Record<string, string> = { "Accept": "application/json" };
+	if (options.body) {
+		headers["Content-Type"] = "application/json";
+	}
+	if (options.token) {
+		headers["Authorization"] = `Bearer ${options.token}`;
+	}
+	const timeout = AbortSignal.timeout(options.timeoutMs ?? requestTimeoutMs);
+	const response = await fetch(new URL(route, controllerUrl), {
+		method: options.body ? "POST" : "GET",
+		headers,
+		body: options.body ? JSON.stringify(options.body) : undefined,
+		signal: options.signal ? AbortSignal.any([options.signal, timeout]) : timeout,
+	});
+	let json;
+	try {
+		json = await response.json();
+	} catch {
+		json = undefined;
+	}
+	return { status: response.status, headers: response.headers, json };
+}
+
+function responseError(response: Response) {
+	const message = typeof response.json?.error === "string"
+		? response.json.error
+		: `Unexpected response from controller: ${response.status}`;
+	return new PairingError(message, response.status);
+}
+
+function pairingResult(response: Response): PairingResult {
+	const { hostId, token, name } = response.json ?? {};
+	if (!Number.isInteger(hostId) || typeof token !== "string" || typeof name !== "string") {
+		throw new PairingError("Invalid response from controller", response.status);
+	}
+	return { hostId, token, name };
+}
+
+/** Responses that are worth trying again later */
+function isTemporary(status: number) {
+	return status === 429 || status >= 500;
+}
+
+function retryDelay(response: Response, defaultMs: number) {
+	const seconds = Number(response.headers.get("retry-after"));
+	return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : defaultMs;
+}
+
+/**
+ * Poll a request until it is approved, rejected or no longer known by the controller
+ *
+ * @returns the result if approved, or null if the controller no longer knows the request.
+ */
+async function poll(
+	requestId: string,
+	options: Required<Pick<PairHostOptions, "pollIntervalMs" | "retryDelayMs">> & PairHostOptions,
+) {
+	let delayMs = options.pollIntervalMs;
+	while (true) {
+		await sleep(delayMs, undefined, { signal: options.signal });
+		delayMs = options.pollIntervalMs;
+		let response;
+		try {
+			response = await request(options.controllerUrl, "api/host-pairing/poll", {
+				body: { requestId }, signal: options.signal,
+			});
+		} catch (err: any) {
+			if (options.signal?.aborted) {
+				throw err;
+			}
+			// Keep polling through network errors, the request is made again if it expired meanwhile.
+			delayMs = options.retryDelayMs;
+			options.onRetry?.(err, delayMs);
+			continue;
+		}
+		if (response.status === 200) {
+			return pairingResult(response);
+		}
+		if (response.status === 404) {
+			return null;
+		}
+		if (isTemporary(response.status)) {
+			delayMs = retryDelay(response, options.retryDelayMs);
+			options.onRetry?.(responseError(response), delayMs);
+			continue;
+		}
+		if (response.status !== 202) {
+			throw responseError(response);
+		}
+	}
+}
+
+/**
+ * Request to join the cluster and wait until the request is approved
+ *
+ * Waits through network errors and controller restarts, making a new request
+ * if the controller no longer knows the previous one.
+ *
+ * @param options - Pairing options.
+ * @returns id, token and approved name of the host.
+ * @throws {PairingError} if the controller refuses or rejects the request.
+ * @throws {DOMException} AbortError if the signal is aborted.
+ */
+export async function pairHost(options: PairHostOptions): Promise<PairingResult> {
+	const opts = { pollIntervalMs: 2000, retryDelayMs: 10e3, version: "", ...options };
+	let requestId: string | undefined;
+	try {
+		while (true) {
+			opts.signal?.throwIfAborted();
+			let response;
+			try {
+				response = await request(opts.controllerUrl, "api/host-pairing", {
+					body: { name: opts.name, version: opts.version }, token: opts.token, signal: opts.signal,
+				});
+			} catch (err: any) {
+				if (opts.signal?.aborted) {
+					throw err;
+				}
+				opts.onRetry?.(err, opts.retryDelayMs);
+				await sleep(opts.retryDelayMs, undefined, { signal: opts.signal });
+				continue;
+			}
+
+			if (response.status === 200) {
+				return pairingResult(response);
+			}
+			if (isTemporary(response.status)) {
+				const delayMs = retryDelay(response, opts.retryDelayMs);
+				opts.onRetry?.(responseError(response), delayMs);
+				await sleep(delayMs, undefined, { signal: opts.signal });
+				continue;
+			}
+			if (response.status === 404) {
+				throw new PairingError("The controller must be updated to a version that supports host pairing", 404);
+			}
+			if (response.status !== 202) {
+				throw responseError(response);
+			}
+			if (typeof response.json?.requestId !== "string" || typeof response.json.code !== "string") {
+				throw new PairingError("Invalid response from controller", response.status);
+			}
+
+			requestId = response.json.requestId as string;
+			opts.onPending?.(response.json.code);
+			const result = await poll(requestId, opts);
+			requestId = undefined;
+			if (result) {
+				return result;
+			}
+		}
+
+	} catch (err) {
+		if (!opts.signal?.aborted) {
+			throw err;
+		}
+		if (requestId !== undefined) {
+			await request(opts.controllerUrl, "api/host-pairing/cancel", {
+				body: { requestId }, timeoutMs: 5e3,
+			}).catch(() => {});
+		}
+		throw new DOMException("Pairing was cancelled", "AbortError");
+	}
+}
+
+/**
+ * Check if the controller accepts a host token
+ *
+ * @param controllerUrl - URL of the controller, ending with a slash.
+ * @param token - Host token to check.
+ * @param signal - Signal to cancel the check with.
+ * @returns true if the token is valid and its host is registered.
+ * @throws {PairingError} if the controller gives an unexpected response.
+ */
+export async function checkHostToken(controllerUrl: string, token: string, signal?: AbortSignal) {
+	const response = await request(controllerUrl, "api/host-pairing/check", { token, signal });
+	if (response.status === 200) {
+		return true;
+	}
+	if (response.status === 401) {
+		return false;
+	}
+	throw responseError(response);
+}
+
+/**
+ * Pair the host with the controller if it has not been paired, or the controller no longer accepts it
+ *
+ * Waits until the request is approved and saves the id, name and token of the host to its config.
+ *
+ * @param config - Config of the host.
+ * @param options - Pairing options.
+ * @throws {PairingError} if the controller refuses or rejects the request.
+ * @throws {DOMException} AbortError if the signal is aborted.
+ */
+export async function ensurePaired(
+	config: lib.HostConfig,
+	options: { version: string, userToken?: string, signal?: AbortSignal },
+) {
+	const controllerUrl = config.get("host.controller_url");
+	const token = config.get("host.controller_token");
+	if (token && token !== unpairedToken) {
+		let valid;
+		try {
+			valid = await checkHostToken(controllerUrl, token, options.signal);
+		} catch (err: any) {
+			if (options.signal?.aborted) {
+				throw err;
+			}
+			// Connecting reports the problem, or authenticates if the controller predates pairing.
+			logger.verbose(`Unable to check if the controller accepts this host: ${err.message}`);
+			return;
+		}
+		if (valid) {
+			return;
+		}
+		logger.warn("The controller no longer accepts this host, requesting to join again");
+	}
+
+	const name = config.get("host.name");
+	logger.info(`Requesting to join the cluster at ${controllerUrl} as "${name}"`);
+	const result = await pairHost({
+		controllerUrl,
+		name,
+		version: options.version,
+		token: options.userToken,
+		signal: options.signal,
+		onPending: code => {
+			const approveUrl = new URL(`hosts?approve=${code}`, controllerUrl).href;
+			// Logged as warnings for the systemd service, which only prints warnings and above.
+			// The link is logged on its own so terminals can make it clickable without it wrapping.
+			logger.warn(`Waiting for approval of "${name}" with code ${code}, open this link to approve it:`);
+			logger.warn(approveUrl);
+			logger.warn(`Or run: npx clusterioctl host approve ${code}`);
+		},
+		onRetry: (err, delayMs) => {
+			logger.warn(`Requesting to join failed: ${err.message}, trying again in ${Math.ceil(delayMs / 1000)}s`);
+		},
+	});
+
+	config.set("host.id", result.hostId);
+	config.set("host.name", result.name);
+	config.set("host.controller_token", result.token);
+	await config.save();
+	logger.info(`Host approved as "${result.name}" with id ${result.hostId}`);
+}

packages/host/src/patch.ts

@@ -1,10 +1,9 @@
 // Library for patching Factorio saves with scenario code.
 
-import events from "events";
 import fs from "node:fs/promises";
-import JSZip from "jszip";
 import path from "path";
 import semver from "semver";
+import { pipeline } from "node:stream/promises";
 import { Type, Static } from "@sinclair/typebox";
 
 import * as lib from "@clusterio/lib";
@@ -72,18 +71,16 @@ export class SaveModule {
 		};
 	}
 
-	static async fromSave(json: Static<typeof SaveModule.jsonSchema>, root: JSZip) {
+	static async fromSave(json: Static<typeof SaveModule.jsonSchema>, zip: lib.ZipArchive, root: string) {
 		const module = new this(lib.ModuleInfo.fromJSON(json));
-		module.files = new Map(await Promise.all(json.files
-			.map(filename => ({filename, file: root.file(filename)}))
-			.filter(({filename, file}) => {
-				if (file === null) {
-					lib.logger.warn(`Missing file ${filename} in save`);
-				}
-				return file !== null;
-			})
-			.map(async ({filename, file}) => [filename, await file!.async("nodebuffer")] as const)
-		));
+		for (const filename of json.files) {
+			const content = await zip.readFile(path.posix.join(root, filename));
+			if (content === null) {
+				lib.logger.warn(`Missing file ${filename} in save`);
+				continue;
+			}
+			module.files.set(filename, content);
+		}
 		return module;
 	}
 
@@ -205,7 +202,7 @@ export class PatchInfo {
 		};
 	}
 
-	static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, root: JSZip) {
+	static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, zip: lib.ZipArchive, root: string) {
 		if (json.version === undefined) {
 			interface ScenarioInfoV0 {
 				name: string;
@@ -229,7 +226,8 @@ export class PatchInfo {
 		return new this(
 			json.patch_number,
 			lib.ModuleInfo.fromJSON(json.scenario),
-			await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))),
+			await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))),
+			json.version,
 		);
 	}
 }
@@ -384,6 +382,72 @@ const knownScenarios: Record<string, lib.ModuleInfo> = {
 	"3af547d2f4db3728b75ac5d1b3a4f47830dc48e7": new lib.ModuleInfo("freeplay", "2.0.29", [], ["scenario"]),
 };
 
+/**
+ * Names of the zip entries added by patching for the modules in the patch info
+ */
+function moduleEntries(zip: lib.ZipArchive, patchInfo: PatchInfo, root: string) {
+	const entries = new Set<string>();
+	if (patchInfo.version === 0) {
+		const modulesPath = path.posix.join(root, "modules/");
+		for (const name of zip.entries.keys()) {
+			if (name.startsWith(modulesPath)) {
+				entries.add(name);
+			}
+		}
+	} else {
+		for (const module of patchInfo.modules) {
+			for (const filepath of module.files.keys()) {
+				entries.add(path.posix.join(root, filepath));
+			}
+		}
+	}
+	return entries;
+}
+
+/**
+ * Write a copy of the save next to it with entries removed and added
+ *
+ * Keeps the order of the entries in the original.
+ *
+ * @returns Path to the temporary file written.
+ */
+async function rewriteSave(
+	zip: lib.ZipArchive, savePath: string, removed: Set<string>, added: Map<string, Buffer>
+) {
+	added = new Map(added);
+	const output = new lib.ZipWriter();
+	const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
+	const writing = pipeline(output.outputStream, writeStream);
+	// Awaited below, but a write error can reject it before then.
+	writing.catch(() => {});
+	try {
+		for (const entry of zip.entries.values()) {
+			const name = entry.fileName;
+			if (removed.has(name)) {
+				continue;
+			}
+			const contents = added.get(name);
+			if (contents !== undefined) {
+				await output.addBuffer(contents, name);
+				added.delete(name);
+			} else {
+				await output.addEntry(zip, entry);
+			}
+		}
+		for (const [name, contents] of added) {
+			await output.addBuffer(contents, name);
+		}
+		await output.end();
+		await writing;
+	} catch (err) {
+		output.outputStream.destroy();
+		await writing.catch(() => {});
+		await fs.rm(tempSavePath, { force: true });
+		throw err;
+	}
+	return tempSavePath;
+}
+
 /**
  * Patch a save with the given modules
  *
@@ -396,82 +460,128 @@ const knownScenarios: Record<string, lib.ModuleInfo> = {
  * @param modules - Description of the modules to patch.
  */
 export async function patch(savePath: string, modules: SaveModule[]) {
-	let zip = await JSZip.loadAsync(await fs.readFile(savePath));
-	let root = zip.folder(lib.findRoot(zip))!;
+	const zip = await lib.ZipArchive.fromFile(savePath);
+	let tempSavePath: string;
+	try {
+		const root = lib.findRoot(zip);
+		const rootPath = (relativePath: string) => path.posix.join(root, relativePath);
 
-	let patchInfoFile = root.file("clusterio.json");
-	let patchInfo: PatchInfo;
-	if (patchInfoFile !== null) {
-		let content = await patchInfoFile.async("string");
-		patchInfo = await PatchInfo.fromSave(JSON.parse(content), root);
+		// Files to add or replace in the save, relative to the root
+		const files = new Map<string, Buffer>();
+		let patchInfo: PatchInfo;
+		const patchInfoContent = await zip.readFile(rootPath("clusterio.json"));
+		if (patchInfoContent !== null) {
+			patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root);
+			if (patchInfo.version > PatchInfo.currentVersion) {
+				throw new Error(
+					`Save patch version ${patchInfo.version} is newer than the patch version this ` +
+					`version of Clusterio can load (${PatchInfo.currentVersion})`
+				);
+			}
+
+			// Unpatching put the original control.lua back in place of scenario.lua
+			const scenarioRequired = [...patchInfo.scenario.load, ...patchInfo.scenario.require].includes("scenario");
+			if (scenarioRequired && !zip.file(rootPath("scenario.lua"))) {
+				const control = await zip.readFile(rootPath("control.lua"));
+				if (control === null) {
+					throw new Error("Unable to patch save, missing control.lua file.");
+				}
+				files.set("scenario.lua", control);
+			}
+
+		// No info file present, try to detect if it's a known compatible scenario.
+		} else {
+			const controlFile = zip.file(rootPath("control.lua"));
+			if (!controlFile) {
+				throw new Error("Unable to patch save, missing control.lua file.");
+			}
+			const controlHash = await lib.hashStream(await zip.openStream(controlFile));
+
+			if (controlHash in knownScenarios) {
+				patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
+				files.set("scenario.lua", await zip.readEntry(controlFile));
+			} else {
+				throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
+			}
+		}
+
+		// Increment patch number
+		patchInfo.patchNumber = patchInfo.patchNumber + 1;
+
+		// Remove any existing modules from the save
+		const removed = moduleEntries(zip, patchInfo, root);
+		patchInfo.modules = [];
+
+		reorderDependencies(modules);
+
+		// Add the modules to the save.
+		for (const module of modules) {
+			for (const [relativePath, contents] of module.files) {
+				files.set(relativePath, contents);
+			}
+			patchInfo.modules.push(module);
+		}
+
+		// Add loading code and patch info
+		files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8"));
+		files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8"));
+
+		const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents]));
+		tempSavePath = await rewriteSave(zip, savePath, removed, added);
+	} finally {
+		zip.close();
+	}
+	await fs.rename(tempSavePath, savePath);
+}
+
+/**
+ * Remove the modules patched into a save
+ *
+ * Moves the original control.lua that patching kept as scenario.lua back
+ * and removes the module files.  The patch info is kept with no modules so that
+ * the patch number keeps counting if the save is patched again.
+ *
+ * @param savePath - Path to the Factorio save to unpatch.
+ * @returns true if the save was changed.
+ */
+export async function unpatch(savePath: string) {
+	const zip = await lib.ZipArchive.fromFile(savePath);
+	let tempSavePath: string;
+	try {
+		const root = lib.findRoot(zip);
+		const rootPath = (relativePath: string) => path.posix.join(root, relativePath);
+		const patchInfoContent = await zip.readFile(rootPath("clusterio.json"));
+		if (patchInfoContent === null) {
+			return false;
+		}
+		const patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root);
 		if (patchInfo.version > PatchInfo.currentVersion) {
 			throw new Error(
 				`Save patch version ${patchInfo.version} is newer than the patch version this ` +
-				`version of Clusterio can load (${PatchInfo.currentVersion})`
+				`version of Clusterio can unpatch (${PatchInfo.currentVersion})`
 			);
 		}
-
-	// No info file present, try to detect if it's a known compatible scenario.
-	} else {
-		let controlFile = root.file("control.lua");
-		if (!controlFile) {
-			throw new Error("Unable to patch save, missing control.lua file.");
+		if (patchInfo.version !== 0 && !patchInfo.modules.length) {
+			return false;
 		}
-		let controlStream = controlFile.nodeStream("nodebuffer");
-		let controlHash = await lib.hashStream(controlStream);
 
-		if (controlHash in knownScenarios) {
-			patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
-			root.file("scenario.lua", controlFile.nodeStream("nodebuffer"));
-		} else {
-			throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
-		}
-	}
-
-	// Increment patch number
-	patchInfo.patchNumber = patchInfo.patchNumber + 1;
-
-	// Remove any existing modules from the save
-	if (patchInfo.version === 0) {
-		for (let file of root.file(/^modules\//)) {
-			zip.remove(file.name);
-		}
-	} else {
-		for (let module of patchInfo.modules) {
-			for (let filepath of module.files.keys()) {
-				const file = root.file(filepath);
-				if (file !== null) {
-					zip.remove(file.name);
-				}
-			}
-		}
+		const removed = moduleEntries(zip, patchInfo, root);
 		patchInfo.modules = [];
-	}
-
-	reorderDependencies(modules);
-
-	// Add the modules to the save.
-	for (let module of modules) {
-		for (let [relativePath, contents] of module.files) {
-			root.file(relativePath, contents);
+		const scenario = await zip.readFile(rootPath("scenario.lua"));
+		if (scenario !== null) {
+			removed.add(rootPath("scenario.lua"));
 		}
-		patchInfo.modules.push(module);
-	}
-
-	// Add loading code and patch info
-	root.file("control.lua", generateLoader(patchInfo));
-	root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
-
-	// Write back the save
-	let stream = zip.generateNodeStream({ compression: "DEFLATE" });
-	const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
-	try {
-		let pipe = stream.pipe(writeStream);
-		await events.once(pipe, "finish");
+		// Scenarios made for Clusterio have no scenario.lua, keep loading them without modules
+		const control = scenario ?? Buffer.from(generateLoader(patchInfo), "utf8");
+		tempSavePath = await rewriteSave(zip, savePath, removed, new Map([
+			[rootPath("control.lua"), control],
+			[rootPath("clusterio.json"), Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8")],
+		]));
 	} finally {
-		writeStream.destroy();
+		zip.close();
 	}
 	await fs.rename(tempSavePath, savePath);
+	return true;
 }
 
 // For testing only

packages/host/src/server.ts

@@ -1,9 +1,12 @@
 // Factorio Server interface
 import fs from "node:fs/promises";
-import { type FSWatcher, watch as fsWatch, writeFileSync } from "node:fs";
+import { type FSWatcher, createWriteStream, watch as fsWatch, writeFileSync } from "node:fs";
 import child_process from "child_process";
+import dgram from "dgram";
 import path from "path";
-import JSZip from "jszip";
+import stream from "node:stream";
+import { pipeline } from "node:stream/promises";
+import type { ReadableStream } from "node:stream/web";
 import events from "events";
 import util from "util";
 import crypto from "crypto";
@@ -112,29 +115,38 @@ async function downloadAndExtractZip(url: string, targetDir: string) {
 		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
 	}
 
-  	// Load the ZIP with JSZip
-	const buffer = Buffer.from(await res.arrayBuffer());
-	const zip = await JSZip.loadAsync(buffer);
+	if (!res.body) {
+		throw new Error(`Failed to fetch ${url}: empty response`);
+	}
 
-  	// Extract each file/folder
-	await fs.mkdir(targetDir, { recursive: true });
-	await Promise.all(
-		Object.values(zip.files).map(async (entry) => {
-			const parts = entry.name.split("/").filter(Boolean);
-			const strippedPath = parts.slice(1).join("/");
-			if (!strippedPath) {
-				return; // This copies the behaviour of tar --strip-components 1
-			}
+	// Write the download to a temp file, zip files need random access to read
+	const tmpFilePath = `${targetDir}.tmp`;
+	await fs.writeFile(tmpFilePath, stream.Readable.fromWeb(res.body as ReadableStream));
+	try {
+		const zip = await lib.ZipArchive.fromFile(tmpFilePath);
+		try {
+			await fs.mkdir(targetDir, { recursive: true });
+			for (const entry of zip.entries.values()) {
+				const parts = entry.fileName.split("/").filter(Boolean);
+				const strippedPath = parts.slice(1).join("/");
+				if (!strippedPath) {
+					continue; // This copies the behaviour of tar --strip-components 1
+				}
 
-			const entryPath = path.join(targetDir, strippedPath);
-			if (entry.dir) {
-				await fs.mkdir(entryPath, { recursive: true });
-			} else {
-				await fs.mkdir(path.dirname(entryPath), { recursive: true });
-				await fs.writeFile(entryPath, await entry.async("nodebuffer"));
+				const entryPath = path.join(targetDir, strippedPath);
+				if (entry.fileName.endsWith("/")) {
+					await fs.mkdir(entryPath, { recursive: true });
+				} else {
+					await fs.mkdir(path.dirname(entryPath), { recursive: true });
+					await pipeline(await zip.openStream(entry), createWriteStream(entryPath));
+				}
 			}
-		})
-	);
+		} finally {
+			zip.close();
+		}
+	} finally {
+		await fs.rm(tmpFilePath);
+	}
 }
 
 /**
@@ -170,6 +182,76 @@ async function downloadAndExtractTar(url: string, targetDir: string) {
 	}
 }
 
+// Factorio downloads in progress by target directory, shared by every server in the process
+const downloadsInProgress = new Map<string, Promise<void>>();
+
+/**
+ * Download the newest Factorio version matching the target if not installed
+ *
+ * Call before {@link FactorioServer.fromOptions} for it to find the new version.
+ *
+ * @param factorioDir - Path to Factorio installation dirs.
+ * @param targetVersion - Version to look for, or "latest".
+ * @param availableVersions - Versions available for download.
+ * @param logger - Logger to report the download on.
+ */
+export async function checkForUpdates(
+	factorioDir: string,
+	targetVersion: lib.TargetVersion,
+	availableVersions: lib.ExternalFactorioVersion[],
+	logger: lib.Logger = lib.logger,
+) {
+
+	const latestVersion = availableVersions
+		.filter(v => (
+			targetVersion === "latest" || v.version === targetVersion || v.version.startsWith(targetVersion)
+		))
+		.sort((v1, v2) => versionOrder(v1.version, v2.version))
+		[0];
+
+	if (!latestVersion) {
+		return;
+	}
+
+	// A version that is still being extracted can already look installed
+	const dst = path.join(factorioDir, latestVersion.version);
+	const installedVersions = await listFactorioVersions(factorioDir);
+	if (!installedVersions.versions.has(latestVersion.version) || downloadsInProgress.has(dst)) {
+		const v = latestVersion.version;
+		// Can't download if not linux or this is a direct install
+		if (installedVersions.direct) {
+			logger.info(
+				`A newer version of factorio is available (${v}) but cannot be downloaded automatically ` +
+				`because ${factorioDir} is a direct (single-version) Factorio install. Use a versioned ` +
+				`layout, where ${factorioDir} contains a subdirectory per version, to enable automatic ` +
+				"downloads."
+			);
+			return;
+		}
+		if (process.platform !== "linux") {
+			logger.info(
+				`A newer version of factorio is available (${v}) but must be manually downloaded ` +
+				"(automatic downloads are only supported on Linux)."
+			);
+			return;
+		}
+
+		// Download the newer version, or wait for another server already downloading it
+		let download = downloadsInProgress.get(dst);
+		if (download) {
+			logger.info(`A newer version of factorio is available (${v}), waiting for download...`);
+		} else {
+			logger.info(`A newer version of factorio is available (${v}), starting download...`);
+			download = downloadAndExtractTar(latestVersion.headlessUrl, dst).finally(() => {
+				downloadsInProgress.delete(dst);
+			});
+			downloadsInProgress.set(dst, download);
+		}
+		await download;
+		logger.info(`Downloaded factorio version ${v}`);
+	}
+}
+
 /**
  * Find factorio data directory of the given version
  *
@@ -180,15 +262,33 @@ async function downloadAndExtractTar(url: string, targetDir: string) {
  * @param targetVersion -
  *     Version to look for, supports the special value "latest" for the
  *     latest version available.
+ * @param headless -
+ *     Only consider headless installs if true, or only full installs if
+ *     false.
  * @returns Array with path to data dir and version found.
  */
 export async function findVersion(
 	factorioDir: string,
 	targetVersion: lib.TargetVersion,
+	headless?: boolean,
 ): Promise<[string, lib.FullVersion]> {
+	let kind = "";
+	if (headless !== undefined) {
+		kind = headless ? "headless " : "full ";
+	}
+	const checkInstallType = async (dir: string) => (
+		headless === undefined || await isHeadless(path.join(dir, "data")) === headless
+	);
+
 	// Check if this is a direct installation dir
 	const directVersion = await getFactorioVersion(factorioDir);
 	if (directVersion !== null) {
+		if (!await checkInstallType(factorioDir)) {
+			throw new Error(
+				`Unable to find a ${kind}install of Factorio ${targetVersion}: ${factorioDir} is a direct ` +
+				`(single-version) install of the ${headless ? "full" : "headless"} build.`
+			);
+		}
 		if (targetVersion === "latest" || directVersion === targetVersion || directVersion.startsWith(targetVersion)) {
 			return [path.join(factorioDir, "data"), directVersion];
 		}
@@ -205,7 +305,7 @@ export async function findVersion(
 		// Avoid any fancy directory detection in order to follow symlinks
 		const fullPath = path.join(factorioDir, name);
 		const version = await getFactorioVersion(fullPath);
-		if (version === null) {
+		if (version === null || !await checkInstallType(fullPath)) {
 			continue;
 		}
 
@@ -217,7 +317,7 @@ export async function findVersion(
 	}
 
 	if (!versions.size) {
-		throw new Error(`Unable to find any Factorio install in ${factorioDir}`);
+		throw new Error(`Unable to find any ${kind}Factorio install in ${factorioDir}`);
 	}
 
 	if (lib.isPartialVersion(targetVersion)) {
@@ -233,7 +333,26 @@ export async function findVersion(
 		return [path.join(factorioDir, versions.get(latest)!, "data"), latest];
 	}
 
-	throw new Error(`Unable to find Factorio version ${targetVersion}`);
+	throw new Error(`Unable to find ${kind}Factorio version ${targetVersion}`);
+}
+
+/**
+ * Check if a Factorio install is the headless build
+ *
+ * The headless build has no icons, which is what tells it apart.
+ *
+ * @param dataDir - Path to the data dir of the install.
+ */
+export async function isHeadless(dataDir: string) {
+	try {
+		await fs.access(path.join(dataDir, "base", "graphics", "icons"));
+		return false;
+	} catch (err: any) {
+		if (err.code === "ENOENT" || err.code === "ENOTDIR") {
+			return true;
+		}
+		throw err;
+	}
 }
 
 /**
@@ -289,6 +408,15 @@ async function generatePassword(length: number) {
 	}
 }
 
+function unescapeChannel(channel: Buffer) {
+	return channel
+		.toString("utf-8")
+		.replace(/\\x([0-9a-f]{2})/g, (match, p1) => (
+			String.fromCharCode(parseInt(p1, 16))
+		))
+	;
+}
+
 /**
  * Interpret lines of output from Factorio
  *
@@ -411,6 +539,9 @@ const outputHeuristics: Heuristic[] = [
 		filter: { type: "log", message: /^Starting RCON interface/ },
 		action: function() {
 			this._startRcon().catch((err) => { this.emit("error", err); });
+			if (this.enableLuaUdp) {
+				this._startUdp().catch((err) => { this.emit("error", err); });
+			}
 		},
 	},
 
@@ -542,6 +673,10 @@ export interface FactorioServerOptions {
 	rconPort?: number,
 	/** Password use for RCON. */
 	rconPassword?: string,
+	/** Enable UDP messaging with the Factorio server. */
+	enableLuaUdp?: boolean,
+	/** UDP port the Factorio server receives Lua UDP messages on. */
+	luaUdpPort?: number,
 	/** Turn on whitelisting. */
 	enableWhitelist?: boolean,
 	/** Enable Factorio.com based multiplayer bans. */
@@ -586,8 +721,9 @@ type FactorioServerEvents = {
 	"_saving": [],
 	"_saved": [],
 
-	// IPS events
+	// IPC events
 	[ipcEvent: `ipc-${string}`]: [ event: any ],
+	[udpEvent: `udp-${string}`]: [ data: Buffer ],
 };
 
 /**
@@ -607,6 +743,8 @@ type FactorioServerEvents = {
  * - autosave-fnished - invoked when the autosave finished
  * - save-finished - invoked when the server has finished a manual save
  * - exit - invoked when the sterver has exited
+ * - ipc-<channel> - invoked with data sent with send_json in-game
+ * - udp-<channel> - invoked with data sent with send_udp in-game
  * @extends events.EventEmitter
  */
 export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
@@ -618,6 +756,10 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	rconPort: number;
 	/** Password used for RCON on the Factorio game server */
 	rconPassword: string;
+	/** Enable UDP messaging with the Factorio game server */
+	enableLuaUdp: boolean;
+	/** UDP port the Factorio game server receives Lua UDP messages on */
+	luaUdpPort: number;
 	/** Enable player whitelist */
 	enableWhitelist: boolean;
 	/** Enable Factorio.com based multiplayer bans **/
@@ -632,9 +774,8 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	_factorioDir: string;
 	_writeDir: string;
 
-	// Resolved in init
-	_version: lib.FullVersion | null = null;
-	_dataDir: string | null = null;
+	_version: lib.FullVersion;
+	_dataDir: string;
 
 	// Due to inconsistencies in the factorio api, we must manually watch the whitelist
 	// https://forums.factorio.com/viewtopic.php?t=123673
@@ -642,11 +783,11 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	_whitelist = new Set<string>();
 
 	_logger: lib.Logger;
-	_targetVersion: lib.TargetVersion;
-	_state: "new" | "init" | "create" | "running" | "stopping" = "new";
+	_state: "init" | "create" | "running" | "stopping" = "init";
 	_server: child_process.ChildProcessWithoutNullStreams | null = null;
 	_rconClient: Rcon | null= null;
 	_rconReady = false;
+	_udpSocket: dgram.Socket | null = null;
 	_gameReady = false;
 	_stripRegExp?: RegExp;
 	_maxConcurrentCommands = 5;
@@ -658,27 +799,40 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 
 
 	/**
-	 * Create a Factorio server interface
+	 * Create a Factorio server interface for the version found by {@link fromOptions}
 	 *
 	 * @param factorioDir - Directory of the Factorio install(s).
 	 * @param writeDir - Directory to write runtime data to.
-	 * @param options - Optional parameters.
+	 * @param dataDir - Data directory of the Factorio version to run.
+	 * @param version - Factorio version to run.
+	 * @param options - Optional parameters, `version` is ignored.
 	 */
-	constructor(factorioDir: string, writeDir: string, options: FactorioServerOptions) {
+	constructor(
+		factorioDir: string,
+		writeDir: string,
+		dataDir: string,
+		version: lib.FullVersion,
+		options: FactorioServerOptions & { rconPassword: string },
+	) {
 		super();
 
 		this._factorioDir = factorioDir;
 		this._writeDir = writeDir;
+		this._dataDir = dataDir;
+		this._version = version;
 
 		this._logger = options.logger || lib.logger;
-		this._targetVersion = options.version || "latest";
 		this.executablePath = options.executablePath;
 		/** UDP port used for hosting the Factorio game server on */
 		this.gamePort = options.gamePort || randomDynamicPort();
 		/** TCP port used for RCON on the Factorio game server */
 		this.rconPort = options.rconPort || randomDynamicPort();
 		/** Password used for RCON on the Factorio game server */
-		this.rconPassword = options.rconPassword as string; // init will generate one if not there
+		this.rconPassword = options.rconPassword;
+		/** Enable UDP messaging with the Factorio game server */
+		this.enableLuaUdp = options.enableLuaUdp || false;
+		/** UDP port the Factorio game server receives Lua UDP messages on */
+		this.luaUdpPort = options.luaUdpPort || randomDynamicPort();
 		/** Enable player whitelist */
 		this.enableWhitelist = options.enableWhitelist || false;
 		/** Enable Factorio.com based multiplayer bans **/
@@ -741,19 +895,24 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		}));
 	}
 
+	handleUdp(channel: string, handler: (data: Buffer) => Promise<void>) {
+		// Runs from the socket callback, so synchronous throws must be caught here too
+		this.on(`udp-${channel}`, async (data) => {
+			try {
+				await handler(data);
+			} catch (err: any) {
+				this._logger.error(`Error handling udp event:\n${err.stack ?? err.message}`);
+			}
+		});
+	}
+
 	async _handleIpc(line: Buffer) {
 		let channelEnd = line.indexOf("?");
 		if (channelEnd === -1) {
 			throw new Error(`Malformed IPC line "${line.toString()}"`);
 		}
 
-		let channel = line
-			.subarray(6, channelEnd)
-			.toString("utf-8")
-			.replace(/\\x([0-9a-f]{2})/g, (match, p1) => (
-				String.fromCharCode(parseInt(p1, 16))
-			))
-		;
+		let channel = unescapeChannel(line.subarray(6, channelEnd));
 
 		let type = line.subarray(channelEnd + 1, channelEnd + 2).toString("utf-8");
 		let content;
@@ -791,6 +950,74 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		}
 	}
 
+	_handleUdpMessage(message: Buffer, rinfo: dgram.RemoteInfo) {
+		// Factorio sends from the socket bound to the Lua UDP port, anything
+		// else is some other local process.
+		if (rinfo.port !== this.luaUdpPort) {
+			this._logger.warn(`Ignoring UDP packet from unexpected source ${rinfo.address}:${rinfo.port}`);
+			return;
+		}
+
+		let channelEnd = message.indexOf("?");
+		if (channelEnd === -1) {
+			this._logger.warn(`Ignoring malformed UDP packet "${message.subarray(0, 100).toString("utf-8")}"`);
+			return;
+		}
+
+		let channel = unescapeChannel(message.subarray(0, channelEnd));
+		let data = message.subarray(channelEnd + 1);
+		if (!this.emit(`udp-${channel}`, data)) {
+			this._logger.warn(`Warning: Unhandled udp-${channel}`);
+		}
+	}
+
+	async _startUdp() {
+		if (this._udpSocket !== null) {
+			throw Error("UDP socket is already open");
+		}
+
+		const socket = dgram.createSocket("udp4");
+		socket.on("message", (message, rinfo) => this._handleUdpMessage(message, rinfo));
+		try {
+			await new Promise<void>((resolve, reject) => {
+				socket.once("error", reject);
+				socket.bind(0, "127.0.0.1", () => {
+					socket.off("error", reject);
+					resolve();
+				});
+			});
+		} catch (err: any) {
+			this._logger.error(`Failed to open UDP socket:\n${err?.stack ?? err?.message ?? err}`);
+			socket.close();
+			return;
+		}
+		socket.on("error", err => this.emit("error", err));
+		this._udpSocket = socket;
+	}
+
+	// Factorio before 2.1.10 crashes when a packet is received on a headless server
+	_checkLuaUdpVersion() {
+		if (this.enableLuaUdp && lib.integerFullVersion(this._version) < lib.integerFullVersion("2.1.10")) {
+			throw new Error("Lua UDP requires Factorio 2.1.10 or later");
+		}
+	}
+
+	_stopUdp() {
+		if (this._udpSocket) {
+			this._udpSocket.close();
+			this._udpSocket = null;
+		}
+	}
+
+	/**
+	 * UDP port on localhost the host receives Lua UDP messages on
+	 *
+	 * Only available while the server is running with Lua UDP enabled.
+	 */
+	get hostUdpPort(): number | undefined {
+		return this._udpSocket?.address().port;
+	}
+
 	_handleOutput(rawLine: Buffer, source: "stdout" | "stderr") {
 		if (rawLine.subarray(0, 6).equals(Buffer.from("\f$ipc:"))) {
 			this._handleIpc(rawLine).catch(err => this.emit("error", err));
@@ -959,84 +1186,25 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	}
 
 	/**
-	 * Override the target version of Factorio to run
+	 * Create a Factorio server interface for an installed version
 	 *
-	 * Used to substitute a concrete version for a release channel target
-	 * (e.g. "stable" or "experimental") before {@link checkForUpdates} and
-	 * {@link init} run. Must be called before the server is initialized.
-	 */
-	setTargetVersion(version: lib.TargetVersion) {
-		this._check(["new"]);
-		this._targetVersion = version;
-	}
-
-	/**
-	 * Initialize class instance
-	 *
-	 * Must be called before instances of this class can be used.
+	 * @param factorioDir - Directory of the Factorio install(s).
+	 * @param writeDir - Directory to write runtime data to.
+	 * @param options - Optional parameters.
 	 * @throws {Error} if the requested version of Factorio was not found.
 	 */
-	async init() {
-		this._check(["new"]);
-		[this._dataDir, this._version] = await findVersion(this._factorioDir, this._targetVersion);
-		this.rconPassword = this.rconPassword as string | undefined ?? await generatePassword(10);
-		this._state = "init";
-	}
-
-	/**
-	 * Check of updated versions to download
-	 *
-	 * Should be called before init for the new version to be detected during init
-	 */
-	async checkForUpdates(availableVersions: lib.ExternalFactorioVersion[]) {
-		const targetVersion = this._targetVersion;
-		const latestVersion = availableVersions
-			.filter(v => (
-				targetVersion === "latest" || v.version === targetVersion || v.version.startsWith(targetVersion)
-			))
-			.sort((v1, v2) => versionOrder(v1.version, v2.version))
-			[0];
-
-		if (!latestVersion) {
-			return;
-		}
-
-		const installedVersions = await listFactorioVersions(this._factorioDir);
-		if (!installedVersions.versions.has(latestVersion.version)) {
-			const v = latestVersion.version;
-			// Can't download if not linux or this is a direct install
-			if (installedVersions.direct) {
-				this._logger.info(
-					`A newer version of factorio is available (${v}) but cannot be downloaded automatically ` +
-					`because ${this._factorioDir} is a direct (single-version) Factorio install. Use a versioned ` +
-					`layout, where ${this._factorioDir} contains a subdirectory per version, to enable automatic ` +
-					"downloads."
-				);
-				return;
-			}
-			if (process.platform !== "linux") {
-				this._logger.info(
-					`A newer version of factorio is available (${v}) but must be manually downloaded ` +
-					"(automatic downloads are only supported on Linux)."
-				);
-				return;
-			}
-
-			// Download the newer version
-			this._logger.info(`A newer version of factorio is available (${v}), starting download...`);
-			const dst = path.join(this._factorioDir, latestVersion.version);
-			await downloadAndExtractTar(latestVersion.headlessUrl, dst);
-			this._logger.info(`Downloaded factorio version ${v}`);
-		}
+	static async fromOptions(factorioDir: string, writeDir: string, options: FactorioServerOptions) {
+		const [dataDir, version] = await findVersion(factorioDir, options.version || "latest");
+		const rconPassword = options.rconPassword ?? await generatePassword(10);
+		return new this(factorioDir, writeDir, dataDir, version, { ...options, rconPassword });
 	}
 
 	/**
 	 * The version of Factorio in use. This will be the actual version if
-	 * "latest" (the default) was specified as `factorioVersion` to the constructor.
-	 * This will be undefined before the server is initialized, or if init failed.
+	 * "latest" (the default) was passed to {@link fromOptions}.
 	 */
-	get version(): lib.FullVersion | undefined {
-		return this._version ?? undefined;
+	get version(): lib.FullVersion {
+		return this._version;
 	}
 
 	/**
@@ -1063,6 +1231,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		this._server = null;
 		this._rconClient = null;
 		this._rconReady = false;
+		this._udpSocket = null;
 		this._gameReady = false;
 		this._unexpected = [];
 		this._killed = false;
@@ -1108,6 +1277,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 			if (this._rconClient) {
 				this._rconClient.end().catch(() => {});
 			}
+			this._stopUdp();
 
 			if (this._whitelistWatcher) {
 				this._whitelistWatcher.close();
@@ -1126,6 +1296,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 			if (this._rconClient) {
 				this._rconClient.end().catch(() => {});
 			}
+			this._stopUdp();
 
 			if (this._whitelistWatcher) {
 				this._whitelistWatcher.close();
@@ -1224,6 +1395,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	 */
 	async start(save: string) {
 		this._check(["init"]);
+		this._checkLuaUdpVersion();
 		this._state = "running";
 
 		try {
@@ -1239,6 +1411,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 					"--port", String(this.gamePort),
 					"--rcon-port", String(this.rconPort),
 					"--rcon-password", this.rconPassword,
+					...(this.enableLuaUdp ? ["--enable-lua-udp", String(this.luaUdpPort)] : []),
 					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
 					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
 					...(this.verboseLogging ? ["--verbose"] : []),
@@ -1276,6 +1449,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	 */
 	async startScenario(scenario: string, seed?: number, mapGenSettings?: object, mapSettings?: object) {
 		this._check(["init"]);
+		this._checkLuaUdpVersion();
 		this._state = "running";
 
 		try {
@@ -1297,6 +1471,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 					"--port", String(this.gamePort),
 					"--rcon-port", String(this.rconPort),
 					"--rcon-password", this.rconPassword,
+					...(this.enableLuaUdp ? ["--enable-lua-udp", String(this.luaUdpPort)] : []),
 					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
 					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
 					...(this.verboseLogging ? ["--verbose"] : []),
@@ -1377,6 +1552,27 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		return response;
 	}
 
+	/**
+	 * Send UDP message to the server
+	 *
+	 * Sends a UDP packet to the Lua UDP port of the Factorio server, which
+	 * is delivered in-game as an on_udp_packet_received event.  Delivery is
+	 * not guaranteed.  Requires Lua UDP to be enabled.
+	 *
+	 * @param message - payload of the packet to send.
+	 */
+	async sendUdp(message: string | Buffer) {
+		this._check(["running", "stopping"]);
+		if (!this._udpSocket) {
+			throw new Error("UDP socket is not open");
+		}
+
+		const socket = this._udpSocket;
+		await new Promise<void>((resolve, reject) => {
+			socket.send(message, this.luaUdpPort, "127.0.0.1", err => (err ? reject(err) : resolve()));
+		});
+	}
+
 	/**
 	 * Timeout in ms to wait after a shutdown is requested before killing the
 	 * process. Defaults to 0 meaning no timeout
@@ -1412,6 +1608,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		if (this._rconClient) {
 			this._rconClient.end().catch(() => {});
 		}
+		this._stopUdp();
 		if (this._whitelistWatcher) {
 			this._whitelistWatcher.close();
 		}
@@ -1526,14 +1723,13 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	 * Return path in data directory
 	 *
 	 * Creates a path using path.join with the given parts that's relative to
-	 * the data directory of the Factorio server.  Not valid before init has
-	 * been called.
+	 * the data directory of the Factorio server.
 	 *
 	 * @param parts - Extra parts to add to the data path.
 	 * @returns Data directory path.
 	 */
 	dataPath(...parts: string[]) {
-		return path.join(this._dataDir!, ...parts);
+		return path.join(this._dataDir, ...parts);
 	}
 
 	/**

packages/host/src/set_lazy_schema_compilation.ts

@@ -0,0 +1,3 @@
+// This needs to be a separate file due to ECMAScript Modules evaluating code in
+// imported modules before any code in the file that imports it.
+(global as any).lazySchemaCompilation = true;

packages/host/typedoc.json

@@ -1,4 +0,0 @@
-{
-	"entryPoints": ["index.ts"],
-	"includeVersion": true
-}

packages/lib/browser.ts

@@ -1,3 +1,4 @@
+import "./src/browser_polyfills";
 export * from "./src/api";
 export * from "./src/AsyncHook";
 export * from "./src/SyncHook";

packages/lib/build_mod.d.ts

@@ -6,5 +6,5 @@ export interface BuildArgs {
 	build?: boolean;
 	outputDir: string;
 	sourceDir: string;
-};
+}
 export function build(args: BuildArgs): Promise<void>;

packages/lib/build_mod.js

@@ -1,13 +1,40 @@
 /* eslint-disable no-console */
 import fs from "node:fs/promises";
-import JSZip from "jszip";
 import path from "node:path";
-import stream from "node:stream";
-import util from "node:util";
+import { pipeline } from "node:stream/promises";
 import yargs from "yargs";
+import yazl from "yazl";
 
-const finished = util.promisify(stream.finished);
 
+async function addModFiles(args, info, modName, zip) {
+	for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) {
+		if (dirent.isFile()) {
+			const itemPath = path.join(dirent.parentPath, dirent.name);
+			// On Windows the path created uses backslashes as the directory sepparator
+			// but the zip file needs to use forward slashes.  We can't use the posix
+			// version of relative here as it doesn't work with Windows style paths.
+			let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/");
+			if (basePath === "info.json") {
+				// The info.json file is overridden later.
+				continue;
+			}
+			zip.addFile(itemPath, path.posix.join(modName, basePath));
+		}
+	}
+
+	for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
+		let filePath = path.join(args.sourceDir, ...pathParts);
+		try {
+			await fs.access(filePath);
+		} catch (err) {
+			throw new Error(`Error reading additional file ${filePath}`, { cause: err });
+		}
+		zip.addFile(filePath, path.posix.join(modName, fileName));
+	}
+	delete info.additional_files;
+
+	zip.addBuffer(Buffer.from(JSON.stringify(info, null, "\t")), path.posix.join(modName, "info.json"));
+}
 
 async function buildMod(args, info, modName) {
 	if (args.build) {
@@ -15,43 +42,24 @@ async function buildMod(args, info, modName) {
 		modName = args.modName ?? `${info.name}_${info.version}`;
 
 		if (args.pack) {
-			let zip = new JSZip();
-			for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) {
-				if (dirent.isFile()) {
-					const itemPath = path.join(dirent.parentPath, dirent.name);
-					// On Windows the path created uses backslashes as the directory sepparator
-					// but the zip file needs to use forward slashes.  We can't use the posix
-					// version of relative here as it doesn't work with Windows style paths.
-					let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/");
-					// eslint-disable-next-line max-depth
-					if (basePath === "info.json") {
-						// The info.json file is overridden later.
-						continue;
-					}
-					zip.file(path.posix.join(modName, basePath), (await fs.open(itemPath)).createReadStream());
-				}
-			}
-
-			for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
-				let filePath = path.join(args.sourceDir, ...pathParts);
-				let fileStream;
-				try {
-					fileStream = (await fs.open(filePath)).createReadStream();
-				} catch (err) {
-					throw new Error(`Error reading additional file ${filePath}`, { cause: err });
-				}
-				zip.file(path.posix.join(modName, fileName), fileStream);
-			}
-			delete info.additional_files;
-
-			zip.file(path.posix.join(modName, "info.json"), JSON.stringify(info, null, "\t"));
-
 			let modPath = path.join(args.outputDir, `${modName}.zip`);
 			console.log(`Writing ${modPath}`);
-			let writeStream = zip.generateNodeStream().pipe(
-				(await fs.open(modPath, "w")).createWriteStream()
-			);
-			await finished(writeStream);
+			let zip = new yazl.ZipFile();
+			// yazl reports errors on the zip object, not the output stream.
+			zip.on("error", err => zip.outputStream.destroy(err));
+			// addFile() can fail before the traversal is done, so start writing first.
+			const writing = pipeline(zip.outputStream, (await fs.open(modPath, "w")).createWriteStream());
+			writing.catch(() => {});
+			try {
+				await addModFiles(args, info, modName, zip);
+				zip.end();
+				await writing;
+			} catch (err) {
+				zip.outputStream.destroy();
+				await writing.catch(() => {});
+				await fs.rm(modPath, { force: true });
+				throw err;
+			}
 
 		} else {
 			let modDir = path.join(args.outputDir, modName);

packages/lib/index.ts

@@ -35,6 +35,7 @@ export * from "./src/subscriptions.js";
 export * from "./src/datastore.js";
 export * from "./src/load_plugin_list.js";
 export * from "./src/rce_ops.js";
+export * from "./src/npm_registry.js";
 export * from "./src/LockFile.js";
 export * from "./src/AsyncHook.js";
 export * from "./src/SyncHook.js";

packages/lib/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/lib",
-	"version": "2.0.0",
+	"version": "2.0.3",
 	"description": "Shared library for Clusterio",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -46,21 +46,24 @@
 		"compile_validator": "node scripts/compile_validator.js"
 	},
 	"engines": {
-		"node": ">=22"
+		"node": "^22.18.0 || >=24.2.0"
 	},
 
 	"dependencies": {
 		"@sinclair/typebox": "catalog:",
+		"@types/yauzl": "catalog:",
+		"@types/yazl": "catalog:",
 		"ajv": "^8.20.0",
 		"chalk": "catalog:",
 		"fast-deep-equal": "^3.1.3",
-		"jszip": "catalog:",
 		"set-blocking": "catalog:",
 		"triple-beam": "^1.4.1",
 		"winston": "catalog:",
 		"winston-transport": "catalog:",
 		"ws": "catalog:",
-		"yargs": "catalog:"
+		"yargs": "catalog:",
+		"yauzl": "catalog:",
+		"yazl": "catalog:"
 	},
 	"devDependencies": {
 		"@types/node": "catalog:",

packages/lib/src/LockFile.ts

@@ -1,5 +1,5 @@
-import { FileHandle, open, unlink, readFile } from "node:fs/promises";
-import { unlinkSync } from "node:fs";
+import fs, { type FileHandle } from "node:fs/promises";
+import { readFileSync, unlinkSync } from "node:fs";
 import assert from "assert";
 
 /**
@@ -24,23 +24,99 @@ export class LockFileBusyError extends Error {
 	}
 }
 
+/**
+ * Error thrown when a process exited while clearing a stale lock, which needs the recovery file removed by hand.
+ */
+export class LockFileRecoveryError extends Error {
+	constructor(
+		readonly recoveryPath: string
+	) {
+		super(`Lock file recovery was interrupted, remove ${recoveryPath} to allow the lock to be acquired`);
+		this.name = "LockFileRecoveryError";
+	}
+}
+
 type LockState = "idle" | "acquiring" | "acquired" | "releasing";
+type LockSnapshot = { content: string, ino: bigint, mtimeNs: bigint };
 
 /**
  * Cross-platform lock file utility for Node.js.
  */
 export class LockFile {
+	/**
+	 * Content written to the lock file. The start time tells this process apart from an earlier one with
+	 * the same pid, and is the same for every copy of lib loaded in this process.
+	 */
+	private static readonly content = `${process.pid} ${performance.timeOrigin}\n`;
+	// A lock file without a valid pid is left alone for this long, as it may still be being written
+	private static readonly writeGraceMs = 10e3;
+
 	private handle: FileHandle | null = null;
 	private state: LockState = "idle";
 
+	/**
+	 * @param filePath - Path of the lock file
+	 * @param handleSignals -
+	 *     Release the lock on SIGINT, SIGTERM, SIGHUP and uncaught exceptions. Listening to these
+	 *     changes how the process exits, so without it the lock is only removed on exit.
+	 */
 	constructor(
-		private readonly filePath: string
+		private readonly filePath: string,
+		private readonly handleSignals = true,
 	) {}
 
+	/** Content and identity of a lock file, read through one handle so both belong to the same file */
+	private static async readLock(filePath: string): Promise<LockSnapshot | undefined> {
+		let handle;
+		try {
+			handle = await fs.open(filePath, "r");
+		} catch (err: any) {
+			if (err.code === "ENOENT") {
+				return undefined;
+			}
+			throw err;
+		}
+		try {
+			const stat = await handle.stat({ bigint: true });
+			return { content: await handle.readFile("utf8"), ino: stat.ino, mtimeNs: stat.mtimeNs };
+		} finally {
+			await handle.close();
+		}
+	}
+
+	private static isSameLock(a: LockSnapshot, b: LockSnapshot) {
+		return a.content === b.content && a.ino === b.ino && a.mtimeNs === b.mtimeNs;
+	}
+
+	private static lockPid(lock: LockSnapshot) {
+		const pid = Number(lock.content.split(" ")[0]);
+		return Number.isInteger(pid) && pid > 0 ? pid : undefined;
+	}
+
+	/** True if the lock names a process that is still running, or may still be written to */
+	private static lockIsLive(lock: LockSnapshot) {
+		const pid = LockFile.lockPid(lock);
+		if (pid === undefined) {
+			return Date.now() - Number(lock.mtimeNs / 1000000n) < LockFile.writeGraceMs;
+		}
+		if (pid === process.pid) {
+			// Held by this process, or left by an earlier process with the same pid
+			return lock.content === LockFile.content;
+		}
+		try {
+			// Signal 0 only checks that the process exists
+			process.kill(pid, 0);
+			return true;
+		} catch (err: any) {
+			return err.code !== "ESRCH";
+		}
+	}
+
 	/**
 	 * Acquire the lock file.
 	 * Throws LockFileExistsError if already locked.
 	 * Throws LockFileBusyError if not in idle state.
+	 * Throws LockFileRecoveryError if clearing a stale lock was interrupted.
 	 */
 	async acquire(): Promise<void> {
 		if (this.state !== "idle") {
@@ -49,13 +125,10 @@ export class LockFile {
 
 		this.state = "acquiring";
 
-		if (await this.isStale()) {
-			await unlink(this.filePath).catch(() => {}); // ignore errors, may already be removed
-		}
-
 		try {
-			this.handle = await open(this.filePath, "wx");
-			await this.handle.write(`${process.pid}\n`);
+			await this.removeStale();
+			this.handle = await fs.open(this.filePath, "wx");
+			await this.handle.write(LockFile.content);
 			this.attachExitHandlers();
 			this.state = "acquired";
 		} catch (err: any) {
@@ -81,7 +154,9 @@ export class LockFile {
 
 		// Ignored: file may have already been removed by another process
 		try { await this.handle?.close(); } catch {}
-		try { await unlink(this.filePath); } catch {}
+		if (await this.isHeldFile()) {
+			await fs.unlink(this.filePath).catch(() => {});
+		}
 
 		this.handle = null;
 		this.state = "idle";
@@ -89,19 +164,20 @@ export class LockFile {
 
 	/**
 	 * Check whether the lock is currently held by this process.
-	 * True only if the lock is acquired and the PID in the file
-	 * matches the current process.
+	 * True only if the lock is acquired and the lock file was written by this process.
 	 */
 	async isHeld(): Promise<boolean> {
 		if (this.state !== "acquired") {
 			return false;
 		}
+		return await this.isHeldFile();
+	}
 
+	private async isHeldFile() {
 		try {
-			const pidInLock = Number(await readFile(this.filePath, "utf8"));
-			return pidInLock === process.pid;
+			return (await LockFile.readLock(this.filePath))?.content === LockFile.content;
 		} catch {
-    		// Ignored: file may not exist or be unreadable, so lock is not held
+			// Ignored: file may be unreadable, so lock is not held
 			return false;
 		}
 	}
@@ -110,14 +186,45 @@ export class LockFile {
 	 * Determine whether an existing lock is stale.
 	 */
 	async isStale(): Promise<boolean> {
+		const lock = await LockFile.readLock(this.filePath);
+		return lock === undefined || !LockFile.lockIsLive(lock);
+	}
+
+	/** Pid of the process holding the lock, if it has written it */
+	async holderPid(): Promise<number | undefined> {
+		const lock = await LockFile.readLock(this.filePath);
+		return lock && LockFile.lockPid(lock);
+	}
+
+	/**
+	 * Remove a stale lock file if it is still the same file. Clearing is done while holding a separate
+	 * recovery lock, so a lock another process took after the stale check is never removed.
+	 */
+	private async removeStale(): Promise<void> {
+		const stale = await LockFile.readLock(this.filePath);
+		if (!stale || LockFile.lockIsLive(stale)) {
+			return;
+		}
+		const recoveryPath = `${this.filePath}.recovery`;
 		try {
-			const pidInLock = Number(await readFile(this.filePath, "utf8"));
-			if (!Number.isFinite(pidInLock)) { return true; }
-			// signal 0 is a special case to test if a process exists
-			process.kill(pidInLock, 0);
-			return false;
-		} catch {
-			return true;
+			await fs.writeFile(recoveryPath, LockFile.content, { flag: "wx" });
+		} catch (err: any) {
+			if (err.code !== "EEXIST") {
+				throw err;
+			}
+			const holder = await LockFile.readLock(recoveryPath);
+			if (holder && !LockFile.lockIsLive(holder)) {
+				throw new LockFileRecoveryError(recoveryPath);
+			}
+			return; // Another process is clearing it
+		}
+		try {
+			const current = await LockFile.readLock(this.filePath);
+			if (current && LockFile.isSameLock(current, stale)) {
+				await fs.unlink(this.filePath);
+			}
+		} finally {
+			await fs.unlink(recoveryPath).catch(() => {});
 		}
 	}
 
@@ -126,10 +233,12 @@ export class LockFile {
 	 */
 	private attachExitHandlers(): void {
 		assert(this.state === "acquiring");
-		process.on("SIGINT", this.handleExit);
-		process.on("SIGTERM", this.handleExit);
-		process.on("SIGHUP", this.handleExit);
-		process.on("uncaughtException", this.handleExit);
+		if (this.handleSignals) {
+			process.on("SIGINT", this.handleExit);
+			process.on("SIGTERM", this.handleExit);
+			process.on("SIGHUP", this.handleExit);
+			process.on("uncaughtException", this.handleExit);
+		}
 		// eslint-disable-next-line node/no-sync
 		process.on("exit", this.handleExitSync);
 	}
@@ -137,7 +246,7 @@ export class LockFile {
 	/**
 	 * Detach process handlers, called when transitioning to "idle"
 	 */
-	private async detachExitHandlers(): Promise<void> {
+	private detachExitHandlers(): void {
 		assert(this.state === "releasing");
 		process.off("SIGINT", this.handleExit);
 		process.off("SIGTERM", this.handleExit);
@@ -147,17 +256,21 @@ export class LockFile {
 		process.off("exit", this.handleExitSync);
 	}
 
-  	// Event handler for async signals / exceptions
+	// Event handler for async signals / exceptions
 	private handleExit = async (): Promise<void> => {
 		await this.release();
 	};
 
-  	// Event handler for synchronous process exit
+	// Event handler for synchronous process exit, readLock can't be used as it is async
 	private handleExitSync = (): void => {
 		try {
-			unlinkSync(this.filePath);
+			// eslint-disable-next-line node/no-sync
+			if (readFileSync(this.filePath, "utf8") === LockFile.content) {
+				// eslint-disable-next-line node/no-sync
+				unlinkSync(this.filePath);
+			}
 		} catch (err) {
-      		// Ignored: file may have already been removed by another process
+			// Ignored: file may have already been removed by another process
 		}
 	};
 }