Prepare 2.0.3

Add a LogEntry type for log entries sent over links and returned by
log queries. It names the fields every entry has (level, message,
timestamp) and leaves the rest as unknown.

The console now prefixes each line with the entry's time in the
browser's local time zone, which makes it possible to tell when an
error happened after the fact.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Default owners for all files in the repository.
+* @psihius @Danielv123 @Hornwitser @Cooldude2606 @ILLISIS

.github/workflows/ci-changes.yml

@@ -0,0 +1,126 @@
+name: Classify CI changes
+
+on:
+  workflow_call:
+    outputs:
+      build_and_test:
+        description: Run the test and coverage matrix, compatibility tests, and mod build.
+        value: ${{ jobs.changes.outputs.build_and_test }}
+      js_lint:
+        description: Run the JavaScript/TypeScript linter.
+        value: ${{ jobs.changes.outputs.js_lint }}
+      lua_lint:
+        description: Run the Lua linter.
+        value: ${{ jobs.changes.outputs.lua_lint }}
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      build_and_test: ${{ steps.syntax.outputs.build_and_test || steps.paths.outputs.code }}
+      js_lint: ${{ steps.lint.outputs.js_lint }}
+      lua_lint: ${{ steps.lint.outputs.lua_lint }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: dorny/paths-filter@v4
+        id: paths
+        with:
+          token: ''
+          base: ${{ github.event.pull_request.base.sha || github.event.before }}
+          ref: ${{ github.sha }}
+          list-files: json
+          predicate-quantifier: every
+          filters: |
+            # Files requiring validation beyond documentation checks.
+            code:
+              - '**'
+              - '!**/*.md'
+              - '!docs/**/*.{png,jpg,svg,gif}'
+            # Sources and configuration owned by each linter.
+            # Workflow scripts and shared build configuration run both linters.
+            js:
+              - '**/*.{js,cjs,mjs,jsx,ts,cts,mts,tsx}'
+              - '!.github/**'
+              - '!**/{webpack,babel,rollup,vite,jest}.config.*'
+            lua:
+              - '{*.lua,**/*.lua,.emmyrc.json}'
+              - '!.github/**'
+            # Modified JS/TS source eligible for the comment-only shortcut.
+            # TypeScript type changes still count as substantive changes.
+            syntax:
+              - modified: '**/*.{js,cjs,mjs,jsx,ts,cts,mts,tsx}'
+              # Source fixtures and templates always receive full validation.
+              - '!test/file/**'
+              - '!packages/create/templates/**'
+              - '!.github/**'
+              - '!**/{eslint,webpack,babel,rollup,vite,jest}.config.*'
+      - name: Select linters
+        id: lint
+        env:
+          CODE_FILES: ${{ steps.paths.outputs.code_files }}
+          JS_FILES: ${{ steps.paths.outputs.js_files }}
+          LUA_FILES: ${{ steps.paths.outputs.lua_files }}
+        run: |
+          # Unclassified non-documentation files can affect either language.
+          jq -nr --argjson code "$CODE_FILES" --argjson js "$JS_FILES" --argjson lua "$LUA_FILES" '
+            ($code - $js - $lua | length > 0) as $shared |
+            "js_lint=\($shared or ($js | length > 0))",
+            "lua_lint=\($shared or ($lua | length > 0))"
+          ' >> "$GITHUB_OUTPUT"
+      - name: Install Difftastic
+        if: steps.paths.outputs.syntax == 'true'
+        env:
+          DIFFT_VERSION: 0.71.0
+          DIFFT_SHA256: 61aea5394a53c56f144637cf39b3a0c0dafa27769731745e5851ff276e6478da
+        run: |
+          mkdir -p "$RUNNER_TEMP/difftastic"
+          curl -fsSL "https://github.com/Wilfred/difftastic/releases/download/$DIFFT_VERSION/difft-$DIFFT_VERSION-x86_64-unknown-linux-gnu.tar.gz" -o "$RUNNER_TEMP/difftastic.tar.gz"
+          echo "$DIFFT_SHA256  $RUNNER_TEMP/difftastic.tar.gz" | sha256sum -c
+          tar -xzf "$RUNNER_TEMP/difftastic.tar.gz" -C "$RUNNER_TEMP/difftastic"
+          echo "$RUNNER_TEMP/difftastic" >> "$GITHUB_PATH"
+      - name: Check for substantive changes
+        id: syntax
+        if: steps.paths.outputs.code == 'true'
+        env:
+          CODE_FILES: ${{ steps.paths.outputs.code_files }}
+          SYNTAX_FILES: ${{ steps.paths.outputs.syntax_files }}
+          BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
+          HEAD_SHA: ${{ github.sha }}
+        run: |
+          # Skip only when every non-documentation file is an ordinary source edit.
+          echo 'build_and_test=true' >> "$GITHUB_OUTPUT"
+          # A new branch has no previous commit; keep full validation on its first push.
+          if [[ "$BASE" == 0000000000000000000000000000000000000000 ]]; then
+            exit 0
+          fi
+          if ! jq -en --argjson code "$CODE_FILES" --argjson syntax "$SYNTAX_FILES" \
+            '$code - $syntax | length == 0' > /dev/null; then
+            exit 0
+          fi
+          git cat-file -e "$BASE^{commit}"
+          jq -j '.[] | ., "\u0000"' <<< "$SYNTAX_FILES" > "$RUNNER_TEMP/syntax-files"
+          while IFS= read -r -d '' file; do
+            # Mode changes and symlinks are substantive even when text is identical.
+            [[ -z "$(git diff --summary --no-renames "$BASE" "$HEAD_SHA" -- "$file")" ]] || exit 0
+            git ls-tree "$HEAD_SHA" -- "$file" | grep -qE '^100(644|755) blob ' || exit 0
+            before="$RUNNER_TEMP/before.${file##*.}"
+            after="$RUNNER_TEMP/after.${file##*.}"
+            git show "$BASE:$file" > "$before"
+            git show "$HEAD_SHA:$file" > "$after"
+            # Difftastic ignores ALL comments; retain compiler/bundler/coverage directives.
+            if grep -Eq '@ts-|@jsx|@internal|<reference\b|<amd-|sourceMappingURL|sourceURL|webpack|vite|[#@]__|/\*!|(istanbul|c8|v8)[[:space:]]+ignore' "$before" "$after"; then
+              exit 0
+            fi
+            case "$file" in
+              *.js|*.cjs|*.mjs|*.jsx)
+                # JavaScript JSDoc can contain type information.
+                if grep -Fq '/**' "$before" "$after"; then exit 0; fi ;;
+            esac
+            if ! difft --ignore-comments --check-only --exit-code "$before" "$after"; then
+              exit 0
+            fi
+          done < "$RUNNER_TEMP/syntax-files"
+          echo 'build_and_test=false' >> "$GITHUB_OUTPUT"

.github/workflows/lua-lint.yml

@@ -8,7 +8,12 @@ env:
   EMMYLUA_CHECK_VERSION: "0.25.1"
 
 jobs:
+  changes:
+    uses: ./.github/workflows/ci-changes.yml
+
   lint:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.lua_lint != 'false' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7

.github/workflows/node.js.yml

@@ -11,14 +11,20 @@ env:
   NODE_VERSION: 24.x
 
 jobs:
+  changes:
+    uses: ./.github/workflows/ci-changes.yml
+
   test:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.build_and_test != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
-        # Supported node versions are active lts and maintenance see https://nodejs.org/en/about/releases/
-        node-version: [24.x]
+        # Test Node.js 24 and 26 across the Factorio matrix; Node.js 22 runs coverage below.
+        # Node.js release schedule: https://nodejs.org/en/about/releases/
+        node-version: [24.x, 26.x]
 
         # Supported factorio versions are latest stable and one minor version before stable (even if from a previous major) 
         factorio-version: [1.1.110, 2.0.77, 2.1.17]
@@ -57,6 +63,8 @@ jobs:
         path: temp/test/logs
 
   version-compat:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.build_and_test != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     env:
@@ -86,9 +94,11 @@ jobs:
         node scripts/version-compat-test.js
         --controller ${{ matrix.controller }}
         --host ${{ matrix.host }}
-        --published-version $PUBLISHED_VERSION
+        --published-version "$PUBLISHED_VERSION"
 
   lint:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.js_lint != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -100,6 +110,8 @@ jobs:
     - run: pnpm run lint
 
   build:
+    needs: changes
+    if: ${{ !cancelled() && needs.changes.outputs.build_and_test != 'false' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:

AGENTS.md

@@ -0,0 +1,36 @@
+# Working on Clusterio
+
+Clusterio manages distributed Factorio servers through a controller, hosts, and plugins.
+
+Read [contributing](docs/contributing.md) for the project map, setup, commands, debugging, tests, and code style.
+For PR reviews, follow the [review guide](docs/pr-review-guide.md), including its candidate ledger and final evidence audit.
+Use the detailed guides for the affected area:
+
+- [Communication and routing](docs/communication.md), [wire protocol](docs/devs/protocol.md), and [configuration](docs/config-system.md).
+- [Plugins](docs/writing-plugins.md), [plugin migration](docs/devs/plugin-migration-to-2.0.md), and [Factorio/Lua](docs/developing-for-clusterio.md).
+- [Compatibility policies](docs/decisions.md) and [releases](docs/release-workflow.md).
+
+
+## Working rules
+
+- Inspect `git status` before editing.
+  Preserve unrelated work and external plugin repositories; continue an existing task branch.
+- Read the relevant source and tests before changing behavior.
+  If documentation and implementation disagree, report the discrepancy and distinguish observed behavior from intended policy.
+  Update affected documentation in the same change.
+- Follow nearby code and `eslint.config.mjs`: tabs, double quotes, semicolons, and `.js` extensions in relative TypeScript imports.
+  Import shared public APIs through `@clusterio/lib` and check Node/browser exports when changing them.
+- Run commands from the root using the pinned pnpm version and the [documented Node requirements](docs/contributing.md#supported-nodejs-and-factorio-versions).
+  Install with `pnpm install --no-frozen-lockfile`; the lockfile is intentionally ignored.
+- `pnpm build` compiles TypeScript.
+  It does not regenerate validators or production web bundles; use the appropriate package's `prepare` script.
+  Edit sources, not generated outputs.
+- Start validation with focused tests, then run `pnpm test` and `pnpm lint` for code changes when possible.
+  Root tests include integration setup even with fast/external exclusions.
+  The harness needs Factorio, recreates `temp/test/`, and shares ports 8880/4443; do not run competing suites.
+- Report checks, failures, skips, and unverified behavior precisely.
+  Documentation-only changes need path, command, and formatting checks.
+- Preview cleanup with `node clean.js --dry-run`; see the [cleanup scopes](docs/contributing.md#cleaning-generated-and-runtime-files).
+  Runtime-data deletion and release publication require explicit authorization.
+  Keep local configs, credentials, runtime data, and generated output out of commits.
+- Follow the [PR template](.github/pull_request_template.md), including `### Changelog` (`None.` when appropriate).

CHANGELOG.md

@@ -31,6 +31,63 @@ Many thanks to the following for contributing to this release:
 [@username](https://github.com/username)
 -->
 
+## Version 2.0.3
+
+### Features
+
+- Added per instance options to choose which parts of the player inventory sync covers. [#968](<https://github.com/clusterio/clusterio/issues/968>)
+- Added opt-in UDP messaging between instances and the Factorio server via `factorio.enable_lua_udp`, with a `sendUdp` plugin API, `clusterio_api.send_udp` in-game and a `LuaUdp` plugin feature flag. [[#969](<https://github.com/clusterio/clusterio/issues/969>)](<https://github.com/clusterio/clusterio/issues/969>)
+- Added notes that can be attached to hosts, instances, users, roles, mod packs, mods and saves in the web interface and through ctl. [#903](<https://github.com/clusterio/clusterio/issues/903>)
+- Added tools to inspect, replace, delete and unlock player data stored by inventory sync, in the web interface and as `clusterioctl inventory-sync` commands. [#1003](<https://github.com/clusterio/clusterio/issues/1003>)
+- Notes in the web interface render as Markdown and link `@<type>/<id>` mentions to the resource. [#1000](<https://github.com/clusterio/clusterio/pull/1000>)
+- Added timestamps to log lines in the web interface console. [#1137](<https://github.com/clusterio/clusterio/pull/1137>)
+- Added syncing of quick bar slots that hold a specific blueprint, blueprint book or planner, including ones from the player's blueprint library, on Factorio 2.1. [#1092](<https://github.com/clusterio/clusterio/issues/1092>)
+- Added a `set_destination` remote interface to Player Sync. A plugin that moves a player to another instance calls it before `connect_to_server`, and the player's data is sent to that instance while the client reconnects, so the join applies it without a download. [#1099](<https://github.com/clusterio/clusterio/issues/1099>)
+- Added a cluster research queue to research_sync: servers can follow one research queue, set from the research screen on any following server, with a checkbox on the lab window. [#1115](<https://github.com/clusterio/clusterio/pull/1115>)
+
+### Fixes
+
+- Fixed instances that start at the same time each downloading a new Factorio version into the same files and leaving a corrupt install. [[#1105](<https://github.com/clusterio/clusterio/issues/1105>)](<https://github.com/clusterio/clusterio/issues/1105>)
+- Fixed `controller.max_send_buffer_size` counting buffered messages in UTF-16 code units instead of UTF-8 bytes, which let non-ASCII messages go over the limit. [[#1108](<https://github.com/clusterio/clusterio/issues/1108>)](<https://github.com/clusterio/clusterio/issues/1108>)
+- Removed incompatible Server Select and Discord Bridge releases from the installer's plugin choices so selecting all offered plugins no longer installs them. [#1102](<https://github.com/clusterio/clusterio/issues/1102>)
+- Fixed reading map exchange strings from Factorio 2.1.18 and later, which gave wrong enemy evolution and expansion settings and made `instance load-scenario --map-exchange-string` fail. [[#1131](<https://github.com/clusterio/clusterio/issues/1131>)](<https://github.com/clusterio/clusterio/issues/1131>)
+- Fixed Player Sync erroring when the controller returned no data for a downloading player, which left the player waiting for the timeout. [#1094](<https://github.com/clusterio/clusterio/issues/1094>)
+- Fixed spoilable items arriving fresh after being serialized, the spoil percent is now kept. [#1093](<https://github.com/clusterio/clusterio/issues/1093>)
+- Fixed named personal logistic groups losing their requests and trash rules when joining a server where the group was missing or empty. [#1091](<https://github.com/clusterio/clusterio/issues/1091>)
+- Fixed Player Sync leaving quick bar slots from an earlier visit in place, which made moved or cleared slots come back after changing server. [#1092](<https://github.com/clusterio/clusterio/issues/1092>)
+- Fixed research_sync sending research it received from the controller back to it, so every instance got the research again each time an instance started. [#1110](<https://github.com/clusterio/clusterio/issues/1110>)
+- Fixed items in main inventory slots from armor equipment such as toolbelts not fitting in the inventory when joining another server. [#1100](<https://github.com/clusterio/clusterio/issues/1100>)
+
+### Changes
+
+- Replaced jszip with yauzl and yazl so saves and mods are no longer loaded fully into memory when patched, read or exported. [#1002](<https://github.com/clusterio/clusterio/issues/1002>)
+- Bumped dependency versions, including typebox 0.34 and react-router 7. Plugins using `@sinclair/typebox` should update to ^0.34. [#1113](<https://github.com/clusterio/clusterio/pull/1113>)
+
+### Contributors
+
+Many thanks to the following for contributing to this release:
+[@psihius](<https://github.com/psihius>), [@ILLISIS](<https://github.com/ILLISIS>), [@bbassie](<https://github.com/bbassie>), [@ColumbusLabs](<https://github.com/ColumbusLabs>)
+
+### Statistics
+
+Commits: 101  Pull Requests: 29  Files Changed: 135  Additions: +8,830  Deletions: -994
+
+## Version 2.0.2
+
+### Fixes
+
+- Fixed installation with selected plugins by relying on automatic plugin discovery, and ensured installer failures return a nonzero exit status. [#1082](<https://github.com/clusterio/clusterio/pull/1082>), [#1084](<https://github.com/clusterio/clusterio/issues/1084>)
+- Fixed web plugin `controllerConnectionEvent` handlers receiving the connector's event data instead of the event name. [#1083](<https://github.com/clusterio/clusterio/issues/1083>)
+- Updated installation instructions to recommend latest Node.js LTS and corrected outdated mod management, CLI, plugin, and release documentation. [#1086](<https://github.com/clusterio/clusterio/pull/1086>)
+
+### Meta
+
+- Made player-auth login tests independent of wall-clock timing and added verification-token expiry coverage. [#1087](<https://github.com/clusterio/clusterio/pull/1087>)
+- Added Node.js 26 to the CI test matrix across all tested Factorio versions. [#1088](<https://github.com/clusterio/clusterio/pull/1088>)
+
+Many thanks to the following for contributing to this release:
+[@psihius](<https://github.com/psihius>), [@bbassie](<https://github.com/bbassie>)
+
 ## Version 2.0.1
 
 ### Fixes
@@ -673,7 +730,7 @@ Many thanks to the following for contributing to this release:
 ## Version 2.0.0-alpha.14
 
 This release does not retain backwards compatibility with 2.0.0-alpha.13 meaning all parts of the cluster needs to updated at the same time.
-See the [migration guide](/docs/migrating-from-2.0.0-alpha.13.md) for how to update.
+See the [migration guide](https://github.com/clusterio/clusterio/blob/e0755cb4ee7bce0068c844ab6271da481a843e62/docs/migrating-from-2.0.0-alpha.13.md) for how to update.
 
 ### Major Features
 

README.md

@@ -11,6 +11,7 @@ Discord for development/support/play: https://discord.gg/5XuDkje
 ## Important notice
 
 Clusterio 2.0 is the current stable release and requires Node.js 22 or newer.
+For new installations, use the latest Node.js LTS release (see [Installation](#installation)).
 The previous stable series is no longer supported.
 If you are upgrading from a 2.0 alpha, review the [changelog](CHANGELOG.md) before upgrading.
 Plugin authors should also follow the [2.0 migration guide](docs/devs/plugin-migration-to-2.0.md).
@@ -41,7 +42,7 @@ Plugin authors should also follow the [2.0 migration guide](docs/devs/plugin-mig
 * [Setting up remote hosts](#setting-up-remote-hosts)
 * [Setting up clusterioctl](#setting-up-clusterioctl)
 * [Known issues](#known-issues)
-* [Common problems](#Common-problems)
+* [Common problems](#common-problems)
 
 
 ## Introduction
@@ -72,7 +73,7 @@ Plugins add functionality to Factorio servers, Clusterio itself or both.
 These are the plugins supported and maintained by the Clusterio developers:
 
 - [Global Chat](/plugins/global_chat/README.md): share the in-game chat between servers.
-- [Research Sync](/plugins/research_sync/README.md): synchronize research progress and technologies unlocked between servers.
+- [Research Sync](/plugins/research_sync/README.md): synchronize research progress and technologies unlocked and the research queue between servers.
 - [Statistics Exporter](/plugins/statistics_exporter/README.md): collect in-game statistics from all the servers and makes it available to the Prometheus endpoint on the controller.
 - [Subspace Storage](https://github.com/clusterio/subspace_storage): Provide shared storage that can transport items between servers via teleport chests.
 - [Player Auth](/plugins/player_auth/README.md): Provides authentication to the cluster via logging into a Factorio server.
@@ -88,6 +89,9 @@ There's also plugins developed and maintained by the community:
   See repo readme for plugin breakdown. Maintained by Cooldude2606.
 - [Chat Relay](https://github.com/PHIDIAS0303/chat_relay) (@PHIDIAS0303/chat_relay): Chat relay of Discord and Factorio, seperate channel.
 
+The installer offers the six plugins maintained in this repository.
+Community plugins are versioned and released independently of Clusterio; check their compatibility before installing them manually.
+
 Want to make your own plugin?
 Check out the documentation on [Writing Plugins](/docs/writing-plugins.md) for where to start.
 If you have a plugin written for an earlier 2.0 alpha, see [Migrating Plugins to 2.0](/docs/devs/plugin-migration-to-2.0.md).
@@ -96,8 +100,9 @@ Note that from 2.0 onwards minor versions of Clusterio may contain breaking chan
 
 ## Installation
 
-Clusterio runs on all LTS version of NodeJS, it is distributed via npm and comes with a guided install script.
-If you already have a recent Node.js installed, you can set it up in a new directory with:
+Install the latest [Node.js LTS release](https://nodejs.org/en/download), which is recommended for new Clusterio installations.
+Clusterio is distributed via npm and comes with a guided install script.
+If you already have a supported Node.js version installed, you can set it up in a new directory with:
 
     npm init "@clusterio"
 
@@ -106,12 +111,14 @@ Check out all our of decisions for [node, factorio, and operation support](/docs
 
 ### Ubuntu setup
 
-1.  Install Node.js v18 or higher.
-    Note the version of Node.js provided by the Ubuntu/debian repos are too old and you will have to use the nodesource PPA
+1.  Install the latest Node.js LTS release using the NodeSource repository for Ubuntu/Debian.
+    The distribution's default repository may provide an older version.
 
-        curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - &&\
+        curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - &&\
         sudo apt-get install -y nodejs
 
+    Verify the installation with `node --version` and `npm --version`.
+
 2.  Log in as the user you wish to install Clusterio as (do not use root), and then create a new directory and run the Clusterio installer:
 
         mkdir clusterio
@@ -143,7 +150,7 @@ Your instances (save files etc) will be stored there.
 
 ### Windows setup
 
-1.  Download and install the latest LTS release from http://nodejs.org.
+1.  Download and install the latest [Node.js LTS release](https://nodejs.org/en/download).
 
 2.  Create a new empty directory for the installation and navigate into it.
     Open a PowerShell window in this new directory by Shift+Right clicking inside it and choosing "Open PowerShell window here" and then run the following command.
@@ -157,7 +164,8 @@ Your instances (save files etc) will be stored there.
 
 ### MacOS Setup
 
-1.  Install the latest Node.js LTS release from http://nodejs.org or use brew (`brew install node`).
+1.  Install the latest [Node.js LTS release](https://nodejs.org/en/download).
+    If you use [nvm](https://github.com/nvm-sh/nvm#installing-and-updating), run `nvm install --lts`.
 
 2.  Open Terminal or Command prompt in the directory you want to install to and run the following commands.
 
@@ -211,7 +219,11 @@ See the [readme for @clusterio/controller](/packages/controller/README.md) for m
 
 ## Managing Factorio mods
 
-Mods in clusterio are managed as parts of modpacks. There is currently no mod portal integration, so you first need to navigate to the mods section of the web UI and upload your mods. You can then create a modpack from the uploaded mods by navigating to the mods page, hitting `create`. Once you have a modpack, you can assign it to an instance under Instance config -> Factorio -> Mod pack. You can also set a default modpack for new instances in the controller settings.
+Mods in Clusterio are managed as parts of mod packs.
+On the Mods page of the web interface, use **Search** to browse the Factorio Mod Portal and download mods to the controller, or upload mod zip files directly.
+Mod Portal downloads require the controller's [Factorio account credentials](docs/configuration.md#controllerfactorio_username) to be configured.
+Use **Create** in the Mod Packs section to create a mod pack, then add the mods you want to use.
+Assign the mod pack to an instance under Instance config -> Factorio -> Mod pack, or set a default mod pack for new instances in the controller settings.
 
 ![Select mod pack for instance](docs/assets/config_select_mod_pack.png)
 
@@ -227,7 +239,9 @@ Once this is done, the mod settings will show up under the modpack configuration
 ## Setting up shared storage
 
 The chests teleporting items to and from the shared storage on the controller has been moved into the Subspace Storage mod and plugin.
-To get this to work you will need to install the Subspace Storage plugin and copy the [Subspace Storage mod](https://mods.factorio.com/mod/subspace_storage) and the [Clusterio Lib mod](https://mods.factorio.com/mod/clusterio_lib) into the `sharedMods` folder on each host installation.
+To get this to work, install the Subspace Storage plugin on the controller and hosts using the [plugin installation instructions](#installing-plugins).
+Download or upload the [Subspace Storage mod](https://mods.factorio.com/mod/subspace_storage) and the [Clusterio Lib mod](https://mods.factorio.com/mod/clusterio_lib) to the controller, add them to a mod pack, and assign it to each participating instance as described in [Managing Factorio mods](#managing-factorio-mods).
+Hosts fetch the mods from the controller and synchronize each instance's mods folder with its assigned mod pack when the instance starts.
 
 Do not install the [Clusterio mod](https://mods.factorio.com/mod/clusterio), this was for the old 1.2.x version of Clusterio and is not compatible with Clusterio 2.0.
 
@@ -275,20 +289,13 @@ If you want to use a different user for the command line interface, you can gene
 
 ## Known Issues
 
-- When the `server_select` plugin is installed you may see `[error] Unhandled event server_select:UpdateInstancesEvent` logged when starting an instance.
-  This should be ignored as it does not indicate an error occured.
+- `@hornwitser/server_select@2.0.1` fails to load with Clusterio 2.0 because its entrypoints omit the required `.js` extension.
+  `@hornwitser/discord_bridge@0.2.5` is not automatically discovered because it lacks the `clusterio-plugin` keyword.
+  The Clusterio 2.0.2 installer still offers these versions; leave them unchecked until compatible plugin releases are available.
+  If already installed, stop the controller and host, run `npm uninstall @hornwitser/server_select @hornwitser/discord_bridge` in each affected installation directory, and restart them to continue with the core plugins.
+  See [#1102](https://github.com/clusterio/clusterio/issues/1102).
 - After updating Factorio you may get the error `Error: Expected empty response but got "Cannot execute command. Error: [string "clusterio_private.update_instance(..."]:1: attempt to index global 'clusterio_private' (a nil value)?` on instance start.
   This is due to Factorio applying a migration replacing the patched content of the save with the original scenario, attempting to start the instance again should resolve the issue.
-- In the mod pack view of the web interface the "base" mod is displayed with an error complaining about the mod being missing from the controller storage.
-  This is due to the "base" mod currently not being connectly handled as a built-in mod to Factorio in the web interface, this error message should be ignored.
-- After migrating from Alpha 13 to Alpha 14 running an instance data export fails with the error `Upload failed: 401 Unauthorized` due to the host token being issued to a slave.
-  To workaround this find the id of the host in the host config and generate a new host token with the same id.
-  Stop the host and replace the value of the `host.controller_token` field in `config-host.json` with the new host token before starting it up again.
-- After migrating from Alpha 13 to Alpha 14 you get kicked out from the web interface due to the master to controller rename.
-  If you don't have the authentication token readily available you can manually migrate the token by opening the Inspect/Developer Tools on the page and running the following piece of JavaScript in the Console:
-  ```
-  localStorage.controller_token = localStorage.master_token
-  ```
 
 
 ## Common problems

docs/assets/clusterio_2.0_block_diagram.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2023-07-15T21:19:41.336Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/20.8.16 Chrome/106.0.5249.199 Electron/21.4.0 Safari/537.36" etag="WQippoIg5xK5LnRPeLWb" version="20.8.16" type="device"><diagram id="wKlty5uROKaaHdmDaOKw" name="Page-1">5Vtdk6I4FP01Ps4UEEB9bO12Pmp3qmvdqnlOQ4R0R+KE2Or++k0kgBBEelTQoR9suFwgOeee5N4YB2C63H5hcBX+TX1EBpbhbwfgcWBZznAsPqVhlxhsABJDwLCfmMzcMMf/IWU0lHWNfRQXHDmlhONV0ejRKEIeL9ggY3RTdFtQUnzrCgZIM8w9SHTrT+zzUHXLMHL7V4SDkJcuLGHqqwxxCH26OTCBpwGYMkp5crTcThGR0KWwJPfNjlzN2sVQxJvc4Lz9M3Hs7zvzZR7gHwEh3/59/OQodt4hWasOq9byXYoAo+vIR/IpxgBMNiHmaL6Cnry6EZQLW8iXRJyZ4jDrpfQNCIxjdRy/Ie6F6mSBCZlSQpk4j2gkHjWBBAeROCVowaU7Z/QNlXx8zATFmEq/mK4lwBMdBoXMO2IcbQ9MCpYviC4RZzvhsqkgNKwiE6ogCrJ7c6DFgcL6A7hnjWyEu3ka90NEBxbwIRotPAkZjMPsGUVQhZvrjdDLojFtK8Sw6D9ish04CpQZMk/p1j2TEHU1lYwaMobq9ICuzOWQLmBdiy5XY2tKIwEmIYgJ+zOjHhKQnaWcEoOG4RhoUUWaYZiPk6m8Q7ThwL7Y/12GAbvIwFhnwDSrGLgWAeZYAxf5YpxWp5TxkAY0guQpt06K8Oc+f1G6UqC/Is53KnjhmtNaSswXaCKrmhL36WFWB70YrpiHanqoOsghCxA/HYqy97VEMkQgx+/FWayKl/2tD4zB3YHDiuKIxwdPfpaGPD7cYnyYdmnu+ZC7OEjen8dH1pEzhlj3BkJmAd2RYVSFzHQInP0VFPkPMk/JpzlhmWHZ2X0jXtfLVdpo5fD7UWaBhmE2vokwGxXjxho6LcQN0Ab7GfQ4ZZgK6xyxdznmX3CkbxAjVxzps3E8xdjteqy3bmGsv4Bwz1CpfVcqHRYjCJhtqNTutUqB0blKR7ek0lNaPE7aGSp17kqlpRwMOG2o1Om3Su2uVQp0Amrw7t06Q7kuGTVcachWJy9OmG3fwLB61UIXuA2HzduodLOZNo0Rt77UHTt17tcZZsHwBoKm04wZjBoGFbiRqLI6qGxTkHoyG4s3FDHuvLIFva9s04TobnRazujaqG1TkHqq0+5rW9vU4O2bTq0706nRQXWbgtRXnXZe3dq38BXPVYslu+ka08VleB4x+rLDT/QiDN8iUdUvZLjLBHgmPj2yjoUNU4+Ti4qlAS9XFEv5q6uqb9DdVrVStdPHJVzBUMDd/bWm6YVP8V4HD8JBQLLNL4qjQP6fpgRKj8+i+caEUO9N/H/EMGBwmb5GNDt5U3KfRrZAlhcZrdwEdHrvkGQJe5A8KPMS+/5e+lUBVAyxZutTlxhMS7VJxcLTuM3wcP7MDUnWhegyh26BLxvohDkVfF1tR5Kj5x7P4S6WYX889+izwqoYq8xWrkeZ/pWlnr7kGb0nscFeraoabBI7iuTJVOFEaKe2jyX2WuZum6MiT3YJ/yRHUnfVLLza7okHJTmU9qBLVQNOZc4zF1Mhkq1U+5Pl5tVey9IaWqd1CVrVpb67s2bqu7ednNrMVbEK2u7MNdTg/kF99PlVlPdHd9L2SiLlmauCsbHbImNAF8hXGvMauu5XLtqWw84XI/W67Y8F324Pe3Ga//QmSQHyny+Bp/8B</diagram></mxfile>

docs/assets/clusterio_2.0_block_diagram.svg

@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg
+  PUBLIC '-//W3C//DTD SVG 1.1//EN'
+  'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd'>
+<!-- Exported from clusterio_2.0_block_diagram.drawio with draw.io Desktop 32.4.1. Export settings: SVG, light theme, white background, width 1000. Native SVG labels: disable Formatted Text and Word Wrap in an export copy, preserving the title at font size 20. -->
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="background: #ffffff; background-color: #ffffff; color-scheme: light;" version="1.1" width="1001px" height="1001px" viewBox="0 0 1001 1001" role="img" aria-labelledby="diagram-title diagram-description">
+  <desc id="diagram-description">A controller and two hosts run across two physical servers. Each host manages three Factorio servers. The hosts and web or command-line clients connect to the controller over WebSocket.</desc>
+  <title id="diagram-title">Clusterio 2.0 system architecture</title>
+  <defs/>
+  <rect fill="#ffffff" width="100%" height="100%" x="0" y="0" style="fill: rgb(255, 255, 255);"/>
+  <g>
+    <g data-cell-id="0">
+      <g data-cell-id="1">
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-59">
+          <g>
+            <rect x="0" y="0" width="1000" height="1000" fill="none" stroke="none" transform="rotate(90,500,500)" pointer-events="all"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-21">
+          <g>
+            <rect x="40" y="140" width="440" height="640" rx="26.4" ry="26.4" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-dasharray="6 6" pointer-events="all" style="fill: rgb(218, 232, 252); stroke: rgb(108, 142, 191);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-6">
+          <g>
+            <rect x="80" y="180" width="220" height="60" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="190" y="220">Controller Process</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-19">
+          <g>
+            <path d="M 120 400 L 120 280 L 120 252.74" fill="none" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 110, 175);"/>
+            <path d="M 120 242.24 L 127 256.24 L 120 252.74 L 113 256.24 Z" fill="#006eaf" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 110, 175); stroke: rgb(0, 110, 175);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-26">
+          <g>
+            <path d="M 220 550 L 160 550 L 160 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-23">
+          <g>
+            <rect x="220" y="520" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="330" y="560">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-29">
+          <g>
+            <path d="M 220 630 L 140 630 L 140 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-24">
+          <g>
+            <rect x="220" y="600" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="330" y="640">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-28">
+          <g>
+            <path d="M 220 710 L 120 710 L 120 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-25">
+          <g>
+            <rect x="220" y="680" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="330" y="720">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-35">
+          <g>
+            <rect x="520" y="360" width="440" height="420" rx="25.2" ry="25.2" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-dasharray="6 6" pointer-events="all" style="fill: rgb(218, 232, 252); stroke: rgb(108, 142, 191);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-44">
+          <g>
+            <path d="M 600 400 L 600 320 L 190 320 L 190 252.74" fill="none" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 110, 175);"/>
+            <path d="M 190 242.24 L 197 256.24 L 190 252.74 L 183 256.24 Z" fill="#006eaf" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 110, 175); stroke: rgb(0, 110, 175);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-37">
+          <g>
+            <path d="M 700 550 L 640 550 L 640 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-38">
+          <g>
+            <rect x="700" y="520" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="810" y="560">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-39">
+          <g>
+            <path d="M 700 630 L 620 630 L 620 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-40">
+          <g>
+            <rect x="700" y="600" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="810" y="640">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-41">
+          <g>
+            <path d="M 700 710 L 600 710 L 600 460" fill="none" stroke="#c73500" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(199, 53, 0);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-42">
+          <g>
+            <rect x="700" y="680" width="220" height="60" fill="#fa6800" stroke="#c73500" stroke-width="2" pointer-events="all" style="fill: rgb(250, 104, 0); stroke: rgb(199, 53, 0);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="810" y="720">Factorio Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-46">
+          <g>
+            <path d="M 560 210 L 312.74 210" fill="none" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 110, 175);"/>
+            <path d="M 302.24 210 L 316.24 203 L 312.74 210 L 316.24 217 Z" fill="#006eaf" stroke="#006eaf" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 110, 175); stroke: rgb(0, 110, 175);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-45">
+          <g>
+            <rect x="560" y="180" width="320" height="60" fill="#1ba1e2" stroke="#006eaf" stroke-width="2" pointer-events="all" style="fill: rgb(27, 161, 226); stroke: rgb(0, 110, 175);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="720" y="220">Web Interface / clusterioctl</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-49">
+          <g>
+            <rect x="60" y="40" width="580" height="60" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="40px" style="fill: rgb(0, 0, 0);">
+              <text x="64" y="88">Clusterio 2.0 Block Diagram</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-51">
+          <g>
+            <rect x="352" y="860" width="100" height="40" rx="8" ry="8" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-dasharray="6 6" pointer-events="all" style="fill: rgb(218, 232, 252); stroke: rgb(108, 142, 191);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-52">
+          <g>
+            <rect x="60" y="860" width="220" height="40" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="24px" style="fill: rgb(0, 0, 0);">
+              <text x="64" y="890">Physical Server</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-54">
+          <g>
+            <path d="M 836 880 L 923.26 880" fill="none" stroke="#001dbc" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke" style="stroke: rgb(0, 29, 188);"/>
+            <path d="M 933.76 880 L 919.76 887 L 923.26 880 L 919.76 873 Z" fill="#001dbc" stroke="#001dbc" stroke-width="2" stroke-miterlimit="10" pointer-events="all" style="fill: rgb(0, 29, 188); stroke: rgb(0, 29, 188);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-55">
+          <g>
+            <rect x="544" y="860" width="260" height="40" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="24px" style="fill: rgb(0, 0, 0);">
+              <text x="548" y="890">WebSocket connection</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-56">
+          <g>
+            <rect x="352" y="920" width="100" height="40" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-57">
+          <g>
+            <rect x="60" y="920" width="192" height="40" fill="none" stroke="none" pointer-events="all"/>
+          </g>
+          <g>
+            <g fill="#000000" font-family="Helvetica" text-anchor="start" font-size="24px" style="fill: rgb(0, 0, 0);">
+              <text x="64" y="950">Node.js Process</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-36">
+          <g>
+            <rect x="560" y="400" width="220" height="60" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="670" y="440">Host Process</text>
+            </g>
+          </g>
+        </g>
+        <g data-cell-id="5kRB54Jy1bSgiNgllITD-9">
+          <g>
+            <rect x="80" y="400" width="220" height="60" fill="#0050ef" stroke="#001dbc" stroke-width="2" pointer-events="all" style="fill: rgb(0, 80, 239); stroke: rgb(0, 29, 188);"/>
+          </g>
+          <g>
+            <g fill="#ffffff" font-family="Helvetica" text-anchor="middle" font-size="24px" style="fill: rgb(255, 255, 255);">
+              <text x="190" y="440">Host Process</text>
+            </g>
+          </g>
+        </g>
+      </g>
+    </g>
+  </g>
+</svg>

docs/communication.md

@@ -1,36 +1,57 @@
 # Communication Layer
 
-<sub>This document describes the communication layer in Clusterio and unless you're developing Clusterio it's probably not going to be very useful for you.</sub>
-
-Communication is facilitated by the lib/link library and is based on three core abstractions: links, connectors and messages.
-Links represents one side of a two way communication pipe, connectors deal with connecting and establishing communication, and messages are the content sent over the link.
+Clusterio uses links, connectors, and message classes to communicate between controllers, hosts, instances, and control clients.
+A link sends and receives messages through a connector, while message classes define the requests and events that can be exchanged.
+See the [system overview](how-it-works.md#system-overview) for a diagram of the processes and their connections, and the [protocol reference](devs/protocol.md) for wire formats and session flow.
 
 
-## Link
+## Links and routing
 
-The Link class represents one side of a two way communication channel where the actual communication is done by a connector associated with the link.
-The link also knows what the endpoint types it's connected to are via the source and target attributes.
+[Link](../packages/lib/src/link/link.ts) handles request/response matching, event dispatch, message registration, and routing through an attached router.
+Its connector identifies the local and remote endpoints through `connector.src` and `connector.dst`, both `Address` objects.
 
-The endpoint types currently implemented are controller, host, instance, and control.
-Although host and instances run in the same Node.js program they use virtual links to communicate between them in order to simplify the communication architecture.
+The controller's `WsServer` stores one link for each host or control connection.
+On a host, the `Host` object is a link to the controller, and each `Instance` is linked to a corresponding `InstanceConnection` stored on the host.
+The command-line client's `Ctl` object and the web interface's `Control` object each act as a link to the controller.
 
-Upon creation the link will register with the connector in order to receive and process messages from it.
-The messages received are validated using the validator registered for the type of the message.
-A message for which there is no validator for is considered an error.
+Requests can pass through several links before reaching their destination.
+Each receiving link checks the message name, allowed endpoint types, and source address before routing it.
+The controller checks the user's permissions before validating the message data; permission callbacks therefore receive untrusted data.
+See [message permissions](writing-plugins.md#static-permission).
+
+At the destination, the registered schema validates the payload and `fromJSON` restores the message before its handler runs.
+
+Routing can perform work before the destination handler runs.
+In particular, the [host router](../packages/host/src/Host.ts) can start or wake an instance for messages marked as instance-starting.
 
 
-## Connector
+## Connectors and sessions
 
-The connector used with a link is responsible for establishing and/or maintain the connection that messages are sent over.
-Reconnect logic is expected to be implemented in the connector, though this has not yet been done.
+[Connectors](../packages/lib/src/link/connectors.ts) transport messages, assign sequence numbers, and deliver incoming messages to links.
+WebSocket connectors connect hosts and control clients to the controller and validate incoming message structures, including their type and addresses.
+Hosts and instances share a Node.js process and communicate in memory through paired `VirtualConnector` objects.
 
-There's also the virtual connector which lets two links in the same program be connected to each other without having to establish a network connection.
+The protocol reference describes [connection and recovery behavior](devs/protocol.md#connection-flow), [connector states and events](devs/protocol.md#connector-flow), and [graceful disconnects](devs/protocol.md#graceful-disconnect).
+For handling these events in a plugin, including queueing work and preparing for disconnection, see [connection handling](writing-plugins.md#handling-connection-events).
 
 
-## Message
+## Message definitions and registration
 
-There's two types of messages currently implemented apart from the Link builtin close message and that's Request and Event.
-The builtin messages are defined in [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) and are attached to links via the attachAllMessages function, which is usally called in the Link subclass constructor.
+Core request and event classes live under [packages/lib/src/data/](../packages/lib/src/data/).
+Their static `type`, `src`, and `dst` properties specify the message type and allowed endpoint types, while their schema and permission fields define the payload and access rules.
+Plugin classes also supply a `plugin` name.
 
-The messages define which links they are valid on and only attaches handlers for those links.
-There's also a forwarding mechanism for when a request or event is to be forwarded another hop.
+`Link.register` registers request and event classes and compiles their payload validators.
+Core classes are registered by the link module; `registerPluginMessages` registers each plugin's `messages` array.
+Incoming messages invoke handlers attached with `link.handle(MessageClass, handler)`.
+Plugins attach these handlers from their entrypoint functions rather than relying on method-name conventions.
+
+Send a message with `link.sendTo(destination, new MessageClass(...))`.
+Requests return a promise for the response; events do not return a handler result.
+Use broadcast destinations only for events.
+When using a request's source as a later event destination, remove its request ID with `Address.withoutRequestId()`.
+See [writing plugins](writing-plugins.md#defining-link-messages) for a complete example.
+
+When changing a wire message structure in [messages_core.ts](../packages/lib/src/data/messages_core.ts), rebuild the packages with `pnpm -r run prepare` to regenerate the standalone validator.
+When changing a request or event class, keep serialization, allowed endpoints, permission checks, registration, and handlers consistent.
+The [contributor guide](contributing.md#checking-your-code-for-errors) explains how to run focused tests.

docs/config-system.md

@@ -3,18 +3,18 @@
 <sub>This document describes the implementation and inner working of the configuration system in Clusterio, and unless you're developing Clusterio it's probably not going to be very useful for you.</sub>
 
 Clusterio uses a field based configuration system to manage settings for the controller, instances, hosts, and the clusterioctl utility.
-Plugins can add their own config fields for the controller, host and instance configs.
+Plugins can add their own config fields for the controller, host, instance, and ctl configs.
 The fields are defined early in the startup, and fields for disabled plugins will still be created.
 
-The built-in config fields are defined in [packages/lib/src/config/definitions.ts](/packages/lib/src/config/definitions.ts) while plugins can add their own config field definitions to the `controllerConfigFields`, `hostConfigFields` and `instanceConfigFields` properties of the `plugin` export.
-A `<plugin_name>.load_plugin` field is automatically created for each plugin and it's used to enable/disable the loading plugins on startup.
+The built-in config fields are defined in [packages/lib/src/config/definitions.ts](/packages/lib/src/config/definitions.ts) while plugins add fields through `controllerConfigFields`, `hostConfigFields`, `instanceConfigFields`, and `ctlConfigFields` on the `plugin` export.
+Controller, host, and instance configs also receive `<plugin_name>.load_plugin` to enable/disable loading on startup and a hidden `<plugin_name>.version` for migrations.
 
 
-## Defing Fields
+## Defining Fields
 
 Fields are defined as POJS object properties where the name of the property correspond to the name of the configuration field.
 The property name is the primary way the field is accessed both in code and on the command line, as such it should follow the lower\_case\_underscore style and consist of a group name and a field name separated by a dot.
-For plugins the group name is the name of the plugin, e.g., the `level` field of the `foo_frobber` plugin would be named `foo_frobber.lavel`.
+For plugins the group name is the name of the plugin, e.g., the `level` field of the `foo_frobber` plugin would be named `foo_frobber.level`.
 
 **access**:
     The locations this config field can be read and modified from.
@@ -44,7 +44,7 @@ For plugins the group name is the name of the plugin, e.g., the `level` field of
 
 **initialValue** (optional if optional is true):
     Value this config field should take in a newly initialized config.
-    This can also be an async function returning the value to use.
+    This can also be a synchronous function returning the value to use.
 
 **validator** (optional):
     Function called with the value being set and the config it belongs to.
@@ -83,7 +83,22 @@ For plugins the group name is the name of the plugin, e.g., the `level` field of
 
 **inputComponent** (optional):
     Name of the input component the web interface should render for this field instead of the default for its type.
-    Plugins can add their own components through `inputComponents` on their web plugin class.
+    Plugins register components with the `control.hooks.inputComponents` hook in their web entrypoint.
+    The hook returns an object whose keys match the field's `inputComponent` value.
+
+For example, with `inputComponent: "foo_frobber_level"` on a field:
+
+```js
+import LevelInput from "./LevelInput.js";
+
+export default async function({ control, plugin }) {
+    control.hooks.inputComponents.attach(plugin.name, () => ({
+        foo_frobber_level: LevelInput,
+    }));
+}
+```
+
+`LevelInput` must implement the `InputComponent` contract exported by `@clusterio/web_ui`.
 
 If the value for a field stored in on disk somehow ends up being invalid the field will take on the initial\_value instead when the config is loaded.
 

docs/configuration.md

@@ -606,6 +606,23 @@ Like factorio.enable_save_patching, plugins that need script commands can't be l
 Defaults to true.
 
 
+### factorio.enable_lua_udp
+
+Enable UDP messaging between the host and the Factorio server by passing `--enable-lua-udp` to Factorio.
+This lets plugins send UDP packets to the server and receive data sent with `clusterio_api.send_udp` in-game.
+Requires Factorio 2.1.10 or later, earlier versions crash when a packet is received on a headless server.
+
+Defaults to false.
+
+
+### factorio.lua_udp_port
+
+UDP port on localhost the Factorio server receives Lua UDP messages on when `factorio.enable_lua_udp` is enabled.
+If null a random port in the dynamic range is assigned each time the instance starts.
+
+Defaults to null.
+
+
 ### factorio.enable_whitelist
 
 Turn on the whitelist on the server.

docs/contributing.md

@@ -3,23 +3,27 @@
 You are probably reading this because you want to contribute code to Clusterio.
 Great!  We need people like you.
 Though we have some conventions and workflows that we kindly ask you to follow.
-We have included all the pratical considerations in this document.
-Although, we recomend checking out out our [past decisions](/docs/decisions.md) if you have the time.
+This document covers the development workflow, commands, and repository layout.
+Read our [past decisions](/docs/decisions.md) for compatibility policies and design rationale.
 
 
 ## Contents
 
+- [Project map](#project-map)
 - [Workflow](#workflow)
     - [Starting a Feature Branch](#starting-a-feature-branch)
     - [Setting up the Development Environment](#setting-up-the-development-environment)
+    - [Build outputs](#build-outputs)
+    - [Debugging](#debugging)
     - [Checking your code for errors](#checking-your-code-for-errors)
+    - [Cleaning generated and runtime files](#cleaning-generated-and-runtime-files)
     - [Submitting a Pull Request](#submitting-a-pull-request)
     - [Editing your Feature Branch](#editing-your-feature-branch)
     - [Resolving Merge Conflicts](#resolving-merge-conflicts)
     - [Updating Your GitHub Fork](#updating-your-github-fork)
     - [Publishing releases to npm](#publishing-releases-to-npm)
 - [Changelog](#changelog)
-- [Supported Node.js and Factorio Version](#supported-node.js-and-factorio-version)
+- [Supported Node.js and Factorio Versions](#supported-nodejs-and-factorio-versions)
 - [Code Style](#code-style)
     - [Naming Style](#naming-style)
     - [Units](#units)
@@ -28,6 +32,28 @@ Although, we recomend checking out out our [past decisions](/docs/decisions.md)
     - [Markdown](#markdown)
 
 
+## Project map
+
+Clusterio manages distributed Factorio servers through a controller, hosts, and plugins.
+
+| Location | Responsibility |
+| --- | --- |
+| `packages/controller/` | Cluster state, users and roles, connections, HTTP API, and serving the web UI. Start with `src/Controller.ts` and `src/ControlConnection.ts`. |
+| `packages/host/` | Instance lifecycle, Factorio processes, save patching, and exports. Start with `src/Host.ts`, `src/Instance.ts`, `src/server.ts`, and `src/patch.ts`. |
+| `packages/ctl/` | Command-line administration and plugin commands. |
+| `packages/lib/` | Shared messages, routing, config, permissions, plugin loading, logging, and utilities for Node and browser consumers. |
+| `packages/web_ui/` | React/Ant Design interface and browser control connection; bundled through the controller's webpack build. |
+| `packages/create/` | Installer and JavaScript/TypeScript plugin templates. |
+| `plugins/` | Bundled plugins, including their Node, web, Lua, and test code. |
+| `external_plugins/` | Plugins developed in independent repositories and included in the local workspace. |
+| `packages/host/modules/` | Lua modules injected into saves. |
+| `packages/host/lua/` | Factorio mod sources, including `clusterio_lib`. |
+| `test/` | Root test suite, fixtures, helpers, integration harness, and manual tests. Plugin tests also live under `plugins/*/test/`. |
+
+Start with [how it works](how-it-works.md), [communication](communication.md), and the [config system](config-system.md) for architecture.
+See [writing plugins](writing-plugins.md) and [developing for Clusterio](developing-for-clusterio.md) for plugin and Factorio contracts.
+
+
 ## Workflow
 
 While we don't require you to follow this workflow, it makes things much simpler for both you and us if you choose to do so.
@@ -52,16 +78,20 @@ Do not run `git pull upstream` it will mess thing up!
 
 ### Setting up the Development Environment
 
-The project is organized into several packages that are all tied together using pnpm's workspace feature.
-In order to get the development environment up and running you will need to run the following command:
+Run commands from the repository root unless stated otherwise.
+Use the latest Node.js LTS release and the pnpm version pinned by `packageManager` in root `package.json`.
+Install dependencies with:
 
-    pnpm install
+    pnpm install --no-frozen-lockfile
 
-This installs dependencies needed by tests and links the packages up so they work from the git work tree.
-To start a cluster from the repo you will first need to initialize it, this can be done using the installer:
+This installs dependencies and builds the packages.
+`pnpm-lock.yaml` is intentionally ignored; do not force-add it or assume a frozen-lockfile install is supported.
+For a fresh development cluster, run the installer and select standalone mode to create `config-controller.json`, `config-host.json`, and the credentials needed to connect them:
 
     node packages/create --dev
 
+You can skip this step if you already have a working controller and host configuration.
+
 To add plugins to use in the development environment use
 
     node packages/ctl plugin add ./path/to/plugin
@@ -79,9 +109,10 @@ By default node does not enable source maps, to get stack traces pointing to the
 
 Note that to use `clusterioctl` you will have to create a ctl config first:
 
-    node packages/controller bootstrap create-ctl-config [admin-user]
+    node packages/controller bootstrap create-ctl-config <admin-user>
 
-Once you've set up the cluster you can use `node packages/controller run` to run the controller and `node packages/host run` to run the host which connects to the controller.
+Use the name of an existing admin user.
+Run `node --enable-source-maps packages/controller run` and `node --enable-source-maps packages/host run` in separate terminals.
 
 When editing TypeScript files ran by Node.js you need to run `pnpm build` to compile the files to JavaScript before restarting the controller/host/ctl for changes take effect.
 To have TypeScript watch for changes and rebuild immediadly you may run `pnpm watch` instead.
@@ -94,14 +125,46 @@ For web development on the controller there are also the following flags:
 
 These options can be combined and multiple plugins can be actived for dev mode.
 Keep in mind if you're working on external plugins the `--dev-plugin` option is likely to only work if the plugin is placed inside the `external_plugins` directory and its dependencies are installed with pnpm as part of the clusterio repository workspace.
-Note that webpack is configured to delete the web interface build when using these options, so you will have to run `pnpm run -r --if-present prepare` before you start the controller without these flags again.
+Note that webpack is configured to delete the web interface build when using these options, so you will have to run `pnpm -r run prepare` before you start the controller without these flags again.
+
+
+### Build outputs
+
+| Command | Output |
+| --- | --- |
+| `pnpm build` | Compiled JavaScript and type declarations in each package's `dist/node/` and, where applicable, `dist/browser/`. |
+| `pnpm watch` | TypeScript compilation as files change; restart affected Node processes afterward. |
+| `pnpm -r run prepare` | Compiled packages, generated validators, and production web bundles. |
+| `pnpm build-mod` | Factorio mod archives in root `dist/`. |
+| `pnpm docs` | Generated API documentation. |
+
+`pnpm build` alone does not regenerate the standalone validator or webpack bundles.
+After changing `packages/lib/src/data/messages_core.ts`, run `pnpm -r run prepare` to compile the schema and regenerate its validator.
+
+
+### Debugging
+
+Establish the controller, host, instance, Factorio version, enabled plugins, and exact reproduction first.
+Enable source maps and confirm the affected process is running rebuilt code.
+Use the contextual logger so host, instance, and plugin metadata remain available.
+Development logs are in `logs/`; integration logs are in `temp/test/logs/` and are uploaded by CI.
+
+For a failing request or event, start with its handler and the input that triggers the failure.
+Handlers are registered with `handle(MessageClass, handler)`; searching for the message class will help you find its handler.
+Inspect the input and the state used by the failing operation, then try to reproduce the failure in a [focused test](#checking-your-code-for-errors).
+If the handler is never reached, investigate message registration, routing, and permission checks using the [communication guide](communication.md).
+For errors reported by Factorio, start with the Lua stack trace and the module or mod it identifies; the [developing-for-Clusterio guide](developing-for-clusterio.md) explains how that code communicates with the host.
+
+For connection and recovery bugs, test disconnect, reconnect, timeout, and shutdown as applicable.
+A cached controller status does not prove the current state of a host or Factorio process.
+Recovery mode disables normal plugin loading; account for it when diagnosing missing plugin behavior and [Lua state migrations](developing-for-clusterio.md#initialisation-and-recovery).
+Verify UI behavior in the browser: successful compilation does not verify rendering, permissions, or subscriptions.
 
 
 ### Starting a Feature Branch
 
 Development is done exclusively in feature branches.
-This means the first thing to do before starting any development at all is to make a new branch starting from the master branch of the main repository.
-To do this check out that branch from the main repository.
+To start a new feature branch, first check out master from the main repository.
 
     git fetch upstream
     git checkout upstream/master
@@ -117,18 +180,61 @@ But it's nice to have its name be somewhat descriptive of what it implements.
 
 ### Checking your code for errors
 
-There's automated tests and linting setup for the project.
-Please do add tests for any code you add and run both:
+Tests use Mocha and Node assertions, mostly as JavaScript ESM importing compiled packages.
+Run the test suite and linter from the repository root:
 
     pnpm run test
     pnpm run lint
 
-To check that your changes pass the integration tests as well as the ESLint rules set up for the project.
+`pnpm test` builds TypeScript and runs root and bundled-plugin tests, excluding `test/manual/`.
+Run tests from the root: individual package `test` scripts are often placeholders that deliberately fail.
+Use `pnpm cover` for the full suite with c8 coverage.
 
 The full test suite can take a while to run.
 Use `pnpm run fast-test` to skip tests marked as slow, or `pnpm run no-external-test` to skip tests that depend on external APIs such as the Factorio mod portal or GitHub.
 These correspond to setting the `FAST_TEST` and `NO_EXTERNAL_TEST` environment variables and can be combined.
-When writing tests that take a lot of time call `slowTest(this)` at the start of the test, and for tests that call out to external APIs call `externalTest(this)`, both are exported from `test/integration/index.js`.
+For example, `FAST_TEST=y NO_EXTERNAL_TEST=y pnpm test` combines both exclusions.
+These flags still run the remaining integration setup and tests; neither produces an isolated unit-test run.
+When writing slow tests call `slowTest(this)`, and for external API tests call `externalTest(this)`; both are exported from `test/common.js`.
+For tests that need a Factorio installation, call `requiresFactorio(this)` in the test or pass the suite to `requiresFactorio`, imported from `test/integration/index.js`.
+`test/common.js` also provides JSON round-trip/schema helpers, and `test/mock.js` provides shared mocks.
+
+For a focused test, build first and select the file explicitly instead of appending it to the full-suite script:
+
+```sh
+pnpm build
+NODE_EXTRA_CA_CERTS="$PWD/test/file/tls/cert.pem" pnpm exec mocha \
+  --enable-source-maps --check-leaks test/lib/helpers.js
+```
+
+Replace the example file with the relevant test; use `--grep` to narrow it further.
+Preserve the test TLS CA configuration for HTTPS integration tests.
+Use `--require ./test/setup_logging.js` when an isolated test needs the shared logger configured.
+Some tests import `test/integration/index.js`, which installs root hooks and starts a controller and host even for a targeted invocation.
+
+The integration harness recreates its configs, database, mods, and instances under `temp/test/` and uses HTTP port 8880 and HTTPS port 4443.
+Do not run competing integration suites in the same checkout or on those ports.
+It locates Factorio through `host.factorio_directory` in root `config-host.json`, defaulting to `factorio/`.
+The full harness currently requires a usable Factorio installation: a missing directory fails discovery, and an empty directory fails setup while selecting a version.
+Some individual tests have conditional skips, but these do not make a full run without Factorio succeed.
+
+Use the relevant production bundle, mod build, or [manual scenario](../test/manual/README.md) for changes affecting those surfaces.
+For documentation changes, check that links, paths, and example commands are correct.
+
+
+### Cleaning generated and runtime files
+
+Inspect `node clean.js --help` and preview the selected scope with `--dry-run` before cleaning.
+
+| Command | Removal scope |
+| --- | --- |
+| `node clean.js --dry-run` | Preview workspace package/plugin build output and webpack caches; omit `--dry-run` to clean, or use `pnpm clean`. |
+| `node clean.js tests --dry-run` | Also preview root `.nyc_output/`, `coverage/`, the whole `temp/` directory, and `static/`. |
+| `node clean.js all --dry-run` | Also preview runtime databases, instances, logs, mods, local configs, `FactorioAdminToken.txt`, and `plugin-list.json`. |
+
+`all` deletes cluster runtime data, including configurations and saves; back up anything you want to keep before using it.
+The cleaner excludes `external_plugins/` and does not remove mod archives in root `dist/`.
+Do not commit local configs, tokens, databases, saves, logs, or generated build output.
 
 
 ### Submitting a Pull Request
@@ -200,12 +306,14 @@ The format of this log should be self explanatory, please follow it carefully. P
 
 ## Supported Node.js and Factorio Versions
 
-Clusterio run on Node.js v18 and up, make sure to check that the Node builtin API's and npm packages you use are supported on these versions.
-The GitHub Actions tests are run on whatever version of Node.js v18 and v20 is the default.
+For development, use the latest Node.js LTS release, following the [installation instructions](../README.md#installation).
+Make sure the Node.js built-in APIs and npm packages you use remain compatible with the supported versions.
+The GitHub Actions tests currently run on Node.js 24 and 26 across the Factorio matrix, with Node.js 22 used for coverage; see the [CI workflow](../.github/workflows/node.js.yml) for the version matrix.
 
-For Factorio Clusterio 2.0 aims to support version 0.17.69 and up, including the latest experimental release.
-It's recommended that you use the lua API reference for 0.17.69, as there's no information on what version Factorio API's were introduced in.
-The GitHub Actions tests are run against the latest experimental release.
+For Factorio, active support covers the latest stable version and the previous major version, using version lines such as 2.1, 2.0 and 1.1 as described in the [Factorio support policy](decisions.md#factorio).
+Older versions from 0.17 onwards receive passive support where it does not restrict actively supported versions.
+Check the Lua API documentation for the Factorio versions your changes target and preserve compatibility with the actively supported versions.
+The GitHub Actions tests use pinned Factorio releases; see the [CI workflow](../.github/workflows/node.js.yml) for the current version matrix.
 
 Check out all our of decisions for [node, factorio, and operation support](/docs/decisions.md#supportability).
 
@@ -226,13 +334,10 @@ Fields in configs uses lowercase_underscore.
 JavaScript files are named using lowercase_underscore for files that export multiple items.
 Files containing and exporting a single class should be named the same as the class in PascalCase.
 
-Relative imports in lib package are prefixed with lib to make them easier to distinguish, as the names tend to be very generic.
-E.g.:
-
-    import libLink from "./link";
-    import libLuaTools from "./lua_tools";
-
-Note that no sub-modules of the lib package is imported directly from the outside of it with the exception of `build_mod`.
+Use ESM imports, including `.js` extensions in TypeScript relative imports.
+For namespace imports within lib, use the `lib` prefix to distinguish generic module names, for example `import * as libFileOps from "./file_ops.js";`.
+Production consumers should import shared public APIs through `@clusterio/lib` rather than reaching into its internals.
+When adding public shared exports, check both `packages/lib/index.ts` and `packages/lib/browser.ts` and keep Node-only dependencies out of browser code.
 
 For lua code lowercase_underscore is used for everything.
 
@@ -241,7 +346,7 @@ For lua code lowercase_underscore is used for everything.
 When naming variables whos unit is not a base SI unit like seconds or metres the unit used should be suffixed in the variable name:
 E.g.:
 
-    const startTimeMs = date.now();
+    const startTimeMs = Date.now();
     const endNs = process.hrtime();
 
 This makes it easier to spot different unit bases are being mixed in the code.
@@ -282,7 +387,7 @@ For example:
 ### Markdown
 
 Markdown files are formatted using one line per sentence to improve diff reports when making changes.
-Headings have two spaces before them and on space after them to make them stand out, the heading at the top of the file does not have any spaces before it, and heading immediadly followed by another heading have one space between them.
+Headings have two blank lines before them and one after them; the top heading has no blank lines before it, and consecutive headings have one blank line between them.
 For example:
 
 ```md

docs/decisions.md

@@ -452,11 +452,13 @@ While we aim to increase the number of useful checks performed by CI in the futu
 
 🧾 Time Immemorial
 
-> We do not use any CD actions, and have no plans of implementing them.
+> We publish packages and mods when a maintainer publishes a stable GitHub release.
 
-Unlike CI, we do not find continuous deployment useful at our current stage of the project.
-This is due to the frequent breaking changes merged into the main branch, which we prefer to bundle together as a single version.
-In the future, we may automate the deployment process, but it will always require manual activation.
+Changes merged into the main branch are bundled into releases before publication.
+Publishing a stable GitHub release triggers the [Publish workflow](../.github/workflows/publish.yml), which publishes npm packages and Factorio mods.
+Publishing a prerelease or pushing a tag alone does not trigger publication.
+Running clusters are updated by their operators.
+See the [release workflow](release-workflow.md) for the publication procedure.
 
 
 ### Mocha
@@ -475,5 +477,5 @@ However, we are always looking for a better framework which is worth the effort
 
 > We perform all tests through GitHub actions using Ubuntu latest, using all supported versions of node and all supported versions of factorio.
 
-We perform tests on all versions of NodeJS and Factorio that we support.
+The [current CI workflow](../.github/workflows/node.js.yml) tests selected Node.js major versions and pinned Factorio releases.
 However, we do not conduct tests on all supported operating systems, as the added demand on our testing resources would be significant, and the value it would provide for a cross-platform engine like NodeJS would be limited.

docs/developing-for-clusterio.md

@@ -10,7 +10,9 @@ This document describes the various different types of content that can be creat
     - [event_handler interface](#event_handler-interface)
 - [Communicating with Clusterio](#communicating-with-clusterio)
 - [Clusterio Modules](#clusterio-modules)
+    - [Initialisation and recovery](#initialisation-and-recovery)
 - [Clusterio Lua API](#clusterio-lua-api)
+    - [Compatibility library](#compatibility-library)
 - [Clusterio Plugins](#clusterio-plugins)
 
 
@@ -73,7 +75,7 @@ The following entries are recognized by `add_lib`, all of which are optional:
     Callback called when the callback of `script.on_configuration_changed` is invoked.
 - `events`:
     Table mapping event ids as defined in `defines.events` to callbacks.
-    For example the table `{ [defines.events.on_player_died] = foo }` will call bar with the usual `event` table as argument every time a player dies.
+    For example the table `{ [defines.events.on_player_died] = foo }` will call foo with the usual `event` table as argument every time a player dies.
 - `on_nth_tick`:
     Table mapping the nth tick number to a callback.
     For example the table `{ [30] = bar }` will call bar every 30th tick.
@@ -92,7 +94,7 @@ For example a module could be defined as
 ```lua
 -- example.lua
 local function my_handler(event)
-    local name = game.player[event.player_index].name
+    local name = game.get_player(event.player_index).name
     game.print("It appears that " .. name .. " has died")
 end
 
@@ -176,12 +178,27 @@ This means always declearing your top level variables and functions local and ex
 You can require other files in your module by prefixing your require paths with `modules/your_module_name/`.
 It's also possible to require files from other modules this way too.
 
-The global variables and data stored in the global table is shared by all Clusterio modules as well as the scenario, so it's important that you use unique names.
-It's recommended to prefix the global variables and data in the global table that your module has with the name of your module.
+Persistent state is shared by all save-patched modules and the scenario.
+Use `compat.script_data` to access the version-appropriate persistent table (`global` or `storage`), and keep your state under a unique module name.
+See the [compatibility library](#compatibility-library) below.
+
+
+### Initialisation and recovery
 
 Because modules can be patched into an existing game you cannot rely on the `on_init` callback to be called in Clusterio Modules.
 Nor can you rely on the `on_configuration_changed` callback, as this is not called when level code changes.
-The Clusterio Lua API provides the custom `on_server_startup` event that can be used as a substitute, see the next section.
+The Clusterio Lua API provides `on_server_startup` for setup and migrations, but some Factorio events can run before it.
+In particular, player-leaving, console-command, and configuration-change handlers must not assume startup has initialized their state.
+
+Use an idempotent state accessor in handlers that can run early.
+It should migrate any legacy namespace before creating fresh state, fill missing fields, and return the same state on repeated calls.
+A guard that returns early because the new namespace is absent can silently discard recovery work on an older save.
+See [player_sync's state accessor](../plugins/player_sync/module/get_script_data.lua) for an implementation that handles a renamed namespace and [the module template](../packages/create/templates/common/module/control.lua) for lifecycle setup.
+
+Use `on_load` only to restore local references and other operations permitted by Factorio's load lifecycle; do not create or migrate persistent state there.
+Perform state changes in startup or other events that permit them.
+Recovery mode skips normal Node plugin loading, so save data may still use an older plugin namespace when Lua handlers run.
+Test old and current saves, early events, repeated access, and save/load behavior on the relevant supported Factorio versions.
 
 
 ## Clusterio Lua API
@@ -190,6 +207,29 @@ Clusterio provides a save patched module as well as a regular Factorio mod for i
 This includes tools sending data to plugins, script events to listen for, and an item serialization module.
 
 
+### Compatibility library
+
+Use the compatibility helpers for APIs that differ between supported Factorio versions:
+
+```lua
+-- In a save-patched module:
+local compat = require("modules/clusterio/compat")
+-- In a mod, use require("__clusterio_lib__/compat") instead.
+
+local function encode_example()
+    local script_data = compat.script_data
+    local encoded = compat.table_to_json({ example = true })
+    local decoded = compat.json_to_table(encoded)
+end
+```
+
+`script_data` selects the persistent state table, while the JSON helpers select the available runtime API.
+Access persistent state from the appropriate lifecycle callbacks, after initialisation or loading has made it available.
+The JSON helpers also work before runtime; other compatibility helpers may require `game` to be available.
+Legacy examples using `global` or `game.table_to_json` directly are version-specific.
+See [compat.lua](../packages/host/modules/clusterio/compat.lua) and the [Factorio support policy](decisions.md#factorio) before introducing version checks.
+
+
 ### clusterio_api library
 
 **require path**  
@@ -228,7 +268,8 @@ Event data:
 
 Raised when Clusterio is starting up the server.
 It can be used to initialize clusterio related features in a mod, and as a stand-in for the `on_init` and `on_configuration_changed` event for modules.
-It is invoked on the first tick the server runs after the save has been patched, before most other events.
+It is invoked on the first tick the server runs after the save has been patched, before most other events but not all of them.
+See [initialisation and recovery](#initialisation-and-recovery) for handlers that can run earlier.
 
 Use this event to initialize and/or migrate the data structures you need.
 Keep in mind that both mods and Clusterio modules can switch from any version to any version so it should be able to handle both forwards and backwards migrations gracefully.
@@ -236,7 +277,7 @@ Keep in mind that both mods and Clusterio modules can switch from any version to
 #### clusterio_api.send_json(channel, data)
 
 Send JSON data to Clusterio over the given channel.
-The `data` argument should be table that can be serialized with `game.table_to_json`.
+The `data` argument should be a table that can be serialized with `compat.table_to_json`.
 Clusterio plugins can listen to channels and will receive an event with the data sent here.
 See the [Communicating with Factorio section](writing-plugins.md#communicating-with-factorio) in the Writing Plugins document for more information.
 
@@ -249,7 +290,20 @@ Strings embedded into the tables sent must be valid UTF-8 text.
 
 Parameters:
 - `channel`: string identifying which channel to send it on.
-- `data`: table that can be converted to JSON with game.table_to_json
+- `data`: table that can be converted to JSON with `compat.table_to_json`
+
+#### clusterio_api.send_udp(channel, data)
+
+Send data to Clusterio over the given channel using UDP instead of stdout.
+Clusterio plugins receive it as `udp-channel_name` events, see the [Communicating over UDP section](writing-plugins.md#communicating-over-udp) in the Writing Plugins document.
+Requires `factorio.enable_lua_udp` to be enabled on the instance and Factorio 2.1.10 or later, otherwise the data is silently dropped.
+The port to send to is passed to the module with a script command on startup, so `factorio.enable_script_commands` must be enabled as well.
+
+**Note**: Delivery is not guaranteed and payloads should be kept well below 32 kB.
+
+Parameters:
+- `channel`: string identifying which channel to send it on.
+- `data`: string with the payload to send, any bytes are allowed.
 
 
 ### serialize library
@@ -260,7 +314,7 @@ From a module
 From a mod  
 `local serialize = require("__clusterio_lib__/serialize")`
 
-Serializes inventories and item stacks into a table that in turn can be converted into JSON with `game.table_to_json`.
+Serializes inventories and item stacks into a table that in turn can be converted into JSON with `compat.table_to_json`.
 Properly handles the extra data for most items.
 
 #### serialize.serialize_equipment_grid(LuaEquipmentGrid)

docs/devs/plugin-migration-to-2.0.md

@@ -18,6 +18,12 @@ See [Breaking Changes](/docs/decisions.md#breaking-changes) in the decision log.
 Clusterio's packages are now ES modules and plugins should be too.
 Add `"type": "module"` to the plugin's `package.json` and replace `require()` and `module.exports` with `import` and `export`.
 
+Keep `"clusterio-plugin"` in the package's `"keywords"` for automatic discovery, and keep `"main"` pointing to the plugin info module (or use the default `index.js`).
+An `"exports"` map is not required for ES modules.
+If you add one, expose `"./package.json": "./package.json"`; otherwise Clusterio cannot discover or load the plugin by its package name.
+Clusterio currently reads the implementation path from `"main"` rather than conditional exports.
+See [Plugin Structure](../writing-plugins.md#plugin-structure) for a compatible package example.
+
 Relative imports must include the full filename with the `.js` extension.
 This also applies in TypeScript, where you import `./messages.js` even though the source file is `messages.ts`.
 Importing a directory no longer resolves to its `index.js`.

docs/devs/protocol.md

@@ -2,38 +2,32 @@
 
 Clusterio communicates using a simple homebrew message passing protocol based on JSON over WebSocket with limited support for reliable transport of messages.
 The protocol is implemented in [packages/lib/src/link](/packages/lib/src/link) and uses the `ws` library to handle the WebSocket connections on the Node.js side and the native WebSocket API on the browser side.
+The wire schemas are defined in [messages_core.ts](/packages/lib/src/data/messages_core.ts); see the [communication overview](../communication.md) for how links, connectors, and routing fit together.
 
 Two general kinds of messages are supported: events and requests.
 Events are messages sent by one party and received by the other and offer no notification for the sender on whether it was actually received.
 Requests are pairs of request-response messages where one side sends the requests and the other side replies with a response.
 Apart from the handshake used to establish connections the protocol is directionless, and either party can send or receive both requests and events.
 
-Events and requests are defined as instances of the `Request` and `Event` classes, and specify a unique type string, who can send and receive the message, optional forwarding of the message and the format of the data payload as a JSON schema.
-All messages sent and received are validated with JSON schemas to catch potential errors in sending messages and avoid having to write lots of validation code in message handlers.
-
-Messages are sent by calling the `.send()` method of the message instance and passing a `Link` instance to send it over and the data to send.
-For the controller there's a `Link` instance for each host and control connection stored in the `WsServer` class.
-For hosts the `Host` instance is a link to the controller, and each `Instance` instance is an in-memory link to the `Host` instance and has a corresponding `InstanceConnection` link stored on the `Host` instance.
-For ctl there's a `Ctl` instance and web_ui there's a `Control` instance that acts as the link to the controller.
-
-Messages received invokes message handlers that were registered when the link was brought up.
-These handlers are named after the name of the property the message is stored under with `EventHandler` appended for events and `RequestHandler` appended for requests.
-The handlers are defined on the link class itself for core messages, and on the plugin class in the case of plugin defined messages.
-
 
 ## Connection flow
 
 Clients connect via WebSocket to the `/api/socket` endpoint and upon establishing connection to the server the connection is in the handshake phase and the server will send a `hello` message to the client.
-If the client has no previous session with the server it sends a `register_host` or `register_control` message depending on what kind of client it is to the server.
-If the register is valid then the server will respond with a `ready` message with a session token and the connection has left the handshake phase, otherwise the server will close the connection with a WebSocket close code indicating the kind of failure.
+If the client has no previous session with the server it sends a `registerHost` or `registerControl` message depending on what kind of client it is to the server.
+If the registration is valid, the server responds with a `ready` message containing the client's address, a session token, and session parameters.
+The connection has then left the handshake phase; otherwise the server closes it with a WebSocket close code indicating the kind of failure.
 
 Upon leaving the handshake phase both parties should start sending heartbeat messages with the sequence number of the last received message.
-Messages sent over the link should be stored in a send buffer and only cleared from the send buffer when a heartbeat with a `last_seq` equal or or higher is received or the session is invalidated.
+Routable messages are stored in a send buffer until a heartbeat acknowledges an equal or higher sequence number, or the session ends.
+The heartbeat's top-level `seq` acknowledges receipt by the transport, not successful completion of a message handler.
 
-If the connection goes stale or is unexpectedly closed the client can reastablish the same session by connecting to the server again and sending a `resume` message after received the `hello` from the server.
+If the connection goes stale or is unexpectedly closed, the client can re-establish the same session by connecting again and sending a `resume` message after receiving `hello`.
 If the session resume is successful then the server will respond with a `continue` message and the connection is active again.
-The client should then re-send all messages it has in its send buffer that is greater than the `last_seq` number in the `continue` message.
-Otherwise the server will respond with `invalidate` and the client is expected to drop its send buffer start a new session with `register_host` or `register_control`.
+The client then re-sends buffered messages newer than the `lastSeq` in `continue`.
+The server likewise re-sends messages newer than the `lastSeq` supplied by the client in `resume`.
+Otherwise the server responds with `invalidate`, and the client discards the old send buffer and registers a new session.
+Session expiry, buffer overflow, or process failure can lose pending work; plugins must not treat transport recovery as an application-level exactly-once guarantee.
+The [send-buffer setting](../configuration.md#controllermax_send_buffer_size) controls how much unacknowledged message data the controller retains for a connection.
 
 ![Connection flow](../assets/connection_flow.svg)
 
@@ -41,36 +35,42 @@ Otherwise the server will respond with `invalidate` and the client is expected t
 ## Connector flow
 
 To manage the complexity of connecting, reconnecting, resuming or invalidating dropped sessions the WebSocket connection for a `Link` class is managed by a connector.
-There's three kinds of connectors defined: `WebSocketClientConnector`, `WebSocketServerConnector` and `VirtualConnector`.
+There's three kinds of connectors defined: `WebSocketClientConnector`, `WsServerConnector` and `VirtualConnector`.
 The virtual connector is for in-memory connections and doesn't have any connection state logic.
 
 For the WebSocket connectors there's four states they can exist in:
+
 - `closed` - The connector has just been created or has been explicitly closed.
 - `connecting` (client only) - The connector is in the process of trying to connect to the server.
 - `connected` - The connector is successfully connected.
 - `resuming` - The connection dropped and the connector is in the process of attempting to resume it.
 
 Along with the following events emitted when transition between states:
-- `connect` From `connecting` (client) or `closed` (server) to `connected`
-- `drop` - From `connected` to `resuming`
-- `resume` - From `resuming` to `connected`
-- `invalidate` - From `resuming` to `connecting` or `closed`
-- `close` - From any state to `closed`
+
+- `connect` - A new session is established, moving from `connecting` (client) or `closed` (server) to `connected`.
+- `drop` - The transport was lost but the session is retained for resumption, moving from `connected` to `resuming`.
+- `resume` - The retained session is reconnected and unacknowledged buffered messages are replayed, moving from `resuming` to `connected`.
+- `invalidate` - The previous session cannot be reused, moving from `resuming` to `connecting` (client only); server session expiry emits `close` instead.
+- `close` - The connector is closed, moving from any state to `closed`.
+
+Invalidation or closure rejects pending requests with `SessionLost`.
+Plugins must handle lost sessions, stale state, and application retries where required; see [connection handling](../writing-plugins.md#handling-connection-events) for the plugin hooks and queueing behavior.
 
 ![Conector States](../assets/connector_states.svg)
 
 
 ### WsClientConnector
 
-The client connector will keep trying to establish and mantain a connection after `.connect()` is called on it, stopping only in the case of authentication failure or `.close()`/`.setClosing()` being called.
-Other than that, the logic is pretty straight forward.
+The client connector, named `WebSocketClientConnector` in the source, keeps trying to establish and maintain a connection after `.connect()` is called, stopping on terminal failures such as authentication or policy rejection, or when `.close()`/`.setClosing()` is called.
+For example, policy violation (1008) stops retries, while try-again-later (1013), used for send-buffer overflow, permits reconnecting.
+Connection retries use exponential backoff.
 
 ![WsClientConnector](../assets/WsClientConnector.svg)
 
 
 ### WsServerConnector
 
-The server connector doesn't do the work of connecting or validating the handshake.
+The [server connector](/packages/controller/src/WsServerConnector.ts) doesn't do the work of connecting or validating the handshake.
 Instead it's handed over a WebSocket connection from `WsServer` once there's only the step of sending the final message left to do.
 Because of this there's no `connecting` state, and once a session times out from the `resuming` state it goes directly to closed.
 
@@ -79,19 +79,26 @@ Because of this there's no `connecting` state, and once a session times out from
 
 ## Wire format
 
-All communication in the protocol is done via WebSocket text frames containing a JSON payload with the following properties:
+All communication uses WebSocket text frames containing a JSON object.
+Every frame has a `type` string; the other fields depend on that type.
+Requests, responses, and events carry an integer transport sequence number in `seq`, plus source and destination addresses in `src` and `dst`.
+Message-specific data is carried in `data` when present.
 
-- `seq: integer | null` - The sequence number of the message, can only be null for handshake and heartbeats.
-- `type: string` - The type of message
-- `data: object` - Message data
+An address is `[type, id]`, optionally followed by a request ID: `[type, id, requestId]`.
+Address types are 0 for controller, 1 for host, 2 for instance, 3 for control, and 4 for broadcast.
+The controller's ID is 0; a broadcast address uses its ID to identify the target endpoint type.
+For example, `[2, 123]` addresses instance 123, while `[4, 2]` addresses all instances.
+Broadcast destinations are for events.
 
-Before a connection is established it's in the handshake phase and the `seq` property is `null`.
-The `type` property is used to distinguish which of handshake, heartbeat, events or requests this message is and determines the expected format of the `data` property.
+A request puts its request ID in `src`; the response returns that ID in `dst` so the sender can match it to the pending request.
+This ID is separate from transport `seq`, which is used for acknowledgements and buffering.
 
 
 ### Handshake Messages
 
-The following messages are the only valid messages that can be sent during the handshake phase.
+The following messages are used during the handshake phase.
+Their listed fields belong inside the frame's `data` object, except for `invalidate`, which has no data.
+Property names are case-sensitive.
 
 
 #### `hello`
@@ -102,7 +109,7 @@ Sent by the controller when the WebSocket connection has been opened.
 - `plugins: Object<string, string>` -
     Object mapping plugin names to plugin version for plugins that are loaded on the controller.
 
-#### `register_host`
+#### `registerHost`
 
 Host handshake for establishing a new connection session.
 
@@ -112,7 +119,7 @@ Host handshake for establishing a new connection session.
 - `plugins: Object<string, string>` -
     Object mapping plugin names to plugin version for plugins that are available on the host.
 
-#### `register_control`
+#### `registerControl`
 
 Control handshake for enstablishing a new connection session.
 
@@ -123,29 +130,32 @@ Control handshake for enstablishing a new connection session.
 
 Handshake for resuming an existing connection session where the connection had dropped unexpectedly.
 
-- `session_token: string` - Session token to resume session with.
-- `last_seq: integer | null` - Last message seq received by the client.
+- `sessionToken: string` - Session token to resume session with.
+- `lastSeq: integer` (optional) - Last transport sequence number received by the client; omitted if none has been received.
 
 #### `invalidate`
 
 Sent by the controller in reply to a resume for a session that cannot be resumed.
-The client should start a new session with a register\_\* message in response.
+The frame is `{"type":"invalidate"}`, without a `data` object.
+The client should start a new session with `registerHost` or `registerControl` in response.
 
 #### `continue`
 
 Sent by the controller to signal resuming the session was successful.
 
-- `last_seq: integer | null` - Last message seq received by the server.
-- `session_timeout: integer` - Session timeout in seconds.
-- `heartbeat_interval: number` - Rate client should send heartbeats at.
+- `lastSeq: integer` (optional) - Last transport sequence number received by the server; omitted if none has been received.
+- `sessionTimeout: number` - Session timeout in seconds.
+- `heartbeatInterval: number` - Interval between heartbeats in seconds.
 
 #### `ready`
 
 Sent by the controller to signal the connection has been established with a new session.
 
-- `session_token: string` - Token to resume the session with.
-- `session_timeout: integer` - Session timeout in seconds.
-- `heartbeat_interval: number` - Rate client should send heartbeats at.
+- `src: Address` - The client's address, used as the source of messages it sends.
+- `sessionToken: string` - Token to resume the session with.
+- `sessionTimeout: number` - Session timeout in seconds.
+- `heartbeatInterval: number` - Interval between heartbeats in seconds.
+- `account` (optional) - Account details for a control client: a `name` and a `roles` array whose entries contain `name`, `id`, and `permissions`.
 
 
 ### Heartbeat message
@@ -158,64 +168,83 @@ The heartbeat should contain the `seq` of the last received message, and receivi
 
 Sent by both the controller and clients at the interval given in the continue/ready messages.
 
-- `last_seq: integer` - Last message seq received over the link.
+- `seq: integer` (optional, top-level) - Last transport sequence number received over the link.
+
+For example, `{"type":"heartbeat","seq":123}` acknowledges messages through sequence 123.
+Before receiving any messages, send `{"type":"heartbeat"}`; there is no `data.last_seq` field.
 
 ### Events
 
-Events are messages send in one direction over the link, that the receiver is expected to act upon, but not make any replies (for that there's [Requests](#requests).
-By convention message types ending in `*_event` are events.
-The data payload of an event consists solely of event specific properties.
+Events are messages sent in one direction that the receiver acts upon without replying; use [requests](#requests) when a response is needed.
+Their wire `type` is `"event"`, with a separate `name` identifying the registered event class.
+Plugin names are prefixed, for example `foo_frobber:SomeEvent`.
+The `data` payload, when present, contains event-specific properties.
 
-See [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) for the recognized core events and their contents.
-Plugins may defined their own events as well.
+See [packages/lib/src/data/](/packages/lib/src/data/) for core event definitions.
+Plugins may define their own events as well.
 
-#### `*_event`
+#### `event`
 
-- any properties defined by the event type.
+- `seq: integer` - Transport sequence number.
+- `src: Address` - Source address.
+- `dst: Address` - Destination address, including broadcast destinations.
+- `name: string` - Registered event name.
+- `data` (optional) - Payload defined by the event class.
 
 
 ### Requests
 
-Requests are messages which expects a response in return.
-They function a lot like HTTP requests except that both parties of a connection can inniate a request that is expected to be responded to.
-By convention message types ending in `*_request` are requests, and is expected to be replied to with a corresponding `*_response` message.
-If an error occured while processing the request an error response containing an error property in the data payload is sent instead.
+Requests expect a response in return.
+They function like HTTP requests except that either party of a connection can initiate one.
+The wire `type` is `"request"`; a successful reply uses `"response"`, and a failed request uses `"responseError"`.
+The request's `name` identifies the registered class, prefixed with the plugin name for plugin messages, such as `foo_frobber:Frobnicate`.
 
-See [packages/lib/src/link/messages.ts](/packages/lib/src/link/messages.ts) for the recognized core requests and their contents.
+See [packages/lib/src/data/](/packages/lib/src/data/) for core request definitions.
 Plugins may define their own requests as well.
 
-#### `*_request`
+#### `request`
 
-- any properties defined by the request type.
+- `seq: integer` - Transport sequence number.
+- `src: Address` - Source address, including the request ID.
+- `dst: Address` - Destination address.
+- `name: string` - Registered request name.
+- `data` (optional) - Payload defined by the request class.
 
-#### `*_response` success
+#### `response` success
 
-- `seq: integer` - The seq number of the request message.
-- any additional properties defined by the request type.
+- `seq: integer` - This response's transport sequence number, not the request's sequence number.
+- `src: Address` - Responding endpoint.
+- `dst: Address` - Requester's address, including the original request ID.
+- `data` (optional) - Payload defined by the request class's `Response` definition.
 
-#### `*_response` error
+#### `responseError`
 
-- `seq: integer` - The seq number of the request message.
-- `error: string` - Human readable text message describing the error.
+- `seq: integer` - This error response's transport sequence number.
+- `src: Address` - Responding endpoint.
+- `dst: Address` - Requester's address, including the original request ID.
+- `data.message: string` - Human-readable error description.
+- `data.code: string` (optional) - Error code.
+- `data.stack: string` (optional) - Error stack.
 
 
 ### Graceful Disconnect
 
-To ensure no data is lost when closing the link it's important to send a prepare disconnect request and wait for it to complete before closing the WebSocket.
-Either party of the link can send this request and the party sending the request should be the one that initates the close of the WebSocket.
+To reduce data loss when closing a link, ask the peer to prepare and wait for it to finish before closing the WebSocket.
+Either party can initiate the disconnect and is then responsible for closing the socket.
+These are connector control frames, not registered request/response classes; forced closure and process crashes can still lose work.
 
-#### `prepare_disconnect_request`
+#### `disconnect` with `data: "prepare"`
 
-Request graceful disconnect of the connection.
+The frame `{"type":"disconnect","data":"prepare"}` requests a graceful disconnect.
 The receiving party should stop tasks that send events and requests over the link and wait for pending requests it has over the link to complete, but it should not stop responding to requests sent to it.
 Only after this process is done should the response be sent.
 
-#### `prepare_disconnect_response`
+#### `disconnect` with `data: "ready"`
 
-Signals the preparation for disconnect is complete and that the connection can be closed.
-The party that initiatied the prepare disconnect request may still send other requests/events over the link to finish its part of the disconnection proceedure.
-
-If the error response variant is received it should be logged if possible and treated as a success.
+The frame `{"type":"disconnect","data":"ready"}` signals that preparation is complete and the connection can be closed.
+When using `WebSocketBaseConnector.disconnect()`, the initiator must stop producing new work and await its own `Link.prepareDisconnect()` before calling `disconnect()`.
+The connector closes as soon as the peer sends `ready` or the timeout expires; it does not drain the initiator's pending requests itself.
+There is no error-response variant; the connector also has a timeout so an unresponsive peer cannot block shutdown indefinitely.
 
 
 ### Example handshake
@@ -224,10 +253,10 @@ Server hello after opening the WebSocket.
 
 ```json
 {
-    "seq": null,
     "type": "hello",
     "data": {
-        "version": "2.0.0"
+        "version": "2.0.0",
+        "plugins": {}
     }
 }
 ```
@@ -236,12 +265,12 @@ Host registering a new connection session.
 
 ```json
 {
-    "seq": null,
-    "type": "register_host",
+    "type": "registerHost",
     "data": {
         "token": "xyzabc123",
         "version": "2.0.0",
-        "id": 123
+        "id": 123,
+        "plugins": {}
     }
 }
 ```
@@ -250,12 +279,12 @@ Controller reply signaling the session is established and ready.
 
 ```json
 {
-    "seq": null,
     "type": "ready",
     "data": {
-        "session_token": "xyz.azy.bzy",
-        "session_timeout": 60,
-        "heartbeat_interval": 15
+        "src": [1, 123],
+        "sessionToken": "xyz.azy.bzy",
+        "sessionTimeout": 60,
+        "heartbeatInterval": 15
     }
 }
 ```

docs/how-it-works.md

@@ -1,3 +1,25 @@
+# How Clusterio works
+
+Clusterio runs a group of Factorio servers as a cluster, managed through a web interface or the `clusterioctl` command-line client.
+The controller is the central point of contact: clients connect to it to manage the cluster, and hosts connect to it to receive commands and exchange messages.
+Each host manages its local Factorio server processes, with each server belonging to an instance.
+
+
+## System overview
+
+The diagram shows one possible layout with two physical servers, each running a host and three Factorio servers.
+The controller shares a machine with one host in this example, but it can run on a separate machine.
+The number of hosts and instances can vary with the needs of the cluster.
+
+![Clusterio system diagram showing a controller, web and command-line clients, and two hosts managing three Factorio servers each](assets/clusterio_2.0_block_diagram.svg)
+
+The blue arrows point from the hosts and clients to the controller they connect to; messages can travel in both directions over those WebSocket connections.
+The orange lines connect each host to the Factorio processes it manages.
+Open the [full-size diagram](assets/clusterio_2.0_block_diagram.svg) to zoom in, or use the [Draw.io source](assets/clusterio_2.0_block_diagram.drawio) to edit it.
+
+For how messages are handled and routed between these components, see the [communication guide](communication.md).
+
+
 ## Save Patching
 
 In order to efficiently get data in and out of running Factorio servers and to provide custom interfaces Clusterio uses save patching<sup>[1]</sup> to add code to the game.

docs/managing-a-cluster.md

@@ -30,9 +30,10 @@ Create a host config. `--id` id the host id. `--name` is the host name. `--gener
 
 ### Update Clusterio
 
-    ctl> host update [--id <host id>] [--name <host name>] [--restart]
+    ctl> host update <host> [--all] [--restart]
 
-If remote updates are enabled on the host, this will attempt to update `@clusterio/host` on the machine. Changes will not be applied until the host is restarted. If the `--restart` flag is provided, the host will automatically restart once the update is complete.
+Specify the host by name or ID.
+If remote updates are enabled on the host, this will attempt to update `@clusterio/host` on the machine. Add `--all` to also update all npm-installed plugins. Changes will not be applied until the host is restarted. If the `--restart` flag is provided, the host will automatically restart once the update is complete.
 
 Remote updates are enabled by default but can be disabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the following command locally while the host is offline:
 
@@ -40,7 +41,7 @@ Remote updates are enabled by default but can be disabled on a per-machine basis
 
 The controller also supports remote updates, following the same logic. The corresponding commands are:
 
-    ctl> controller update [--restart]
+    ctl> controller update [--all] [--restart]
     controller> config set controller.allow_remote_updates true/false
 
 ### Mark restart required
@@ -116,7 +117,7 @@ Edit instance config in a text editor. Once file is saved and editor has been ex
 
 ### Assign instance to host
 
-    ctl> instace assign <instance> [host]
+    ctl> instance assign <instance> [host]
 
 Assigns instance to host. `instance` is instance to assign, `host` is host to assign to. If `host` is not provided, instance will be unassigned.
 
@@ -230,6 +231,38 @@ Note: If the player joins the cluster again a new account will be made for them
 To be written.
 
 
+## Notes
+
+Free text notes can be attached to hosts, instances, users, roles, mod packs, mods and saves.
+They are stored on the controller and shown to everyone with access to the resource in the web interface, and can for example be used to record who runs a host or what an instance is meant for.
+
+The web interface renders notes as Markdown.
+A mention of the form `@<type>/<id>` becomes a link to that resource showing its name, for example `@instance/778528005` or `@user/cooldude2606`.
+This works for hosts, instances, users, roles and mod packs, and the id is shown if the name is not available.
+The note itself stores the mention as written.
+
+### List notes
+
+    ctl> note list [--type <type>]
+
+Lists all notes, optionally limited to one resource type.
+
+### Set note
+
+    ctl> note set <type> <id> <content>
+
+Sets the note of a resource.
+The type is one of host, instance, user, role, mod_pack, mod or save.
+The id is the numeric id for hosts, instances, roles and mod packs, the lower case name for users, `<name>_<version>` for mods and `<instance id>/<file name>` for saves.
+
+### Delete note
+
+    ctl> note delete <type> <id>
+
+Removes the note of a resource.
+Notes are also removed when the resource they are attached to is deleted.
+
+
 ## Plugins (Local)
 
 To be written.

docs/migrating-from-2.0.0-alpha.13.md

@@ -1,46 +0,0 @@
-# Migrating from 2.0.0-alpha.13 to 2.0.0-alpha.14
-
-Alpha 14 comes with significant breaking changes for existing clusters, most notably the rename of master and slave to controller and host.
-Please study the entire migration guide carefully before attempting to update an existing cluster.
-
-## Before updating
-
-If you're using mods you need to create mod pack string for the mod configurations you're currently running.
-You can do this by running the follow RCON command on an instance that is currently running the mod configuration you want to use (you can also run this command in single player and find the string printed in Factorio's [Log file](https://wiki.factorio.com/Log_file)):
-
-    /sc local m={name="pack",description="",factorio_version=game.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}} for n,v in pairs(game.active_mods) do table.insert(m.mods,{name=n,enabled=true,version=v}) end local function set(s,c) for n,v in pairs(s) do m.settings[c][n]=v end end set(settings.startup,"startup") set(settings.global,"runtime-global") if game.player then set(game.player.mod_settings,"runtime-per-user") else set(settings.player,"runtime-per-user") end local e=game.encode_string(game.table_to_json(m)) log(e) rcon.print(e)
-
-If you for some reason want to run this function in 2.0+, use the following command:
-
-    /sc local a={name="pack",description="",factorio_version=script.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}}for b,c in pairs(script.active_mods)do table.insert(a.mods,{name=b,enabled=true,version=c})end;local function d(e,f)for b,c in pairs(e)do a.settings[f][b]=c end end;d(settings.startup,"startup")d(settings.global,"runtime-global")if game.player then d(game.player.mod_settings,"runtime-per-user")else d(settings.player,"runtime-per-user")end;local g=helpers.encode_string(helpers.table_to_json(a))log(g)rcon.print(g)
-
-This should produce a [Mod Pack String](https://forums.factorio.com/viewtopic.php?f=96&t=103578) which you need to copy and paste somewhere to keep it for later.
-Mod Pack strings looks something like this:
-
-    eNqFketuwyAMhd/Fv0OUVpW25lWmqSLESdAIRNh0l4h3n9mlitZt5QdCx/7so8MKXs8ILSzaPEEFPZKJdmEbvIgiDNpwiDaczhjpU93Vu/r+Tmpz6Anah/V7RqcJRUavO4c9tBwTVnAF5upCGJeIsYx3tvsPbQTdb0lKHYlnPJHY0+ONvcdjvW8gP1ZAyGz9KL5XINaR0yLPXEFMnu2ManSh0+6j/GOFitqXG4mjNSUidVm5wlm7JMYG7Qhl3BX8FjyqZ9vztOk+NM2fvRPaceLbzbN+UejQFFOWXzfArvk6v2HWD9ZLv5JfxA1UwsubPBaMKhHGklLO76vAwfQ=
-
-If you have multiple mod configuration create a mod pack string for each and note them down.
-
-Double check that you are on a supported version of Node.js by running `node --version`, the version should be v18 or later, if not you will need to update Node.js before continuing.
-
-
-## Migrate installations
-
-**Note:** Check that plugins you are using have updates avaialble for alpha 14 before attempting to update, if they do not either wait for the plugin to be updated or uninstall the plugin from all places it is installed on first.
-
-Shutdown all slaves and the master server and then migrate each installations by running the following command:
-
-    npm exec @clusterio/create -y -- --migrate-rename --log-level=verbose
-
-This will convert the existing installation's usage of the master and slave names in packages, configs, database and logs to the new controller and host terms and update the installed packages.
-
-If you're using systemd then copy the generated clusteriocontroller.service and/or clusteriohost.service files from the systemd folder to /etc/systemd/system/ and remove the old clusteriomaster.service and clusterioslave.service files.
-
-Once the installations have been migrated start up the controller and hosts again.
-
-
-## Post installation
-
-Import the Mod Pack strings that was created beforehand on the controller and upload the mods needed by those mod packs to the controller.
-You can assign a default mod pack in the controller config which instances use by default or set the mod pack on the per instance basis in the instance config.
-Once you have assigned a mod pack to an instance run a data export on the instance in order to properly populate the mod settings in the mod and get item icons and item names resolved in the web interface.

docs/pr-review-guide.md

@@ -0,0 +1,69 @@
+# Reviewing pull requests
+
+Review observable behavior across the whole PR, including callers, consumers, and lifecycle transitions.
+Reading every changed file and passing tests do not establish that those behaviors were covered.
+Use [contributing](contributing.md) for setup and validation commands.
+
+
+## Establish the scope
+
+- Record the PR's base and current head; preserve unrelated work with an isolated checkout when needed.
+- Read the requirements and relevant documentation at the reviewed revision.
+  Distinguish intended policy from observed behavior when they disagree.
+- Map each affected behavior from input through authorization, routing, state changes, and its consumers.
+  On a follow-up, verify the fixes and reassess the whole PR's affected behavior; explicitly label a deliberately narrower review.
+- Compare the PR base, previously reviewed head, and current head when attributing a follow-up finding.
+  Distinguish a new regression, an incomplete fix, and a previously missed defect.
+  Existing producer behavior can become unsafe through a new consumer; reject a finding as pre-existing only after comparing the complete failing path.
+- Trace the actual entrypoint and earlier guards before declaring a failure reachable.
+  Check supported configurations and compatibility policy; a merged revision or earlier clean review is not proof of correctness.
+
+
+## Exercise the boundaries
+
+- Write a small matrix of relevant states and transitions before probing.
+  Distinguish intended state, eligibility, and observed state, and vary the conditions that can make them differ.
+  Exercise transitions through the components that produce and consume the state; constructing the final state alone does not verify the transition.
+- Check adverse paths as well as success: failure, retry, removal, restart, reconnect, and overlapping operations where applicable.
+  Select relevant cases from these recurring review gaps:
+
+  | Boundary | Questions to check |
+  | --- | --- |
+  | Data contracts | Does real serializer output satisfy every consumer, including Lua restoration and inspection UI? Check omitted, empty, partial, malformed, reserved, and numeric-limit values before durable mutation. |
+  | State and identity | Do delayed reads and mutation completions still belong to the selected entity? Do disabled components retain their local state? Can an identifier or generation be reused after recovery? |
+  | Persistence and synchronization | Do save/reload, migration, deletion, and resubscription preserve versions and tombstones? Compare full snapshots, deltas, and optimized paths with ordinary transfers. |
+  | Ownership and errors | Can concurrent work remove another operation's lock, handle, or cache? Check synchronous throws, early promise rejection, partial mutation, and cleanup after each failure. |
+  | Protocol and compatibility | What does the receiver do with the message, close code, or acknowledgement? Check early startup, mixed versions, and supported older APIs. |
+  | Distribution | Do generated validators, browser bundles, and installed packages behave like the source checkout? Check actual export maps and plugin discovery. |
+
+- Use focused reproductions with concrete inputs and expected versus actual results.
+  Keep the logic under review real: stubbing permissions, effects, or state transitions can erase the defect being investigated.
+  Check that the harness reaches the intended path; compare a normal case and the base revision where practical.
+  Identify assumptions introduced by fixtures, caches, shared identities, or execution order, and vary those relevant to the change.
+- Control concurrency with explicit gates and assertions about ownership and completion, rather than timing alone.
+  Check both that invalid actions are prevented and that valid work eventually makes progress.
+  Vary delays and completion order, including changes to an action's preconditions between initiation and execution.
+
+
+## Account for every candidate
+
+- Keep a compact ledger of candidate issues and uncovered scenarios throughout the review.
+  Give each candidate a disposition: reported, rejected with evidence, or unresolved with the next check needed.
+- Before finishing, compare probe outputs with the final findings.
+  A reproduced anomaly must not disappear just because several other issues were already found.
+- Carry unresolved candidates and coverage gaps into handoffs and context summaries, alongside published findings.
+  For each finding, identify the trigger, consequence, source location, and supporting evidence.
+  Reconcile the previous ledger at every follow-up; an unconfirmed candidate remains unresolved until evidence rejects it or confirms it.
+  Name remaining material gaps in the final report even when other findings already block merging.
+
+
+## Report and follow up
+
+- Separate source inspection, focused probes, full suites, hosted CI, browser checks, and Factorio runtime evidence.
+  Record failures, skips, and unavailable validation; investigate external-service failures before attributing them to the PR.
+- Reproduce the original failure when checking a fix, including adjacent cases exposed by the change.
+  A fix for one malformed field calls for checking the remaining consumer requirements, not just that example.
+  Limit conclusions to the scenarios exercised, and state which relevant assumptions remain unverified.
+  Resolve a review thread only when its exact concern is addressed.
+- When authorized to publish, refresh the head and existing comments, avoid duplicates, and anchor feedback to the reviewed code.
+  Read back each posted inline comment to verify its body and location.

docs/release-workflow.md

@@ -11,16 +11,21 @@ If the script fails due to GitHub's 60/hour public access rate limit, you can se
 
 ## Packages
 
-The [Publish workflow](../.github/workflows/publish.yml) publishes npm packages and Lua mods when a GitHub release is published.
+The [Publish workflow](../.github/workflows/publish.yml) publishes npm packages and Lua mods on GitHub's `release: released` event for a stable release.
+Publishing a prerelease does not trigger it; see [GitHub's release event documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#release).
+The npm release includes the packages under `packages/` and the six core plugins under `plugins/`.
+Community plugins hosted in separate repositories have independent versions and releases.
 For this workflow, first merge the changelog and package version updates into `master`, then create and push the release tag on that merged commit.
 Create a draft GitHub release for the existing tag and use the GitHub Release Markdown from `scripts/pr_to_changelog.mts` as its notes.
-Publishing the draft triggers the workflow; pushing a tag alone does not.
+Publishing the draft as a stable release triggers the workflow; pushing a tag alone does not.
+Preparing a release PR, pushing a tag, and publishing a release or packages are separate actions; obtain authorization for the publication steps before executing them on someone else's behalf.
 The steps below describe publishing npm packages manually instead.
 
 > [!NOTE]
 > You will need commit access to the repository as well publish access to the @clusterio org on npm to do this.
 
-1. Ensure you are on the latest version of master branch with no untracked files
+1. Inspect `git status` and `git remote -v`, verify which remote is the main repository, and use a clean release worktree on its latest master.
+   The commands below assume that remote is named `upstream`; preserve unrelated work and external plugin checkouts.
 
     ```sh
     git switch master
@@ -28,31 +33,45 @@ The steps below describe publishing npm packages manually instead.
     git status
     ```
 
-2. Bump the version in the `package.json` files
+2. Create a release branch and bump the version in the `package.json` files. The commands below use 2.0.1 to 2.0.2 as an example; substitute the versions for the release you are preparing.
 
     ```sh
-    sed -i 's/2\.0\.0-alpha\.XX/2\.0\.0-alpha\.YY/' {packages,plugins}/*/package.json
-    ````
+    git switch -c release/2.0.2
+    sed -i 's/"version": "2\.0\.1"/"version": "2.0.2"/' {packages,plugins}/*/package.json
+    pnpm install --no-frozen-lockfile
+    ```
 
-3. Tag and push the release to github
+3. Review and commit the version and changelog changes, then open and merge a release pull request.
 
     ```sh
     git add -p
-    git commit -m v2.0.0-alpha.XX
-    git tag v2.0.0-alpha.XX
+    git commit -m "Prepare 2.0.2 release"
     ```
 
-4. Perform a clean install and publish the packages.
+4. After the release pull request is merged, tag the merged commit and push the tag.
 
     ```sh
+    git switch master
+    git pull --ff-only upstream master
+    git tag v2.0.2
+    git push upstream v2.0.2
+    ```
+
+5. For manual publication, perform a clean install and publish the packages. For automated publication, publish the GitHub release as described above instead.
+
+    The repository does not track `pnpm-lock.yaml`, so allow pnpm to generate or update it during installation.
+
+    ```sh
+    node clean.js --dry-run
     pnpm clean
+    pnpm install --no-frozen-lockfile
     pnpm -r --filter='./packages/*' --filter='./plugins/*' --fail-if-no-match publish
     ```
 
 ## Plugins
 
 1. Bump the version in the `package.json` file
-2. Run `pnpm install` in the repository root
+2. Run `pnpm install --no-frozen-lockfile` in the repository root
 3. Run `pnpm adduser` to authenticate
 4. Run `pnpm publish` in the plugin/package directory
 

docs/writing-plugins.md

@@ -39,6 +39,11 @@ The basic file structure of a plugin is the following.
 Clusterio plugins are Node.js packages that can be published on npm for ease of installation and distribution.
 The usual guides for creating such packages apply.
 At minimum the `package.json` file must contain a version entry.
+For automatic discovery, include `"clusterio-plugin"` in the package's `"keywords"` array.
+Installed npm plugins are discovered from the installation's dependencies when controller, host or ctl loads its configuration, and added to `plugin-list.json`.
+Discovery starts with the root package's `dependencies`, not `devDependencies`, and follows dependencies of recognized plugins.
+It does not search arbitrary nested dependency trees for plugins.
+For tooling that needs the same discovery behavior, reuse [loadPluginList](../packages/lib/src/load_plugin_list.ts) rather than maintaining another package scanner.
 
 A possible workflow for developing plugins is to place the plugin in a sub-directory of where clusterio has been installed, and rely on Node.js searching up the folder heirarchy for it to find `@clusterio/lib`.
 After creating a directory for your plugin, the boilerplate can be generated automaticlly from a set of tempaltes using `npm exec @clusterio/create -y -- --plugin-template` after which you will be asked which areas your plugin will interact with and your plugin's name.
@@ -47,26 +52,50 @@ Note that it's important that the path starts with ./ or ../ (use .\ or ..\ on W
 
 For a plugin to be recognised by Clusterio it needs to export an entry named `plugin` from its main entrypoint.
 By default the main entrypoint is the `index.js` file, but this may be changed by setting the `"main"` entry to a different file in `package.json`.
+Keep this `"main"` entry when using a package `"exports"` map: Clusterio currently loads the plugin implementation from `"main"` (or `index.js`), rather than selecting a conditional export.
+
+If a plugin published on npm defines an `"exports"` map, it must also expose `"./package.json"`.
+Both automatic discovery and runtime loading resolve `<package-name>/package.json` to read the plugin's metadata.
+Packages that hide this file are currently skipped by discovery, and manually adding their package names to `plugin-list.json` does not make them loadable.
+The generated plugin templates omit `"exports"`, which leaves `package.json` accessible.
+For example, a plugin that needs an export map can use:
+
+```json
+{
+    "name": "example-clusterio-plugin",
+    "version": "1.0.0",
+    "type": "module",
+    "main": "./index.js",
+    "keywords": ["clusterio-plugin"],
+    "exports": {
+        ".": "./index.js",
+        "./package.json": "./package.json"
+    }
+}
+```
+
 Here's an example of `index.js`:
 
 ```js
+import * as messages from "./messages.js";
+
 export const plugin = {
     name: "foo_frobber",
     title: "Foo Frobber",
     description: "Does advanced frobnication",
     instanceEntrypoint: "instance.js",
     controllerEntrypoint: "controller.js",
-    messages: {
-        /* See below */
-    },
+    messages: [messages.Frobnicate],
 };
 ```
 
 The following properties are recognized:
 
 **name**:
-    Internal name of the plugin.
-    Must be the same as the plugin's directory name
+    Stable internal identifier of the plugin, used as its key in the plugin list.
+    For local plugins automatically discovered under `plugins/` or `external_plugins/`, it must match the plugin's directory name.
+    It does not have to match the npm package name.
+    For example, the package `@clusterio/plugin-subspace_storage` declares the name `subspace_storage`.
 
 **title**:
     Name of the plugin as shown to users.
@@ -90,26 +119,39 @@ The following properties are recognized:
     This is an optional parameter.
     A plugin can be made that only runs on the controller.
 
-**ControllerConfigGroup**:
-    Object mapping controller configuration fields to `FieldDefinitions`for this plugin.
+**controllerConfigFields**:
+    Object mapping controller configuration fields to `FieldDefinition` objects for this plugin.
     See [Plugin Configuration](#plugin-configuration)
 
+**hostEntrypoint**:
+    Optional path to the Node.js module exporting the host entrypoint function.
+
+**hostConfigFields**:
+    Object mapping host configuration fields to `FieldDefinition` objects.
+
 **ctlEntrypoint**:
     Path to a Node.js module relative to the plugin directory which exports the clusterioctl entrypoint function for this plugin.
     This is an optional paramater.
     A plugin can be made that only runs on the clusterioctl side.
 
+**ctlConfigFields**:
+    Object mapping ctl configuration fields to `FieldDefinition` objects.
+
+**webEntrypoint**:
+    Optional Module Federation exposure key for the web entrypoint, such as `"./web"`.
+    It must match the key in the plugin's webpack `exposes` configuration; it is not a Node.js file path.
+
 **migrationEntrypoint**:
     Path to a Node.js module relative to the plugin directory which exports the migration entrypoint function for this plugin.
     This is an optional parameter.
     See [Migrating Stored Data](#migrating-stored-data)
 
 **features**:
-    Array of features the plugin needs from the instances it is loaded on, `"SavePatching"` and/or `"ScriptCommands"`.
+    Array of features the plugin needs from the instances it is loaded on, any of `"SavePatching"`, `"ScriptCommands"` and `"LuaUdp"`.
     See [Handling Invalid Configuration](#handling-invalid-configuration)
 
 **messages**:
-    Object with link messages definitions for this plugin.
+    Array of request and event classes for this plugin.
     See guide for [defining link messages](#defining-link-messages) below.
 
 **permissions**:
@@ -123,7 +165,8 @@ The only restriction imposed on modules embedded into plugins is that they must
 While there is no standard for how to organize a plugin it's recommended to put the controller entrypoint into controller.js and the instance entrypoint into instance.js.
 You can put them into whatever file you want (even the same one for both).
 
-The entrypoint paths must include the .js extension and use forward slashes for directory separators if any.
+Node.js entrypoint paths must include the `.js` extension and use forward slashes for directory separators.
+This rule does not apply to the `webEntrypoint` exposure key.
 
 
 ## Defining the plugin entrypoint
@@ -210,7 +253,7 @@ try {
     // Operation that should not throw but may end up throwing
 } catch (err) {
     logger.error(`Operation failed:\n${err.stack}`);
-)
+}
 ```
 
 For plugin hooks expections thrown are automatically catched and logged, but for event handlers registered on EventEmitters it's critical that exceptions are catched and handled appropriately.
@@ -219,7 +262,7 @@ For plugin hooks expections thrown are automatically catched and logged, but for
 ## Plugin Configuration
 
 Clusterio provides a configuration system that handles storing, distributing, editing and validating config fields for you.
-You can take advantage of it by declaring config fields under either `controllerConfigFields` or `instanceConfigFields` in the `plugin` export.
+Declare fields under `controllerConfigFields`, `hostConfigFields`, `instanceConfigFields`, or `ctlConfigFields` in the `plugin` export.
 For example in index.js:
 
 ```ts
@@ -260,7 +303,7 @@ See [Configuration System](config-system.md) for more details on how this system
 
 ### Handling Invalid Configuration
 
-If the plugin requires save patching or script commands to function, which can be disabled to run vanilla or scenarios not compatible with Clusterio, declare it under `features` in the `plugin` export:
+If the plugin requires save patching, script commands or Lua UDP to function, which can be disabled to run vanilla or scenarios not compatible with Clusterio, declare it under `features` in the `plugin` export:
 
 ```js
 export const plugin = {
@@ -314,6 +357,8 @@ In JavaScript the `declare module` block is not needed.
 
 Permissions declared this way are registered when the plugin is loaded.
 The older `lib.definePermission()` function still works, but a plugin should not use both for the same permission name.
+Controller-side permission checks are authoritative; hiding a web UI control does not authorize or protect its underlying request.
+Declare the [permission on control-origin messages](#static-permission) and test denied requests as well as allowed ones.
 
 
 ## Migrating Stored Data
@@ -341,7 +386,7 @@ They run synchronously, a handler that throws stops the controller or host from
 `lib.renameConfigField`, `lib.renameConfigPrefix`, `lib.renamePermission` and `lib.renamePermissionPrefix` cover the common case of renaming.
 
 Handlers run every time the data is loaded and must leave data that is already migrated unchanged.
-For changes that can't be repeated, each config has a hidden `<plugin>.version` field with the version of the plugin it was last loaded with, which is updated after the migrations have run:
+For changes that can't be repeated, controller, host, and instance configs have a hidden `<plugin>.version` field with the version of the plugin they were last loaded with, which is updated after the migrations have run:
 
 ```js
 hooks.instanceConfig.attach(plugin.name, config => {
@@ -448,7 +493,7 @@ async function handleFoo(content) {
 **Note:** This only handles IPC events, all other events must use `on`. You should not include the `ipc-` prefix when using this method. Error logging is handled for you, if you want custom error handling use `on`.
 
 It's recommended to either use the plugin name as the channel name or to prefix the channel name with the name of the plugin if you need multiple channels.
-It's also important to catch any errors that might occur as they will otherwise be propogated to the instance code and kill the server.
+With raw `server.on(...)` listeners, catch errors in the handler; `server.handle(...)` already handles error logging for IPC handlers.
 
 Data out from Factorio does not have the same limits as data into Factorio, RCON responses can be in 100kB range without causing issues, and payloads to the `send_json` API can be in the 4MB range provided the server has a fast enough storage system.
 
@@ -456,6 +501,43 @@ Data out from Factorio does not have the same limits as data into Factorio, RCON
 For `send_json` it's possible that payloads greater than 4kB are received after payloads that were sent at a later point in time.
 For RCON, commands longer than 50 characters may end up being executed after shorter commands sent after it.
 
+### Communicating over UDP
+
+Instances can optionally exchange UDP packets with the Factorio server for low latency fire-and-forget communication, using the `--enable-lua-udp` option of Factorio.
+This needs Factorio 2.1.10 or later, earlier versions crash when a packet is received on a headless server.
+This is disabled by default and enabled by setting `factorio.enable_lua_udp` on the instance.
+Sending data out of Factorio also needs `factorio.enable_script_commands`, as the port to send to is passed to the module with a script command on startup.
+Plugins requiring it should declare the `LuaUdp` feature.
+Note that UDP does not guarantee delivery or ordering, and packets should be kept well below 32 kB.
+
+Data out from Factorio is sent with the `send_udp` API of the Clusterio module.
+Unlike `send_json` the payload is an arbitrary string, and it is delivered to plugins as a `udp-channel_name` event with the payload as a Buffer:
+
+```lua
+clusterio_api.send_udp("my_plugin_foo", "123")
+```
+
+```js
+export default async function(context) {
+    const { instance, logger } = context;
+    instance.server.handleUdp("my_plugin_foo", async data => {
+        logger.info(`got ${data.toString()}`);
+    });
+}
+```
+
+Like `handle` for `send_json`, `handleUdp` logs errors thrown by the handler.
+Use `instance.server.on("udp-my_plugin_foo", ...)` for custom error handling.
+
+Data into Factorio is sent with the `sendUdp` method on the instance and delivered in-game as `defines.events.on_udp_packet_received` events with the packet in `event.payload`.
+The Clusterio module polls for received packets every tick, all modules and mods receive every packet so make sure the payload identifies your plugin.
+
+```js
+instance.hooks.start.attach(plugin.name, async () => {
+    await instance.sendUdp("my_plugin_foo:123");
+});
+```
+
 
 ## Defining Link Messages
 
@@ -470,59 +552,59 @@ Events are simple one-way notifications that invoke a handler on the target it's
 Requests are pairs of request and response messages where the request is sent to the target and the response is the reply back from the target.
 The requests are similar to HTTP requests, except both parties of a link may initiate one.
 
-Messages are defined as items of the `plugin.messages` array exported by `index.js`.
-For example, the following could be defined in `plugin`:
+Export message classes from `messages.js` and list them in the `plugin.messages` array in `index.js`, as in the [plugin structure example](#plugin-structure).
+For example, `messages.js` can contain:
 
 ```js
-messages: [
-    class Frobnicate {
-        static plugin = "foo_frobber"; // Plugin name
-        static type = "request";
-        static src = "controller";
-        static dst = "instance";
-        static jsonSchema = {
+export class Frobnicate {
+    static plugin = "foo_frobber";
+    static type = "request";
+    static src = "instance";
+    static dst = "controller";
+    static jsonSchema = {
+        type: "object",
+        properties: {
+            verbosity: { type: "integer" },
+            special: { type: "boolean" },
+        },
+        required: ["verbosity"],
+        additionalProperties: false,
+    };
+    constructor(json) {
+        this.verbosity = json.verbosity;
+        this.special = json.special;
+    }
+    static fromJSON(json) {
+        return new Frobnicate(json);
+    }
+    toJSON() {
+        return {
+            verbosity: this.verbosity,
+            special: this.special,
+        };
+    }
+    static Response = {
+        jsonSchema: {
             type: "object",
             properties: {
-                "verbosity": { type: "integer" },
-                "special": { type: "boolean" },
-            },
-            required: ["verbosity"],
-            additionalProperties: false,
-        };
-        constructor(json) {
-            this.verbosity = json.verbosity;
-            this.special = json.special;
-        }
-        static fromJSON(json) {
-            return new Frobnicate(json);
-        }
-        toJSON() {
-            return {
-                verbosity: this.verbosity,
-                special: this.special,
-            };
-        }
-        static Response = {
-            jsonSchema: {
-                type: "object",
-                properties: {
-                    "report": {
-                        type: "array",
-                        items: { type: "string" },
-                    },
+                report: {
+                    type: "array",
+                    items: { type: "string" },
                 },
             },
-            fromJSON(json) {
-                return json
-            }
-        }
-    }
-]
+            required: ["report"],
+            additionalProperties: false,
+        },
+        fromJSON(json) {
+            return json;
+        },
+    };
+}
 ```
 
-This specifies a request that can be sent from the controller to an instance, being routed through the correct host behind the scenes.
+This specifies a request sent from an instance to the controller through its host.
 The request data must contain the property `verbosity` with an integer number as the value, and it may also contain a boolean `special` property.
-It also defines that the response sent may contain a `report` property mapping to an array of strings.
+The response must contain a `report` property containing an array of strings.
 
 The following properties are recognized as part of a message class:
 
@@ -551,6 +633,34 @@ If the target is multiple hops away it the message will be forwarded towards its
 For example a message sent from a control connection an instance will be forwarded by the controller to the relevant host and from the host to the correct instance.
 Messages sent to a different destination will be rejected.
 
+#### static permission
+
+Required for request and event classes whose `src` includes `"control"`.
+Use a permission name, a synchronous permission-checking function, or `null` when every authenticated control client is deliberately allowed to send the message.
+Registration rejects control-origin messages with no permission declaration.
+
+A callback receives `(user, message)`, where `message` is the raw wire envelope.
+It runs on the controller before destination payload validation and deserialization, so `message.data` is untrusted and may be missing or malformed.
+Check any required payload shape before inspecting fields, and use `user.checkPermission(...)` or throw `PermissionError` for a normal denial.
+For example, this checks the payload shape needed to choose a permission; the destination schema still validates the full payload:
+
+```js
+import { PermissionError } from "@clusterio/lib";
+
+function checkFrobnicatePermission(user, message) {
+    const data = message.data;
+    if (!data || typeof data !== "object" || typeof data.special !== "boolean") {
+        throw new PermissionError("Expected a boolean special field");
+    }
+    user.checkPermission(data.special ? "foo_frobber.special" : "foo_frobber.frobnicate");
+}
+```
+
+To use this callback, declare both permissions, require `special` in the message schema, and set `static permission = checkFrobnicatePermission` on a class that accepts `"control"` sources.
+The `Frobnicate` example above only accepts instances and therefore does not need a permission declaration.
+Do not make permission callbacks async: their result is not awaited, and returning `false` does not deny access.
+Unexpected exceptions are contained and logged as errors; they are not a substitute for intentional permission denials.
+
 #### static jsonSchema
 
 A JSON schema that specifies what's valid to send in the message.
@@ -562,7 +672,8 @@ This should return an object that's valid according to `jsonSchema` and accepted
 
 #### static fromJSON
 
-Required function to restore the class from a JSON object.
+Function to restore the class from a JSON object; it must be provided together with `jsonSchema` for messages carrying data.
+Messages without data can omit both and are constructed without arguments.
 This function is called when the message reaches its destination and is expected to return an identical instance of the class.
 I.e. the result of `fromJSON(JSON.parse(JSON.stringify(instance)))` should be a complete copy of the message.
 
@@ -571,54 +682,62 @@ I.e. the result of `fromJSON(JSON.parse(JSON.stringify(instance)))` should be a
 Optional [JSON sterilisable class](/docs/devs/json-serialisable-classes.md) that specifies the response to the request.
 Required properties are `jsonSchema` and `fromJSON`.
 
+Test JSON round trips, invalid payloads, allowed source/destination types, and authorization failures when changing messages.
+Keep registration, schemas, serialization, permissions, and handlers consistent.
+
+
 ## Sending Link Messages
 
 Link messages are sent by calling the `.sendTo()` method on a connection object with the destination and the data you want to send, where the data is an instance of one of the defined message classes in plugin.messages.
 For example:
 
 ```js
+import * as messages from "./messages.js";
+
 // In an instance entrypoint
 async function frobnicate(instance) {
-    const response = await instance.sendTo("controller", new messages.Frobnicate({ 
+    const response = await instance.sendTo("controller", new messages.Frobnicate({
         verbosity: 2,
         special: false,
     }));
     console.log(response); // { report: [...] }
 }
 ```
-Because the send and handle methods rely on this, you should always call them directly on the connection object.
-
-If you extract them into a variable, destructure them, or pass them as unbound callbacks, they lose their this context. The method will eventually crash with an error like Cannot read properties of undefined (reading 'sendRequest').
-
-While it's fine to type cast a direct method invocation, casting and then storing the result causes that same dangerous loss of context.
-
-Here is what that looks like in practice:
+Register the corresponding handler on the controller:
 
 ```js
-// ❌ BAD: The extracted method loses its `this` binding and crashes when called.
-const send = instance.sendTo;
-send("controller", new messages.Frobnicate({ foo: "bar" }));
+import * as messages from "./messages.js";
 
-// ✅ GOOD: Call it directly on the connection object. (Cast args/results here if needed).
-instance.sendTo("controller", new messages.Frobnicate({ foo: "bar" }));
-
-// ✅ GOOD: Explicitly bind the receiver if you need to pass a method as a value (like a handler).
-instance.handle(messages.Frobnicate, plugin.handleFrobnicate.bind(plugin));
+export default async function({ controller }) {
+    controller.handle(messages.Frobnicate, async request => ({
+        report: [`Frobnication verbosity: ${request.verbosity}`],
+    }));
+}
 ```
 
+The send and handle methods depend on `this`.
+Call them on their connection object, or explicitly bind them when passing a method as a value:
+
+```js
+const send = instance.sendTo.bind(instance);
+const response = await send("controller", new messages.Frobnicate({ verbosity: 2, special: false }));
+```
+
+Extracting an unbound method loses its receiver; storing a returned response does not.
 For classes with `static type = "request"` the send method is async and returns a promise that resolves to the response data received from the target it was sent to, or rejects with an error if the request failed.
-The destination specification can either be the ID of a particular control, host or instance, or one of the keywords used to send to multiple targets at once.
+The destination can identify a particular control, host, or instance.
+The `allHosts`, `allControls`, and `allInstances` broadcast destinations are for events, not requests.
 
 Example destinations:
 
 ```js
-sendTo("controller", message);
-sendTo("allHosts", message);
-sendTo("allControls", message);
-sendTo("allInstances", message);
-sendTo({ hostId: 123 }, message);
-sendTo({ instanceId: 123 }, message);
-sendTo({ controlId: 123 }, message);
+connection.sendTo("controller", message);
+connection.sendTo("allHosts", event);
+connection.sendTo("allControls", event);
+connection.sendTo("allInstances", event);
+connection.sendTo({ hostId: 123 }, message);
+connection.sendTo({ instanceId: 123 }, message);
+connection.sendTo({ controlId: 123 }, message);
 ```
 
 ### Handling connection events
@@ -630,20 +749,23 @@ This is signaled to controller plugins via the `prepareHostDisconnect` hook and
 After the prepare disconnect the connection will be closed, which will result in pending requests and events being dropped.
 Plugins must respond to the prepare disconnect by stopping any processess it does that send events or requests over the link in question.
 This can be accomplished either through listening for the prepare disconnect hook, or by checking the `connected` property of the `HostConnection` class and `Host` class on the controller and host respectively.
-For example the sending of an event from an instance plugin can be stopped while the connection is not connected, not in the dropped state, and not in the process of discunnecting by using the following code:
+For example, an instance plugin can avoid starting a request while the host is disconnected or preparing to disconnect:
 
 ```js
 if (host.connected) {
-    instance.sendTo("controller", new messages.Frobnicate({ foo: "bar" }));
+    await instance.sendTo("controller", new messages.Frobnicate({ verbosity: 2, special: false }));
 }
 ```
 
 If the event or request needs to be sent to the controller it can be put into a queue and sent out when the connection is established again.
-The re-establishement of the connection is  notified to plugins via the `connect` event to the `controllerConnectionEvent` and `hostConnectionEvent` hooks.
+The connection hooks report `connect` for a new session and `resume` for a retained session.
+Keep application queues separate from messages already accepted into the connector buffer; blindly sending both again can duplicate an operation.
 
 The second connection event which is of lesser importance to respond to is the `drop` connection event served through `controllerConnectionEvent` for instance plugins and through `hostConnectionEvent` for controller plugins.
 This is raised when the connection between the controller and host in question is lost, most likely due to networking issues.
-When in the dropped state the host will keep trying to reconnect to the controller in order to re-establish it, and if successful no events or requests will be lost.
+When in the dropped state the host retries the connection and attempts to resume the retained session.
+Successful resumption replays unacknowledged buffered messages; expiry, invalidation, or buffer overflow can instead end the session.
+This is not an application-level exactly-once guarantee; stateful plugins need their own recovery and duplicate-handling strategy.
 However while in the dropped state any requests and events sent gets queued up in memory until the connection is either re-established or the session times out.
 This means that if your plugin sends a lot of events or requests, they can end up being queued up in a buffer and sent out all at once the connection is re-estabished.
 To avoid this you should be throtteling and/or stopping your requests/events after `drop` has been raised, and continue back as normal when `resume` is raised.
@@ -673,7 +795,7 @@ It's recommended that plugin metrics follow `clusterio_<plugin_name>_<metric_nam
 For metrics that are per-instance, you must define an `instance_id` label and set it accordingly, for example:
 
 ```js
-import { Counter } from "@clusterio/lib";
+import { Gauge } from "@clusterio/lib";
 
 const barMetric = new Gauge(
     "clusterio_foo_frobber_bar_metric", "Bar instance level",

logo-square.svg

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg width="100%" height="100%" viewBox="0 0 90 90" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:1.41421;"><g transform="translate(0.84,24.491)"><path d="M48.48,19.309l-9.917,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l2.718,0l0.002,-8.867l0.96,-0.96l3.117,0l1.68,1.68l0,10.293l-2.64,2.64ZM32.828,19.309l-9.918,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l8.478,0.009l0,2.137l-2.64,2.64ZM17.116,19.309l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.942l0,0.018l-7.596,0l0,5.04l8.556,0l0,2.146l-2.64,2.64ZM64.301,19.309l-9.997,0l-1.68,-1.68l0,-2.142l0.96,-0.96l7.597,-0.002l0,-0.993l-8.557,0l0,-6.196l1.68,-1.68l0,0l0.96,-0.96c2.029,0 7.282,0 9.997,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.96l-7.597,0.002l0,0.957l8.557,0l0,6.227l-0.96,0.96l0,0l-1.68,1.68ZM76.745,19.309l-3.12,0l-1.68,-1.68l0,-6.216l0.96,-0.96l6.48,0l0,6.216l-2.64,2.64ZM78.185,11.653l-6,0l0,5.877l1.54,1.539l2.921,0l1.539,-1.539l0,-5.877ZM17.017,5.896l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-6.48l8.556,0l0,-1.807l-1.539,-1.54ZM25.788,5.896l-2.778,0l-1.54,1.54l0,10.094l1.54,1.539l9.719,0l1.539,-1.539l0,-1.799l-8.478,-0.008l-0.002,-9.827ZM41.441,5.896l-2.779,0l-1.539,1.54l0,10.094l1.539,1.539l9.719,0l1.539,-1.539l0,-10.094l-1.539,-1.54l-2.778,0l-0.003,9.827l-4.157,0l-0.002,-9.827ZM64.201,5.896l-9.797,0l-0.82,0.82l-0.72,0.72l0,5.856l8.557,0l0,2.432l-8.557,0.003l0,1.803l1.54,1.539l9.797,0l1.54,-1.539l0,-5.888l-8.557,0l0,-2.396l8.557,-0.003l0,-1.807l-1.54,-1.54ZM81.921,9.497l-13.471,0l0,-2.161l2.64,-2.64l10.111,0l1.68,1.68c0,1.034 0,2.161 0,2.161l-0.96,0.96ZM80.141,5.896l-9.912,0l-1.539,1.54l0,1.821l12.991,0l0,-1.821l-1.54,-1.54Z" style="fill:#141414;"/><path d="M73.625,19.309l3.12,0l-3.12,0ZM76.646,19.069l-2.921,0l-1.54,-1.539l0,-5.877l6,0l0,5.877l-1.539,1.539ZM17.017,19.069l-9.798,0l-1.539,-1.539l0,-10.094l1.539,-1.54l9.798,0l1.539,1.54l0,1.807l-8.556,0l0,6.48l8.556,0l0,1.807l-1.539,1.539ZM32.729,19.069l-9.719,0l-1.54,-1.539l0,-10.094l1.54,-1.54l2.778,0l0.002,9.827l8.478,0.008l0,1.799l-1.539,1.539ZM48.381,19.069l-9.719,0l-1.539,-1.539l0,-10.094l1.539,-1.54l2.779,0l0.002,9.827l4.157,0l0.003,-9.827l2.778,0l1.539,1.54l0,10.094l-1.539,1.539ZM64.201,19.069l-9.797,0l-1.54,-1.539l0,-1.803l8.557,-0.003l0,-2.432l-8.557,0l0,-5.856l1.54,-1.54l9.797,0l1.54,1.54l0,1.807l-8.557,0.003l0,2.396l8.557,0l0,5.888l-1.54,1.539ZM78.425,11.413l0,6.216l0,-6.216ZM68.45,7.336l1.68,-1.68l-1.68,1.68c0,1.034 0,2.161 0,2.161l0,-2.161ZM81.681,9.257l-12.991,0l0,-1.821l1.539,-1.54l9.912,0l1.54,1.54l0,1.821Z" style="fill:#d58b36;"/></g><g transform="translate(-71.277,41.509)"><path d="M105.009,19.309l-3.12,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68l0,6.2l-0.96,0.96l-7.596,-0.003l0,4.816l-0.96,0.953l0,0.007ZM111.885,19.309l-3.12,0l0,-3.944l0.96,-0.96l4.8,0l0,2.212l-2.64,2.692ZM96.07,19.309l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-1.086,0.96l-7.47,0l0,0.959l5.841,0l0,2.13l-0.96,0.96l-4.881,0l0,0.991l8.556,0l0,2.146l-2.64,2.64ZM116.135,11.413l0.96,-0.96l6.481,0l0,5.953l-2.64,2.903l-3.121,0l-1.68,-1.943l0,-5.953ZM137.546,19.309l-2.872,0l0,-1.178c-1.382,-0.123 -2.749,-0.431 -4.028,-0.977l-0.056,-0.024l-1.307,0.804l-2.03,-1.25l1.277,-0.786c-0.81,-0.666 -1.487,-1.522 -1.761,-2.534l-1.749,0l0,-1.768l0.96,-0.96l1.141,0c0.332,-0.616 0.809,-1.156 1.35,-1.61l-1.198,-0.78l0.94,-0.93l2.03,-1.25l1.227,0.755c0.949,-0.424 2.033,-0.741 3.204,-0.922l0,-0.248l0.013,0l0.947,-0.96l2.872,0l0,1.116c1.505,0.156 2.889,0.535 4.059,1.079l1.332,-0.82l2.03,1.25l-1.12,1.059c0.701,0.671 1.181,1.437 1.377,2.261l1.976,0l0,1.768l-0.96,0.962l-0.002,-0.002l-1.327,0c-0.228,0.624 -0.547,1.236 -1.053,1.681l1.109,0.679l-0.968,0.978l-0.015,-0.004l-2.007,1.236l-1.411,-0.868c-0.958,0.432 -1.978,0.728 -3.02,0.908l0,0.375l-0.96,0.93l0,0.03ZM122.376,11.653l-6.001,0l0,5.624l1.55,1.792l2.901,0l1.55,-1.792l0,-5.624ZM113.325,15.605l-4.32,0l0,3.464l2.778,0l1.542,-1.59l0,-1.874ZM95.971,5.896l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-2.431l5.841,0l0,-1.65l-5.841,0l0,-2.399l8.556,0l0,-1.807l-1.539,-1.54ZM111.786,5.896l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l2.781,0l0,-5.777l8.556,0.004l0,-5.86l-1.539,-1.54ZM140.11,13.349c0.327,-0.363 0.421,-0.95 0.212,-1.53c-0.459,-1.279 -2.339,-2.021 -4.278,-2.122c-0.058,0 -0.058,0 -0.115,0c-2.792,0.201 -3.386,1.31 -3.433,1.781c-0.151,1.505 1.994,2.74 4.45,2.74c1.269,0 2.7,-0.357 3.164,-0.869ZM109.009,11.642l0,-2.398l-4.24,0.002l0,2.396l4.24,0ZM105.969,9.485l0,0.957l2.8,0l0,-0.958l-2.8,0.001ZM122.527,9.449l-6.392,0l0,-2.113Line truncated

package.json

@@ -49,18 +49,19 @@
 		"@swc/core": "catalog:",
 		"@types/node": "catalog:",
 		"c8": "^12.0.0",
-		"@typescript-eslint/eslint-plugin": "^8.70.0",
-		"@typescript-eslint/parser": "^8.70.0",
-		"eslint": "^10.10.0",
+		"@typescript-eslint/eslint-plugin": "^8.71.2",
+		"@typescript-eslint/parser": "^8.71.2",
+		"eslint": "^10.13.0",
 		"eslint-plugin-node": "^11.1.0",
 		"express": "catalog:",
 		"form-data": "^4.0.6",
-		"globals": "^17.12.0",
+		"globals": "^17.13.0",
 		"jsonwebtoken": "catalog:",
-		"jszip": "catalog:",
 		"mocha": "^11.8.0",
 		"typedoc": "^0.28.20",
 		"typescript": "catalog:",
-		"yargs": "catalog:"
+		"yargs": "catalog:",
+		"yauzl": "catalog:",
+		"yazl": "catalog:"
 	}
 }

packages/controller/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/controller",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Implementation of Clusterio controller",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -43,7 +43,6 @@
 		"express": "catalog:",
 		"finalhandler": "^2.1.1",
 		"jsonwebtoken": "catalog:",
-		"jszip": "catalog:",
 		"semver": "catalog:",
 		"winston": "catalog:",
 		"winston-daily-rotate-file": "catalog:",
@@ -53,7 +52,7 @@
 	"devDependencies": {
 		"@clusterio/web_ui": "workspace:^",
 		"@swc/core": "catalog:",
-		"@types/busboy": "^1.5.4",
+		"@types/busboy": "^1.5.5",
 		"@types/compression": "^1.8.1",
 		"@types/express": "catalog:",
 		"@types/finalhandler": "catalog:",
@@ -68,7 +67,7 @@
 		"typescript": "catalog:",
 		"webpack": "catalog:",
 		"webpack-cli": "catalog:",
-		"webpack-dev-middleware": "^6.1.3",
+		"webpack-dev-middleware": "^8.3.0",
 		"webpack-merge": "catalog:"
 	}
 }

packages/controller/src/ControlConnection.ts

@@ -119,6 +119,8 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.RoleUpdateRequest, this.handleRoleUpdateRequest.bind(this));
 		this.handle(lib.RoleGrantDefaultPermissionsRequest, this.handleRoleGrantDefaultPermissionsRequest.bind(this));
 		this.handle(lib.RoleDeleteRequest, this.handleRoleDeleteRequest.bind(this));
+		this.handle(lib.NoteListRequest, this.handleNoteListRequest.bind(this));
+		this.handle(lib.NoteSetRequest, this.handleNoteSetRequest.bind(this));
 		this.handle(lib.UserGetRequest, this.handleUserGetRequest.bind(this));
 		this.handle(lib.UserListRequest, this.handleUserListRequest.bind(this));
 		this.handle(lib.UserCreateRequest, this.handleUserCreateRequest.bind(this));
@@ -313,6 +315,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleInstanceDeleteRequest(request: lib.InstanceDeleteRequest) {
 		await this._controller.instances.deleteInstance(request.instanceId);
+		this._controller.deleteNote("instance", request.instanceId);
 	}
 
 	async handleInstanceConfigGetRequest(request: lib.InstanceConfigGetRequest) {
@@ -500,6 +503,7 @@ export default class ControlConnection extends BaseConnection {
 			throw new lib.RequestError(`Mod pack with ID ${id} does not exist`);
 		}
 		this._controller.modPacks.delete(modPack);
+		this._controller.deleteNote("mod_pack", id);
 	}
 
 	async handleModGetRequest(request: lib.ModGetRequest) {
@@ -863,6 +867,7 @@ export default class ControlConnection extends BaseConnection {
 
 	async handleModDeleteRequest(request: lib.ModDeleteRequest) {
 		await this._controller.modStore.deleteMod(request.name, request.version);
+		this._controller.deleteNote("mod", `${request.name}_${request.version}`);
 	}
 
 	async handleLogSetSubscriptionsRequest(request: lib.LogSetSubscriptionsRequest) {
@@ -973,6 +978,7 @@ export default class ControlConnection extends BaseConnection {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
 		this._controller.roles.delete(role);
+		this._controller.deleteNote("role", id);
 
 		for (const user of this._controller.users.values()) {
 			if (user.removeRole(id)) {
@@ -981,6 +987,18 @@ export default class ControlConnection extends BaseConnection {
 		}
 	}
 
+	async handleNoteListRequest() {
+		return [...this._controller.notes.values()].filter(note => !note.isDeleted);
+	}
+
+	async handleNoteSetRequest(request: lib.NoteSetRequest) {
+		const { resourceType, resourceId, content } = request;
+		if (!resourceType || !resourceId) {
+			throw new lib.RequestError("resourceType and resourceId must not be empty");
+		}
+		this._controller.setNote(resourceType, resourceId, content, this.user.name);
+	}
+
 	async handleUserGetRequest(request: lib.UserGetRequest): Promise<lib.UserDetails> {
 		let name = request.name;
 		let user = this._controller.users.getByName(name);
@@ -1095,6 +1113,7 @@ export default class ControlConnection extends BaseConnection {
 				controlConnection.connector.terminate();
 			}
 		}
+		this._controller.deleteNote("user", user.id);
 
 		if (user.isAdmin) {
 			this._controller.sendTo("allInstances", new lib.InstanceAdminlistUpdateEvent(name, false));

packages/controller/src/Controller.ts

@@ -205,6 +205,11 @@ export default class Controller {
 			undefined, this.finaliseUsers,
 		).bootstrap());
 
+		const notes = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "notes.json"),
+			lib.Note.fromJSON.bind(lib.Note),
+		).bootstrap());
+
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.mkdir(modsDirectory, { recursive: true });
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
@@ -227,6 +232,7 @@ export default class Controller {
 			modStore,
 			roles,
 			users,
+			notes,
 		] as const;
 	}
 
@@ -260,6 +266,8 @@ export default class Controller {
 		public roles = new lib.SubscribableDatastore<lib.Role>(),
 		/** Mapping of user id to user record */
 		users = new lib.SubscribableDatastore<UserRecord>(),
+		/** Mapping of note id to note attached to a resource */
+		public notes = new lib.SubscribableDatastore<lib.Note>(),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -291,6 +299,7 @@ export default class Controller {
 		this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this));
 		this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this));
 		this.subscriptions.handle(lib.RoleUpdatesEvent, this.handleRoleSubscription.bind(this));
+		this.subscriptions.handle(lib.NoteUpdatesEvent, this.handleNoteSubscription.bind(this));
 
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
@@ -300,6 +309,7 @@ export default class Controller {
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
 		this.roles.on("update", this.rolesUpdated.bind(this));
 		this.users.records.on("update", this.usersUpdated.bind(this));
+		this.notes.on("update", this.notesUpdated.bind(this));
 	}
 
 	get authSecret() {
@@ -744,6 +754,7 @@ export default class Controller {
 			this.roles.save(),
 			this.users.records.save(),
 			this.hooks.save.invoke(),
+			this.notes.save(),
 		]);
 	}
 
@@ -1137,6 +1148,60 @@ export default class Controller {
 		return roles.length ? new lib.RoleUpdatesEvent(roles) : null;
 	}
 
+	notesUpdated(notes: lib.Note[]) {
+		this.subscriptions.broadcast(new lib.NoteUpdatesEvent(notes));
+	}
+
+	async handleNoteSubscription(request: lib.SubscriptionRequest) {
+		if (request.action === "unsubscribe") {
+			return null;
+		}
+		// Deleted notes are kept as tombstones so resubscribing clients learn of the deletion
+		const notes = [...this.notes.values()].filter(
+			note => note.updatedAtMs > request.lastRequestTimeMs && (request.lastRequestTimeMs > 0 || !note.isDeleted),
+		);
+		return notes.length ? new lib.NoteUpdatesEvent(notes) : null;
+	}
+
+	/**
+	 * Set or remove the note attached to a resource
+	 * @param resourceType - Type of resource the note belongs to.
+	 * @param resourceId - Id of the resource within its type.
+	 * @param content - New text of the note, empty removes the note.
+	 * @param updatedBy - Name of the user making the change.
+	 */
+	setNote(resourceType: string, resourceId: string, content: string, updatedBy = "") {
+		const existing = this.notes.getMutable(lib.Note.idFor(resourceType, resourceId));
+		if (content === "") {
+			this.deleteNote(resourceType, resourceId, updatedBy);
+			return;
+		}
+		if (existing) {
+			existing.content = content;
+			existing.updatedBy = updatedBy;
+			existing.isDeleted = false;
+			this.notes.set(existing);
+		} else {
+			this.notes.set(new lib.Note(resourceType, resourceId, content, updatedBy));
+		}
+	}
+
+	/**
+	 * Remove the note attached to a resource, if any
+	 * @param resourceType - Type of resource the note belongs to.
+	 * @param resourceId - Id of the resource within its type.
+	 * @param updatedBy - Name of the user making the change.
+	 */
+	deleteNote(resourceType: string, resourceId: string | number, updatedBy = "") {
+		const note = this.notes.getMutable(lib.Note.idFor(resourceType, resourceId));
+		if (note && !note.isDeleted) {
+			note.content = "";
+			note.updatedBy = updatedBy;
+			note.isDeleted = true;
+			this.notes.set(note);
+		}
+	}
+
 	async loadPlugins() {
 		const context = { metrics, logger, controller: this };
 		for (let pluginInfo of this.pluginInfos) {

packages/controller/src/routes.ts

@@ -5,7 +5,6 @@ import busboy from "busboy";
 import crypto from "crypto";
 import events from "events";
 import fs from "node:fs/promises";
-import JSZip from "jszip";
 import jwt from "jsonwebtoken";
 import path from "path";
 import nodeStream from "stream";
@@ -266,9 +265,7 @@ async function uploadExport(req: Request, res: Response) {
 	for await (let chunk of req) {
 		data.push(chunk);
 	}
-	let buffer: Buffer | null = Buffer.concat(data);
-	let zip = await JSZip.loadAsync(buffer);
-	buffer = null;
+	let zip = await lib.ZipArchive.fromBuffer(Buffer.concat(data));
 
 	// This is hardcoded to prevent path expansion attacks
 	let exportFiles = [
@@ -288,14 +285,15 @@ async function uploadExport(req: Request, res: Response) {
 			continue;
 		}
 
+		let content = await zip.readEntry(file);
 		if (filePath === "export/settings.json") {
-			settingPrototypes = JSON.parse(await file.async("text"));
+			settingPrototypes = JSON.parse(content.toString("utf8"));
 		}
 
 		let { name, ext } = path.posix.parse(filePath);
-		let hash = await lib.hashStream(file.nodeStream());
+		let hash = await lib.hashStream(await zip.openStream(file));
 		assets[name] = `${name}.${hash}${ext}`;
-		await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
+		await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), content);
 	}
 
 	modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString());

packages/create/README.md

@@ -2,4 +2,5 @@
 
 Installer for Clusterio.
 
-See [the Clusterio README.md](https://github.com/clusterio/clusterio#readme) the for installation guide.
+Install the latest [Node.js LTS release](https://nodejs.org/en/download) before running the installer.
+See [the Clusterio installation guide](https://github.com/clusterio/clusterio#installation) for platform-specific instructions.

packages/create/create.js

@@ -35,10 +35,6 @@ const availablePlugins = [
 	{ name: "Research Sync", value: "@clusterio/plugin-research_sync" },
 	{ name: "Statistics Exporter", value: "@clusterio/plugin-statistics_exporter" },
 	{ name: "Subspace Storage", value: "@clusterio/plugin-subspace_storage" },
-
-	// Comunity plugins
-	{ name: "Discord Bridge", value: "@hornwitser/discord_bridge" },
-	{ name: "Server Select", value: "@hornwitser/server_select" },
 ];
 
 const factorioLocations = {
@@ -392,31 +388,7 @@ async function installClusterio(mode, plugins) {
 		throw new InstallError(`Failed to install: ${err.message}`);
 	}
 
-	if (plugins.length) {
-		logger.info("Setting up plugins");
-		let pluginList;
-		try {
-			pluginList = new Map(JSON.parse(await fs.readFile("plugin-list.json")));
-		} catch (err) {
-			if (err.code === "ENOENT") {
-				pluginList = new Map();
-			} else {
-				throw new InstallError(`Error loading plugin-list.json: ${err.message}`);
-			}
-		}
-		for (let plugin of plugins) {
-			if (!pluginList.has(plugin)) {
-
-				let pluginInfo = (await import(plugin)).plugin;
-				pluginList.set(pluginInfo.name, plugin);
-			}
-		}
-		try {
-			await safeOutputFile("plugin-list.json", JSON.stringify([...pluginList], null, "\t"));
-		} catch (err) {
-			throw new InstallError(`Error writing plugin-list.json: ${err.message}`);
-		}
-	}
+	// Controller, host and ctl discover installed plugins when loading their configuration.
 }
 
 async function groupIdToName(gid) {
@@ -905,6 +877,7 @@ async function main() {
 
 if (import.meta.main) {
 	main().catch(err => {
+		process.exitCode = 1;
 		if (err instanceof InstallError) {
 			logger.error(err.message);
 		} else {

packages/create/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/create",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Installer for clusterio",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -33,7 +33,7 @@
 
 	"dependencies": {
 		"chalk": "catalog:",
-		"inquirer": "^8.2.4",
+		"inquirer": "^8.2.7",
 		"yargs": "catalog:"
 	}
 }

packages/ctl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/ctl",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Clusterio command line management tool",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",

packages/ctl/src/commands.ts

@@ -8,6 +8,7 @@ import { modPackCommands } from "./commands_mod_pack.js";
 import { modCommands } from "./commands_mod.js";
 import { permissionCommands } from "./commands_permission.js";
 import { roleCommands } from "./commands_role.js";
+import { noteCommands } from "./commands_note.js";
 import { userCommands } from "./commands_user.js";
 import { logCommands } from "./commands_log.js";
 import { debugCommands } from "./commands_debug.js";
@@ -22,6 +23,7 @@ export async function registerCommands(ctlHooks: CtlHooks, yargs: Argv) {
 	rootCommands.add(modCommands);
 	rootCommands.add(permissionCommands);
 	rootCommands.add(roleCommands);
+	rootCommands.add(noteCommands);
 	rootCommands.add(userCommands);
 	rootCommands.add(logCommands);
 	rootCommands.add(debugCommands);

packages/ctl/src/commands_note.ts

@@ -0,0 +1,72 @@
+import asTableModule from "as-table";
+
+import * as lib from "@clusterio/lib";
+import type { Ctl } from "../ctl.js";
+import { print } from "./command_ops.js";
+
+const asTable = asTableModule.configure({ delimiter: " | " });
+
+export const noteCommands = new lib.CommandTree({
+	name: "note", description: "Notes attached to hosts, instances, users and other resources",
+});
+noteCommands.add(new lib.Command({
+	definition: [["list", "l"], "List notes in the cluster", (yargs) => {
+		yargs.options({
+			"type": { describe: "Only list notes for this resource type", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(args: { type?: string }, ctl: Ctl) {
+		let notes = await ctl.send(new lib.NoteListRequest());
+		if (args.type !== undefined) {
+			notes = notes.filter(note => note.resourceType === args.type);
+		}
+		print(asTable(notes.map(note => ({
+			type: note.resourceType,
+			id: note.resourceId,
+			updatedBy: note.updatedBy,
+			content: note.content,
+		}))));
+	},
+}));
+
+noteCommands.add(new lib.Command({
+	definition: ["get <type> <id>", "Show the note of a resource", (yargs) => {
+		yargs.positional("type", {
+			describe: "Resource type, one of host, instance, user, role, mod_pack, mod, save", type: "string",
+		});
+		yargs.positional("id", { describe: "Id of the resource", type: "string" });
+	}],
+	handler: async function(args: { type: string, id: string }, ctl: Ctl) {
+		const notes = await ctl.send(new lib.NoteListRequest());
+		const note = notes.find(n => n.resourceType === args.type && n.resourceId === args.id);
+		if (!note) {
+			throw new lib.CommandError(`No note for ${args.type} ${args.id}`);
+		}
+		print(note.content);
+	},
+}));
+
+noteCommands.add(new lib.Command({
+	definition: ["set <type> <id> <content>", "Set the note of a resource", (yargs) => {
+		yargs.positional("type", {
+			describe: "Resource type, one of host, instance, user, role, mod_pack, mod, save", type: "string",
+		});
+		yargs.positional("id", { describe: "Id of the resource", type: "string" });
+		yargs.positional("content", { describe: "Text of the note", type: "string" });
+	}],
+	handler: async function(args: { type: string, id: string, content: string }, ctl: Ctl) {
+		await ctl.send(new lib.NoteSetRequest(args.type, args.id, args.content));
+	},
+}));
+
+noteCommands.add(new lib.Command({
+	definition: ["delete <type> <id>", "Remove the note of a resource", (yargs) => {
+		yargs.positional("type", {
+			describe: "Resource type, one of host, instance, user, role, mod_pack, mod, save", type: "string",
+		});
+		yargs.positional("id", { describe: "Id of the resource", type: "string" });
+	}],
+	handler: async function(args: { type: string, id: string }, ctl: Ctl) {
+		await ctl.send(new lib.NoteSetRequest(args.type, args.id, ""));
+	},
+}));

packages/host/lua/clusterio_lib/api.lua

@@ -38,17 +38,20 @@ function api.get_instance_id()
 end
 
 
+local function escape_channel(channel)
+	-- Escape bad characters.  The question mark is used for separating the
+	-- channel name from the payload.
+	return channel:gsub("([\x00-\x1f?\\])", function(match)
+		return "\\x" .. string.format("%02x", match:byte())
+	end)
+end
+
 function api.send_json(channel, data)
 	if not remote.interfaces.clusterio_api then
 		return
 	end
 
-	-- Escape bad characters.  The question mark is used for separating the
-	-- channel name from the payload.
-	channel = channel:gsub("([\x00-\x1f?\\])", function(match)
-		return "\\x" .. string.format("%02x", match:byte())
-	end)
-
+	channel = escape_channel(channel)
 	data = compat.table_to_json(data)
 
 	-- If there's more than about 4kB of data users running with the Windows
@@ -64,5 +67,14 @@ function api.send_json(channel, data)
 	end
 end
 
+function api.send_udp(channel, data)
+	local port = call_api("get_host_udp_port")
+	if not port then
+		return -- Lua UDP is not enabled for this instance
+	end
+
+	helpers.send_udp(port, escape_channel(channel) .. "?" .. data, 0)
+end
+
 
 return api

packages/host/modules/clusterio/api.lua

@@ -17,14 +17,16 @@ function api.get_instance_id()
 	return compat.script_data.clusterio.instance_id
 end
 
-function api.send_json(channel, data)
-
+local function escape_channel(channel)
 	-- Escape bad characters.  The question mark is used for separating the
 	-- channel name from the payload.
-	channel = channel:gsub("([\x00-\x1f?\\])", function(match)
+	return channel:gsub("([\x00-\x1f?\\])", function(match)
 		return "\\x" .. string.format("%02x", match:byte())
 	end)
+end
 
+function api.send_json(channel, data)
+	channel = escape_channel(channel)
 	data = compat.table_to_json(data)
 
 	-- If there's more than about 4kB of data users running with the Windows
@@ -41,5 +43,15 @@ function api.send_json(channel, data)
 	end
 end
 
+function api.send_udp(channel, data)
+	local clusterio = compat.script_data.clusterio
+	local port = clusterio and clusterio.host_udp_port
+	if not port then
+		return -- Lua UDP is not enabled for this instance
+	end
+
+	helpers.send_udp(port, escape_channel(channel) .. "?" .. data, 0)
+end
+
 
 return api

packages/host/modules/clusterio/impl.lua

@@ -12,16 +12,27 @@ local function check_patch()
 	end
 end
 
+-- Added in Factorio 2.0.59, indexing a missing key on a LuaObject errors
+local has_recv_udp = helpers ~= nil and pcall(function() return helpers.recv_udp end)
+
 local impl = {}
 impl.events = {}
 
-impl.events[defines.events.on_tick] = check_patch
+impl.events[defines.events.on_tick] = function()
+	check_patch()
+	-- Dispatch on_udp_packet_received for packets sent by the host, this is
+	-- a no-op when the server was started without --enable-lua-udp
+	if has_recv_udp then
+		helpers.recv_udp(0)
+	end
+end
 
 impl.events[api.events.on_server_startup] = function()
 	if not compat.script_data.clusterio then
 		compat.script_data.clusterio = {
 			instance_id = nil,
 			instance_name = nil,
+			host_udp_port = nil,
 		}
 	end
 
@@ -74,11 +85,12 @@ end
 
 -- Internal API
 clusterio_private = {}
-function clusterio_private.update_instance(new_id, new_name)
+function clusterio_private.update_instance(new_id, new_name, host_udp_port)
 	check_patch()
 	local script_data = compat.script_data
 	script_data.clusterio.instance_id = new_id
 	script_data.clusterio.instance_name = new_name
+	script_data.clusterio.host_udp_port = host_udp_port
 	script.raise_event(api.events.on_instance_updated, {
 		instance_id = new_id,
 		instance_name = new_name,
@@ -105,6 +117,12 @@ remote.add_interface('clusterio_api', {
 		return compat.script_data.clusterio.instance_name
 	end,
 
+	get_host_udp_port = function()
+		-- Not initialised until the first tick after the save is patched
+		local clusterio = compat.script_data.clusterio
+		return clusterio and clusterio.host_udp_port
+	end,
+
 	get_file_no = function()
 		local script_data = compat.script_data
 		script_data.clusterio_file_no = (script_data.clusterio_file_no or 0) + 1

packages/host/modules/clusterio/serialize.lua

@@ -17,6 +17,7 @@ end
 
 local has_create_grid = compat.version_ge("1.1.7")
 local has_quality = compat.version_ge("2.0.0")
+local has_spoilage = compat.version_ge("2.0.0")
 
 -- Equipment Grids are serialized into an array of equipment entries
 -- where ench entry is a table with the following fields:
@@ -106,6 +107,7 @@ end
 --	 c: count
 --	 q: quality (optional)
 --	 h: health (optional)
+--	 p: spoil percent (optional)
 --	 d: durability (optional)
 --	 a: ammo count (optional)
 --	 l: label (optional)
@@ -139,6 +141,7 @@ function serialize.serialize_item_stack(slot, entry)
 	entry.c = slot.count
 	if has_quality and slot.quality.level ~= 0 then entry.q = slot.quality.name end
 	if slot.health < 1 then entry.h = slot.health end
+	if has_spoilage and slot.spoil_percent > 0 then entry.p = slot.spoil_percent end
 	if slot.type == "tool" and slot.durability then entry.d = slot.durability end
 	if slot.type == "ammo" then entry.a = slot.ammo end
 	if slot.is_item_with_label then
@@ -177,6 +180,7 @@ function serialize.deserialize_item_stack(slot, entry)
 	}
 	if entry.q then item_stack.quality = entry.q end
 	if entry.h then item_stack.health = entry.h end
+	if entry.p then item_stack.spoil_percent = entry.p end
 	if entry.d then item_stack.durability = entry.d end
 	if entry.a then item_stack.ammo = entry.a end
 
@@ -288,10 +292,10 @@ function serialize.deserialize_inventory(inventory, serialized)
 			if entry.f then
 				local call_success, call_return = pcall(inventory.set_filter, index, entry.f)
 				if not call_success then
-					print("Error: '" .. call_return .. "' thrown setting filter " .. entry.f)
+					print("Error: '" .. call_return .. "' thrown setting filter " .. serpent.line(entry.f))
 
 				elseif not call_return then
-					print("Error: Failed to set filter " .. entry.f)
+					print("Error: Failed to set filter " .. serpent.line(entry.f))
 				end
 			end
 

packages/host/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/host",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Implementation of Clusterio host server",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -40,14 +40,14 @@
 		"@clusterio/lib": "workspace:^",
 		"@sinclair/typebox": "catalog:",
 		"jimp": "^1.6.1",
-		"jszip": "catalog:",
-		"pidusage": "^4.0.1",
+		"pidusage": "^4.1.0",
 		"rcon-client": "^4.2.5",
 		"semver": "catalog:",
 		"set-blocking": "catalog:",
 		"winston": "catalog:",
 		"winston-daily-rotate-file": "catalog:",
-		"yargs": "catalog:"
+		"yargs": "catalog:",
+		"yazl": "catalog:"
 	},
 	"devDependencies": {
 		"@types/node": "catalog:",
@@ -55,6 +55,7 @@
 		"@types/semver": "catalog:",
 		"@types/set-blocking": "catalog:",
 		"@types/yargs": "catalog:",
+		"@types/yazl": "catalog:",
 		"typescript": "catalog:"
 	}
 }

packages/host/src/BaseInstancePlugin.ts

@@ -259,6 +259,22 @@ export class BaseInstancePlugin {
 		return await this.instance.sendRcon(message, expectEmpty, this.info.name);
 	}
 
+	/**
+	 * Send UDP message to instance
+	 *
+	 * Send a UDP packet to the Factorio server, which is delivered in-game
+	 * as an on_udp_packet_received event.  Delivery is not guaranteed and
+	 * the payload should be kept well below 64 kB.  Requires the LuaUdp
+	 * plugin feature.
+	 *
+	 * This should not be called before onStart or after onStop.
+	 *
+	 * @param message - payload of the packet to send.
+	 */
+	async sendUdp(message: string | Buffer): Promise<void> {
+		await this.instance.sendUdp(message);
+	}
+
 	/**
 	 * Send serially ordered RCON message to instance
 	 *

packages/host/src/Instance.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs/promises";
 import path from "path";
 import pidusage from "pidusage";
+import stream from "node:stream";
 import util from "util";
 import type { Static } from "@sinclair/typebox";
 import { exec } from "child_process";
@@ -232,6 +233,8 @@ export default class Instance extends lib.Link {
 			gamePort: this.config.get("factorio.game_port") ?? host.assignGamePort(this.id),
 			rconPort: this.config.get("factorio.rcon_port") ?? undefined,
 			rconPassword: this.config.get("factorio.rcon_password") ?? undefined,
+			enableLuaUdp: this.config.get("factorio.enable_lua_udp"),
+			luaUdpPort: this.config.get("factorio.lua_udp_port") ?? undefined,
 			enableWhitelist: this.config.get("factorio.enable_whitelist"),
 			enableAuthserverBans: this.config.get("factorio.enable_authserver_bans"),
 			verboseLogging: this.config.get("factorio.verbose_logging"),
@@ -505,6 +508,23 @@ end`.replace(/\r?\n/g, " ");
 		}
 	}
 
+	/**
+	 * Send UDP message to the Factorio server
+	 *
+	 * Delivered in-game as an on_udp_packet_received event.  Requires
+	 * factorio.enable_lua_udp to be enabled on the instance.
+	 *
+	 * @param message - payload of the packet to send.
+	 */
+	async sendUdp(message: string | Buffer) {
+		if (!this.config.get("factorio.enable_lua_udp")) {
+			throw new Error(
+				"Attempted to send UDP message while Lua UDP is disabled. See config factorio.enable_lua_udp."
+			);
+		}
+		await this.server.sendUdp(message);
+	}
+
 	static async listSaves(instanceId: number, savesDir: string, loadedSave: string | null) {
 		let defaultSave = null;
 		if (loadedSave === null) {
@@ -1089,7 +1109,8 @@ end`.replace(/\r?\n/g, " ");
 	 */
 	async updateInstanceData() {
 		let name = lib.escapeString(this.name);
-		await this.sendRcon(`/sc clusterio_private.update_instance(${this.id}, "${name}")`, true);
+		let hostUdpPort = this.server.hostUdpPort ?? "nil";
+		await this.sendRcon(`/sc clusterio_private.update_instance(${this.id}, "${name}", ${hostUdpPort})`, true);
 	}
 
 	async updateFactorioSettings(current: Record<string, unknown>, previous: Record<string, unknown>) {
@@ -1401,13 +1422,12 @@ end`.replace(/\r?\n/g, " ");
 			await this.syncMods();
 			let zip = await exportData(this.server);
 
-			let content = await zip.generateAsync({ type: "nodebuffer" });
 			let url = new URL(this._host.config.get("host.controller_url"));
 			url.pathname += "api/upload-export";
 			url.searchParams.set("mod_pack_id", String(this.activeModPack.id));
 			let response = await fetch(url, {
 				method: "PUT",
-				body: content,
+				body: stream.Readable.toWeb(lib.zipOutputStream(zip)),
 				duplex: "half",
 				headers: {
 					"Content-Type": "application/zip",

packages/host/src/create_scenario.ts

@@ -1,8 +1,9 @@
 // Implementation of the create-scenario command
 import fs from "node:fs/promises";
 import os from "node:os";
-import JSZip from "jszip";
 import path from "path";
+import { pipeline } from "node:stream/promises";
+import yazl from "yazl";
 
 import * as lib from "@clusterio/lib";
 import { logger } from "@clusterio/lib";
@@ -61,33 +62,50 @@ export async function createScenario(
 
 async function zipDirectory(dirPath: string, zipPath: string) {
 	const root = path.basename(path.resolve(dirPath));
-	const zip = new JSZip();
-	let dirs: [string, string][] = [[dirPath, root]];
-	while (dirs.length) {
-		let [dir, relativeDir] = dirs.pop()!;
-		for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
-			let fsPath = path.join(dir, entry.name);
-			let relativePath = path.posix.join(relativeDir, entry.name);
-			if (entry.isFile()) {
-				zip.file(relativePath, await fs.readFile(fsPath));
-			} else if (entry.isDirectory()) {
-				dirs.push([fsPath, relativePath]);
+	const zip = new yazl.ZipFile();
+	const output = lib.zipOutputStream(zip);
+	const fd = await fs.open(zipPath, "w");
+	// addFile() can fail before the traversal is done, so start writing first.
+	const writing = pipeline(output, fd.createWriteStream());
+	writing.catch(() => {});
+	try {
+		let dirs: [string, string][] = [[dirPath, root]];
+		while (dirs.length) {
+			let [dir, relativeDir] = dirs.pop()!;
+			for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
+				let fsPath = path.join(dir, entry.name);
+				let relativePath = path.posix.join(relativeDir, entry.name);
+				if (entry.isFile()) {
+					zip.addFile(fsPath, relativePath);
+				} else if (entry.isDirectory()) {
+					dirs.push([fsPath, relativePath]);
+				}
 			}
 		}
+		zip.end();
+	} catch (err) {
+		output.destroy();
+		await writing.catch(() => {});
+		throw err;
 	}
-	await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" }));
+	await writing;
 }
 
 async function extractZip(zipPath: string, dirPath: string) {
-	const zip = await JSZip.loadAsync(await fs.readFile(zipPath));
-	const root = lib.findRoot(zip);
-	for (let file of Object.values(zip.files)) {
-		if (file.dir || !file.name.startsWith(`${root}/`)) {
-			continue;
+	const zip = await lib.ZipArchive.fromFile(zipPath);
+	try {
+		const root = lib.findRoot(zip);
+		for (let entry of zip.entries.values()) {
+			if (entry.fileName.endsWith("/") || !entry.fileName.startsWith(`${root}/`)) {
+				continue;
+			}
+			const fsPath = path.join(dirPath, entry.fileName.slice(root.length + 1));
+			await fs.mkdir(path.dirname(fsPath), { recursive: true });
+			const fd = await fs.open(fsPath, "w");
+			await pipeline(await zip.openStream(entry), fd.createWriteStream());
 		}
-		const fsPath = path.join(dirPath, file.name.slice(root.length + 1));
-		await fs.mkdir(path.dirname(fsPath), { recursive: true });
-		await fs.writeFile(fsPath, await file.async("nodebuffer"));
+	} finally {
+		zip.close();
 	}
 }
 

packages/host/src/export.ts

@@ -2,7 +2,7 @@
 import fs from "node:fs/promises";
 import path from "path";
 import { Jimp, JimpMime, type JimpInstance } from "jimp";
-import JSZip from "jszip";
+import yazl from "yazl";
 
 import * as lib from "@clusterio/lib";
 import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js";
@@ -64,21 +64,38 @@ async function generateExportMod(server: FactorioServer) {
 	});
 }
 
-let zipCache = new Map<string, JSZip>();
-async function loadZip(server: FactorioServer, modVersions: Map<string, string>, mod: string) {
+interface ModZip {
+	zip: lib.ZipArchive;
+	root: string;
+}
+type ZipCache = Map<string, ModZip>;
+async function loadZip(server: FactorioServer, modVersions: Map<string, string>, zipCache: ZipCache, mod: string) {
 	let modVersion = modVersions.get(mod);
 	if (!modVersion) {
 		throw new Error(`Got path for unknown mod ${mod}`);
 	}
 
 	let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`);
-	let zip = zipCache.get(zipPath);
-	if (!zip) {
-		zip = await JSZip.loadAsync(await fs.readFile(zipPath));
-		zipCache.set(zipPath, zip);
+	let modZip = zipCache.get(zipPath);
+	if (!modZip) {
+		const zip = await lib.ZipArchive.fromFile(zipPath);
+		try {
+			modZip = { zip, root: lib.findRoot(zip) };
+		} catch (err) {
+			zip.close();
+			throw err;
+		}
+		zipCache.set(zipPath, modZip);
 	}
 
-	return zip.folder(lib.findRoot(zip))!;
+	return modZip;
+}
+
+function closeZipCache(zipCache: ZipCache) {
+	for (const { zip } of zipCache.values()) {
+		zip.close();
+	}
+	zipCache.clear();
 }
 
 /**
@@ -86,11 +103,17 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
  *
  * @param server - The server to load the file from.
  * @param modVersions - Mapping of mod to version used.
+ * @param zipCache - Mod zips opened by this export.
  * @param modPath - Factorio style path to the file to load.
  * @returns The content of the file or null if not found.
  * @internal
  */
-async function loadFile(server: FactorioServer, modVersions: Map<string, string>, modPath: string) {
+async function loadFile(
+	server: FactorioServer,
+	modVersions: Map<string, string>,
+	zipCache: ZipCache,
+	modPath: string,
+) {
 	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
 	if (!match) {
 		server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`);
@@ -111,9 +134,9 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
 		}
 	}
 
-	let zip;
+	let modZip;
 	try {
-		zip = await loadZip(server, modVersions, mod);
+		modZip = await loadZip(server, modVersions, zipCache, mod);
 	} catch (err: any) {
 		if (err.code === "ENOENT") {
 			return null;
@@ -121,12 +144,7 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
 		throw err;
 	}
 
-	let file = zip.file(filePath);
-	if (!file) {
-		return null;
-	}
-
-	return await file.async("nodebuffer");
+	return await modZip.zip.readFile(path.posix.join(modZip.root, filePath));
 }
 
 type Image = JimpInstance;
@@ -134,6 +152,7 @@ type IconCache = Map<string, Image | null>;
 async function loadIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	zipCache: ZipCache,
 	iconPath: string,
 	iconSize: number,
 	iconMipmaps: number,
@@ -141,7 +160,7 @@ async function loadIcon(
 ) {
 	let icon = iconCache.get(iconPath);
 	if (icon === undefined) {
-		let fileContent = await loadFile(server, modVersions, iconPath);
+		let fileContent = await loadFile(server, modVersions, zipCache, iconPath);
 		if (fileContent) {
 			icon = await Jimp.fromBuffer(fileContent) as Image;
 			icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize });
@@ -157,11 +176,12 @@ async function loadIcon(
 async function loadSimpleIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	zipCache: ZipCache,
 	item: SimpleIconSpecification,
 	size: number,
 	iconCache: IconCache,
 ): Promise<Image | null> {
-	let icon = await loadIcon(server, modVersions, item.icon, item.icon_size ?? 64, 0, iconCache);
+	let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache);
 	if (icon) {
 		let iconScale = size / (item.icon_size ?? 64);
 		if (iconScale !== 1) {
@@ -175,6 +195,7 @@ async function loadSimpleIcon(
 async function loadLayeredIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	zipCache: ZipCache,
 	item: LayeredIconSpecification,
 	size: number,
 	iconCache: IconCache,
@@ -190,7 +211,9 @@ async function loadLayeredIcon(
 
 	for (let layer of item.icons) {
 		let layerSize = (layer.icon_size || item.icon_size) ?? 64;
-		let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
+		let iconLayer = await loadIcon(
+			server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
+		);
 
 		if (!iconLayer) {
 			continue;
@@ -308,6 +331,7 @@ async function growSheet(state: SheetState) {
 async function exportIcon(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	zipCache: ZipCache,
 	state: SheetState,
 	prototype: PrototypeWithIcon,
 ) {
@@ -318,7 +342,7 @@ async function exportIcon(
 
 	if (prototype.icons) {
 		icon = await loadLayeredIcon(
-			server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
+			server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
 		);
 		iconPos = state.pos;
 		iconPath = prototype.icons[0].icon;
@@ -328,7 +352,7 @@ async function exportIcon(
 		iconPath = prototype.icon;
 		if (iconPos === undefined) {
 			icon = await loadSimpleIcon(
-				server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
+				server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
 			);
 			if (icon) {
 				iconPos = state.pos;
@@ -364,6 +388,7 @@ async function exportIcon(
  *
  * @param server - The server to load icons from.
  * @param modVersions - Mapping of mod name to version.
+ * @param zipCache - Mod zips opened by this export.
  * @param prototypes - Array of prototypes with icon to export for this base type.
  * @param baseName - name of base type (used in metadata).
  * @param state - Shared spritesheet state (mutated in place).
@@ -373,6 +398,7 @@ async function exportIcon(
 async function exportIcons(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	zipCache: ZipCache,
 	prototypes: PrototypeWithIcon[],
 	baseName: string,
 	state: SheetState,
@@ -381,7 +407,7 @@ async function exportIcons(
 	state.metadata[baseName] ??= metadata;
 	let count = 0;
 	for (let prototype of prototypes) {
-		const icon = await exportIcon(server, modVersions, state, prototype);
+		const icon = await exportIcon(server, modVersions, zipCache, state, prototype);
 		if (icon) {
 			count += 1;
 		}
@@ -404,6 +430,7 @@ async function exportIcons(
  *
  * @param server - The server to export the locale from.
  * @param modVersions - Mapping of mod name to version to export locale from.
+ * @param zipCache - Mod zips opened by this export.
  * @param modOrder - Load order of the mods.
  * @param languageCode - Language to export locale for.
  * @returns merged locale information
@@ -412,6 +439,7 @@ async function exportIcons(
 async function exportLocale(
 	server: FactorioServer,
 	modVersions: Map<string, string>,
+	zipCache: ZipCache,
 	modOrder: string[],
 	languageCode: string
 ) {
@@ -445,18 +473,21 @@ async function exportLocale(
 			continue;
 		}
 
-		let zip;
+		let modZip;
 		try {
-			zip = await loadZip(server, modVersions, mod);
+			modZip = await loadZip(server, modVersions, zipCache, mod);
 		} catch (err: any) {
 			if (err.code === "ENOENT") {
 				continue;
 			}
 			throw err;
 		}
-		for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) {
-			let content = await file.async("nodebuffer");
-			mergeLocale(lib.parse(content.toString("utf8")));
+		const localePath = `${path.posix.join(modZip.root, "locale", languageCode)}/`;
+		for (const entry of modZip.zip.entries.values()) {
+			if (entry.fileName.startsWith(localePath) && entry.fileName.endsWith(".cfg")) {
+				let content = await modZip.zip.readEntry(entry);
+				mergeLocale(lib.parse(content.toString("utf8")));
+			}
 		}
 	}
 
@@ -470,6 +501,16 @@ async function exportLocale(
  * @returns zip file with exported data.
  */
 export async function exportData(server: FactorioServer) {
+	const zipCache: ZipCache = new Map();
+	try {
+		return await buildExport(server, zipCache);
+	} finally {
+		// Free up the zip files loaded during the export.
+		closeZipCache(zipCache);
+	}
+}
+
+async function buildExport(server: FactorioServer, zipCache: ZipCache) {
 	await generateExportMod(server);
 
 	let settings: lib.ExportSettings = {};
@@ -549,16 +590,14 @@ export async function exportData(server: FactorioServer) {
 		}
 	}
 
-	let locale = await exportLocale(server, modVersions, modOrder, "en");
+	let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en");
 
-	// Free up the memory used by zip files loaded during the export.
-	zipCache.clear();
-
-	let zip = new JSZip();
-	zip.file("export/settings.json", JSON.stringify(settings satisfies lib.ExportSettings));
-	zip.file("export/prototypes.json", JSON.stringify(prototypes satisfies lib.ExportPrototypes));
-	zip.file("export/locale.json", JSON.stringify([...locale.entries()] satisfies lib.ExportLocale));
-	zip.file("export/defines.json", JSON.stringify(defines satisfies lib.ExportDefines));
+	let zip = new yazl.ZipFile();
+	const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name);
+	addJson("export/settings.json", settings satisfies lib.ExportSettings);
+	addJson("export/prototypes.json", prototypes satisfies lib.ExportPrototypes);
+	addJson("export/locale.json", [...locale.entries()] satisfies lib.ExportLocale);
+	addJson("export/defines.json", defines satisfies lib.ExportDefines);
 
 	// Build a single unified spritesheet across all prototypes with icons.
 	const state = await createSheetState();
@@ -572,15 +611,16 @@ export async function exportData(server: FactorioServer) {
 		if (prototypesWithIcons.length === 0) {
 			continue;
 		}
-		const count = await exportIcons(server, modVersions, prototypesWithIcons, baseName, state);
+		const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state);
 		server._logger.info(`Exported ${count} ${baseName} icons`);
 	}
 
 	// Crop sheet to actual used height and write single spritesheet + metadata.
 	const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
 	state.sheet.crop({ x: 0, y: 0, w: SHEET_WIDTH, h: Math.max(usedRows * SHEET_ICON_SIZE, 1) });
-	zip.file("export/spritesheet.png", await state.sheet.getBuffer(JimpMime.png));
-	zip.file("export/metadata.json", JSON.stringify(state.metadata satisfies lib.ExportMetadata));
+	zip.addBuffer(await state.sheet.getBuffer(JimpMime.png), "export/spritesheet.png");
+	addJson("export/metadata.json", state.metadata satisfies lib.ExportMetadata);
+	zip.end();
 
 	server._logger.info(`Export complete: ${state.pos} icons on ${usedRows} row(s)`);
 

packages/host/src/patch.ts

@@ -1,10 +1,9 @@
 // Library for patching Factorio saves with scenario code.
 
-import events from "events";
 import fs from "node:fs/promises";
-import JSZip from "jszip";
 import path from "path";
 import semver from "semver";
+import { pipeline } from "node:stream/promises";
 import { Type, Static } from "@sinclair/typebox";
 
 import * as lib from "@clusterio/lib";
@@ -72,18 +71,16 @@ export class SaveModule {
 		};
 	}
 
-	static async fromSave(json: Static<typeof SaveModule.jsonSchema>, root: JSZip) {
+	static async fromSave(json: Static<typeof SaveModule.jsonSchema>, zip: lib.ZipArchive, root: string) {
 		const module = new this(lib.ModuleInfo.fromJSON(json));
-		module.files = new Map(await Promise.all(json.files
-			.map(filename => ({filename, file: root.file(filename)}))
-			.filter(({filename, file}) => {
-				if (file === null) {
-					lib.logger.warn(`Missing file ${filename} in save`);
-				}
-				return file !== null;
-			})
-			.map(async ({filename, file}) => [filename, await file!.async("nodebuffer")] as const)
-		));
+		for (const filename of json.files) {
+			const content = await zip.readFile(path.posix.join(root, filename));
+			if (content === null) {
+				lib.logger.warn(`Missing file ${filename} in save`);
+				continue;
+			}
+			module.files.set(filename, content);
+		}
 		return module;
 	}
 
@@ -205,7 +202,7 @@ export class PatchInfo {
 		};
 	}
 
-	static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, root: JSZip) {
+	static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, zip: lib.ZipArchive, root: string) {
 		if (json.version === undefined) {
 			interface ScenarioInfoV0 {
 				name: string;
@@ -229,7 +226,7 @@ export class PatchInfo {
 		return new this(
 			json.patch_number,
 			lib.ModuleInfo.fromJSON(json.scenario),
-			await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))),
+			await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))),
 		);
 	}
 }
@@ -396,80 +393,111 @@ const knownScenarios: Record<string, lib.ModuleInfo> = {
  * @param modules - Description of the modules to patch.
  */
 export async function patch(savePath: string, modules: SaveModule[]) {
-	let zip = await JSZip.loadAsync(await fs.readFile(savePath));
-	let root = zip.folder(lib.findRoot(zip))!;
+	const zip = await lib.ZipArchive.fromFile(savePath);
+	let tempSavePath: string;
+	try {
+		const root = lib.findRoot(zip);
+		const rootPath = (relativePath: string) => path.posix.join(root, relativePath);
 
-	let patchInfoFile = root.file("clusterio.json");
-	let patchInfo: PatchInfo;
-	if (patchInfoFile !== null) {
-		let content = await patchInfoFile.async("string");
-		patchInfo = await PatchInfo.fromSave(JSON.parse(content), root);
-		if (patchInfo.version > PatchInfo.currentVersion) {
-			throw new Error(
-				`Save patch version ${patchInfo.version} is newer than the patch version this ` +
-				`version of Clusterio can load (${PatchInfo.currentVersion})`
-			);
-		}
+		// Files to add or replace in the save, relative to the root
+		const files = new Map<string, Buffer>();
+		let patchInfo: PatchInfo;
+		const patchInfoContent = await zip.readFile(rootPath("clusterio.json"));
+		if (patchInfoContent !== null) {
+			patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root);
+			if (patchInfo.version > PatchInfo.currentVersion) {
+				throw new Error(
+					`Save patch version ${patchInfo.version} is newer than the patch version this ` +
+					`version of Clusterio can load (${PatchInfo.currentVersion})`
+				);
+			}
 
-	// No info file present, try to detect if it's a known compatible scenario.
-	} else {
-		let controlFile = root.file("control.lua");
-		if (!controlFile) {
-			throw new Error("Unable to patch save, missing control.lua file.");
-		}
-		let controlStream = controlFile.nodeStream("nodebuffer");
-		let controlHash = await lib.hashStream(controlStream);
-
-		if (controlHash in knownScenarios) {
-			patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
-			root.file("scenario.lua", controlFile.nodeStream("nodebuffer"));
+		// No info file present, try to detect if it's a known compatible scenario.
 		} else {
-			throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
-		}
-	}
+			const controlFile = zip.file(rootPath("control.lua"));
+			if (!controlFile) {
+				throw new Error("Unable to patch save, missing control.lua file.");
+			}
+			const controlHash = await lib.hashStream(await zip.openStream(controlFile));
 
-	// Increment patch number
-	patchInfo.patchNumber = patchInfo.patchNumber + 1;
-
-	// Remove any existing modules from the save
-	if (patchInfo.version === 0) {
-		for (let file of root.file(/^modules\//)) {
-			zip.remove(file.name);
-		}
-	} else {
-		for (let module of patchInfo.modules) {
-			for (let filepath of module.files.keys()) {
-				const file = root.file(filepath);
-				if (file !== null) {
-					zip.remove(file.name);
-				}
+			if (controlHash in knownScenarios) {
+				patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
+				files.set("scenario.lua", await zip.readEntry(controlFile));
+			} else {
+				throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
 			}
 		}
-		patchInfo.modules = [];
-	}
 
-	reorderDependencies(modules);
+		// Increment patch number
+		patchInfo.patchNumber = patchInfo.patchNumber + 1;
 
-	// Add the modules to the save.
-	for (let module of modules) {
-		for (let [relativePath, contents] of module.files) {
-			root.file(relativePath, contents);
+		// Remove any existing modules from the save
+		const removed = new Set<string>();
+		if (patchInfo.version === 0) {
+			const modulesPath = rootPath("modules/");
+			for (const name of zip.entries.keys()) {
+				if (name.startsWith(modulesPath)) {
+					removed.add(name);
+				}
+			}
+		} else {
+			for (const module of patchInfo.modules) {
+				for (const filepath of module.files.keys()) {
+					removed.add(rootPath(filepath));
+				}
+			}
+			patchInfo.modules = [];
 		}
-		patchInfo.modules.push(module);
-	}
 
-	// Add loading code and patch info
-	root.file("control.lua", generateLoader(patchInfo));
-	root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
+		reorderDependencies(modules);
 
-	// Write back the save
-	let stream = zip.generateNodeStream({ compression: "DEFLATE" });
-	const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
-	try {
-		let pipe = stream.pipe(writeStream);
-		await events.once(pipe, "finish");
+		// Add the modules to the save.
+		for (const module of modules) {
+			for (const [relativePath, contents] of module.files) {
+				files.set(relativePath, contents);
+			}
+			patchInfo.modules.push(module);
+		}
+
+		// Add loading code and patch info
+		files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8"));
+		files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8"));
+
+		// Write back the save, keeping the order of the entries in the original
+		const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents]));
+		const output = new lib.ZipWriter();
+		let writeStream;
+		[tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
+		const writing = pipeline(output.outputStream, writeStream);
+		// Awaited below, but a write error can reject it before then.
+		writing.catch(() => {});
+		try {
+			for (const entry of zip.entries.values()) {
+				const name = entry.fileName;
+				if (removed.has(name)) {
+					continue;
+				}
+				const contents = added.get(name);
+				if (contents !== undefined) {
+					await output.addBuffer(contents, name);
+					added.delete(name);
+				} else {
+					await output.addEntry(zip, entry);
+				}
+			}
+			for (const [name, contents] of added) {
+				await output.addBuffer(contents, name);
+			}
+			await output.end();
+			await writing;
+		} catch (err) {
+			output.outputStream.destroy();
+			await writing.catch(() => {});
+			await fs.rm(tempSavePath, { force: true });
+			throw err;
+		}
 	} finally {
-		writeStream.destroy();
+		zip.close();
 	}
 	await fs.rename(tempSavePath, savePath);
 }

packages/host/src/server.ts

@@ -1,9 +1,12 @@
 // Factorio Server interface
 import fs from "node:fs/promises";
-import { type FSWatcher, watch as fsWatch, writeFileSync } from "node:fs";
+import { type FSWatcher, createWriteStream, watch as fsWatch, writeFileSync } from "node:fs";
 import child_process from "child_process";
+import dgram from "dgram";
 import path from "path";
-import JSZip from "jszip";
+import stream from "node:stream";
+import { pipeline } from "node:stream/promises";
+import type { ReadableStream } from "node:stream/web";
 import events from "events";
 import util from "util";
 import crypto from "crypto";
@@ -112,29 +115,38 @@ async function downloadAndExtractZip(url: string, targetDir: string) {
 		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
 	}
 
-  	// Load the ZIP with JSZip
-	const buffer = Buffer.from(await res.arrayBuffer());
-	const zip = await JSZip.loadAsync(buffer);
+	if (!res.body) {
+		throw new Error(`Failed to fetch ${url}: empty response`);
+	}
 
-  	// Extract each file/folder
-	await fs.mkdir(targetDir, { recursive: true });
-	await Promise.all(
-		Object.values(zip.files).map(async (entry) => {
-			const parts = entry.name.split("/").filter(Boolean);
-			const strippedPath = parts.slice(1).join("/");
-			if (!strippedPath) {
-				return; // This copies the behaviour of tar --strip-components 1
-			}
+	// Write the download to a temp file, zip files need random access to read
+	const tmpFilePath = `${targetDir}.tmp`;
+	await fs.writeFile(tmpFilePath, stream.Readable.fromWeb(res.body as ReadableStream));
+	try {
+		const zip = await lib.ZipArchive.fromFile(tmpFilePath);
+		try {
+			await fs.mkdir(targetDir, { recursive: true });
+			for (const entry of zip.entries.values()) {
+				const parts = entry.fileName.split("/").filter(Boolean);
+				const strippedPath = parts.slice(1).join("/");
+				if (!strippedPath) {
+					continue; // This copies the behaviour of tar --strip-components 1
+				}
 
-			const entryPath = path.join(targetDir, strippedPath);
-			if (entry.dir) {
-				await fs.mkdir(entryPath, { recursive: true });
-			} else {
-				await fs.mkdir(path.dirname(entryPath), { recursive: true });
-				await fs.writeFile(entryPath, await entry.async("nodebuffer"));
+				const entryPath = path.join(targetDir, strippedPath);
+				if (entry.fileName.endsWith("/")) {
+					await fs.mkdir(entryPath, { recursive: true });
+				} else {
+					await fs.mkdir(path.dirname(entryPath), { recursive: true });
+					await pipeline(await zip.openStream(entry), createWriteStream(entryPath));
+				}
 			}
-		})
-	);
+		} finally {
+			zip.close();
+		}
+	} finally {
+		await fs.rm(tmpFilePath);
+	}
 }
 
 /**
@@ -170,6 +182,9 @@ async function downloadAndExtractTar(url: string, targetDir: string) {
 	}
 }
 
+// Factorio downloads in progress by target directory, shared by every server in the process
+const downloadsInProgress = new Map<string, Promise<void>>();
+
 /**
  * Find factorio data directory of the given version
  *
@@ -289,6 +304,15 @@ async function generatePassword(length: number) {
 	}
 }
 
+function unescapeChannel(channel: Buffer) {
+	return channel
+		.toString("utf-8")
+		.replace(/\\x([0-9a-f]{2})/g, (match, p1) => (
+			String.fromCharCode(parseInt(p1, 16))
+		))
+	;
+}
+
 /**
  * Interpret lines of output from Factorio
  *
@@ -411,6 +435,9 @@ const outputHeuristics: Heuristic[] = [
 		filter: { type: "log", message: /^Starting RCON interface/ },
 		action: function() {
 			this._startRcon().catch((err) => { this.emit("error", err); });
+			if (this.enableLuaUdp) {
+				this._startUdp().catch((err) => { this.emit("error", err); });
+			}
 		},
 	},
 
@@ -542,6 +569,10 @@ export interface FactorioServerOptions {
 	rconPort?: number,
 	/** Password use for RCON. */
 	rconPassword?: string,
+	/** Enable UDP messaging with the Factorio server. */
+	enableLuaUdp?: boolean,
+	/** UDP port the Factorio server receives Lua UDP messages on. */
+	luaUdpPort?: number,
 	/** Turn on whitelisting. */
 	enableWhitelist?: boolean,
 	/** Enable Factorio.com based multiplayer bans. */
@@ -586,8 +617,9 @@ type FactorioServerEvents = {
 	"_saving": [],
 	"_saved": [],
 
-	// IPS events
+	// IPC events
 	[ipcEvent: `ipc-${string}`]: [ event: any ],
+	[udpEvent: `udp-${string}`]: [ data: Buffer ],
 };
 
 /**
@@ -607,6 +639,8 @@ type FactorioServerEvents = {
  * - autosave-fnished - invoked when the autosave finished
  * - save-finished - invoked when the server has finished a manual save
  * - exit - invoked when the sterver has exited
+ * - ipc-<channel> - invoked with data sent with send_json in-game
+ * - udp-<channel> - invoked with data sent with send_udp in-game
  * @extends events.EventEmitter
  */
 export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
@@ -618,6 +652,10 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	rconPort: number;
 	/** Password used for RCON on the Factorio game server */
 	rconPassword: string;
+	/** Enable UDP messaging with the Factorio game server */
+	enableLuaUdp: boolean;
+	/** UDP port the Factorio game server receives Lua UDP messages on */
+	luaUdpPort: number;
 	/** Enable player whitelist */
 	enableWhitelist: boolean;
 	/** Enable Factorio.com based multiplayer bans **/
@@ -647,6 +685,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	_server: child_process.ChildProcessWithoutNullStreams | null = null;
 	_rconClient: Rcon | null= null;
 	_rconReady = false;
+	_udpSocket: dgram.Socket | null = null;
 	_gameReady = false;
 	_stripRegExp?: RegExp;
 	_maxConcurrentCommands = 5;
@@ -679,6 +718,10 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		this.rconPort = options.rconPort || randomDynamicPort();
 		/** Password used for RCON on the Factorio game server */
 		this.rconPassword = options.rconPassword as string; // init will generate one if not there
+		/** Enable UDP messaging with the Factorio game server */
+		this.enableLuaUdp = options.enableLuaUdp || false;
+		/** UDP port the Factorio game server receives Lua UDP messages on */
+		this.luaUdpPort = options.luaUdpPort || randomDynamicPort();
 		/** Enable player whitelist */
 		this.enableWhitelist = options.enableWhitelist || false;
 		/** Enable Factorio.com based multiplayer bans **/
@@ -741,19 +784,24 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		}));
 	}
 
+	handleUdp(channel: string, handler: (data: Buffer) => Promise<void>) {
+		// Runs from the socket callback, so synchronous throws must be caught here too
+		this.on(`udp-${channel}`, async (data) => {
+			try {
+				await handler(data);
+			} catch (err: any) {
+				this._logger.error(`Error handling udp event:\n${err.stack ?? err.message}`);
+			}
+		});
+	}
+
 	async _handleIpc(line: Buffer) {
 		let channelEnd = line.indexOf("?");
 		if (channelEnd === -1) {
 			throw new Error(`Malformed IPC line "${line.toString()}"`);
 		}
 
-		let channel = line
-			.subarray(6, channelEnd)
-			.toString("utf-8")
-			.replace(/\\x([0-9a-f]{2})/g, (match, p1) => (
-				String.fromCharCode(parseInt(p1, 16))
-			))
-		;
+		let channel = unescapeChannel(line.subarray(6, channelEnd));
 
 		let type = line.subarray(channelEnd + 1, channelEnd + 2).toString("utf-8");
 		let content;
@@ -791,6 +839,74 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		}
 	}
 
+	_handleUdpMessage(message: Buffer, rinfo: dgram.RemoteInfo) {
+		// Factorio sends from the socket bound to the Lua UDP port, anything
+		// else is some other local process.
+		if (rinfo.port !== this.luaUdpPort) {
+			this._logger.warn(`Ignoring UDP packet from unexpected source ${rinfo.address}:${rinfo.port}`);
+			return;
+		}
+
+		let channelEnd = message.indexOf("?");
+		if (channelEnd === -1) {
+			this._logger.warn(`Ignoring malformed UDP packet "${message.subarray(0, 100).toString("utf-8")}"`);
+			return;
+		}
+
+		let channel = unescapeChannel(message.subarray(0, channelEnd));
+		let data = message.subarray(channelEnd + 1);
+		if (!this.emit(`udp-${channel}`, data)) {
+			this._logger.warn(`Warning: Unhandled udp-${channel}`);
+		}
+	}
+
+	async _startUdp() {
+		if (this._udpSocket !== null) {
+			throw Error("UDP socket is already open");
+		}
+
+		const socket = dgram.createSocket("udp4");
+		socket.on("message", (message, rinfo) => this._handleUdpMessage(message, rinfo));
+		try {
+			await new Promise<void>((resolve, reject) => {
+				socket.once("error", reject);
+				socket.bind(0, "127.0.0.1", () => {
+					socket.off("error", reject);
+					resolve();
+				});
+			});
+		} catch (err: any) {
+			this._logger.error(`Failed to open UDP socket:\n${err?.stack ?? err?.message ?? err}`);
+			socket.close();
+			return;
+		}
+		socket.on("error", err => this.emit("error", err));
+		this._udpSocket = socket;
+	}
+
+	// Factorio before 2.1.10 crashes when a packet is received on a headless server
+	_checkLuaUdpVersion() {
+		if (this.enableLuaUdp && lib.integerFullVersion(this._version!) < lib.integerFullVersion("2.1.10")) {
+			throw new Error("Lua UDP requires Factorio 2.1.10 or later");
+		}
+	}
+
+	_stopUdp() {
+		if (this._udpSocket) {
+			this._udpSocket.close();
+			this._udpSocket = null;
+		}
+	}
+
+	/**
+	 * UDP port on localhost the host receives Lua UDP messages on
+	 *
+	 * Only available while the server is running with Lua UDP enabled.
+	 */
+	get hostUdpPort(): number | undefined {
+		return this._udpSocket?.address().port;
+	}
+
 	_handleOutput(rawLine: Buffer, source: "stdout" | "stderr") {
 		if (rawLine.subarray(0, 6).equals(Buffer.from("\f$ipc:"))) {
 			this._handleIpc(rawLine).catch(err => this.emit("error", err));
@@ -1001,8 +1117,10 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 			return;
 		}
 
+		// A version that is still being extracted can already look installed
+		const dst = path.join(this._factorioDir, latestVersion.version);
 		const installedVersions = await listFactorioVersions(this._factorioDir);
-		if (!installedVersions.versions.has(latestVersion.version)) {
+		if (!installedVersions.versions.has(latestVersion.version) || downloadsInProgress.has(dst)) {
 			const v = latestVersion.version;
 			// Can't download if not linux or this is a direct install
 			if (installedVersions.direct) {
@@ -1022,10 +1140,18 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 				return;
 			}
 
-			// Download the newer version
-			this._logger.info(`A newer version of factorio is available (${v}), starting download...`);
-			const dst = path.join(this._factorioDir, latestVersion.version);
-			await downloadAndExtractTar(latestVersion.headlessUrl, dst);
+			// Download the newer version, or wait for another server already downloading it
+			let download = downloadsInProgress.get(dst);
+			if (download) {
+				this._logger.info(`A newer version of factorio is available (${v}), waiting for download...`);
+			} else {
+				this._logger.info(`A newer version of factorio is available (${v}), starting download...`);
+				download = downloadAndExtractTar(latestVersion.headlessUrl, dst).finally(() => {
+					downloadsInProgress.delete(dst);
+				});
+				downloadsInProgress.set(dst, download);
+			}
+			await download;
 			this._logger.info(`Downloaded factorio version ${v}`);
 		}
 	}
@@ -1063,6 +1189,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		this._server = null;
 		this._rconClient = null;
 		this._rconReady = false;
+		this._udpSocket = null;
 		this._gameReady = false;
 		this._unexpected = [];
 		this._killed = false;
@@ -1108,6 +1235,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 			if (this._rconClient) {
 				this._rconClient.end().catch(() => {});
 			}
+			this._stopUdp();
 
 			if (this._whitelistWatcher) {
 				this._whitelistWatcher.close();
@@ -1126,6 +1254,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 			if (this._rconClient) {
 				this._rconClient.end().catch(() => {});
 			}
+			this._stopUdp();
 
 			if (this._whitelistWatcher) {
 				this._whitelistWatcher.close();
@@ -1224,6 +1353,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	 */
 	async start(save: string) {
 		this._check(["init"]);
+		this._checkLuaUdpVersion();
 		this._state = "running";
 
 		try {
@@ -1239,6 +1369,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 					"--port", String(this.gamePort),
 					"--rcon-port", String(this.rconPort),
 					"--rcon-password", this.rconPassword,
+					...(this.enableLuaUdp ? ["--enable-lua-udp", String(this.luaUdpPort)] : []),
 					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
 					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
 					...(this.verboseLogging ? ["--verbose"] : []),
@@ -1276,6 +1407,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	 */
 	async startScenario(scenario: string, seed?: number, mapGenSettings?: object, mapSettings?: object) {
 		this._check(["init"]);
+		this._checkLuaUdpVersion();
 		this._state = "running";
 
 		try {
@@ -1297,6 +1429,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 					"--port", String(this.gamePort),
 					"--rcon-port", String(this.rconPort),
 					"--rcon-password", this.rconPassword,
+					...(this.enableLuaUdp ? ["--enable-lua-udp", String(this.luaUdpPort)] : []),
 					...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
 					...(this.enableAuthserverBans ? ["--use-authserver-bans"] : []),
 					...(this.verboseLogging ? ["--verbose"] : []),
@@ -1377,6 +1510,27 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		return response;
 	}
 
+	/**
+	 * Send UDP message to the server
+	 *
+	 * Sends a UDP packet to the Lua UDP port of the Factorio server, which
+	 * is delivered in-game as an on_udp_packet_received event.  Delivery is
+	 * not guaranteed.  Requires Lua UDP to be enabled.
+	 *
+	 * @param message - payload of the packet to send.
+	 */
+	async sendUdp(message: string | Buffer) {
+		this._check(["running", "stopping"]);
+		if (!this._udpSocket) {
+			throw new Error("UDP socket is not open");
+		}
+
+		const socket = this._udpSocket;
+		await new Promise<void>((resolve, reject) => {
+			socket.send(message, this.luaUdpPort, "127.0.0.1", err => (err ? reject(err) : resolve()));
+		});
+	}
+
 	/**
 	 * Timeout in ms to wait after a shutdown is requested before killing the
 	 * process. Defaults to 0 meaning no timeout
@@ -1412,6 +1566,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		if (this._rconClient) {
 			this._rconClient.end().catch(() => {});
 		}
+		this._stopUdp();
 		if (this._whitelistWatcher) {
 			this._whitelistWatcher.close();
 		}

packages/lib/build_mod.js

@@ -1,13 +1,40 @@
 /* eslint-disable no-console */
 import fs from "node:fs/promises";
-import JSZip from "jszip";
 import path from "node:path";
-import stream from "node:stream";
-import util from "node:util";
+import { pipeline } from "node:stream/promises";
 import yargs from "yargs";
+import yazl from "yazl";
 
-const finished = util.promisify(stream.finished);
 
+async function addModFiles(args, info, modName, zip) {
+	for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) {
+		if (dirent.isFile()) {
+			const itemPath = path.join(dirent.parentPath, dirent.name);
+			// On Windows the path created uses backslashes as the directory sepparator
+			// but the zip file needs to use forward slashes.  We can't use the posix
+			// version of relative here as it doesn't work with Windows style paths.
+			let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/");
+			if (basePath === "info.json") {
+				// The info.json file is overridden later.
+				continue;
+			}
+			zip.addFile(itemPath, path.posix.join(modName, basePath));
+		}
+	}
+
+	for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
+		let filePath = path.join(args.sourceDir, ...pathParts);
+		try {
+			await fs.access(filePath);
+		} catch (err) {
+			throw new Error(`Error reading additional file ${filePath}`, { cause: err });
+		}
+		zip.addFile(filePath, path.posix.join(modName, fileName));
+	}
+	delete info.additional_files;
+
+	zip.addBuffer(Buffer.from(JSON.stringify(info, null, "\t")), path.posix.join(modName, "info.json"));
+}
 
 async function buildMod(args, info, modName) {
 	if (args.build) {
@@ -15,43 +42,24 @@ async function buildMod(args, info, modName) {
 		modName = args.modName ?? `${info.name}_${info.version}`;
 
 		if (args.pack) {
-			let zip = new JSZip();
-			for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) {
-				if (dirent.isFile()) {
-					const itemPath = path.join(dirent.parentPath, dirent.name);
-					// On Windows the path created uses backslashes as the directory sepparator
-					// but the zip file needs to use forward slashes.  We can't use the posix
-					// version of relative here as it doesn't work with Windows style paths.
-					let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/");
-					// eslint-disable-next-line max-depth
-					if (basePath === "info.json") {
-						// The info.json file is overridden later.
-						continue;
-					}
-					zip.file(path.posix.join(modName, basePath), (await fs.open(itemPath)).createReadStream());
-				}
-			}
-
-			for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
-				let filePath = path.join(args.sourceDir, ...pathParts);
-				let fileStream;
-				try {
-					fileStream = (await fs.open(filePath)).createReadStream();
-				} catch (err) {
-					throw new Error(`Error reading additional file ${filePath}`, { cause: err });
-				}
-				zip.file(path.posix.join(modName, fileName), fileStream);
-			}
-			delete info.additional_files;
-
-			zip.file(path.posix.join(modName, "info.json"), JSON.stringify(info, null, "\t"));
-
 			let modPath = path.join(args.outputDir, `${modName}.zip`);
 			console.log(`Writing ${modPath}`);
-			let writeStream = zip.generateNodeStream().pipe(
-				(await fs.open(modPath, "w")).createWriteStream()
-			);
-			await finished(writeStream);
+			let zip = new yazl.ZipFile();
+			// yazl reports errors on the zip object, not the output stream.
+			zip.on("error", err => zip.outputStream.destroy(err));
+			// addFile() can fail before the traversal is done, so start writing first.
+			const writing = pipeline(zip.outputStream, (await fs.open(modPath, "w")).createWriteStream());
+			writing.catch(() => {});
+			try {
+				await addModFiles(args, info, modName, zip);
+				zip.end();
+				await writing;
+			} catch (err) {
+				zip.outputStream.destroy();
+				await writing.catch(() => {});
+				await fs.rm(modPath, { force: true });
+				throw err;
+			}
 
 		} else {
 			let modDir = path.join(args.outputDir, modName);

packages/lib/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/lib",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Shared library for Clusterio",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",
@@ -51,16 +51,19 @@
 
 	"dependencies": {
 		"@sinclair/typebox": "catalog:",
+		"@types/yauzl": "catalog:",
+		"@types/yazl": "catalog:",
 		"ajv": "^8.20.0",
 		"chalk": "catalog:",
 		"fast-deep-equal": "^3.1.3",
-		"jszip": "catalog:",
 		"set-blocking": "catalog:",
 		"triple-beam": "^1.4.1",
 		"winston": "catalog:",
 		"winston-transport": "catalog:",
 		"ws": "catalog:",
-		"yargs": "catalog:"
+		"yargs": "catalog:",
+		"yauzl": "catalog:",
+		"yazl": "catalog:"
 	},
 	"devDependencies": {
 		"@types/node": "catalog:",

packages/lib/src/config/classes.ts

@@ -109,8 +109,8 @@ export interface FieldDefinition<
 	 */
 	autoComplete?: string;
 	/**
-	 * Web UI component to use for rending this input. Can be extended by
-	 * @{link "@clusterio/web_ui".BaseWebPlugin.inputComponents}
+	 * Web UI component to use for rendering this input. Plugins register components
+	 * through the inputComponents hook on control.hooks in their web entrypoint.
 	 */
 	inputComponent?: string;
 	/**
@@ -119,7 +119,8 @@ export interface FieldDefinition<
 	optional?: Type extends null ? true : false;
 	/**
 	 * Value this config field should take in a newly initialized config.
-	 * This can also be an function returning the value to use.
+	 * This can also be a synchronous function returning the value to use.
+	 * Promise results are not awaited.
 	 */
 	initialValue?: unknown | (() => unknown);
 	/**

packages/lib/src/config/definitions.ts

@@ -482,6 +482,8 @@ export interface InstanceConfigFields {
 	"factorio.host_assigned_game_port": number | null;
 	"factorio.rcon_port": number | null;
 	"factorio.rcon_password": string | null;
+	"factorio.enable_lua_udp": boolean;
+	"factorio.lua_udp_port": number | null;
 	"factorio.player_online_autosave_slots": number;
 	"factorio.mod_pack_id": number | null;
 	"factorio.enable_save_patching": boolean;
@@ -606,6 +608,21 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			type: "string",
 			optional: true,
 		},
+		"factorio.enable_lua_udp": {
+			description:
+				"Enable UDP messaging between the Factorio server and the host. " +
+				"Required by plugins using UDP. Requires Factorio 2.1.10 or later.",
+			restartRequired: true, // Because of the cli option "--enable-lua-udp"
+			type: "boolean",
+			initialValue: false,
+		},
+		"factorio.lua_udp_port": {
+			description: "UDP port for the Factorio server to receive Lua UDP messages on, uses a random port if null",
+			restartRequired: true,
+			type: "number",
+			optional: true,
+			validator: validators.optional(validators.greaterThanEqualZero),
+		},
 		"factorio.player_online_autosave_slots": {
 			description:
 				"Rename autosaves where players have been online since the previous autosave into a separate " +
@@ -795,7 +812,12 @@ export function addPluginFieldDefinitions(
 			fieldDef.dependsOn.push("factorio.enable_script_commands");
 		}
 
-		if (scriptCommands || savePatching) {
+		const luaUdp = pluginInfo.features.includes("LuaUdp");
+		if (luaUdp) {
+			fieldDef.dependsOn.push("factorio.enable_lua_udp");
+		}
+
+		if (scriptCommands || savePatching || luaUdp) {
 			fieldDef.validator = (enabled, config) => {
 				if (!enabled) {
 					return;
@@ -806,6 +828,9 @@ export function addPluginFieldDefinitions(
 				if (scriptCommands && !config.get("factorio.enable_script_commands")) {
 					throw new Error(`Plugin ${pluginInfo.name} requires script commands`);
 				}
+				if (luaUdp && !config.get("factorio.enable_lua_udp")) {
+					throw new Error(`Plugin ${pluginInfo.name} requires Lua UDP`);
+				}
 			};
 		}
 	}

packages/lib/src/data/ModInfo.ts

@@ -1,10 +1,10 @@
 import fs from "node:fs/promises";
-import JSZip from "jszip";
+import path from "path";
 import { Type, Static } from "@sinclair/typebox";
 
 import * as libHash from "../hash.js";
 import * as libSchema from "../schema.js";
-import { findRoot } from "../zip_ops.js";
+import { ZipArchive, findRoot } from "../zip_ops.js";
 import { ModRecord } from "./ModPack.js";
 
 import {
@@ -352,18 +352,15 @@ export default class ModInfo {
 
 	static async fromModFile(modPath: string) {
 		let modInfo: Static<typeof ModInfo.jsonSchema>;
-		{
-			// XXX: JSZip needs the whole archive loaded in memory to work.
-			// This is clearly untenable and will be replaced later.
-			let zip = await JSZip.loadAsync(await fs.readFile(modPath));
-			let root = zip.folder(findRoot(zip))!;
-
-			let infoFile = root.file("info.json");
-			if (!infoFile) {
+		let zip = await ZipArchive.fromFile(modPath);
+		try {
+			let content = await zip.readFile(path.posix.join(findRoot(zip), "info.json"));
+			if (content === null) {
 				throw new Error("Mod contains no info.json file");
 			}
-
-			modInfo = JSON.parse(await infoFile.async("string"));
+			modInfo = JSON.parse(content.toString("utf8"));
+		} finally {
+			zip.close();
 		}
 
 		let valid = this.validateInfo(modInfo);

packages/lib/src/data/Note.ts

@@ -0,0 +1,74 @@
+import { Type, Static } from "@sinclair/typebox";
+
+/**
+ * Free text note attached to a resource in the cluster.
+ *
+ * Notes are keyed by the type of resource and the id of the resource within
+ * that type. Core resource types are "host", "instance", "user", "role",
+ * "mod_pack", "mod" and "save", plugins may define their own.
+ */
+export default class Note {
+	constructor(
+		/** Type of resource this note is attached to. */
+		public resourceType: string,
+		/** Id of the resource within its type, as a string. */
+		public resourceId: string,
+		/** Text of the note. */
+		public content: string,
+		/** Name of the user that last changed the note. */
+		public updatedBy = "",
+		/** Millisecond Unix timestamp this entry was last updated at. */
+		public updatedAtMs = 0,
+		/** True if this note has been removed. */
+		public isDeleted = false,
+	) { }
+
+	/** Unique key for this note, resource type and id joined with a slash. */
+	get id() {
+		return Note.idFor(this.resourceType, this.resourceId);
+	}
+
+	static idFor(resourceType: string, resourceId: string | number) {
+		return `${resourceType}/${resourceId}`;
+	}
+
+	static jsonSchema = Type.Object({
+		"resourceType": Type.String(),
+		"resourceId": Type.String(),
+		"content": Type.String(),
+		"updatedBy": Type.Optional(Type.String()),
+		"updatedAtMs": Type.Optional(Type.Number()),
+		"isDeleted": Type.Optional(Type.Boolean()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.resourceType,
+			json.resourceId,
+			json.content,
+			json.updatedBy,
+			json.updatedAtMs,
+			json.isDeleted,
+		);
+	}
+
+	toJSON() {
+		const json = {
+			resourceType: this.resourceType,
+			resourceId: this.resourceId,
+			content: this.content,
+		} as Static<typeof Note.jsonSchema>;
+
+		if (this.updatedBy !== "") {
+			json.updatedBy = this.updatedBy;
+		}
+		if (this.updatedAtMs !== 0) {
+			json.updatedAtMs = this.updatedAtMs;
+		}
+		if (this.isDeleted !== false) {
+			json.isDeleted = this.isDeleted;
+		}
+
+		return json;
+	}
+}

packages/lib/src/data/index.ts

@@ -12,6 +12,7 @@ export { default as ModuleInfo } from "./ModuleInfo.js";
 export { default as Permission } from "./Permission.js";
 export { default as PlayerStats } from "./PlayerStats.js";
 export { default as Role } from "./Role.js";
+export { default as Note } from "./Note.js";
 export { default as InstanceDetails, InstanceStatus } from "./InstanceDetails.js";
 export { default as HostDetails } from "./HostDetails.js";
 export { default as UserDetails } from "./UserDetails.js";
@@ -24,5 +25,6 @@ export * from "./messages_instance.js";
 export * from "./messages_mod.js";
 export * from "./messages_user.js";
 export * from "./messages_role.js";
+export * from "./messages_note.js";
 export * from "./messages_plugin.js";
 export * from "./version.js";

packages/lib/src/data/messages_controller.ts

@@ -1,9 +1,15 @@
 import { Type, Static } from "@sinclair/typebox";
 import { JsonString, StringEnum, StringKey, plainJson } from "./composites.js";
-import { levels } from "../logging.js";
+import { type LogEntry, levels } from "../logging.js";
 import { ControllerConfig, HostConfig } from "../config/index.js";
 import { ExternalFactorioVersionSchema, LatestReleasesSchema } from "../external/index.js";
 
+const LogEntrySchema = Type.Object({
+	"level": StringKey(levels),
+	"message": Type.String(),
+	"timestamp": Type.String(),
+});
+
 export class ControllerStopRequest {
 	declare ["constructor"]: typeof ControllerStopRequest;
 	static type = "request" as const;
@@ -236,11 +242,11 @@ export class LogQueryRequest {
 
 	static Response = class Response {
 		constructor(
-			public log: object[],
+			public log: LogEntry[],
 		) { }
 
 		static jsonSchema = Type.Object({
-			"log": Type.Array(Type.Object({})),
+			"log": Type.Array(LogEntrySchema),
 		});
 
 		static fromJSON(json: Static<typeof this.jsonSchema>) {
@@ -256,14 +262,11 @@ export class LogMessageEvent {
 	static dst = ["controller", "control"] as const;
 
 	constructor(
-		public info: { level: keyof typeof levels, message: string },
+		public info: LogEntry,
 	) { }
 
 	static jsonSchema = Type.Object({
-		"info": Type.Object({
-			"level": StringKey(levels),
-			"message": Type.String(),
-		}),
+		"info": LogEntrySchema,
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {

packages/lib/src/data/messages_note.ts

@@ -0,0 +1,59 @@
+import { Type, Static } from "@sinclair/typebox";
+import { jsonArray } from "./composites.js";
+import Note from "./Note.js";
+
+export class NoteListRequest {
+	declare ["constructor"]: typeof NoteListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.note.list" as const;
+	static Response = jsonArray(Note);
+}
+
+export class NoteUpdatesEvent {
+	declare ["constructor"]: typeof NoteUpdatesEvent;
+	static type = "event" as const;
+	static src = "controller" as const;
+	static dst = "control" as const;
+	static permission = "core.note.subscribe" as const;
+
+	constructor(
+		public updates: Note[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		"updates": Type.Array(Note.jsonSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.updates.map(update => Note.fromJSON(update)));
+	}
+}
+
+/**
+ * Set the note of a resource. An empty content removes the note.
+ */
+export class NoteSetRequest {
+	declare ["constructor"]: typeof NoteSetRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.note.update" as const;
+
+	constructor(
+		public resourceType: string,
+		public resourceId: string,
+		public content: string,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"resourceType": Type.String(),
+		"resourceId": Type.String(),
+		"content": Type.String(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.resourceType, json.resourceId, json.content);
+	}
+}

packages/lib/src/factorio/exchange_string.ts

@@ -239,7 +239,9 @@ function readPollution(state: MapReaderState) {
 		pollution_with_max_forest_damage: readOptional(state, readDouble),
 		pollution_per_tree_damage: readOptional(state, readDouble),
 		pollution_restored_per_tree_damage: readOptional(state, readDouble),
-		max_pollution_to_restore_trees: readOptional(state, readDouble),
+		...state.versionAtLeast(2, 1, 18) ? {} : {
+			max_pollution_to_restore_trees: readOptional(state, readDouble), // removed v2.1.18
+		},
 		enemy_attack_pollution_consumption_modifier: readOptional(state, readDouble),
 	};
 }

packages/lib/src/link/connectors.ts

@@ -171,10 +171,11 @@ export abstract class WebSocketBaseConnector<
 	_heartbeatId: ReturnType<typeof setInterval> | null = null;
 	_heartbeatInterval: number | null = null;
 	_lastReceivedSeq = undefined;
-	/** Serialised messages sent but not yet acknowledged by the other side */
-	_sendBuffer: { seq: number, text: string }[] = [];
+	/** Serialised messages sent but not yet acknowledged by the other side, with their UTF-8 size in bytes */
+	_sendBuffer: { seq: number, text: string, size: number }[] = [];
+	/** Total UTF-8 size in bytes of the messages in the send buffer */
 	_sendBufferSize = 0;
-	/** Session is ended if the unacknowledged messages exceed this many characters, 0 for no limit */
+	/** Session is ended if the unacknowledged messages exceed this many bytes, 0 for no limit */
 	maxSendBufferSize = 0;
 
 	_reset() {
@@ -225,7 +226,7 @@ export abstract class WebSocketBaseConnector<
 		for (let index = 0; index < this._sendBuffer.length; index++) {
 			if (this._sendBuffer[index].seq <= seq) {
 				dropCount += 1;
-				this._sendBufferSize -= this._sendBuffer[index].text.length;
+				this._sendBufferSize -= this._sendBuffer[index].size;
 			} else {
 				break;
 			}
@@ -334,8 +335,9 @@ export abstract class WebSocketBaseConnector<
 		}
 
 		const text = JSON.stringify(message);
-		this._sendBuffer.push({ seq: message.seq, text });
-		this._sendBufferSize += text.length;
+		const size = Buffer.byteLength(text, "utf8");
+		this._sendBuffer.push({ seq: message.seq, text, size });
+		this._sendBufferSize += size;
 		if (this.maxSendBufferSize && this._sendBufferSize > this.maxSendBufferSize) {
 			this._sendBufferOverflow();
 			return;

packages/lib/src/link/messages.ts

@@ -5,6 +5,7 @@ import * as instance from "../data/messages_instance.js";
 import * as mod from "../data/messages_mod.js";
 import * as user from "../data/messages_user.js";
 import * as role from "../data/messages_role.js";
+import * as note from "../data/messages_note.js";
 import * as plugin from "../data/messages_plugin.js";
 import * as subscriptions from "../subscriptions.js";
 import type { RequestClass, EventClass } from "./link.js";
@@ -126,6 +127,10 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	role.RoleGrantDefaultPermissionsRequest,
 	role.RoleDeleteRequest,
 
+	note.NoteListRequest,
+	note.NoteUpdatesEvent,
+	note.NoteSetRequest,
+
 	user.UserGetRequest,
 	user.UserListRequest,
 	user.UserCreateRequest,

packages/lib/src/logging.ts

@@ -93,6 +93,18 @@ export class WebConsoleFormat {
 	}
 }
 
+/**
+ * Log entry as produced by the logger and sent over links
+ */
+export interface LogEntry {
+	level: keyof typeof levels;
+	message: string;
+	/** ISO 8601 time the entry was logged at. */
+	timestamp: string;
+	/** Other fields like host_id, instance_id, plugin and parsed vary by source. */
+	[key: string]: unknown;
+}
+
 /**
  * Filter object for logs
  */

packages/lib/src/permissions.ts

@@ -462,6 +462,24 @@ export const corePermissions = [
 		description: "Permanently delete roles.",
 	},
 
+	{
+		name: "core.note.list",
+		title: "List notes",
+		description: "Get the full list of notes attached to resources in the cluster.",
+		grantByDefault: true,
+	},
+	{
+		name: "core.note.subscribe",
+		title: "Subscribe to note updates",
+		description: "Subscribe to be notified when notes attached to resources are updated.",
+		grantByDefault: true,
+	},
+	{
+		name: "core.note.update",
+		title: "Update notes",
+		description: "Write, edit and remove notes attached to resources in the cluster.",
+	},
+
 	{
 		name: "core.user.get",
 		title: "Get user",

packages/lib/src/plugin.ts

@@ -14,6 +14,8 @@ export const PluginFeatureFlags = [
 	"SavePatching",
 	/** The plugin requires access to script commands over rcon */
 	"ScriptCommands",
+	/** The plugin requires UDP messaging with the Factorio server */
+	"LuaUdp",
 ] as const;
 
 /** Used to define the plugin export in plugins */

packages/lib/src/subscriptions.ts

@@ -598,7 +598,7 @@ export class EventSubscriber<E, S = null> {
 			// By delaying the unsubscription here the subscription happens
 			// before the unsubscription, thus preventing the redundant
 			// updates from being sent out.
-			setImmediate(() => {
+			setTimeout(() => {
 				let index = this._callbacks.lastIndexOf(handler);
 				if (index === -1) {
 					return;

packages/lib/src/zip_ops.ts

@@ -1,4 +1,122 @@
-import type JSZip from "jszip";
+/**
+ * Zip file helpers
+ * @module lib/zip_ops
+ */
+import events from "events";
+import { PassThrough, type Readable } from "stream";
+import util from "util";
+import zlib from "zlib";
+import yauzl from "yauzl";
+import type yazl from "yazl";
+
+const deflateRaw = util.promisify(zlib.deflateRaw);
+
+
+/**
+ * Zip archive opened for random access reading
+ *
+ * Only the central directory is read up front, the content of entries is
+ * streamed from the archive on demand.  Archives opened from a file hold
+ * the file open until close() is called.
+ */
+export class ZipArchive {
+	private constructor(
+		/** Underlying yauzl zip file. */
+		public zipfile: yauzl.ZipFile,
+		/** Entries by file name in the order they appear in the central directory. */
+		public entries: Map<string, yauzl.Entry>,
+	) { }
+
+	/**
+	 * Open a zip archive from the file system
+	 *
+	 * @param zipPath - Path to the zip file to open.
+	 */
+	static async fromFile(zipPath: string) {
+		return this.load(await yauzl.openPromise(zipPath, { autoClose: false }));
+	}
+
+	/**
+	 * Open a zip archive held in memory
+	 *
+	 * @param buffer - Content of the zip file.
+	 */
+	static async fromBuffer(buffer: Buffer) {
+		return this.load(await yauzl.fromBufferPromise(buffer));
+	}
+
+	private static async load(zipfile: yauzl.ZipFile) {
+		const entries = new Map<string, yauzl.Entry>();
+		try {
+			for await (const entry of zipfile.eachEntry()) {
+				entries.set(entry.fileName, entry);
+			}
+		} catch (err) {
+			zipfile.close();
+			throw err;
+		}
+		return new this(zipfile, entries);
+	}
+
+	/**
+	 * Look up a file in the archive
+	 *
+	 * @param filePath - Path of the file in the archive.
+	 * @returns entry for the file, or null if it does not exist.
+	 */
+	file(filePath: string) {
+		const entry = this.entries.get(filePath);
+		if (!entry || entry.fileName.endsWith("/")) {
+			return null;
+		}
+		return entry;
+	}
+
+	/**
+	 * Open a stream of the decompressed content of an entry
+	 *
+	 * @param entry - Entry to read, must belong to this archive.
+	 */
+	openStream(entry: yauzl.Entry): Promise<Readable> {
+		return this.zipfile.openReadStreamPromise(entry);
+	}
+
+	/**
+	 * Read the decompressed content of an entry
+	 *
+	 * @param entry - Entry to read, must belong to this archive.
+	 */
+	async readEntry(entry: yauzl.Entry) {
+		const chunks: Buffer[] = [];
+		for await (const chunk of await this.openStream(entry)) {
+			chunks.push(chunk);
+		}
+		return Buffer.concat(chunks);
+	}
+
+	/**
+	 * Read the content of a file in the archive
+	 *
+	 * @param filePath - Path of the file in the archive.
+	 * @returns content of the file, or null if it does not exist.
+	 */
+	async readFile(filePath: string) {
+		const entry = this.file(filePath);
+		if (!entry) {
+			return null;
+		}
+		return await this.readEntry(entry);
+	}
+
+	/**
+	 * Close the archive
+	 *
+	 * Streams already opened are still readable until they end.
+	 */
+	close() {
+		this.zipfile.close();
+	}
+}
 
 /**
  * Returns the root folder in the zip file
@@ -12,8 +130,8 @@ import type JSZip from "jszip";
  * @param zip - Zip to search through.
  * @returns name of the root folder.
  */
-export function findRoot(zip: JSZip) {
-	const relativePath = Object.keys(zip.files)[0];
+export function findRoot(zip: ZipArchive) {
+	const relativePath = zip.entries.keys().next().value;
 	if (relativePath === undefined) {
 		throw new Error("Empty zip file");
 	}
@@ -25,3 +143,263 @@ export function findRoot(zip: JSZip) {
 
 	return relativePath.slice(0, index);
 }
+
+/**
+ * Returns the output stream of a zip file being written
+ *
+ * yazl reports errors on the ZipFile object and leaves the output stream
+ * hanging.  The stream returned here is destroyed with the error instead so
+ * that consumers like stream.pipeline see it.
+ *
+ * @param zipFile - zip file being written.
+ */
+export function zipOutputStream(zipFile: yazl.ZipFile) {
+	const output = zipFile.outputStream as Readable;
+	zipFile.on("error", err => output.destroy(err));
+	return output;
+}
+
+const zip64Limit = 0xffffffff;
+
+function dosDateTime(date: Date) {
+	if (date.getFullYear() < 1980) {
+		return { date: 0x21, time: 0 };
+	}
+	const year = Math.min(date.getFullYear(), 2107);
+	return {
+		date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
+		time: (date.getHours() << 11) | (date.getMinutes() << 5) | (date.getSeconds() >> 1),
+	};
+}
+
+interface WriterEntry {
+	fileName: Buffer;
+	fileComment: Buffer;
+	versionMadeBy: number;
+	generalPurposeBitFlag: number;
+	compressionMethod: number;
+	lastModFileTime: number;
+	lastModFileDate: number;
+	crc32: number;
+	compressedSize: number;
+	uncompressedSize: number;
+	externalFileAttributes: number;
+	offset: number;
+}
+
+export interface ZipWriterOptions {
+	/** Use ZIP64 structures even when the archive does not need them. */
+	forceZip64?: boolean;
+}
+
+export interface ZipWriterBufferOptions {
+	/** Modification time to record for the file, defaults to now. */
+	mtime?: Date;
+}
+
+/**
+ * Streaming zip file writer
+ *
+ * Writes entries to outputStream as they are added.  Entries from an
+ * existing archive are copied without decompressing them.  Sizes and
+ * offsets past 4 GiB and more than 65535 entries use ZIP64 structures.
+ */
+export class ZipWriter {
+	/** Content of the zip file, ends when end() has been called. */
+	outputStream = new PassThrough();
+	private offset = 0;
+	private entries: WriterEntry[] = [];
+	private forceZip64: boolean;
+
+	constructor(options: ZipWriterOptions = {}) {
+		this.forceZip64 = options.forceZip64 ?? false;
+	}
+
+	/**
+	 * Copy an entry from an archive
+	 *
+	 * The compressed content is copied as is along with the metadata
+	 * Factorio writes.  Extra fields are dropped.
+	 *
+	 * @param zip - Archive to copy from.
+	 * @param entry - Entry to copy, must belong to zip.
+	 */
+	async addEntry(zip: ZipArchive, entry: yauzl.Entry) {
+		const writerEntry: WriterEntry = {
+			fileName: Buffer.from(entry.fileName, "utf8"),
+			fileComment: Buffer.from(entry.fileComment, "utf8"),
+			versionMadeBy: entry.versionMadeBy,
+			// Sizes are written in the local header so no data descriptor is
+			// needed, and names are re-encoded as UTF-8.
+			generalPurposeBitFlag: (entry.generalPurposeBitFlag & ~0x8) | 0x800,
+			compressionMethod: entry.compressionMethod,
+			lastModFileTime: entry.lastModFileTime,
+			lastModFileDate: entry.lastModFileDate,
+			crc32: entry.crc32,
+			compressedSize: entry.compressedSize,
+			uncompressedSize: entry.uncompressedSize,
+			externalFileAttributes: entry.externalFileAttributes,
+			offset: this.offset,
+		};
+		await this.write(this.localFileHeader(writerEntry));
+		const stream = await zip.zipfile.openReadStreamPromise(entry, { decodeFileData: false });
+		for await (const chunk of stream) {
+			await this.write(chunk);
+		}
+		this.entries.push(writerEntry);
+	}
+
+	/**
+	 * Add a file with the given content
+	 *
+	 * @param buffer - Content of the file.
+	 * @param fileName - Path of the file in the archive.
+	 * @param options - Options for the entry.
+	 */
+	async addBuffer(buffer: Buffer, fileName: string, options: ZipWriterBufferOptions = {}) {
+		const deflated = await deflateRaw(buffer);
+		const { date, time } = dosDateTime(options.mtime ?? new Date());
+		const writerEntry: WriterEntry = {
+			fileName: Buffer.from(fileName, "utf8"),
+			fileComment: Buffer.alloc(0),
+			versionMadeBy: 20,
+			generalPurposeBitFlag: 0x800,
+			compressionMethod: 8,
+			lastModFileTime: time,
+			lastModFileDate: date,
+			crc32: zlib.crc32(buffer),
+			compressedSize: deflated.length,
+			uncompressedSize: buffer.length,
+			externalFileAttributes: 0,
+			offset: this.offset,
+		};
+		await this.write(this.localFileHeader(writerEntry));
+		await this.write(deflated);
+		this.entries.push(writerEntry);
+	}
+
+	/**
+	 * Write the central directory and end the output stream
+	 */
+	async end() {
+		const start = this.offset;
+		for (const entry of this.entries) {
+			await this.write(this.centralDirectoryHeader(entry));
+		}
+		const size = this.offset - start;
+		const zip64 = this.forceZip64 || this.entries.length >= 0xffff || start >= zip64Limit || size >= zip64Limit;
+		if (zip64) {
+			const record = Buffer.alloc(56);
+			record.writeUInt32LE(0x06064b50, 0);
+			record.writeBigUInt64LE(44n, 4);
+			record.writeUInt16LE(45, 12); // version made by
+			record.writeUInt16LE(45, 14); // version needed to extract
+			record.writeUInt32LE(0, 16); // number of this disk
+			record.writeUInt32LE(0, 20); // disk with the central directory
+			record.writeBigUInt64LE(BigInt(this.entries.length), 24);
+			record.writeBigUInt64LE(BigInt(this.entries.length), 32);
+			record.writeBigUInt64LE(BigInt(size), 40);
+			record.writeBigUInt64LE(BigInt(start), 48);
+			const locator = Buffer.alloc(20);
+			locator.writeUInt32LE(0x07064b50, 0);
+			locator.writeUInt32LE(0, 4); // disk with the zip64 end of central directory
+			locator.writeBigUInt64LE(BigInt(this.offset), 8);
+			locator.writeUInt32LE(1, 16); // total number of disks
+			await this.write(record);
+			await this.write(locator);
+		}
+		const eocd = Buffer.alloc(22);
+		eocd.writeUInt32LE(0x06054b50, 0);
+		eocd.writeUInt16LE(0, 4); // number of this disk
+		eocd.writeUInt16LE(0, 6); // disk with the central directory
+		eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 8);
+		eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 10);
+		eocd.writeUInt32LE(zip64 ? zip64Limit : size, 12);
+		eocd.writeUInt32LE(zip64 ? zip64Limit : start, 16);
+		eocd.writeUInt16LE(0, 20); // comment length
+		await this.write(eocd);
+		this.outputStream.end();
+	}
+
+	private localFileHeader(entry: WriterEntry) {
+		const zip64 = this.forceZip64
+			|| entry.compressedSize >= zip64Limit
+			|| entry.uncompressedSize >= zip64Limit;
+		const extra = zip64 ? Buffer.alloc(20) : Buffer.alloc(0);
+		if (zip64) {
+			extra.writeUInt16LE(0x0001, 0);
+			extra.writeUInt16LE(16, 2);
+			extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4);
+			extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12);
+		}
+		const header = Buffer.alloc(30);
+		header.writeUInt32LE(0x04034b50, 0);
+		header.writeUInt16LE(zip64 ? 45 : 20, 4); // version needed to extract
+		header.writeUInt16LE(entry.generalPurposeBitFlag, 6);
+		header.writeUInt16LE(entry.compressionMethod, 8);
+		header.writeUInt16LE(entry.lastModFileTime, 10);
+		header.writeUInt16LE(entry.lastModFileDate, 12);
+		header.writeUInt32LE(entry.crc32, 14);
+		header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 18);
+		header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 22);
+		header.writeUInt16LE(entry.fileName.length, 26);
+		header.writeUInt16LE(extra.length, 28);
+		return Buffer.concat([header, entry.fileName, extra]);
+	}
+
+	private centralDirectoryHeader(entry: WriterEntry) {
+		const zip64 = this.forceZip64
+			|| entry.compressedSize >= zip64Limit
+			|| entry.uncompressedSize >= zip64Limit
+			|| entry.offset >= zip64Limit;
+		const extra = zip64 ? Buffer.alloc(28) : Buffer.alloc(0);
+		if (zip64) {
+			extra.writeUInt16LE(0x0001, 0);
+			extra.writeUInt16LE(24, 2);
+			extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4);
+			extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12);
+			extra.writeBigUInt64LE(BigInt(entry.offset), 20);
+		}
+		const header = Buffer.alloc(46);
+		header.writeUInt32LE(0x02014b50, 0);
+		header.writeUInt16LE(entry.versionMadeBy, 4);
+		header.writeUInt16LE(zip64 ? 45 : 20, 6); // version needed to extract
+		header.writeUInt16LE(entry.generalPurposeBitFlag, 8);
+		header.writeUInt16LE(entry.compressionMethod, 10);
+		header.writeUInt16LE(entry.lastModFileTime, 12);
+		header.writeUInt16LE(entry.lastModFileDate, 14);
+		header.writeUInt32LE(entry.crc32, 16);
+		header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 20);
+		header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 24);
+		header.writeUInt16LE(entry.fileName.length, 28);
+		header.writeUInt16LE(extra.length, 30);
+		header.writeUInt16LE(entry.fileComment.length, 32);
+		header.writeUInt16LE(0, 34); // disk number start
+		header.writeUInt16LE(0, 36); // internal file attributes
+		header.writeUInt32LE(entry.externalFileAttributes, 38);
+		header.writeUInt32LE(zip64 ? zip64Limit : entry.offset, 42);
+		return Buffer.concat([header, entry.fileName, extra, entry.fileComment]);
+	}
+
+	private async write(buffer: Buffer) {
+		if (this.outputStream.destroyed) {
+			throw this.outputStream.errored ?? new Error("Zip output stream was closed");
+		}
+		if (!this.outputStream.write(buffer)) {
+			// once() rejects if the stream errors, close covers a plain destroy.
+			const controller = new AbortController();
+			const { signal } = controller;
+			try {
+				await Promise.race([
+					events.once(this.outputStream, "drain", { signal }),
+					events.once(this.outputStream, "close", { signal }).then(() => {
+						throw this.outputStream.errored ?? new Error("Zip output stream was closed");
+					}),
+				]);
+			} finally {
+				controller.abort();
+			}
+		}
+		this.offset += buffer.length;
+	}
+}

packages/web_ui/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/web_ui",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Clusterio web interface implementation",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser",
@@ -62,7 +62,7 @@
 		"util": "^0.12.5",
 		"webpack": "catalog:",
 		"webpack-cli": "catalog:",
-		"webpack-manifest-plugin": "^5.0.0"
+		"webpack-manifest-plugin": "^6.0.1"
 	},
 	"devDependencies": {
 		"@types/node": "catalog:",

packages/web_ui/src/components/HostViewPage.tsx

@@ -16,6 +16,7 @@ import { useHost } from "../model/host";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import ResourceNotes from "./ResourceNotes";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
 	MetricDiskUsed, MetricDiskRatio, MetricRelativeDate,
@@ -182,6 +183,7 @@ export default function HostViewPage() {
 					: null
 			}
 		</Descriptions>
+		<ResourceNotes resourceType="host" resourceId={hostId} />
 		{account.hasPermission("core.instance.list") && <>
 			<Title level={5} style={{ marginTop: 16 }}>Instances</Title>
 			<InstanceList instances={hostInstances} size="small" hideAssignedHost />

packages/web_ui/src/components/InstanceViewPage.tsx

@@ -15,6 +15,7 @@ import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import ResourceNotes from "./ResourceNotes";
 import InstanceConfigTree from "./InstanceConfigTree";
 import LogConsole, { SelectMaxLogLevel } from "./LogConsole";
 import InstanceRcon from "./InstanceRcon";
@@ -236,6 +237,7 @@ export default function InstanceViewPage() {
 			extra={<InstanceButtons instance={instance} />}
 		/>
 		<InstanceDescription host={host} instance={instance} />
+		<ResourceNotes resourceType="instance" resourceId={instanceId} />
 
 		{account.hasPermission("core.user.list") && <OnlinePlayersList instanceId={instanceId} />}
 

packages/web_ui/src/components/LogConsole.tsx

@@ -54,16 +54,28 @@ function formatParsedOutput(parsed: Parsed) {
 	return <>{time}{info}{parsed.message}</>;
 }
 
-type Info = {
-	level: keyof typeof lib.levels;
-	message: string;
+type Info = lib.LogEntry & {
 	parsed?: Parsed;
 };
 
+/**
+ * Format an ISO 8601 timestamp as local YYYY-MM-DD HH:MM:SS
+ */
+function formatTime(timestamp: string) {
+	const date = new Date(timestamp);
+	if (Number.isNaN(date.getTime())) {
+		return timestamp;
+	}
+	const pad = (value: number) => String(value).padStart(2, "0");
+	return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ` +
+		`${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
+}
+
 function formatLog(info: Info, key: number): ReactElement {
+	let time = <span className="log-time">{formatTime(info.timestamp)}</span>;
 	let level = <span className={`log-${info.level}`}>{info.level}</span>;
 	let message = info.level === "server" && info.parsed ? formatParsedOutput(info.parsed) : info.message;
-	return <span key={key}>[{level}] {message}<br/></span>;
+	return <span key={key}>{time} [{level}] {message}<br/></span>;
 }
 
 /**
@@ -186,10 +198,10 @@ export default function LogConsole(props: LogConsoleProps) {
 			setPastLines([]);
 		}
 
-		function logHandler(info: Info) {
-			if (!props.actionsOnly || passesActionsOnly(info)) {
+		function logHandler(info: lib.LogEntry) {
+			if (!props.actionsOnly || passesActionsOnly(info as Info)) {
 				setLines(currentLines => currentLines.concat(
-					[formatLog(info, currentLines.length)]
+					[formatLog(info as Info, currentLines.length)]
 				));
 			}
 		}

packages/web_ui/src/components/ModDetails.tsx

@@ -6,6 +6,7 @@ import FileExclamationOutlined from "@ant-design/icons/FileExclamationOutlined";
 import FileSyncOutlined from "@ant-design/icons/FileSyncOutlined";
 
 import * as lib from "@clusterio/lib";
+import ResourceNotes from "./ResourceNotes";
 
 const { useBreakpoint } = Grid;
 
@@ -58,6 +59,9 @@ export default function ModDetails<T extends lib.ModInfo | lib.ModRecord>(props:
 			&& <Descriptions.Item label="Description" span={2}>{mod.description}</Descriptions.Item>
 		}
 		{Boolean(mod.author) && <Descriptions.Item label="Author" span={2}>{mod.author}</Descriptions.Item>}
+		<Descriptions.Item label="Notes" span={2}>
+			<ResourceNotes resourceType="mod" resourceId={`${mod.name}_${mod.version}`} compact />
+		</Descriptions.Item>
 
 		<Descriptions.Item label="Mod&nbsp;Portal" span={2}>
 			<Typography.Link href={`https://mods.factorio.com/mod/${mod.name}`}>

packages/web_ui/src/components/ModPackViewPage.tsx

@@ -26,6 +26,7 @@ import notify, { notifyErrorHandler } from "../util/notify";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import SectionHeader from "./SectionHeader";
+import ResourceNotes from "./ResourceNotes";
 import ModDetails from "./ModDetails";
 import { InputPartialVersion } from "./InputVersion";
 import Link from "./Link";
@@ -1399,5 +1400,6 @@ export default function ModPackViewPage() {
 				</Col>
 			</Row>
 		</div>
+		<ResourceNotes resourceType="mod_pack" resourceId={modPackId} />
 	</PageLayout>;
 }

packages/web_ui/src/components/ResourceMention.tsx

@@ -0,0 +1,71 @@
+import React from "react";
+
+import Link from "./Link";
+import { useHost } from "../model/host";
+import { useInstance } from "../model/instance";
+import { useModPack } from "../model/mod_pack";
+import { useRole } from "../model/roles";
+import { useUser } from "../model/user";
+
+/** Matches @type/id references to resources that have a view page. */
+export const mentionPattern = /(?<![\w@])@(?:(host|instance|role|mod_pack)\/(\d+)|(user)\/(#?[\w.-]*[\w-]))/g;
+
+export function mentionPath(resourceType: string, resourceId: string) {
+	switch (resourceType) {
+		case "host": return `/hosts/${resourceId}/view`;
+		case "instance": return `/instances/${resourceId}/view`;
+		case "role": return `/roles/${resourceId}/view`;
+		case "mod_pack": return `/mods/mod-packs/${resourceId}/view`;
+		case "user": return `/users/${encodeURIComponent(resourceId)}/view`;
+		default: return undefined;
+	}
+}
+
+type ResourceMentionProps = {
+	resourceType: string;
+	resourceId: string;
+};
+
+/**
+ * Link to a resource showing its name, or its id if the name is not
+ * available to this client.
+ */
+export default function ResourceMention(props: ResourceMentionProps) {
+	switch (props.resourceType) {
+		case "host": return <HostMention id={props.resourceId} />;
+		case "instance": return <InstanceMention id={props.resourceId} />;
+		case "role": return <RoleMention id={props.resourceId} />;
+		case "mod_pack": return <ModPackMention id={props.resourceId} />;
+		case "user": return <UserMention id={props.resourceId} />;
+		default: return <>@{props.resourceType}/{props.resourceId}</>;
+	}
+}
+
+function MentionLink(props: { resourceType: string, id: string, name?: string }) {
+	return <Link to={mentionPath(props.resourceType, props.id)!}>{props.name ?? props.id}</Link>;
+}
+
+function HostMention(props: { id: string }) {
+	const [host] = useHost(Number(props.id));
+	return <MentionLink resourceType="host" id={props.id} name={host?.name} />;
+}
+
+function InstanceMention(props: { id: string }) {
+	const [instance] = useInstance(Number(props.id));
+	return <MentionLink resourceType="instance" id={props.id} name={instance?.name} />;
+}
+
+function RoleMention(props: { id: string }) {
+	const [role] = useRole(Number(props.id));
+	return <MentionLink resourceType="role" id={props.id} name={role?.name} />;
+}
+
+function ModPackMention(props: { id: string }) {
+	const [modPack] = useModPack(Number(props.id));
+	return <MentionLink resourceType="mod_pack" id={props.id} name={modPack?.name} />;
+}
+
+function UserMention(props: { id: string }) {
+	const [user] = useUser(props.id);
+	return <MentionLink resourceType="user" id={props.id} name={user?.name} />;
+}

packages/web_ui/src/components/ResourceNotes.tsx

@@ -0,0 +1,183 @@
+import React, { useContext, useEffect, useState } from "react";
+import { Button, Input, Space, Typography } from "antd";
+import ReactMarkdown, { Components } from "react-markdown";
+
+import * as lib from "@clusterio/lib";
+import { notifyErrorHandler } from "../util/notify";
+import { useAccount } from "../model/account";
+import { useNote } from "../model/notes";
+import { formatTimestamp } from "../util/time_format";
+import ControlContext from "./ControlContext";
+import ResourceMention, { mentionPath, mentionPattern } from "./ResourceMention";
+import SectionHeader from "./SectionHeader";
+
+const { Text } = Typography;
+
+/**
+ * Remark plugin turning @type/id mentions into links and line breaks into
+ * hard breaks. Text inside code and existing links is left alone.
+ */
+function remarkNoteText() {
+	function split(value: string) {
+		const nodes: any[] = [];
+		let last = 0;
+		function pushText(text: string) {
+			text.split("\n").forEach((line, index) => {
+				if (index > 0) {
+					nodes.push({ type: "break" });
+				}
+				if (line) {
+					nodes.push({ type: "text", value: line });
+				}
+			});
+		}
+		for (const match of value.matchAll(mentionPattern)) {
+			pushText(value.slice(last, match.index));
+			const resourceType = match[1] ?? match[3];
+			const resourceId = match[2] ?? match[4];
+			nodes.push({
+				type: "link",
+				url: mentionPath(resourceType, resourceId),
+				children: [{ type: "text", value: match[0] }],
+				data: { hProperties: { dataResourceType: resourceType, dataResourceId: resourceId } },
+			});
+			last = match.index + match[0].length;
+		}
+		pushText(value.slice(last));
+		return nodes;
+	}
+	function visit(node: any) {
+		if (!node.children || node.type === "link" || node.type === "linkReference") {
+			return;
+		}
+		node.children = node.children.flatMap((child: any) => {
+			if (child.type === "text") {
+				return split(child.value);
+			}
+			visit(child);
+			return [child];
+		});
+	}
+	return visit;
+}
+
+const noteComponents: Components = {
+	a({ node, ...props }) {
+		const resourceType = node?.properties.dataResourceType;
+		const resourceId = node?.properties.dataResourceId;
+		if (typeof resourceType === "string" && typeof resourceId === "string") {
+			return <ResourceMention resourceType={resourceType} resourceId={resourceId} />;
+		}
+		return <a {...props} target="_blank" rel="noopener noreferrer" />;
+	},
+};
+
+type ResourceNotesProps = {
+	/** Type of resource, such as "host" or "instance". */
+	resourceType: string;
+	/** Id of the resource within its type. */
+	resourceId: string | number;
+	/** Render without the section header, for use inside tables and modals. */
+	compact?: boolean;
+};
+
+/**
+ * Shows the note attached to a resource with an editor for users that
+ * have the core.note.update permission.
+ */
+export default function ResourceNotes(props: ResourceNotesProps) {
+	// Remount on resource change so a draft is never saved to another resource
+	return <ResourceNotesEditor key={lib.Note.idFor(props.resourceType, props.resourceId)} {...props} />;
+}
+
+function ResourceNotesEditor(props: ResourceNotesProps) {
+	const control = useContext(ControlContext);
+	const account = useAccount();
+	const [note, synced] = useNote(props.resourceType, props.resourceId);
+	const [editing, setEditing] = useState(false);
+	const [draft, setDraft] = useState("");
+	const [saving, setSaving] = useState(false);
+	const canUpdate = account.hasPermission("core.note.update");
+	const canView = account.hasPermission("core.note.subscribe");
+
+	// Pick up changes made by others while not editing
+	const content = note?.content ?? "";
+	useEffect(() => {
+		if (!editing) {
+			setDraft(content);
+		}
+	}, [content, editing]);
+
+	if (!canView && !canUpdate) {
+		return null;
+	}
+
+	function save() {
+		setSaving(true);
+		control.send(new lib.NoteSetRequest(props.resourceType, String(props.resourceId), draft))
+			.then(() => setEditing(false))
+			.catch(notifyErrorHandler("Error saving note"))
+			.finally(() => setSaving(false));
+	}
+
+	function cancel() {
+		setDraft(content);
+		setEditing(false);
+	}
+
+	const buttons = editing
+		? <Space>
+			<Button size="small" onClick={cancel} disabled={saving}>Cancel</Button>
+			<Button size="small" type="primary" onClick={save} loading={saving}>Save</Button>
+		</Space>
+		: <Button size="small" onClick={() => setEditing(true)} disabled={!synced}>
+			{note ? "Edit" : "Add note"}
+		</Button>;
+
+	const body = editing
+		? <Input.TextArea
+			autoSize={{ minRows: 3 }}
+			value={draft}
+			onChange={e => setDraft(e.target.value)}
+			onKeyDown={e => {
+				if (e.key === "Escape") {
+					cancel();
+				} else if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) {
+					save();
+				}
+			}}
+			placeholder={
+				"Write a note visible to everyone with access to this page. " +
+				"Supports Markdown and links like @instance/<id> or @user/<name>"
+			}
+			autoFocus
+		/>
+		: <>
+			{note
+				? <div style={{ overflowWrap: "anywhere" }}>
+					<ReactMarkdown remarkPlugins={[remarkNoteText]} components={noteComponents}>
+						{note.content}
+					</ReactMarkdown>
+				</div>
+				: <Text type="secondary">{synced ? "No note" : "Loading"}</Text>
+			}
+			{note && (note.updatedBy || note.updatedAtMs) && <div>
+				<Text type="secondary" style={{ fontSize: "0.85em" }}>
+					Last edited{note.updatedBy ? ` by ${note.updatedBy}` : ""}
+					{note.updatedAtMs ? ` on ${formatTimestamp(note.updatedAtMs)}` : ""}
+				</Text>
+			</div>}
+		</>;
+
+	if (props.compact) {
+		return <Space direction="vertical" style={{ width: "100%" }}>
+			{body}
+			{canUpdate && buttons}
+		</Space>;
+	}
+
+	return <>
+		<SectionHeader title="Notes" extra={canUpdate ? buttons : undefined} />
+		{body}
+	</>;
+}

packages/web_ui/src/components/RoleViewPage.tsx

@@ -11,6 +11,7 @@ import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import ResourceNotes from "./ResourceNotes";
 import { notifyErrorHandler } from "../util/notify";
 
 
@@ -327,6 +328,8 @@ export default function RoleViewPage() {
 				<Input value={description} disabled={!canUpdate} onChange={e => setDescription(e.target.value)} />
 			</div>
 
+			<ResourceNotes resourceType="role" resourceId={roleId} />
+
 			<PluginExtra component="RoleViewPageBeforePermissions" role={role} />
 
 			<h3>Permissions</h3>

packages/web_ui/src/components/SavesList.tsx

@@ -13,6 +13,7 @@ import useTableQueryState from "../util/useTableQueryState";
 import ControlContext from "./ControlContext";
 import CreateSaveModal from "./CreateSaveModal";
 import SectionHeader from "./SectionHeader";
+import ResourceNotes from "./ResourceNotes";
 import { useInstances } from "../model/instance";
 import { useSavesOfInstance } from "../model/saves";
 import { notifyErrorHandler } from "../util/notify";
@@ -249,58 +250,61 @@ export default function SavesList(props: { instance: lib.InstanceDetails }) {
 		expandable={{
 			columnWidth: 33,
 			expandRowByClick: true,
-			expandedRowRender: save => <Space wrap style={{ marginBottom: 0 }}>
-				{account.hasPermission("core.instance.start") && <Button
-					loading={starting}
-					disabled={props.instance.status !== "stopped"}
-					onClick={() => {
-						setStarting(true);
-						control.sendTo(
-							{ instanceId: props.instance.id! },
-							new lib.InstanceStartRequest(save.name),
-						).catch(
-							notifyErrorHandler("Error loading save")
-						).finally(
-							() => { setStarting(false); }
-						);
-					}}
-				>Load save</Button>}
-				{account.hasPermission("core.instance.save.rename") && <RenameModal
-					disabled={hostOffline} instanceId={props.instance.id!} save={save}
-				/>}
-				{account.hasPermission("core.instance.save.copy") && <CopyModal
-					disabled={hostOffline} instanceId={props.instance.id!} save={save}
-				/>}
-				{account.hasPermission("core.instance.save.download") && <Button
-					disabled={hostOffline}
-					onClick={() => {
-						control.send(
-							new lib.InstanceDownloadSaveRequest(props.instance.id!, save.name)
-						).then(streamId => {
-							let url = new URL(webRoot, document.location.href);
-							url.pathname += `api/stream/${streamId}`;
-							document.location.assign(url);
-						}).catch(
-							notifyErrorHandler("Error downloading save")
-						);
-					}}
-				>Download</Button>}
-				{account.hasPermission("core.instance.save.transfer") && <TransferModal
-					disabled={hostOffline} instanceId={props.instance.id!} save={save}
-				/>}
-				{account.hasPermission("core.instance.save.delete") && <Popconfirm
-					title="Permanently delete save?"
-					okText="Delete"
-					placement="top"
-					okButtonProps={{ danger: true }}
-					onConfirm={() => {
-						control.send(
-							new lib.InstanceDeleteSaveRequest(props.instance.id!, save.name)
-						).catch(notifyErrorHandler("Error deleting save"));
-					}}
-				>
-					<Button danger disabled={hostOffline}>Delete</Button>
-				</Popconfirm>}
+			expandedRowRender: save => <Space direction="vertical" style={{ width: "100%" }}>
+				<Space wrap>
+					{account.hasPermission("core.instance.start") && <Button
+						loading={starting}
+						disabled={props.instance.status !== "stopped"}
+						onClick={() => {
+							setStarting(true);
+							control.sendTo(
+								{ instanceId: props.instance.id! },
+								new lib.InstanceStartRequest(save.name),
+							).catch(
+								notifyErrorHandler("Error loading save")
+							).finally(
+								() => { setStarting(false); }
+							);
+						}}
+					>Load save</Button>}
+					{account.hasPermission("core.instance.save.rename") && <RenameModal
+						disabled={hostOffline} instanceId={props.instance.id!} save={save}
+					/>}
+					{account.hasPermission("core.instance.save.copy") && <CopyModal
+						disabled={hostOffline} instanceId={props.instance.id!} save={save}
+					/>}
+					{account.hasPermission("core.instance.save.download") && <Button
+						disabled={hostOffline}
+						onClick={() => {
+							control.send(
+								new lib.InstanceDownloadSaveRequest(props.instance.id!, save.name)
+							).then(streamId => {
+								let url = new URL(webRoot, document.location.href);
+								url.pathname += `api/stream/${streamId}`;
+								document.location.assign(url);
+							}).catch(
+								notifyErrorHandler("Error downloading save")
+							);
+						}}
+					>Download</Button>}
+					{account.hasPermission("core.instance.save.transfer") && <TransferModal
+						disabled={hostOffline} instanceId={props.instance.id!} save={save}
+					/>}
+					{account.hasPermission("core.instance.save.delete") && <Popconfirm
+						title="Permanently delete save?"
+						okText="Delete"
+						placement="top"
+						okButtonProps={{ danger: true }}
+						onConfirm={() => {
+							control.send(
+								new lib.InstanceDeleteSaveRequest(props.instance.id!, save.name)
+							).catch(notifyErrorHandler("Error deleting save"));
+						}}
+					>
+						<Button danger disabled={hostOffline}>Delete</Button>
+					</Popconfirm>}
+				</Space>
+				<ResourceNotes resourceType="save" resourceId={save.id} compact />
 			</Space>,
 		}}
 	/>;

packages/web_ui/src/components/UserViewPage.tsx

@@ -17,6 +17,7 @@ import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
 import SectionHeader from "./SectionHeader";
+import ResourceNotes from "./ResourceNotes";
 import useRowNavigation from "../util/useRowNavigation";
 import Link from "./Link";
 import notify, { notifyErrorHandler } from "../util/notify";
@@ -290,6 +291,7 @@ export default function UserViewPage() {
 					&& <Form.Item label="Ban reason">{user.banReason}</Form.Item>
 			}
 		</Form>
+		<ResourceNotes resourceType="user" resourceId={user.id} />
 		<SectionHeader title="Player stats" />
 		<Descriptions size="small" bordered column={{ xs: 1, sm: 2, md: 2, lg: 2, xl: 2, xxl: 2 }}>
 			<Descriptions.Item label="Total online time">

packages/web_ui/src/index.ts

@@ -12,6 +12,7 @@ export * from "./model/instance";
 export * from "./model/is_dragging";
 export * from "./model/mod_pack";
 export * from "./model/mods";
+export * from "./model/notes";
 export * from "./model/roles";
 export * from "./model/saves";
 export * from "./model/system";
@@ -38,6 +39,7 @@ export { default as PageHeader } from "./components/PageHeader";
 export { default as PageLayout } from "./components/PageLayout";
 export { default as SavesList } from "./components/SavesList";
 export { default as SectionHeader } from "./components/SectionHeader";
+export { default as ResourceNotes } from "./components/ResourceNotes";
 
 export { default as Link } from "./components/Link";
 

packages/web_ui/src/model/notes.ts

@@ -0,0 +1,16 @@
+import { useCallback, useContext, useSyncExternalStore } from "react";
+import ControlContext from "../components/ControlContext";
+
+import * as lib from "@clusterio/lib";
+
+export function useNote(resourceType: string, resourceId?: string | number) {
+	const [notes, synced] = useNotes();
+	const note = resourceId !== undefined ? notes.get(lib.Note.idFor(resourceType, resourceId)) : undefined;
+	return [note, synced] as const;
+}
+
+export function useNotes() {
+	const control = useContext(ControlContext);
+	const subscribe = useCallback((callback: () => void) => control.notes.subscribe(callback), [control]);
+	return useSyncExternalStore(subscribe, () => control.notes.getSnapshot());
+}

packages/web_ui/src/style.css

@@ -195,6 +195,10 @@ body {
 	white-space: pre-wrap;
 }
 
+.instance-console .log-time {
+	color: #808080;
+}
+
 .instance-console .factorio-time {
 	color: #afaf1f;
 }

packages/web_ui/src/util/websocket.ts

@@ -11,7 +11,7 @@ type saveListHandler = (saveListEvent: lib.SaveDetails) => void;
 type modPackHandler = (modPack: lib.ModPack) => void;
 type modInfoHandler = (modInfo: lib.ModInfo) => void;
 type userHandler = (rawUser: lib.UserDetails) => void;
-type logHandler = (info: { level: keyof typeof lib.levels, message: string }) => void;
+type logHandler = (info: lib.LogEntry) => void;
 
 /**
  * Connector for control connection to controller
@@ -73,6 +73,7 @@ export class Control extends lib.Link {
 	mods = new lib.MapSubscriber(lib.ModUpdatesEvent, this);
 	users = new lib.MapSubscriber(lib.UserUpdatesEvent, this);
 	roles = new lib.MapSubscriber(lib.RoleUpdatesEvent, this);
+	notes = new lib.MapSubscriber(lib.NoteUpdatesEvent, this);
 
 	/** Cache for factorio versions to avoid repeat calls to the controller */
 	factorioVersions = new lib.ValueCache(this.requestFactorioVersions.bind(this));
@@ -106,7 +107,7 @@ export class Control extends lib.Link {
 		});
 
 		for (let event of ["connect", "drop", "resume", "close"] as const) {
-			this.connector.on(event, this.hooks.controllerConnectionEvent.listener);
+			this.connector.on(event, () => this.hooks.controllerConnectionEvent.invoke(event));
 		}
 
 		this.handle(lib.AccountUpdateEvent, this.handleAccountUpdateEvent.bind(this));

plugins/global_chat/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-global_chat",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Clusterio plugin forwarding between Factorio servers",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",

plugins/player_auth/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-player_auth",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Clusterio plugin authenticating logged in players to the web interface",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Hornwitser <github@hornwitser.no>",

plugins/player_auth/test/plugin.js

@@ -1,4 +1,5 @@
 import assert from "node:assert/strict";
+import { mock as nodeMock } from "node:test";
 import jwt from "jsonwebtoken";
 
 import * as mock from "../../../test/mock.js";
@@ -87,6 +88,17 @@ describe("player_auth", function() {
 			let endpoint;
 			let fetchPlayerCode;
 			let setVerifyCode;
+			let now;
+			let dateNow;
+
+			beforeEach(function() {
+				// Keep code expiry and JWT timestamps independent of HTTP request timing.
+				now = Date.now();
+				dateNow = nodeMock.method(Date, "now", () => now);
+			});
+			afterEach(function() {
+				dateNow.mock.restore();
+			});
 
 			// Adds a pending code for player through the request handlers
 			async function addPlayer(player, { timeout = 1, verifyCode } = {}) {
@@ -334,6 +346,8 @@ describe("player_auth", function() {
 
 			describe("integration", function() {
 				it("should verify a full login flow", async function() {
+					// A one-second timeout leaves only 1ms before the JWT's rounded expiry here.
+					now = Math.floor(now / 1000) * 1000 + 999;
 					let { playerCode } = await fetchPlayerCode(new FetchPlayerCodeRequest("test"));
 
 					const playerCodeResult = await postJSON(
@@ -353,6 +367,22 @@ describe("player_auth", function() {
 					assert.equal(verifyResult.status, 200);
 					assert.deepEqual(await verifyResult.json(), { verified: true, token: token });
 				});
+				it("should reject a verification token at its expiry", async function() {
+					const playerCode = await addPlayer("test");
+					const playerCodeResult = await postJSON(
+						`${controllerUrl}/api/player_auth/player_code`, { player_code: playerCode }
+					);
+					const { verify_code, verify_token } = await playerCodeResult.json();
+					await setVerifyCode(new SetVerifyCodeRequest("test", verify_code));
+
+					now = jwt.decode(verify_token).exp * 1000;
+					const verifyResult = await postJSON(
+						`${controllerUrl}/api/player_auth/verify`,
+						{ player_code: playerCode, verify_code, verify_token }
+					);
+					assert.equal(verifyResult.status, 200);
+					assert.deepEqual(await verifyResult.json(), { error: true, message: "jwt expired" });
+				});
 			});
 		});
 	});

plugins/player_sync/README.md

@@ -30,6 +30,33 @@ Remove the old package and install the new one in its place:
 The `inventory_sync.*` config fields and the `inventory_sync.inventory.view` permission are renamed on load.
 The controller moves `inventories.json` in the database directory to `player_sync.json`, and saves keep their player data the next time they are started with the new module.
 
+## Fixing broken inventories
+
+The controller stores one entry per player in `player_sync.json` and keeps a lock on the player while an instance has them online.
+When sync goes wrong for a player, the Player sync page in the web interface lists every stored entry with its generation, size and the instance holding the lock.
+Click an entry to see a summary of its inventories, export it as JSON, import an edited JSON file in its place, delete it, or release the lock.
+
+The same operations are available from clusterioctl:
+
+    npx clusterioctl player-sync list
+    npx clusterioctl player-sync show <player>
+    npx clusterioctl player-sync export <player> [file]
+    npx clusterioctl player-sync import <player> <file>
+    npx clusterioctl player-sync delete <player>
+    npx clusterioctl player-sync release <player>
+
+Importing and deleting both start a new epoch for the player.
+An instance holding a copy from an earlier epoch throws it away the next time the player joins, even if it is newer than what the controller has, and an upload from an earlier epoch is discarded.
+After an import the instance downloads the new data.
+After a delete the next join turns whatever the player has on that instance into the new synced inventory.
+Deleted players stay in `player_sync.json` as an entry with `deleted: true` so their epoch survives a controller restart.
+Hosts running a version of this plugin from before epochs keep comparing generations only, so they can still overwrite an import with a copy they failed to upload.
+Update the plugin on every host to get the full guarantee.
+Both are refused while an instance holds the lock on the player, because the instance uploads its own copy when the player leaves and that would undo the change.
+Have the player leave first, or release the lock if the instance is stuck holding it.
+
+Viewing and exporting need the `player_sync.inventory.view` permission. Importing, deleting and releasing need `player_sync.inventory.modify`, which is not granted by default.
+
 ## Method of operation
 
 This plugin does event based synchronization of inventories.
@@ -53,11 +80,36 @@ The greatly simplified data flow is as follows:
 When the player leaves the inventory is uploaded if it's a synced inventory and the exclusive access the scenario script holds is released.
 Should an error occur during this process the player is given the option to use a temporary inventory instead, which will be merged back into the synced inventory the next time the sync succeeds on that instance.
 
+Instances can leave parts of the player out of the sync with the `player_sync.sync_*` options below.
+A part which is not synced is left as it is when the player joins, and the instance's copy of it is not uploaded, so the stored copy from other instances stays as it was.
+A player online when the options change keeps the old ones until they leave and join again, and a part that was turned on is downloaded when they join.
+Turning off `player_sync.sync_inventories` leaves the items players have on the instance in place while the stored inventory still holds them too.
+Items follow the player to whatever controller they end up with, a character, god or ghost, and what does not fit is put into a corpse below them.
+
 Communication between the server and instance goes over stdout or rcon, depending on the size of the data.
 Communication between the instance and the controller goes over websockets.
 Overall, we are able to achieve a latency between 3 and infinite ticks from server join, depending on the size of the inventory. The major limiter is rcon transfer speeds with larger inventories, especially if they contain blueprints.
 
 
+## Sending player data ahead
+
+A plugin that moves a player to another instance with `LuaPlayer.connect_to_server` can tell player sync where the player is going, just before the call:
+
+```lua
+local ps = remote.interfaces.player_sync
+if ps and ps.set_destination then
+	pcall(remote.call, "player_sync", "set_destination", player.name, instance_id) -- a failure never stops the connect
+end
+player.connect_to_server({ address = address, name = name })
+```
+
+`instance_id` is the Clusterio instance id of the destination. The call only records it, and a later call replaces it.
+When the player then leaves with the reason `switching_servers`, the upload carries the destination and the controller sends the stored data on to that instance while the client reconnects.
+On joining, a copy that the acquire reply confirms is applied without a download.
+The destination's `player_sync.sync_*` options apply to the copy as to a download.
+A cancelled dialog, a quit or a crash sends nothing.
+If the destination isn't running, has player sync off or is the player's current instance, the join downloads as before.
+
 ## Controller Configuration
 
 ### player_sync.player_lock_timeout
@@ -77,3 +129,55 @@ Size in characters of the chunks the serialized inventory is split into when sen
 Smaller chunks take more commands to transfer an inventory but hold up the RCON command pipe for less time each.
 
 Defaults to `1000`.
+
+### player_sync.sync_controller
+
+Sync whether the player is a character, god or spectator, their respawn timer and cheat mode.
+
+Defaults to `true`.
+
+### player_sync.sync_force
+
+Sync the force the player is on.
+
+Defaults to `true`.
+
+### player_sync.sync_appearance
+
+Sync the player color, chat color and tag.
+
+Defaults to `true`.
+
+### player_sync.sync_inventories
+
+Sync the items the player holds, the character bonuses and the crafting queue.
+This covers the character inventories, the god inventory and the items a dead player will respawn with.
+A spectator holds no items, so the synced items are put into a corpse when the player is a spectator on the instance.
+A player without a character gets the ingredients of the crafting queue as items instead.
+With this off and the controller synced, the items the player has on the instance move to the synced controller the same way.
+
+Defaults to `true`.
+
+### player_sync.sync_logistics
+
+Sync the personal logistic requests.
+
+Defaults to `true`.
+
+### player_sync.sync_quick_bar
+
+Sync the quick bar.
+
+Defaults to `true`.
+
+### player_sync.sync_settings
+
+Sync the shortcut toggles, game view settings and flashlight.
+
+Defaults to `true`.
+
+### player_sync.sync_recipe_notifications
+
+Sync which new recipe notifications have been seen.
+
+Defaults to `true`.

plugins/player_sync/components.ts

@@ -0,0 +1,82 @@
+import type * as lib from "@clusterio/lib";
+import type { IpcPlayerData } from "./messages.js";
+
+/** Parts of the player data an instance can choose to sync, and the fields of the player data holding them. */
+export const components = {
+	controller: ["controller", "ticks_to_respawn", "cheat_mode"],
+	force: ["force"],
+	appearance: ["color", "chat_color", "tag"],
+	// inventories holds the god inventory and the hidden ghost inventory
+	inventories: ["character", "crafting_queue", "inventories"],
+	logistics: ["personal_logistic_slots"],
+	quick_bar: ["quick_bar", "hotbar"],
+	settings: ["flashlight", "shortcuts", "game_view_settings"],
+	recipe_notifications: ["recipe_notifications"],
+} as const;
+
+export type Component = keyof typeof components;
+
+function componentSet(isEnabled: (component: Component) => boolean) {
+	return new Set((Object.keys(components) as Component[]).filter(isEnabled));
+}
+
+/** Components synced on an instance. */
+export function enabledComponents(config: lib.InstanceConfig) {
+	return componentSet(component => Boolean(config.get(`player_sync.sync_${component}`)));
+}
+
+/**
+ * Components the scenario sent with an upload or download, all of them when not given.
+ *
+ * Lua may encode an empty table as an array, which reads as no components.
+ */
+export function scenarioComponents(sent: Record<string, boolean> | unknown[] | undefined) {
+	if (sent === undefined) {
+		return componentSet(() => true);
+	}
+	return componentSet(component => !Array.isArray(sent) && sent[component] === true);
+}
+
+/** Remove the fields of components which are not synced. */
+export function stripPlayerData(playerData: IpcPlayerData, enabled: Set<Component>) {
+	const fields = playerData as Record<string, unknown>;
+	for (const [component, keys] of Object.entries(components)) {
+		if (!enabled.has(component as Component)) {
+			for (const key of keys) {
+				delete fields[key];
+			}
+		}
+	}
+	return playerData;
+}
+
+/**
+ * Combine data uploaded by an instance with the stored data.
+ *
+ * Components the upload was not serialized with keep the stored fields.
+ * Logistic requests are kept when the player has nowhere to hold them.
+ */
+export function mergePlayerData(stored: IpcPlayerData | undefined, uploaded: IpcPlayerData) {
+	const { components: uploadComponents, ...merged } = uploaded as IpcPlayerData & Record<string, unknown>;
+	const enabled = scenarioComponents(uploadComponents);
+	function keep(keys: readonly string[]) {
+		for (const key of keys) {
+			const value = (stored as Record<string, unknown> | undefined)?.[key];
+			if (value !== undefined) {
+				merged[key] = value;
+			} else {
+				delete merged[key];
+			}
+		}
+	}
+
+	for (const [component, keys] of Object.entries(components)) {
+		if (!enabled.has(component as Component)) {
+			keep(keys);
+		}
+	}
+	if (!uploaded.character && !uploaded.personal_logistic_slots) {
+		keep(["personal_logistic_slots"]);
+	}
+	return merged as IpcPlayerData;
+}

plugins/player_sync/controller.ts

@@ -1,11 +1,16 @@
 import type { ControllerPluginContext, InstanceRecord } from "@clusterio/controller";
 import type { IpcPlayerData } from "./messages.js";
+import { mergePlayerData } from "./components.js";
 
+import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "path";
 import * as lib from "@clusterio/lib";
 import * as msg from "./messages.js";
 
+// Left behind by deleted player data so instances can tell their copy is stale
+type Tombstone = { generation: number, epoch: number, deleted: true };
+
 async function loadDatabase(config: lib.ControllerConfig, logger: lib.Logger) {
 	let databaseDirectory = config.get("controller.database_directory");
 	let itemsPath = path.resolve(databaseDirectory, "player_sync.json");
@@ -21,47 +26,76 @@ async function loadDatabase(config: lib.ControllerConfig, logger: lib.Logger) {
 			}
 		}
 	}
+	let entries: [string, IpcPlayerData | Tombstone][] = [];
 	logger.verbose(`Loading ${itemsPath}`);
 	try {
 		let content = await fs.readFile(itemsPath, { encoding: "utf8" });
-		return new Map(JSON.parse(content));
+		entries = JSON.parse(content);
 
 	} catch (err: any) {
-		if (err.code === "ENOENT") {
-			logger.verbose("Creating new player data database");
-			return new Map();
+		if (err.code !== "ENOENT") {
+			throw err;
 		}
-		throw err;
+		logger.verbose("Creating new player data database");
 	}
+
+	let playerDatastore = new Map<string, IpcPlayerData>();
+	let tombstones = new Map<string, Tombstone>();
+	for (let [name, entry] of entries) {
+		if ("deleted" in entry && entry.deleted === true) {
+			tombstones.set(name, entry);
+		} else {
+			// Imports used to keep the marker
+			delete (entry as any).deleted;
+			playerDatastore.set(name, entry as IpcPlayerData);
+		}
+	}
+	return { playerDatastore, tombstones };
 }
 
+// Tombstones share the file with the player data so a delete is written in one go
 async function saveDatabase(
 	controllerConfig: lib.ControllerConfig,
-	playerDatastore: Map<string, IpcPlayerData> | undefined,
+	playerDatastore: Map<string, IpcPlayerData>,
+	tombstones: Map<string, Tombstone>,
 	logger: lib.Logger,
 ) {
-	if (playerDatastore) {
-		let file = path.resolve(controllerConfig.get("controller.database_directory"), "player_sync.json");
-		logger.verbose(`writing ${file}`);
-		let content = JSON.stringify(Array.from(playerDatastore));
-		await lib.safeOutputFile(file, content);
-	}
+	let file = path.resolve(controllerConfig.get("controller.database_directory"), "player_sync.json");
+	logger.verbose(`writing ${file}`);
+	let content = JSON.stringify([...playerDatastore, ...tombstones]);
+	await lib.safeOutputFile(file, content);
 }
 
 export default async function(context: ControllerPluginContext) {
 	const { controller, logger, plugin } = context;
+	// Differs on every controller start: a prefetched copy holding another run's token may be one this run never saved
+	const runToken = crypto.randomBytes(8).toString("hex");
 	const acquiredPlayers = new Map<string, { instanceId: number, expiresMs?: number }>();
-	const playerDatastore: Map<string, IpcPlayerData> = await loadDatabase(controller.config, logger);
+	const { playerDatastore, tombstones } = await loadDatabase(controller.config, logger);
 	let playerDatastoreDirty = false;
 
-	function acquire(instanceId: number, playerName: string): boolean {
+	// Generation and epoch instances should compare their copy against
+	function storedVersion(playerName: string) {
+		let stored = playerDatastore.get(playerName) ?? tombstones.get(playerName);
+		return { generation: stored?.generation ?? 0, epoch: stored?.epoch ?? 0 };
+	}
+
+	// Returns the acquisition record for the player if it's still in effect
+	function getAcquisition(playerName: string) {
 		let acquisitionRecord = acquiredPlayers.get(playerName);
 		if (
 			!acquisitionRecord
-			|| acquisitionRecord.instanceId === instanceId
 			|| !controller.instances.has(acquisitionRecord.instanceId)
 			|| acquisitionRecord.expiresMs && acquisitionRecord.expiresMs < Date.now()
 		) {
+			return undefined;
+		}
+		return acquisitionRecord;
+	}
+
+	function acquire(instanceId: number, playerName: string): boolean {
+		let acquisitionRecord = getAcquisition(playerName);
+		if (!acquisitionRecord || acquisitionRecord.instanceId === instanceId) {
 			acquiredPlayers.set(playerName, { instanceId });
 			return true;
 		}
@@ -69,6 +103,39 @@ export default async function(context: ControllerPluginContext) {
 		return false;
 	}
 
+	// Modifying stored data while an instance holds the player is pointless
+	// as the instance uploads its own copy when the player leaves.
+	function checkNotAcquired(playerName: string) {
+		let acquisitionRecord = getAcquisition(playerName);
+		if (acquisitionRecord) {
+			let instanceName = controller.instances.get(acquisitionRecord.instanceId)!.config.get("instance.name");
+			throw new lib.RequestError(
+				`${playerName} is currently acquired by ${instanceName}, ` +
+				"have the player leave or release the lock first"
+			);
+		}
+	}
+
+	// Send stored player data on to the instance the player left to switch to (set_destination in the Lua module).
+	// Only another running instance with player_sync on: a copy applied on the source would rebuild the body there.
+	function prefetch(sourceId: number, targetId: number, playerData: IpcPlayerData) {
+		const target = controller.instances.get(targetId);
+		if (
+			!target || targetId === sourceId || target.status !== "running"
+			|| !target.config.get("player_sync.load_plugin")
+		) {
+			logger.verbose(`Not prefetching ${playerData.name} to instance ${targetId}`);
+			return;
+		}
+		try {
+			controller.sendTo({ instanceId: targetId }, new msg.PrefetchEvent(playerData, runToken));
+			logger.verbose(`Prefetching ${playerData.name} to ${target.config.get("instance.name")}`);
+		} catch (err: any) {
+			// sendEvent throws when the instance's host isn't connected; the upload is stored either way
+			logger.warn(`Failed to prefetch ${playerData.name} to instance ${targetId}: ${err.message}`);
+		}
+	}
+
 	controller.hooks.instanceStatusChanged.attach(plugin.name, async (instance: InstanceRecord) => {
 		let instanceId = instance.id;
 		if (["unassigned", "deleted"].includes(instance.status)) {
@@ -100,7 +167,7 @@ export default async function(context: ControllerPluginContext) {
 	controller.hooks.save.attach(plugin.name, async () => {
 		if (playerDatastoreDirty) {
 			playerDatastoreDirty = false;
-			await saveDatabase(controller.config, playerDatastore, logger);
+			await saveDatabase(controller.config, playerDatastore, tombstones, logger);
 		}
 	});
 
@@ -115,11 +182,14 @@ export default async function(context: ControllerPluginContext) {
 			};
 		}
 
-		let playerData = playerDatastore.get(playerName);
+		let { generation, epoch } = storedVersion(playerName);
 		return new msg.AcquireRequest.Response(
 			"acquired",
-			playerData ? playerData.generation : 0,
-			Boolean(playerData),
+			generation,
+			playerDatastore.has(playerName),
+			undefined,
+			epoch || undefined,
+			runToken,
 		);
 	});
 
@@ -154,19 +224,39 @@ export default async function(context: ControllerPluginContext) {
 		}
 
 		acquiredPlayers.delete(playerName);
-		let oldPlayerData = playerDatastore.get(playerName);
-		if (store && oldPlayerData && oldPlayerData.generation >= playerData.generation) {
+		let stored = storedVersion(playerName);
+		if (playerData.epoch === undefined) {
+			// Instances from before epochs were added never send one, they keep the rules they had before
+			stored.generation = playerDatastore.get(playerName)?.generation ?? 0;
+			if (stored.epoch) {
+				playerData.epoch = stored.epoch;
+			}
+		}
+		if (store && (playerData.epoch ?? 0) !== stored.epoch) {
 			logger.warn(
-				`${instanceName} uploaded generation ${playerData.generation} while the stored` +
-				`generation is ${oldPlayerData.generation} for ${playerName}`
+				`${instanceName} uploaded ${playerName} from epoch ${playerData.epoch} while the stored ` +
+				`epoch is ${stored.epoch}, discarding the outdated copy`
+			);
+			store = false;
+
+		} else if (store && stored.generation >= playerData.generation) {
+			logger.warn(
+				`${instanceName} uploaded generation ${playerData.generation} while the stored ` +
+				`generation is ${stored.generation} for ${playerName}`
 			);
 			store = false;
 		}
 
 		if (store) {
 			logger.verbose(`Received player data for ${playerName} from ${instanceName}`);
-			playerDatastore.set(playerName, playerData);
+			const merged = mergePlayerData(playerDatastore.get(playerName), playerData);
+			playerDatastore.set(playerName, merged);
+			tombstones.delete(playerName);
 			playerDatastoreDirty = true;
+			if (request.destinationInstanceId !== undefined) {
+				// The stored copy: components the upload left out are the controller's
+				prefetch(instanceId, request.destinationInstanceId, merged);
+			}
 		}
 	});
 
@@ -201,4 +291,71 @@ export default async function(context: ControllerPluginContext) {
 			},
 		);
 	});
+
+	controller.handle(msg.ListPlayersRequest, async () => {
+		let entries = new Map<string, msg.PlayerEntry>();
+		for (let [name, playerData] of playerDatastore) {
+			let size = JSON.stringify(playerData).length;
+			entries.set(name, new msg.PlayerEntry(name, playerData.generation, size, undefined, playerData.force));
+		}
+		for (let name of acquiredPlayers.keys()) {
+			let acquisitionRecord = getAcquisition(name);
+			if (!acquisitionRecord) {
+				continue;
+			}
+			let entry = entries.get(name) ?? new msg.PlayerEntry(name);
+			entry.instanceId = acquisitionRecord.instanceId;
+			entries.set(name, entry);
+		}
+		return [...entries.values()];
+	});
+
+	controller.handle(msg.GetPlayerDataRequest, async (request: msg.GetPlayerDataRequest) => {
+		let { playerName } = request;
+		return new msg.GetPlayerDataRequest.Response(
+			playerDatastore.get(playerName) ?? null,
+			getAcquisition(playerName)?.instanceId,
+		);
+	});
+
+	controller.handle(msg.SetPlayerDataRequest, async (request: msg.SetPlayerDataRequest) => {
+		let { playerName, playerData } = request;
+		checkNotAcquired(playerName);
+
+		// A new epoch makes every instance drop its copy, including ones with an unconfirmed upload
+		let stored = storedVersion(playerName);
+		playerData.name = playerName;
+		// Start over from generations at or past the limit, instances that know epochs download the import anyway
+		let imported = playerData.generation < msg.generationLimit ? playerData.generation : 0;
+		let next = stored.generation < msg.generationLimit ? stored.generation + 1 : 1;
+		playerData.generation = Math.max(imported, next);
+		playerData.epoch = stored.epoch + 1;
+		playerDatastore.set(playerName, playerData);
+		tombstones.delete(playerName);
+		playerDatastoreDirty = true;
+		logger.info(`Replaced player data for ${playerName} with generation ${playerData.generation}`);
+		return new msg.SetPlayerDataRequest.Response(playerData.generation);
+	});
+
+	controller.handle(msg.DeletePlayerDataRequest, async (request: msg.DeletePlayerDataRequest) => {
+		let { playerName } = request;
+		if (!playerDatastore.has(playerName)) {
+			throw new lib.RequestError(`No player data stored for ${playerName}`);
+		}
+		checkNotAcquired(playerName);
+
+		let { generation, epoch } = storedVersion(playerName);
+		tombstones.set(playerName, { generation, epoch: epoch + 1, deleted: true });
+		playerDatastore.delete(playerName);
+		playerDatastoreDirty = true;
+		logger.info(`Deleted player data for ${playerName}`);
+	});
+
+	controller.handle(msg.ForceReleaseRequest, async (request: msg.ForceReleaseRequest) => {
+		let { playerName } = request;
+		if (!acquiredPlayers.delete(playerName)) {
+			throw new lib.RequestError(`${playerName} is not acquired by any instance`);
+		}
+		logger.info(`Released lock on ${playerName}`);
+	});
 }

plugins/player_sync/ctl.ts

@@ -0,0 +1,130 @@
+import fs from "node:fs/promises";
+import * as lib from "@clusterio/lib";
+import type { Ctl, CtlPluginContext } from "@clusterio/ctl";
+import * as msg from "./messages.js";
+import { summarizePlayerInventories } from "./player_data.js";
+
+function print(...content: any) {
+	// eslint-disable-next-line no-console
+	console.log(...content);
+}
+
+async function instanceName(ctl: Ctl, instanceId: number | undefined) {
+	if (instanceId === undefined) {
+		return "-";
+	}
+	try {
+		let instance = await ctl.send(new lib.InstanceDetailsGetRequest(instanceId));
+		return `${instance.name} (${instanceId})`;
+	} catch (err) {
+		// Viewing player data does not require core.instance.get
+		return String(instanceId);
+	}
+}
+
+const playerSyncCommands = new lib.CommandTree({
+	name: "player-sync", description: "Player sync plugin commands",
+});
+playerSyncCommands.add(new lib.Command({
+	definition: [["list", "l"], "List player data stored on the controller"],
+	handler: async function(args: object, ctl: Ctl) {
+		let players = await ctl.send(new msg.ListPlayersRequest());
+		players.sort((a, b) => a.name.localeCompare(b.name));
+		print("name | generation | size | acquired by");
+		for (let player of players) {
+			print(
+				`${player.name} | ${player.generation ?? "-"} | ` +
+				`${player.size === undefined ? "-" : lib.formatBytes(player.size)} | ` +
+				`${await instanceName(ctl, player.instanceId)}`
+			);
+		}
+	},
+}));
+
+playerSyncCommands.add(new lib.Command({
+	definition: ["show <player>", "Show a summary of the stored player data", (yargs) => {
+		yargs.positional("player", { describe: "Name of the player", type: "string" });
+	}],
+	handler: async function(args: { player: string }, ctl: Ctl) {
+		let response = await ctl.send(new msg.GetPlayerDataRequest(args.player));
+		print(`acquired by: ${await instanceName(ctl, response.instanceId)}`);
+		let playerData = response.playerData;
+		if (!playerData) {
+			print(`No player data stored for ${args.player}`);
+			return;
+		}
+		print(`generation: ${playerData.generation}`);
+		print(`size: ${lib.formatBytes(JSON.stringify(playerData).length)}`);
+		print(`controller: ${playerData.controller}`);
+		print(`force: ${playerData.force}`);
+		for (let inventory of summarizePlayerInventories(playerData)) {
+			print(`${inventory.name}:`);
+			for (let item of inventory.items) {
+				let quality = item.quality ? ` (${item.quality})` : "";
+				let size = item.exportSize ? ` ${lib.formatBytes(item.exportSize)}` : "";
+				print(`  ${item.count} ${item.name}${quality}${size}`);
+			}
+		}
+	},
+}));
+
+playerSyncCommands.add(new lib.Command({
+	definition: ["export <player> [file]", "Export stored player data as JSON", (yargs) => {
+		yargs.positional("player", { describe: "Name of the player", type: "string" });
+		yargs.positional("file", { describe: "File to write to, prints to stdout if omitted", type: "string" });
+	}],
+	handler: async function(args: { player: string, file?: string }, ctl: Ctl) {
+		let response = await ctl.send(new msg.GetPlayerDataRequest(args.player));
+		if (!response.playerData) {
+			throw new lib.CommandError(`No player data stored for ${args.player}`);
+		}
+		let content = JSON.stringify(response.playerData, null, "\t");
+		if (args.file) {
+			await fs.writeFile(args.file, content);
+			print(`Wrote player data for ${args.player} to ${args.file}`);
+		} else {
+			print(content);
+		}
+	},
+}));
+
+playerSyncCommands.add(new lib.Command({
+	definition: ["import <player> <file>", "Replace stored player data with JSON from a file", (yargs) => {
+		yargs.positional("player", { describe: "Name of the player", type: "string" });
+		yargs.positional("file", { describe: "File to read player data from", type: "string" });
+	}],
+	handler: async function(args: { player: string, file: string }, ctl: Ctl) {
+		let playerData = JSON.parse(await fs.readFile(args.file, "utf8"));
+		if (typeof playerData !== "object" || playerData === null || Array.isArray(playerData)) {
+			throw new lib.CommandError("Player data must be a JSON object");
+		}
+		playerData.name = args.player;
+		playerData.generation ??= 0;
+		let response = await ctl.send(new msg.SetPlayerDataRequest(args.player, playerData));
+		print(`Stored player data for ${args.player} as generation ${response.generation}`);
+	},
+}));
+
+playerSyncCommands.add(new lib.Command({
+	definition: ["delete <player>", "Delete stored player data", (yargs) => {
+		yargs.positional("player", { describe: "Name of the player", type: "string" });
+	}],
+	handler: async function(args: { player: string }, ctl: Ctl) {
+		await ctl.send(new msg.DeletePlayerDataRequest(args.player));
+	},
+}));
+
+playerSyncCommands.add(new lib.Command({
+	definition: ["release <player>", "Release the lock an instance holds on a player", (yargs) => {
+		yargs.positional("player", { describe: "Name of the player", type: "string" });
+	}],
+	handler: async function(args: { player: string }, ctl: Ctl) {
+		await ctl.send(new msg.ForceReleaseRequest(args.player));
+	},
+}));
+
+export default async function(context: CtlPluginContext) {
+	context.hooks.addCommands.attach(context.plugin.name, async (rootCommand) => {
+		rootCommand.add(playerSyncCommands);
+	});
+}

plugins/player_sync/index.ts

@@ -4,12 +4,21 @@ import * as messages from "./messages.js";
 declare module "@clusterio/lib" {
 	export interface InstanceConfigFields {
 		"player_sync.rcon_chunk_size": number;
+		"player_sync.sync_controller": boolean;
+		"player_sync.sync_force": boolean;
+		"player_sync.sync_appearance": boolean;
+		"player_sync.sync_inventories": boolean;
+		"player_sync.sync_logistics": boolean;
+		"player_sync.sync_quick_bar": boolean;
+		"player_sync.sync_settings": boolean;
+		"player_sync.sync_recipe_notifications": boolean;
 	}
 	export interface ControllerConfigFields {
 		"player_sync.player_lock_timeout": number;
 	}
 	export interface Permissions {
 		"player_sync.inventory.view": never;
+		"player_sync.inventory.modify": never;
 	}
 }
 
@@ -28,9 +37,62 @@ export const plugin: lib.PluginDeclaration = {
 			type: "number",
 			initialValue: 1000,
 		},
+		"player_sync.sync_controller": {
+			title: "Sync controller",
+			description:
+				"Sync whether the player is a character, god or spectator, their respawn timer, cheat mode " +
+				"and god inventory. Only synced while inventories are synced too.",
+			type: "boolean",
+			initialValue: true,
+		},
+		"player_sync.sync_force": {
+			title: "Sync force",
+			description: "Sync the force the player is on.",
+			type: "boolean",
+			initialValue: true,
+		},
+		"player_sync.sync_appearance": {
+			title: "Sync appearance",
+			description: "Sync the player color, chat color and tag.",
+			type: "boolean",
+			initialValue: true,
+		},
+		"player_sync.sync_inventories": {
+			title: "Sync inventories",
+			description:
+				"Sync the character inventories, character bonuses and crafting queue. Without the " +
+				"controller synced a player without a character keeps their local inventory.",
+			type: "boolean",
+			initialValue: true,
+		},
+		"player_sync.sync_logistics": {
+			title: "Sync logistics",
+			description: "Sync the personal logistic requests.",
+			type: "boolean",
+			initialValue: true,
+		},
+		"player_sync.sync_quick_bar": {
+			title: "Sync quick bar",
+			description: "Sync the quick bar.",
+			type: "boolean",
+			initialValue: true,
+		},
+		"player_sync.sync_settings": {
+			title: "Sync settings",
+			description: "Sync the shortcut toggles, game view settings and flashlight.",
+			type: "boolean",
+			initialValue: true,
+		},
+		"player_sync.sync_recipe_notifications": {
+			title: "Sync recipe notifications",
+			description: "Sync which new recipe notifications have been seen.",
+			type: "boolean",
+			initialValue: true,
+		},
 	},
 
 	controllerEntrypoint: "dist/node/controller.js",
+	ctlEntrypoint: "dist/node/ctl.js",
 	controllerConfigFields: {
 		"player_sync.player_lock_timeout": {
 			title: "Player Lock Timeout",
@@ -52,7 +114,13 @@ export const plugin: lib.PluginDeclaration = {
 		messages.ReleaseRequest,
 		messages.UploadRequest,
 		messages.DownloadRequest,
+		messages.PrefetchEvent,
 		messages.DatabaseStatsRequest,
+		messages.ListPlayersRequest,
+		messages.GetPlayerDataRequest,
+		messages.SetPlayerDataRequest,
+		messages.DeletePlayerDataRequest,
+		messages.ForceReleaseRequest,
 	],
 	permissions: [
 		{
@@ -61,6 +129,11 @@ export const plugin: lib.PluginDeclaration = {
 			description: "View player inventories",
 			grantByDefault: true,
 		},
+		{
+			name: "player_sync.inventory.modify",
+			title: "Modify player inventories",
+			description: "Replace, delete and release the lock on player inventories stored on the controller",
+		},
 	],
 	webEntrypoint: "./web",
 	routes: ["/inventory"],

plugins/player_sync/instance.ts

@@ -4,7 +4,9 @@ import * as lib from "@clusterio/lib";
 import type { InstancePluginContext } from "@clusterio/host";
 import {
 	AcquireRequest, AcquireResponse, ReleaseRequest, UploadRequest, DownloadRequest, DownloadResponse, IpcPlayerData,
+	PrefetchEvent,
 } from "./messages.js";
+import { type Component, enabledComponents, stripPlayerData } from "./components.js";
 
 type IpcPlayerName = {
 	player_name: string
@@ -15,12 +17,19 @@ type IpcDownloadRequest = {
 	recipe_notifications?: string,
 }
 
+/** Components as an escaped JSON object for the scenario. */
+function componentsJson(enabled: Set<Component>) {
+	return lib.escapeString(JSON.stringify(Object.fromEntries([...enabled].map(name => [name, true]))));
+}
+
 type IpcAcquireResponse = {
 	player_name: string,
 	status: string,
 	generation?: number,
 	has_data?: boolean,
 	message?: string,
+	epoch?: number,
+	token?: string,
 }
 
 /**
@@ -53,6 +62,8 @@ async function encodeLuaString(text: string): Promise<string> {
 export type RecipeNotificationDelta = {
 	add?: string[],
 	remove?: string[],
+	/** add is the whole stored list, sent without the instance's: the save works out what to remove */
+	full?: true,
 }
 
 /**
@@ -79,6 +90,7 @@ export function recipeNotificationDelta(stored: string[], current: string[]): Re
 /**
  * Replace the stored recipe notifications with the difference from the
  * current state on the instance to reduce the amount of data sent to Lua.
+ * Without a current state the whole stored list is sent, marked full.
  */
 export async function applyRecipeNotificationDelta(
 	playerData: IpcPlayerData,
@@ -89,10 +101,10 @@ export async function applyRecipeNotificationDelta(
 		return;
 	}
 	try {
-		const delta = recipeNotificationDelta(
-			JSON.parse(await decodeLuaString(playerData.recipe_notifications)),
-			current ? JSON.parse(await decodeLuaString(current)) : [],
-		);
+		const stored = JSON.parse(await decodeLuaString(playerData.recipe_notifications));
+		const delta: RecipeNotificationDelta = current
+			? recipeNotificationDelta(stored, JSON.parse(await decodeLuaString(current)))
+			: { ...recipeNotificationDelta(stored, []), full: true };
 		playerData.recipe_notifications = await encodeLuaString(JSON.stringify(delta));
 	} catch (err: any) {
 		logger.warn(`Dropping invalid recipe notifications for ${playerData.name}:\n${err.stack}`);
@@ -112,6 +124,7 @@ export default async function(context: InstancePluginContext) {
 			message: "Controller is temporarily unavailable",
 			has_data: undefined,
 			generation: undefined,
+			epoch: undefined,
 		};
 
 		if (host.connector.connected && !disconnecting) {
@@ -126,6 +139,8 @@ export default async function(context: InstancePluginContext) {
 					generation: acquireResponse.generation,
 					has_data: acquireResponse.hasData,
 					message: acquireResponse.message,
+					epoch: acquireResponse.epoch,
+					token: acquireResponse.token,
 				};
 			} catch (err: any) {
 				if (!(err instanceof lib.SessionLost)) {
@@ -163,11 +178,16 @@ export default async function(context: InstancePluginContext) {
 			return;
 		}
 
+		// Set by the Lua module when the player left to switch servers (set_destination, which only records integers).
+		// It isn't player data.
+		const destinationInstanceId = player_data.destination_instance_id;
+		delete player_data.destination_instance_id;
+
 		logger.verbose(`Uploading ${player_data.name} (${JSON.stringify(player_data).length / 1000}kB)`);
 		try {
 			await instance.sendTo(
 				"controller",
-				new UploadRequest(instance.id, player_data.name, player_data),
+				new UploadRequest(instance.id, player_data.name, player_data, destinationInstanceId),
 			);
 
 		} catch (err: any) {
@@ -182,6 +202,39 @@ export default async function(context: InstancePluginContext) {
 		);
 	}
 
+	/**
+	 * Send player data to the save in chunks of player_sync.rcon_chunk_size, stripped to the components synced here
+	 * and with its recipe notifications as a delta from current. command(chunk, number, total, extra) gives the RCON
+	 * command for each escaped chunk, numbered from 1; extra passes the components the data was stripped by with
+	 * chunk 1 and is empty for the rest. With stopOnReply a non-empty reply (the save refusing or ignoring the data)
+	 * ends the transfer; without it any reply throws, as sendRcon's expectEmpty does.
+	 */
+	async function sendPlayerData(
+		playerData: IpcPlayerData,
+		current: string | undefined,
+		command: (chunk: string, number: number, total: number, extra: string) => string,
+		stopOnReply: boolean,
+	) {
+		const enabled = enabledComponents(instance.config);
+		stripPlayerData(playerData, enabled);
+		await applyRecipeNotificationDelta(playerData, current, logger);
+
+		const chunks = chunkify(instance.config.get("player_sync.rcon_chunk_size"), JSON.stringify(playerData));
+		logger.verbose(`Sending inventory for ${playerData.name} in ${chunks.length} chunks`);
+		// The scenario restarts a download, and drops a prefetched copy at the join, sent with other components
+		const components = componentsJson(enabled);
+		for (let i = 0; i < chunks.length; i++) {
+			const extra = i === 0 ? `,'${components}'` : "";
+			const reply = await instance.sendRcon(
+				command(lib.escapeString(chunks[i]), i + 1, chunks.length, extra), !stopOnReply, plugin.name,
+			);
+			if (stopOnReply && reply !== "") {
+				logger.verbose(`Stopped sending ${playerData.name}: ${reply.trim()}`);
+				return;
+			}
+		}
+	}
+
 	async function handleDownload(request: IpcDownloadRequest) {
 		const playerName = request.player_name;
 		logger.verbose(`Downloading ${playerName}`);
@@ -198,19 +251,9 @@ export default async function(context: InstancePluginContext) {
 			return;
 		}
 
-		await applyRecipeNotificationDelta(response.playerData, request.recipe_notifications, logger);
-
-		const chunkSize = instance.config.get("player_sync.rcon_chunk_size");
-		const chunks = chunkify(chunkSize, JSON.stringify(response.playerData));
-		logger.verbose(`Sending inventory for ${playerName} in ${chunks.length} chunks`);
-		for (let i = 0; i < chunks.length; i++) {
-			const chunk = lib.escapeString(chunks[i]);
-			await instance.sendRcon(
-				`/sc player_sync.download_inventory('${playerName}','${chunk}',${i + 1},${chunks.length})`,
-				true,
-				plugin.name,
-			);
-		}
+		await sendPlayerData(response.playerData, request.recipe_notifications, (chunk, number, total, extra) => (
+			`/sc player_sync.download_inventory('${playerName}','${chunk}',${number},${total}${extra})`
+		), false);
 	}
 
 	// Handle IPC from scenario script
@@ -239,6 +282,36 @@ export default async function(context: InstancePluginContext) {
 		),
 	);
 
+	// A copy of a player's stored data, sent ahead while they switch to this instance (player_sync.prefetch_inventory)
+	async function handlePrefetch(event: PrefetchEvent) {
+		const { playerData, token } = event;
+		const { name, generation } = playerData;
+		const epoch = playerData.epoch ?? 0;
+		logger.verbose(`Prefetching ${name}`);
+		// This side doesn't know the save's cleared recipe list, so the whole stored list goes as a full delta
+		await sendPlayerData(playerData, undefined, (chunk, number, total, extra) => {
+			// Later chunks have no components argument, so nil holds its place
+			const copyArgs = number === 1 ? `,'${token}',${epoch}` : `,nil,'${token}',${epoch}`;
+			const args = `'${name}',${generation},'${chunk}',${number},${total}${extra}${copyArgs}`;
+			return `/sc player_sync.prefetch_inventory(${args})`;
+		}, true);
+	}
+	instance.handle(PrefetchEvent, handlePrefetch);
+
+	/** Tell the scenario which components are synced. */
+	async function sendComponents() {
+		const json = componentsJson(enabledComponents(instance.config));
+		await instance.sendRcon(`/sc player_sync.set_components("${json}")`, true, plugin.name);
+	}
+
+	instance.hooks.start.attach(plugin.name, sendComponents);
+
+	instance.hooks.instanceConfigFieldChanged.attach(plugin.name, async (field) => {
+		if (field.startsWith("player_sync.sync_") && instance.status === "running") {
+			await sendComponents();
+		}
+	});
+
 	instance.hooks.prepareControllerDisconnect.attach(plugin.name, async () => {
 		disconnecting = true;
 	});

plugins/player_sync/messages.ts

@@ -1,31 +1,192 @@
-import { Type, Static } from "@sinclair/typebox";
-import { StringEnum } from "@clusterio/lib";
+import { Type, Static, TSchema } from "@sinclair/typebox";
+import { StringEnum, jsonArray } from "@clusterio/lib";
 
 // .\module\serialize.lua:serialize.serialize_player()
 export type IpcPlayerData = {
 	generation: number,
-	controller: string,
+	/**
+	 * Bumped when the stored data is imported or deleted, invalidating copies held by instances.
+	 * Instances from before epochs were added leave it out of uploads.
+	 */
+	epoch?: number,
 	name: string,
-	color: number[],
-	chat_color: number[],
-	tag: string,
-	force: string,
-	cheat_mode: boolean,
-	flashlight: boolean,
+	controller?: string,
+	color?: number[],
+	chat_color?: number[],
+	tag?: string,
+	force?: string,
+	cheat_mode?: boolean,
+	flashlight?: boolean,
 	shortcuts?: Record<string, boolean>,
 	game_view_settings?: Record<string, boolean>,
 	ticks_to_respawn?: number,
 	character?: any,
 	inventories?: any,
-	hotbar?: string[],
-	personal_logistic_slots?: {name:string, min:number, max:number}[],
+	quick_bar?: any,
+	hotbar?: Record<string, string>,
+	personal_logistic_slots?: any,
 	crafting_queue?: any,
 	recipe_notifications?: string,
+	// Components the upload was serialized with, all when missing
+	components?: Record<string, boolean> | unknown[],
+	destination_instance_id?: number, // set_destination's instance, on the leave upload only; not player data
 }
 
+// Marks tombstones in player_sync.json
+const jsonReserved = { deleted: Type.Optional(Type.Never()) };
 const jsonPlayerData = Type.Object({
-	generation: Type.Number(),
+	generation: Type.Integer({ minimum: 0 }),
+	epoch: Type.Optional(Type.Integer({ minimum: 0 })),
 	name: Type.String(),
+	...jsonReserved,
+});
+
+// The parts of an import that module/serialize.lua relies on, item stacks are checked by the Lua deserializer
+// An empty Lua table is encoded as an object
+function jsonList<T extends TSchema>(item: T) {
+	return Type.Union([Type.Array(item), Type.Object({}, { maxProperties: 0 })]);
+}
+// Item stack format of packages/host/modules/clusterio/serialize.lua, the nested parts are left to Lua
+const jsonInventory = Type.Object({
+	b: Type.Optional(Type.Integer({ minimum: 0 })),
+	i: jsonList(Type.Object({
+		s: Type.Optional(Type.Integer({ minimum: 1 })),
+		r: Type.Optional(Type.Integer({ minimum: 0 })),
+		f: Type.Optional(Type.Union([Type.String(), Type.Object({})])),
+		e: Type.Optional(Type.String()),
+		n: Type.Optional(Type.String()),
+		c: Type.Optional(Type.Integer({ minimum: 1 })),
+		q: Type.Optional(Type.String()),
+		h: Type.Optional(Type.Number({ minimum: 0 })),
+		p: Type.Optional(Type.Number({ minimum: 0 })),
+		d: Type.Optional(Type.Number({ minimum: 0 })),
+		a: Type.Optional(Type.Number({ minimum: 0 })),
+	})),
+});
+const jsonInventories = Type.Record(Type.String(), jsonInventory);
+const jsonColor = Type.Union([
+	Type.Object({ r: Type.Number(), g: Type.Number(), b: Type.Number(), a: Type.Optional(Type.Number()) }),
+	Type.Array(Type.Number(), { minItems: 3, maxItems: 4 }),
+]);
+// Missing stats are left as they are, the minimums are the ones Factorio enforces
+const jsonCharacter = Type.Object({
+	character_crafting_speed_modifier: Type.Optional(Type.Number({ minimum: -1 })),
+	character_mining_speed_modifier: Type.Optional(Type.Number({ minimum: -1 })),
+	character_additional_mining_categories: Type.Optional(jsonList(Type.String())),
+	character_running_speed_modifier: Type.Optional(Type.Number({ minimum: -1 })),
+	character_build_distance_bonus: Type.Optional(Type.Number({ minimum: 0 })),
+	character_item_drop_distance_bonus: Type.Optional(Type.Number({ minimum: 0 })),
+	character_reach_distance_bonus: Type.Optional(Type.Number({ minimum: 0 })),
+	character_resource_reach_distance_bonus: Type.Optional(Type.Number({ minimum: 0 })),
+	character_item_pickup_distance_bonus: Type.Optional(Type.Number({ minimum: 0 })),
+	character_loot_pickup_distance_bonus: Type.Optional(Type.Number({ minimum: 0 })),
+	character_inventory_slots_bonus: Type.Optional(Type.Integer({ minimum: 0 })),
+	character_trash_slot_count_bonus: Type.Optional(Type.Integer({ minimum: 0 })),
+	character_maximum_following_robot_count_bonus: Type.Optional(Type.Integer({ minimum: 0 })),
+	character_health_bonus: Type.Optional(Type.Number({ minimum: 0 })),
+	character_personal_logistic_requests_enabled: Type.Optional(Type.Boolean()),
+	allow_dispatching_robots: Type.Optional(Type.Boolean()),
+	inhibit_movement_bonus: Type.Optional(Type.Boolean()),
+	inventories: Type.Optional(jsonInventories),
+});
+const jsonLogisticSlot = Type.Object({
+	name: Type.String(),
+	min: Type.Optional(Type.Integer({ minimum: 0 })),
+	max: Type.Optional(Type.Integer({ minimum: 0 })),
+});
+const jsonComparator = StringEnum(["=", ">", "<", "≥", ">=", "≤", "<=", "≠", "!="]);
+// Limits enforced by LuaLogisticSection.filters
+const jsonLogisticFilter = Type.Object({
+	value: Type.Optional(Type.Union([
+		Type.String(),
+		Type.Object({
+			type: Type.Optional(StringEnum([
+				"item", "fluid", "virtual", "entity", "recipe", "space-location", "asteroid-chunk", "quality",
+			])),
+			name: Type.String(),
+			quality: Type.Optional(Type.String()),
+			comparator: Type.Optional(jsonComparator),
+		}),
+	])),
+	min: Type.Optional(Type.Integer({ minimum: -(2 ** 31), maximum: 2 ** 31 - 1 })),
+	max: Type.Optional(Type.Integer({ minimum: 0, maximum: 2 ** 32 - 1 })),
+	minimum_delivery_count: Type.Optional(Type.Integer({ minimum: 0, maximum: 2 ** 32 - 1 })),
+	// Exported as null, the prototype does not convert to JSON
+	import_from: Type.Optional(Type.Union([Type.String(), Type.Null()])),
+});
+const jsonPersonalLogisticSlots = Type.Union([
+	Type.Object({
+		enabled: Type.Boolean(),
+		trash_not_requested: Type.Boolean(),
+		sections: jsonList(Type.Object({
+			group: Type.String(),
+			active: Type.Boolean(),
+			multiplier: Type.Number({ minimum: 0 }),
+			filters: jsonList(jsonLogisticFilter),
+		})),
+	}),
+	// From before 2.0, slot index to request
+	Type.Record(Type.String({ pattern: "^[1-9][0-9]*$" }), jsonLogisticSlot, { additionalProperties: false }),
+	Type.Array(jsonLogisticSlot),
+]);
+// QuickBarSlotEncoded in module/serialize.lua
+const jsonQuickBarSlot = Type.Union([
+	Type.String(),
+	Type.Object({
+		t: Type.Literal("f"),
+		n: Type.String(),
+		q: Type.Optional(Type.String()),
+		c: Type.Optional(jsonComparator),
+	}),
+	Type.Object({
+		t: Type.Literal("i"),
+		n: Type.String(),
+		s: Type.Integer({ minimum: 1 }),
+		b: Type.Optional(jsonList(Type.Integer({ minimum: 1 }))),
+	}),
+	Type.Object({
+		t: Type.Literal("r"),
+		p: jsonList(Type.Integer({ minimum: 1 })),
+		y: Type.String(),
+		l: Type.Optional(Type.String()),
+	}),
+]);
+const jsonCraftingQueue = Type.Object({
+	crafting_queue: jsonList(Type.Object({ recipe: Type.String(), count: Type.Integer({ minimum: 1 }) })),
+	crafting_queue_progress: Type.Number({ minimum: 0, maximum: 1 }),
+	ingredients: jsonList(Type.Object({
+		name: Type.String(),
+		count: Type.Integer({ minimum: 1 }),
+		quality: Type.Optional(Type.String()),
+	})),
+});
+const jsonTable = Type.Union([Type.Array(Type.Unknown()), Type.Object({})]);
+// Lua's JSON loses precision past this, generations from here on are not counted up
+export const generationLimit = 2 ** 50;
+const jsonImportedPlayerData = Type.Object({
+	generation: Type.Integer({ minimum: 0 }),
+	epoch: Type.Optional(Type.Integer({ minimum: 0 })),
+	name: Type.String(),
+	// Any physical controller, remote view is never stored and a cutscene needs waypoints to restore
+	controller: Type.Optional(StringEnum(["character", "god", "editor", "spectator", "ghost"])),
+	color: Type.Optional(jsonColor),
+	chat_color: Type.Optional(jsonColor),
+	tag: Type.Optional(Type.String()),
+	force: Type.Optional(Type.String()),
+	cheat_mode: Type.Optional(Type.Boolean()),
+	flashlight: Type.Optional(Type.Boolean()),
+	shortcuts: Type.Optional(Type.Record(Type.String(), Type.Boolean())),
+	game_view_settings: Type.Optional(Type.Record(Type.String(), Type.Boolean())),
+	ticks_to_respawn: Type.Optional(Type.Integer({ minimum: 0 })),
+	character: Type.Optional(jsonCharacter),
+	inventories: Type.Optional(jsonInventories),
+	quick_bar: Type.Optional(Type.Record(Type.String(), jsonQuickBarSlot)),
+	hotbar: Type.Optional(Type.Record(Type.String(), Type.String())),
+	personal_logistic_slots: Type.Optional(jsonPersonalLogisticSlots),
+	crafting_queue: Type.Optional(jsonCraftingQueue),
+	recipe_notifications: Type.Optional(Type.String()),
+	components: Type.Optional(jsonTable),
+	...jsonReserved,
 });
 
 export class AcquireResponse {
@@ -34,6 +195,8 @@ export class AcquireResponse {
 		public generation?: number,
 		public hasData?: boolean,
 		public message?: string,
+		public epoch?: number,
+		public token?: string,
 	) {
 	}
 
@@ -42,10 +205,12 @@ export class AcquireResponse {
 		"generation": Type.Optional(Type.Integer()),
 		"hasData": Type.Optional(Type.Boolean()),
 		"message": Type.Optional(Type.String()),
+		"epoch": Type.Optional(Type.Integer()),
+		"token": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof AcquireResponse.jsonSchema>): AcquireResponse {
-		return new this(json.status, json.generation, json.hasData, json.message);
+		return new this(json.status, json.generation, json.hasData, json.message, json.epoch, json.token);
 	}
 };
 export class AcquireRequest {
@@ -106,6 +271,7 @@ export class UploadRequest {
 		public instanceId: number,
 		public playerName: string,
 		public playerData: IpcPlayerData,
+		public destinationInstanceId?: number,
 	) {
 	}
 
@@ -113,10 +279,36 @@ export class UploadRequest {
 		"instanceId": Type.Integer(),
 		"playerName": Type.String(),
 		"playerData": jsonPlayerData,
+		"destinationInstanceId": Type.Optional(Type.Integer()),
 	});
 
 	static fromJSON(json: Static<typeof UploadRequest.jsonSchema>) {
-		return new this(json.instanceId, json.playerName, json.playerData as IpcPlayerData);
+		return new this(json.instanceId, json.playerName, json.playerData as IpcPlayerData, json.destinationInstanceId);
+	}
+}
+
+/** The stored data of a player switching to the instance it is sent to, so the join there needs no download */
+export class PrefetchEvent {
+	declare ["constructor"]: typeof PrefetchEvent;
+	static type = "event" as const;
+	static src = "controller" as const;
+	static dst = "instance" as const;
+	static plugin = "player_sync" as const;
+
+	constructor(
+		public playerData: IpcPlayerData,
+		/** The controller run that sent it, which a held copy must share with the acquire reply to be applied */
+		public token: string,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"playerData": jsonPlayerData,
+		"token": Type.String(),
+	});
+
+	static fromJSON(json: Static<typeof PrefetchEvent.jsonSchema>): PrefetchEvent {
+		return new this(json.playerData as IpcPlayerData, json.token);
 	}
 }
 
@@ -190,3 +382,160 @@ export class DatabaseStatsRequest {
 	static permission = "player_sync.inventory.view" as const;
 	static Response = DatabaseStatsResponse;
 }
+
+export class PlayerEntry {
+	constructor(
+		public name: string,
+		public generation?: number,
+		public size?: number,
+		public instanceId?: number,
+		public force?: string,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"name": Type.String(),
+		"generation": Type.Optional(Type.Integer()),
+		"size": Type.Optional(Type.Integer()),
+		"instanceId": Type.Optional(Type.Integer()),
+		"force": Type.Optional(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof PlayerEntry.jsonSchema>): PlayerEntry {
+		return new this(json.name, json.generation, json.size, json.instanceId, json.force);
+	}
+}
+
+export class ListPlayersRequest {
+	declare ["constructor"]: typeof ListPlayersRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static plugin = "player_sync" as const;
+	static permission = "player_sync.inventory.view" as const;
+	static Response = jsonArray(PlayerEntry);
+}
+
+export class GetPlayerDataResponse {
+	constructor(
+		public playerData: IpcPlayerData | null,
+		public instanceId?: number,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"playerData": Type.Union([jsonPlayerData, Type.Null()]),
+		"instanceId": Type.Optional(Type.Integer()),
+	});
+
+	static fromJSON(json: Static<typeof GetPlayerDataResponse.jsonSchema>): GetPlayerDataResponse {
+		return new this(json.playerData as IpcPlayerData | null, json.instanceId);
+	}
+}
+
+export class GetPlayerDataRequest {
+	declare ["constructor"]: typeof GetPlayerDataRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static plugin = "player_sync" as const;
+	static permission = "player_sync.inventory.view" as const;
+	static Response = GetPlayerDataResponse;
+
+	constructor(
+		public playerName: string,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"playerName": Type.String(),
+	});
+
+	static fromJSON(json: Static<typeof GetPlayerDataRequest.jsonSchema>) {
+		return new this(json.playerName);
+	}
+}
+
+export class SetPlayerDataResponse {
+	constructor(
+		public generation: number,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"generation": Type.Integer(),
+	});
+
+	static fromJSON(json: Static<typeof SetPlayerDataResponse.jsonSchema>): SetPlayerDataResponse {
+		return new this(json.generation);
+	}
+}
+
+export class SetPlayerDataRequest {
+	declare ["constructor"]: typeof SetPlayerDataRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static plugin = "player_sync" as const;
+	static permission = "player_sync.inventory.modify" as const;
+	static Response = SetPlayerDataResponse;
+
+	constructor(
+		public playerName: string,
+		public playerData: IpcPlayerData,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"playerName": Type.String(),
+		"playerData": jsonImportedPlayerData,
+	});
+
+	static fromJSON(json: Static<typeof SetPlayerDataRequest.jsonSchema>) {
+		return new this(json.playerName, json.playerData as IpcPlayerData);
+	}
+}
+
+export class DeletePlayerDataRequest {
+	declare ["constructor"]: typeof DeletePlayerDataRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static plugin = "player_sync" as const;
+	static permission = "player_sync.inventory.modify" as const;
+
+	constructor(
+		public playerName: string,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"playerName": Type.String(),
+	});
+
+	static fromJSON(json: Static<typeof DeletePlayerDataRequest.jsonSchema>) {
+		return new this(json.playerName);
+	}
+}
+
+export class ForceReleaseRequest {
+	declare ["constructor"]: typeof ForceReleaseRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static plugin = "player_sync" as const;
+	static permission = "player_sync.inventory.modify" as const;
+
+	constructor(
+		public playerName: string,
+	) {
+	}
+
+	static jsonSchema = Type.Object({
+		"playerName": Type.String(),
+	});
+
+	static fromJSON(json: Static<typeof ForceReleaseRequest.jsonSchema>) {
+		return new this(json.playerName);
+	}
+}

plugins/player_sync/module/download_inventory.lua

@@ -2,26 +2,38 @@ local progress_dialog = require("modules/player_sync/gui/progress_dialog")
 local ensure_character = require("modules/player_sync/ensure_character")
 local restore_position = require("modules/player_sync/restore_position")
 local get_script_data = require("modules/player_sync/get_script_data")
+local serialize = require("modules/player_sync/serialize")
+local compat = require("modules/clusterio/compat")
 
 
-local function download_inventory(player_name, data, number, total)
+--- @param components string? JSON of the components the instance sent the data with, given with the first part
+local function download_inventory(player_name, data, number, total, components)
+	local script_data = get_script_data()
+	local record = script_data.active_downloads[player_name]
 	local player = game.get_player(player_name)
-	if player == nil then
+	-- A prefetched copy (player_sync.prefetch_inventory) may be for a player who doesn't exist here yet
+	if player == nil and not (record and record.prefetch) then
 		rcon.print("Player " .. player_name .. " does not exist")
 		return
 	end
 
-	local script_data = get_script_data()
-	local record = script_data.active_downloads[player_name]
 	if record == nil then
 		rcon.print("No active download is in progress for " .. player_name)
 		return
 	end
+	-- For the linter, which can't narrow through the check above. A prefetch record never reaches a line that indexes
+	-- player while it is nil: it has no restart, never gets total 0, and shown below is false for it.
+	--- @cast player LuaPlayer
 
 	local player_record = script_data.players[player_name]
+	-- The components changed after the download was started
+	local sent_components = number == 1 and components and compat.json_to_table(components)
+	if sent_components and not serialize.same_components(record.components, sent_components) then
+		record.restart = true
+	end
 	if record.restart then
 		rcon.print("Restarting outdated download")
-		player_sync.initiate_inventory_download(player, player_record, record.generation)
+		player_sync.initiate_inventory_download(player, player_record, record.generation, record.epoch)
 		return
 	end
 
@@ -33,43 +45,59 @@ local function download_inventory(player_name, data, number, total)
 		progress_dialog.remove(player)
 
 		-- Give the player a character and pretend that's the synced player data
-		ensure_character(player)
+		if serialize.syncs(record.components, "controller") or player_record.local_controller then
+			ensure_character(player)
+		end
+		player_record.local_controller = nil
 
 		-- Restore player position and driving state
-		restore_position(record, player)
+		restore_position(player, record)
 
 		script_data.active_downloads[player_name] = nil
 		player_record.dirty = player.connected
+		player_record.sync_in_progress = false
 		player_record.sync = true
+		player_record.components = record.components
 		player_record.generation = record.generation
+		player_record.epoch = record.epoch
 		return
 	end
 
 	record.data = record.data .. data
+	-- A prefetched copy shows progress, and is applied, only once the acquire reply has confirmed it for the player
+	-- online here (player_sync.sync_player). A copy that finishes before then waits in finished_downloads for that reply.
+	local shown = not record.prefetch or (record.confirmed and player.connected)
 
 	-- Show progress in console
 	-- player.print("Downloaded "..(number + 1).."/"..total.." parts")
 	if number ~= total then
 		-- Show progress in GUI
-		progress_dialog.display(player, number, total)
+		if shown then
+			progress_dialog.display(player, number, total)
+		end
 
 		-- Update activity to indicate this session is still going
 		record.last_active = game.ticks_played
 		return
 	end
 
-	-- Download is complete
-	progress_dialog.remove(player)
+	-- Download is complete. The dialog goes even when this copy isn't shown now: it may have been shown before a leave
+	if player then
+		progress_dialog.remove(player)
+	end
 
 	-- Remove download session and store finished download
 	script_data.active_downloads[player_name] = nil
 	script_data.finished_downloads[player_name] = record
 
 	local ticks = game.ticks_played - record.started
-	player.print("Imported player data in " .. ticks .. " ticks")
-	log("Imported player data for " .. player_name .. " in " .. ticks .. " ticks")
+	if shown then
+		player.print("Imported player data in " .. ticks .. " ticks")
+	end
+	local kind = record.prefetch and "Prefetched" or "Imported"
+	log(kind .. " player data for " .. player_name .. " in " .. ticks .. " ticks")
 
-	if player.connected then
+	if shown and player.connected then
 		player_sync.finish_download(player, record)
 	end
 end

plugins/player_sync/module/get_script_data.lua

@@ -18,6 +18,7 @@ return function(no_early_return)
 	player_sync.players_waiting_for_acquire = player_sync.players_waiting_for_acquire or {}
 	player_sync.players_in_cutscene_to_sync = player_sync.players_in_cutscene_to_sync or {}
 	player_sync.failed_deserialization = player_sync.failed_deserialization or {}
+	player_sync.inventory_bonus_raises = player_sync.inventory_bonus_raises or {}
 
 	player_sync.active_downloads = player_sync.active_downloads or {}
 	player_sync.finished_downloads = player_sync.finished_downloads or {}

plugins/player_sync/module/gui/dialog_failed_download.lua

@@ -131,6 +131,8 @@ dialog_failed_download.events = {
 			player.gui.screen.dialog_failed_download.destroy()
 			player_sync.players[player.name].dirty = true
 			player_sync.players[player.name].sync = false
+			player_sync.players[player.name].sync_in_progress = false
+			player_sync.players[player.name].local_controller = nil
 
 		elseif event.element.name == "player_sync_failed_download_retry" then
 			-- Retry acquiring the player from the controller

plugins/player_sync/module/player_sync.lua

@@ -24,11 +24,25 @@ local function is_in_cutscene(player)
 	return player.controller_type == defines.controllers.cutscene
 end
 
+-- Remember the components a player's synced session uses, uploads are serialized with these
+local function start_sync(player_record, components)
+	player_record.sync = true
+	player_record.components = components
+end
+
+-- Remember the controller the player had before sync code replaced it, restored when the controller is not synced
+local function save_local_controller(player, player_record, components)
+	if not player_record.local_controller then
+		player_record.local_controller = serialize.serialize_local_controller(player, components)
+	end
+end
+
 -- Create player record for bookkeeping
 local function create_player(player, dirty)
 	local script_data = get_script_data(true) -- no early return to support loading from single player
 	script_data.players[player.name] = {
 		dirty = dirty, -- Player inventory has changes that should be persisted
+		sync_in_progress = false, -- Player data is being downloaded, the current state should not be persisted
 		sync = false, -- Player inventory is synced from the controller
 		generation = 0,
 	}
@@ -46,11 +60,77 @@ local function remove_player(player)
 	script_data.players[player.name] = nil
 end
 
+-- Remove the temporary inventory bonus a deserialized character got to fit its main inventory
+local function remove_inventory_bonus_raise(script_data, player_name)
+	local raises = script_data.inventory_bonus_raises
+	local raise = raises and raises[player_name]
+	if raise then
+		raises[player_name] = nil
+		local character = raise.character
+		if character.valid then
+			character.character_inventory_slots_bonus = character.character_inventory_slots_bonus - raise.bonus
+		end
+	end
+end
+
+-- Turn a synced player into a spectator while their player data is downloaded or applied, if this instance syncs
+-- inventories, and keep where their body stood in record for restore_position. The old body goes, with its crafting
+-- queue. Until the data is applied, the player's state isn't persisted.
+local function spectate(player, player_record, record)
+	-- Indicate the current state shouldn't be persisted
+	player_record.sync_in_progress = true
+
+	if player_record.sync and serialize.syncs(record.components, "inventories") then
+		-- Store original position to teleport back to. Only once per record: a rejoin while a prefetched copy is held
+		-- spectates again, and the spectator's position then isn't where the body stood
+		if record.position == nil then
+			if v2_remote_controller then
+				record.surface = player.physical_surface
+				record.position = player.physical_position
+			else
+				record.surface = player.surface
+				record.position = player.position
+			end
+			if player.driving then
+				record.vehicle = player.vehicle
+			end
+		end
+
+		save_local_controller(player, player_record, record.components)
+		local character = player.character
+		player.set_controller({ type = defines.controllers.spectator })
+		if character ~= nil then character.destroy() end
+	end
+end
+
 
 player_sync = {}
 -- This function is called by instance.js as /sc player_sync.download_inventory("danielv", Escaped JSON string, package_number, total_packages_count) with data from the controller
 player_sync.download_inventory = download_inventory
 
+-- This function is called by instance.js with the components synced on this instance
+function player_sync.set_components(data)
+	get_script_data().components = assert(compat.json_to_table(data))
+end
+
+-- For plugins that move players to another instance: remote.call("player_sync", "set_destination", player_name,
+-- instance_id) just before LuaPlayer.connect_to_server, with the destination's Clusterio instance id. It only records
+-- it; if the player then leaves to switch servers, the upload carries it and the controller sends the player's data
+-- on to that instance (README, "Sending player data ahead"). A later call replaces it, and every leave clears it.
+remote.add_interface("player_sync", {
+	set_destination = function(player_name, instance_id)
+		-- Integers only: table_to_json writes inf and nan bare, which the host can't parse, so the upload carrying it
+		-- would fail on every retry and the session would be lost (inf % 1 and nan % 1 are nan, never 0)
+		if type(instance_id) ~= "number" or instance_id % 1 ~= 0 then
+			return
+		end
+		local player_record = get_script_data().players[player_name]
+		if player_record then
+			player_record.destination = instance_id
+		end
+	end,
+})
+
 -- This function is called internally when a player leaves the game to
 -- serialize the player for upload.
 function player_sync.serialize_player(player, player_record)
@@ -78,8 +158,19 @@ function player_sync.serialize_player(player, player_record)
 	end
 
 	local failed_deserialization = get_script_data().failed_deserialization[player.name] or {}
-	local serialized_player = serialize.serialize_player(player, failed_deserialization)
+	local serialized_player = serialize.serialize_player(player, failed_deserialization, player_record.components)
 	serialized_player.generation = player_record.generation
+	-- Always sent so the controller can tell this instance supports epochs
+	serialized_player.epoch = player_record.epoch or 0
+
+	-- Leaving on the tick of a download the temporary bonus is still there and must not be uploaded. Removing it
+	-- from the character here would shrink the main inventory before the armor's bonus applies and spill items.
+	local raises = get_script_data().inventory_bonus_raises
+	local raise = raises and raises[player.name]
+	local character = serialized_player.character
+	if raise and character and raise.character == player.character then
+		character.character_inventory_slots_bonus = character.character_inventory_slots_bonus - raise.bonus
+	end
 
 	return serialized_player
 end
@@ -95,11 +186,14 @@ function player_sync.deserialize_player(player, finished_record)
 		return
 	end
 
-	-- Stash temporary inventory if it exists
-	local stashed_corpse
 	local script_data = get_script_data()
 	local player_record = script_data.players[player.name]
-	if player_record.dirty and player.character then
+	local local_controller = player_record.local_controller
+		or serialize.serialize_local_controller(player, finished_record.components)
+
+	-- Stash temporary inventory if it exists, without inventory sync the current inventory is the one to keep
+	local stashed_corpse
+	if serialize.syncs(finished_record.components, "inventories") and player_record.dirty and player.character then
 		local character = player.character
 		local surface = character.surface
 		local position = character.position
@@ -112,10 +206,22 @@ function player_sync.deserialize_player(player, finished_record)
 	-- Deserialize downloaded player data
 	local serialized_player = compat.json_to_table(finished_record.data)
 	assert(type(serialized_player) == "table", "wrong type for serialized_player")
-	script_data.failed_deserialization[player.name] = serialize.deserialize_player(player, serialized_player)
+	-- An earlier raise not removed yet is replaced, the character's inventory bonus is restored with the data
+	script_data.inventory_bonus_raises[player.name] = nil
+	local failed, spill, inventory_bonus_raise =
+		serialize.deserialize_player(player, serialized_player, finished_record.components, local_controller)
+	script_data.failed_deserialization[player.name] = failed
+	player_record.local_controller = nil
+	if inventory_bonus_raise and inventory_bonus_raise > 0 then
+		script_data.inventory_bonus_raises[player.name] = {
+			character = player.character,
+			bonus = inventory_bonus_raise,
+		}
+	end
 
 	-- Restore player position and driving state
 	restore_position(player, finished_record)
+	serialize.spill_items(player, spill)
 
 	-- Transfer items from stashed inventory
 	if stashed_corpse then
@@ -228,6 +334,19 @@ function player_sync.check_players_in_cutscene()
 	end
 end
 
+-- Whether a join should take the stored data: there is some, and this save hasn't synced the player in that epoch at
+-- that generation or newer with every component synced here now. A player without a record here has nothing to keep.
+local function download_needed(player_record, acquire_response, script_data)
+	return acquire_response.has_data
+		and (
+			not player_record
+			or not player_record.sync
+			or (player_record.epoch or 0) ~= (acquire_response.epoch or 0)
+			or acquire_response.generation > player_record.generation
+			or serialize.newly_synced(player_record.components, script_data.components)
+		)
+end
+
 function player_sync.sync_player(acquire_response)
 	local player = assert(game.get_player(acquire_response.player_name))
 
@@ -243,9 +362,48 @@ function player_sync.sync_player(acquire_response)
 	end
 
 	local script_data = get_script_data()
+	-- Epoch is bumped when an admin replaces or deletes the stored data
+	local epoch = acquire_response.epoch or 0
+	-- A prefetched copy (player_sync.prefetch_inventory) is used only if this reply acquires the player at the copy's
+	-- generation and epoch, from the controller run that sent it (token), and the copy was sent with the components
+	-- synced here now. The generation alone isn't enough: a controller that crashed before saving it restarts at the
+	-- last saved generation, and another instance's upload can then reuse the same number with other data. Otherwise
+	-- the copy goes, and the join carries on as if it had never come: today's download or failure dialog, never the
+	-- restart path below, which only sees a different generation or epoch. A copy this save doesn't need goes too,
+	-- keeping the local data: an upload from an instance with older data can take the generation this save already
+	-- holds.
+	-- A copy confirmed at an earlier join already took the body (spectate keeps where it stood), and the local data
+	-- went with it, so a join that drops that copy downloads.
+	local body_taken -- the dropped copy that took the body
+	for _, downloads in ipairs({ script_data.finished_downloads, script_data.active_downloads }) do
+		local record = downloads[acquire_response.player_name]
+		if record and record.prefetch and (not (
+			acquire_response.status == "acquired" and record.generation == acquire_response.generation
+			and (record.epoch or 0) == epoch and record.token == acquire_response.token
+			and serialize.same_components(record.components, script_data.components)
+		) or not download_needed(script_data.players[acquire_response.player_name], acquire_response, script_data)) then
+			if record.position ~= nil then
+				body_taken = record
+			end
+			downloads[acquire_response.player_name] = nil
+		end
+	end
+
 	local finished_record = script_data.finished_downloads[acquire_response.player_name]
 	if finished_record then
-		if acquire_response.status == "acquired" and finished_record.generation == acquire_response.generation then
+		if
+			acquire_response.status == "acquired"
+			and finished_record.generation == acquire_response.generation
+			and (finished_record.epoch or 0) == epoch
+		then
+			if finished_record.prefetch then
+				-- As a download does: the body left here goes, and the copy makes a new character where it stood
+				spectate(player, script_data.players[player.name], finished_record)
+				if player_sync.finish_download(player, finished_record) then
+					log("Applied prefetched player data for " .. player.name)
+				end
+				return
+			end
 			player_sync.finish_download(player, finished_record)
 			return
 		end
@@ -258,7 +416,18 @@ function player_sync.sync_player(acquire_response)
 	local download_record = script_data.active_downloads[acquire_response.player_name]
 	if download_record then
 		-- Ignore the response if it affirms the current active download
-		if acquire_response.status == "acquired" and download_record.generation == acquire_response.generation then
+		if
+			acquire_response.status == "acquired"
+			and download_record.generation == acquire_response.generation
+			and (download_record.epoch or 0) == epoch
+		then
+			if download_record.prefetch then
+				-- The copy is still arriving: spectate as a download does, and its last chunk applies it
+				spectate(player, script_data.players[player.name], download_record)
+				progress_dialog.display(player, 0, 1)
+				download_record.confirmed = true
+				return
+			end
 			game.print("Continuing active download")
 			return
 		end
@@ -279,6 +448,8 @@ function player_sync.sync_player(acquire_response)
 			-- download was started. Restart it.
 			game.print("Restarting download")
 			download_record.restart = true
+			download_record.generation = acquire_response.generation
+			download_record.epoch = epoch
 			return
 		end
 
@@ -291,23 +462,32 @@ function player_sync.sync_player(acquire_response)
 
 	local player_record = script_data.players[acquire_response.player_name]
 	if acquire_response.status ~= "acquired" then
-		if player_record.sync and not player_record.dirty then
-			if player.controller_type == defines.controllers.character then
+		if player_record.sync and (not player_record.dirty or player_record.sync_in_progress) then
+			if serialize.syncs(player_record.components, "inventories")
+				and player.controller_type == defines.controllers.character
+			then
+				save_local_controller(player, player_record, player_record.components)
 				local character = player.character
 				player.set_controller({ type = defines.controllers.spectator })
 				if character then character.destroy() end
 			end
 			player_record.sync = false
 		end
+		player_record.sync_in_progress = false
 		dialog_failed_download.create(player, acquire_response)
 		return
 	end
 
-	if
-		acquire_response.has_data
-		and (not player_record.sync or acquire_response.generation > player_record.generation)
-	then
-		player_sync.initiate_inventory_download(player, player_record, acquire_response.generation)
+	local stale = (player_record.epoch or 0) ~= epoch
+	if body_taken or download_needed(player_record, acquire_response, script_data) then
+		player_sync.initiate_inventory_download(player, player_record, acquire_response.generation, epoch)
+		if body_taken then
+			-- spectate kept the spectator's place, the body stood where the dropped copy kept
+			local download = script_data.active_downloads[player.name]
+			download.surface = body_taken.surface
+			download.position = body_taken.position
+			download.vehicle = body_taken.vehicle
+		end
 
 	else
 		if acquire_response.generation == 0 then
@@ -318,11 +498,21 @@ function player_sync.sync_player(acquire_response)
 			)
 		end
 
-		player_record.sync = true
+		start_sync(player_record, script_data.components)
 		player_record.dirty = true
+		player_record.sync_in_progress = false
+		-- Deleted data keeps its generation and epoch so the next upload is accepted as the new baseline
+		player_record.epoch = epoch
+		-- A copy from another epoch has nothing to do with the stored generation
+		if stale then
+			player_record.generation = acquire_response.generation
+		else
+			player_record.generation = math.max(player_record.generation, acquire_response.generation)
+		end
 	end
 end
 
+--- @return boolean? applied true when the player data was deserialized
 function player_sync.finish_download(player, finished_record)
 	local script_data = get_script_data()
 	script_data.finished_downloads[player.name] = nil
@@ -335,12 +525,16 @@ function player_sync.finish_download(player, finished_record)
 		-- Treat this as a temporary inventory so the incomplete state is not uploaded over the synced data
 		player_record.dirty = true
 		player_record.sync = false
+		player_record.sync_in_progress = false
 		return
 	end
 
 	player_record.dirty = true
-	player_record.sync = true
+	player_record.sync_in_progress = false
+	start_sync(player_record, finished_record.components)
 	player_record.generation = finished_record.generation
+	player_record.epoch = finished_record.epoch
+	return true
 end
 
 -- Download inventory from controller
@@ -359,6 +553,12 @@ player_sync.events[defines.events.on_player_joined_game] = function(event)
 	script_data.active_uploads[player.name] = nil
 end
 
+-- Raised for a restored armor after its equipment's inventory bonus applied, the temporary bonus is no longer needed
+player_sync.events[defines.events.on_player_armor_inventory_changed] = function(event)
+	local player = assert(game.get_player(event.player_index))
+	remove_inventory_bonus_raise(get_script_data(), player.name)
+end
+
 player_sync.on_nth_tick = {}
 player_sync.on_nth_tick[33] = function(event)
 	-- Periodically check players currently in the process of syncing
@@ -388,14 +588,24 @@ player_sync.events[defines.events.on_pre_player_left_game] = function(event)
 	local script_data = get_script_data()
 	local player = assert(game.get_player(event.player_index))
 	local player_record = script_data.players[player.name]
+	-- Recorded by set_destination. Every leave clears it; only an uploading switching_servers leave sends it on.
+	local destination = player_record.destination
+	player_record.destination = nil
 
-	-- If player has an active download release our acquisition but let the download continue
-	if script_data.active_downloads[player.name] then
+	local download = script_data.active_downloads[player.name]
+	if download and download.prefetch then
+		-- A prefetched copy is confirmed per join: the next join's acquire reply confirms it again. Until then it has
+		-- replaced nothing here, so the checks below decide the upload. One whose join took the body (spectate) left
+		-- sync_in_progress set, and uploads nothing.
+		download.confirmed = nil
+
+	elseif download then
+		-- If player has an active download release our acquisition but let the download continue
 		clusterio_api.send_json("player_sync_release", { player_name = player.name })
 		return
 	end
 
-	if not player_record.dirty or not player_record.sync then
+	if not player_record.dirty or not player_record.sync or player_record.sync_in_progress then
 		-- Nothing to upload, release the acquisition so other instances are not blocked
 		clusterio_api.send_json("player_sync_release", { player_name = player.name })
 		return
@@ -408,6 +618,11 @@ player_sync.events[defines.events.on_pre_player_left_game] = function(event)
 		player.print("ERROR: Serializing player data failed: " .. result)
 		return
 	end
+	-- The controller sends the stored data on to that instance while the client reconnects. In the serialized table,
+	-- so a retried upload (check_active_uploads) carries it too.
+	if destination and event.reason == defines.disconnect_reason.switching_servers then
+		result.destination_instance_id = destination
+	end
 	script_data.active_uploads[player.name] = {
 		serialized = result,
 		last_attempt = game.ticks_played,
@@ -441,20 +656,22 @@ function player_sync.confirm_upload(player_name, generation)
 	player_record.dirty = false
 end
 
-function player_sync.initiate_inventory_download(player, player_record, generation)
+function player_sync.initiate_inventory_download(player, player_record, generation, epoch)
 	player.print("Initiating inventory download...")
 	local record = {
 		started = game.ticks_played,
 		last_active = game.ticks_played,
 		generation = generation,
-		data = ""
+		epoch = epoch,
+		data = "",
 	}
 	local script_data = get_script_data()
+	record.components = script_data.components
 	script_data.active_downloads[player.name] = record
 
 	-- The plugin only sends back what differs from the current notification state
 	local recipe_notifications
-	if recipe_notifications_api then
+	if recipe_notifications_api and serialize.syncs(record.components, "recipe_notifications") then
 		recipe_notifications = serialize.serialize_crafting_notifications(player)
 	end
 
@@ -463,28 +680,7 @@ function player_sync.initiate_inventory_download(player, player_record, generati
 		recipe_notifications = recipe_notifications,
 	})
 
-	-- If this is a synced player turn them into a spectator while the
-	-- player data is downloading
-	if player_record.sync then
-		-- Store original position to teleport back to
-		if v2_remote_controller then
-			record.surface = player.physical_surface
-			record.position = player.physical_position
-		else
-			record.surface = player.surface
-			record.position = player.position
-		end
-		if player.driving then
-			record.vehicle = player.vehicle
-		end
-
-		local character = player.character
-		player.set_controller({ type = defines.controllers.spectator })
-		if character ~= nil then character.destroy() end
-
-		-- Indicate this state shouldn't be persisted
-		player_record.dirty = false
-	end
+	spectate(player, player_record, record)
 
 	progress_dialog.display(player, 0, 1)
 end
@@ -531,4 +727,73 @@ function player_sync.check_active_uploads()
 	end
 end
 
+-- Called by instance.ts as /sc player_sync.prefetch_inventory('name', generation, 'chunk', number, total, components,
+-- token, epoch) with a copy of a player's stored data that the controller sent ahead (PrefetchEvent) while they switch
+-- to this server. It holds the copy for a player who is offline here or doesn't exist yet. Every chunk carries the
+-- token and epoch, so chunks of another copy never land in this one. It prints only when it refuses or ignores a chunk,
+-- which stops the host sending the rest.
+--- @param components string? JSON of the components the instance stripped the copy by, given with the first chunk
+--- @param token string? The controller run that sent the copy
+--- @param epoch integer? The stored data's epoch, bumped when an admin replaces or deletes it
+function player_sync.prefetch_inventory(player_name, generation, data, number, total, components, token, epoch)
+	epoch = epoch or 0
+	local script_data = get_script_data()
+	if number == 1 then
+		local player = game.get_player(player_name)
+		local active = script_data.active_downloads[player_name]
+		local finished = script_data.finished_downloads[player_name]
+		if player and player.connected then
+			rcon.print("Refused prefetch for " .. player_name .. ": online here, their join downloads")
+			return
+		end
+		if active and not active.prefetch then
+			-- download_inventory has no generation check, so its remaining chunks would land in the copy
+			rcon.print("Refused prefetch for " .. player_name .. ": a download is in progress")
+			return
+		end
+		-- Two copies can reach here out of order (applyRecipeNotificationDelta awaits zlib). A held finished copy of the
+		-- same generation is replaced: the newer copy of the same number is what the controller holds now. A copy
+		-- with another token came from an earlier controller run, which may have crashed before saving that
+		-- generation, and one with another epoch from the other side of an admin's import or delete, which can start
+		-- the generations over. Either way the generations don't compare, and the arriving copy replaces the held one
+		local function comparable(held)
+			return held.token == token and (held.epoch or 0) == epoch
+		end
+		local active_newer = active and comparable(active) and active.generation >= generation
+		local finished_newer = finished and comparable(finished) and finished.generation > generation
+		if active_newer or finished_newer then
+			rcon.print("Refused prefetch for " .. player_name .. ": generation " .. generation .. " is not newer")
+			return
+		end
+		-- A held copy (or offline-finished download) whose join took the body knows where it stood, and that the body
+		-- is gone (sync_player's body_taken): the copy replacing it keeps both
+		local held = active or finished
+		script_data.finished_downloads[player_name] = nil
+		script_data.active_downloads[player_name] = {
+			started = game.ticks_played,
+			last_active = game.ticks_played,
+			generation = generation,
+			data = "",
+			prefetch = true,
+			token = token,
+			epoch = epoch,
+			components = components and compat.json_to_table(components),
+			surface = held and held.surface,
+			position = held and held.position,
+			vehicle = held and held.vehicle,
+		}
+	end
+
+	local record = script_data.active_downloads[player_name]
+	if not (
+		record and record.prefetch and record.generation == generation and record.token == token
+		and (record.epoch or 0) == epoch
+	) then
+		-- Refused at chunk 1, dropped by the acquire reply or the inactivity check, or replaced by a newer copy
+		rcon.print("Ignored prefetch chunk " .. number .. " for " .. player_name)
+		return
+	end
+	download_inventory(player_name, data, number, total)
+end
+
 return player_sync

plugins/player_sync/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-player_sync",
-	"version": "2.0.1",
+	"version": "2.0.3",
 	"description": "Clusterio plugin forwarding between Factorio servers",
 	"homepage": "https://github.com/clusterio/clusterio#readme",
 	"author": "Danielv <danielv@danielv.no>",
@@ -35,6 +35,7 @@
 
 	"peerDependencies": {
 		"@clusterio/controller": "workspace:^",
+		"@clusterio/ctl": "workspace:^",
 		"@clusterio/host": "workspace:^",
 		"@clusterio/lib": "workspace:^"
 	},
@@ -50,7 +51,9 @@
 		"@sinclair/typebox": "catalog:"
 	},
 	"devDependencies": {
+		"@ant-design/icons": "^5.6.1",
 		"@clusterio/controller": "workspace:^",
+		"@clusterio/ctl": "workspace:^",
 		"@clusterio/host": "workspace:^",
 		"@clusterio/lib": "workspace:^",
 		"@clusterio/web_ui": "workspace:^",

plugins/player_sync/player_data.ts

@@ -0,0 +1,61 @@
+import type { IpcPlayerData } from "./messages.js";
+
+// See packages/host/modules/clusterio/serialize.lua for the item stack format
+type SerializedItem = {
+	n?: string,
+	c?: number,
+	q?: string,
+	r?: number,
+	e?: string,
+};
+
+// Factorio serializes an empty table as {}
+type SerializedInventory = { i?: SerializedItem[] | Record<string, SerializedItem> };
+
+export type ItemSummary = {
+	name: string,
+	quality?: string,
+	count: number,
+	/** Length of the export string for blueprints and other exportable items */
+	exportSize?: number,
+};
+
+export type InventorySummary = {
+	name: string,
+	items: ItemSummary[],
+};
+
+export function summarizeInventory(inventory: SerializedInventory): ItemSummary[] {
+	let items: ItemSummary[] = [];
+	let stacks = new Map<string, ItemSummary>();
+	for (let entry of Object.values(inventory?.i ?? [])) {
+		// Uploads are not validated as strictly as imports
+		if (typeof entry !== "object" || entry === null) {
+			continue;
+		}
+		let repeat = Number.isInteger(entry.r) ? entry.r! + 1 : 1;
+		if (typeof entry.e === "string") {
+			items.push({ name: "exported item", count: repeat, exportSize: entry.e.length });
+			continue;
+		}
+		if (typeof entry.n !== "string") {
+			continue;
+		}
+		let key = `${entry.n}\0${entry.q ?? ""}`;
+		let stack = stacks.get(key);
+		if (!stack) {
+			stack = { name: entry.n, quality: entry.q, count: 0 };
+			stacks.set(key, stack);
+			items.push(stack);
+		}
+		stack.count += (typeof entry.c === "number" ? entry.c : 0) * repeat;
+	}
+	return items;
+}
+
+export function summarizePlayerInventories(playerData: IpcPlayerData): InventorySummary[] {
+	let inventories = playerData.character?.inventories ?? playerData.inventories ?? {};
+	return Object.entries(inventories).map(
+		([name, inventory]) => ({ name, items: summarizeInventory(inventory as SerializedInventory) })
+	);
+}

plugins/player_sync/test/components.js

@@ -0,0 +1,161 @@
+import assert from "node:assert/strict";
+
+import {
+	components, enabledComponents, mergePlayerData, stripPlayerData, scenarioComponents,
+} from "../dist/node/components.js";
+
+const allComponents = Object.keys(components);
+
+function configWith(disabled = []) {
+	return { get: name => !disabled.includes(name.replace("player_sync.sync_", "")) };
+}
+
+const character = { inventories: { main: [] } };
+const stored = {
+	generation: 1,
+	name: "test",
+	controller: "god",
+	cheat_mode: true,
+	force: "player",
+	color: [1, 0, 0],
+	chat_color: [1, 0, 0],
+	tag: "stored",
+	inventories: { main: ["stored god item"] },
+	quick_bar: ["stored-item"],
+	recipe_notifications: "stored",
+};
+const uploaded = {
+	generation: 2,
+	name: "test",
+	controller: "character",
+	cheat_mode: false,
+	force: "enemy",
+	color: [0, 1, 0],
+	chat_color: [0, 1, 0],
+	tag: "uploaded",
+	character,
+	crafting_queue: [],
+	personal_logistic_slots: [],
+	quick_bar: ["uploaded-item"],
+	recipe_notifications: "uploaded",
+};
+
+describe("player_sync components", function() {
+	describe("enabledComponents()", function() {
+		it("should enable every component by default", function() {
+			assert.deepEqual([...enabledComponents(configWith())], allComponents);
+		});
+		it("should leave out disabled components", function() {
+			const enabled = enabledComponents(configWith(["force", "quick_bar"]));
+			assert.equal(enabled.has("force"), false);
+			assert.equal(enabled.has("quick_bar"), false);
+			assert.equal(enabled.size, allComponents.length - 2);
+		});
+		it("should sync the controller without the inventories", function() {
+			const enabled = enabledComponents(configWith(["inventories"]));
+			assert.equal(enabled.has("controller"), true);
+		});
+	});
+
+	describe("stripPlayerData()", function() {
+		it("should keep everything when all components are synced", function() {
+			const playerData = structuredClone(uploaded);
+			stripPlayerData(playerData, new Set(allComponents));
+			assert.deepEqual(playerData, uploaded);
+		});
+		it("should remove the fields of disabled components", function() {
+			const playerData = structuredClone(uploaded);
+			stripPlayerData(playerData, new Set(allComponents.filter(c => !["appearance", "inventories"].includes(c))));
+			for (const key of ["color", "chat_color", "tag", "character", "crafting_queue"]) {
+				assert.equal(key in playerData, false, key);
+			}
+			assert.equal(playerData.force, "enemy");
+		});
+		it("should remove the god inventory with the inventories", function() {
+			const playerData = structuredClone(stored);
+			stripPlayerData(playerData, new Set(allComponents.filter(c => c !== "inventories")));
+			assert.equal("inventories" in playerData, false);
+			assert.equal(playerData.controller, "god");
+		});
+	});
+
+	describe("scenarioComponents()", function() {
+		it("should sync everything when no components are sent", function() {
+			assert.deepEqual([...scenarioComponents(undefined)], allComponents);
+		});
+		it("should sync the components marked true", function() {
+			const enabled = scenarioComponents({ force: true, quick_bar: true, settings: false });
+			assert.deepEqual([...enabled], ["force", "quick_bar"]);
+		});
+		it("should sync nothing for an empty table encoded as an array", function() {
+			assert.equal(scenarioComponents([]).size, 0);
+		});
+		it("should sync the controller without the inventories", function() {
+			assert.deepEqual([...scenarioComponents({ controller: true })], ["controller"]);
+		});
+	});
+
+	describe("mergePlayerData()", function() {
+		function withComponents(data, disabled) {
+			const enabled = allComponents.filter(c => !disabled.includes(c));
+			return { ...data, components: Object.fromEntries(enabled.map(c => [c, true])) };
+		}
+
+		it("should store the upload as is when it has no components", function() {
+			assert.deepEqual(mergePlayerData(stored, uploaded), uploaded);
+		});
+		it("should not store the components of the upload", function() {
+			assert.deepEqual(mergePlayerData(stored, withComponents(uploaded, [])), uploaded);
+		});
+		it("should keep the stored fields of disabled components", function() {
+			const merged = mergePlayerData(stored, withComponents(uploaded, ["force", "appearance"]));
+			assert.equal(merged.force, "player");
+			assert.deepEqual(merged.color, [1, 0, 0]);
+			assert.equal(merged.tag, "stored");
+			assert.deepEqual(merged.quick_bar, ["uploaded-item"]);
+			assert.equal(merged.generation, 2);
+		});
+		it("should drop disabled components without stored data", function() {
+			const merged = mergePlayerData(undefined, withComponents(uploaded, ["force"]));
+			assert.equal("force" in merged, false);
+			assert.equal(merged.tag, "uploaded");
+		});
+		it("should take the items but not the controller when the controller is not synced", function() {
+			const merged = mergePlayerData(stored, withComponents(uploaded, ["controller"]));
+			assert.equal(merged.controller, "god");
+			assert.equal(merged.cheat_mode, true);
+			assert.equal(merged.character, character);
+			assert.equal("inventories" in merged, false, "the god inventory moved to the character");
+		});
+		it("should take the items of a player without a character", function() {
+			const storedCharacter = {
+				...stored, character, crafting_queue: ["stored"], personal_logistic_slots: [{ name: "stored" }],
+			};
+			const god = { ...uploaded, controller: "god", inventories: { main: ["new god item"] } };
+			delete god.character;
+			delete god.crafting_queue;
+			delete god.personal_logistic_slots;
+			const merged = mergePlayerData(storedCharacter, withComponents(god, ["controller"]));
+			assert.equal("character" in merged, false);
+			assert.equal("crafting_queue" in merged, false);
+			assert.deepEqual(merged.inventories, { main: ["new god item"] });
+			assert.deepEqual(merged.personal_logistic_slots, [{ name: "stored" }], "nowhere to hold logistics");
+		});
+		it("should drop logistic requests a character uploaded without", function() {
+			const storedSlots = { ...stored, personal_logistic_slots: [{ name: "stored" }] };
+			const noSlots = { ...uploaded };
+			delete noSlots.personal_logistic_slots;
+			const merged = mergePlayerData(storedSlots, withComponents(noSlots, []));
+			assert.equal("personal_logistic_slots" in merged, false);
+		});
+		it("should take cleared logistic requests from an upload without inventories", function() {
+			const storedSlots = { ...stored, personal_logistic_slots: [{ name: "stored" }] };
+			// Factorio 1.1 serializes a character without requests as an empty table
+			const cleared = { ...uploaded, personal_logistic_slots: {} };
+			delete cleared.character;
+			delete cleared.crafting_queue;
+			const merged = mergePlayerData(storedSlots, withComponents(cleared, ["inventories"]));
+			assert.deepEqual(merged.personal_logistic_slots, {});
+		});
+	});
+});

plugins/player_sync/test/controller.js

@@ -3,10 +3,12 @@ import fs from "node:fs/promises";
 import path from "node:path";
 
 import * as lib from "@clusterio/lib";
+import { InstanceRecord } from "@clusterio/controller";
 import * as mock from "../../../test/mock.js";
 import controllerEntrypoint from "../dist/node/controller.js";
+import { components as allComponents } from "../dist/node/components.js";
 import { plugin as info } from "../dist/node/index.js";
-import { DatabaseStatsRequest } from "../dist/node/messages.js";
+import * as msg from "../dist/node/messages.js";
 
 describe("player_sync controller", function() {
 	const databaseDir = path.join("temp", "test", "player_sync");
@@ -35,7 +37,7 @@ describe("player_sync controller", function() {
 		await fs.writeFile(path.join(databaseDir, "inventories.json"), JSON.stringify([["test", playerData]]));
 
 		const controller = await load();
-		const stats = await mock.getHandler(controller, DatabaseStatsRequest)();
+		const stats = await mock.getHandler(controller, msg.DatabaseStatsRequest)();
 		assert.equal(stats.databaseEntries, 1);
 		assert.equal(stats.largestEntry.name, "test");
 		await assert.rejects(fs.access(path.join(databaseDir, "inventories.json")), { code: "ENOENT" });
@@ -46,8 +48,435 @@ describe("player_sync controller", function() {
 		await fs.writeFile(path.join(databaseDir, "player_sync.json"), JSON.stringify([["new", { name: "new" }]]));
 
 		const controller = await load();
-		const stats = await mock.getHandler(controller, DatabaseStatsRequest)();
+		const stats = await mock.getHandler(controller, msg.DatabaseStatsRequest)();
 		assert.equal(stats.databaseEntries, 1);
 		assert.equal(stats.largestEntry.name, "new");
 	});
+
+	describe("UploadRequest", function() {
+		let controller;
+		let disabled;
+		beforeEach(async function() {
+			disabled = [];
+			const stored = { generation: 1, name: "test", force: "player", tag: "stored" };
+			await fs.writeFile(path.join(databaseDir, "player_sync.json"), JSON.stringify([["test", stored]]));
+			controller = await load();
+			controller.instances = new Map([[1, {
+				config: {
+					get: name => {
+						if (name === "instance.name") { return "test instance"; }
+						return !disabled.includes(name.replace("player_sync.sync_", ""));
+					},
+				},
+			}]]);
+		});
+
+		async function upload(playerData) {
+			await mock.getHandler(controller, msg.UploadRequest)({ instanceId: 1, playerName: "test", playerData });
+			return (await mock.getHandler(controller, msg.DownloadRequest)({ instanceId: 1, playerName: "test" }))
+				.playerData;
+		}
+
+		it("should store the upload as is when it has no components", async function() {
+			const playerData = { generation: 2, name: "test", force: "enemy", tag: "uploaded" };
+			assert.deepEqual(await upload(playerData), playerData);
+		});
+		it("should merge by the payload's components and ignore the instance config", async function() {
+			disabled = ["appearance"];
+			const components = Object.fromEntries(Object.keys(allComponents).map(c => [c, c !== "force"]));
+			assert.deepEqual(
+				await upload({ generation: 2, name: "test", tag: "uploaded", components }),
+				{ generation: 2, name: "test", force: "player", tag: "uploaded" },
+			);
+		});
+	});
+
+	describe("player data tools", function() {
+		function playerData(name, generation, extra = {}) {
+			return { generation, epoch: 0, name, controller: "character", force: "player", ...extra };
+		}
+
+		let controller;
+		function handler(Class) {
+			return mock.getHandler(controller, Class);
+		}
+		async function acquire(instanceId, playerName) {
+			const response = await handler(msg.AcquireRequest)(new msg.AcquireRequest(instanceId, playerName));
+			return response.status === "acquired";
+		}
+		async function saved() {
+			await controller.hooks.save.invoke();
+			return new Map(JSON.parse(await fs.readFile(path.join(databaseDir, "player_sync.json"), "utf8")));
+		}
+		async function upload(instanceId, data) {
+			await handler(msg.UploadRequest)(new msg.UploadRequest(instanceId, data.name, data));
+		}
+		async function acquireResponse(instanceId, playerName) {
+			const response = await handler(msg.AcquireRequest)(new msg.AcquireRequest(instanceId, playerName));
+			// Random per controller run, the run token tests cover it
+			response.token = undefined;
+			return response;
+		}
+
+		async function loadWithInstances() {
+			controller = await mock.loadControllerPlugin(async (context) => {
+				context.controller.mockConfigEntries.set("controller.database_directory", databaseDir);
+				// Negative so locks expire as soon as their instance stops
+				context.controller.mockConfigEntries.set("player_sync.player_lock_timeout", -1);
+				await controllerEntrypoint(context);
+			}, info);
+			for (const [id, name, status] of [[1, "Running", "running"], [2, "Stopped", "stopped"]]) {
+				controller.instances.records.set(id, new InstanceRecord(
+					new lib.InstanceConfig("controller", { "instance.id": id, "instance.name": name }), status,
+				));
+			}
+		}
+
+		beforeEach(async function() {
+			await loadWithInstances();
+
+			await handler(msg.UploadRequest)(new msg.UploadRequest(1, "stored", playerData("stored", 3)));
+			assert(await acquire(1, "stored"));
+			assert(await acquire(1, "online"));
+			assert(await acquire(2, "expired"));
+			await controller.hooks.instanceStatusChanged.invoke(controller.instances.get(2), "running");
+			assert(await acquire(99, "gone"));
+			await saved();
+		});
+
+		describe("ListPlayersRequest", function() {
+			it("should list stored data and live acquisitions", async function() {
+				const entries = await handler(msg.ListPlayersRequest)(new msg.ListPlayersRequest());
+				entries.sort((a, b) => a.name.localeCompare(b.name));
+				assert.deepEqual(entries, [
+					new msg.PlayerEntry("online", undefined, undefined, 1),
+					new msg.PlayerEntry("stored", 3, JSON.stringify(playerData("stored", 3)).length, 1, "player"),
+				]);
+			});
+		});
+
+		describe("GetPlayerDataRequest", function() {
+			async function get(name) {
+				return await handler(msg.GetPlayerDataRequest)(new msg.GetPlayerDataRequest(name));
+			}
+			it("should return stored data and acquisition", async function() {
+				assert.deepEqual(await get("stored"), new msg.GetPlayerDataResponse(playerData("stored", 3), 1));
+			});
+			it("should return null for unknown players", async function() {
+				assert.deepEqual(await get("unknown"), new msg.GetPlayerDataResponse(null, undefined));
+			});
+			it("should ignore expired and dangling acquisitions", async function() {
+				for (const name of ["expired", "gone"]) {
+					assert.equal((await get(name)).instanceId, undefined);
+				}
+			});
+		});
+
+		const acquiredError = new lib.RequestError(
+			"stored is currently acquired by Running, have the player leave or release the lock first"
+		);
+
+		describe("SetPlayerDataRequest", function() {
+			async function set(name, data) {
+				return await handler(msg.SetPlayerDataRequest)(new msg.SetPlayerDataRequest(name, data));
+			}
+			async function release(name) {
+				await handler(msg.ForceReleaseRequest)(new msg.ForceReleaseRequest(name));
+			}
+			it("should store data with a bumped generation and epoch", async function() {
+				await release("stored");
+				assert.deepEqual(await set("stored", playerData("other", 1)), new msg.SetPlayerDataResponse(4));
+				assert.deepEqual((await saved()).get("stored"), playerData("stored", 4, { epoch: 1 }));
+			});
+			it("should invalidate unconfirmed uploads from instances", async function() {
+				// Instance 1 left with generation 4 but the upload never reached the controller
+				await release("stored");
+				await set("stored", playerData("stored", 0));
+				await upload(1, playerData("stored", 4));
+				assert.deepEqual((await saved()).get("stored"), playerData("stored", 4, { epoch: 1 }));
+				assert.deepEqual(
+					await acquireResponse(1, "stored"), new msg.AcquireResponse("acquired", 4, true, undefined, 1),
+				);
+			});
+			it("should ignore the epoch in imported data", async function() {
+				await release("stored");
+				await set("stored", playerData("stored", 0, { epoch: 7 }));
+				assert.equal((await saved()).get("stored").epoch, 1);
+			});
+			it("should keep a higher provided generation", async function() {
+				await release("stored");
+				assert.equal((await set("stored", playerData("stored", 10))).generation, 10);
+			});
+			it("should store data for new players", async function() {
+				assert.equal((await set("new", playerData("new", 0))).generation, 1);
+				assert.equal((await saved()).get("new").generation, 1);
+			});
+			it("should reject when the player is acquired", async function() {
+				await assert.rejects(set("stored", playerData("stored", 3)), acquiredError);
+				assert.equal((await saved()).get("stored").generation, 3);
+			});
+
+			// The link validates requests before the handler runs
+			const fromJSON = lib.Link.requestFromJSON(msg.SetPlayerDataRequest, "SetPlayerDataRequest");
+			async function setJSON(name, data) {
+				const request = fromJSON({ playerName: name, playerData: data });
+				return await handler(msg.SetPlayerDataRequest)(request);
+			}
+			it("should keep the stored data when an import fails validation", async function() {
+				await release("stored");
+				await setJSON("stored", playerData("stored", 0));
+				const before = (await saved()).get("stored");
+				assert.equal(before.epoch, 1);
+				const badFilter = { value: { name: "iron-plate" }, min: "many" };
+				for (const extra of [
+					{ crafting_queue: {} },
+					{ deleted: false },
+					{ deleted: true },
+					{ controller: "cutscene" },
+					{ inventories: { main: { i: [{ e: null }] } } },
+					{ personal_logistic_slots: {
+						enabled: true,
+						trash_not_requested: false,
+						sections: [{ group: "", active: true, multiplier: 1, filters: [badFilter] }],
+					} },
+				]) {
+					await assert.rejects(
+						setJSON("stored", playerData("stored", 0, extra)),
+						{ message: "Request SetPlayerDataRequest failed validation" },
+					);
+				}
+				assert.deepEqual((await saved()).get("stored"), before);
+			});
+			it("should start over from imported generations past the limit", async function() {
+				await release("stored");
+				assert.equal((await setJSON("stored", playerData("stored", 2 ** 53))).generation, 4);
+				await upload(1, playerData("stored", 5, { epoch: 1 }));
+				assert.deepEqual((await saved()).get("stored"), playerData("stored", 5, { epoch: 1 }));
+			});
+			for (const deleted of [false, true]) {
+				it(`should recover ${deleted ? "deleted" : "stored"} data past the generation limit`, async function() {
+					// Could be stored by uploads or by imports from before the limit
+					for (const generation of [msg.generationLimit, 2 ** 53]) {
+						const stuck = { ...playerData("stored", generation), ...deleted ? { deleted } : {} };
+						const entries = [["stored", stuck]];
+						await fs.writeFile(path.join(databaseDir, "player_sync.json"), JSON.stringify(entries));
+						await loadWithInstances();
+						// Importing the exported data as is
+						assert.equal((await setJSON("stored", playerData("stored", generation))).generation, 1);
+						await upload(1, playerData("stored", 2, { epoch: 1 }));
+						assert.deepEqual((await saved()).get("stored"), playerData("stored", 2, { epoch: 1 }));
+					}
+				});
+			}
+			it("should keep counting up below the generation limit", async function() {
+				const stored = playerData("stored", msg.generationLimit - 1);
+				await fs.writeFile(path.join(databaseDir, "player_sync.json"), JSON.stringify([["stored", stored]]));
+				await loadWithInstances();
+				const response = await setJSON("stored", playerData("stored", 0));
+				assert.equal(response.generation, msg.generationLimit);
+			});
+			it("should only load entries marked deleted: true as tombstones", async function() {
+				// Imports from before the marker was reserved could store deleted: false
+				const data = playerData("stored", 4, { epoch: 1 });
+				const entries = [["stored", { ...data, deleted: false }]];
+				await fs.writeFile(path.join(databaseDir, "player_sync.json"), JSON.stringify(entries));
+				controller = await load();
+				const response = await handler(msg.GetPlayerDataRequest)(new msg.GetPlayerDataRequest("stored"));
+				assert.deepEqual(response.playerData, data);
+			});
+		});
+
+		describe("DeletePlayerDataRequest", function() {
+			async function del(name) {
+				await handler(msg.DeletePlayerDataRequest)(new msg.DeletePlayerDataRequest(name));
+			}
+			it("should delete stored data", async function() {
+				await handler(msg.ForceReleaseRequest)(new msg.ForceReleaseRequest("stored"));
+				await del("stored");
+				const response = await handler(msg.GetPlayerDataRequest)(new msg.GetPlayerDataRequest("stored"));
+				assert.equal(response.playerData, null);
+			});
+			it("should keep the generation and bump the epoch", async function() {
+				await handler(msg.ForceReleaseRequest)(new msg.ForceReleaseRequest("stored"));
+				await del("stored");
+				assert.deepEqual(
+					await acquireResponse(2, "stored"), new msg.AcquireResponse("acquired", 3, false, undefined, 1),
+				);
+				// In the same file as the player data so the delete and its tombstone are written together
+				assert.deepEqual((await saved()).get("stored"), { generation: 3, epoch: 1, deleted: true });
+
+				controller = await load();
+				assert.deepEqual(
+					await acquireResponse(2, "stored"), new msg.AcquireResponse("acquired", 3, false, undefined, 1),
+				);
+			});
+			it("should accept the next upload from the new epoch only", async function() {
+				await handler(msg.ForceReleaseRequest)(new msg.ForceReleaseRequest("stored"));
+				await del("stored");
+				await upload(1, playerData("stored", 11));
+				assert((await saved()).get("stored").deleted);
+				await upload(2, playerData("stored", 4, { epoch: 1 }));
+				assert.deepEqual((await saved()).get("stored"), playerData("stored", 4, { epoch: 1 }));
+
+				// The other instance downloads the baseline instead of uploading its stale copy
+				await upload(1, playerData("stored", 11));
+				assert.equal((await saved()).get("stored").generation, 4);
+			});
+			it("should reject unknown players", async function() {
+				await assert.rejects(del("unknown"), new lib.RequestError("No player data stored for unknown"));
+			});
+			it("should reject when the player is acquired", async function() {
+				await assert.rejects(del("stored"), acquiredError);
+				assert.deepEqual((await saved()).get("stored"), playerData("stored", 3));
+			});
+		});
+
+		describe("uploads without an epoch", function() {
+			// What instances running a version from before epochs send
+			function legacyData(generation) {
+				const data = playerData("stored", generation);
+				delete data.epoch;
+				return data;
+			}
+			beforeEach(async function() {
+				await handler(msg.ForceReleaseRequest)(new msg.ForceReleaseRequest("stored"));
+			});
+			it("should store them by generation", async function() {
+				await upload(1, legacyData(3));
+				await upload(1, legacyData(4));
+				assert.deepEqual((await saved()).get("stored"), legacyData(4));
+			});
+			it("should keep accepting them after an import", async function() {
+				const request = new msg.SetPlayerDataRequest("stored", playerData("stored", 0));
+				await handler(msg.SetPlayerDataRequest)(request);
+				await upload(1, legacyData(4));
+				assert.equal((await saved()).get("stored").epoch, 1);
+				await upload(1, legacyData(5));
+				assert.deepEqual((await saved()).get("stored"), playerData("stored", 5, { epoch: 1 }));
+			});
+			it("should accept any generation after a delete", async function() {
+				await handler(msg.DeletePlayerDataRequest)(new msg.DeletePlayerDataRequest("stored"));
+				await upload(1, legacyData(1));
+				assert.deepEqual((await saved()).get("stored"), playerData("stored", 1, { epoch: 1 }));
+			});
+		});
+
+		describe("ForceReleaseRequest", function() {
+			async function release(name) {
+				await handler(msg.ForceReleaseRequest)(new msg.ForceReleaseRequest(name));
+			}
+			it("should remove the acquisition", async function() {
+				await release("online");
+				assert(await acquire(2, "online"));
+			});
+			it("should reject players that are not acquired", async function() {
+				await assert.rejects(
+					release("unknown"), new lib.RequestError("unknown is not acquired by any instance"),
+				);
+			});
+		});
+
+		describe("AcquireRequest", function() {
+			it("should refuse players held by another instance", async function() {
+				assert(!await acquire(2, "online"));
+				assert(await acquire(1, "online"));
+			});
+			it("should take over expired and dangling acquisitions", async function() {
+				assert(await acquire(1, "expired"));
+				assert(await acquire(1, "gone"));
+				const response = await handler(msg.GetPlayerDataRequest)(new msg.GetPlayerDataRequest("gone"));
+				assert.equal(response.instanceId, 1);
+			});
+		});
+	});
+
+	describe("UploadRequest with destinationInstanceId", function() {
+		// Instances 1-3 run player_sync, 4 is stopped, 5 runs with player_sync off; 1 is the source in every case
+		async function setup() {
+			const controller = await load();
+			for (const [id, status, loadPlugin] of [
+				[1, "running", true], [2, "running", true], [3, "running", true],
+				[4, "stopped", true], [5, "running", false],
+			]) {
+				const fields = { "instance.name": `instance ${id}`, "player_sync.load_plugin": loadPlugin };
+				controller.instances.records.set(id, { id, status, config: { get: field => fields[field] } });
+			}
+			const sent = [];
+			controller.sendTo = (address, message) => { sent.push({ address, message }); };
+			return { controller, sent, upload: mock.getHandler(controller, msg.UploadRequest) };
+		}
+		const data = generation => ({ name: "test", generation });
+		async function stored(controller) {
+			const download = mock.getHandler(controller, msg.DownloadRequest);
+			return (await download(new msg.DownloadRequest(2, "test"))).playerData;
+		}
+
+		it("should send the stored data to a running destination with player_sync on", async function() {
+			const { sent, upload } = await setup();
+			await upload(new msg.UploadRequest(1, "test", data(1), 2));
+			assert.equal(sent.length, 1);
+			assert.deepEqual(sent[0].address, { instanceId: 2 });
+			assert.ok(sent[0].message instanceof msg.PrefetchEvent);
+			assert.deepEqual(sent[0].message.playerData, data(1));
+		});
+		it("should tag the prefetch and acquire reply with one token per controller run", async function() {
+			const { controller, sent, upload } = await setup();
+			await upload(new msg.UploadRequest(1, "test", data(1), 2));
+			const acquire = await mock.getHandler(controller, msg.AcquireRequest)(new msg.AcquireRequest(2, "test"));
+			assert.equal(acquire.status, "acquired");
+			assert.ok(sent[0].message.token);
+			assert.equal(acquire.token, sent[0].message.token);
+
+			const again = await setup();
+			await again.upload(new msg.UploadRequest(1, "test", data(1), 2));
+			assert.notEqual(again.sent[0].message.token, sent[0].message.token);
+		});
+		it("should not need a token on a busy reply", async function() {
+			const { controller } = await setup();
+			const acquire = mock.getHandler(controller, msg.AcquireRequest);
+			await acquire(new msg.AcquireRequest(3, "test"));
+			assert.equal((await acquire(new msg.AcquireRequest(2, "test"))).token, undefined);
+		});
+		it("should send the stored copy, with the parts the upload's components left out", async function() {
+			const { sent, upload } = await setup();
+			await upload(new msg.UploadRequest(1, "test", { ...data(1), force: "player", tag: "first" }));
+			const components = Object.fromEntries(Object.keys(allComponents).map(c => [c, c !== "force"]));
+			await upload(new msg.UploadRequest(1, "test", { ...data(2), tag: "second", components }, 2));
+			assert.equal(sent.length, 1);
+			assert.deepEqual(sent[0].message.playerData, { ...data(2), force: "player", tag: "second" });
+		});
+		it("should send nothing without a destination", async function() {
+			const { sent, upload } = await setup();
+			await upload(new msg.UploadRequest(1, "test", data(1)));
+			assert.equal(sent.length, 0);
+		});
+		it("should send nothing when an older generation isn't stored", async function() {
+			const { sent, upload } = await setup();
+			await upload(new msg.UploadRequest(1, "test", data(3)));
+			await upload(new msg.UploadRequest(1, "test", data(2), 2));
+			assert.equal(sent.length, 0);
+		});
+		it("should send nothing when another instance's lock rejects the upload", async function() {
+			const { controller, sent, upload } = await setup();
+			await mock.getHandler(controller, msg.AcquireRequest)(new msg.AcquireRequest(3, "test"));
+			await upload(new msg.UploadRequest(1, "test", data(1), 2));
+			assert.equal(sent.length, 0);
+		});
+		it("should store but not send to a stopped, source, plugin-off or unknown instance", async function() {
+			const { controller, sent, upload } = await setup();
+			let generation = 0;
+			for (const id of [4, 1, 5, 99]) {
+				generation += 1;
+				await upload(new msg.UploadRequest(1, "test", data(generation), id));
+			}
+			assert.equal(sent.length, 0);
+			assert.equal((await stored(controller)).generation, 4);
+		});
+		it("should store the upload and not fail it when sending throws", async function() {
+			const { controller, upload } = await setup();
+			controller.sendTo = () => { throw new Error("Host containing instance is not connected"); };
+			await upload(new msg.UploadRequest(1, "test", data(1), 2));
+			assert.deepEqual(await stored(controller), data(1));
+		});
+	});
 });

plugins/player_sync/test/instance.js

@@ -5,7 +5,8 @@ import * as lib from "@clusterio/lib";
 
 import * as mock from "../../../test/mock.js";
 import loadInstancePlugin, { recipeNotificationDelta, applyRecipeNotificationDelta } from "../dist/node/instance.js";
-import { DownloadResponse } from "../dist/node/messages.js";
+import { AcquireResponse, DownloadResponse, PrefetchEvent } from "../dist/node/messages.js";
+import { components } from "../dist/node/components.js";
 import { plugin as info } from "../dist/node/index.js";
 
 const inflate = util.promisify(zlib.inflate);
@@ -53,10 +54,15 @@ describe("player_sync", function() {
 			await apply(playerData, await encode(["a"]));
 			assert.deepEqual(await decode(playerData.recipe_notifications), {});
 		});
-		it("should add everything without a current list", async function() {
+		it("should send the whole list marked full without a current list", async function() {
 			const playerData = { name: "test", recipe_notifications: await encode(["a"]) };
 			await apply(playerData, undefined);
-			assert.deepEqual(await decode(playerData.recipe_notifications), { add: ["a"] });
+			assert.deepEqual(await decode(playerData.recipe_notifications), { add: ["a"], full: true });
+		});
+		it("should mark an empty list full without a current list", async function() {
+			const playerData = { name: "test", recipe_notifications: await encode([]) };
+			await apply(playerData, undefined);
+			assert.deepEqual(await decode(playerData.recipe_notifications), { full: true });
 		});
 		it("should do nothing without stored notifications", async function() {
 			const playerData = { name: "test" };
@@ -84,6 +90,9 @@ describe("player_sync", function() {
 		});
 		beforeEach(function() {
 			instance.server.reset();
+			for (const component of Object.keys(components)) {
+				instance.mockConfigEntries.set(`player_sync.sync_${component}`, true);
+			}
 		});
 
 		function respondWith(playerData) {
@@ -100,9 +109,10 @@ describe("player_sync", function() {
 				await lib.wait(1);
 			}
 		}
-		function downloadCommand(playerData) {
+		function downloadCommand(playerData, enabled = Object.keys(components)) {
 			const json = lib.escapeString(JSON.stringify(playerData));
-			return `/sc player_sync.download_inventory('test','${json}',1,1)`;
+			const sent = lib.escapeString(JSON.stringify(Object.fromEntries(enabled.map(c => [c, true]))));
+			return `/sc player_sync.download_inventory('test','${json}',1,1,'${sent}')`;
 		}
 
 		it("should send an empty download when the controller has no data", async function() {
@@ -113,10 +123,12 @@ describe("player_sync", function() {
 				["/sc player_sync.download_inventory('test',nil,0,0)"]
 			);
 		});
-		it("should send everything as a delta when no snapshot was given", async function() {
+		it("should send everything as a full delta when no snapshot was given", async function() {
 			respondWith({ generation: 1, name: "test", recipe_notifications: await encode(["a"]) });
 			await download({ player_name: "test" });
-			const expected = { generation: 1, name: "test", recipe_notifications: await encode({ add: ["a"] }) };
+			const expected = {
+				generation: 1, name: "test", recipe_notifications: await encode({ add: ["a"], full: true }),
+			};
 			assert.deepEqual(instance.server.rconCommands, [downloadCommand(expected)]);
 		});
 		it("should send a delta when a snapshot was given", async function() {
@@ -125,5 +137,208 @@ describe("player_sync", function() {
 			const expected = { generation: 1, name: "test", recipe_notifications: await encode({ add: ["a"] }) };
 			assert.deepEqual(instance.server.rconCommands, [downloadCommand(expected)]);
 		});
+		it("should leave out components which are not synced", async function() {
+			for (const component of Object.keys(components)) {
+				instance.mockConfigEntries.set(`player_sync.sync_${component}`, component === "appearance");
+			}
+			respondWith({ generation: 1, name: "test", force: "enemy", tag: "tag", recipe_notifications: "x" });
+			await download({ player_name: "test" });
+			const expected = { generation: 1, name: "test", tag: "tag" };
+			assert.deepEqual(instance.server.rconCommands, [downloadCommand(expected, ["appearance"])]);
+		});
+		it("should send the components it stripped by when the config changes during the download", async function() {
+			instance.connector.once("send", message => {
+				instance.mockConfigEntries.set("player_sync.sync_force", false);
+				instance.connector.emit("message", new lib.MessageResponse(
+					1, message.dst, message.src,
+					new DownloadResponse({ generation: 1, name: "test", force: "enemy", tag: "tag" }),
+				));
+			});
+			await download({ player_name: "test" });
+			const expected = { generation: 1, name: "test", tag: "tag" };
+			const enabled = Object.keys(components).filter(c => c !== "force");
+			assert.deepEqual(instance.server.rconCommands, [downloadCommand(expected, enabled)]);
+		});
+		it("should only send the components with the first chunk", async function() {
+			instance.mockConfigEntries.set("player_sync.rcon_chunk_size", 10);
+			try {
+				respondWith({ generation: 1, name: "test" });
+				await download({ player_name: "test" });
+				for (let i = 0; i < 50 && instance.server.rconCommands.length < 3; i++) {
+					await lib.wait(1);
+				}
+				const [first, ...rest] = instance.server.rconCommands;
+				assert.match(first, /,1,3,'.+'\)$/);
+				for (const command of rest) {
+					assert.match(command, /,\d,3\)$/);
+				}
+			} finally {
+				instance.mockConfigEntries.set("player_sync.rcon_chunk_size", 100000);
+			}
+		});
+	});
+
+	describe("set_components", function() {
+		let instance;
+		before(async function() {
+			({ instance } = await mock.loadInstancePlugin(loadInstancePlugin, info));
+		});
+		beforeEach(function() {
+			instance.server.reset();
+			for (const component of Object.keys(components)) {
+				instance.mockConfigEntries.set(`player_sync.sync_${component}`, true);
+			}
+		});
+
+		it("should send the synced components on start", async function() {
+			instance.mockConfigEntries.set("player_sync.sync_quick_bar", false);
+			await instance.hooks.start.invoke();
+			const enabled = Object.fromEntries(
+				Object.keys(components).filter(c => c !== "quick_bar").map(c => [c, true])
+			);
+			const json = lib.escapeString(JSON.stringify(enabled));
+			assert.deepEqual(
+				instance.server.rconCommands,
+				[`/sc player_sync.set_components("${json}")`]
+			);
+		});
+		it("should send the synced components when one is changed", async function() {
+			await instance.hooks.instanceConfigFieldChanged.invoke("player_sync.rcon_chunk_size", 1, 2);
+			assert.equal(instance.server.rconCommands.length, 0);
+			await instance.hooks.instanceConfigFieldChanged.invoke("player_sync.sync_force", false, true);
+			assert.equal(instance.server.rconCommands.length, 1);
+		});
+		it("should not send the components when the instance is not running", async function() {
+			instance.status = "stopped";
+			try {
+				await instance.hooks.instanceConfigFieldChanged.invoke("player_sync.sync_force", false, true);
+			} finally {
+				instance.status = "running";
+			}
+			assert.equal(instance.server.rconCommands.length, 0);
+		});
+	});
+
+	describe("ipc-player_sync_upload", function() {
+		let instance;
+		before(async function() {
+			try {
+				lib.registerPluginMessages([info]);
+			} catch (err) {
+				// Already registered by the full test suite
+			}
+			({ instance } = await mock.loadInstancePlugin(loadInstancePlugin, info));
+		});
+		beforeEach(function() {
+			instance.connector.sentMessages = [];
+		});
+
+		// The UploadRequest the instance sends the controller (no response is given; only the request is checked)
+		async function upload(playerData) {
+			instance.server.emit("ipc-player_sync_upload", playerData);
+			for (let i = 0; i < 100 && !instance.connector.sentMessages.length; i++) {
+				await lib.wait(1);
+			}
+			return instance.connector.sentMessages[0].data;
+		}
+
+		it("should pass destination_instance_id on as destinationInstanceId, not as player data", async function() {
+			const request = await upload({ name: "test", generation: 2, destination_instance_id: 7 });
+			assert.equal(request.destinationInstanceId, 7);
+			assert.deepEqual(request.playerData, { name: "test", generation: 2 });
+		});
+		it("should send no destination without the field", async function() {
+			const request = await upload({ name: "test", generation: 2 });
+			assert.equal(request.destinationInstanceId, undefined);
+		});
+	});
+
+	describe("PrefetchEvent", function() {
+		let instance;
+		before(async function() {
+			try {
+				lib.registerPluginMessages([info]);
+			} catch (err) {
+				// Already registered by the full test suite
+			}
+			({ instance } = await mock.loadInstancePlugin(loadInstancePlugin, info));
+			instance.mockConfigEntries.set("player_sync.rcon_chunk_size", 10);
+		});
+		beforeEach(function() {
+			instance.server.reset();
+			for (const component of Object.keys(components)) {
+				instance.mockConfigEntries.set(`player_sync.sync_${component}`, true);
+			}
+		});
+
+		const prefetch = playerData => mock.getHandler(instance, PrefetchEvent)(new PrefetchEvent(playerData, "tok"));
+		function prefetchCommands(playerData, enabled = Object.keys(components), epoch = 0) {
+			const chunks = JSON.stringify(playerData).match(/.{1,10}/g);
+			const sent = lib.escapeString(JSON.stringify(Object.fromEntries(enabled.map(c => [c, true]))));
+			return chunks.map((chunk, i) => "/sc player_sync.prefetch_inventory(" +
+				`'test',${playerData.generation},'${lib.escapeString(chunk)}',${i + 1},${chunks.length}` +
+				`${i === 0 ? `,'${sent}','tok',${epoch}` : `,nil,'tok',${epoch}`})`);
+		}
+
+		it("should send prefetch_inventory chunks with the generation and the full recipe list", async function() {
+			await prefetch({ generation: 3, name: "test", recipe_notifications: await encode(["a"]) });
+			const expected = {
+				generation: 3, name: "test", recipe_notifications: await encode({ add: ["a"], full: true }),
+			};
+			assert.ok(instance.server.rconCommands.length > 1, "more than one chunk");
+			assert.deepEqual(instance.server.rconCommands, prefetchCommands(expected));
+		});
+		it("should leave out components which are not synced and send the components with chunk 1", async function() {
+			instance.mockConfigEntries.set("player_sync.sync_force", false);
+			await prefetch({ generation: 3, name: "test", force: "enemy", tag: "tag" });
+			const enabled = Object.keys(components).filter(c => c !== "force");
+			const commands = prefetchCommands({ generation: 3, name: "test", tag: "tag" }, enabled);
+			assert.ok(commands.length > 1, "more than one chunk");
+			assert.deepEqual(instance.server.rconCommands, commands);
+		});
+		it("should send the token and epoch with every chunk", async function() {
+			await prefetch({ generation: 3, name: "test" });
+			const commands = instance.server.rconCommands;
+			assert.ok(commands[0].endsWith(",'tok',0)"));
+			assert.ok(commands.length > 1 && commands.slice(1).every(command => command.endsWith(",nil,'tok',0)")));
+		});
+		it("should send the stored data's epoch", async function() {
+			const playerData = { generation: 3, epoch: 2, name: "test" };
+			await prefetch(playerData);
+			assert.deepEqual(instance.server.rconCommands, prefetchCommands(playerData, undefined, 2));
+		});
+		it("should stop when the save refuses the first chunk", async function() {
+			const playerData = { generation: 3, name: "test" };
+			const commands = prefetchCommands(playerData);
+			instance.server.rconCommandResults.set(commands[0], "Refused prefetch for test: online here");
+			await prefetch(playerData);
+			assert.deepEqual(instance.server.rconCommands, commands.slice(0, 1));
+		});
+		it("should stop at the first non-empty reply mid-stream", async function() {
+			const playerData = { generation: 3, name: "test" };
+			const commands = prefetchCommands(playerData);
+			assert.equal(commands.length, 3);
+			instance.server.rconCommandResults.set(commands[1], "Ignored prefetch chunk 2 for test");
+			await prefetch(playerData);
+			assert.deepEqual(instance.server.rconCommands, commands.slice(0, 2));
+		});
+	});
+
+	describe("acquire reply", function() {
+		it("should pass the controller's token to acquire_response", async function() {
+			try {
+				lib.registerPluginMessages([info]);
+			} catch (err) {
+				// Already registered by the full test suite
+			}
+			const { instance, host } = await mock.loadInstancePlugin(loadInstancePlugin, info);
+			host.connector.connected = true;
+			instance.sendTo = async () => new AcquireResponse("acquired", 4, true, undefined, undefined, "tok");
+			instance.server.emit("ipc-player_sync_acquire", { player_name: "test" });
+			await lib.wait(10);
+			const [command] = instance.server.rconCommands;
+			assert.ok(command.startsWith("/sc player_sync.acquire_response("), command);
+			assert.ok(command.includes(lib.escapeString('"token":"tok"')), command);
+		});
 	});
 });

plugins/player_sync/test/plugin.js

@@ -0,0 +1,419 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import * as lib from "@clusterio/lib";
+import { CtlHooks } from "@clusterio/ctl";
+
+import * as mock from "../../../test/mock.js";
+import { testRoundTripJsonSerialisable, testMatrix } from "../../../test/common.js";
+import * as msg from "../dist/node/messages.js";
+import { summarizeInventory, summarizePlayerInventories } from "../dist/node/player_data.js";
+import ctlEntrypoint from "../dist/node/ctl.js";
+import { plugin as info } from "../dist/node/index.js";
+
+
+function playerData(name, generation, extra = {}) {
+	return { generation, name, controller: "character", force: "player", ...extra };
+}
+
+describe("player_sync plugin", function() {
+	describe("messages", function() {
+		it("PlayerEntry should be round trip json serialisable", function() {
+			testRoundTripJsonSerialisable(msg.PlayerEntry, testMatrix(
+				["player"], // name
+				[undefined, 0, 4], // generation
+				[undefined, 0, 1234], // size
+				[undefined, 1], // instanceId
+				[undefined, "player"], // force
+			));
+		});
+		it("GetPlayerDataResponse should be round trip json serialisable", function() {
+			testRoundTripJsonSerialisable(msg.GetPlayerDataResponse, testMatrix(
+				[null, playerData("player", 1)], // playerData
+				[undefined, 1], // instanceId
+			));
+		});
+		it("AcquireResponse should be round trip json serialisable", function() {
+			testRoundTripJsonSerialisable(msg.AcquireResponse, testMatrix(
+				["acquired", "busy"], // status
+				[undefined, 3], // generation
+				[undefined, true], // hasData
+				[undefined, "Instance"], // message
+				[undefined, 2], // epoch
+			));
+		});
+		it("should reject non-integer generations", function() {
+			const validate = lib.compile(msg.SetPlayerDataRequest.jsonSchema);
+			assert(validate({ playerName: "player", playerData: playerData("player", 5) }));
+			for (const generation of [5.5, -1, "5"]) {
+				const request = { playerName: "player", playerData: playerData("player", generation) };
+				assert(!validate(request), `accepted ${generation}`);
+			}
+		});
+		it("should reject uploads carrying the tombstone marker", function() {
+			const validate = lib.compile(msg.UploadRequest.jsonSchema);
+			assert(validate({ instanceId: 1, playerName: "player", playerData: playerData("player", 5) }));
+			for (const deleted of [false, true]) {
+				const data = playerData("player", 5, { deleted });
+				assert(!validate({ instanceId: 1, playerName: "player", playerData: data }), `accepted ${deleted}`);
+			}
+		});
+		describe("SetPlayerDataRequest schema", function() {
+			const validate = lib.compile(msg.SetPlayerDataRequest.jsonSchema);
+			// Shaped like module/serialize.lua output
+			const exported = playerData("player", 7, {
+				epoch: 2,
+				color: { r: 1, g: 0.5, b: 0, a: 0.5 },
+				chat_color: { r: 1, g: 0.5, b: 0, a: 1 },
+				tag: "",
+				cheat_mode: false,
+				flashlight: true,
+				shortcuts: { "toggle-personal-roboport": true },
+				game_view_settings: { show_minimap: true },
+				character: {
+					character_running_speed_modifier: 0,
+					character_additional_mining_categories: {},
+					allow_dispatching_robots: true,
+					inventories: {
+						main: { b: 80, i: [
+							{ n: "iron-plate", c: 100, r: 2 },
+							{ s: 10, f: { name: "coal" } },
+							{ f: { quality: "rare", comparator: "=" } },
+							{ f: "stone" },
+							{ n: "firearm-magazine", c: 10, q: "rare", a: 5 },
+							{ n: "automation-science-pack", c: 3, d: 0.5, p: 0.25, h: 0.75 },
+							{ e: "0eNq" },
+						] },
+						armor: { i: {} },
+					},
+				},
+				personal_logistic_slots: {
+					enabled: true,
+					trash_not_requested: false,
+					sections: [{
+						group: "",
+						active: true,
+						multiplier: 1,
+						// Empty slots export as {}
+						filters: [
+							{},
+							{ value: { name: "iron-plate", type: "item" }, min: 3 },
+							{ value: { name: "stone", type: "item" }, min: -5, max: 10 },
+							{
+								value: { name: "coal", type: "item", quality: "normal", comparator: "=" },
+								min: 5,
+								max: 10,
+								minimum_delivery_count: 2,
+								import_from: null,
+							},
+						],
+					}],
+				},
+				quick_bar: {
+					1: "iron-plate",
+					2: { t: "f", n: "iron-gear-wheel", q: "normal", c: "=" },
+					3: { t: "i", n: "main", s: 4, b: [2] },
+					4: { t: "r", p: [1], y: "blueprint" },
+				},
+				crafting_queue: {
+					crafting_queue: [{ recipe: "iron-gear-wheel", count: 5 }],
+					crafting_queue_progress: 0.25,
+					ingredients: [{ name: "iron-plate", count: 10 }],
+				},
+				recipe_notifications: "",
+				components: { controller: true, inventories: true },
+			});
+			function accepts(data) {
+				return validate({ playerName: "player", playerData: data });
+			}
+
+			it("should accept exported and partial player data", function() {
+				assert(accepts(exported), JSON.stringify(validate.errors));
+				assert(accepts({ name: "player", generation: 0 }));
+				assert(accepts(playerData("player", 0, { controller: "ghost", ticks_to_respawn: 300 })));
+				assert(accepts(playerData("player", 0, { color: [1, 0, 0] })));
+				assert(accepts(playerData("player", 0, { chat_color: [1, 0, 0], tag: "tag" })));
+				assert(accepts(playerData("player", 0, {
+					crafting_queue: { crafting_queue: {}, crafting_queue_progress: 0, ingredients: {} },
+					personal_logistic_slots: { 1: { name: "iron-plate", min: 50 } },
+					hotbar: { 1: "iron-plate" },
+				})));
+			});
+			it("should reject data the scenario can't restore", function() {
+				for (const [field, value] of [
+					["controller", "not-a-controller"],
+					["controller", "remote"],
+					["controller", "cutscene"],
+					["deleted", false],
+					["deleted", true],
+					["color", "red"],
+					["force", 1],
+					["ticks_to_respawn", -1],
+					["inventories", { main: { i: "iron-plate" } }],
+					["inventories", { main: [] }],
+					["inventories", { main: { i: [{ e: null }] } }],
+					["inventories", { main: { i: [{ n: 5, c: 1 }] } }],
+					["inventories", { main: { i: [{ n: "wood", c: "many" }] } }],
+					["inventories", { main: { i: [{ n: "wood", c: 0 }] } }],
+					["character", { inventories: { main: { i: [{ s: 0 }] } } }],
+					["shortcuts", { "toggle-personal-roboport": "yes" }],
+					["character", { character_health_bonus: "100" }],
+					["character", { character_inventory_slots_bonus: -1 }],
+					["character", { character_reach_distance_bonus: -1 }],
+					["character", { character_crafting_speed_modifier: -5 }],
+					["quick_bar", { 1: { t: "f", n: "iron-plate", q: "normal", c: "~" } }],
+					["crafting_queue", {}],
+					["crafting_queue", { crafting_queue: [], crafting_queue_progress: 0 }],
+					["crafting_queue", {
+						crafting_queue: [{ recipe: "iron-gear-wheel" }], crafting_queue_progress: 0, ingredients: [],
+					}],
+					["quick_bar", { 1: { t: "i", n: "main" } }],
+					["quick_bar", { 1: { t: "r", y: "blueprint" } }],
+					["quick_bar", { 1: 5 }],
+					["personal_logistic_slots", { enabled: true, sections: [] }],
+					["personal_logistic_slots", { enabled: true, trash_not_requested: false, sections: [{}] }],
+					["personal_logistic_slots", { 1: { min: 50 } }],
+					["personal_logistic_slots", {
+						enabled: true,
+						trash_not_requested: false,
+						sections: [{ group: "", active: true, multiplier: 1, filters: [{ value: 5 }] }],
+					}],
+				]) {
+					assert(!accepts({ ...exported, [field]: value }), `accepted ${field} ${JSON.stringify(value)}`);
+				}
+				for (const filter of [
+					{ value: { name: "iron-plate" }, min: "many" },
+					{ value: { name: "iron-plate" }, min: 2 ** 31 },
+					{ value: { name: "iron-plate" }, max: -1 },
+					{ value: { name: "iron-plate" }, minimum_delivery_count: -1 },
+					{ value: { type: "item" }, min: 1 },
+					{ value: { name: "iron-plate", type: "nope" }, min: 1 },
+					{ value: { name: "iron-plate", comparator: "~" }, min: 1 },
+					{ value: { name: "iron-plate", quality: 5 }, min: 1 },
+				]) {
+					const slots = {
+						enabled: true,
+						trash_not_requested: false,
+						sections: [{ group: "", active: true, multiplier: 1, filters: [filter] }],
+					};
+					assert(
+						!accepts({ ...exported, personal_logistic_slots: slots }), `accepted ${JSON.stringify(filter)}`,
+					);
+				}
+			});
+		});
+		it("SetPlayerDataResponse should be round trip json serialisable", function() {
+			testRoundTripJsonSerialisable(msg.SetPlayerDataResponse, testMatrix([1, 5]));
+		});
+		it("requests should be round trip json serialisable", function() {
+			testRoundTripJsonSerialisable(msg.GetPlayerDataRequest, testMatrix(["player"]));
+			testRoundTripJsonSerialisable(msg.DeletePlayerDataRequest, testMatrix(["player"]));
+			testRoundTripJsonSerialisable(msg.ForceReleaseRequest, testMatrix(["player"]));
+			testRoundTripJsonSerialisable(msg.SetPlayerDataRequest, testMatrix(
+				["player"], // playerName
+				[playerData("player", 3)], // playerData
+			));
+		});
+	});
+
+	describe("summarizeInventory()", function() {
+		it("should merge stacks and expand repeats", function() {
+			assert.deepEqual(summarizeInventory({ i: [
+				{ n: "iron-plate", c: 100, r: 2 },
+				{ n: "iron-plate", c: 50, q: "rare" },
+				{ n: "iron-plate", c: 7, s: 10 },
+				{ e: "0eNq", r: 1 },
+				{ f: "copper-plate" },
+			] }), [
+				{ name: "iron-plate", quality: undefined, count: 307 },
+				{ name: "iron-plate", quality: "rare", count: 50 },
+				{ name: "exported item", count: 2, exportSize: 4 },
+			]);
+		});
+		it("should skip malformed entries", function() {
+			assert.deepEqual(summarizeInventory({ i: [
+				{ e: null },
+				null,
+				"wood",
+				{ n: 5, c: 1 },
+				{ n: "wood", c: "many", r: "x" },
+				{ n: "wood", c: 2, r: 1 },
+				{ e: "0eNq", r: 1e9 },
+			] }), [
+				{ name: "wood", quality: undefined, count: 4 },
+				{ name: "exported item", count: 1e9 + 1, exportSize: 4 },
+			]);
+			assert.deepEqual(
+				summarizePlayerInventories(playerData("player", 1, { inventories: { main: null } })),
+				[{ name: "main", items: [] }],
+			);
+		});
+		it("should handle empty inventories", function() {
+			assert.deepEqual(summarizeInventory({}), []);
+			assert.deepEqual(summarizeInventory(JSON.parse('{"i":{}}')), []);
+			assert.deepEqual(
+				summarizeInventory({ i: [{ n: "wood" }] }),
+				[{ name: "wood", quality: undefined, count: 0 }],
+			);
+		});
+	});
+
+	describe("summarizePlayerInventories()", function() {
+		it("should prefer character inventories", function() {
+			const data = playerData("player", 1, {
+				character: { inventories: { main: { i: [{ n: "wood", c: 1 }] } } },
+				inventories: { main: { i: [{ n: "stone", c: 1 }] } },
+			});
+			assert.deepEqual(summarizePlayerInventories(data), [
+				{ name: "main", items: [{ name: "wood", quality: undefined, count: 1 }] },
+			]);
+		});
+		it("should fall back to player inventories", function() {
+			const data = playerData("player", 1, { inventories: { main: { i: [{ n: "stone", c: 1 }] } } });
+			assert.deepEqual(summarizePlayerInventories(data), [
+				{ name: "main", items: [{ name: "stone", quality: undefined, count: 1 }] },
+			]);
+		});
+		it("should handle empty serialized inventories", function() {
+			const data = JSON.parse('{"generation":1,"name":"player","character":{"inventories":{"armor":{"i":{}}}}}');
+			assert.deepEqual(summarizePlayerInventories(data), [{ name: "armor", items: [] }]);
+		});
+		it("should handle missing inventories", function() {
+			assert.deepEqual(summarizePlayerInventories(playerData("player", 1)), []);
+			assert.deepEqual(summarizePlayerInventories(playerData("player", 1, { character: {} })), []);
+		});
+	});
+
+	describe("ctl commands", function() {
+		let commands;
+		let sent;
+		let printed;
+		let responses;
+		const ctl = {
+			async send(message) {
+				sent.push(message);
+				const handler = responses.get(message.constructor);
+				if (!handler) {
+					throw new Error(`Unexpected ${message.constructor.name}`);
+				}
+				return handler(message);
+			},
+		};
+		const storedData = playerData("alice", 2, {
+			character: { inventories: { main: { i: [{ n: "wood", c: 5, q: "rare" }, { e: "0eNq" }] } } },
+		});
+		let tempDir;
+
+		before(async function() {
+			const root = new lib.CommandTree({ name: "clusterioctl", description: "root" });
+			const hooks = new CtlHooks(new mock.MockLogger());
+			await ctlEntrypoint({ hooks, logger: new mock.MockLogger(), plugin: info });
+			await hooks.addCommands.invoke(root);
+			commands = root.subCommands.get("player-sync").subCommands;
+			tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "player_sync-ctl-"));
+		});
+		after(async function() {
+			await fs.rm(tempDir, { recursive: true, force: true });
+		});
+		beforeEach(function() {
+			sent = [];
+			printed = [];
+			responses = new Map([
+				[lib.InstanceDetailsGetRequest, request => ({ id: request.instanceId, name: "Test" })],
+				[msg.ListPlayersRequest, () => [
+					new msg.PlayerEntry("bob", undefined, undefined, 1),
+					new msg.PlayerEntry("alice", 2, 100),
+				]],
+				[msg.GetPlayerDataRequest, request => new msg.GetPlayerDataResponse(
+					request.playerName === "alice" ? storedData : null, request.playerName === "alice" ? 1 : undefined
+				)],
+				[msg.SetPlayerDataRequest, () => new msg.SetPlayerDataResponse(3)],
+				[msg.DeletePlayerDataRequest, () => undefined],
+				[msg.ForceReleaseRequest, () => undefined],
+			]);
+		});
+
+		async function run(name, args = {}) {
+			/* eslint-disable no-console */
+			const original = console.log;
+			console.log = (...values) => printed.push(values.join(" "));
+			try {
+				await commands.get(name).run(args, ctl);
+			} finally {
+				console.log = original;
+			}
+			/* eslint-enable no-console */
+		}
+
+		it("list should print sorted entries", async function() {
+			await run("list");
+			assert.deepEqual(printed, [
+				"name | generation | size | acquired by",
+				`alice | 2 | ${lib.formatBytes(100)} | -`,
+				"bob | - | - | Test (1)",
+			]);
+		});
+		it("show should print a summary", async function() {
+			await run("show", { player: "alice" });
+			assert.deepEqual(printed, [
+				"acquired by: Test (1)",
+				"generation: 2",
+				`size: ${lib.formatBytes(JSON.stringify(storedData).length)}`,
+				"controller: character",
+				"force: player",
+				"main:",
+				"  5 wood (rare)",
+				`  1 exported item ${lib.formatBytes(4)}`,
+			]);
+		});
+		it("should show the instance ID when details can't be fetched", async function() {
+			responses.set(lib.InstanceDetailsGetRequest, () => {
+				throw new lib.PermissionError("Permission denied");
+			});
+			await run("list");
+			assert.equal(printed[2], "bob | - | - | 1");
+			printed = [];
+			await run("show", { player: "alice" });
+			assert.equal(printed[0], "acquired by: 1");
+		});
+		it("show should handle players without data", async function() {
+			await run("show", { player: "carol" });
+			assert.deepEqual(printed, ["acquired by: -", "No player data stored for carol"]);
+		});
+		it("export should print or write JSON", async function() {
+			await run("export", { player: "alice" });
+			assert.deepEqual(JSON.parse(printed.join("\n")), storedData);
+			const file = path.join(tempDir, "alice.json");
+			await run("export", { player: "alice", file });
+			assert.deepEqual(JSON.parse(await fs.readFile(file, "utf8")), storedData);
+			await assert.rejects(
+				run("export", { player: "carol" }),
+				new lib.CommandError("No player data stored for carol"),
+			);
+		});
+		it("import should send the file contents", async function() {
+			const file = path.join(tempDir, "import.json");
+			await fs.writeFile(file, JSON.stringify({ name: "other", controller: "god" }));
+			await run("import", { player: "alice", file });
+			assert.deepEqual(sent, [
+				new msg.SetPlayerDataRequest("alice", { name: "alice", controller: "god", generation: 0 }),
+			]);
+			assert.deepEqual(printed, ["Stored player data for alice as generation 3"]);
+		});
+		it("import should reject non-object JSON", async function() {
+			const file = path.join(tempDir, "bad.json");
+			await fs.writeFile(file, "[1]");
+			await assert.rejects(
+				run("import", { player: "alice", file }),
+				new lib.CommandError("Player data must be a JSON object"),
+			);
+		});
+		it("delete and release should send requests", async function() {
+			await run("delete", { player: "alice" });
+			await run("release", { player: "alice" });
+			assert.deepEqual(sent, [new msg.DeletePlayerDataRequest("alice"), new msg.ForceReleaseRequest("alice")]);
+		});
+	});
+});

plugins/player_sync/tsconfig.browser.json

@@ -4,5 +4,5 @@
 		{ "path": "../../packages/lib/tsconfig.browser.json" },
 		{ "path": "../../packages/web_ui/tsconfig.browser.json" },
 	],
-	"include": [ "web/**/*.tsx", "web/**/*.ts", "index.ts", "messages.ts", "package.json" ],
+	"include": [ "web/**/*.tsx", "web/**/*.ts", "index.ts", "messages.ts", "player_data.ts", "package.json" ],
 }