From 5238a0fbb384661bac3e4e0ae839da909cfa6db8 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:02:52 +0000 Subject: [PATCH 01/44] Report missing base game icons when exporting from headless Factorio The headless build has no graphics, so export-data found none of the base game's icon files. Plain icons were skipped silently and layered icons were drawn blank and counted as exported. Skip a layered icon when any of its layers is missing, and log one warning with the number of base game icon files that were not found. Add host.factorio_graphics_directory to read builtin mod graphics from a full install when the instance runs the headless build. Fixes #1068 Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 9 ++ packages/host/src/Instance.ts | 2 +- packages/host/src/export.ts | 122 +++++++++++++----- packages/lib/src/config/definitions.ts | 9 ++ .../base/graphics/icons/graphics-icon.png | Bin 0 -> 206 bytes test/host/export.js | 66 +++++++++- 6 files changed, 171 insertions(+), 37 deletions(-) create mode 100644 test/file/factorio-graphics/base/graphics/icons/graphics-icon.png diff --git a/docs/configuration.md b/docs/configuration.md index 1818a357..b7064174 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -339,6 +339,15 @@ This can either point to a Factorio server directory, or to a directory containi Defaults to "factorio". +### host.factorio_graphics_directory + +Directory to read the base game's icons from when exporting data. +The headless build of Factorio has no graphics, so instances running it export none of the base game's icons unless this points to a full install of Factorio. +Like `host.factorio_directory` this can either point to a Factorio directory, or to a directory containing multiple versions, in which case the latest one matching the major and minor version of the instance is used. + +Defaults to null. + + ### host.mods_directory Directory the host stores mods downloaded from the controller in. diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index f70080d3..7596a8fc 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -1469,7 +1469,7 @@ end`.replace(/\r?\n/g, " "); this.logger.info("Exporting data ....."); await this.syncMods(); - let zip = await exportData(this.server); + let zip = await exportData(this.server, this._host.config.get("host.factorio_graphics_directory")); let url = new URL(this._host.config.get("host.controller_url")); url.pathname += "api/upload-export"; diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index cb7f73d5..9554e5c5 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -6,7 +6,7 @@ import yazl from "yazl"; import * as lib from "@clusterio/lib"; import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js"; -import type { FactorioServer } from "./server.js"; +import { findVersion, type FactorioServer } from "./server.js"; import { definesPrototypesFallback } from "./export_fallback.js"; interface SimpleIconSpecification { @@ -91,6 +91,13 @@ async function loadZip(server: FactorioServer, modVersions: Map, return modZip; } +/** Where icon files are read from during an export. */ +interface ExportFiles { + zipCache: ZipCache; + /** Data dir of a full install to fall back to for builtin mod files. */ + graphicsDataDir?: string; +} + function closeZipCache(zipCache: ZipCache) { for (const { zip } of zipCache.values()) { zip.close(); @@ -103,7 +110,7 @@ function closeZipCache(zipCache: ZipCache) { * * @param server - The server to load the file from. * @param modVersions - Mapping of mod to version used. - * @param zipCache - Mod zips opened by this export. + * @param files - Where to read files from. * @param modPath - Factorio style path to the file to load. * @returns The content of the file or null if not found. * @internal @@ -111,7 +118,7 @@ function closeZipCache(zipCache: ZipCache) { async function loadFile( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, modPath: string, ) { let match = /^__([^\/]+)__\/(.*)$/.exec(modPath); @@ -124,19 +131,25 @@ async function loadFile( const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)]; if (builtinModNames.includes(mod)) { - try { - return await fs.readFile(server.dataPath(mod, filePath)); - } catch (err: any) { - if (err.code === "ENOENT") { - return null; - } - throw err; + const dataDirs = [server.dataPath()]; + if (files.graphicsDataDir) { + dataDirs.push(files.graphicsDataDir); } + for (const dataDir of dataDirs) { + try { + return await fs.readFile(path.join(dataDir, mod, filePath)); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + } + } + return null; } let modZip; try { - modZip = await loadZip(server, modVersions, zipCache, mod); + modZip = await loadZip(server, modVersions, files.zipCache, mod); } catch (err: any) { if (err.code === "ENOENT") { return null; @@ -152,7 +165,7 @@ type IconCache = Map; async function loadIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, iconPath: string, iconSize: number, iconMipmaps: number, @@ -160,7 +173,7 @@ async function loadIcon( ) { let icon = iconCache.get(iconPath); if (icon === undefined) { - let fileContent = await loadFile(server, modVersions, zipCache, iconPath); + let fileContent = await loadFile(server, modVersions, files, iconPath); if (fileContent) { icon = await Jimp.fromBuffer(fileContent) as Image; icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize }); @@ -176,12 +189,12 @@ async function loadIcon( async function loadSimpleIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, item: SimpleIconSpecification, size: number, iconCache: IconCache, ): Promise { - let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache); + let icon = await loadIcon(server, modVersions, files, item.icon, item.icon_size ?? 64, 0, iconCache); if (icon) { let iconScale = size / (item.icon_size ?? 64); if (iconScale !== 1) { @@ -195,11 +208,11 @@ async function loadSimpleIcon( async function loadLayeredIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, item: LayeredIconSpecification, size: number, iconCache: IconCache, -): Promise { +): Promise { let baseLayerSize = (item.icons[0].icon_size || item.icon_size) ?? 64; let icon = new Jimp({ width: size, height: size }); @@ -212,11 +225,12 @@ async function loadLayeredIcon( for (let layer of item.icons) { let layerSize = (layer.icon_size || item.icon_size) ?? 64; let iconLayer = await loadIcon( - server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache, + server, modVersions, files, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache, ); + // A partial icon would be wrong, skip it like a missing simple icon if (!iconLayer) { - continue; + return null; } iconLayer = iconLayer.clone(); @@ -331,7 +345,7 @@ async function growSheet(state: SheetState) { async function exportIcon( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, state: SheetState, prototype: PrototypeWithIcon, ) { @@ -342,9 +356,11 @@ async function exportIcon( if (prototype.icons) { icon = await loadLayeredIcon( - server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache, + server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache, ); - iconPos = state.pos; + if (icon) { + iconPos = state.pos; + } iconPath = prototype.icons[0].icon; } else if (prototype.icon) { @@ -352,7 +368,7 @@ async function exportIcon( iconPath = prototype.icon; if (iconPos === undefined) { icon = await loadSimpleIcon( - server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache, + server, modVersions, files, prototype, SHEET_ICON_SIZE, state.iconCache, ); if (icon) { iconPos = state.pos; @@ -388,7 +404,7 @@ async function exportIcon( * * @param server - The server to load icons from. * @param modVersions - Mapping of mod name to version. - * @param zipCache - Mod zips opened by this export. + * @param files - Where to read icon files from. * @param prototypes - Array of prototypes with icon to export for this base type. * @param baseName - name of base type (used in metadata). * @param state - Shared spritesheet state (mutated in place). @@ -398,7 +414,7 @@ async function exportIcon( async function exportIcons( server: FactorioServer, modVersions: Map, - zipCache: ZipCache, + files: ExportFiles, prototypes: PrototypeWithIcon[], baseName: string, state: SheetState, @@ -407,7 +423,7 @@ async function exportIcons( state.metadata[baseName] ??= metadata; let count = 0; for (let prototype of prototypes) { - const icon = await exportIcon(server, modVersions, zipCache, state, prototype); + const icon = await exportIcon(server, modVersions, files, state, prototype); if (icon) { count += 1; } @@ -498,19 +514,57 @@ async function exportLocale( * Export locale, icons and metadata for the given factorio server * * @param server - The server to export the data from. + * @param graphicsDirectory - + * Full Factorio install, or directory of installs, to read builtin mod + * graphics from when the server's install lacks them. * @returns zip file with exported data. */ -export async function exportData(server: FactorioServer) { - const zipCache: ZipCache = new Map(); +export async function exportData(server: FactorioServer, graphicsDirectory?: string | null) { + const files: ExportFiles = { zipCache: new Map() }; + if (graphicsDirectory) { + const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion; + [files.graphicsDataDir] = await findVersion(graphicsDirectory, majorMinor); + } try { - return await buildExport(server, zipCache); + return await buildExport(server, files); } finally { // Free up the zip files loaded during the export. - closeZipCache(zipCache); + closeZipCache(files.zipCache); } } -async function buildExport(server: FactorioServer, zipCache: ZipCache) { +/** + * Warn about builtin mod icon files that could not be found + * + * @param server - The server the export ran on. + * @param files - Where icon files were read from. + * @param iconCache - Icons loaded by the export, null for missing files. + * @internal + */ +function warnMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, iconCache: IconCache) { + const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)]; + let missing = 0; + for (const [iconPath, icon] of iconCache) { + const mod = /^__([^\/]+)__\//.exec(iconPath)?.[1]; + if (icon === null && mod && builtinModNames.includes(mod)) { + missing += 1; + } + } + if (!missing) { + return; + } + + const dirs = [server.dataPath(), files.graphicsDataDir].filter(Boolean).join(" or "); + const hint = files.graphicsDataDir + ? "Check that host.factorio_graphics_directory points to a full Factorio install" + : "Set host.factorio_graphics_directory to a full Factorio install to read them from there"; + server._logger.warn( + `${missing} icon files of the base game were not found in ${dirs}, icons using them were not exported. ` + + `The headless build of Factorio has no graphics. ${hint}, or export the data on a host with a full install.` + ); +} + +async function buildExport(server: FactorioServer, files: ExportFiles) { await generateExportMod(server); let settings: lib.ExportSettings = {}; @@ -590,7 +644,7 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) { } } - let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en"); + let locale = await exportLocale(server, modVersions, files.zipCache, modOrder, "en"); let zip = new yazl.ZipFile(); const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name); @@ -611,9 +665,10 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) { if (prototypesWithIcons.length === 0) { continue; } - const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state); + const count = await exportIcons(server, modVersions, files, prototypesWithIcons, baseName, state); server._logger.info(`Exported ${count} ${baseName} icons`); } + warnMissingBuiltinIcons(server, files, state.iconCache); // Crop sheet to actual used height and write single spritesheet + metadata. const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE)); @@ -632,3 +687,4 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) { export const _exportLocale = exportLocale; export const _loadSimpleIcon = loadSimpleIcon; export const _loadLayeredIcon = loadLayeredIcon; +export const _warnMissingBuiltinIcons = warnMissingBuiltinIcons; diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 5d19897a..93a7a43e 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -324,6 +324,7 @@ export interface HostConfigFields { "host.name": string; "host.id": number; "host.factorio_directory": string; + "host.factorio_graphics_directory": string | null; "host.mods_directory": string; "host.instances_directory": string; "host.controller_url": string; @@ -372,6 +373,14 @@ export class HostConfig extends classes.Config { initialValue: "factorio", validator: validators.filePath, }, + "host.factorio_graphics_directory": { + description: + "Path to a full Factorio install, or directory of installs, to read base game icons from when " + + "exporting data. Needed when running the headless build, which has no graphics.", + type: "string", + optional: true, + validator: validators.optional(validators.filePath), + }, "host.mods_directory": { title: "Mods Directory", description: "Path to directory where mods for instances are cached.", diff --git a/test/file/factorio-graphics/base/graphics/icons/graphics-icon.png b/test/file/factorio-graphics/base/graphics/icons/graphics-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..b2f06cccc1810f3be8ee688a20407b7a133fd628 GIT binary patch literal 206 zcmeAS@N?(olHy`uVBq!ia0vp^2|($9&iw7G06K< zeecwRz)9r=e{MWB=w0C!qP$A*YDbof=_08>I&pt7R538u=|BHz1au&Sr>mdKI;Vst E07RZjv;Y7A literal 0 HcmV?d00001 diff --git a/test/host/export.js b/test/host/export.js index 43109532..c371145f 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -2,7 +2,9 @@ import assert from "node:assert/strict"; import path from "node:path"; import { FactorioServer } from "@clusterio/host/dist/node/src/server.js"; -import { _exportLocale, _loadLayeredIcon, _loadSimpleIcon } from "@clusterio/host/dist/node/src/export.js"; +import { + _exportLocale, _loadLayeredIcon, _loadSimpleIcon, _warnMissingBuiltinIcons, +} from "@clusterio/host/dist/node/src/export.js"; describe("host/src/export", function() { @@ -30,20 +32,37 @@ describe("host/src/export", function() { // test-icon.png is a red 64px icon followed by a blue 32px mipmap const red = 0xff0000ff; + // Stands in for the data dir of a full install + const graphicsDataDir = path.join("test", "file", "factorio-graphics"); describe("loadSimpleIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadSimpleIcon( - testServer, new Map(), new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map() + testServer, new Map(), { zipCache: new Map() }, + { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map(), ); assert.equal(icon.bitmap.width, 32); assert.equal(icon.bitmap.height, 32); assert.equal(icon.getPixelColor(31, 16), red); }); + it("returns null when the icon file is missing", async function() { + const icon = await _loadSimpleIcon( + testServer, new Map(), { zipCache: new Map() }, + { icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(), + ); + assert.equal(icon, null); + }); + it("falls back to graphicsDataDir for builtin mod files", async function() { + const icon = await _loadSimpleIcon( + testServer, new Map(), { zipCache: new Map(), graphicsDataDir }, + { icon: "__base__/graphics/icons/graphics-icon.png" }, 32, new Map(), + ); + assert.equal(icon.getPixelColor(31, 16), red); + }); }); describe("loadLayeredIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadLayeredIcon( - testServer, new Map(), new Map(), + testServer, new Map(), { zipCache: new Map() }, { icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map(), ); assert.equal(icon.bitmap.width, 32); @@ -51,5 +70,46 @@ describe("host/src/export", function() { assert.equal(icon.getPixelColor(0, 0), red); assert.equal(icon.getPixelColor(31, 31), red); }); + it("returns null when a layer is missing", async function() { + const icon = await _loadLayeredIcon( + testServer, new Map(), { zipCache: new Map() }, + { icons: [ + { icon: "__base__/graphics/icons/test-icon.png" }, + { icon: "__base__/graphics/icons/missing.png" }, + ] }, 32, new Map(), + ); + assert.equal(icon, null); + }); + }); + describe("warnMissingBuiltinIcons()", function() { + let warnings; + let logger; + beforeEach(function() { + warnings = []; + logger = testServer._logger; + testServer._logger = { warn: msg => warnings.push(msg) }; + }); + afterEach(function() { + testServer._logger = logger; + }); + + it("warns once with the number of missing builtin files", function() { + _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ + ["__base__/graphics/icons/a.png", null], + ["__core__/graphics/icons/b.png", null], + ["__base__/graphics/icons/c.png", {}], + ["__some_mod__/graphics/icons/d.png", null], + ])); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /^2 icon files of the base game were not found/); + assert.match(warnings[0], /headless/); + }); + it("does not warn when no builtin files are missing", function() { + _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ + ["__base__/graphics/icons/c.png", {}], + ["__some_mod__/graphics/icons/d.png", null], + ])); + assert.deepEqual(warnings, []); + }); }); }); From 87d4f5c64e25a615a9d4b33bc1a73c1c1a3d1be3 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 06:11:32 +0100 Subject: [PATCH 02/44] Replace host tokens with host pairing Hosts join the cluster by running the installer, which shows a code and waits until the request is approved under Hosts > Add host in the web interface or with clusterioctl host approve. An admin can pass their user token to the installer to approve the host right away. The token is only used for that and is not saved. The controller only accepts hosts that have a record, and only approving a host creates one. Revoking a host deletes its record, so its token can no longer be used to reconnect. Hosts that have connected before keep working. Removes host generate-token, revoke-token and create-config from clusterioctl, bootstrap generate-host-token, the host token messages and the core.host.generate_token, revoke_token and create_config permissions, which are also stripped from stored roles. Adds clusterioctl host pending, approve, reject and revoke, and the core.host.approve and core.host.revoke permissions. Standalone installs add the host to the controller database directly, since the controller is not running during the install. --- README.md | 17 +- docs/configuration.md | 8 +- docs/devs/plugin-migration-to-2.1.md | 20 + docs/devs/protocol.md | 61 ++ docs/managing-a-cluster.md | 43 +- docs/setting-up-tls.md | 2 +- packages/controller/controller.ts | 12 - packages/controller/index.ts | 1 + packages/controller/src/ControlConnection.ts | 57 +- packages/controller/src/Controller.ts | 29 +- packages/controller/src/HostConnection.ts | 34 +- packages/controller/src/HostPairingManager.ts | 361 ++++++++++ packages/controller/src/WsServer.ts | 12 +- packages/controller/src/routes.ts | 144 +++- packages/create/README.md | 116 ++++ packages/create/create.js | 303 ++++++-- packages/create/pair.js | 275 ++++++++ packages/ctl/src/commands_host.ts | 107 +-- packages/host/README.md | 3 + packages/host/host.ts | 7 +- packages/lib/src/data/HostPairingDetails.ts | 73 ++ packages/lib/src/data/index.ts | 1 + packages/lib/src/data/messages_controller.ts | 96 ++- packages/lib/src/data/messages_host.ts | 6 +- packages/lib/src/link/messages.ts | 8 +- packages/lib/src/permissions.ts | 19 +- .../web_ui/src/components/HostViewPage.tsx | 63 +- packages/web_ui/src/components/HostsPage.tsx | 304 ++++---- packages/web_ui/src/model/host.ts | 13 + packages/web_ui/src/util/host_pairing.ts | 72 ++ packages/web_ui/src/util/websocket.ts | 8 + scripts/version-compat-test.js | 50 +- test/controller/ControlConnection.test.js | 214 +++++- test/controller/HostConnection.test.js | 98 ++- test/controller/HostPairingManager.test.js | 648 ++++++++++++++++++ test/controller/WsServer.js | 143 +++- test/controller/routes.js | 325 ++++++++- test/create.js | 331 +++++++++ test/create/pair.js | 343 +++++++++ test/ctl/commands_host.test.js | 145 ++++ test/integration/clusterio.js | 342 +++++++-- test/integration/index.js | 45 +- test/integration/instance.js | 12 +- test/integration/link.js | 36 +- test/integration/routing.js | 4 +- test/lib/data/HostPairingDetails.test.js | 85 +++ test/messages/controller.test.js | 77 +++ test/messages/host.test.js | 13 + test/web_ui/host_pairing.js | 142 ++++ 49 files changed, 4855 insertions(+), 473 deletions(-) create mode 100644 packages/controller/src/HostPairingManager.ts create mode 100644 packages/create/pair.js create mode 100644 packages/lib/src/data/HostPairingDetails.ts create mode 100644 packages/web_ui/src/util/host_pairing.ts create mode 100644 test/controller/HostPairingManager.test.js create mode 100644 test/create/pair.js create mode 100644 test/ctl/commands_host.test.js create mode 100644 test/lib/data/HostPairingDetails.test.js create mode 100644 test/web_ui/host_pairing.js diff --git a/README.md b/README.md index 52ac01bc..e40a34c4 100644 --- a/README.md +++ b/README.md @@ -272,10 +272,23 @@ If you have copied these files over to `/etc/systemd/system/` then you can start ## Setting up remote hosts -Run the installer as described in the installation section and choose "Host only" as the operating mode to install, it'll ask for a controller URL and an authentication token. -The URL is the same as what is needed to connect to the web interface, and the authentication token can be generated on the Hosts page in the web interface. +Run the installer as described in the installation section and choose "Host only" as the operating mode to install, it'll ask for a name for the host and a controller URL. +The URL is the same as what is needed to connect to the web interface. +Once it has installed everything the installer prints a code and waits for the host to be approved: + + Waiting for approval. Approve "my-host" with code ABC-DEF on the Hosts page of https://example.com/, or run: npx clusterioctl host approve ABC-DEF + +Approve it in the web interface by clicking Add host on the Hosts page and approving the request with the same code, or with `npx clusterioctl host approve `. +The installer then saves the connection details for the host and exits. +Only approve a request if its code matches the one shown by the installer, approving gives the host full access to the cluster. +Approving requires the `core.host.approve` permission, which the Cluster Admin role has. +If you have that permission you can skip the wait by passing your own authentication token to the installer with `--controller-token `, it is only used to approve the host and is not saved. Once you start up the host it should show up in the Hosts list and be available for assigning and running instances on. +To remove a host from the cluster, unassign its instances and click Revoke host access on the page for the host in the web interface, or run `npx clusterioctl host revoke `. +This disconnects the host and its saved connection details stop working. +To add the computer back to the cluster run the installer again in the same directory, it reuses the existing installation and asks for approval again. + ## Setting up clusterioctl diff --git a/docs/configuration.md b/docs/configuration.md index 1818a357..2ea6bf06 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -93,7 +93,7 @@ Defaults to null meaning the header is never trusted. ### controller.public_url Public address the controller is accessible on. -Used as the URL in configs generated with `clusteriocontroller bootstrap create-ctl-config` and `clusterioctl host create-config`, and by plugins like Player Auth that need to show the address of the web interface in-game. +Used as the URL in configs generated with `clusteriocontroller bootstrap create-ctl-config`, and by plugins like Player Auth that need to show the address of the web interface in-game. This should be a full URL ending with a /. Defaults to null meaning assume localhost. @@ -277,6 +277,7 @@ Defaults to 1000. Whether the `controller update` command in clusterioctl and the update button in the web interface are allowed to update the `@clusterio/controller` package with npm. For security reasons this can only be changed locally with `clusteriocontroller config set` while the controller is stopped, and it is hidden in the web interface. +The installer sets this, controller.allow_plugin_updates and controller.allow_plugin_install from one question about remote updates and plugin installs. See [Managing a Cluster](/docs/managing-a-cluster.md) for details on remote updates. Defaults to true. @@ -326,6 +327,7 @@ Defaults to "New Host". ### host.id Immutable numeric id of the host which uniquely identifies this host in the cluster. +The controller picks the id when the host is approved and the installer writes it here, the controller does not accept hosts with an id it has not approved. Hidden in the web interface. Defaults to a random 31 bit number. @@ -365,7 +367,8 @@ Defaults to "http://localhost:8080/". ### host.controller_token Access token used for authenticating with the controller. -You can generate an access token with `clusterioctl host generate-token --id `, or use the `clusterioctl host create-config` to create a new host config with the correct url and token in it. +The installer writes it together with host.id when the host is approved, see [Setting up remote hosts](/README.md#setting-up-remote-hosts). +The token only works while the host exists on the controller, if the host is revoked run the installer again in the host directory to get it approved again. The value can only be read and changed from the host itself. Defaults to "enter token here". @@ -418,6 +421,7 @@ Defaults to 60. Whether the `host update` command in clusterioctl and the update button in the web interface are allowed to update the `@clusterio/host` package with npm. For security reasons this can only be changed locally with `clusteriohost config set` while the host is stopped, and it is hidden in the web interface. +The installer sets this, host.allow_plugin_updates and host.allow_plugin_install from one question about remote updates and plugin installs. See [Managing a Cluster](/docs/managing-a-cluster.md) for details on remote updates. Defaults to true. diff --git a/docs/devs/plugin-migration-to-2.1.md b/docs/devs/plugin-migration-to-2.1.md index 30545fdf..2161e4a9 100644 --- a/docs/devs/plugin-migration-to-2.1.md +++ b/docs/devs/plugin-migration-to-2.1.md @@ -59,3 +59,23 @@ Imports of `assert`, `buffer`, `events`, `os`, `path`, `process`, `stream`, `uti The `container` property of `WebPluginContext` is now the plugin's ES module namespace, `await container.get("./web")` returns the module instead of a factory function. `lib.PluginWebpackEnvInfo` is renamed to `lib.PluginWebEnvInfo`. + + +## Host pairing + +Host tokens are no longer generated by users. +A host joins the cluster by running the installer and having the code it shows approved, see [Host pairing](protocol.md#host-pairing) for the HTTP endpoints it uses. + +- `HostGenerateTokenRequest` and `HostConfigCreateRequest` are removed. + Pending requests are listed with `HostPairingListRequest`, approved with `HostPairingApproveRequest` and rejected with `HostPairingRejectRequest`, and changes to them are sent to subscribers of `HostPairingUpdatesEvent`. +- `HostRevokeTokensRequest` is replaced by `HostRevokeRequest`, which deletes the host instead of only invalidating its tokens. + It throws `ConflictError` while instances are assigned to the host. +- The permissions `core.host.generate_token`, `core.host.revoke_token` and `core.host.create_config` are removed. + Check `core.host.approve` and `core.host.revoke` instead. +- `clusteriocontroller bootstrap generate-host-token` is removed, scripts that set up hosts should run the installer with `--controller-token` to approve the host with a user token. + +Host records are no longer created when a host connects, only by approving it, and the controller rejects hosts it has no record of. +Revoking a host deletes its record and note. +Plugins that keep data per host should remove it when the host is deleted. +On the controller the `update` event of `controller.hosts` gives the record with `isDeleted` set to true, and subscribers of `HostUpdatesEvent` receive the host with `isDeleted` set to true. +A host that is revoked and installed again gets a new id. diff --git a/docs/devs/protocol.md b/docs/devs/protocol.md index 0f9cf15e..8408d2cb 100644 --- a/docs/devs/protocol.md +++ b/docs/devs/protocol.md @@ -32,6 +32,63 @@ The [send-buffer setting](../configuration.md#controllermax_send_buffer_size) co ![Connection flow](../assets/connection_flow.svg) +## Host pairing + +The host id and token sent in `registerHost` come from the installer, which gets them from the controller over HTTP before the host is started. +Hosts are only added to the controller by approving a pairing request, and `registerHost` is rejected with close code 3000 (Unauthorized) for a host id the controller has no record of. +Revoking a host deletes its record, so its token stops working and the installer has to pair it again, which gives it a new id. +See [Setting up remote hosts](/README.md#setting-up-remote-hosts) for how this looks to the user. + +The endpoints below take and return JSON bodies and are sent with `Cache-Control: no-store`. +The `POST` endpoints return errors as `{"error": ""}` and respond with 503 while the controller is in recovery mode. +A request body that is not valid JSON or is larger than 4 kB is rejected with 400 or 413 before it reaches the endpoint, without a JSON body. +`GET /api/host-pairing/check` also works in recovery mode, and responds to a token that is not valid with a plain 401 without a JSON body. +A user or host token is sent in either an `Authorization: Bearer ` header or an `X-Access-Token` header. + +### `POST /api/host-pairing` + +Creates a request for a host to join the cluster. + +- `name: string` - Name of the host, at most 100 characters after trimming whitespace, without control characters. +- `version: string` (optional) - Version of the installer. + +Responds with 202 and `{"requestId": string, "code": string}`. +The `code` is shown to the user to approve the request with, formatted as six characters like `ABC-DEF`. +Codes are matched ignoring case and anything that is not a letter or digit, so `abcdef` matches `ABC-DEF`. +The `requestId` is a secret used to poll for the result, it is never shown to users. + +If a user token is sent and the user has the `core.host.approve` permission the host is created right away and the response is 200 with the same body as an approved poll. +An invalid token gives 401 and a user without the permission gives 403, a token is never ignored to create a pending request instead. + +A missing or too long name gives 400. +Too many requests from the same address or in total gives 429 with a `Retry-After` header in seconds. + +### `POST /api/host-pairing/poll` + +- `requestId: string` - Id returned when the request was created. + +Responds with 202 and `{"status": "pending"}` while the request is waiting for approval. +Once approved it responds with 200 and `{"hostId": integer, "token": string, "name": string}`, where `name` is the name the host was approved with, which can differ from the one asked for. +Rejected requests give 403, and unknown or expired requests give 404. +The result of an approved or rejected request is only returned once, after which the request is gone. + +A pending request expires 30 seconds after it was last polled, and after an hour of waiting. +The installer polls every 2 seconds. + +### `POST /api/host-pairing/cancel` + +- `requestId: string` - Id returned when the request was created. + +Removes a pending request, sent by the installer when it stops waiting. +Responds with 204 whether or not the request existed. + +### `GET /api/host-pairing/check` + +Checks a host token. +Responds with 200 and `{"valid": true}` if the token is valid and the host it was issued to still exists, and with 401 otherwise. +The installer uses this to tell whether an existing host config still works when it is run again. + + ## Connector flow To manage the complexity of connecting, reconnecting, resuming or invalidating dropped sessions the WebSocket connection for a `Link` class is managed by a connector. @@ -119,6 +176,10 @@ Host handshake for establishing a new connection session. - `plugins: Object` - Object mapping plugin names to plugin version for plugins that are available on the host. +The controller closes the connection with code 3000 (Unauthorized) if the token is invalid, was issued for another id, or if there is no host with this id, see [Host pairing](#host-pairing). +If this happens when the host starts it exits with code 8, and the run scripts created by the installer do not restart it. +A host that is revoked while connected is disconnected, is rejected when it tries to connect again, and shuts down. + #### `registerControl` Control handshake for enstablishing a new connection session. diff --git a/docs/managing-a-cluster.md b/docs/managing-a-cluster.md index b96c57e8..20ff255a 100644 --- a/docs/managing-a-cluster.md +++ b/docs/managing-a-cluster.md @@ -16,17 +16,40 @@ The easiest way to do this is to run `npx clusteriocontroller bootstrap create-c Lists all hosts. -### Generate token +Hosts are added to the cluster by running the installer on the computer the host runs on and approving the code it shows, see [Setting up remote hosts](/README.md#setting-up-remote-hosts). +Listing, approving and rejecting installers waiting for approval needs the `core.host.approve` permission. - ctl> host generate-token [--id ] +### List pending hosts -Generate a token for a host. `--id` is the host id. + ctl> host pending -### Create config +Lists installers waiting for approval with the code they show, the name they asked for, the address they connected from and how long they have been waiting. +A request goes away when its installer stops waiting, and after waiting for an hour. - ctl> host create-config [--id ] [--name ] [--generate-token ] [--output ] +### Approve host -Create a host config. `--id` id the host id. `--name` is the host name. `--generate-token` decides whether to also generate a token, defaulting to `0`. `--output` is the filename for the output of the config; Default is "config-host.json", "-" for stdout. + ctl> host approve [--name ] + +Approves the installer showing the code and prints the id of the new host. +The code is not case sensitive and the dash can be left out. +`--name` gives the host a different name than the one entered in the installer. +Approving gives the host full access to the cluster, check with `host pending` that the name and address match the computer you are setting up. + +### Reject host + + ctl> host reject + +Rejects the installer showing the code, the installer then exits with an error. + +### Revoke host + + ctl> host revoke + +Specify the host by name or ID. +Removes the host from the cluster, disconnects it and makes its token stop working. +Instances assigned to the host have to be unassigned first. +Requires the `core.host.revoke` permission. +To add the computer back to the cluster, run the installer again in the directory the host is installed in. ### Update Clusterio @@ -270,9 +293,11 @@ To be written. ## Plugins (Remote) -By default, remote updates are enabled unless you selected "no" during installation. -Remote updates allow ctl and the web UI to update plugins on machines within the cluster. -However, remote installations are disabled by default to prevent the risk of arbitrary code execution through the installation of malicious npm packages onto the machine. See below if you wish to enable this feature. +The installer asks "Allow remote updates and plugin installs from the web interface?" and the answer sets `allow_remote_updates`, `allow_plugin_updates` and `allow_plugin_install` together on that machine, pass `--remote-npm` or `--no-remote-npm` to answer it from the command line. +Remote updates allow ctl and the web UI to update Clusterio and plugins on machines within the cluster. +Remote installs allow them to install any package from npm onto the machine, which is a risk of arbitrary code execution through malicious npm packages, so only allow them if you trust everyone with access to install plugins. +In a config created without the installer remote updates are enabled and remote installs are disabled by default. +See below for changing these settings after installing. ### List diff --git a/docs/setting-up-tls.md b/docs/setting-up-tls.md index 8c2524db..a063b868 100644 --- a/docs/setting-up-tls.md +++ b/docs/setting-up-tls.md @@ -63,7 +63,7 @@ This creates a private key file named `key.pem` and a public certificate file na npx clusteriocontroller config set controller.tls_certificate cert.pem npx clusteriocontroller config set controller.tls_private_key key.pem -Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure node to trust it by setting the `NODE_EXTRA_CA_CERTS` environment variable to a path to this file before the host is started. You can do this by passing `--self-signed-cert path/to/cert` to the installer when setting up a host, or manually by editing run-host.cmd, run-host.sh and/or clusteriohost.service by adding the following line next to the one setting `NODE_OPTIONS`: +Additionally you will need copy the `cert.pem` file to all of the computers you want to set up as hosts and configure node to trust it by setting the `NODE_EXTRA_CA_CERTS` environment variable to a path to this file before the host is started. You can do this by passing `--self-signed-cert path/to/cert` to the installer when setting up a host, which also makes the installer trust the certificate when it asks the controller to approve the host, or manually by editing run-host.cmd, run-host.sh and/or clusteriohost.service by adding the following line next to the one setting `NODE_OPTIONS`: **For run-host.cmd** diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 46b04cb0..c21ecb94 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -23,7 +23,6 @@ import util from "util"; import winston from "winston"; import DailyRotateFile from "winston-daily-rotate-file"; -import jwt from "jsonwebtoken"; // homebrew modules import * as lib from "@clusterio/lib"; @@ -186,14 +185,6 @@ async function handleBootstrapCommand( // eslint-disable-next-line no-console console.log(userManager.signUserToken(user)); - } else if (subCommand === "generate-host-token") { - // eslint-disable-next-line no-console - console.log(jwt.sign( - { aud: "host", host: args.id }, - // Auth secret is never null after startup - Buffer.from(controllerConfig.get("controller.auth_secret")!, "base64") - )); - } else if (subCommand === "create-ctl-config") { let admin = userManager.getByName(args.name); if (!admin) { @@ -322,9 +313,6 @@ async function initialize(): Promise { .command("generate-user-token ", "Generate authentication token for the given user", yargs => { yargs.positional("name", { describe: "Name of the user", type: "string" }); }) - .command("generate-host-token ", "Generate authentication token for the given host", yargs => { - yargs.positional("id", { describe: "ID of the host", type: "number" }); - }) .command("create-ctl-config ", "Create clusterioctl config for the given user", yargs => { yargs.option("output", { describe: "Path to output config (- for stdout)", type: "string", diff --git a/packages/controller/index.ts b/packages/controller/index.ts index 6a8937b6..4b3e8f8a 100644 --- a/packages/controller/index.ts +++ b/packages/controller/index.ts @@ -6,6 +6,7 @@ export { default as HostConnection } from "./src/HostConnection.js"; export { default as InstanceRecord } from "./src/InstanceRecord.js"; export { default as InstanceManager } from "./src/InstanceManager.js"; export { default as HostRecord } from "./src/HostRecord.js"; +export { default as HostPairingManager, HostPairingError } from "./src/HostPairingManager.js"; export { BaseControllerPlugin, ControllerHooks, type ControllerPluginContext } from "./src/BaseControllerPlugin.js"; export { default as UserManager } from "./src/UserManager.js"; export { default as UserRecord } from "./src/UserRecord.js"; diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index d0f7b636..81849632 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -81,10 +81,11 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.ControllerConfigSetRequest, this.handleControllerConfigSetRequest.bind(this)); this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this)); this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this)); - this.handle(lib.HostRevokeTokensRequest, this.handleHostRevokeTokensRequest.bind(this)); + this.handle(lib.HostRevokeRequest, this.handleHostRevokeRequest.bind(this)); this.handle(lib.HostListRequest, this.handleHostListRequest.bind(this)); - this.handle(lib.HostGenerateTokenRequest, this.handleHostGenerateTokenRequest.bind(this)); - this.handle(lib.HostConfigCreateRequest, this.handleHostConfigCreateRequest.bind(this)); + this.handle(lib.HostPairingListRequest, this.handleHostPairingListRequest.bind(this)); + this.handle(lib.HostPairingApproveRequest, this.handleHostPairingApproveRequest.bind(this)); + this.handle(lib.HostPairingRejectRequest, this.handleHostPairingRejectRequest.bind(this)); this.handle(lib.InstanceDetailsGetRequest, this.handleInstanceDetailsGetRequest.bind(this)); this.handle(lib.InstanceDetailsListRequest, this.handleInstanceDetailsListRequest.bind(this)); this.handle(lib.InstanceCreateRequest, this.handleInstanceCreateRequest.bind(this)); @@ -248,49 +249,49 @@ export default class ControlConnection extends BaseConnection { this._controller.config.setProp(field as keyof lib.ControllerConfigFields, prop, value, "control"); } - async handleHostRevokeTokensRequest(request: lib.HostRevokeTokensRequest) { + async handleHostRevokeRequest(request: lib.HostRevokeRequest) { const host = this._controller.hosts.getMutable(request.hostId); if (!host) { - throw new Error(`Unknown host id (${request.hostId})`); + throw new lib.NotFoundError(`Unknown host id (${request.hostId})`); } - host.tokenValidAfter = Math.floor(Date.now() / 1000); + const assigned = [...this._controller.instances.values()].filter( + instance => instance.config.get("instance.assigned_host") === request.hostId + ); + if (assigned.length) { + const names = assigned.map(instance => instance.config.get("instance.name")).join(", "); + throw new lib.ConflictError( + `Cannot revoke host ${host.name} while instances are assigned to it: ${names}` + ); + } + this._controller.hosts.delete(host); + this._controller.deleteNote("host", request.hostId, this.user.name); + + // Disconnect before saving so the host is cut off even if the save fails. const hostConnection = this._controller.wsServer.hostConnections.get(request.hostId); if (hostConnection) { hostConnection.connector.terminate(); } - - this._controller.hosts.set(host); + logger.audit(`User ${this.user.name} revoked host ${host.name} (${host.id})`); + await this._controller.hosts.save(); } async handleHostListRequest(): Promise { return [...this._controller.hosts.values()].map(host => host.toHostDetails()); } - async handleHostGenerateTokenRequest(message: lib.HostGenerateTokenRequest) { - let hostId = message.hostId; - if (hostId === undefined) { - hostId = Math.random() * 2**31 | 0; - } - return this._controller.generateHostToken(hostId); + async handleHostPairingListRequest(): Promise { + this._controller.hostPairing.prune(); + return [...this._controller.hostPairing.requests.values()]; } - async handleHostConfigCreateRequest(request: lib.HostConfigCreateRequest) { - const hostConfig = new lib.HostConfig("host"); + async handleHostPairingApproveRequest(request: lib.HostPairingApproveRequest) { + return await this._controller.hostPairing.approve(request.code, request.name, this.user.name); + } - hostConfig.set("host.controller_url", this._controller.getControllerUrl()); - if (request.id !== undefined) { - hostConfig.set("host.id", request.id); - } - if (request.name !== undefined) { - hostConfig.set("host.name", request.name); - } - if (request.generateToken) { - this.user.checkPermission("core.host.generate_token"); - hostConfig.set("host.controller_token", this._controller.generateHostToken(hostConfig.get("host.id"))); - } - return hostConfig.toJSON(); + async handleHostPairingRejectRequest(request: lib.HostPairingRejectRequest) { + this._controller.hostPairing.reject(request.code, this.user.name); } async handleInstanceDetailsGetRequest(request: lib.InstanceDetailsGetRequest) { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index bbd2286d..5908c62e 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -29,6 +29,7 @@ import UserRecord from "./UserRecord.js"; import UserManager from "./UserManager.js"; import WsServer from "./WsServer.js"; import HostRecord from "./HostRecord.js"; +import HostPairingManager from "./HostPairingManager.js"; import { BaseControllerPlugin, ControllerHooks } from "./BaseControllerPlugin.js"; import ControllerRouter from "./ControllerRouter.js"; import InstanceManager from "./InstanceManager.js"; @@ -91,6 +92,8 @@ export default class Controller { users: UserManager; /** Instance manager for the cluster, also contains access to the user records */ instances: InstanceManager; + /** Pending requests from host installers to join the cluster */ + hostPairing: HostPairingManager; // Possible states are new, starting, running, stopping, stopped private _state: string = "new"; @@ -296,6 +299,7 @@ export default class Controller { this.users = new UserManager(users, roles, config); this.instances = new InstanceManager(instances, this); + this.hostPairing = new HostPairingManager(this); this.modStore.on("change", mod => { this.modsUpdated([mod]); @@ -305,6 +309,7 @@ export default class Controller { this.subscriptions = new lib.SubscriptionController(config.get("controller.max_subscription_filters")); this.subscriptions.handle(lib.SystemInfoUpdateEvent, this.handleSystemInfoSubscription.bind(this)); this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this)); + this.subscriptions.handle(lib.HostPairingUpdatesEvent, this.handleHostPairingSubscription.bind(this)); this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this)); this.subscriptions.handle( lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsSubscription.bind(this) @@ -318,6 +323,7 @@ export default class Controller { // Handle updates for datastores this.systems.on("update", this.systemsUpdated.bind(this)); this.hosts.on("update", this.hostsUpdated.bind(this)); + this.hostPairing.requests.on("update", this.hostPairingRequestsUpdated.bind(this)); this.instances.records.on("update", this.instanceUpdated.bind(this)); this.saves.on("update", this.savesUpdated.bind(this)); this.modPacks.on("update", this.modPacksUpdated.bind(this)); @@ -367,6 +373,7 @@ export default class Controller { async _startInternal(args: ControllerArgs) { this.logDirectory = args.logDirectory; + this.hostPairing.start(); this.clusterLogIndex = await lib.LogIndex.load(path.join(this.logDirectory, "cluster")); this.clusterLogBuildInterval = setInterval(() => { @@ -582,6 +589,8 @@ export default class Controller { this.autosaveInterval = undefined; } + this.hostPairing.stop(); + if (this.clusterLogIndex) { await this.clusterLogIndex.save(); } @@ -869,6 +878,10 @@ export default class Controller { static migrateRoles(rawJson: unknown[]): Static[] { const serialized = rawJson as Static[]; + // migrate: host token permissions were removed with host pairing in 2.1 + const removedPermissions = new Set([ + "core.host.generate_token", "core.host.revoke_token", "core.host.create_config", + ]); return serialized.map(json => { json.permissions = json.permissions.map(permission => { // migrate: core.instance.save.list.subscribe was renamed in alpha 17 @@ -876,7 +889,7 @@ export default class Controller { return "core.instance.save.subscribe"; } return permission; - }); + }).filter(permission => !removedPermissions.has(permission)); return lib.Role.pluginMigrations.reduce(json); }); } @@ -1017,6 +1030,20 @@ export default class Controller { this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates)); } + hostPairingRequestsUpdated(requests: lib.HostPairingDetails[]) { + this.subscriptions.broadcast(new lib.HostPairingUpdatesEvent(requests)); + } + + async handleHostPairingSubscription(request: lib.SubscriptionRequest) { + if (request.action === "unsubscribe") { + return null; + } + const requests = [...this.hostPairing.requests.values()].filter( + pairing => pairing.updatedAtMs > request.lastRequestTimeMs, + ); + return requests.length ? new lib.HostPairingUpdatesEvent(requests) : null; + } + async handleHostSubscription(request: lib.SubscriptionRequest) { if (request.action === "unsubscribe") { return null; diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 56d5d489..59eedfef 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -34,19 +34,16 @@ export default class HostConnection extends BaseConnection { this.plugins = new Map(Object.entries(registerData.plugins)); - const previousHostInfo = this._controller.hosts.get(registerData.id); - this.info = new HostRecord( - registerData.id, - previousHostInfo?.name ?? "", - registerData.version, - this.plugins, - true, - remoteAddress, - previousHostInfo?.publicAddress, - previousHostInfo?.tokenValidAfter, - 0, - false, - ); + // Hosts are only created by approving them, WsServer rejects hosts without a record. + const record = this._controller.hosts.getMutable(registerData.id); + if (!record) { + throw new Error(`Host ${registerData.id} is not registered`); + } + record.version = registerData.version; + record.plugins = this.plugins; + record.connected = true; + record.remoteAddress = remoteAddress; + this.info = record; this._controller.hosts.set(this.info); this._checkPluginVersions(); @@ -71,7 +68,7 @@ export default class HostConnection extends BaseConnection { this._controller.instances.records.setMany(instances); this.info.connected = false; - this._controller.hosts.set(this.info); + this._saveInfo(); }); this.handle(lib.GetFactorioCredentialsRequest, this.handleGetFactorioCredentialsRequest.bind(this)); @@ -175,10 +172,17 @@ export default class HostConnection extends BaseConnection { return {}; } + /** Store changes to the host record, unless the host was revoked */ + private _saveInfo() { + if (this._controller.hosts.getMutable(this.id) === this.info) { + this._controller.hosts.set(this.info); + } + } + async handleHostInfoUpdateEvent(event: lib.HostInfoUpdateEvent) { this.info.name = event.update.name; this.info.publicAddress = event.update.publicAddress; - this._controller.hosts.set(this.info); + this._saveInfo(); } async handleInstanceStatusChangedEvent(request: lib.InstanceStatusChangedEvent) { diff --git a/packages/controller/src/HostPairingManager.ts b/packages/controller/src/HostPairingManager.ts new file mode 100644 index 00000000..b420ca67 --- /dev/null +++ b/packages/controller/src/HostPairingManager.ts @@ -0,0 +1,361 @@ +import type Controller from "./Controller.js"; + +import crypto from "crypto"; +import net from "net"; + +import * as lib from "@clusterio/lib"; +import { logger } from "@clusterio/lib"; + +import HostRecord from "./HostRecord.js"; + +/** Symbols used in pairing codes, avoids 0/O and 1/I/L */ +const codeAlphabet = "23456789ABCDEFGHJKMNPQRSTUVWXYZ"; +const codeLength = 6; + +/** Time a request is kept after the last poll from the installer */ +export const pollTimeoutMs = 30e3; +/** Maximum time a request can stay pending */ +export const maxPendingMs = 60 * 60e3; +/** Maximum number of pending requests in total */ +export const maxPending = 50; +/** Maximum number of pending requests from one address */ +export const maxPendingPerAddress = 3; +/** Maximum number of requests created from one address per minute */ +export const maxCreatesPerMinute = 10; +/** Maximum length of a host name */ +export const maxNameLength = 100; +/** Control and bidirectional override characters refused in host names */ +const nameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u; + +/** Result of an approved request, sent to the installer */ +export interface HostPairingResult { + hostId: number, + token: string, + name: string, +} + +/** Error with an HTTP status code to respond to the installer with */ +export class HostPairingError extends Error { + constructor( + message: string, + public status: number, + /** Seconds to wait before trying again, for status 429 */ + public retryAfter?: number, + ) { + super(message); + } +} + +interface PairingRequest { + /** Secret the installer polls with, never shown to users */ + secret: string, + details: lib.HostPairingDetails, + /** Address with IPv6 addresses grouped by /64, used for limits */ + addressKey: string, + lastPollMs: number, + state: "pending" | "approving" | "approved" | "rejected", + result?: HostPairingResult, +} + +/** + * Strips the IPv4 mapped prefix and groups IPv6 addresses by their /64 + * + * @param address - Address to make a key for + * @returns key used to count requests from an address. + */ +export function addressKey(address: string) { + if (address.startsWith("::ffff:") && net.isIPv4(address.slice(7))) { + return address.slice(7); + } + if (!net.isIPv6(address)) { + return address; + } + const [head, tail = ""] = address.split("::"); + const headParts = head ? head.split(":") : []; + const tailParts = tail ? tail.split(":") : []; + const missing = 8 - headParts.length - tailParts.length; + const parts = address.includes("::") + ? [...headParts, ...new Array(missing).fill("0"), ...tailParts] + : headParts; + return `${parts.slice(0, 4).map(part => part.toLowerCase().replace(/^0+(?=.)/, "")).join(":")}::/64`; +} + +/** + * Normalise a pairing code typed by a user + * + * @param code - Code as typed, may contain dashes and lower case letters. + * @returns code formatted as shown by the installer. + */ +export function normaliseCode(code: string) { + const plain = code.toUpperCase().replace(/[^0-9A-Z]/g, ""); + return `${plain.slice(0, 3)}-${plain.slice(3)}`; +} + +function generateCode() { + let code = ""; + for (let i = 0; i < codeLength; i++) { + code += codeAlphabet[crypto.randomInt(codeAlphabet.length)]; + } + return normaliseCode(code); +} + +/** + * Keeps track of requests from host installers to join the cluster + * + * Requests only live in memory, and only while the installer keeps polling + * for the result. + */ +export default class HostPairingManager { + /** Pending requests by code, broadcast to subscribed control links */ + requests = new lib.SubscribableDatastore(); + private _bySecret = new Map(); + private _byCode = new Map(); + /** Creation times of recent requests by address key */ + private _recentCreates = new Map(); + private _pruneInterval?: ReturnType; + + constructor( + private _controller: Controller, + ) { } + + start() { + this._pruneInterval = setInterval(() => this.prune(), 5e3); + this._pruneInterval.unref(); + } + + stop() { + clearInterval(this._pruneInterval); + this._pruneInterval = undefined; + } + + /** + * Check and trim a host name given by an installer or a user + * + * @param name - Name to check. + * @returns the trimmed name. + */ + static validateName(name: unknown) { + if (typeof name !== "string" || !name.trim()) { + throw new HostPairingError("Host name is required", 400); + } + const trimmed = name.trim(); + if (trimmed.length > maxNameLength) { + throw new HostPairingError(`Host name must be at most ${maxNameLength} characters`, 400); + } + // Names come from unauthenticated requests and are printed to terminals and logs. + if (nameControlCharacters.test(trimmed)) { + throw new HostPairingError("Host name may not contain control characters", 400); + } + return trimmed; + } + + /** + * Create an approved host with a new id and token + * + * The host record is saved to disk before this returns. + * + * @param name - Name of the host. + * @returns the id, token and name of the new host. + */ + async createHost(name: string): Promise { + const hosts = this._controller.hosts; + let hostId; + do { + hostId = crypto.randomInt(1, 2 ** 31); + } while (hosts.has(hostId)); + + const record = new HostRecord( + hostId, name, "", new Map(), false, "", "", Math.floor(Date.now() / 1000), + ); + hosts.set(record); + try { + await hosts.save(); + } catch (err) { + hosts.delete(record); + throw err; + } + return { hostId, token: this._controller.generateHostToken(hostId), name }; + } + + /** + * Create a pending request + * + * @param name - Name the installer asked for. + * @param version - Version of the installer. + * @param remoteAddress - Address the request came from. + * @returns the secret to poll with and the code to show the user. + */ + create(name: string, version: string, remoteAddress: string) { + this.prune(); + const key = addressKey(remoteAddress); + const now = Date.now(); + const recent = (this._recentCreates.get(key) ?? []).filter(ms => ms > now - 60e3); + if (recent.length >= maxCreatesPerMinute) { + throw new HostPairingError( + "Too many requests from this address", 429, Math.ceil((recent[0] + 60e3 - now) / 1000), + ); + } + const pendingFromAddress = [...this._bySecret.values()].filter( + request => request.addressKey === key && request.state === "pending" + ).length; + if (pendingFromAddress >= maxPendingPerAddress) { + throw new HostPairingError("Too many pending requests from this address", 429, 30); + } + if (this.requests.size >= maxPending) { + throw new HostPairingError("Too many pending requests", 429, 30); + } + recent.push(now); + this._recentCreates.set(key, recent); + + let code; + do { + code = generateCode(); + } while (this._byCode.has(code)); + + const request: PairingRequest = { + secret: crypto.randomBytes(32).toString("base64url"), + details: new lib.HostPairingDetails(code, name, remoteAddress, version, now), + addressKey: key, + lastPollMs: now, + state: "pending", + }; + this._bySecret.set(request.secret, request); + this._byCode.set(code, request); + this.requests.set(request.details); + logger.verbose(`Host pairing request ${code} for "${name}" from ${remoteAddress}`); + return { requestId: request.secret, code }; + } + + /** + * Poll for the result of a request + * + * @param secret - Request id returned when the request was created. + * @returns the result if the request was approved, or null if still pending. + */ + poll(secret: string): HostPairingResult | null { + this.prune(); + const request = this._bySecret.get(secret); + if (!request) { + throw new HostPairingError("Request not found", 404); + } + request.lastPollMs = Date.now(); + if (request.state === "approved") { + this._remove(request); + return request.result!; + } + if (request.state === "rejected") { + this._remove(request); + throw new HostPairingError("Request was rejected", 403); + } + return null; + } + + /** + * Cancel a pending request + * + * @param secret - Request id returned when the request was created. + */ + cancel(secret: string) { + const request = this._bySecret.get(secret); + if (request && request.state === "pending") { + logger.verbose(`Host pairing request ${request.details.code} cancelled by the installer`); + this._remove(request); + } + } + + private _pending(code: string) { + this.prune(); + const request = this._byCode.get(normaliseCode(code)); + if (!request || request.state !== "pending") { + throw new lib.NotFoundError(`No pending host request with code ${code}`); + } + return request; + } + + /** + * Approve a pending request, creating the host + * + * @param code - Code of the request. + * @param name - Name to give the host, defaults to the requested name. + * @param approvedBy - Name of the user approving it, for logging. + * @returns the id of the new host. + */ + async approve(code: string, name: string | undefined, approvedBy: string) { + const request = this._pending(code); + let hostName; + try { + hostName = name === undefined ? request.details.name : HostPairingManager.validateName(name); + } catch (err: any) { + throw new lib.ValidationError(err.message); + } + + request.state = "approving"; + let result; + try { + result = await this.createHost(hostName); + } catch (err) { + request.state = "pending"; + throw err; + } + + request.state = "approved"; + request.result = result; + this.requests.delete(request.details); + logger.audit( + `User ${approvedBy} approved host ${result.name} (${result.hostId}) with code ` + + `${request.details.code} from ${request.details.remoteAddress}` + ); + return result.hostId; + } + + /** + * Reject a pending request + * + * @param code - Code of the request. + * @param rejectedBy - Name of the user rejecting it, for logging. + */ + reject(code: string, rejectedBy: string) { + const request = this._pending(code); + request.state = "rejected"; + this.requests.delete(request.details); + logger.audit( + `User ${rejectedBy} rejected host ${request.details.name} with code ` + + `${request.details.code} from ${request.details.remoteAddress}` + ); + } + + private _remove(request: PairingRequest) { + this._bySecret.delete(request.secret); + this._byCode.delete(request.details.code); + if (this.requests.has(request.details.code)) { + this.requests.delete(request.details); + } + } + + /** Remove requests the installer stopped polling for or that waited too long */ + prune() { + const now = Date.now(); + for (const request of this._bySecret.values()) { + if (request.state === "approving") { + continue; + } + const expired = now - request.lastPollMs > pollTimeoutMs + || (request.state === "pending" && now - request.details.createdAtMs > maxPendingMs); + if (!expired) { + continue; + } + if (request.state === "approved") { + logger.warn( + `Approved host ${request.result!.name} (${request.result!.hostId}) was never picked up ` + + "by its installer, revoke it on the hosts page if it is not needed" + ); + } + this._remove(request); + } + for (const [key, times] of this._recentCreates) { + if (!times.some(ms => ms > now - 60e3)) { + this._recentCreates.delete(key); + } + } + } +} diff --git a/packages/controller/src/WsServer.ts b/packages/controller/src/WsServer.ts index ce4bb2b3..6320a338 100644 --- a/packages/controller/src/WsServer.ts +++ b/packages/controller/src/WsServer.ts @@ -264,7 +264,10 @@ ${err.stack}` } let host = this.controller.hosts.get(data.id); - if (!tokenPayload.iat || tokenPayload.iat < (host?.tokenValidAfter ?? 0)) { + if (!host) { + throw new Error("host is not registered, run the installer again in the host directory"); + } + if (!tokenPayload.iat || tokenPayload.iat < host.tokenValidAfter) { throw new Error("invalid token"); } } catch (err: any) { @@ -303,6 +306,13 @@ ${err.stack}` ); } + // The host may have been revoked while waiting for the old connection to close. + if (!this.controller.hosts.has(data.id)) { + wsRejectedConnectionsCounter.inc(); + socket.close(lib.ConnectionClosed.Unauthorized, "Authentication failed: host was revoked"); + return; + } + let [connector, sessionToken] = this.createSession(new lib.Address(lib.Address.host, data.id)); // Handle close before HostConnection does. This avoids a host diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 9b7f96fa..b9ce7803 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -1,5 +1,6 @@ -import type { Application, Request, Response } from "express"; +import express, { type Application, type Request, type Response } from "express"; import Controller from "./Controller.js"; +import HostPairingManager, { HostPairingError } from "./HostPairingManager.js"; import busboy from "busboy"; import crypto from "crypto"; @@ -215,19 +216,7 @@ function validateUserToken(req: Request, res: Response, next: any) { } try { - const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "user" }); - if (typeof tokenPayload === "string") { - throw new Error("unexpected JsonWebToken type"); - } - let user = req.app.locals.controller.users.getByName(tokenPayload.user); - if (!user) { - throw new Error("invalid user"); - } - if (!tokenPayload.iat || tokenPayload.iat < user.tokenValidAfter) { - throw new Error("invalid token"); - } - res.locals.user = user; - + res.locals.user = userFromToken(req.app.locals.controller, token); } catch (err) { res.sendStatus(401); return; @@ -236,6 +225,118 @@ function validateUserToken(req: Request, res: Response, next: any) { next(); } +function userFromToken(controller: Controller, token: string) { + const tokenPayload = jwt.verify(token, controller.authSecret, { audience: "user" }); + if (typeof tokenPayload === "string") { + throw new Error("unexpected JsonWebToken type"); + } + let user = controller.users.getByName(tokenPayload.user); + if (!user) { + throw new Error("invalid user"); + } + if (!tokenPayload.iat || tokenPayload.iat < user.tokenValidAfter) { + throw new Error("invalid token"); + } + return user; +} + +// Responds with the status of a pairing error, returns false for other errors. +function sendHostPairingError(res: Response, err: unknown) { + if (!(err instanceof HostPairingError)) { + return false; + } + if (err.retryAfter !== undefined) { + res.set("Retry-After", String(err.retryAfter)); + } + res.status(err.status).json({ error: err.message }); + return true; +} + +// Checks recovery mode and the request id for the pairing endpoints, returns false if responded to. +function checkHostPairingRequest(req: Request, res: Response, needsRequestId: boolean) { + if (req.app.locals.controller.recoveryMode) { + res.status(503).json({ error: "Controller is in recovery mode" }); + return false; + } + if (needsRequestId && typeof req.body?.requestId !== "string") { + res.status(400).json({ error: "requestId is required" }); + return false; + } + return true; +} + +// Request from an installer for a host to join the cluster. +async function createHostPairing(req: Request, res: Response) { + if (!checkHostPairingRequest(req, res, false)) { + return; + } + const controller = req.app.locals.controller; + const remoteAddress = controller.wsServer.remoteAddr(req); + try { + const name = HostPairingManager.validateName(req.body?.name); + const version = typeof req.body?.version === "string" ? req.body.version.slice(0, 64) : ""; + + // A user allowed to approve hosts can skip the wait by passing their token. + const token = getAccessToken(req); + if (token !== undefined) { + let user; + try { + user = userFromToken(controller, token); + } catch (err) { + res.status(401).json({ error: "Invalid user token" }); + return; + } + try { + user.checkPermission("core.host.approve"); + } catch (err) { + res.status(403).json({ error: `User ${user.name} is not allowed to approve hosts` }); + return; + } + const result = await controller.hostPairing.createHost(name); + logger.audit( + `User ${user.name} approved host ${result.name} (${result.hostId}) from ${remoteAddress} ` + + "with their token" + ); + res.status(200).json(result); + return; + } + + res.status(202).json(controller.hostPairing.create(name, version, remoteAddress)); + } catch (err) { + if (!sendHostPairingError(res, err)) { + throw err; + } + } +} + +// Poll from an installer for the result of its request. +function pollHostPairing(req: Request, res: Response) { + if (!checkHostPairingRequest(req, res, true)) { + return; + } + try { + const result = req.app.locals.controller.hostPairing.poll(req.body.requestId); + if (result) { + res.status(200).json(result); + } else { + res.status(202).json({ status: "pending" }); + } + } catch (err) { + if (!sendHostPairingError(res, err)) { + throw err; + } + } +} + +// Cancel from an installer that stopped waiting. +function cancelHostPairing(req: Request, res: Response) { + if (!checkHostPairingRequest(req, res, true)) { + return; + } + req.app.locals.controller.hostPairing.cancel(req.body.requestId); + res.sendStatus(204); +} + // Handle an uploaded export package. async function uploadExport(req: Request, res: Response) { if (req.get("Content-Type") !== "application/zip") { @@ -686,6 +787,21 @@ export function addRouteHandlers(app: Application) { ); app.get("/api/plugins", getPlugins); app.get("/api/cluster-name", getClusterName); + // Set before parsing so errors from the body parser and token check are not cached either. + const noStore = (req: Request, res: Response, next: any) => { + res.set("Cache-Control", "no-store"); + next(); + }; + const pairingJson = express.json({ limit: "4kb" }); + app.post("/api/host-pairing", + noStore, pairingJson, + (req:Request, res:Response, next:any) => createHostPairing(req, res).catch(next) + ); + app.post("/api/host-pairing/poll", noStore, pairingJson, pollHostPairing); + app.post("/api/host-pairing/cancel", noStore, pairingJson, cancelHostPairing); + app.get("/api/host-pairing/check", noStore, validateHostToken, (req: Request, res: Response) => { + res.json({ valid: true }); + }); app.put("/api/upload-export", validateHostToken, (req:Request, res:Response, next:any) => uploadExport(req, res).catch(next) diff --git a/packages/create/README.md b/packages/create/README.md index 2483f03b..166e449b 100644 --- a/packages/create/README.md +++ b/packages/create/README.md @@ -4,3 +4,119 @@ Installer for Clusterio. Install the latest [Node.js LTS release](https://nodejs.org/en/download) before running the installer. See [the Clusterio installation guide](https://github.com/clusterio/clusterio#installation) for platform-specific instructions. + +Run the installer in a new empty directory: + + npm init "@clusterio" + +It asks which components to install and how to set them up. +Every question can be answered with an option instead, pass options to the installer after `--`: + + npm init "@clusterio" -- --mode host --host-name my-host --controller-url https://example.com/ + +The installer runs without prompts when all the questions for the mode are answered with options. + + +## Modes + +- `standalone`: controller and host on this computer. + The host is added to the controller directly and does not need to be approved. +- `controller`: controller only. +- `host`: host only, joins the cluster of an existing controller, see [Setting up a host](#setting-up-a-host). +- `ctl`: the clusterioctl command line interface only. +- `plugins`: plugins only, for adding plugins to an existing installation. +- `plugin-template`: files for writing a new plugin. + + +## Options + +| Option | Modes | Description | +| --- | --- | --- | +| `--mode ` | all | Operating mode to install. | +| `--admin ` | standalone, controller | Name of the admin account to create. | +| `--http-port ` | standalone, controller | Port the controller web interface listens on, defaults to 8080. | +| `--host-name ` | host | Name to request for the host, defaults to the name of the computer. | +| `--controller-url ` | host, ctl | URL of the controller, the same as for its web interface. | +| `--controller-token ` | host, ctl | ctl: authentication token saved for clusterioctl. host: authentication token of a user that approves the host without waiting, it is not saved. | +| `--no-pair` | host | Install the host without pairing it with the controller. | +| `--public-address
` | standalone, host | DNS name or IP address players connect to this computer with. | +| `--factorio-dir ` | standalone, host | Path to the Factorio installation. | +| `--self-signed-cert ` | standalone, host | Certificate to trust for a controller using a self-signed certificate. It is used for pairing and added to the run scripts. | +| `--remote-npm`, `--no-remote-npm` | standalone, controller, host | Allow or deny remote updates, plugin updates and plugin installs from the web interface. | +| `--plugins ` | all | Plugins to install, pass `--plugins` without packages to install none. | +| `--plugin-template `, `--plugin-name ` | plugin-template | Templates to use and name of the new plugin. | +| `--log-level ` | all | Log level to print, defaults to info. | +| `--allow-install-as-root` | all | Linux only, allow installing as root (not recommended). | +| `--migrate-rename` | | Migrate an installation from before the slave/master rename. | + + +## Setting up a host + +A new host has to be approved before it can join the cluster: + +1. Run the installer with the host mode on the computer for the host. + It asks for a name for the host, the controller URL, the public address, the Factorio installation, whether to allow remote updates and which plugins to install. + +2. Once it has installed everything it prints a code and waits for the host to be approved: + + Waiting for approval. Approve "my-host" with code ABC-DEF on the Hosts page of https://example.com/, or run: npx clusterioctl host approve ABC-DEF + +3. Approve the request by clicking Add host on the Hosts page of the web interface, or run `npx clusterioctl host approve ABC-DEF`. + Only approve a request if its code matches the one shown by the installer. + Approving requires the `core.host.approve` permission. + +4. The installer saves `host.id`, `host.name`, `host.controller_url` and `host.controller_token` to `config-host.json` and exits. + Start the host with `run-host.cmd`, `run-host.sh` or the systemd service in the `systemd` folder. + +Press Ctrl+C while the installer waits to cancel the request. +A request that is not approved within an hour expires. + +Pass `--controller-token ` with the token of a user that has the `core.host.approve` permission to approve the host without waiting. +The token is only used for approving the host and is not saved. + +If the controller uses a self-signed certificate pass it with `--self-signed-cert `, see [setting up TLS](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate). + +When a host is revoked in the web interface or with `npx clusterioctl host revoke ` its token stops working and a running host shuts down. +Starting it again fails with exit code 8, which the run scripts do not restart on. + + +## Running the installer again + +Running the installer with the host mode in a directory with `config-host.json` in it does not install anything, it only pairs the host again if needed: + +- If the controller accepts the saved token it prints "Host is already set up" and exits with code 0. +- If it does not, for example because the host was revoked or never approved, it asks for the host name and controller URL, defaulting to the saved ones, and pairs the host again. + The host gets a new id, other settings in `config-host.json` are kept. +- If the controller cannot be reached it prints a warning, keeps the saved config and exits with code 0. + +The token is only checked with the controller in `config-host.json`. +Passing a different `--controller-url` pairs the host with that controller instead, without sending it the saved token, and the host gets a new id. +If the same controller only moved to a new address, change the address with `npx clusteriohost config set host.controller_url ` instead, which keeps the host. + + +## Docker + +To run a host in a container install it when building the image with all answers given as options and `--no-pair`. +Then run the installer again with the host mode when the container starts, it pairs the host on the first start and exits straight away on later starts. + +```dockerfile +FROM node:22 +# Install the Factorio headless server to /opt/factorio here +RUN mkdir /clusterio && chown node:node /clusterio +USER node +WORKDIR /clusterio +RUN npm init "@clusterio" -- --mode host --no-pair --public-address factorio.example.com \ + --factorio-dir /opt/factorio --no-remote-npm --plugins +VOLUME /clusterio +CMD npm init "@clusterio" -- --mode host --controller-url "$CONTROLLER_URL" --host-name "$HOST_NAME" \ + ${CONTROLLER_TOKEN:+--controller-token "$CONTROLLER_TOKEN"} && exec ./run-host.sh +``` + + docker run -d --name my-host -v my-host:/clusterio -p 34100-34199:34100-34199/udp \ + -e CONTROLLER_URL=https://example.com/ -e HOST_NAME=my-host my-host-image + docker logs my-host + +- The volume holds `config-host.json`, without it every new container pairs as a new host. + Docker copies the installation from the image into a new empty volume, later image updates do not change the installation in an existing volume. +- The code to approve is printed to the container log, or set `CONTROLLER_TOKEN` to approve the host without waiting. +- Stop the container after revoking its host, otherwise a restart runs the installer again and the host asks to join again. diff --git a/packages/create/create.js b/packages/create/create.js index 342fa6a6..8a8aac9a 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -10,6 +10,7 @@ import util from "node:util"; import yargs from "yargs"; import { levels, logger, setLogLevel } from "./logging.js"; +import { checkHostToken, createLocalHost, pairHost, PairingError } from "./pair.js"; import { copyPluginTemplates } from "./template.js"; let dev = false; const scriptExt = process.platform === "win32" ? ".cmd" : ""; @@ -161,22 +162,193 @@ async function execCtl(args) { return await execFile(path.join("node_modules", ".bin", `clusterioctl${scriptExt}`), args); } -function validateHostToken(token) { - let parts = token.split("."); - if (parts.length !== 3) { - throw new InstallError("Invalid token"); +function validateHostName(input) { + if (!input || !input.trim()) { + return "Host name may not be empty"; } + if (input.trim().length > 100) { + return "Host name may be at most 100 characters"; + } + if (hostNameControlCharacters.test(input)) { + return "Host name may not contain control characters"; + } + return true; +} - let parsed; +// Same characters as the controller refuses in host names. +const hostNameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u; + +function validateControllerUrl(input) { + let url; try { - parsed = JSON.parse(Buffer.from(parts[1], "base64")); + url = new URL(input); } catch (err) { - throw new InstallError("Invalid token"); + return "Controller URL must be a URL like https://example.com/"; + } + if (!["http:", "https:"].includes(url.protocol)) { + return "Controller URL must start with http:// or https://"; + } + return true; +} + +// The host requires the controller URL to end with a slash. +function normaliseControllerUrl(input) { + const url = new URL(input); + if (!url.pathname.endsWith("/")) { + url.pathname += "/"; + } + return url.href; +} + +async function readSelfSignedCert(args) { + if (!args.selfSignedCert) { + return undefined; + } + try { + return await fs.readFile(args.selfSignedCert); + } catch (err) { + throw new InstallError(`Failed to read --self-signed-cert ${args.selfSignedCert}: ${err.message}`); + } +} + +async function installerVersion() { + return JSON.parse(await fs.readFile(path.join(import.meta.dirname, "package.json"))).version; +} + +// Remote updates, plugin updates and plugin installs are allowed or denied together. +async function setRemoteUpdates(exec, component, allow) { + for (let field of ["allow_remote_updates", "allow_plugin_updates", "allow_plugin_install"]) { + await exec(["config", "set", `${component}.${field}`, allow]); + } +} + +// Turn an error from pairing into a message for the user. +function pairingFailed(err) { + const retry = "Run the installer again with --mode host in this directory to retry."; + if (err.name === "AbortError") { + return new InstallError(`Pairing cancelled. ${retry}`); + } + if (err instanceof PairingError) { + return new InstallError(`Pairing failed: ${err.message}. ${retry}`); + } + if (typeof err.code === "string") { + const hint = /CERT|SELF_SIGNED|UNABLE_TO_VERIFY/.test(err.code) + ? " Pass the certificate with --self-signed-cert if the controller uses a self-signed one." + : ""; + return new InstallError(`Failed to connect to the controller: ${err.message}.${hint} ${retry}`); + } + return err; +} + +// Request to join the cluster and save the id and token of the host once approved. +async function pairWithController(args, { hostName, controllerUrl }) { + const ca = await readSelfSignedCert(args); + await execHost(["config", "set", "host.name", hostName]); + await execHost(["config", "set", "host.controller_url", controllerUrl]); + + // Ctrl-C may arrive twice under npm init, as npm forwards it to the installer, so keep + // listening until the request has been cancelled. + const abort = new AbortController(); + const onSigint = () => { abort.abort(); }; + process.on("SIGINT", onSigint); + // The close patch for inquirer on Windows leaves the console in raw mode, where Ctrl-C is + // read as input instead of raising SIGINT. No prompts follow, so it can be turned off. + if (process.platform === "win32" && process.stdin.isTTY && process.stdin.isRaw) { + process.stdin.setRawMode(false); + } + let result; + try { + logger.info(`Requesting to join the cluster at ${controllerUrl}`); + result = await pairHost({ + controllerUrl, + name: hostName, + version: await installerVersion(), + token: args.controllerToken, + ca, + signal: abort.signal, + onPending: ({ code }) => { + // eslint-disable-next-line no-console + console.log( + `Waiting for approval. Approve "${hostName}" with code ${code} on the Hosts page of ` + + `${controllerUrl}, or run: npx clusterioctl host approve ${code}` + ); + }, + }); + } catch (err) { + throw pairingFailed(err); + } finally { + process.off("SIGINT", onSigint); } - if (parsed.aud !== "host" || !Number.isInteger(parsed.host)) { - throw new InstallError("Invalid token"); + logger.info(`Host approved as "${result.name}" with id ${result.hostId}`); + await execHost(["config", "set", "host.id", result.hostId]); + if (result.name !== hostName) { + await execHost(["config", "set", "host.name", result.name]); } + // Saved last so that an interrupted install is paired again when the installer is run again. + await execHost(["config", "set", "host.controller_token", result.token]); +} + +// Pair a host that is already installed again, unless the controller still accepts it. +async function rerunHost(args) { + let config; + try { + config = JSON.parse(await fs.readFile("config-host.json")); + } catch (err) { + throw new InstallError(`Failed to read config-host.json: ${err.message}`); + } + // Let a rerun fix run scripts written before the certificate was passed. + if (!dev && args.selfSignedCert) { + logger.info("Writing run scripts"); + await writeScripts("host", args.selfSignedCert); + } + if (!args.pair) { + // eslint-disable-next-line no-console + console.log("Host is already installed"); + return; + } + + const token = config["host.controller_token"]; + const controllerUrl = config["host.controller_url"]; + let otherController = false; + if (args.controllerUrl !== undefined) { + const result = validateControllerUrl(args.controllerUrl); + if (result !== true) { + throw new InstallError(result); + } + const savedUrl = validateControllerUrl(controllerUrl) === true ? normaliseControllerUrl(controllerUrl) : ""; + otherController = normaliseControllerUrl(args.controllerUrl) !== savedUrl; + } + // The token is never sent to another controller, it would give that controller access to this cluster. + if (otherController) { + logger.info(`Pairing with ${args.controllerUrl} instead of the controller in config-host.json`); + } else if (token && token !== "enter token here") { + let valid; + try { + valid = await checkHostToken({ controllerUrl, token, ca: await readSelfSignedCert(args) }); + } catch (err) { + if (err instanceof InstallError) { + throw err; + } + logger.warn(`Could not check if ${controllerUrl} accepts this host: ${err.message}`); + logger.warn("Keeping the existing host config, pass --controller-url to pair with another controller"); + return; + } + if (valid) { + // eslint-disable-next-line no-console + console.log("Host is already set up"); + return; + } + logger.info("The controller no longer accepts this host, pairing it again"); + } + + const answers = await inquirerPairingArgs(args, { + hostName: config["host.name"], + controllerUrl: config["host.controller_url"], + }); + await pairWithController(args, answers); + // eslint-disable-next-line no-console + console.log("Successfully set up host"); } async function validateInstallDir() { @@ -509,6 +681,37 @@ WantedBy=multi-user.target } } +async function inquirerPairingArgs(args, defaults = {}) { + let answers = {}; + if (args.hostName) { answers.hostName = args.hostName; } + if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; } + answers = await inquirer.prompt([ + { + type: "input", + name: "hostName", + message: "Name of host", + default: defaults.hostName ?? os.hostname(), + validate: validateHostName, + }, + { + type: "input", + name: "controllerUrl", + message: "Controller URL", + default: defaults.controllerUrl, + validate: validateControllerUrl, + }, + ], answers); + + // Inquirer does not validate answers given as arguments + for (let result of [validateHostName(answers.hostName), validateControllerUrl(answers.controllerUrl)]) { + if (result !== true) { + throw new InstallError(result); + } + } + answers.controllerUrl = normaliseControllerUrl(answers.controllerUrl); + return answers; +} + // eslint-disable-next-line complexity async function inquirerMissingArgs(args) { let answers = {}; @@ -568,17 +771,16 @@ async function inquirerMissingArgs(args) { } if (answers.mode === "host") { - if (args.hostName) { answers.hostName = args.hostName; } - answers = await inquirer.prompt([ - { - type: "input", - name: "hostName", - message: "Name of host", - }, - ], answers); + if (await pathExists("config-host.json")) { + // An installed host is only paired again, see rerunHost + return { mode: "host", rerun: true }; + } + if (args.pair) { + answers = { ...answers, ...await inquirerPairingArgs(args) }; + } } - if (["host", "ctl"].includes(answers.mode)) { + if (answers.mode === "ctl") { if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; } answers = await inquirer.prompt([ { @@ -598,10 +800,6 @@ async function inquirerMissingArgs(args) { ], answers); } - if (answers.mode === "host") { - validateHostToken(answers.controllerToken); - } - if (["standalone", "host"].includes(answers.mode)) { let myIp = "localhost"; if (args.publicAddress) { @@ -663,7 +861,7 @@ async function inquirerMissingArgs(args) { { type: "confirm", name: "remoteNpm", - message: "Allow remote updates via npm?", + message: "Allow remote updates and plugin installs from the web interface?", default: true, }, ], answers); @@ -749,7 +947,12 @@ async function main() { nargs: 1, describe: "Controller URL [host/ctl]", type: "string", }) .option("controller-token", { - nargs: 1, describe: "Controller authentication token [host/ctl]", type: "string", + nargs: 1, type: "string", describe: + "User token for ctl, or of a user that approves the host without waiting (not saved) [host/ctl]", + }) + .option("pair", { + describe: "Pair the host with the controller, use --no-pair to only install it [host]", + default: true, type: "boolean", }) .option("public-address", { nargs: 1, describe: "DNS/IP Address to connect to this server [standalone/host]", type: "string", @@ -761,7 +964,8 @@ async function main() { nargs: 1, describe: "Path to self-signed certificate of the controller [standalone/host]", type: "string", }) .option("remote-npm", { - nargs: 0, description: "Allow remote updates via npm [standalone/controller/host]", type: "boolean", + nargs: 0, type: "boolean", description: + "Allow remote updates and plugin installs from the web interface [standalone/controller/host]", }) .option("plugins", { array: true, describe: "Plugins to install", type: "string", @@ -801,6 +1005,11 @@ async function main() { let answers = await inquirerMissingArgs(args); logger.verbose(JSON.stringify(answers)); + if (answers.rerun) { + await rerunHost(args); + return; + } + if (answers.pluginTemplate) { await copyPluginTemplates(answers.pluginName, answers.pluginTemplate); return; @@ -816,46 +1025,36 @@ async function main() { await execController(["config", "create"]); await execController(["bootstrap", "create-admin", answers.admin]); await execController(["config", "set", "controller.http_port", answers.httpPort]); - await execController(["config", "set", "controller.allow_remote_updates", answers.remoteNpm]); - await execController(["config", "set", "controller.allow_plugin_updates", answers.remoteNpm]); + await setRemoteUpdates(execController, "controller", answers.remoteNpm); let result = await execController(["bootstrap", "generate-user-token", answers.admin]); adminToken = result.stdout.split("\n").slice(-2)[0]; } - if (answers.mode === "standalone") { + if (["standalone", "host"].includes(answers.mode)) { logger.info("Setting up host"); await execHost(["config", "create"]); - await execHost(["config", "set", "host.name", "local"]); + } + if (answers.mode === "standalone") { + // The controller is not running, so the host is added to its database directly. The id + // from the host config is kept so running the installer again reuses the same host. let result = await execHost(["config", "show", "host.id"]); let hostId = Number.parseInt(result.stdout.split("\n").slice(-2)[0], 10); - - result = await execController(["bootstrap", "generate-host-token", hostId]); - let hostToken = result.stdout.split("\n").slice(-2)[0]; - + let host = await createLocalHost({ name: "local", id: Number.isInteger(hostId) ? hostId : undefined }); + await execHost(["config", "set", "host.id", host.hostId]); + await execHost(["config", "set", "host.name", host.name]); // Default to localhost on correct port for host in standalone mode await execHost(["config", "set", "host.controller_url", `http://localhost:${answers.httpPort}/`]); - await execHost(["config", "set", "host.controller_token", hostToken]); - await execHost(["config", "set", "host.public_address", answers.publicAddress]); - await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]); - await execHost(["config", "set", "host.allow_remote_updates", answers.remoteNpm]); - await execHost(["config", "set", "host.allow_plugin_updates", answers.remoteNpm]); + await execHost(["config", "set", "host.controller_token", host.token]); } - if (answers.mode === "host") { - logger.info("Setting up host"); - let hostId = JSON.parse(Buffer.from(answers.controllerToken.split(".")[1], "base64")).host; - await execHost(["config", "create"]); - await execHost(["config", "set", "host.id", hostId]); - await execHost(["config", "set", "host.name", answers.hostName]); - await execHost(["config", "set", "host.controller_url", answers.controllerUrl]); - await execHost(["config", "set", "host.controller_token", answers.controllerToken]); + if (["standalone", "host"].includes(answers.mode)) { await execHost(["config", "set", "host.public_address", answers.publicAddress]); await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]); - await execHost(["config", "set", "host.allow_remote_updates", answers.remoteNpm]); - await execHost(["config", "set", "host.allow_plugin_updates", answers.remoteNpm]); + await setRemoteUpdates(execHost, "host", answers.remoteNpm); } + // Written before pairing as running the installer again on an installed host only pairs it. if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) { logger.info("Writing run scripts"); await writeScripts(answers.mode, args.selfSignedCert); @@ -867,6 +1066,14 @@ async function main() { await execCtl(["config", "set", "ctl.controller_token", answers.controllerToken]); } + if (answers.mode === "host") { + if (args.pair) { + await pairWithController(args, answers); + } else { + logger.info("Run the installer again with --mode host in this directory to pair the host"); + } + } + /* eslint-disable no-console */ console.log(`Successfully installed ${answers.mode}`); if (adminToken) { diff --git a/packages/create/pair.js b/packages/create/pair.js new file mode 100644 index 00000000..c290c0ad --- /dev/null +++ b/packages/create/pair.js @@ -0,0 +1,275 @@ +// Pairing of hosts with the controller, used by the installer. +// Only uses Node.js builtins as it runs before or without Clusterio being installed. +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import http from "node:http"; +import https from "node:https"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +/** Time to keep polling through network errors, the controller drops requests not polled for 30 seconds */ +const pollRetryMs = 30e3; +/** Time to wait for a response from the controller */ +const requestTimeoutMs = 30e3; +/** Largest response accepted from the controller */ +const maxResponseSize = 64 * 1024; + +/** Error responded by the controller, or for an invalid controller URL */ +export class PairingError extends Error { + /** + * @param {string} message - Description of the error. + * @param {number=} status - HTTP status the controller responded with. + */ + constructor(message, status) { + super(message); + this.name = "PairingError"; + /** @type {number | undefined} */ + this.status = status; + } +} + +function apiUrl(controllerUrl, route) { + let url; + try { + url = new URL(route, controllerUrl.endsWith("/") ? controllerUrl : `${controllerUrl}/`); + } catch (err) { + throw new PairingError(`Invalid controller URL ${controllerUrl}`); + } + if (!["http:", "https:"].includes(url.protocol)) { + throw new PairingError(`Controller URL ${controllerUrl} must start with http:// or https://`); + } + return url; +} + +function request(url, { method = "POST", body, token, ca, signal, timeoutMs = requestTimeoutMs }) { + return new Promise((resolve, reject) => { + const data = body === undefined ? undefined : JSON.stringify(body); + const headers = { "Accept": "application/json" }; + if (data !== undefined) { + headers["Content-Type"] = "application/json"; + headers["Content-Length"] = Buffer.byteLength(data); + } + if (token) { + headers["Authorization"] = `Bearer ${token}`; + } + const options = { method, headers, signal, timeout: timeoutMs }; + if (url.protocol === "https:" && ca) { + options.ca = ca; + } + + const req = (url.protocol === "https:" ? https : http).request(url, options, res => { + const chunks = []; + let size = 0; + res.on("data", chunk => { + size += chunk.length; + if (size > maxResponseSize) { + req.destroy(new PairingError("Response from controller is too large", res.statusCode)); + return; + } + chunks.push(chunk); + }); + res.on("error", reject); + res.on("end", () => { + let json; + try { + json = JSON.parse(Buffer.concat(chunks).toString()); + } catch (err) { + json = undefined; + } + resolve({ status: res.statusCode, headers: res.headers, json }); + }); + }); + req.on("timeout", () => { + const err = new Error(`Request to ${url} timed out`); + err.code = "ETIMEDOUT"; + req.destroy(err); + }); + req.on("error", reject); + req.end(data); + }); +} + +function responseError(response) { + let message = typeof response.json?.error === "string" + ? response.json.error + : `Unexpected response from controller: ${response.status} ${http.STATUS_CODES[response.status] ?? ""}`.trim(); + const retryAfter = response.headers["retry-after"]; + if (response.status === 429 && retryAfter) { + message += `, try again in ${retryAfter} seconds`; + } + return new PairingError(message, response.status); +} + +function pairingResult(response) { + const { hostId, token, name } = response.json ?? {}; + if (!Number.isInteger(hostId) || typeof token !== "string" || typeof name !== "string") { + throw new PairingError("Invalid response from controller", response.status); + } + return { hostId, token, name }; +} + +async function pollUntilDone(controllerUrl, requestId, ca, pollIntervalMs, signal) { + const url = apiUrl(controllerUrl, "api/host-pairing/poll"); + let lastResponseMs = Date.now(); + while (true) { + await sleep(pollIntervalMs, undefined, { signal }); + let response; + try { + response = await request(url, { body: { requestId }, ca, signal }); + } catch (err) { + if (signal?.aborted || Date.now() - lastResponseMs > pollRetryMs) { + throw err; + } + continue; + } + lastResponseMs = Date.now(); + if (response.status === 200) { + return pairingResult(response); + } + if (response.status !== 202) { + throw responseError(response); + } + } +} + +/** + * Request to join the cluster and wait until the request is approved + * + * @param {object} options - Pairing options. + * @param {string} options.controllerUrl - URL of the controller, with or without a trailing slash. + * @param {string} options.name - Name to request for the host. + * @param {string=} options.version - Version of the installer, shown with the request. + * @param {string=} options.token - + * Token of a user allowed to approve hosts, approves the host without waiting. + * @param {(string|Buffer)=} options.ca - Certificate to trust for a controller using a self-signed certificate. + * @param {number=} options.pollIntervalMs - Interval to poll for the result at. + * @param {function({ code: string }): void=} options.onPending - + * Called once with the code to approve the request with if it has to wait for approval. + * @param {AbortSignal=} options.signal - Signal to cancel the request with. + * @returns {Promise<{ hostId: number, token: string, name: string }>} id, token and approved name of the host. + * @throws {PairingError} if the controller refuses or rejects the request. + * @throws {DOMException} AbortError if the signal is aborted. + */ +export async function pairHost({ + controllerUrl, name, version = "", token, ca, pollIntervalMs = 2000, onPending, signal, +}) { + let requestId; + try { + signal?.throwIfAborted(); + const response = await request(apiUrl(controllerUrl, "api/host-pairing"), { + body: { name, version }, token, ca, signal, + }); + if (response.status === 200) { + return pairingResult(response); + } + if (response.status === 404) { + throw new PairingError("The controller must be updated to a version that supports host pairing", 404); + } + if (response.status !== 202) { + throw responseError(response); + } + if (typeof response.json?.requestId !== "string" || typeof response.json.code !== "string") { + throw new PairingError("Invalid response from controller", response.status); + } + requestId = response.json.requestId; + onPending?.({ code: response.json.code }); + return await pollUntilDone(controllerUrl, requestId, ca, pollIntervalMs, signal); + + } catch (err) { + if (!signal?.aborted) { + throw err; + } + if (requestId !== undefined) { + await request(apiUrl(controllerUrl, "api/host-pairing/cancel"), { + body: { requestId }, ca, timeoutMs: 5e3, + }).catch(() => {}); + } + throw new DOMException("Pairing was cancelled", "AbortError"); + } +} + +/** + * Check if the controller accepts a host token + * + * @param {object} options - Check options. + * @param {string} options.controllerUrl - URL of the controller, with or without a trailing slash. + * @param {string} options.token - Host token to check. + * @param {(string|Buffer)=} options.ca - Certificate to trust for a controller using a self-signed certificate. + * @returns {Promise} true if the token is valid and its host is registered. + * @throws {PairingError} if the controller gives an unexpected response. + */ +export async function checkHostToken({ controllerUrl, token, ca }) { + const response = await request(apiUrl(controllerUrl, "api/host-pairing/check"), { + method: "GET", token, ca, + }); + if (response.status === 200) { + return true; + } + if (response.status === 401) { + return false; + } + throw responseError(response); +} + +function signHostToken(payload, secret) { + const encode = value => Buffer.from(JSON.stringify(value)).toString("base64url"); + const unsigned = `${encode({ alg: "HS256", typ: "JWT" })}.${encode(payload)}`; + const signature = crypto.createHmac("sha256", secret).update(unsigned).digest("base64url"); + return `${unsigned}.${signature}`; +} + +/** + * Add a host to the database of a controller installed in the same directory + * + * The controller must not be running, as it would overwrite the database with + * what it has in memory. + * + * @param {object} options - Host options. + * @param {string=} options.directory - Directory with config-controller.json in it. + * @param {string} options.name - Name of the host. + * @param {number=} options.id - + * Id to give the host, picks a random unused id if not given. If a host with + * this id exists, a new token is signed for it and its name is kept. + * @param {string=} options.version - Version to record for the host. + * @returns {Promise<{ hostId: number, token: string, name: string }>} id, token and name of the host. + */ +export async function createLocalHost({ directory = ".", name, id, version = "" }) { + const configPath = path.join(directory, "config-controller.json"); + const config = JSON.parse(await fs.readFile(configPath, "utf8")); + const secret = config["controller.auth_secret"]; + if (typeof secret !== "string" || !secret) { + throw new Error(`${configPath} has no controller.auth_secret`); + } + + const databaseDirectory = path.resolve(directory, config["controller.database_directory"] ?? "database"); + const hostsPath = path.join(databaseDirectory, "hosts.json"); + let hosts = []; + try { + hosts = JSON.parse(await fs.readFile(hostsPath, "utf8")); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + const now = Math.floor(Date.now() / 1000); + const existing = id === undefined ? undefined : hosts.find(host => host.id === id); + let hostId = id; + if (existing) { + // Running the installer again signs a new token for the same host. + name = existing.name; + } else { + const ids = new Set(hosts.map(host => host.id)); + while (hostId === undefined || ids.has(hostId)) { + hostId = crypto.randomInt(1, 2 ** 31); + } + hosts.push({ id: hostId, name, version, plugins: {}, token_valid_after: now }); + await fs.mkdir(databaseDirectory, { recursive: true }); + const temporary = path.join(databaseDirectory, "hosts.tmp.json"); + await fs.writeFile(temporary, JSON.stringify(hosts, null, "\t"), { flush: true }); + await fs.rename(temporary, hostsPath); + } + + const token = signHostToken({ aud: "host", host: hostId, iat: now }, Buffer.from(secret, "base64")); + return { hostId, token, name }; +} diff --git a/packages/ctl/src/commands_host.ts b/packages/ctl/src/commands_host.ts index 3d20e050..a644b164 100644 --- a/packages/ctl/src/commands_host.ts +++ b/packages/ctl/src/commands_host.ts @@ -5,7 +5,6 @@ import os from "os"; import child_process from "child_process"; import * as lib from "@clusterio/lib"; -import { logger } from "@clusterio/lib"; import type { Ctl } from "../ctl.js"; import { print } from "./command_ops.js"; import { serializedConfigToString, getEditor, configToKeyVal } from "./config_ops.js"; @@ -51,62 +50,78 @@ hostCommands.add(new lib.Command({ }, })); +/** Format a duration in milliseconds as a short age like "1m 5s" */ +function formatAge(ms: number) { + const seconds = Math.max(0, Math.floor(ms / 1000)); + if (seconds < 60) { + return `${seconds}s`; + } + if (seconds < 3600) { + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; + } + return `${Math.floor(seconds / 3600)}h ${Math.floor(seconds / 60) % 60}m`; +} + +/** Strip case and separators the same way the controller does when matching pairing codes */ +function plainCode(code: string) { + return code.toUpperCase().replace(/[^0-9A-Z]/g, ""); +} + hostCommands.add(new lib.Command({ - definition: ["generate-token", "Generate token for a host", (yargs) => { - yargs.option("id", { type: "number", nargs: 1, describe: "Host id" }); - }], - handler: async function(args: { id?: number }, ctl: Ctl) { - let hostId = typeof args.id === "number" ? args.id : undefined; - let response = await ctl.send(new lib.HostGenerateTokenRequest(hostId)); - print(response); + definition: ["pending", "List host installers waiting for approval"], + handler: async function(args: object, ctl: Ctl) { + let requests = await ctl.send(new lib.HostPairingListRequest()); + if (!requests.length) { + print("No pending host requests"); + return; + } + let now = Date.now(); + print(asTable(requests.map(request => ({ + code: request.code, + name: request.name, + address: request.remoteAddress, + age: formatAge(now - request.createdAtMs), + })))); }, })); hostCommands.add(new lib.Command({ - definition: ["revoke-token ", "Revoke all tokens for a host", (yargs) => { - yargs.positional("host", { describe: "Host to revoke tokens for", type: "string" }); + definition: ["approve ", "Approve a host installer waiting for approval", (yargs) => { + yargs.positional("code", { describe: "Code shown by the installer", type: "string" }); + yargs.option("name", { + type: "string", nargs: 1, describe: "Name to give the host instead of the one it asked for", + }); + }], + handler: async function(args: { code: string, name?: string }, ctl: Ctl) { + let name = args.name?.trim(); + if (name === undefined) { + let requests = await ctl.send(new lib.HostPairingListRequest()); + name = requests.find(request => plainCode(request.code) === plainCode(args.code))?.name; + } + let hostId = await ctl.send(new lib.HostPairingApproveRequest(args.code, args.name)); + print(name ? `Approved host ${name} with id ${hostId}` : `Approved host with id ${hostId}`); + }, +})); + +hostCommands.add(new lib.Command({ + definition: ["reject ", "Reject a host installer waiting for approval", (yargs) => { + yargs.positional("code", { describe: "Code shown by the installer", type: "string" }); + }], + handler: async function(args: { code: string }, ctl: Ctl) { + await ctl.send(new lib.HostPairingRejectRequest(args.code)); + }, +})); + +hostCommands.add(new lib.Command({ + definition: ["revoke ", "Remove a host from the cluster and disconnect it", (yargs) => { + yargs.positional("host", { describe: "Host to revoke", type: "string" }); }], handler: async function(args: { host: string }, ctl: Ctl) { let hostId = await lib.resolveHost(ctl, args.host); - await ctl.send(new lib.HostRevokeTokensRequest(hostId)); + await ctl.send(new lib.HostRevokeRequest(hostId)); }, })); -hostCommands.add(new lib.Command({ - definition: ["create-config", "Create host config", (yargs) => { - yargs.option("id", { type: "number", nargs: 1, describe: "Host id" }); - yargs.option("name", { type: "string", nargs: 1, describe: "Host name" }); - yargs.option("generate-token", { - type: "boolean", nargs: 0, describe: "Generate authentication token", default: false, - }); - yargs.option("output", { - type: "string", nargs: 1, describe: "Path to output config (- for stdout)", default: "config-host.json", - }); - }], - handler: async function( - args: { id?: number, name?: string, generateToken: boolean, output: string }, - ctl: Ctl - ) { - let config = await ctl.send( - new lib.HostConfigCreateRequest(args.id, args.name, args.generateToken) - ); - - let content = JSON.stringify(config, null, "\t"); - if (args.output === "-") { - print(content); - } else { - logger.info(`Writing ${args.output}`); - try { - await fs.writeFile(args.output, content, { flag: "wx", mode: 0o600 }); - } catch (err: any) { - if (err.code === "EEXIST") { - throw new lib.CommandError(`File ${args.output} already exists`); - } - throw err; - } - } - }, -})); hostCommands.add(new lib.Command({ definition: ["update ", "Update the host", (yargs) => { yargs.positional("host", { describe: "Host to update", type: "string" }); diff --git a/packages/host/README.md b/packages/host/README.md index d305077e..95fe9ac8 100644 --- a/packages/host/README.md +++ b/packages/host/README.md @@ -132,6 +132,9 @@ Lists up all configuration entries with their currently configured values. ### `run` Runs the host. +The host has to be approved on the controller before it can connect, which is done by running the installer in the host directory, see Setting up remote hosts in the readme of the main repository. +If the controller rejects the token of the host when it starts, for example because the host was revoked, it exits with code 8 and the run scripts created by the installer do not restart it. +A host that is revoked while it runs shuts down. ## See Also diff --git a/packages/host/host.ts b/packages/host/host.ts index 11ea2ba8..d3deccd7 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -216,10 +216,9 @@ async function startHost() { if (hostConfig.get("host.controller_token") === "enter token here") { logger.fatal("ERROR invalid config!"); logger.fatal( - "Controller requires an access token for socket operations. As clusterio\n"+ - "hosts depends upon this, please set your token using the command npx\n"+ - "clusteriohost config set host.controller_token . You can generate an\n"+ - "auth token using npx clusterioctl generate-host-token." + "This host has not been set up to connect to a controller. Run the installer\n"+ + "again in this directory with npm init @clusterio and approve the host in the\n"+ + "web interface." ); process.exitCode = 1; return; diff --git a/packages/lib/src/data/HostPairingDetails.ts b/packages/lib/src/data/HostPairingDetails.ts new file mode 100644 index 00000000..69524efd --- /dev/null +++ b/packages/lib/src/data/HostPairingDetails.ts @@ -0,0 +1,73 @@ +import { Type, Static } from "@sinclair/typebox"; + +/** + * A pending request from a host installer to join the cluster + * + * Requests are kept in memory on the controller only while the installer + * waits for them to be approved or rejected. + */ +export default class HostPairingDetails { + constructor( + /** Code shown by the installer, used to identify the request. */ + public code: string, + /** Name the installer asked for the host to have. */ + public name: string, + /** Address the request was made from. */ + public remoteAddress: string, + /** Version of the installer that made the request. */ + public version = "", + /** Millisecond Unix timestamp the request was made at. */ + public createdAtMs = 0, + /** Millisecond Unix timestamp this entry was last updated at. */ + public updatedAtMs = 0, + /** True if this request is no longer pending. */ + public isDeleted = false, + ) { } + + get id() { + return this.code; + } + + static jsonSchema = Type.Object({ + "code": Type.String(), + "name": Type.String(), + "remoteAddress": Type.String(), + "version": Type.Optional(Type.String()), + "createdAtMs": Type.Optional(Type.Number()), + "updatedAtMs": Type.Optional(Type.Number()), + "isDeleted": Type.Optional(Type.Boolean()), + }); + + static fromJSON(json: Static) { + return new this( + json.code, + json.name, + json.remoteAddress, + json.version, + json.createdAtMs, + json.updatedAtMs, + json.isDeleted, + ); + } + + toJSON() { + const json: Static = { + code: this.code, + name: this.name, + remoteAddress: this.remoteAddress, + }; + if (this.version !== "") { + json.version = this.version; + } + if (this.createdAtMs !== 0) { + json.createdAtMs = this.createdAtMs; + } + if (this.updatedAtMs !== 0) { + json.updatedAtMs = this.updatedAtMs; + } + if (this.isDeleted) { + json.isDeleted = this.isDeleted; + } + return json; + } +} diff --git a/packages/lib/src/data/index.ts b/packages/lib/src/data/index.ts index 19ce86eb..41571354 100644 --- a/packages/lib/src/data/index.ts +++ b/packages/lib/src/data/index.ts @@ -15,6 +15,7 @@ export { default as Role } from "./Role.js"; export { default as Note } from "./Note.js"; export { default as InstanceDetails, InstanceStatus } from "./InstanceDetails.js"; export { default as HostDetails } from "./HostDetails.js"; +export { default as HostPairingDetails } from "./HostPairingDetails.js"; export { default as UserDetails } from "./UserDetails.js"; export type { IUser } from "./UserDetails.js"; export * from "./composites.js"; diff --git a/packages/lib/src/data/messages_controller.ts b/packages/lib/src/data/messages_controller.ts index 41f0ec1d..e7d534b8 100644 --- a/packages/lib/src/data/messages_controller.ts +++ b/packages/lib/src/data/messages_controller.ts @@ -1,9 +1,10 @@ import { Type, Static } from "@sinclair/typebox"; -import { JsonString, StringEnum, StringKey, plainJson } from "./composites.js"; +import { JsonNumber, StringEnum, StringKey, jsonArray, plainJson } from "./composites.js"; import { type LogEntry, levels } from "../logging.js"; -import { ControllerConfig, HostConfig } from "../config/index.js"; +import { ControllerConfig } from "../config/index.js"; import { ExternalFactorioVersionSchema, LatestReleasesSchema } from "../external/index.js"; import { PluginDetails } from "./messages_plugin.js"; +import HostPairingDetails from "./HostPairingDetails.js"; const LogEntrySchema = Type.Object({ "level": StringKey(levels), @@ -120,52 +121,81 @@ export class ControllerConfigSetPropRequest { } } -export class HostGenerateTokenRequest { - declare ["constructor"]: typeof HostGenerateTokenRequest; +export class HostPairingListRequest { + declare ["constructor"]: typeof HostPairingListRequest; static type = "request" as const; static src = "control" as const; static dst = "controller" as const; - static permission = "core.host.generate_token" as const; - - constructor( - public hostId?: number, - ) { } - - static jsonSchema = Type.Object({ - "hostId": Type.Optional(Type.Integer()), - }); - - static fromJSON(json: Static) { - return new this(json.hostId); - } - - static Response = JsonString; + static permission = "core.host.approve" as const; + static Response = jsonArray(HostPairingDetails); } -export class HostConfigCreateRequest { - declare ["constructor"]: typeof HostConfigCreateRequest; - static type = "request" as const; - static src = "control" as const; - static dst = "controller" as const; - static permission = "core.host.create_config" as const; +export class HostPairingUpdatesEvent { + declare ["constructor"]: typeof HostPairingUpdatesEvent; + static type = "event" as const; + static src = "controller" as const; + static dst = "control" as const; + static permission = "core.host.approve" as const; constructor( - public id: number | undefined, - public name: string | undefined, - public generateToken: boolean, + public updates: HostPairingDetails[], ) { } static jsonSchema = Type.Object({ - "id": Type.Optional(Type.Integer()), - "name": Type.Optional(Type.String()), - "generateToken": Type.Boolean(), + "updates": Type.Array(HostPairingDetails.jsonSchema), }); static fromJSON(json: Static) { - return new this(json.id, json.name, json.generateToken); + return new this(json.updates.map(update => HostPairingDetails.fromJSON(update))); + } +} + +export class HostPairingApproveRequest { + declare ["constructor"]: typeof HostPairingApproveRequest; + static type = "request" as const; + static src = "control" as const; + static dst = "controller" as const; + static permission = "core.host.approve" as const; + + constructor( + /** Code of the pending request, case and dashes are ignored. */ + public code: string, + /** Name to give the host instead of the one it asked for. */ + public name?: string, + ) { } + + static jsonSchema = Type.Object({ + "code": Type.String(), + "name": Type.Optional(Type.String()), + }); + + static fromJSON(json: Static) { + return new this(json.code, json.name); } - static Response = plainJson(HostConfig.jsonSchema); + /** Id of the host created. */ + static Response = JsonNumber; +} + +export class HostPairingRejectRequest { + declare ["constructor"]: typeof HostPairingRejectRequest; + static type = "request" as const; + static src = "control" as const; + static dst = "controller" as const; + static permission = "core.host.approve" as const; + + constructor( + /** Code of the pending request, case and dashes are ignored. */ + public code: string, + ) { } + + static jsonSchema = Type.Object({ + "code": Type.String(), + }); + + static fromJSON(json: Static) { + return new this(json.code); + } } export class GetFactorioCredentialsRequest { diff --git a/packages/lib/src/data/messages_host.ts b/packages/lib/src/data/messages_host.ts index 41e8e107..e89b236a 100644 --- a/packages/lib/src/data/messages_host.ts +++ b/packages/lib/src/data/messages_host.ts @@ -259,12 +259,12 @@ export class SyncUserListsEvent { } } -export class HostRevokeTokensRequest { - declare["constructor"]: typeof HostRevokeTokensRequest; +export class HostRevokeRequest { + declare ["constructor"]: typeof HostRevokeRequest; static type = "request" as const; static src = "control" as const; static dst = "controller" as const; - static permission = "core.host.revoke_token" as const; + static permission = "core.host.revoke" as const; constructor( public hostId: number, diff --git a/packages/lib/src/link/messages.ts b/packages/lib/src/link/messages.ts index c4d95dcb..f4e15976 100644 --- a/packages/lib/src/link/messages.ts +++ b/packages/lib/src/link/messages.ts @@ -29,8 +29,10 @@ export const dataClasses: (RequestClass | EventClass) controller.ControllerConfigSetRequest, controller.ControllerConfigSetFieldRequest, controller.ControllerConfigSetPropRequest, - controller.HostGenerateTokenRequest, - controller.HostConfigCreateRequest, + controller.HostPairingListRequest, + controller.HostPairingUpdatesEvent, + controller.HostPairingApproveRequest, + controller.HostPairingRejectRequest, controller.GetFactorioCredentialsRequest, controller.LogSetSubscriptionsRequest, controller.LogQueryRequest, @@ -59,7 +61,7 @@ export const dataClasses: (RequestClass | EventClass) host.ControllerConnectionEvent, host.PrepareControllerDisconnectRequest, host.SyncUserListsEvent, - host.HostRevokeTokensRequest, + host.HostRevokeRequest, instance.InstanceDetailsGetRequest, instance.InstanceDetailsListRequest, diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index 16c1d8e6..aeadc779 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -196,19 +196,16 @@ export const corePermissions = [ grantByDefault: true, }, { - name: "core.host.generate_token", - title: "Generate host token", - description: "Generate tokens for hosts to connect to the cluster with.", + name: "core.host.approve", + title: "Approve hosts", + description: + "List, approve and reject requests from host installers to join the cluster, and approve hosts " + + "directly by passing a token to the installer. Approving gives the new host full access to the cluster.", }, { - name: "core.host.revoke_token", - title: "Revoke host token", - description: "Revoke access token and terminate existing connection for hosts.", - }, - { - name: "core.host.create_config", - title: "Create host config", - description: "Create host configs via the controller.", + name: "core.host.revoke", + title: "Revoke hosts", + description: "Remove hosts from the cluster, disconnecting them and making their access tokens invalid.", }, { diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index 6e1cf16f..6e21f95f 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -1,7 +1,7 @@ import React, { useContext, useState } from "react"; import { Descriptions, Spin, Typography, Button, Space, Popconfirm, Flex, Tooltip } from "antd"; import { ExclamationCircleOutlined } from "@ant-design/icons"; -import { useParams } from "react-router-dom"; +import { useNavigate, useParams } from "react-router-dom"; import * as lib from "@clusterio/lib"; import notify, { notifyErrorHandler } from "../util/notify"; @@ -21,7 +21,6 @@ import { MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, MetricDiskUsed, MetricDiskRatio, MetricRelativeDate, } from "./system_metrics"; -import { formatTimestamp } from "../util/time_format"; import { useSystems } from "../model/system"; import { hasNpmButtonPermission, NpmButton } from "./NpmButton"; import VariableDropdownButton, { VariableDropdownButtonProps } from "./VariableDropdownButton"; @@ -98,10 +97,42 @@ function HostControlButton({ hostId, canRestart, restartRequired }: HostControlB return ; } +type RevokeHostButtonProps = { host: lib.HostDetails, instances: ReadonlyMap }; + +function RevokeHostButton({ host, instances }: RevokeHostButtonProps) { + const control = useContext(ControlContext); + const navigate = useNavigate(); + const assigned = [...instances.values()].map(instance => instance.name); + + if (assigned.length) { + return + + ; + } + + return + The host is removed from the cluster and disconnected.
+ To add it back, run the installer again in its directory. + } + placement="bottomRight" + okText="Revoke" + okButtonProps={{ danger: true }} + onConfirm={() => control.send( + new lib.HostRevokeRequest(host.id) + ).then(() => { + notify(`Revoked access for host ${host.name}`, "success"); + navigate("/hosts"); + }).catch(notifyErrorHandler(`Error revoking access for host ${host.name}`))} + > + + ; +} + export default function HostViewPage() { let params = useParams(); let hostId = Number(params.id); - let control = useContext(ControlContext); let account = useAccount(); let [instances] = useInstances(); const [systems] = useSystems(); @@ -125,24 +156,7 @@ export default function HostViewPage() { let hostButtons = - { - account.hasPermission("core.host.revoke_token") - && { - control.send(new lib.HostRevokeTokensRequest(host.id!)) - .then(() => notify("Host tokens revoked")) - .catch(notifyErrorHandler(`Error revoking tokens for host id:${host.id}`)); - }} - > - - - } + {account.hasPermission("core.host.revoke") && } {canShowNpm && ( )} @@ -175,13 +189,6 @@ export default function HostViewPage() { {host.publicAddress} - { - host.tokenValidAfter - ? - {formatTimestamp(host.tokenValidAfter*1000)} - - : null - } {account.hasPermission("core.instance.list") && <> diff --git a/packages/web_ui/src/components/HostsPage.tsx b/packages/web_ui/src/components/HostsPage.tsx index 40684bc5..2947e2e5 100644 --- a/packages/web_ui/src/components/HostsPage.tsx +++ b/packages/web_ui/src/components/HostsPage.tsx @@ -1,6 +1,6 @@ -import React, { useContext, useRef, useState, useEffect } from "react"; -import { Button, Flex, Form, InputNumber, Modal, Space, Switch, Table, Tooltip, Typography } from "antd"; -import { ExclamationCircleOutlined, CopyOutlined } from "@ant-design/icons"; +import React, { useContext, useState, useEffect } from "react"; +import { Badge, Button, Flex, Input, Modal, Space, Switch, Table, Tooltip, Typography } from "antd"; +import { ExclamationCircleOutlined, CopyOutlined, PlusOutlined } from "@ant-design/icons"; import * as lib from "@clusterio/lib"; @@ -16,9 +16,10 @@ import { MetricDiskUsed, MetricDiskRatio, RestartRequired, } from "./system_metrics"; -import { useHosts } from "../model/host"; +import { useHostPairings, useHosts } from "../model/host"; import { useSystems } from "../model/system"; import notify, { notifyErrorHandler } from "../util/notify"; +import { formatDuration } from "../util/time_format"; import useTableQueryState from "../util/useTableQueryState"; import useColumnSearch from "../util/useColumnSearch"; import useRowNavigation from "../util/useRowNavigation"; @@ -27,133 +28,186 @@ import Link from "./Link"; const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare; -function GenerateHostTokenButton() { - let control = useContext(ControlContext); - let [open, setOpen] = useState(false); - let [token, setToken] = useState(null); - let [hostId, setHostId] = useState(null); - let [form] = Form.useForm(); - let [pluginList, setPluginList] = useState([]); +/** Current time in milliseconds, updated every second while active */ +function useNow(active: boolean) { + const [now, setNow] = useState(Date.now); useEffect(() => { - (async () => { - let response = await fetch(`${webRoot}api/plugins`); - if (response.ok) { - const plugins = await response.json(); - setPluginList(plugins); - } else { - notify("Failed to load plugin list"); - } - })(); - }, []); - - async function generateToken() { - let id; - let values = form.getFieldsValue(); - if (values.hostId) { - id = Number.parseInt(values.hostId, 10); - if (Number.isNaN(id)) { - form.setFields([{ name: "hostId", errors: ["Must be an integer"] }]); - return; - } - form.setFields([{ name: "hostId", errors: [] }]); + if (!active) { + return undefined; } + setNow(Date.now()); + const interval = setInterval(() => { setNow(Date.now()); }, 1000); + return () => { clearInterval(interval); }; + }, [active]); + return now; +} - let newToken = await control.send(new lib.HostGenerateTokenRequest(id)); - setToken(newToken); - setHostId(id ?? null); +function AddHostModal(props: { open: boolean, onClose: () => void }) { + const control = useContext(ControlContext); + const [requests, synced] = useHostPairings(); + const [names, setNames] = useState>(new Map()); + const [busyCodes, setBusyCodes] = useState>(new Set()); + const now = useNow(props.open); + + const controllerUrl = new URL(webRoot, document.location.href).href; + const installCommand = `npm init "@clusterio" -- --mode host --controller-url ${controllerUrl}`; + + function setName(code: string, name?: string) { + setNames(current => { + const next = new Map(current); + if (name === undefined) { + next.delete(code); + } else { + next.set(code, name); + } + return next; + }); } - // Generate a new random - useEffect(() => { - if (open) { - generateToken().catch(notifyErrorHandler("Error generating token")); - } - }, [open]); + function setBusy(code: string, busy: boolean) { + setBusyCodes(current => { + const next = new Set(current); + if (busy) { + next.add(code); + } else { + next.delete(code); + } + return next; + }); + } - const pluginString = pluginList.filter(p => p.npmPackage).map(p => `"${p.npmPackage}"`).join(" "); + function nameOf(request: Readonly) { + return (names.get(request.code) ?? request.name).trim(); + } + + function approve(request: Readonly) { + const name = nameOf(request); + setBusy(request.code, true); + control.send( + new lib.HostPairingApproveRequest(request.code, name !== request.name ? name : undefined) + ).then(() => { + setName(request.code); + notify(`Approved host ${name}`, "success"); + }).catch( + notifyErrorHandler(`Error approving host ${request.name}`) + ).finally(() => { + setBusy(request.code, false); + }); + } + + function reject(request: Readonly) { + setBusy(request.code, true); + control.send( + new lib.HostPairingRejectRequest(request.code) + ).then(() => { + setName(request.code); + }).catch( + notifyErrorHandler(`Error rejecting host ${request.name}`) + ).finally(() => { + setBusy(request.code, false); + }); + } + + return + + Run the installer in an empty directory on the machine the host will run on. + It asks for a name for the host, then shows a code and waits for the request to be approved here. + +
+ +

> mkdir clusterio

+

> cd clusterio

+

+ > npm init "@clusterio" -- --mode host + --controller-url {controllerUrl} +

+
+ + Once approved the installer saves the connection details of the host and exits. + The host can then be started with the run-host script in that directory. + + Waiting for approval + + Only approve a request if its code matches the code shown by the installer. + An approved host has full access to the cluster. + + a.createdAtMs - b.createdAtMs)} + locale={{ emptyText: "No hosts are waiting for approval" }} + columns={[ + { + title: "Name", + key: "name", + render: (_, request) => { setName(request.code, event.target.value); }} + />, + }, + { + title: "Code", + dataIndex: "code", + render: (_, request) => {request.code}, + }, + { + title: "Address", + dataIndex: "remoteAddress", + }, + { + title: "Age", + key: "age", + render: (_, request) => + {formatDuration(Math.max(0, now - request.createdAtMs))} + , + }, + { + key: "action", + align: "right", + render: (_, request) => { + const busy = busyCodes.has(request.code); + return + + + ; + }, + }, + ]} + /> + ; +} + +function AddHostButton() { + const [open, setOpen] = useState(false); + const [requests] = useHostPairings(); return <> - - { - setOpen(false); - setToken(null); - form.resetFields(); - }} - width="700px" - > -
- - { - generateToken().catch(notifyErrorHandler("Error generating token")); - }} /> - - - - Host IDs are randomly generated and unique, leave blank to use a new ID, - otherwise the ID of an existing host can be found using: - -
- - npx clusteriohost config get host.id -
- {token !== null && <> - - Host auth token: - -
- - {token} -
- - You can set the token on an existing host with the following command: - -
- - npx clusteriohost config set host.id {token} -
- - A new host can be setup using the following commands: - -
- -

> mkdir clusterio

-

> cd clusterio

-

- > npm init "@clusterio" -- - --controller-token {token} - --mode "host" - --download-headless - --controller-url {document.location.origin}/ - --host-name {hostId ? `Host ${hostId}` : "Host"} - --public-address localhost - {pluginString.length ? "--plugins" : ""} {pluginString} -

-

> ./run-host

-
- } - + + + + { setOpen(false); }} /> ; } @@ -216,7 +270,7 @@ export default function HostsPage() { return : undefined} + extra={account.hasPermission("core.host.approve") ? : undefined} />
void) => control.hosts.subscribe(callback), [control]); return useSyncExternalStore(subscribe, () => control.hosts.getSnapshot()); } + +/** Requests from host installers waiting to be approved, empty if the account may not approve hosts */ +export function useHostPairings() { + const control = useContext(ControlContext); + const account = useAccount(); + const permitted = Boolean(account.hasPermission("core.host.approve")); + const subscribe = useCallback( + (callback: () => void) => (permitted ? control.hostPairings.subscribe(callback) : () => {}), + [control, permitted], + ); + return useSyncExternalStore(subscribe, () => control.hostPairings.getSnapshot()); +} diff --git a/packages/web_ui/src/util/host_pairing.ts b/packages/web_ui/src/util/host_pairing.ts new file mode 100644 index 00000000..c8cb79b0 --- /dev/null +++ b/packages/web_ui/src/util/host_pairing.ts @@ -0,0 +1,72 @@ +import * as lib from "@clusterio/lib"; + +/** + * Subscriber for the requests from host installers waiting to be approved + * + * Only accounts that may approve hosts can subscribe to the requests, so + * this does not resubscribe on connect or account changes for other + * accounts, which would fail with a permission error. The requests only + * live in the controller's memory and it does not tell a new session about + * requests removed before it started, so the requests are cleared when a + * new session starts and fetched again. The same applies to requests removed + * while nothing was subscribed, so they are also cleared when subscribing + * again after the last handler unsubscribed. + */ +export class HostPairingSubscriber extends lib.MapSubscriber { + private _permitted: () => boolean; + private _handlers = 0; + + /** + * @param control - Link to the controller. + * @param permitted - Returns true if the account may approve hosts. + */ + constructor(control: lib.Link, permitted: () => boolean) { + super(lib.HostPairingUpdatesEvent, control); + this._permitted = permitted; + } + + override handleConnectionEvent(event: "connect" | "drop" | "resume" | "close") { + if (event === "connect") { + this.values.clear(); + this.lastUpdatedMs = -1; + } + if (event === "connect" || event === "resume") { + // The account of a new session is reported after the connect event. + queueMicrotask(() => { + if (this._permitted()) { + super.handleConnectionEvent(event); + } + }); + return; + } + super.handleConnectionEvent(event); + } + + override handleAccountEvent() { + if (this._permitted()) { + super.handleAccountEvent(); + } + } + + override subscribe(handler: lib.EventSubscriberCallback) { + if (this._handlers === 0) { + this.values.clear(); + this.lastUpdatedMs = -1; + } + this._handlers += 1; + const unsubscribe = super.subscribe(handler); + let unsubscribed = false; + return () => { + if (unsubscribed) { + return; + } + unsubscribed = true; + unsubscribe(); + // Deferred like the unsubscribe in EventSubscriber, so moving between pages + // that both subscribe keeps the requests. + setTimeout(() => { + this._handlers -= 1; + }); + }; + } +} diff --git a/packages/web_ui/src/util/websocket.ts b/packages/web_ui/src/util/websocket.ts index 20d3b8eb..805899a4 100644 --- a/packages/web_ui/src/util/websocket.ts +++ b/packages/web_ui/src/util/websocket.ts @@ -1,9 +1,16 @@ import * as lib from "@clusterio/lib"; import packageJson from "../../package.json"; import * as WebPlugin from "../BaseWebPlugin"; +import { HostPairingSubscriber } from "./host_pairing"; const { logFilter, logger } = lib; +function rolesHavePermission(roles: lib.AccountRole[] | null, permission: lib.PermissionName) { + return (roles ?? []).some( + role => role.permissions.includes("core.admin") || role.permissions.includes(permission) + ); +} + type accountHandler = (account: lib.AccountDetails) => void; type hostHandler = (hostDetails: lib.HostDetails) => void; type instanceHandler = (instanceDetails: lib.InstanceDetails) => void; @@ -67,6 +74,7 @@ export class Control extends lib.Link { /** Updates handled by the subscription service */ systems = new lib.MapSubscriber(lib.SystemInfoUpdateEvent, this); hosts = new lib.MapSubscriber(lib.HostUpdatesEvent, this); + hostPairings = new HostPairingSubscriber(this, () => rolesHavePermission(this.accountRoles, "core.host.approve")); instances = new lib.MapSubscriber(lib.InstanceDetailsUpdatesEvent, this); saves = new lib.MapSubscriber(lib.InstanceSaveDetailsUpdatesEvent, this); modPacks = new lib.MapSubscriber(lib.ModPackUpdatesEvent, this); diff --git a/scripts/version-compat-test.js b/scripts/version-compat-test.js index b300833e..408e7362 100644 --- a/scripts/version-compat-test.js +++ b/scripts/version-compat-test.js @@ -32,6 +32,8 @@ import readline from "node:readline"; import util from "node:util"; import { loadBreaks, matchBreaks, selectBreaks } from "./version-compat-breaks.js"; +// Only uses Node builtins, so it does not load a second copy of the lib. +import { pairHost, PairingError } from "../packages/create/pair.js"; const execFile = util.promisify(child_process.execFile); @@ -279,6 +281,44 @@ async function pollCtl(ctl, cliArgs, expectRegex, description, timeoutMs = 30e3) } } +/** + * Pair the host with the running controller and write its config + * + * The pairing API is probed instead of checking which side is local, as releases that support + * pairing no longer have ctl host create-config to fall back on. + * + * @returns true if the host was paired, false if the controller predates host pairing. + */ +async function pairHostWithController() { + // Written by bootstrap create-ctl-config, holds the admin token which approves the host directly. + const ctlConfig = JSON.parse(await fs.readFile(path.join(controllerDir, "config-ctl.json"), "utf8")); + const controllerUrl = ctlConfig["ctl.controller_url"]; + console.log(`| pairing host with ${controllerUrl}`); + let pairing; + try { + pairing = await pairHost({ controllerUrl, name: "host", token: ctlConfig["ctl.controller_token"] }); + } catch (err) { + if (err instanceof PairingError && err.status === 404) { + console.log("| controller does not support host pairing"); + return false; + } + throw err; + } + console.log(`| paired host ${pairing.name} with id ${pairing.hostId}`); + + // The same fields the installer sets, which every host version reads. + const hostConfig = { + "host.id": pairing.hostId, + "host.name": pairing.name, + "host.controller_url": controllerUrl, + "host.controller_token": pairing.token, + }; + await fs.writeFile( + path.join(hostDir, "config-host.json"), JSON.stringify(hostConfig, null, "\t"), { flag: "wx", mode: 0o600 } + ); + return true; +} + async function main() { const options = parseArgs(); const breaks = await loadBreaks(); @@ -337,10 +377,12 @@ async function main() { ); try { - await run("ctl", ctl, [ - "host", "create-config", "--id", "1", "--name", "host", "--generate-token", - "--output", path.join(hostDir, "config-host.json"), - ], controllerDir); + if (!await pairHostWithController()) { + await run("ctl", ctl, [ + "host", "create-config", "--id", "1", "--name", "host", "--generate-token", + "--output", path.join(hostDir, "config-host.json"), + ], controllerDir); + } // Started host is only logged after the connection to the controller is established. const hostProc = await spawnDaemon("host", host, ["run"], hostDir, /Started host/); diff --git a/test/controller/ControlConnection.test.js b/test/controller/ControlConnection.test.js index 6cb50dd8..5516cc4e 100644 --- a/test/controller/ControlConnection.test.js +++ b/test/controller/ControlConnection.test.js @@ -1,8 +1,11 @@ import assert from "node:assert/strict"; import events from "node:events"; import * as lib from "@clusterio/lib"; -import { Controller, ControlConnection, ControllerHooks, InstanceManager } from "@clusterio/controller"; +import { + Controller, ControlConnection, ControllerHooks, HostConnection, HostRecord, InstanceManager, InstanceRecord, +} from "@clusterio/controller"; import { MockConnector, MockController, MockLogger } from "../mock.js"; +import "../setup_logging.js"; const addr = lib.Address.fromShorthand; @@ -272,4 +275,213 @@ describe("controller/src/ControlConnection", function() { assert.equal(player.connector.terminated, true); }); }); + + describe("host management", function() { + /** @type {Controller} */ + let controller; + /** @type {ControlConnection} */ + let connection; + let audits; + let originalAudit; + + beforeEach(function() { + const config = new lib.ControllerConfig("controller"); + config.set("controller.auth_secret", Buffer.from("TestSecretDoNotUse").toString("base64")); + controller = new Controller(lib.logger, [], config); + const connector = new lib.VirtualConnector(addr("controller"), addr({ controlId: 1 })); + const user = controller.users.getOrCreateUser("admin"); + connection = new ControlConnection({ version: "2.0.0" }, connector, controller, user, 1); + audits = []; + originalAudit = lib.logger.audit; + lib.logger.audit = msg => { audits.push(msg); }; + }); + afterEach(function() { + lib.logger.audit = originalAudit; + }); + + it("registers the host pairing and revoke handlers", function() { + for (const Request of [ + lib.HostRevokeRequest, + lib.HostPairingListRequest, + lib.HostPairingApproveRequest, + lib.HostPairingRejectRequest, + ]) { + assert(connection._requestHandlers.has(Request), `${Request.name} is not handled`); + } + }); + + describe(".handleHostRevokeRequest()", function() { + let record; + let broadcasts; + + beforeEach(function() { + record = new HostRecord(10, "host-10", "", new Map()); + controller.hosts.set(record); + broadcasts = []; + controller.subscriptions.broadcast = event => { + if (event instanceof lib.HostUpdatesEvent) { + broadcasts.push(...event.updates.map(host => ({ id: host.id, isDeleted: host.isDeleted }))); + } + }; + }); + + function assignInstance(id, name, hostId) { + const instanceConfig = new lib.InstanceConfig("controller"); + instanceConfig.set("instance.id", id); + instanceConfig.set("instance.name", name); + instanceConfig.set("instance.assigned_host", hostId); + controller.instances.records.set(new InstanceRecord(instanceConfig, "stopped")); + } + + async function revoke(hostId = 10) { + await connection.handleHostRevokeRequest(new lib.HostRevokeRequest(hostId)); + } + + it("deletes and saves the host record", async function() { + let savedWithoutHost; + const originalSave = controller.hosts.save; + controller.hosts.save = async function() { + savedWithoutHost = !this.has(10); + return await originalSave.call(this); + }; + await revoke(); + assert.equal(controller.hosts.has(10), false); + assert.equal(savedWithoutHost, true, "hosts were not saved after deleting the record"); + assert.deepEqual(broadcasts, [{ id: 10, isDeleted: true }]); + assert.equal(audits.length, 1); + assert.match(audits[0], /User admin revoked host host-10 \(10\)/); + }); + it("deletes the note of the host", async function() { + controller.setNote("host", "10", "rack 3", "someone"); + controller.setNote("host", "11", "rack 4", "someone"); + await revoke(); + const note = controller.notes.get("host/10"); + assert.equal(note.isDeleted, true); + assert.equal(note.content, ""); + assert.equal(note.updatedBy, "admin"); + assert.equal(controller.notes.get("host/11").isDeleted, false); + }); + it("terminates the connection of the host", async function() { + let terminated = false; + controller.wsServer.hostConnections.set(10, { connector: { terminate() { terminated = true; } } }); + await revoke(); + assert.equal(terminated, true); + }); + it("terminates the connection of the host when saving fails", async function() { + let terminated = false; + controller.wsServer.hostConnections.set(10, { connector: { terminate() { terminated = true; } } }); + controller.hosts.save = async () => { throw new Error("EPERM: operation not permitted"); }; + await assert.rejects(revoke(), /EPERM/); + assert.equal(terminated, true, "revoked host kept its connection"); + assert.equal(controller.hosts.has(10), false); + assert.equal(controller.notes.get("host/10"), undefined); + }); + it("does not leave a host record behind when the connection closes", async function() { + const connector = new MockConnector(addr("controller"), addr({ hostId: 10 })); + connector.terminate = () => { connector.emit("close"); }; + const hostConnection = new HostConnection( + new lib.RegisterHostData("", "2.0.0", 10, {}), connector, controller, "10.0.0.1" + ); + controller.wsServer.hostConnections.set(10, hostConnection); + assert.equal(controller.hosts.get(10).connected, true); + + await revoke(); + assert.equal(controller.hosts.has(10), false, "host record was recreated"); + assert.deepEqual(broadcasts.at(-1), { id: 10, isDeleted: true }); + }); + it("refuses to revoke a host with assigned instances", async function() { + controller.setNote("host", "10", "rack 3", "someone"); + let terminated = false; + controller.wsServer.hostConnections.set(10, { connector: { terminate() { terminated = true; } } }); + assignInstance(101, "alpha", 10); + assignInstance(102, "beta", 10); + assignInstance(201, "gamma", 20); + + await assert.rejects( + revoke(), + new lib.ConflictError("Cannot revoke host host-10 while instances are assigned to it: alpha, beta"), + ); + assert.equal(controller.hosts.get(10), record); + assert.equal(controller.notes.get("host/10").isDeleted, false); + assert.equal(terminated, false); + assert.deepEqual(broadcasts, []); + }); + it("ignores instances assigned to other hosts", async function() { + assignInstance(201, "gamma", 20); + await revoke(); + assert.equal(controller.hosts.has(10), false); + }); + it("throws NotFoundError for an unknown host", async function() { + await assert.rejects(revoke(99), new lib.NotFoundError("Unknown host id (99)")); + assert.equal(controller.hosts.has(10), true); + }); + }); + + describe("host pairing requests", function() { + it("lists pending requests", async function() { + const first = controller.hostPairing.create("Host A", "2.0.0", "10.0.0.1"); + const second = controller.hostPairing.create("Host B", "2.0.0", "10.0.0.2"); + const list = await connection.handleHostPairingListRequest(new lib.HostPairingListRequest()); + assert(list.every(details => details instanceof lib.HostPairingDetails)); + assert.deepEqual(list.map(details => details.code).sort(), [first.code, second.code].sort()); + assert.equal(list.find(details => details.code === first.code).name, "Host A"); + }); + it("leaves out expired, approved and rejected requests when listing", async function() { + const expired = controller.hostPairing.create("Host A", "", "10.0.0.1"); + const approved = controller.hostPairing.create("Host B", "", "10.0.0.2"); + const rejected = controller.hostPairing.create("Host C", "", "10.0.0.3"); + const pending = controller.hostPairing.create("Host D", "", "10.0.0.4"); + controller.hostPairing._bySecret.get(expired.requestId).lastPollMs -= 60e3; + await controller.hostPairing.approve(approved.code, undefined, "admin"); + controller.hostPairing.reject(rejected.code, "admin"); + + const list = await connection.handleHostPairingListRequest(new lib.HostPairingListRequest()); + assert.deepEqual(list.map(details => details.code), [pending.code]); + }); + it("approves a request and returns the new host id", async function() { + const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); + const hostId = await connection.handleHostPairingApproveRequest( + new lib.HostPairingApproveRequest(code.toLowerCase().replace("-", "")) + ); + assert.equal(typeof hostId, "number"); + assert.equal(controller.hosts.get(hostId).name, "Host A"); + assert.equal(controller.hostPairing.poll(requestId).hostId, hostId); + assert(audits.some(msg => msg.startsWith(`User admin approved host Host A (${hostId})`))); + }); + it("approves a request with a different name", async function() { + const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); + const hostId = await connection.handleHostPairingApproveRequest( + new lib.HostPairingApproveRequest(code, "Renamed") + ); + assert.equal(controller.hosts.get(hostId).name, "Renamed"); + assert.equal(controller.hostPairing.poll(requestId).name, "Renamed"); + }); + it("rejects approving an unknown code or with an invalid name", async function() { + await assert.rejects( + connection.handleHostPairingApproveRequest(new lib.HostPairingApproveRequest("ZZZ-ZZZ")), + new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), + ); + const { code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); + await assert.rejects( + connection.handleHostPairingApproveRequest(new lib.HostPairingApproveRequest(code, " ")), + new lib.ValidationError("Host name is required"), + ); + assert.equal(controller.hosts.size, 0); + }); + it("rejects a request", async function() { + const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); + await connection.handleHostPairingRejectRequest(new lib.HostPairingRejectRequest(code.toLowerCase())); + assert.equal(controller.hostPairing.requests.has(code), false); + assert.throws(() => controller.hostPairing.poll(requestId), /Request was rejected/); + assert.equal(controller.hosts.size, 0); + assert(audits.some(msg => msg.startsWith(`User admin rejected host Host A with code ${code}`))); + }); + it("throws NotFoundError when rejecting an unknown code", async function() { + await assert.rejects( + connection.handleHostPairingRejectRequest(new lib.HostPairingRejectRequest("ZZZ-ZZZ")), + new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), + ); + }); + }); + }); }); diff --git a/test/controller/HostConnection.test.js b/test/controller/HostConnection.test.js index 7c86e278..d8722af2 100644 --- a/test/controller/HostConnection.test.js +++ b/test/controller/HostConnection.test.js @@ -1,6 +1,9 @@ import assert from "node:assert/strict"; import * as lib from "@clusterio/lib"; -import { HostConnection } from "@clusterio/controller"; +import { Controller, HostConnection, HostRecord } from "@clusterio/controller"; +import { MockConnector } from "../mock.js"; + +const addr = lib.Address.fromShorthand; function makeSave(instanceId, name) { return new lib.SaveDetails(instanceId, "file", name, 0, 0, false, false, 0, false); @@ -40,6 +43,99 @@ function makeInstances(assignments, names = new Map()) { } describe("controller/src/HostConnection", function() { + describe("host record", function() { + /** @type {Controller} */ + let controller; + let record; + let connector; + let updates; + + beforeEach(function() { + controller = new Controller(lib.logger, [], new lib.ControllerConfig("controller")); + record = new HostRecord(10, "host-10", "1.0.0", new Map(), false, "", "203.0.113.1", 123); + controller.hosts.set(record); + connector = new MockConnector(addr("controller"), addr({ hostId: 10 })); + updates = []; + controller.hosts.on("update", changed => { + updates.push(...changed.map(host => ({ id: host.id, name: host.name, isDeleted: host.isDeleted }))); + }); + }); + + function connect(id = 10) { + return new HostConnection( + new lib.RegisterHostData("", "2.0.0", id, {}), connector, controller, "10.0.0.1" + ); + } + + function revoke() { + controller.hosts.delete(record); + updates.length = 0; + } + + it("should refuse a host without a record", function() { + assert.throws(() => connect(11), /Host 11 is not registered/); + assert.equal(controller.hosts.has(11), false); + assert.deepEqual(updates, []); + }); + it("should update the existing record on connect", function() { + const connection = connect(); + assert.equal(controller.hosts.get(10), record, "host record was replaced"); + assert.equal(connection.id, 10); + assert.equal(record.connected, true); + assert.equal(record.version, "2.0.0"); + assert.equal(record.remoteAddress, "10.0.0.1"); + assert.equal(record.name, "host-10"); + assert.equal(record.publicAddress, "203.0.113.1"); + assert.equal(record.tokenValidAfter, 123); + assert.deepEqual(updates, [{ id: 10, name: "host-10", isDeleted: false }]); + }); + it("should mark the host disconnected when the connection closes", function() { + connect(); + updates.length = 0; + connector.emit("close"); + assert.equal(controller.hosts.get(10), record); + assert.equal(record.connected, false); + assert.deepEqual(updates, [{ id: 10, name: "host-10", isDeleted: false }]); + }); + it("should save info updates from the host", async function() { + const connection = connect(); + updates.length = 0; + await connection.handleHostInfoUpdateEvent( + new lib.HostInfoUpdateEvent(new lib.HostInfoUpdate("renamed", "198.51.100.1")) + ); + assert.equal(controller.hosts.get(10).name, "renamed"); + assert.equal(controller.hosts.get(10).publicAddress, "198.51.100.1"); + assert.deepEqual(updates, [{ id: 10, name: "renamed", isDeleted: false }]); + }); + it("should not recreate the record of a revoked host when the connection closes", function() { + connect(); + revoke(); + connector.emit("close"); + assert.equal(controller.hosts.has(10), false, "host record was recreated"); + assert.deepEqual(updates, []); + }); + it("should not recreate the record of a revoked host on info updates", async function() { + const connection = connect(); + revoke(); + await connection.handleHostInfoUpdateEvent( + new lib.HostInfoUpdateEvent(new lib.HostInfoUpdate("renamed", "198.51.100.1")) + ); + assert.equal(controller.hosts.has(10), false, "host record was recreated"); + assert.deepEqual(updates, []); + }); + it("should not overwrite a newer record with the same id", function() { + connect(); + revoke(); + const newer = new HostRecord(10, "newer", "", new Map()); + controller.hosts.set(newer); + updates.length = 0; + connector.emit("close"); + assert.equal(controller.hosts.get(10), newer); + assert.equal(newer.name, "newer"); + assert.deepEqual(updates, []); + }); + }); + describe(".handleInstanceSaveDetailsUpdatesEvent()", function() { // Instance 1 is on host 10, instance 2 is on host 20. const assignments = new Map([[1, 10], [2, 20]]); diff --git a/test/controller/HostPairingManager.test.js b/test/controller/HostPairingManager.test.js new file mode 100644 index 00000000..912b4dbe --- /dev/null +++ b/test/controller/HostPairingManager.test.js @@ -0,0 +1,648 @@ +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import jwt from "jsonwebtoken"; + +import * as lib from "@clusterio/lib"; +import { Controller, HostPairingManager, HostPairingError, HostRecord } from "@clusterio/controller"; +import { + addressKey, normaliseCode, pollTimeoutMs, maxPendingMs, maxPending, maxPendingPerAddress, + maxCreatesPerMinute, maxNameLength, +} from "@clusterio/controller/dist/node/src/HostPairingManager.js"; + +import "../setup_logging.js"; + +const codePattern = /^[2-9A-HJKMNP-Z]{3}-[2-9A-HJKMNP-Z]{3}$/; + +function makeController() { + const config = new lib.ControllerConfig("controller"); + config.set("controller.auth_secret", Buffer.from("TestSecretDoNotUse").toString("base64")); + return new Controller(lib.logger, [], config); +} + +/** + * Make crypto.randomInt return the given values before falling back to random ones + * + * @param {number[]} values - Values to return, in order. + * @returns {() => void} function restoring crypto.randomInt. + */ +function stubRandomInt(values) { + const original = crypto.randomInt; + crypto.randomInt = (...args) => (values.length ? values.shift() : original(...args)); + return () => { crypto.randomInt = original; }; +} + +function deferred() { + let resolve, reject; + const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); + return { promise, resolve, reject }; +} + +function assertPairingError(fn, status, message) { + assert.throws(fn, err => { + assert(err instanceof HostPairingError, `expected HostPairingError, got ${err}`); + assert.equal(err.status, status); + if (message) { + assert.equal(err.message, message); + } + return true; + }); +} + +describe("controller/src/HostPairingManager", function() { + describe("addressKey()", function() { + it("should return IPv4 addresses unchanged", function() { + assert.equal(addressKey("10.0.0.1"), "10.0.0.1"); + }); + it("should strip the IPv4 mapped prefix", function() { + assert.equal(addressKey("::ffff:10.0.0.1"), "10.0.0.1"); + }); + it("should group IPv6 addresses by their /64", function() { + assert.equal(addressKey("2001:db8:0:1::5"), "2001:db8:0:1::/64"); + assert.equal(addressKey("2001:db8::1:ffff:0:0:3"), "2001:db8:0:1::/64"); + assert.equal(addressKey("2001:0DB8:0000:0001:aaaa:bbbb:cccc:dddd"), "2001:db8:0:1::/64"); + assert.equal(addressKey("2001:db8:0:2::5"), "2001:db8:0:2::/64"); + }); + it("should expand compressed groups at the start and end", function() { + assert.equal(addressKey("::1"), "0:0:0:0::/64"); + assert.equal(addressKey("::"), "0:0:0:0::/64"); + assert.equal(addressKey("fe80::1"), "fe80:0:0:0::/64"); + assert.equal(addressKey("2001:db8::"), "2001:db8:0:0::/64"); + }); + it("should return other values unchanged", function() { + assert.equal(addressKey(""), ""); + assert.equal(addressKey("not an address"), "not an address"); + assert.equal(addressKey("::ffff:not-ipv4"), "::ffff:not-ipv4"); + }); + }); + + describe("normaliseCode()", function() { + it("should format codes as shown by the installer", function() { + assert.equal(normaliseCode("ABC-DEF"), "ABC-DEF"); + assert.equal(normaliseCode("abcdef"), "ABC-DEF"); + assert.equal(normaliseCode("abc-def"), "ABC-DEF"); + assert.equal(normaliseCode(" abc def "), "ABC-DEF"); + assert.equal(normaliseCode("a.b_c-d e/f"), "ABC-DEF"); + }); + }); + + describe("HostPairingManager.validateName()", function() { + it("should require a non-empty string", function() { + for (const name of [undefined, null, 5, {}, "", " "]) { + assertPairingError(() => HostPairingManager.validateName(name), 400, "Host name is required"); + } + }); + it("should trim the name", function() { + assert.equal(HostPairingManager.validateName(" Host A \t"), "Host A"); + }); + it("should limit the length of the name", function() { + const longest = "x".repeat(maxNameLength); + assert.equal(maxNameLength, 100); + assert.equal(HostPairingManager.validateName(longest), longest); + assert.equal(HostPairingManager.validateName(` ${longest} `), longest); + assertPairingError( + () => HostPairingManager.validateName(`${longest}x`), + 400, "Host name must be at most 100 characters", + ); + }); + it("should refuse control and bidirectional override characters", function() { + for (const name of ["x\u001b]0;title\u0007", "line\nfaked", "a\rb", "del\u007f", "c1\u009b", "x\u202ey"]) { + assertPairingError( + () => HostPairingManager.validateName(name), 400, "Host name may not contain control characters", + ); + } + }); + it("should allow other unicode characters", function() { + for (const name of ["Fabrikk p\u00e5 \u00d8stlandet", "\u5de5\u5382 1", "box \u{1F44D}\u200d"]) { + assert.equal(HostPairingManager.validateName(name), name); + } + }); + }); + + describe("class HostPairingManager", function() { + /** @type {Controller} */ + let controller; + /** @type {HostPairingManager} */ + let manager; + let updates; + let restoreRandomInt; + + beforeEach(function() { + controller = makeController(); + manager = controller.hostPairing; + updates = []; + manager.requests.on("update", changed => { + updates.push(...changed.map(details => ({ code: details.code, isDeleted: details.isDeleted }))); + }); + restoreRandomInt = undefined; + }); + afterEach(function() { + manager.stop(); + if (restoreRandomInt) { + restoreRandomInt(); + } + }); + + function internal(requestId) { + return manager._bySecret.get(requestId); + } + + // Pretend the installer last polled the request ms milliseconds earlier. + function idle(requestId, ms) { + internal(requestId).lastPollMs -= ms; + } + + describe(".createHost()", function() { + it("should create a host record with a valid token", async function() { + const before = Math.floor(Date.now() / 1000); + const result = await manager.createHost("Host A"); + + assert.equal(result.name, "Host A"); + assert(Number.isInteger(result.hostId), "hostId is not an integer"); + assert(result.hostId >= 1 && result.hostId < 2 ** 31, `hostId ${result.hostId} out of range`); + + const record = controller.hosts.get(result.hostId); + assert(record instanceof HostRecord, "host record was not created"); + assert.equal(record.name, "Host A"); + assert.equal(record.connected, false); + assert(record.tokenValidAfter >= before, "tokenValidAfter predates the call"); + assert(record.tokenValidAfter <= Math.floor(Date.now() / 1000), "tokenValidAfter is in the future"); + + const payload = jwt.verify(result.token, controller.authSecret, { audience: "host" }); + assert.equal(payload.host, result.hostId); + assert(payload.iat >= record.tokenValidAfter, "token is not valid for the new record"); + }); + it("should save the hosts before returning", async function() { + const save = deferred(); + let saveCalls = 0; + controller.hosts.save = () => { saveCalls += 1; return save.promise; }; + + let done = false; + const promise = manager.createHost("Host A").then(result => { done = true; return result; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(saveCalls, 1); + assert.equal(done, false, "createHost returned before the hosts were saved"); + save.resolve(); + const result = await promise; + assert(controller.hosts.has(result.hostId)); + }); + it("should remove the record if saving fails", async function() { + restoreRandomInt = stubRandomInt([1234]); + controller.hosts.save = async () => { throw new Error("disk full"); }; + await assert.rejects(manager.createHost("Host A"), /disk full/); + assert.equal(controller.hosts.has(1234), false); + assert.equal(controller.hosts.size, 0); + }); + it("should not reuse the id of an existing host", async function() { + controller.hosts.set(new HostRecord(5, "existing", "", new Map())); + restoreRandomInt = stubRandomInt([5, 5, 6]); + const result = await manager.createHost("Host A"); + assert.equal(result.hostId, 6); + assert.equal(controller.hosts.get(5).name, "existing"); + assert.equal(controller.hosts.get(6).name, "Host A"); + }); + }); + + describe(".create()", function() { + it("should create a pending request", function() { + const before = Date.now(); + const { requestId, code } = manager.create("Host A", "2.0.0", "10.0.0.1"); + + assert.equal(typeof requestId, "string"); + assert(requestId.length >= 32, "requestId is too short to be a secret"); + assert.match(code, codePattern); + assert.notEqual(requestId, code); + + const details = manager.requests.get(code); + assert(details instanceof lib.HostPairingDetails); + assert.equal(details.code, code); + assert.equal(details.name, "Host A"); + assert.equal(details.version, "2.0.0"); + assert.equal(details.remoteAddress, "10.0.0.1"); + assert(details.createdAtMs >= before && details.createdAtMs <= Date.now()); + assert(details.updatedAtMs > 0); + assert.equal(details.isDeleted, false); + assert(!JSON.stringify(details).includes(requestId), "requestId is exposed to users"); + assert.deepEqual(updates, [{ code, isDeleted: false }]); + }); + it("should create unique codes", function() { + const codes = new Set(); + for (let i = 0; i < maxPending; i++) { + const { code } = manager.create(`Host ${i}`, "", `10.1.${i}.1`); + assert.match(code, codePattern); + codes.add(code); + } + assert.equal(codes.size, maxPending); + }); + it("should not reuse the code of a pending request", function() { + restoreRandomInt = stubRandomInt([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]); + assert.equal(manager.create("Host A", "", "10.0.0.1").code, "222-222"); + assert.equal(manager.create("Host B", "", "10.0.0.2").code, "333-333"); + }); + it("should not reuse the code of an approved request not yet picked up", async function() { + restoreRandomInt = stubRandomInt([0, 0, 0, 0, 0, 0, 1234, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]); + const { code } = manager.create("Host A", "", "10.0.0.1"); + assert.equal(code, "222-222"); + assert.equal(await manager.approve(code, undefined, "admin"), 1234); + assert.equal(manager.create("Host B", "", "10.0.0.2").code, "333-333"); + }); + it("should use distinct request ids", function() { + const first = manager.create("Host A", "", "10.0.0.1"); + const second = manager.create("Host A", "", "10.0.0.1"); + assert.notEqual(first.requestId, second.requestId); + assert.notEqual(first.code, second.code); + }); + }); + + describe("limits", function() { + it("should limit pending requests per address", function() { + assert.equal(maxPendingPerAddress, 3); + for (let i = 0; i < maxPendingPerAddress; i++) { + manager.create("Host", "", "10.0.0.1"); + } + assert.throws(() => manager.create("Host", "", "10.0.0.1"), err => { + assert(err instanceof HostPairingError); + assert.equal(err.status, 429); + assert.equal(err.retryAfter, 30); + assert.equal(err.message, "Too many pending requests from this address"); + return true; + }); + // Other addresses are not affected + manager.create("Host", "", "10.0.0.2"); + assert.equal(manager.requests.size, maxPendingPerAddress + 1); + }); + it("should count IPv4 mapped addresses as the IPv4 address", function() { + manager.create("Host", "", "10.0.0.1"); + manager.create("Host", "", "::ffff:10.0.0.1"); + manager.create("Host", "", "10.0.0.1"); + assertPairingError(() => manager.create("Host", "", "::ffff:10.0.0.1"), 429); + }); + it("should group IPv6 addresses by /64", function() { + manager.create("Host", "", "2001:db8:0:1::1"); + manager.create("Host", "", "2001:db8:0:1::2"); + manager.create("Host", "", "2001:db8::1:ffff:0:0:3"); + assertPairingError( + () => manager.create("Host", "", "2001:0db8:0000:0001:0000:0000:0000:0004"), + 429, "Too many pending requests from this address", + ); + manager.create("Host", "", "2001:db8:0:2::1"); + }); + it("should free the slot when a request is no longer pending", async function() { + const first = manager.create("Host", "", "10.0.0.1"); + const second = manager.create("Host", "", "10.0.0.1"); + const third = manager.create("Host", "", "10.0.0.1"); + assertPairingError(() => manager.create("Host", "", "10.0.0.1"), 429); + + manager.cancel(first.requestId); + const fourth = manager.create("Host", "", "10.0.0.1"); + assertPairingError(() => manager.create("Host", "", "10.0.0.1"), 429); + + // Approved and rejected results waiting to be picked up do not count + await manager.approve(second.code, undefined, "admin"); + manager.reject(third.code, "admin"); + manager.create("Host", "", "10.0.0.1"); + manager.create("Host", "", "10.0.0.1"); + assertPairingError(() => manager.create("Host", "", "10.0.0.1"), 429); + assert(manager.requests.has(fourth.code)); + }); + it("should limit the total number of pending requests", function() { + assert.equal(maxPending, 50); + for (let i = 0; i < maxPending; i++) { + manager.create(`Host ${i}`, "", `10.1.${i}.1`); + } + assert.throws(() => manager.create("Host", "", "10.2.0.1"), err => { + assert(err instanceof HostPairingError); + assert.equal(err.status, 429); + assert.equal(err.retryAfter, 30); + assert.equal(err.message, "Too many pending requests"); + return true; + }); + assert.equal(manager.requests.size, maxPending); + }); + it("should limit the number of requests created per minute from an address", function() { + assert.equal(maxCreatesPerMinute, 10); + for (let i = 0; i < maxCreatesPerMinute; i++) { + manager.cancel(manager.create("Host", "", "10.0.0.1").requestId); + } + assert.throws(() => manager.create("Host", "", "10.0.0.1"), err => { + assert(err instanceof HostPairingError); + assert.equal(err.status, 429); + assert.equal(err.message, "Too many requests from this address"); + assert(err.retryAfter >= 1 && err.retryAfter <= 60, `bad retryAfter ${err.retryAfter}`); + return true; + }); + // IPv6 addresses in the same /64 share the limit + for (let i = 0; i < maxCreatesPerMinute; i++) { + manager.cancel(manager.create("Host", "", `2001:db8:0:1::${i + 1}`).requestId); + } + assertPairingError( + () => manager.create("Host", "", "2001:db8:0:1:ffff::1"), 429, "Too many requests from this address" + ); + manager.create("Host", "", "10.0.0.2"); + }); + it("should allow creating requests again after a minute", function() { + for (let i = 0; i < maxCreatesPerMinute; i++) { + manager.cancel(manager.create("Host", "", "10.0.0.1").requestId); + } + assertPairingError(() => manager.create("Host", "", "10.0.0.1"), 429); + for (const times of manager._recentCreates.values()) { + times.forEach((ms, index) => { times[index] = ms - 60e3 - 1; }); + } + manager.create("Host", "", "10.0.0.1"); + }); + it("should not count refused requests towards the rate limit", function() { + const first = manager.create("Host", "", "10.0.0.1"); + manager.create("Host", "", "10.0.0.1"); + manager.create("Host", "", "10.0.0.1"); + const createRefused = () => manager.create("Host", "", "10.0.0.1"); + for (let i = 0; i < maxCreatesPerMinute; i++) { + assertPairingError(createRefused, 429); + } + manager.cancel(first.requestId); + manager.create("Host", "", "10.0.0.1"); + }); + }); + + describe(".poll()", function() { + it("should return null while pending", function() { + const { requestId } = manager.create("Host A", "", "10.0.0.1"); + assert.equal(manager.poll(requestId), null); + assert.equal(manager.poll(requestId), null); + }); + it("should throw 404 for an unknown request", function() { + assertPairingError(() => manager.poll("invalid"), 404, "Request not found"); + }); + it("should not accept the code in place of the request id", function() { + const { code } = manager.create("Host A", "", "10.0.0.1"); + assertPairingError(() => manager.poll(code), 404); + }); + it("should hand over an approved result once", async function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = await manager.approve(code, undefined, "admin"); + const result = manager.poll(requestId); + assert.deepEqual(Object.keys(result).sort(), ["hostId", "name", "token"]); + assert.equal(result.hostId, hostId); + assert.equal(result.name, "Host A"); + const payload = jwt.verify(result.token, controller.authSecret, { audience: "host" }); + assert.equal(payload.host, hostId); + + assertPairingError(() => manager.poll(requestId), 404, "Request not found"); + assert.equal(manager._byCode.has(code), false); + }); + it("should report a rejection once", function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + manager.reject(code, "admin"); + assertPairingError(() => manager.poll(requestId), 403, "Request was rejected"); + assertPairingError(() => manager.poll(requestId), 404, "Request not found"); + }); + }); + + describe(".cancel()", function() { + it("should remove a pending request", function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + manager.cancel(requestId); + assert.equal(manager.requests.has(code), false); + assertPairingError(() => manager.poll(requestId), 404); + assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); + assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); + }); + it("should ignore unknown requests", function() { + manager.cancel("invalid"); + }); + it("should not throw away an approved result", async function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = await manager.approve(code, undefined, "admin"); + manager.cancel(requestId); + assert.equal(manager.poll(requestId).hostId, hostId); + }); + it("should not throw away a rejection", function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + manager.reject(code, "admin"); + manager.cancel(requestId); + assertPairingError(() => manager.poll(requestId), 403); + }); + }); + + describe(".approve()", function() { + it("should create the host with the requested name", async function() { + const { code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = await manager.approve(code, undefined, "admin"); + assert.equal(controller.hosts.get(hostId).name, "Host A"); + assert.equal(manager.requests.has(code), false); + assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); + }); + it("should accept codes in any case and without the dash", async function() { + const { code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = await manager.approve(code.toLowerCase().replace("-", ""), undefined, "admin"); + assert(controller.hosts.has(hostId)); + }); + it("should use and trim the name given", async function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = await manager.approve(code, " Renamed ", "admin"); + assert.equal(controller.hosts.get(hostId).name, "Renamed"); + assert.equal(manager.poll(requestId).name, "Renamed"); + }); + it("should reject an invalid name and keep the request pending", async function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + await assert.rejects( + manager.approve(code, "", "admin"), + new lib.ValidationError("Host name is required"), + ); + await assert.rejects( + manager.approve(code, "x".repeat(maxNameLength + 1), "admin"), + new lib.ValidationError("Host name must be at most 100 characters"), + ); + assert.equal(controller.hosts.size, 0); + assert.equal(manager.poll(requestId), null); + await manager.approve(code, undefined, "admin"); + assert.equal(controller.hosts.size, 1); + }); + it("should throw NotFoundError for an unknown code", async function() { + await assert.rejects( + manager.approve("ZZZ-ZZZ", undefined, "admin"), + new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), + ); + }); + it("should only approve a request once", async function() { + const { code } = manager.create("Host A", "", "10.0.0.1"); + await manager.approve(code, undefined, "admin"); + await assert.rejects(manager.approve(code, undefined, "admin"), lib.NotFoundError); + assert.equal(controller.hosts.size, 1); + }); + it("should not create two hosts when approved twice at the same time", async function() { + const save = deferred(); + controller.hosts.save = () => save.promise; + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + + const first = manager.approve(code, undefined, "admin"); + await assert.rejects(manager.approve(code, undefined, "admin"), lib.NotFoundError); + assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); + + // The request stays while approving, the installer sees it as pending + manager.cancel(requestId); + idle(requestId, pollTimeoutMs + 1); + manager.prune(); + assert.equal(manager.poll(requestId), null); + assert(manager.requests.has(code), "request removed while approving"); + + save.resolve(); + const hostId = await first; + assert.equal(controller.hosts.size, 1); + assert.equal(manager.poll(requestId).hostId, hostId); + }); + it("should keep the request pending if creating the host fails", async function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + const originalSave = controller.hosts.save; + controller.hosts.save = async () => { throw new Error("disk full"); }; + await assert.rejects(manager.approve(code, undefined, "admin"), /disk full/); + assert.equal(controller.hosts.size, 0); + assert(manager.requests.has(code)); + assert.equal(manager.poll(requestId), null); + + controller.hosts.save = originalSave; + const hostId = await manager.approve(code, undefined, "admin"); + assert.equal(manager.poll(requestId).hostId, hostId); + }); + }); + + describe(".reject()", function() { + it("should remove the request from the pending list", function() { + const { code } = manager.create("Host A", "", "10.0.0.1"); + manager.reject(code.toLowerCase(), "admin"); + assert.equal(manager.requests.has(code), false); + assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); + assert.equal(controller.hosts.size, 0); + }); + it("should throw NotFoundError for an unknown code", function() { + assert.throws( + () => manager.reject("ZZZ-ZZZ", "admin"), + new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), + ); + }); + it("should not reject an approved request", async function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = await manager.approve(code, undefined, "admin"); + assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); + assert.equal(manager.poll(requestId).hostId, hostId); + }); + }); + + describe(".prune()", function() { + let warnings; + let originalWarn; + beforeEach(function() { + warnings = []; + originalWarn = lib.logger.warn; + lib.logger.warn = msg => { warnings.push(msg); }; + }); + afterEach(function() { + lib.logger.warn = originalWarn; + }); + + it("should expire a request the installer stopped polling", function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + idle(requestId, pollTimeoutMs + 1); + manager.prune(); + assert.equal(manager.requests.has(code), false); + assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); + assertPairingError(() => manager.poll(requestId), 404); + assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); + }); + it("should keep a request that is still polled", function() { + assert.equal(pollTimeoutMs, 30e3); + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + idle(requestId, pollTimeoutMs - 1000); + manager.prune(); + assert.equal(manager.poll(requestId), null); + idle(requestId, pollTimeoutMs - 1000); + manager.prune(); + assert(manager.requests.has(code)); + }); + it("should expire a request that has been pending for too long", function() { + assert.equal(maxPendingMs, 60 * 60e3); + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + internal(requestId).details.createdAtMs -= maxPendingMs - 1000; + manager.prune(); + assert.equal(manager.poll(requestId), null); + internal(requestId).details.createdAtMs -= 1001; + manager.prune(); + assert.equal(manager.requests.has(code), false); + assertPairingError(() => manager.poll(requestId), 404); + }); + it("should expire requests when creating and polling", function() { + const first = manager.create("Host A", "", "10.0.0.1"); + idle(first.requestId, pollTimeoutMs + 1); + const second = manager.create("Host B", "", "10.0.0.1"); + assert.equal(manager.requests.has(first.code), false); + idle(second.requestId, pollTimeoutMs + 1); + assertPairingError(() => manager.poll(second.requestId), 404); + }); + it("should drop and warn about an approved host never picked up", async function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = await manager.approve(code, undefined, "admin"); + idle(requestId, pollTimeoutMs + 1); + manager.prune(); + assertPairingError(() => manager.poll(requestId), 404); + assert.equal(warnings.length, 1); + assert.match(warnings[0], new RegExp(`Approved host Host A \\(${hostId}\\) was never picked up`)); + // The host stays until revoked + assert(controller.hosts.has(hostId)); + }); + it("should drop a rejection never picked up", function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + manager.reject(code, "admin"); + idle(requestId, pollTimeoutMs + 1); + manager.prune(); + assertPairingError(() => manager.poll(requestId), 404); + assert.equal(warnings.length, 0); + }); + it("should forget creation times older than a minute", function() { + manager.create("Host A", "", "10.0.0.1"); + assert.equal(manager._recentCreates.size, 1); + manager.prune(); + assert.equal(manager._recentCreates.size, 1); + for (const times of manager._recentCreates.values()) { + times.forEach((ms, index) => { times[index] = ms - 60e3 - 1; }); + } + manager.prune(); + assert.equal(manager._recentCreates.size, 0); + }); + }); + + describe(".start()/.stop()", function() { + it("should run an unreferenced prune interval while started", function() { + manager.start(); + assert(manager._pruneInterval, "prune interval not started"); + assert.equal(manager._pruneInterval.hasRef(), false); + manager.stop(); + assert.equal(manager._pruneInterval, undefined); + }); + }); + + describe("Controller integration", function() { + it("should broadcast changes to pending requests", function() { + const broadcasts = []; + controller.subscriptions.broadcast = event => { + if (event instanceof lib.HostPairingUpdatesEvent) { + broadcasts.push(event.updates.map(details => [details.code, details.isDeleted])); + } + }; + const { code } = manager.create("Host A", "", "10.0.0.1"); + manager.reject(code, "admin"); + assert.deepEqual(broadcasts, [[[code, false]], [[code, true]]]); + }); + it("should send pending requests to new subscribers", async function() { + const { code } = manager.create("Host A", "", "10.0.0.1"); + const event = await controller.handleHostPairingSubscription( + new lib.SubscriptionRequest(lib.HostPairingUpdatesEvent.name, true, 0) + ); + assert(event instanceof lib.HostPairingUpdatesEvent); + assert.deepEqual(event.updates.map(details => details.code), [code]); + + const lastUpdate = manager.requests.get(code).updatedAtMs; + assert.equal(await controller.handleHostPairingSubscription( + new lib.SubscriptionRequest(lib.HostPairingUpdatesEvent.name, true, lastUpdate) + ), null); + }); + }); + }); +}); diff --git a/test/controller/WsServer.js b/test/controller/WsServer.js index c366b1ab..655a0b6b 100644 --- a/test/controller/WsServer.js +++ b/test/controller/WsServer.js @@ -1,7 +1,32 @@ import assert from "node:assert/strict"; import events from "node:events"; -import { Controller, WsServer } from "@clusterio/controller"; +import jwt from "jsonwebtoken"; +import * as lib from "@clusterio/lib"; +import { Controller, HostRecord, WsServer } from "@clusterio/controller"; import { ControllerConfig } from "@clusterio/lib"; +import packageJson from "@clusterio/controller/package.json" with { type: "json" }; + +import "../setup_logging.js"; + +class MockWebSocket extends events.EventEmitter { + constructor() { + super(); + this.sent = []; + this.closed = null; + } + + send(message) { + this.sent.push(JSON.parse(message)); + } + + close(code, reason) { + this.closed = { code, reason }; + } + + terminate() { + this.emit("close", 1006, ""); + } +} describe("controller/src/WsServer", function() { describe("class WsServer", function() { @@ -88,4 +113,120 @@ describe("controller/src/WsServer", function() { }); }); }); + describe(".registerHost()", function() { + const secret = Buffer.from("TestSecretDoNotUse"); + const req = { socket: { remoteAddress: "10.0.0.1", remoteFamily: "IPv4" }, headers: {} }; + /** @type {Controller} */ + let controller; + /** @type {WsServer} */ + let wsServer; + let socket; + beforeEach(function() { + const config = new ControllerConfig("controller"); + config.set("controller.auth_secret", secret.toString("base64")); + controller = new Controller(lib.logger, [], config); + wsServer = controller.wsServer; + socket = new MockWebSocket(); + }); + afterEach(function() { + for (const connection of wsServer.hostConnections.values()) { + connection.connector.terminate(); + } + }); + + function addHost(id, tokenValidAfter = 0) { + const record = new HostRecord(id, `host-${id}`, "", new Map(), false, "", "203.0.113.1", tokenValidAfter); + controller.hosts.set(record); + return record; + } + + async function register(token, id = 5) { + await wsServer.registerHost(new lib.RegisterHostData(token, packageJson.version, id, {}), socket, req); + } + + function assertRejected(reason) { + assert.deepEqual(socket.closed, { code: lib.ConnectionClosed.Unauthorized, reason }); + assert.equal(socket.closed.code, 3000); + assert.equal(wsServer.hostConnections.size, 0); + assert(!socket.sent.some(message => message.type === "ready"), "ready was sent"); + } + + it("should reject a validly signed token for a host that is not registered", async function() { + await register(controller.generateHostToken(5)); + assertRejected( + "Authentication failed: host is not registered, run the installer again in the host directory" + ); + assert.equal(controller.hosts.has(5), false, "host record was created"); + }); + it("should accept a registered host", async function() { + const record = addHost(5); + await register(controller.generateHostToken(5)); + assert.equal(socket.closed, null); + assert(socket.sent.some(message => message.type === "ready"), "ready was not sent"); + assert(wsServer.hostConnections.has(5)); + + assert.equal(controller.hosts.size, 1); + assert.equal(controller.hosts.get(5), record, "host record was replaced"); + assert.equal(record.connected, true); + assert.equal(record.version, packageJson.version); + assert.equal(record.remoteAddress, "10.0.0.1"); + assert.equal(record.name, "host-5"); + assert.equal(record.publicAddress, "203.0.113.1"); + + wsServer.hostConnections.get(5).connector.terminate(); + assert.equal(wsServer.hostConnections.has(5), false); + assert.equal(controller.hosts.get(5).connected, false); + }); + it("should reject a token issued before tokenValidAfter", async function() { + const token = controller.generateHostToken(5); + addHost(5, Math.floor(Date.now() / 1000) + 60); + await register(token); + assertRejected("Authentication failed: invalid token"); + }); + it("should accept a token issued at tokenValidAfter", async function() { + const now = Math.floor(Date.now() / 1000); + addHost(5, now); + await register(jwt.sign({ aud: "host", host: 5, iat: now }, secret)); + assert.equal(socket.closed, null); + assert(wsServer.hostConnections.has(5)); + }); + it("should reject a token without an issue time", async function() { + addHost(5); + await register(jwt.sign({ aud: "host", host: 5 }, secret, { noTimestamp: true })); + assertRejected("Authentication failed: invalid token"); + }); + it("should reject a token for another host", async function() { + addHost(5); + addHost(6); + await register(controller.generateHostToken(6), 5); + assertRejected("Authentication failed: missmatched host id"); + }); + it("should reject a token signed with another secret", async function() { + addHost(5); + await register(jwt.sign({ aud: "host", host: 5 }, Buffer.from("OtherSecret"))); + assertRejected("Authentication failed: invalid signature"); + }); + it("should reject a host revoked while its previous connection closes", async function() { + const record = addHost(5); + wsServer.hostConnections.set(5, { + remoteAddress: "10.0.0.1", + async disconnect() { + controller.hosts.delete(record); + wsServer.hostConnections.delete(5); + }, + }); + await register(controller.generateHostToken(5)); + assertRejected("Authentication failed: host was revoked"); + assert.equal(controller.hosts.has(5), false, "host record was recreated"); + }); + it("should reject hosts in recovery mode", async function() { + addHost(5); + controller.recoveryMode = true; + await register(controller.generateHostToken(5)); + assert.deepEqual(socket.closed, { + code: lib.ConnectionClosed.RecoveryMode, + reason: "Connection Refused: Controller in recovery mode", + }); + }); + }); }); diff --git a/test/controller/routes.js b/test/controller/routes.js index d0af23da..94ea5f30 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -3,7 +3,10 @@ import events from "node:events"; import FormData from "form-data"; import http from "node:http"; -import { wait, expositionContentType } from "@clusterio/lib"; +import jwt from "jsonwebtoken"; + +import { SubscribableDatastore, wait, expositionContentType } from "@clusterio/lib"; +import { Controller, HostPairingManager, HostRecord } from "@clusterio/controller"; import * as routes from "@clusterio/controller/dist/node/src/routes.js"; import * as mock from "../mock.js"; @@ -211,6 +214,326 @@ describe("controller/src/routes", function() { assert.deepEqual(data, { name: "Test Cluster" }); }); }); + describe("/api/host-pairing", function() { + let endpoint, adminToken, playerToken; + beforeEach(function() { + endpoint = `http://localhost:${port}/api/host-pairing`; + // The mock controller lacks what pairing needs, use the real implementations. + controller.hosts = new SubscribableDatastore(); + controller.hostPairing = new HostPairingManager(controller); + controller.generateHostToken = Controller.prototype.generateHostToken; + controller.wsServer.remoteAddr = req => req.socket.remoteAddress; + adminToken = controller.users.signUserToken(controller.users.getByName("test")); + playerToken = controller.users.signUserToken(controller.users.getByName("player")); + }); + + function post(path, body, headers = {}) { + return fetch(`${endpoint}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json", ...headers }, + body: typeof body === "string" || body === undefined ? body : JSON.stringify(body), + }); + } + + async function assertResponse(response, status, body) { + assert.equal(response.status, status); + assert.equal(response.headers.get("cache-control"), "no-store"); + if (body !== undefined) { + assert.deepEqual(await response.json(), body); + } + } + + async function createRequest(name = "Host A") { + const response = await post("", { name }); + assert.equal(response.status, 202); + return await response.json(); + } + + function check(headers) { + return fetch(`${endpoint}/check`, { headers }); + } + + function assertNothingCreated() { + assert.equal(controller.hostPairing.requests.size, 0, "a pending request was created"); + assert.equal(controller.hosts.size, 0, "a host was created"); + } + + describe("POST /api/host-pairing", function() { + it("should create a pending request", async function() { + const response = await post("", { name: "Host A", version: "2.0.0" }); + assert.equal(response.status, 202); + assert.equal(response.headers.get("cache-control"), "no-store"); + const body = await response.json(); + assert.deepEqual(Object.keys(body).sort(), ["code", "requestId"]); + assert.match(body.code, /^[2-9A-HJKMNP-Z]{3}-[2-9A-HJKMNP-Z]{3}$/); + + const details = controller.hostPairing.requests.get(body.code); + assert.equal(details.name, "Host A"); + assert.equal(details.version, "2.0.0"); + assert.match(details.remoteAddress, /127\.0\.0\.1|::1/); + assert.equal(controller.hosts.size, 0); + }); + it("should trim the name and limit the version", async function() { + let { code } = await (await post("", { name: " Host A ", version: "x".repeat(100) })).json(); + let details = controller.hostPairing.requests.get(code); + assert.equal(details.name, "Host A"); + assert.equal(details.version, "x".repeat(64)); + + ({ code } = await (await post("", { name: "Host B", version: 5 })).json()); + details = controller.hostPairing.requests.get(code); + assert.equal(details.version, ""); + }); + it("should respond with 400 for a missing or invalid name", async function() { + const required = { error: "Host name is required" }; + await assertResponse(await post("", {}), 400, required); + await assertResponse(await post("", { name: "" }), 400, required); + await assertResponse(await post("", { name: " " }), 400, required); + await assertResponse(await post("", { name: 5 }), 400, required); + await assertResponse(await post("", undefined), 400, required); + await assertResponse(await post("", "name=Host", { "Content-Type": "text/plain" }), 400, required); + await assertResponse( + await post("", { name: "x".repeat(101) }), + 400, { error: "Host name must be at most 100 characters" }, + ); + assertNothingCreated(); + }); + it("should respond with 400 for malformed JSON", async function() { + controller.app.set("env", "test"); // Silence the default error handler + const response = await post("", "{\"name\":"); + assert.equal(response.status, 400); + assertNothingCreated(); + }); + it("should respond with 413 for a body over 4kb", async function() { + controller.app.set("env", "test"); // Silence the default error handler + const response = await post("", { name: "x".repeat(5000) }); + assert.equal(response.status, 413); + assertNothingCreated(); + }); + it("should respond with 429 and Retry-After when limited", async function() { + for (let i = 0; i < 3; i++) { + await createRequest(); + } + const response = await post("", { name: "Host A" }); + assert.equal(response.headers.get("retry-after"), "30"); + await assertResponse(response, 429, { error: "Too many pending requests from this address" }); + assert.equal(controller.hostPairing.requests.size, 3); + }); + it("should respond with 503 in recovery mode", async function() { + controller.recoveryMode = true; + const recovery = { error: "Controller is in recovery mode" }; + await assertResponse(await post("", { name: "Host A" }), 503, recovery); + await assertResponse( + await post("", { name: "Host A" }, { "Authorization": `Bearer ${adminToken}` }), 503, recovery + ); + assertNothingCreated(); + }); + it("should approve directly with the token of a user allowed to approve hosts", async function() { + for (const headers of [ + { "Authorization": `Bearer ${adminToken}` }, + { "Authorization": `bearer ${adminToken}` }, + { "X-Access-Token": adminToken }, + ]) { + const response = await post("", { name: "Host A" }, headers); + assert.equal(response.status, 200); + assert.equal(response.headers.get("cache-control"), "no-store"); + const body = await response.json(); + assert.deepEqual(Object.keys(body).sort(), ["hostId", "name", "token"]); + assert.equal(body.name, "Host A"); + assert.equal(controller.hosts.get(body.hostId).name, "Host A"); + const payload = jwt.verify(body.token, controller.authSecret, { audience: "host" }); + assert.equal(payload.host, body.hostId); + } + assert.equal(controller.hosts.size, 3); + assert.equal(controller.hostPairing.requests.size, 0, "a pending request was created"); + }); + it("should validate the name before approving directly", async function() { + await assertResponse( + await post("", {}, { "Authorization": `Bearer ${adminToken}` }), + 400, { error: "Host name is required" }, + ); + assertNothingCreated(); + }); + it("should respond with 403 for a user not allowed to approve hosts", async function() { + await assertResponse( + await post("", { name: "Host A" }, { "Authorization": `Bearer ${playerToken}` }), + 403, { error: "User player is not allowed to approve hosts" }, + ); + assertNothingCreated(); + }); + it("should respond with 401 for an invalid token without creating a request", async function() { + // The admin user's tokens are revoked after signing this one + const revokedToken = adminToken; + const user = controller.users.getOrCreateUser("test"); + user.tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000); + user.saveRecord(); + controller.hosts.set(new HostRecord(5, "five", "", new Map())); + const hostToken = controller.generateHostToken(5); + controller.hosts.delete(controller.hosts.get(5)); + + for (const headers of [ + { "Authorization": "Bearer invalid" }, + { "X-Access-Token": "invalid" }, + { "Authorization": `Bearer ${controller.users.signUserToken({ id: "nobody" })}` }, + { "Authorization": `Bearer ${revokedToken}` }, + { "Authorization": `Bearer ${hostToken}` }, + ]) { + await assertResponse( + await post("", { name: "Host A" }, headers), 401, { error: "Invalid user token" } + ); + } + assertNothingCreated(); + }); + }); + + describe("POST /api/host-pairing/poll", function() { + it("should respond with 202 while pending", async function() { + const { requestId } = await createRequest(); + await assertResponse(await post("/poll", { requestId }), 202, { status: "pending" }); + await assertResponse(await post("/poll", { requestId }), 202, { status: "pending" }); + }); + it("should hand over the result of an approved request once", async function() { + const { requestId, code } = await createRequest(); + const hostId = await controller.hostPairing.approve(code, "Renamed", "test"); + + const response = await post("/poll", { requestId }); + assert.equal(response.status, 200); + assert.equal(response.headers.get("cache-control"), "no-store"); + const body = await response.json(); + assert.deepEqual(Object.keys(body).sort(), ["hostId", "name", "token"]); + assert.equal(body.hostId, hostId); + assert.equal(body.name, "Renamed"); + assert.equal(jwt.verify(body.token, controller.authSecret, { audience: "host" }).host, hostId); + + await assertResponse(await post("/poll", { requestId }), 404, { error: "Request not found" }); + }); + it("should respond with 403 once for a rejected request", async function() { + const { requestId, code } = await createRequest(); + controller.hostPairing.reject(code, "test"); + await assertResponse(await post("/poll", { requestId }), 403, { error: "Request was rejected" }); + await assertResponse(await post("/poll", { requestId }), 404, { error: "Request not found" }); + }); + it("should respond with 404 for an unknown request", async function() { + const { code } = await createRequest(); + const notFound = { error: "Request not found" }; + await assertResponse(await post("/poll", { requestId: "invalid" }), 404, notFound); + await assertResponse(await post("/poll", { requestId: code }), 404, notFound); + }); + it("should respond with 400 without a requestId in the body", async function() { + const { requestId } = await createRequest(); + const required = { error: "requestId is required" }; + await assertResponse(await post("/poll", {}), 400, required); + await assertResponse(await post("/poll", { requestId: 5 }), 400, required); + await assertResponse(await post(`/poll?requestId=${encodeURIComponent(requestId)}`, {}), 400, required); + }); + it("should respond with 503 in recovery mode", async function() { + const { requestId } = await createRequest(); + controller.recoveryMode = true; + await assertResponse( + await post("/poll", { requestId }), 503, { error: "Controller is in recovery mode" } + ); + }); + }); + + describe("POST /api/host-pairing/cancel", function() { + it("should cancel a pending request", async function() { + const { requestId, code } = await createRequest(); + await assertResponse(await post("/cancel", { requestId }), 204); + assert.equal(controller.hostPairing.requests.has(code), false); + await assertResponse(await post("/poll", { requestId }), 404, { error: "Request not found" }); + }); + it("should respond with 204 for an unknown request", async function() { + const { code } = await createRequest(); + await assertResponse(await post("/cancel", { requestId: "invalid" }), 204); + await assertResponse(await post("/cancel", { requestId: code }), 204); + assert(controller.hostPairing.requests.has(code), "cancelled by the code"); + }); + it("should respond with 400 without a requestId in the body", async function() { + const { requestId, code } = await createRequest(); + await assertResponse(await post("/cancel", {}), 400, { error: "requestId is required" }); + await assertResponse( + await post(`/cancel?requestId=${encodeURIComponent(requestId)}`, {}), + 400, { error: "requestId is required" }, + ); + assert(controller.hostPairing.requests.has(code)); + }); + it("should respond with 503 in recovery mode", async function() { + const { requestId, code } = await createRequest(); + controller.recoveryMode = true; + await assertResponse( + await post("/cancel", { requestId }), 503, { error: "Controller is in recovery mode" } + ); + assert(controller.hostPairing.requests.has(code)); + }); + }); + + describe("request ids", function() { + it("should not be accepted in the URL", async function() { + const { requestId } = await createRequest(); + const id = encodeURIComponent(requestId); + for (const [method, path] of [ + ["GET", `/poll?requestId=${id}`], + ["GET", `/poll/${id}`], + ["POST", `/poll/${id}`], + ["POST", `/${id}`], + ["GET", `/${id}`], + ["POST", `/cancel/${id}`], + ]) { + const response = await fetch(`${endpoint}${path}`, { method }); + assert.equal(response.status, 404, `${method} ${path}`); + } + await assertResponse(await post("/poll", { requestId }), 202, { status: "pending" }); + }); + }); + + describe("GET /api/host-pairing/check", function() { + it("should respond with 200 for a valid host token", async function() { + const { hostId, token } = await controller.hostPairing.createHost("Host A"); + for (const headers of [ + { "Authorization": `Bearer ${token}` }, + { "X-Access-Token": token }, + ]) { + await assertResponse(await check(headers), 200, { valid: true }); + } + assert(controller.hosts.has(hostId)); + }); + it("should respond with 401 without a valid host token", async function() { + await controller.hostPairing.createHost("Host A"); + assert.equal((await check({})).status, 401); + assert.equal((await check({ "Authorization": "Bearer invalid" })).status, 401); + assert.equal((await check({ "Authorization": `Bearer ${adminToken}` })).status, 401); + }); + it("should respond with 401 for a host that is not registered", async function() { + const token = controller.generateHostToken(99); + assert.equal((await check({ "Authorization": `Bearer ${token}` })).status, 401); + }); + it("should respond with 401 once the host is revoked", async function() { + const { hostId, token } = await controller.hostPairing.createHost("Host A"); + await assertResponse(await check({ "Authorization": `Bearer ${token}` }), 200, { valid: true }); + controller.hosts.delete(controller.hosts.get(hostId)); + assert.equal((await check({ "Authorization": `Bearer ${token}` })).status, 401); + }); + it("should respond with 401 for a token issued before tokenValidAfter", async function() { + const { hostId, token } = await controller.hostPairing.createHost("Host A"); + const record = controller.hosts.getMutable(hostId); + record.tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000); + controller.hosts.set(record); + assert.equal((await check({ "Authorization": `Bearer ${token}` })).status, 401); + }); + }); + + describe("pairing flow", function() { + it("should pair a host approved while the installer polls", async function() { + const { requestId, code } = await createRequest("Host A"); + await assertResponse(await post("/poll", { requestId }), 202, { status: "pending" }); + const hostId = await controller.hostPairing.approve(code.toLowerCase(), undefined, "test"); + const result = await (await post("/poll", { requestId })).json(); + assert.deepEqual(result, { hostId, token: result.token, name: "Host A" }); + await assertResponse( + await check({ "Authorization": `Bearer ${result.token}` }), 200, { valid: true } + ); + }); + }); + }); describe("/api/upload-save", function() { let endpoint, testToken; beforeEach(function() { diff --git a/test/create.js b/test/create.js index a52d0304..f5b2590c 100644 --- a/test/create.js +++ b/test/create.js @@ -1,9 +1,15 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; +import crypto from "node:crypto"; +import events from "node:events"; import fs from "node:fs/promises"; +import http from "node:http"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; +import jwt from "jsonwebtoken"; + +import { escapeArg } from "../packages/create/escape_arg.js"; const execFileAsync = promisify(execFile); const installer = path.resolve(import.meta.dirname, "../packages/create/create.js"); @@ -123,4 +129,329 @@ describe("create installer CLI", function() { return true; }); }); + + describe("host setup", function() { + const scriptExt = process.platform === "win32" ? ".cmd" : ""; + const hostBin = path.join("node_modules", ".bin", `clusteriohost${scriptExt}`); + const controllerBin = path.join("node_modules", ".bin", `clusteriocontroller${scriptExt}`); + const hostSet = (field, value) => `${hostBin} config set ${field} ${escapeArg(String(value))}`; + const approved = { hostId: 1234, token: "host-token", name: "Approved" }; + let server; + let controllerUrl; + let requests; + let handler; + + beforeEach(async function() { + requests = []; + handler = () => [500, { error: "No handler" }]; + server = http.createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) { + body += chunk; + } + const entry = { url: req.url, headers: req.headers, body: body ? JSON.parse(body) : undefined }; + requests.push(entry); + const [status, json] = handler(entry); + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(json === undefined ? undefined : JSON.stringify(json)); + }); + server.listen(0, "localhost"); + await events.once(server, "listening"); + controllerUrl = `http://localhost:${server.address().port}/`; + }); + + afterEach(function() { + server.closeAllConnections(); + server.close(); + }); + + // Stub of the controller pairing endpoints, approving after the given number of pending polls. + function pairingHandler(pendingPolls = 0, result = approved) { + let polls = 0; + return ({ url }) => { + if (url === "/api/host-pairing") { + return [202, { requestId: "secret", code: "ABC-DEF" }]; + } + if (url === "/api/host-pairing/poll") { + polls += 1; + return polls > pendingPolls ? [200, result] : [202, { status: "pending" }]; + } + if (url === "/api/host-pairing/cancel") { + return [204, undefined]; + } + return [404, { error: "Not found" }]; + }; + } + + function run(args, env = {}) { + // Record the commands for npm and the clusterio CLIs instead of running them. + const mockExec = ` + import childProcess from "node:child_process"; + import fs from "node:fs"; + import { PassThrough } from "node:stream"; + childProcess.execFile = (command, options, callback) => { + fs.appendFileSync(process.env.CREATE_TEST_COMMANDS, JSON.stringify(command) + "\\n"); + const stdout = new PassThrough(); + const stderr = new PassThrough(); + // Lets tests give the host id a config show prints. + const output = process.env.CREATE_TEST_HOST_ID && command.includes("config show host.id") + ? process.env.CREATE_TEST_HOST_ID + "\\n" : ""; + queueMicrotask(() => { + stdout.end(); + stderr.end(); + callback(null, output, ""); + }); + return { stdout, stderr }; + }; + if (process.env.CREATE_TEST_SIGINT) { + // Stands in for Ctrl-C, which cannot be sent to a child process on Windows. + // Sent twice like when npm init forwards the one it gets to the installer. + const write = process.stdout.write.bind(process.stdout); + process.stdout.write = (chunk, ...rest) => { + if (String(chunk).includes("Waiting for approval")) { + setTimeout(() => { + process.emit("SIGINT", "SIGINT"); + process.emit("SIGINT", "SIGINT"); + }, 50); + } + return write(chunk, ...rest); + }; + } + `; + return execFileAsync(process.execPath, [ + "--import", `data:text/javascript,${encodeURIComponent(mockExec)}`, + installer, + ...(process.platform === "linux" ? ["--allow-install-as-root"] : []), + ...args, + ], { + cwd: directory, + env: { ...process.env, CREATE_TEST_COMMANDS: path.join(directory, "commands.jsonl"), ...env }, + }); + } + + // Commands run by the installer, without the getent lookups made when writing systemd units. + async function commands() { + let content; + try { + content = await fs.readFile(path.join(directory, "commands.jsonl"), "utf8"); + } catch (err) { + if (err.code === "ENOENT") { + return []; + } + throw err; + } + return content.split("\n").filter(Boolean).map(line => JSON.parse(line)) + .filter(command => !command.startsWith("getent ")); + } + + function hostArgs(...extra) { + return [ + "--mode", "host", "--host-name", "Test Host", "--controller-url", controllerUrl, + "--public-address", "host.example.com", "--factorio-dir", "factorio", "--no-remote-npm", + ...extra, "--plugins", + ]; + } + + async function writeHostConfig(token) { + await fs.writeFile(path.join(directory, "config-host.json"), JSON.stringify({ + "host.name": "Test Host", + "host.controller_url": controllerUrl, + "host.controller_token": token, + })); + } + + it("pairs a new host once approved and saves its id and token", async function() { + handler = pairingHandler(0); + const { stdout } = await run(hostArgs()); + assert(stdout.includes( + `Waiting for approval. Approve "Test Host" with code ABC-DEF on the Hosts page of ${controllerUrl}, ` + + "or run: npx clusterioctl host approve ABC-DEF" + ), stdout); + assert.match(stdout, /Successfully installed host/); + + const { version } = JSON.parse(await fs.readFile(path.join(path.dirname(installer), "package.json"))); + assert.deepEqual(requests[0].body, { name: "Test Host", version }); + assert.equal(requests[0].headers.authorization, undefined); + + const recorded = await commands(); + assert.match(recorded[0], /^npm(?:\.cmd)? install @clusterio\/host$/); + assert.deepEqual(recorded.slice(1), [ + `${hostBin} config create`, + hostSet("host.public_address", "host.example.com"), + hostSet("host.factorio_directory", "factorio"), + hostSet("host.allow_remote_updates", false), + hostSet("host.allow_plugin_updates", false), + hostSet("host.allow_plugin_install", false), + hostSet("host.name", "Test Host"), + hostSet("host.controller_url", controllerUrl), + hostSet("host.id", 1234), + hostSet("host.name", "Approved"), + hostSet("host.controller_token", "host-token"), + ]); + await fs.access(path.join(directory, `run-host${process.platform === "win32" ? ".cmd" : ".sh"}`)); + }); + + it("uses --controller-token to approve the host without saving it", async function() { + handler = ({ headers }) => (headers.authorization === "Bearer user-token" + ? [200, { hostId: 99, token: "host-token", name: "Test Host" }] + : [401, { error: "Invalid user token" }] + ); + const { stdout } = await run(hostArgs("--controller-token", "user-token")); + assert.doesNotMatch(stdout, /Waiting for approval/); + const recorded = await commands(); + assert(!recorded.some(command => command.includes("user-token")), "user token must not be saved"); + assert.deepEqual(recorded.slice(-2), [ + hostSet("host.id", 99), + hostSet("host.controller_token", "host-token"), + ]); + }); + + it("cancels the request on Ctrl-C", async function() { + handler = pairingHandler(Infinity); + await assert.rejects(run(hostArgs(), { CREATE_TEST_SIGINT: "1" }), err => { + assert.equal(err.code, 1); + assert.match(err.stdout + err.stderr, /Pairing cancelled/); + return true; + }); + assert.deepEqual(requests.find(r => r.url === "/api/host-pairing/cancel")?.body, { requestId: "secret" }); + assert(!(await commands()).some(command => command.includes("host.controller_token"))); + }); + + it("installs without contacting the controller with --no-pair", async function() { + const { stdout } = await run([ + "--mode", "host", "--no-pair", "--public-address", "localhost", "--factorio-dir", "factorio", + "--remote-npm", "--plugins", + ]); + assert.match(stdout, /Successfully installed host/); + assert.equal(requests.length, 0); + const recorded = await commands(); + assert(recorded.includes(hostSet("host.allow_plugin_install", true))); + assert(!recorded.some(command => command.includes("host.controller_token"))); + }); + + it("exits when run again on a host the controller accepts", async function() { + await writeHostConfig("valid-token"); + handler = ({ url, headers }) => { + const valid = url === "/api/host-pairing/check" && headers.authorization === "Bearer valid-token"; + return valid ? [200, { valid: true }] : [401, undefined]; + }; + const { stdout } = await run(["--mode", "host"]); + assert.match(stdout, /Host is already set up/); + assert.deepEqual(requests.map(r => r.url), ["/api/host-pairing/check"]); + assert.deepEqual(await commands(), []); + }); + + it("pairs again when run again on a revoked host", async function() { + await writeHostConfig("revoked-token"); + const pairing = pairingHandler(0, { hostId: 5, token: "new-token", name: "Renamed" }); + handler = entry => (entry.url === "/api/host-pairing/check" ? [401, undefined] : pairing(entry)); + const { stdout } = await run([ + "--mode", "host", "--host-name", "Renamed", "--controller-url", controllerUrl, + ]); + assert.match(stdout, /Successfully set up host/); + assert.equal(requests[0].headers.authorization, "Bearer revoked-token"); + assert.equal(requests[1].body.name, "Renamed"); + assert.deepEqual(await commands(), [ + hostSet("host.name", "Renamed"), + hostSet("host.controller_url", controllerUrl), + hostSet("host.id", 5), + hostSet("host.controller_token", "new-token"), + ]); + }); + + it("does not send the saved token to another controller", async function() { + await writeHostConfig("old-token"); + // Same stub server under its IP address, which the installer sees as another controller. + const { address, port } = server.address(); + const otherUrl = `http://${address.includes(":") ? `[${address}]` : address}:${port}/`; + handler = pairingHandler(0, { hostId: 6, token: "other-token", name: "Moved" }); + const { stdout } = await run(["--mode", "host", "--host-name", "Moved", "--controller-url", otherUrl]); + assert.match(stdout, /Successfully set up host/); + assert(!requests.some(r => r.url === "/api/host-pairing/check"), "token checked with other controller"); + assert(!requests.some(r => String(r.headers.authorization).includes("old-token"))); + assert.deepEqual(await commands(), [ + hostSet("host.name", "Moved"), + hostSet("host.controller_url", otherUrl), + hostSet("host.id", 6), + hostSet("host.controller_token", "other-token"), + ]); + }); + + it("adds the trailing slash the host requires to the controller URL", async function() { + handler = pairingHandler(0); + const args = hostArgs(); + args[args.indexOf(controllerUrl)] = controllerUrl.slice(0, -1); + await run(args); + assert((await commands()).includes(hostSet("host.controller_url", controllerUrl))); + }); + + it("writes the certificate into the run scripts when run again with one", async function() { + await writeHostConfig("valid-token"); + handler = ({ url }) => (url === "/api/host-pairing/check" ? [200, { valid: true }] : [404, undefined]); + const cert = path.resolve(import.meta.dirname, "file", "tls", "cert.pem"); + const { stdout } = await run(["--mode", "host", "--self-signed-cert", cert]); + assert.match(stdout, /Host is already set up/); + const script = process.platform === "win32" ? "run-host.cmd" : "run-host.sh"; + const content = await fs.readFile(path.join(directory, script), "utf8"); + assert(content.includes(`NODE_EXTRA_CA_CERTS=${cert}`), content); + }); + + it("adds the host of a standalone install to the controller database", async function() { + const secret = crypto.randomBytes(256).toString("base64"); + await fs.writeFile(path.join(directory, "config-controller.json"), JSON.stringify({ + "controller.auth_secret": secret, + })); + const { stdout } = await run([ + "--mode", "standalone", "--admin", "admin", "--http-port", "8123", "--public-address", "localhost", + "--factorio-dir", "factorio", "--remote-npm", "--plugins", + ]); + assert.match(stdout, /Successfully installed standalone/); + + const hosts = JSON.parse(await fs.readFile(path.join(directory, "database", "hosts.json"))); + assert.equal(hosts.length, 1); + assert.equal(hosts[0].name, "local"); + + const recorded = await commands(); + const tokenSet = `${hostBin} config set host.controller_token `; + const token = recorded.find(command => command.startsWith(tokenSet)).slice(tokenSet.length); + const payload = jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }); + assert.equal(payload.host, hosts[0].id); + for (const expected of [ + hostSet("host.id", hosts[0].id), + hostSet("host.name", "local"), + hostSet("host.controller_url", "http://localhost:8123/"), + `${controllerBin} config set controller.allow_remote_updates true`, + `${controllerBin} config set controller.allow_plugin_updates true`, + `${controllerBin} config set controller.allow_plugin_install true`, + hostSet("host.allow_remote_updates", true), + hostSet("host.allow_plugin_updates", true), + hostSet("host.allow_plugin_install", true), + ]) { + assert(recorded.includes(expected), `missing ${expected}`); + } + assert(!recorded.some(command => command.includes("generate-host-token"))); + }); + + it("keeps the host of a standalone install when run again", async function() { + const secret = crypto.randomBytes(256).toString("base64"); + await fs.writeFile(path.join(directory, "config-controller.json"), JSON.stringify({ + "controller.auth_secret": secret, + })); + await fs.mkdir(path.join(directory, "database")); + const existing = [{ id: 1234, name: "local", version: "2.0.3", plugins: {}, token_valid_after: 1 }]; + await fs.writeFile(path.join(directory, "database", "hosts.json"), JSON.stringify(existing)); + const { stdout } = await run([ + "--mode", "standalone", "--admin", "admin", "--http-port", "8123", "--public-address", "localhost", + "--factorio-dir", "factorio", "--remote-npm", "--plugins", + ], { CREATE_TEST_HOST_ID: "1234" }); + assert.match(stdout, /Successfully installed standalone/); + + assert.deepEqual(JSON.parse(await fs.readFile(path.join(directory, "database", "hosts.json"))), existing); + const recorded = await commands(); + assert(recorded.includes(hostSet("host.id", 1234)), "host id was changed"); + const tokenSet = `${hostBin} config set host.controller_token `; + const token = recorded.find(command => command.startsWith(tokenSet)).slice(tokenSet.length); + assert.equal(jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }).host, 1234); + }); + }); }); diff --git a/test/create/pair.js b/test/create/pair.js new file mode 100644 index 00000000..96bf54a0 --- /dev/null +++ b/test/create/pair.js @@ -0,0 +1,343 @@ +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import events from "node:events"; +import fs from "node:fs/promises"; +import http from "node:http"; +import https from "node:https"; +import os from "node:os"; +import path from "node:path"; +import tls from "node:tls"; +import jwt from "jsonwebtoken"; + +import * as lib from "@clusterio/lib"; +import HostRecord from "@clusterio/controller/dist/node/src/HostRecord.js"; +import { + PairingError, checkHostToken, createLocalHost, pairHost, +} from "../../packages/create/pair.js"; + +import "../setup_logging.js"; + +const tlsDir = path.join(import.meta.dirname, "..", "file", "tls"); + +// Stub controller recording requests, responding with what the test's handler returns. +async function startStub(createServer = http.createServer) { + const stub = { + requests: [], + handler: () => [500, { error: "No handler" }], + }; + stub.server = createServer(async (req, res) => { + const chunks = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const entry = { + method: req.method, + url: req.url, + headers: req.headers, + body: chunks.length ? JSON.parse(Buffer.concat(chunks)) : undefined, + }; + stub.requests.push(entry); + const [status, json, headers = {}] = await stub.handler(entry); + res.writeHead(status, { "Content-Type": "application/json", ...headers }); + res.end(json === undefined ? undefined : JSON.stringify(json)); + }); + stub.server.listen(0, "localhost"); + await events.once(stub.server, "listening"); + stub.url = `${stub.server instanceof https.Server ? "https" : "http"}://localhost:${stub.server.address().port}/`; + stub.close = async () => { + stub.server.closeAllConnections(); + stub.server.close(); + await events.once(stub.server, "close"); + }; + return stub; +} + +// Handler for a request that is approved after the given number of pending polls. +function approveAfter(polls, result = { hostId: 1234, token: "host-token", name: "Approved" }) { + let pending = 0; + return ({ url }) => { + if (url === "/api/host-pairing") { + return [202, { requestId: "secret", code: "ABC-DEF" }]; + } + if (url === "/api/host-pairing/poll") { + pending += 1; + return pending > polls ? [200, result] : [202, { status: "pending" }]; + } + return [404, { error: "Not found" }]; + }; +} + +describe("create/pair", function() { + let stub; + beforeEach(async function() { + stub = await startStub(); + }); + afterEach(async function() { + await stub.close(); + }); + + describe("pairHost()", function() { + it("returns the host when approved with a user token", async function() { + stub.handler = () => [200, { hostId: 5, token: "host-token", name: "Host" }]; + let pendingCalls = 0; + const result = await pairHost({ + controllerUrl: stub.url, name: "Host", version: "2.0.3", token: "user-token", + onPending: () => { pendingCalls += 1; }, + }); + assert.deepEqual(result, { hostId: 5, token: "host-token", name: "Host" }); + assert.equal(pendingCalls, 0); + assert.equal(stub.requests.length, 1); + const [created] = stub.requests; + assert.equal(created.method, "POST"); + assert.equal(created.url, "/api/host-pairing"); + assert.equal(created.headers.authorization, "Bearer user-token"); + assert.deepEqual(created.body, { name: "Host", version: "2.0.3" }); + }); + + it("polls until the request is approved", async function() { + stub.handler = approveAfter(2); + const codes = []; + const result = await pairHost({ + controllerUrl: stub.url, name: "Host", pollIntervalMs: 10, + onPending: ({ code }) => { codes.push(code); }, + }); + assert.deepEqual(result, { hostId: 1234, token: "host-token", name: "Approved" }); + assert.deepEqual(codes, ["ABC-DEF"]); + assert.deepEqual(stub.requests.map(r => r.url), [ + "/api/host-pairing", "/api/host-pairing/poll", "/api/host-pairing/poll", "/api/host-pairing/poll", + ]); + assert.equal(stub.requests[0].headers.authorization, undefined); + assert.deepEqual(stub.requests[0].body, { name: "Host", version: "" }); + for (const poll of stub.requests.slice(1)) { + assert.deepEqual(poll.body, { requestId: "secret" }); + } + }); + + ["sub", "sub/"].forEach(controllerPath => { + it(`resolves routes under a controller URL ending in ${controllerPath}`, async function() { + stub.handler = ({ url }) => (url === "/sub/api/host-pairing" + ? [200, { hostId: 5, token: "host-token", name: "Host" }] + : [404, {}] + ); + const result = await pairHost({ controllerUrl: `${stub.url}${controllerPath}`, name: "Host" }); + assert.equal(result.hostId, 5); + }); + }); + + it("throws the error from the controller when the request is rejected", async function() { + const handler = approveAfter(1); + stub.handler = entry => { + if (entry.url === "/api/host-pairing/poll" && stub.requests.length > 2) { + return [403, { error: "Request was rejected" }]; + } + return handler(entry); + }; + await assert.rejects( + pairHost({ controllerUrl: stub.url, name: "Host", pollIntervalMs: 10 }), + err => err instanceof PairingError && err.status === 403 && err.message === "Request was rejected", + ); + }); + + it("throws if the controller does not support pairing", async function() { + stub.handler = () => [404, undefined]; + await assert.rejects( + pairHost({ controllerUrl: stub.url, name: "Host" }), + err => err instanceof PairingError && err.status === 404 && /must be updated/.test(err.message), + ); + }); + + it("throws on an invalid user token", async function() { + stub.handler = () => [401, { error: "Invalid user token" }]; + await assert.rejects( + pairHost({ controllerUrl: stub.url, name: "Host", token: "bad", onPending: () => assert.fail() }), + err => err instanceof PairingError && err.status === 401 && err.message === "Invalid user token", + ); + assert.equal(stub.requests.length, 1); + }); + + it("includes the retry time when rate limited", async function() { + stub.handler = () => [429, { error: "Too many requests from this address" }, { "Retry-After": "42" }]; + await assert.rejects( + pairHost({ controllerUrl: stub.url, name: "Host" }), + err => err instanceof PairingError && err.status === 429 && /try again in 42 seconds/.test(err.message), + ); + }); + + it("throws if the request expired", async function() { + const handler = approveAfter(0); + stub.handler = entry => (entry.url === "/api/host-pairing/poll" + ? [404, { error: "Request not found" }] + : handler(entry) + ); + await assert.rejects( + pairHost({ controllerUrl: stub.url, name: "Host", pollIntervalMs: 10 }), + err => err instanceof PairingError && err.status === 404 && err.message === "Request not found", + ); + }); + + it("cancels the request when aborted", async function() { + const controller = new AbortController(); + stub.handler = entry => { + if (entry.url === "/api/host-pairing/poll") { + controller.abort(); + } + if (entry.url === "/api/host-pairing/cancel") { + return [204, undefined]; + } + return approveAfter(Infinity)(entry); + }; + await assert.rejects( + pairHost({ controllerUrl: stub.url, name: "Host", pollIntervalMs: 10, signal: controller.signal }), + { name: "AbortError" }, + ); + const cancel = stub.requests.find(r => r.url === "/api/host-pairing/cancel"); + assert.deepEqual(cancel?.body, { requestId: "secret" }); + }); + + it("does not send anything if already aborted", async function() { + await assert.rejects( + pairHost({ controllerUrl: stub.url, name: "Host", signal: AbortSignal.abort() }), + { name: "AbortError" }, + ); + assert.equal(stub.requests.length, 0); + }); + + it("throws on invalid controller URLs", async function() { + for (const controllerUrl of ["localhost:8080", "ftp://localhost/", "not a url"]) { + await assert.rejects(pairHost({ controllerUrl, name: "Host" }), PairingError); + } + }); + + it("trusts the certificate passed as ca", async function() { + const [key, cert] = await Promise.all([ + fs.readFile(path.join(tlsDir, "key.pem")), + fs.readFile(path.join(tlsDir, "cert.pem")), + ]); + const httpsStub = await startStub(handler => https.createServer({ key, cert }, handler)); + try { + httpsStub.handler = () => [200, { hostId: 5, token: "host-token", name: "Host" }]; + const result = await pairHost({ controllerUrl: httpsStub.url, name: "Host", ca: cert }); + assert.equal(result.hostId, 5); + + // Passing a different certificate replaces the trusted ones, proving ca is used. + await assert.rejects( + pairHost({ controllerUrl: httpsStub.url, name: "Host", ca: tls.rootCertificates[0] }), + err => typeof err.code === "string" && !(err instanceof PairingError), + ); + } finally { + await httpsStub.close(); + } + }); + }); + + describe("checkHostToken()", function() { + it("returns true if the controller accepts the token", async function() { + stub.handler = () => [200, { valid: true }]; + assert.equal(await checkHostToken({ controllerUrl: stub.url, token: "host-token" }), true); + assert.equal(stub.requests[0].method, "GET"); + assert.equal(stub.requests[0].url, "/api/host-pairing/check"); + assert.equal(stub.requests[0].headers.authorization, "Bearer host-token"); + }); + + it("returns false if the controller does not accept the token", async function() { + stub.handler = () => [401, undefined]; + assert.equal(await checkHostToken({ controllerUrl: stub.url, token: "host-token" }), false); + }); + + it("throws on other responses", async function() { + stub.handler = () => [503, { error: "Controller is in recovery mode" }]; + await assert.rejects( + checkHostToken({ controllerUrl: stub.url, token: "host-token" }), + err => err instanceof PairingError && err.status === 503, + ); + }); + }); +}); + +describe("create/pair createLocalHost()", function() { + let directory; + let secret; + beforeEach(async function() { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-pair-")); + secret = crypto.randomBytes(256).toString("base64"); + }); + afterEach(async function() { + await fs.rm(directory, { recursive: true, force: true, maxRetries: 10 }); + }); + + // Written with the controller's own config class to match its file format. + async function writeControllerConfig(fields = {}) { + const configPath = path.join(directory, "config-controller.json"); + const config = new lib.ControllerConfig("controller", undefined, configPath); + config.set("controller.auth_secret", secret); + for (const [name, value] of Object.entries(fields)) { + config.set(name, value); + } + await config.save(true); + } + + async function loadHosts(databaseDirectory = "database") { + const provider = new lib.JsonIdDatastoreProvider( + path.join(directory, databaseDirectory, "hosts.json"), HostRecord.fromJSON.bind(HostRecord), + ); + return await provider.load(); + } + + it("adds a host the controller loads with a token it accepts", async function() { + await writeControllerConfig(); + const before = Math.floor(Date.now() / 1000); + const result = await createLocalHost({ directory, name: "local" }); + assert.equal(result.name, "local"); + assert(Number.isInteger(result.hostId) && result.hostId >= 1 && result.hostId < 2 ** 31); + + const hosts = await loadHosts(); + assert.deepEqual([...hosts.keys()], [result.hostId]); + const record = hosts.get(result.hostId); + assert.equal(record.name, "local"); + assert.equal(record.version, ""); + assert.deepEqual(record.plugins, new Map()); + assert(record.tokenValidAfter >= before); + + const payload = jwt.verify(result.token, Buffer.from(secret, "base64"), { audience: "host" }); + assert.equal(payload.host, result.hostId); + assert(payload.iat >= record.tokenValidAfter, "token must not be issued before token_valid_after"); + }); + + it("appends to an existing database in the configured directory", async function() { + await writeControllerConfig({ "controller.database_directory": "db" }); + await fs.mkdir(path.join(directory, "db")); + await fs.writeFile(path.join(directory, "db", "hosts.json"), JSON.stringify([ + { id: 1, name: "existing", version: "2.0.0", plugins: {} }, + ])); + const result = await createLocalHost({ directory, name: "local", id: 7, version: "2.0.3" }); + assert.equal(result.hostId, 7); + + const hosts = await loadHosts("db"); + assert.deepEqual([...hosts.keys()], [1, 7]); + assert.equal(hosts.get(1).name, "existing"); + assert.equal(hosts.get(7).version, "2.0.3"); + await assert.rejects(fs.access(path.join(directory, "database")), { code: "ENOENT" }); + }); + + it("signs a new token for an existing host with the same id", async function() { + await writeControllerConfig(); + const first = await createLocalHost({ directory, name: "first", id: 7 }); + const before = await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"); + const second = await createLocalHost({ directory, name: "second", id: 7 }); + assert.equal(second.hostId, 7); + assert.equal(second.name, "first", "the name of the existing host should be kept"); + assert.equal(await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"), before); + + const record = (await loadHosts()).get(7); + for (const token of [first.token, second.token]) { + const payload = jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }); + assert.equal(payload.host, 7); + assert(payload.iat >= record.tokenValidAfter); + } + }); + + it("throws if the controller config has no auth secret", async function() { + await fs.writeFile(path.join(directory, "config-controller.json"), "{}"); + await assert.rejects(createLocalHost({ directory, name: "local" }), /controller\.auth_secret/); + }); +}); diff --git a/test/ctl/commands_host.test.js b/test/ctl/commands_host.test.js new file mode 100644 index 00000000..3390c59d --- /dev/null +++ b/test/ctl/commands_host.test.js @@ -0,0 +1,145 @@ +import assert from "node:assert/strict"; +import * as lib from "@clusterio/lib"; +import { hostCommands } from "@clusterio/ctl/dist/node/src/commands_host.js"; + +/** Stand-in for Ctl that answers requests from a map of handlers and records what was sent */ +class FakeCtl { + constructor(handlers) { + this.handlers = new Map(handlers); + this.sent = []; + } + + async send(request) { + return await this.sendTo("controller", request); + } + + async sendTo(dst, request) { + this.sent.push(request); + const handler = this.handlers.get(request.constructor); + if (!handler) { + throw new Error(`Unexpected ${request.constructor.name}`); + } + return handler(request); + } +} + +async function run(command, args, ctl) { + const lines = []; + const originalLog = console.log; // eslint-disable-line no-console + console.log = (...content) => lines.push(content.join(" ")); // eslint-disable-line no-console + try { + await hostCommands.get(command).run(args, ctl); + } finally { + console.log = originalLog; // eslint-disable-line no-console + } + return lines; +} + +describe("ctl/commands_host", function() { + // Created when the request is handled, the ages printed depend on the current time. + const pending = () => [ + new lib.HostPairingDetails("ABC-DEF", "new-box", "192.0.2.10", "2.0.3", Date.now() - 65500), + new lib.HostPairingDetails("XYZ-234", "other-box", "2001:db8::1", "", Date.now() - 3720500), + ]; + const hosts = [{ id: 7, name: "basement-box" }, { id: 12, name: "attic-box" }]; + + it("no longer has the token based commands", function() { + for (const name of ["generate-token", "revoke-token", "create-config"]) { + assert.equal(hostCommands.get(name), undefined, `host ${name} still exists`); + } + }); + + describe("host pending", function() { + it("should list pending requests with their age", async function() { + const ctl = new FakeCtl([[lib.HostPairingListRequest, pending]]); + const lines = await run("pending", {}, ctl); + const output = lines.join("\n"); + assert.match(output, /code\s*\|\s*name\s*\|\s*address\s*\|\s*age/); + assert.match(output, /ABC-DEF\s*\|\s*new-box\s*\|\s*192\.0\.2\.10\s*\|\s*1m 5s/); + assert.match(output, /XYZ-234\s*\|\s*other-box\s*\|\s*2001:db8::1\s*\|\s*1h 2m/); + }); + + it("should say so when nothing is pending", async function() { + const ctl = new FakeCtl([[lib.HostPairingListRequest, () => []]]); + assert.deepEqual(await run("pending", {}, ctl), ["No pending host requests"]); + }); + }); + + describe("host approve", function() { + it("should approve with the requested name and print it", async function() { + const ctl = new FakeCtl([ + [lib.HostPairingListRequest, pending], + [lib.HostPairingApproveRequest, () => 42], + ]); + const lines = await run("approve", { code: "abcdef" }, ctl); + const approve = ctl.sent.find(request => request instanceof lib.HostPairingApproveRequest); + assert.equal(approve.code, "abcdef"); + assert.equal(approve.name, undefined); + assert.deepEqual(lines, ["Approved host new-box with id 42"]); + }); + + it("should pass --name and print it without listing requests", async function() { + const ctl = new FakeCtl([[lib.HostPairingApproveRequest, () => 43]]); + const lines = await run("approve", { code: "ABC-DEF", name: " renamed " }, ctl); + assert.equal(ctl.sent.length, 1); + assert.equal(ctl.sent[0].code, "ABC-DEF"); + assert.equal(ctl.sent[0].name, " renamed "); + assert.deepEqual(lines, ["Approved host renamed with id 43"]); + }); + + it("should pass on errors from the controller", async function() { + const ctl = new FakeCtl([ + [lib.HostPairingListRequest, () => []], + [lib.HostPairingApproveRequest, () => { + throw new lib.NotFoundError("No pending host request with code AAA-AAA"); + }], + ]); + await assert.rejects( + run("approve", { code: "AAA-AAA" }, ctl), + new lib.NotFoundError("No pending host request with code AAA-AAA"), + ); + }); + }); + + describe("host reject", function() { + it("should send the code to the controller", async function() { + const ctl = new FakeCtl([[lib.HostPairingRejectRequest, () => undefined]]); + assert.deepEqual(await run("reject", { code: "abc-def" }, ctl), []); + assert.equal(ctl.sent.length, 1); + assert(ctl.sent[0] instanceof lib.HostPairingRejectRequest); + assert.equal(ctl.sent[0].code, "abc-def"); + }); + }); + + describe("host revoke", function() { + function revokeCtl() { + return new FakeCtl([ + [lib.HostListRequest, () => hosts], + [lib.HostRevokeRequest, () => undefined], + ]); + } + + it("should revoke a host given by name", async function() { + const ctl = revokeCtl(); + await run("revoke", { host: "basement-box" }, ctl); + const revoke = ctl.sent.at(-1); + assert(revoke instanceof lib.HostRevokeRequest); + assert.equal(revoke.hostId, 7); + }); + + it("should revoke a host given by id", async function() { + const ctl = revokeCtl(); + await run("revoke", { host: "12" }, ctl); + assert.equal(ctl.sent.at(-1).hostId, 12); + }); + + it("should fail for an unknown host", async function() { + const ctl = revokeCtl(); + await assert.rejects( + run("revoke", { host: "missing" }, ctl), + new lib.CommandError("No host named missing"), + ); + assert(!ctl.sent.some(request => request instanceof lib.HostRevokeRequest)); + }); + }); +}); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index afb554ef..b3b18d24 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -10,11 +10,12 @@ import * as libBuildMod from "@clusterio/lib/build_mod.js"; import * as testStrings from "../lib/factorio/test_strings.js"; import { - TestControl, TestControlConnector, url, controlToken, + TestControl, TestControlConnector, TestHostConnector, url, controlToken, hostId, execCtl, execCtlProcess, execController, execHost, sendRcon, getControl, spawnNode, instancesDir, factorioDir, databaseDir, controllerConfigPath, - requiresFactorio, hasFactorio, + requiresFactorio, hasFactorio, pairTestHost, writeHostConfig, } from "./index.js"; +import { pairHost, checkHostToken, PairingError } from "../../packages/create/pair.js"; import { externalTest, slowTest } from "../common.js"; import { getTestRconPort } from "./rcon_port.js"; import { ControllerEcho, HostEchoReceived, InstanceEcho } from "../file/test_plugin/messages.js"; @@ -31,6 +32,18 @@ async function checkInstanceStatus(id, status) { assert.equal(instances.get(44).status, status, "incorrect instance status"); } +// Factorio directory relative to temp/test where the hosts are run from +const hostFactorioDir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); + +/** Id, token and name of the alt-host, paired the first time it starts */ +const altHost = { hostId: undefined, token: undefined, name: undefined }; + +async function pairAltHost() { + if (altHost.hostId === undefined) { + Object.assign(altHost, await pairTestHost("alt-host")); + } +} + async function spawnAltHost(config = "alt-host-config.json") { return await spawnNode("alt-host:", `../../packages/host run --config ${config}`, /Started host/); } @@ -41,10 +54,10 @@ async function startAltHost() { await fs.rm(configPath, { force: true }); await fs.rm(path.join("temp", "test", "alt-instances"), { force: true, recursive: true, maxRetries: 10 }); await fs.rm(path.join("temp", "test", "alt-mods"), { force: true, recursive: true, maxRetries: 10 }); - await execCtl(`host create-config --id 5 --name alt-host --generate-token --output ${config}`); + await pairAltHost(); + await writeHostConfig(configPath, altHost); await execHost(`--config ${config} config set host.instances_directory alt-instances`); - const dir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); - await execHost(`--config ${config} config set host.factorio_directory ${dir}`); + await execHost(`--config ${config} config set host.factorio_directory ${hostFactorioDir}`); await execHost(`--config ${config} config set host.mods_directory alt-mods`); return await spawnAltHost(config); } @@ -57,7 +70,7 @@ async function stopAltHost(hostProcess) { "timeout" ).catch(() => {}); try { - await execCtl("host stop 5"); + await execCtl(`host stop ${altHost.hostId}`); } catch (err) { // Some tests cause the host to stop } @@ -105,6 +118,28 @@ function jsonArg(value) { return `"${JSON.stringify(value).replace(/"/g, process.platform === "win32" ? '""' : '\\"')}"`; } +/** Run a ctl command in this process and return what it printed */ +async function execCtlOutput(command) { + const lines = []; + const originalLog = console.log; // eslint-disable-line no-console + console.log = (...args) => lines.push(args.join(" ")); // eslint-disable-line no-console + try { + await execCtl(command); + } finally { + console.log = originalLog; // eslint-disable-line no-console + } + return lines.join("\n"); +} + +/** Validator for assert.rejects expecting a PairingError with the given HTTP status */ +function pairingError(status) { + return err => { + assert(err instanceof PairingError, `Expected a PairingError, got ${err.stack}`); + assert.equal(err.status, status, err.message); + return true; + }; +} + describe("Integration of Clusterio", function() { describe("clusteriocontroller", function() { describe("bootstrap generate-user-token", function() { @@ -281,11 +316,15 @@ describe("Integration of Clusterio", function() { describe("hostUpdateEventHandler()", function() { it("should trigger when a new host is added", async function() { slowTest(this); + // Pairing sends an update for the new host before it connects, which is not what is tested here. + // The round trip makes sure that update has been received before clearing the updates. + await pairAltHost(); + await getControl().send(new lib.HostListRequest()); getControl().hostUpdates = []; await runWithAltHost(async () => { await execCtl("instance create alt-test --id 99"); - await execCtl("instance assign alt-test 5"); + await execCtl(`instance assign alt-test ${altHost.hostId}`); }); let sawUpdate = false; @@ -311,7 +350,7 @@ describe("Integration of Clusterio", function() { let result = await getControl().send(new lib.HostListRequest()); let hosts = new Map(result.map(instance => [instance.id, instance])); - assert(hosts.has(5), "Host list was not updated"); + assert(hosts.has(altHost.hostId), "Host list was not updated"); }); }); it("should download mods from controller", async function() { @@ -325,7 +364,7 @@ describe("Integration of Clusterio", function() { } assert(!await checkModDownloaded(), "mod was downloaded before the test"); await execCtl("instance create alt-mod --id 98"); - await execCtl("instance assign alt-mod 5"); + await execCtl(`instance assign alt-mod ${altHost.hostId}`); await execCtl("instance save create alt-mod"); assert(await checkModDownloaded(), "mod was not downloaded by the test"); }); @@ -340,7 +379,7 @@ describe("Integration of Clusterio", function() { hostProcess = await startAltHost(); await execCtl("instance create alt-start --id 97"); await execCtl(`instance config set alt-start factorio.rcon_port ${await getTestRconPort()}`); - await execCtl("instance assign alt-start 5"); + await execCtl(`instance assign alt-start ${altHost.hostId}`); await execCtl("instance config set alt-start instance.auto_start true"); const visibility = jsonArg({ lan: false, public: false }); await execCtlProcess(`instance config set-prop alt-start factorio.settings visibility ${visibility}`); @@ -448,7 +487,7 @@ describe("Integration of Clusterio", function() { }); it("should load the plugin on the host", async function() { - const details = await pluginDetails({ hostId: 4 }); + const details = await pluginDetails({ hostId }); assert(details.enabled, "test_plugin is not enabled on the host"); assert(details.loaded, "test_plugin is not loaded on the host"); }); @@ -470,7 +509,7 @@ describe("Integration of Clusterio", function() { }); it("should route a plugin request from control to a host", async function() { - const received = await getControl().sendTo({ hostId: 4 }, new HostEchoReceived("never-broadcast")); + const received = await getControl().sendTo({ hostId }, new HostEchoReceived("never-broadcast")); assert.equal(received, false); }); @@ -480,7 +519,7 @@ describe("Integration of Clusterio", function() { // The broadcast is sent without waiting for the hosts to receive it. let received = false; for (let attempt = 0; attempt < 100 && !received; attempt++) { - received = await getControl().sendTo({ hostId: 4 }, new HostEchoReceived("to-hosts")); + received = await getControl().sendTo({ hostId }, new HostEchoReceived("to-hosts")); if (!received) { await lib.wait(20); } @@ -593,31 +632,31 @@ describe("Integration of Clusterio", function() { describe("host config", function() { it("changes host config", async function() { - await execCtl("host config set 4 host.name My-Host"); - const result = await execCtlProcess("host config list 4"); + await execCtl(`host config set ${hostId} host.name My-Host`); + const result = await execCtlProcess(`host config list ${hostId}`); assert(/My-Host/.test(result.stdout), "New host name not in config output"); }); it("should not allow setting host.controller_token", async function() { await assert.rejects( - execCtl("host config set 4 host.controller_token xyz"), + execCtl(`host config set ${hostId} host.controller_token xyz`), /Field 'host\.controller_token' is not accessible/, ); }); it("should not allow setting host.id", async function() { await assert.rejects( - execCtl("host config set 4 host.id 200"), + execCtl(`host config set ${hostId} host.id 200`), /Setting 'host\.id' while host is running is not supported/, ); }); it("should not leak host.controller_token", async function() { - let result = await getControl().sendTo({ hostId: 4 }, new lib.HostConfigGetRequest()); + let result = await getControl().sendTo({ hostId }, new lib.HostConfigGetRequest()); assert.equal(Object.prototype.hasOwnProperty.call(result, "host.controller_token"), false); }); }); describe("host plugin list", function() { it("runs", async function() { - await execCtl("host plugin list 4"); + await execCtl(`host plugin list ${hostId}`); }); }); @@ -625,7 +664,7 @@ describe("Integration of Clusterio", function() { it("runs", async function() { // In dev plugins have no npm package, so best we can do is get an error from the host await assert.rejects( - execCtl("host plugin update 4 foo"), + execCtl(`host plugin update ${hostId} foo`), /Plugin foo is not installed on this machine/ ); }); @@ -637,7 +676,7 @@ describe("Integration of Clusterio", function() { // Default is to disallow updates, changing this value would require a restart // Additionally, it can not be changed via ctl, so best we can do is get an error from the host await assert.rejects( - execCtl("host plugin install 4 foo"), + execCtl(`host plugin install ${hostId} foo`), /Plugin installs are disabled on this machine/ ); }); @@ -646,15 +685,15 @@ describe("Integration of Clusterio", function() { describe("host update", function() { it("runs", async function() { - await execCtl("host update 4"); + await execCtl(`host update ${hostId}`); }); it("accepts --all", async function() { - await execCtl("host update 4 --all"); + await execCtl(`host update ${hostId} --all`); }); it("accepts --restart", async function() { // We cannot restart the host, so we check for host error instead await assert.rejects( - execCtl("host update 4 --restart"), + execCtl(`host update ${hostId} --restart`), /Cannot restart, host does not have a process monitor to restart it./ ); }); @@ -664,7 +703,7 @@ describe("Integration of Clusterio", function() { it("runs", async function() { // We cannot restart the host, so we check for host error instead await assert.rejects( - execCtl("host restart 4"), + execCtl(`host restart ${hostId}`), /Cannot restart, host does not have a process monitor to restart it./ ); }); @@ -672,41 +711,216 @@ describe("Integration of Clusterio", function() { describe("host mark-restart-required", function() { it("runs", async function() { - await execCtl("host mark-restart-required 4"); + await execCtl(`host mark-restart-required ${hostId}`); }); }); - describe("host generate-token", function() { - it("runs", async function() { - await execCtl("host generate-token --id 42"); + describe("host pending, approve and reject", function() { + let pairings = []; + afterEach(async function() { + // Stop installers left waiting by a failed test + for (const pairing of pairings) { + pairing.abort.abort(); + } + await Promise.allSettled(pairings.map(pairing => pairing.result)); + pairings = []; }); - it("runs without an id", async function() { - await execCtl("host generate-token"); - }); - }); - describe("host revoke-token", async function() { - it("should disconnect existing host", async function() { - slowTest(this); - let sawDisconnected; - await runWithAltHost(async () => { - const control = getControl(); - control.hostUpdates = []; - await wait(50); // Allow for host to update instances - await execCtl("host revoke-token 5"); - await wait(50); // Allow controller to notify control - for (let update of control.hostUpdates) { - if (update.name !== "alt-host") { - continue; - } - - if (!update.connected) { - sawDisconnected = true; - } - } + // Start pairing like an installer without a token, returns once the request is pending. + async function requestPairing(name) { + const abort = new AbortController(); + let onPending; + const pending = new Promise(resolve => { onPending = resolve; }); + const result = pairHost({ + controllerUrl: url, name, pollIntervalMs: 100, signal: abort.signal, onPending, }); + const pairing = { abort, result, code: undefined }; + pairings.push(pairing); + const ended = result.then(() => { + throw new Error("Pairing finished before the request was pending"); + }); + ended.catch(() => {}); // Only of interest while racing the pending request + ({ code: pairing.code } = await Promise.race([pending, ended])); + return pairing; + } - assert(sawDisconnected, "No host update with status disconnected was sent after revoking token"); + it("should list a waiting request and hand the host over when approved", async function() { + const control = getControl(); + control.hostPairingUpdates = []; + const pairing = await requestPairing("pending-host"); + assert.match(pairing.code, /^[2-9A-HJKMNP-Z]{3}-[2-9A-HJKMNP-Z]{3}$/); + + const pendingOutput = await execCtlOutput("host pending"); + const pendingLines = pendingOutput.split("\n"); + assert( + pendingLines.some(line => line.includes(pairing.code) && line.includes("pending-host")), + `Request missing from host pending output:\n${pendingOutput}`, + ); + assert( + control.hostPairingUpdates.some(update => update.code === pairing.code && !update.isDeleted), + "No pairing update was sent for the new request", + ); + + // Codes are matched ignoring case and separators + const typedCode = pairing.code.replace("-", "").toLowerCase(); + const approveOutput = await execCtlOutput(`host approve ${typedCode} --name approved-host`); + const approved = await pairing.result; + assert.equal(approved.name, "approved-host"); + assert(approveOutput.includes(String(approved.hostId)), `Host id missing from: ${approveOutput}`); + assert( + control.hostPairingUpdates.some(update => update.code === pairing.code && update.isDeleted), + "No pairing update was sent for the approved request", + ); + + const requests = await control.send(new lib.HostPairingListRequest()); + assert(!requests.some(request => request.code === pairing.code), "Approved request is still pending"); + const hosts = await control.send(new lib.HostListRequest()); + assert.equal(hosts.find(host => host.id === approved.hostId)?.name, "approved-host"); + assert.equal(await checkHostToken({ controllerUrl: url, token: approved.token }), true); + + await execCtl(`host revoke ${approved.hostId}`); + }); + + it("should make the installer fail when the request is rejected", async function() { + const pairing = await requestPairing("rejected-host"); + await execCtl(`host reject ${pairing.code}`); + await assert.rejects(pairing.result, pairingError(403)); + + const requests = await getControl().send(new lib.HostPairingListRequest()); + assert(!requests.some(request => request.code === pairing.code), "Rejected request is still pending"); + const hosts = await getControl().send(new lib.HostListRequest()); + assert(!hosts.some(host => host.name === "rejected-host"), "Rejected host was added"); + }); + + it("should remove the request when the installer stops waiting", async function() { + const pairing = await requestPairing("cancelled-host"); + pairing.abort.abort(); + await assert.rejects(pairing.result, err => err.name === "AbortError"); + + const requests = await getControl().send(new lib.HostPairingListRequest()); + assert(!requests.some(request => request.code === pairing.code), "Cancelled request is still pending"); + }); + + it("should fail for a code that is not pending", async function() { + await assert.rejects(execCtl("host approve 222-222"), /No pending host request with code 222-222/); + await assert.rejects(execCtl("host reject 222-222"), /No pending host request with code 222-222/); + }); + + it("should refuse an invalid user token instead of waiting for approval", async function() { + await assert.rejects( + pairHost({ controllerUrl: url, name: "invalid-token-host", token: "invalid" }), pairingError(401) + ); + const requests = await getControl().send(new lib.HostPairingListRequest()); + assert(!requests.some(request => request.name === "invalid-token-host"), "Request was created"); + }); + + it("should refuse a user token without permission to approve hosts", async function() { + await getControl().send(new lib.UserCreateRequest("pairing-player")); + try { + const token = jwt.sign( + { aud: "user", user: "pairing-player" }, Buffer.from("TestSecretDoNotUse", "base64") + ); + await assert.rejects( + pairHost({ controllerUrl: url, name: "unapproved-host", token }), pairingError(403) + ); + } finally { + await getControl().send(new lib.UserDeleteRequest("pairing-player")); + } + const requests = await getControl().send(new lib.HostPairingListRequest()); + assert(!requests.some(request => request.name === "unapproved-host"), "Request was created"); + const hosts = await getControl().send(new lib.HostListRequest()); + assert(!hosts.some(host => host.name === "unapproved-host"), "Host was added"); + }); + }); + + describe("host revoke", function() { + it("should remove the host and invalidate its token", async function() { + const pairing = await pairTestHost("revokee-host"); + assert.equal(await checkHostToken({ controllerUrl: url, token: pairing.token }), true); + await execCtl(`note set host ${pairing.hostId} short-lived`); + + const control = getControl(); + control.hostUpdates = []; + await execCtl(`host revoke ${pairing.hostId}`); + assert( + control.hostUpdates.some(update => update.id === pairing.hostId && update.isDeleted), + "No host update with isDeleted was sent", + ); + const hosts = await control.send(new lib.HostListRequest()); + assert(!hosts.some(host => host.id === pairing.hostId), "Host was not removed"); + assert.equal(await checkHostToken({ controllerUrl: url, token: pairing.token }), false); + const notes = await control.send(new lib.NoteListRequest()); + assert(!notes.some(note => note.id === `host/${pairing.hostId}`), "Note of the host was not removed"); + }); + + it("should refuse while instances are assigned to the host", async function() { + const pairing = await pairTestHost("assigned-host"); + // Instances can only be assigned to connected hosts, a link that accepts the assignment is enough. + const connector = new TestHostConnector(url, 0.2); + connector.token = pairing.token; + connector.hostId = pairing.hostId; + const link = new lib.Link(connector); + link.handle(lib.InstanceAssignInternalRequest, async () => {}); + await connector.connect(); + await execCtl("instance create revoke-assigned --id 45"); + try { + await execCtl(`instance assign revoke-assigned ${pairing.hostId}`); + await assert.rejects( + execCtl(`host revoke ${pairing.hostId}`), + /Cannot revoke host assigned-host while instances are assigned to it: revoke-assigned/, + ); + } finally { + await connector.disconnect(); + await execCtl("instance assign revoke-assigned"); + await execCtl("instance delete revoke-assigned"); + } + + await execCtl(`host revoke ${pairing.hostId}`); + const hosts = await getControl().send(new lib.HostListRequest()); + assert(!hosts.some(host => host.id === pairing.hostId), "Host was not removed"); + }); + + it("should stop a running host and prevent it from starting again", async function() { + slowTest(this); + this.timeout(30000); + const config = "revoke-host-config.json"; + const pairing = await pairTestHost("running-revokee"); + await writeHostConfig(path.join("temp", "test", config), pairing, { + "host.factorio_directory": hostFactorioDir, + "host.instances_directory": "revoke-instances", + "host.mods_directory": "revoke-mods", + }); + const hostProcess = await spawnNode( + "revoke-host:", `../../packages/host run --config ${config}`, /Started host/ + ); + try { + const exited = events.once(hostProcess, "exit"); + await execCtl(`host revoke ${pairing.hostId}`); + assert.notEqual( + await lib.timeout(exited, 10e3, "timeout"), "timeout", "Host did not stop after being revoked" + ); + } finally { + if (hostProcess.exitCode === null && hostProcess.signalCode === null) { + hostProcess.kill("SIGINT"); + } + } + + // Exit code 8 tells the run scripts not to restart the host + await assert.rejects(execHost(`--config ${config} run`), err => { + const output = `${err.stdout}${err.stderr}`; + assert.equal(err.code, 8, `Unexpected exit code ${err.code}:\n${output}`); + assert.match(output, /host is not registered/); + return true; + }); + }); + }); + + describe("removed host token commands", function() { + it("should no longer exist", async function() { + slowTest(this); + for (const command of ["generate-token", "revoke-token", "create-config"]) { + await assert.rejects(execCtlProcess(`host ${command}`), new RegExp(`Unknown argument: ${command}`)); + } }); }); @@ -751,7 +965,7 @@ describe("Integration of Clusterio", function() { describe("instance assign", function() { it("creates the instance files", async function() { - await execCtl("instance assign test 4"); + await execCtl(`instance assign test ${hostId}`); await assert.doesNotReject( fs.access(path.join(instancesDir, "test")), "Instance directory was not created" @@ -1046,14 +1260,14 @@ describe("Integration of Clusterio", function() { assert.equal(controllerEntry.host_id, undefined); assert.equal(controllerEntry.instance_id, undefined); - const hostEntry = await findLog({ hostIds: [4] }, "test_plugin host echo log-tagging"); + const hostEntry = await findLog({ hostIds: [hostId] }, "test_plugin host echo log-tagging"); assert.equal(hostEntry.plugin, "test_plugin"); - assert.equal(hostEntry.host_id, 4); + assert.equal(hostEntry.host_id, hostId); assert.equal(hostEntry.instance_id, undefined); const instanceEntry = await findLog({ instanceIds: [44] }, "test_plugin instance echo log-tagging"); assert.equal(instanceEntry.plugin, "test_plugin"); - assert.equal(instanceEntry.host_id, 4); + assert.equal(instanceEntry.host_id, hostId); assert.equal(instanceEntry.instance_id, 44); }); }); @@ -1523,7 +1737,7 @@ describe("Integration of Clusterio", function() { before(async function() { slowTest(this); await execCtlProcess("instance create spam --id 66"); - await execCtlProcess("instance assign spam 4"); + await execCtlProcess(`instance assign spam ${hostId}`); await execCtlProcess("instance create unassign --id 77"); }); after(async function() { @@ -1550,7 +1764,7 @@ describe("Integration of Clusterio", function() { hostProcess = await startAltHost(); await execCtlProcess("instance create save-test --id 88"); - await execCtlProcess("instance assign save-test 5"); + await execCtlProcess(`instance assign save-test ${altHost.hostId}`); }); after(async function() { await stopAltHost(hostProcess); @@ -1978,9 +2192,9 @@ describe("Integration of Clusterio", function() { describe("note set", function() { it("should set the note of the given resource", async function() { - await execCtl("note set host 4 basement-box"); + await execCtl(`note set host ${hostId} basement-box`); const notes = await getControl().send(new lib.NoteListRequest()); - const note = notes.find(n => n.id === "host/4"); + const note = notes.find(n => n.id === `host/${hostId}`); assert.equal(note.content, "basement-box"); assert.equal(note.updatedBy, "test"); }); @@ -2001,7 +2215,7 @@ describe("Integration of Clusterio", function() { const originalLog = console.log; // eslint-disable-line no-console console.log = (...args) => lines.push(args.join(" ")); // eslint-disable-line no-console try { - await execCtl("note get host 4"); + await execCtl(`note get host ${hostId}`); } finally { console.log = originalLog; // eslint-disable-line no-console } @@ -2017,9 +2231,9 @@ describe("Integration of Clusterio", function() { describe("note delete", function() { it("should remove the note of the given resource", async function() { - await execCtl("note delete host 4"); + await execCtl(`note delete host ${hostId}`); const notes = await getControl().send(new lib.NoteListRequest()); - assert(!notes.find(n => n.id === "host/4"), "Note was not deleted"); + assert(!notes.find(n => n.id === `host/${hostId}`), "Note was not deleted"); }); }); diff --git a/test/integration/index.js b/test/integration/index.js index ccb977e7..d6aa7943 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -12,6 +12,7 @@ import { LineSplitter, logger } from "@clusterio/lib"; import { selectTargetCommand, initialize as initializeCtl } from "@clusterio/ctl"; import { _listFactorioVersions } from "@clusterio/host/dist/node/src/server.js"; +import { pairHost } from "../../packages/create/pair.js"; import "../setup_logging.js"; @@ -26,6 +27,7 @@ export class TestControl extends lib.Link { constructor(connector, subscribe = true) { super(connector); this.hostUpdates = []; + this.hostPairingUpdates = []; this.instanceUpdates = []; this.saveUpdates = []; this.modUpdates = []; @@ -39,6 +41,9 @@ export class TestControl extends lib.Link { this.send( new lib.SubscriptionRequest(lib.HostUpdatesEvent.name, true) ).catch(err => logger.error(`Error setting host subscriptions:\n${err.stack}`)); + this.send( + new lib.SubscriptionRequest(lib.HostPairingUpdatesEvent.name, true) + ).catch(err => logger.error(`Error setting host pairing subscriptions:\n${err.stack}`)); this.send( new lib.SubscriptionRequest(lib.InstanceDetailsUpdatesEvent.name, true) ).catch(err => logger.error(`Error setting instance subscriptions:\n${err.stack}`)); @@ -58,6 +63,7 @@ export class TestControl extends lib.Link { this.handle(lib.AccountUpdateEvent); this.handle(lib.HostUpdatesEvent, this.handleHostUpdatesEvent.bind(this)); + this.handle(lib.HostPairingUpdatesEvent, this.handleHostPairingUpdatesEvent.bind(this)); this.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsUpdatesEvent.bind(this)); this.handle(lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsUpdatesEvent.bind(this)); this.handle(lib.ModUpdatesEvent, this.handleModUpdatesEvent.bind(this)); @@ -69,6 +75,10 @@ export class TestControl extends lib.Link { this.hostUpdates.push(...event.updates); } + async handleHostPairingUpdatesEvent(event) { + this.hostPairingUpdates.push(...event.updates); + } + async handleInstanceDetailsUpdatesEvent(event) { this.instanceUpdates.push(...event.updates); } @@ -181,6 +191,8 @@ const baseHostConfig = loadJSON("config-host.json"); export let url = "https://localhost:4443/"; // Signed in the root before hook, as the user has to exist before the token is issued export let controlToken; +// Id of the test host, assigned by the controller when it is paired in the root before hook +export let hostId; export let instancesDir = path.join("temp", "test", "instances"); export let modsDir = path.join("temp", "test", "mods"); export let databaseDir = path.join("temp", "test", "database"); @@ -253,6 +265,35 @@ export function getControl() { return control; } +/** + * Pair a new host with the test controller, approved right away with the admin token + * + * @param {string} name - Name to give the host. + * @returns {Promise<{ hostId: number, token: string, name: string }>} the paired host. + */ +export async function pairTestHost(name) { + return await pairHost({ controllerUrl: url, name, token: controlToken }); +} + +/** + * Write a new host config for a paired host, with the fields the installer sets + * + * @param {string} configPath - Path to write the config to. + * @param {{ hostId: number, token: string, name: string }} pairing - Host returned by pairing. + * @param {Record} fields - Other fields to set in the config. + */ +export async function writeHostConfig(configPath, pairing, fields = {}) { + const hostConfig = new lib.HostConfig("host"); + hostConfig.set("host.id", pairing.hostId); + hostConfig.set("host.name", pairing.name); + hostConfig.set("host.controller_url", url); + hostConfig.set("host.controller_token", pairing.token); + for (const [name, value] of Object.entries(fields)) { + hostConfig.set(name, value); + } + await fs.writeFile(configPath, JSON.stringify(hostConfig, null, "\t"), { mode: 0o600 }); +} + /** @returns {Promise} */ export function spawn(name, cmd, waitFor, options = {}) { @@ -366,7 +407,9 @@ before(async function() { haveFactorioInstall = factorioVersions.size > 0; const relativeFactorioDir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); - await execCtlProcess("host create-config --id 4 --name host --generate-token"); + const pairing = await pairTestHost("host"); + hostId = pairing.hostId; + await writeHostConfig(hostConfigPath, pairing); await execHost("config create"); await execHost(`config set host.factorio_directory ${relativeFactorioDir}`); diff --git a/test/integration/instance.js b/test/integration/instance.js index 25bdf4f0..4220159c 100644 --- a/test/integration/instance.js +++ b/test/integration/instance.js @@ -8,7 +8,7 @@ import fs from "node:fs/promises"; import { slowTest, testMatrix } from "../common.js"; import { execCtl, execCtlProcess, sendRcon, getControl, - requiresFactorio, hasFactorio, getFactorioVersion, instancesDir, + requiresFactorio, hasFactorio, getFactorioVersion, instancesDir, hostId, } from "./index.js"; import { getTestRconPort } from "./rcon_port.js"; @@ -96,7 +96,7 @@ describe("Clusterio Instance", function() { await execCtl(`instance config set ${instName} factorio.enable_whitelist true`); await execCtlProcess(`instance config set-prop ${instName} factorio.settings visibility "${visibility}"`); await execCtl(`instance config set-prop ${instName} factorio.settings require_user_verification false`); - await execCtl(`instance assign ${instName} 4`); + await execCtl(`instance assign ${instName} ${hostId}`); // Exclude main test instance from start-all to prevent interference await execCtl(`instance config set ${instName} instance.exclude_from_start_all true`); @@ -112,7 +112,7 @@ describe("Clusterio Instance", function() { await execCtl(`instance config set ${instAltName} factorio.enable_whitelist true`); await execCtlProcess(`instance config set-prop ${instAltName} factorio.settings visibility "${visibility}"`); await execCtl(`instance config set-prop ${instAltName} factorio.settings require_user_verification false`); - await execCtl(`instance assign ${instAltName} 4`); + await execCtl(`instance assign ${instAltName} ${hostId}`); // Exclude alt test instance from start-all to prevent interference await execCtl(`instance config set ${instAltName} instance.exclude_from_start_all true`); await execCtl(`instance start ${instAltName}`); @@ -340,9 +340,9 @@ describe("start-all and stop-all commands", function() { await execCtl(`instance create ${testInstName3} --id ${testInstId3}`); // Configure instances - await execCtl(`instance assign ${testInstName1} 4`); - await execCtl(`instance assign ${testInstName2} 4`); - await execCtl(`instance assign ${testInstName3} 4`); + await execCtl(`instance assign ${testInstName1} ${hostId}`); + await execCtl(`instance assign ${testInstName2} ${hostId}`); + await execCtl(`instance assign ${testInstName3} ${hostId}`); const visibility = JSON.stringify({ lan: false, public: false }).replace( /"/g, process.platform === "win32" ? '""' : '\\"' ); diff --git a/test/integration/link.js b/test/integration/link.js index 01beb471..0580af8a 100644 --- a/test/integration/link.js +++ b/test/integration/link.js @@ -3,25 +3,36 @@ import events from "node:events"; import jwt from "jsonwebtoken"; import { - TestControlConnector, TestHostConnector, TestControl, url, controlToken, getControl, + TestControlConnector, TestHostConnector, TestControl, url, controlToken, getControl, hostId, pairTestHost, } from "./index.js"; import * as lib from "@clusterio/lib"; import { ConnectionClosed, ProtocolError, PolicyViolation, AuthenticationFailed } from "@clusterio/lib"; import { slowTest } from "../common.js"; -let tokenHost = jwt.sign({ aud: "host", host: 0 }, Buffer.from("TestSecretDoNotUse", "base64")); +// Valid token for a host id that was never approved +let tokenUnregistered = jwt.sign({ aud: "host", host: 0 }, Buffer.from("TestSecretDoNotUse", "base64")); describe("Integration of lib/link", function() { let control; let controlConnector; let hostConnector; + // Hosts can only connect once approved, so the host connections use a paired host + let linkHost; + before(async function() { + linkHost = await pairTestHost("link-test"); + }); + after(async function() { + if (linkHost) { + await getControl().send(new lib.HostRevokeRequest(linkHost.hostId)); + } + }); beforeEach(async function() { controlConnector = new TestControlConnector(url, 0.2); controlConnector.token = controlToken; control = new TestControl(controlConnector, false); hostConnector = new TestHostConnector(url, 0.2); - hostConnector.token = tokenHost; - hostConnector.hostId = 0; + hostConnector.token = linkHost.token; + hostConnector.hostId = linkHost.hostId; }); afterEach(async function() { await controlConnector.disconnect(); @@ -138,13 +149,24 @@ describe("Integration of lib/link", function() { }); it("should refuse connection with mismatched ids", async function() { - hostConnector.hostId = 1; + hostConnector.hostId = linkHost.hostId + 1; await assert.rejects( hostConnector.connect(), new Error("Authentication failed: missmatched host id") ); }); + it("should refuse connection for a host that is not registered", async function() { + hostConnector.token = tokenUnregistered; + hostConnector.hostId = 0; + await assert.rejects( + hostConnector.connect(), + new Error( + "Authentication failed: host is not registered, run the installer again in the host directory" + ) + ); + }); + it("should close old connections from the same address", async function() { slowTest(this); const hostConnectorFirst = new TestHostConnector(url, 0.2); @@ -163,8 +185,8 @@ describe("Integration of lib/link", function() { it("should refuse duplicate host connections", async function() { this.skip(); // Can't be checked because the remote address for both connections is the same - hostConnector.token = jwt.sign({ aud: "host", host: 4 }, Buffer.from("TestSecretDoNotUse", "base64")); - hostConnector.hostId = 4; // Same as already running host + hostConnector.token = jwt.sign({ aud: "host", host: hostId }, Buffer.from("TestSecretDoNotUse", "base64")); + hostConnector.hostId = hostId; // Same as already running host await assert.rejects( hostConnector.connect(), new Error("Authentication failed: missmatched host id") diff --git a/test/integration/routing.js b/test/integration/routing.js index 6a0103bd..19c73dfa 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import * as lib from "@clusterio/lib"; import { Host, Instance, InstanceConnection } from "@clusterio/host"; -import { ControlConnection, Controller, HostConnection, InstanceRecord } from "@clusterio/controller"; +import { ControlConnection, Controller, HostConnection, HostRecord, InstanceRecord } from "@clusterio/controller"; import { MockServer } from "../mock.js"; // Although there are no imports, the file has side-effects such as "before" which can effect tests @@ -19,6 +19,8 @@ class Control extends lib.Link { } function connectHost(controller, hostId, plugins) { const [controllerSide, hostSide] = lib.VirtualConnector.makePair(addr("controller"), addr({ hostId })); const registerData = new lib.RegisterHostData("", "0.0.0", hostId, plugins); + // Hosts can only connect once approved, which creates their record. + controller.hosts.set(new HostRecord(hostId, `host-${hostId}`, "", new Map())); const hostConnection = new HostConnection(registerData, controllerSide, controller, "host-a.test"); controller.wsServer.hostConnections.set(hostId, hostConnection); const hostConfig = new lib.HostConfig("host"); diff --git a/test/lib/data/HostPairingDetails.test.js b/test/lib/data/HostPairingDetails.test.js new file mode 100644 index 00000000..fa0b31be --- /dev/null +++ b/test/lib/data/HostPairingDetails.test.js @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; +import { HostPairingDetails } from "@clusterio/lib"; + +import { testMatrix, testRoundTripJsonSerialisable } from "../../common.js"; + +describe("lib/data/HostPairingDetails", function () { + + it("should be round trip serialisable", function () { + testRoundTripJsonSerialisable(HostPairingDetails, testMatrix( + ["ABC-DEF"], // code (required) + ["New Host", ""], // name (required) + ["10.0.0.1", "2001:db8::1"], // remoteAddress (required) + [undefined, "2.0.0"], // version + [undefined, 123456], // createdAtMs + [undefined, 999999], // updatedAtMs + [undefined, true], // isDeleted + )); + }); + + describe("constructor", function () { + it("should set defaults correctly", function () { + const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1"); + + assert.equal(details.code, "ABC-DEF"); + assert.equal(details.name, "New Host"); + assert.equal(details.remoteAddress, "10.0.0.1"); + assert.equal(details.version, ""); + assert.equal(details.createdAtMs, 0); + assert.equal(details.updatedAtMs, 0); + assert.equal(details.isDeleted, false); + }); + + it("should use the code as the id", function () { + const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1"); + assert.equal(details.id, "ABC-DEF"); + }); + }); + + describe(".toJSON()", function () { + it("should only include required fields when optional fields are defaults", function () { + const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1", "", 0, 0, false); + + assert.deepEqual(details.toJSON(), { + code: "ABC-DEF", + name: "New Host", + remoteAddress: "10.0.0.1", + }); + }); + + it("should include all optional fields when set", function () { + const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1", "2.0.0", 100, 200, true); + + assert.deepEqual(details.toJSON(), { + code: "ABC-DEF", + name: "New Host", + remoteAddress: "10.0.0.1", + version: "2.0.0", + createdAtMs: 100, + updatedAtMs: 200, + isDeleted: true, + }); + }); + + it("should not include the id getter", function () { + const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1"); + assert.equal(Object.hasOwn(JSON.parse(JSON.stringify(details)), "id"), false); + }); + }); + + describe("static fromJSON()", function () { + it("should construct minimal object", function () { + const details = HostPairingDetails.fromJSON({ + code: "ABC-DEF", + name: "New Host", + remoteAddress: "10.0.0.1", + }); + + assert.equal(details.code, "ABC-DEF"); + assert.equal(details.version, ""); + assert.equal(details.createdAtMs, 0); + assert.equal(details.updatedAtMs, 0); + assert.equal(details.isDeleted, false); + }); + }); +}); diff --git a/test/messages/controller.test.js b/test/messages/controller.test.js index 5a3a3ab3..4bf3e9ab 100644 --- a/test/messages/controller.test.js +++ b/test/messages/controller.test.js @@ -186,6 +186,83 @@ describe("messages/controller", function() { }); }); + describe("Host pairing", function() { + const details = [ + new lib.HostPairingDetails("ABC-DEF", "Host A", "10.0.0.1", "2.0.0", 100, 200), + new lib.HostPairingDetails("GHJ-KMN", "Host B", "2001:db8::1", "", 0, 300, true), + ]; + + it("defines the pairing messages for control links approving hosts", function() { + for (const Message of [ + lib.HostPairingListRequest, + lib.HostPairingApproveRequest, + lib.HostPairingRejectRequest, + ]) { + assert.equal(Message.type, "request", Message.name); + assert.equal(Message.src, "control", Message.name); + assert.equal(Message.dst, "controller", Message.name); + assert.equal(Message.permission, "core.host.approve", Message.name); + assert(lib.Link._requestsByClass.has(Message), `${Message.name} is not registered`); + } + assert.equal(lib.HostPairingUpdatesEvent.type, "event"); + assert.equal(lib.HostPairingUpdatesEvent.src, "controller"); + assert.equal(lib.HostPairingUpdatesEvent.dst, "control"); + assert.equal(lib.HostPairingUpdatesEvent.permission, "core.host.approve"); + assert(lib.Link._eventsByClass.has(lib.HostPairingUpdatesEvent)); + }); + it("removes the host token messages", function() { + for (const name of ["HostGenerateTokenRequest", "HostConfigCreateRequest", "HostRevokeTokensRequest"]) { + assert.equal(lib[name], undefined, `${name} is still exported`); + assert(!lib.Link._requestsByName.has(name), `${name} is still registered`); + } + }); + it("defines the host permissions", function() { + for (const name of ["core.host.approve", "core.host.revoke"]) { + assert(lib.permissions.has(name), `${name} is not defined`); + assert.equal(lib.permissions.get(name).grantByDefault, false, `${name} is granted by default`); + } + for (const name of ["core.host.generate_token", "core.host.revoke_token", "core.host.create_config"]) { + assert.equal(lib.permissions.has(name), false, `${name} is still defined`); + } + }); + it("round trips HostPairingListRequest responses", function() { + const validate = lib.compile(lib.HostPairingListRequest.Response.jsonSchema); + const json = JSON.parse(JSON.stringify(details)); + assert(validate(json), JSON.stringify(validate.errors)); + assert.deepEqual(lib.HostPairingListRequest.Response.fromJSON(json), details); + }); + it("round trips HostPairingUpdatesEvent", function() { + testRoundTripJsonSerialisable(lib.HostPairingUpdatesEvent, [[[]], [details]]); + }); + it("round trips HostPairingApproveRequest", function() { + testRoundTripJsonSerialisable(lib.HostPairingApproveRequest, testMatrix( + ["ABC-DEF", "abcdef"], // code + [undefined, "Renamed"], // name + )); + const validate = lib.compile(lib.HostPairingApproveRequest.Response.jsonSchema); + assert(validate(1234)); + }); + it("round trips HostPairingRejectRequest", function() { + testRoundTripJsonSerialisable(lib.HostPairingRejectRequest, [["ABC-DEF"], ["abcdef"]]); + }); + it("approves a pending request", async function() { + controller.config.set("controller.auth_secret", Buffer.from("TestSecretDoNotUse").toString("base64")); + const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); + const list = await controlConnection.handleHostPairingListRequest(new lib.HostPairingListRequest()); + assert.deepEqual(list.map(entry => entry.code), [code]); + const hostId = await controlConnection.handleHostPairingApproveRequest( + new lib.HostPairingApproveRequest(code) + ); + assert.equal(controller.hosts.get(hostId).name, "Host A"); + assert.equal(controller.hostPairing.poll(requestId).hostId, hostId); + }); + it("rejects a pending request", async function() { + const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); + await controlConnection.handleHostPairingRejectRequest(new lib.HostPairingRejectRequest(code)); + assert.throws(() => controller.hostPairing.poll(requestId), /Request was rejected/); + }); + }); + describe("InstanceConfigGetRequest", function() { it("runs", async function() { const config = await controlConnection.handleInstanceConfigGetRequest( diff --git a/test/messages/host.test.js b/test/messages/host.test.js index 1b31129a..c90fd145 100644 --- a/test/messages/host.test.js +++ b/test/messages/host.test.js @@ -18,6 +18,19 @@ describe("messages/host", function() { host = new Host(hostConnector, hostConfig, []); }); + describe("HostRevokeRequest", function() { + it("is round trip json serialisable", function() { + testRoundTripJsonSerialisable(lib.HostRevokeRequest, [[1], [2 ** 31 - 1]]); + }); + it("is a control request requiring core.host.revoke", function() { + assert.equal(lib.HostRevokeRequest.type, "request"); + assert.equal(lib.HostRevokeRequest.src, "control"); + assert.equal(lib.HostRevokeRequest.dst, "controller"); + assert.equal(lib.HostRevokeRequest.permission, "core.host.revoke"); + assert(lib.Link._requestsByClass.has(lib.HostRevokeRequest)); + }); + }); + describe("HostUpdateRequest", function() { it("runs", async function() { host.config.set("host.allow_remote_updates", true); diff --git a/test/web_ui/host_pairing.js b/test/web_ui/host_pairing.js new file mode 100644 index 00000000..0aff5115 --- /dev/null +++ b/test/web_ui/host_pairing.js @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import events from "node:events"; + +import * as lib from "@clusterio/lib"; +import { MockConnector, MockControl } from "../mock.js"; +import "../setup_logging.js"; + +import { HostPairingSubscriber } from "../../packages/web_ui/src/util/host_pairing.ts"; + +const addr = lib.Address.fromShorthand; + +describe("web_ui/util/host_pairing", function() { + describe("class HostPairingSubscriber", function() { + let mockControl; + let permitted; + let subscriber; + + beforeEach(function() { + mockControl = new MockControl(new MockConnector(addr({ controlId: 0 }), addr("controller"))); + permitted = true; + subscriber = new HostPairingSubscriber(mockControl, () => permitted); + }); + + function pairing(code, updatedAtMs) { + return new lib.HostPairingDetails(code, `Host ${code}`, "192.0.2.1", "", updatedAtMs, updatedAtMs); + } + + async function settle() { + await new Promise(resolve => setImmediate(resolve)); + } + + function sentRequests() { + return mockControl.connector.sentMessages.filter(message => message.name === "SubscriptionRequest"); + } + + it("should not send requests on connect or account updates without permission", async function() { + permitted = false; + mockControl.connector.emit("connect"); + subscriber.handleConnectionEvent("resume"); + subscriber.handleAccountEvent(); + await settle(); + assert.deepEqual(sentRequests(), []); + }); + + it("should resubscribe on connect with permission", async function() { + subscriber.subscribe(() => {}); + await events.once(mockControl.connector, "send"); + mockControl.connector.sentMessages.length = 0; + + mockControl.connector.emit("connect"); + await settle(); + assert.equal(sentRequests().length, 1); + assert.deepEqual(sentRequests()[0].data, new lib.SubscriptionRequest( + lib.HostPairingUpdatesEvent.name, "replace", -1, lib.SubscriptionFilters.all() + )); + }); + + it("should check the permission after the connect handlers have run", async function() { + subscriber.subscribe(() => {}); + await events.once(mockControl.connector, "send"); + mockControl.connector.sentMessages.length = 0; + + permitted = false; + mockControl.connector.emit("connect"); + permitted = true; + await settle(); + assert.equal(sentRequests().length, 1); + }); + + it("should resubscribe on account updates with permission", async function() { + subscriber.subscribe(() => {}); + await events.once(mockControl.connector, "send"); + mockControl.connector.sentMessages.length = 0; + + permitted = false; + subscriber.handleAccountEvent(); + await settle(); + assert.deepEqual(sentRequests(), []); + + permitted = true; + subscriber.handleAccountEvent(); + await settle(); + assert.equal(sentRequests().length, 1); + }); + + it("should clear requests when a new session starts", async function() { + await subscriber._handleEvent(new lib.HostPairingUpdatesEvent([pairing("ABC-DEF", 10)])); + assert.equal(subscriber.values.size, 1); + const [before] = subscriber.getSnapshot(); + + mockControl.connector.emit("connect"); + assert.equal(subscriber.values.size, 0); + assert.equal(subscriber.lastUpdatedMs, -1); + const [after] = subscriber.getSnapshot(); + assert.notEqual(after, before); + assert.equal(after.size, 0); + }); + + it("should keep requests when a session is resumed", async function() { + await subscriber._handleEvent(new lib.HostPairingUpdatesEvent([pairing("ABC-DEF", 10)])); + subscriber.handleConnectionEvent("drop"); + subscriber.handleConnectionEvent("resume"); + assert.equal(subscriber.values.size, 1); + assert.equal(subscriber.lastUpdatedMs, 10); + }); + + it("should clear requests when subscribing again after the last unsubscribe", async function() { + const unsubscribe = subscriber.subscribe(() => {}); + await events.once(mockControl.connector, "send"); + await subscriber._handleEvent(new lib.HostPairingUpdatesEvent([pairing("ABC-DEF", 10)])); + unsubscribe(); + await new Promise(resolve => setTimeout(resolve)); + mockControl.connector.sentMessages.length = 0; + + subscriber.subscribe(() => {}); + assert.equal(subscriber.values.size, 0); + await settle(); + assert.equal(sentRequests().length, 1); + assert.equal(sentRequests()[0].data.lastRequestTimeMs, -1); + }); + + it("should keep requests when a page that subscribes replaces another", async function() { + const unsubscribe = subscriber.subscribe(() => {}); + await events.once(mockControl.connector, "send"); + await subscriber._handleEvent(new lib.HostPairingUpdatesEvent([pairing("ABC-DEF", 10)])); + unsubscribe(); + subscriber.subscribe(() => {}); + assert.equal(subscriber.values.size, 1); + assert.equal(subscriber.lastUpdatedMs, 10); + }); + + it("should set synced false on drop and close", async function() { + subscriber.synced = true; + subscriber.handleConnectionEvent("drop"); + assert.equal(subscriber.synced, false); + + subscriber.synced = true; + subscriber.handleConnectionEvent("close"); + assert.equal(subscriber.synced, false); + }); + }); +}); From 37e79d9c41c444b6e25eb4921c2b84753d5fa79f Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 10:11:14 +0100 Subject: [PATCH 03/44] Remove tests that pin implementation details Drops tests that restate message declarations, check that removed commands are gone, test internal helpers or repeat the same behaviour at another layer. --- test/controller/ControlConnection.test.js | 37 ------ test/controller/HostPairingManager.test.js | 72 +--------- test/controller/WsServer.js | 16 --- test/controller/routes.js | 41 ------ test/ctl/commands_host.test.js | 145 --------------------- test/integration/clusterio.js | 9 -- test/lib/data/HostPairingDetails.test.js | 67 ---------- test/messages/controller.test.js | 49 ------- test/messages/host.test.js | 7 - test/web_ui/host_pairing.js | 22 ---- 10 files changed, 1 insertion(+), 464 deletions(-) delete mode 100644 test/ctl/commands_host.test.js diff --git a/test/controller/ControlConnection.test.js b/test/controller/ControlConnection.test.js index 5516cc4e..39c8ee95 100644 --- a/test/controller/ControlConnection.test.js +++ b/test/controller/ControlConnection.test.js @@ -299,17 +299,6 @@ describe("controller/src/ControlConnection", function() { lib.logger.audit = originalAudit; }); - it("registers the host pairing and revoke handlers", function() { - for (const Request of [ - lib.HostRevokeRequest, - lib.HostPairingListRequest, - lib.HostPairingApproveRequest, - lib.HostPairingRejectRequest, - ]) { - assert(connection._requestHandlers.has(Request), `${Request.name} is not handled`); - } - }); - describe(".handleHostRevokeRequest()", function() { let record; let broadcasts; @@ -448,26 +437,6 @@ describe("controller/src/ControlConnection", function() { assert.equal(controller.hostPairing.poll(requestId).hostId, hostId); assert(audits.some(msg => msg.startsWith(`User admin approved host Host A (${hostId})`))); }); - it("approves a request with a different name", async function() { - const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); - const hostId = await connection.handleHostPairingApproveRequest( - new lib.HostPairingApproveRequest(code, "Renamed") - ); - assert.equal(controller.hosts.get(hostId).name, "Renamed"); - assert.equal(controller.hostPairing.poll(requestId).name, "Renamed"); - }); - it("rejects approving an unknown code or with an invalid name", async function() { - await assert.rejects( - connection.handleHostPairingApproveRequest(new lib.HostPairingApproveRequest("ZZZ-ZZZ")), - new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), - ); - const { code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); - await assert.rejects( - connection.handleHostPairingApproveRequest(new lib.HostPairingApproveRequest(code, " ")), - new lib.ValidationError("Host name is required"), - ); - assert.equal(controller.hosts.size, 0); - }); it("rejects a request", async function() { const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); await connection.handleHostPairingRejectRequest(new lib.HostPairingRejectRequest(code.toLowerCase())); @@ -476,12 +445,6 @@ describe("controller/src/ControlConnection", function() { assert.equal(controller.hosts.size, 0); assert(audits.some(msg => msg.startsWith(`User admin rejected host Host A with code ${code}`))); }); - it("throws NotFoundError when rejecting an unknown code", async function() { - await assert.rejects( - connection.handleHostPairingRejectRequest(new lib.HostPairingRejectRequest("ZZZ-ZZZ")), - new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), - ); - }); }); }); }); diff --git a/test/controller/HostPairingManager.test.js b/test/controller/HostPairingManager.test.js index 912b4dbe..ce6c8084 100644 --- a/test/controller/HostPairingManager.test.js +++ b/test/controller/HostPairingManager.test.js @@ -5,7 +5,7 @@ import jwt from "jsonwebtoken"; import * as lib from "@clusterio/lib"; import { Controller, HostPairingManager, HostPairingError, HostRecord } from "@clusterio/controller"; import { - addressKey, normaliseCode, pollTimeoutMs, maxPendingMs, maxPending, maxPendingPerAddress, + pollTimeoutMs, maxPendingMs, maxPending, maxPendingPerAddress, maxCreatesPerMinute, maxNameLength, } from "@clusterio/controller/dist/node/src/HostPairingManager.js"; @@ -49,41 +49,6 @@ function assertPairingError(fn, status, message) { } describe("controller/src/HostPairingManager", function() { - describe("addressKey()", function() { - it("should return IPv4 addresses unchanged", function() { - assert.equal(addressKey("10.0.0.1"), "10.0.0.1"); - }); - it("should strip the IPv4 mapped prefix", function() { - assert.equal(addressKey("::ffff:10.0.0.1"), "10.0.0.1"); - }); - it("should group IPv6 addresses by their /64", function() { - assert.equal(addressKey("2001:db8:0:1::5"), "2001:db8:0:1::/64"); - assert.equal(addressKey("2001:db8::1:ffff:0:0:3"), "2001:db8:0:1::/64"); - assert.equal(addressKey("2001:0DB8:0000:0001:aaaa:bbbb:cccc:dddd"), "2001:db8:0:1::/64"); - assert.equal(addressKey("2001:db8:0:2::5"), "2001:db8:0:2::/64"); - }); - it("should expand compressed groups at the start and end", function() { - assert.equal(addressKey("::1"), "0:0:0:0::/64"); - assert.equal(addressKey("::"), "0:0:0:0::/64"); - assert.equal(addressKey("fe80::1"), "fe80:0:0:0::/64"); - assert.equal(addressKey("2001:db8::"), "2001:db8:0:0::/64"); - }); - it("should return other values unchanged", function() { - assert.equal(addressKey(""), ""); - assert.equal(addressKey("not an address"), "not an address"); - assert.equal(addressKey("::ffff:not-ipv4"), "::ffff:not-ipv4"); - }); - }); - - describe("normaliseCode()", function() { - it("should format codes as shown by the installer", function() { - assert.equal(normaliseCode("ABC-DEF"), "ABC-DEF"); - assert.equal(normaliseCode("abcdef"), "ABC-DEF"); - assert.equal(normaliseCode("abc-def"), "ABC-DEF"); - assert.equal(normaliseCode(" abc def "), "ABC-DEF"); - assert.equal(normaliseCode("a.b_c-d e/f"), "ABC-DEF"); - }); - }); describe("HostPairingManager.validateName()", function() { it("should require a non-empty string", function() { @@ -245,12 +210,6 @@ describe("controller/src/HostPairingManager", function() { assert.equal(await manager.approve(code, undefined, "admin"), 1234); assert.equal(manager.create("Host B", "", "10.0.0.2").code, "333-333"); }); - it("should use distinct request ids", function() { - const first = manager.create("Host A", "", "10.0.0.1"); - const second = manager.create("Host A", "", "10.0.0.1"); - assert.notEqual(first.requestId, second.requestId); - assert.notEqual(first.code, second.code); - }); }); describe("limits", function() { @@ -568,14 +527,6 @@ describe("controller/src/HostPairingManager", function() { assert.equal(manager.requests.has(code), false); assertPairingError(() => manager.poll(requestId), 404); }); - it("should expire requests when creating and polling", function() { - const first = manager.create("Host A", "", "10.0.0.1"); - idle(first.requestId, pollTimeoutMs + 1); - const second = manager.create("Host B", "", "10.0.0.1"); - assert.equal(manager.requests.has(first.code), false); - idle(second.requestId, pollTimeoutMs + 1); - assertPairingError(() => manager.poll(second.requestId), 404); - }); it("should drop and warn about an approved host never picked up", async function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); const hostId = await manager.approve(code, undefined, "admin"); @@ -595,27 +546,6 @@ describe("controller/src/HostPairingManager", function() { assertPairingError(() => manager.poll(requestId), 404); assert.equal(warnings.length, 0); }); - it("should forget creation times older than a minute", function() { - manager.create("Host A", "", "10.0.0.1"); - assert.equal(manager._recentCreates.size, 1); - manager.prune(); - assert.equal(manager._recentCreates.size, 1); - for (const times of manager._recentCreates.values()) { - times.forEach((ms, index) => { times[index] = ms - 60e3 - 1; }); - } - manager.prune(); - assert.equal(manager._recentCreates.size, 0); - }); - }); - - describe(".start()/.stop()", function() { - it("should run an unreferenced prune interval while started", function() { - manager.start(); - assert(manager._pruneInterval, "prune interval not started"); - assert.equal(manager._pruneInterval.hasRef(), false); - manager.stop(); - assert.equal(manager._pruneInterval, undefined); - }); }); describe("Controller integration", function() { diff --git a/test/controller/WsServer.js b/test/controller/WsServer.js index 655a0b6b..91f55567 100644 --- a/test/controller/WsServer.js +++ b/test/controller/WsServer.js @@ -190,22 +190,6 @@ describe("controller/src/WsServer", function() { assert.equal(socket.closed, null); assert(wsServer.hostConnections.has(5)); }); - it("should reject a token without an issue time", async function() { - addHost(5); - await register(jwt.sign({ aud: "host", host: 5 }, secret, { noTimestamp: true })); - assertRejected("Authentication failed: invalid token"); - }); - it("should reject a token for another host", async function() { - addHost(5); - addHost(6); - await register(controller.generateHostToken(6), 5); - assertRejected("Authentication failed: missmatched host id"); - }); - it("should reject a token signed with another secret", async function() { - addHost(5); - await register(jwt.sign({ aud: "host", host: 5 }, Buffer.from("OtherSecret"))); - assertRejected("Authentication failed: invalid signature"); - }); it("should reject a host revoked while its previous connection closes", async function() { const record = addHost(5); wsServer.hostConnections.set(5, { diff --git a/test/controller/routes.js b/test/controller/routes.js index 94ea5f30..aaa0f9a4 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -447,23 +447,6 @@ describe("controller/src/routes", function() { await assertResponse(await post("/cancel", { requestId: code }), 204); assert(controller.hostPairing.requests.has(code), "cancelled by the code"); }); - it("should respond with 400 without a requestId in the body", async function() { - const { requestId, code } = await createRequest(); - await assertResponse(await post("/cancel", {}), 400, { error: "requestId is required" }); - await assertResponse( - await post(`/cancel?requestId=${encodeURIComponent(requestId)}`, {}), - 400, { error: "requestId is required" }, - ); - assert(controller.hostPairing.requests.has(code)); - }); - it("should respond with 503 in recovery mode", async function() { - const { requestId, code } = await createRequest(); - controller.recoveryMode = true; - await assertResponse( - await post("/cancel", { requestId }), 503, { error: "Controller is in recovery mode" } - ); - assert(controller.hostPairing.requests.has(code)); - }); }); describe("request ids", function() { @@ -502,36 +485,12 @@ describe("controller/src/routes", function() { assert.equal((await check({ "Authorization": "Bearer invalid" })).status, 401); assert.equal((await check({ "Authorization": `Bearer ${adminToken}` })).status, 401); }); - it("should respond with 401 for a host that is not registered", async function() { - const token = controller.generateHostToken(99); - assert.equal((await check({ "Authorization": `Bearer ${token}` })).status, 401); - }); it("should respond with 401 once the host is revoked", async function() { const { hostId, token } = await controller.hostPairing.createHost("Host A"); await assertResponse(await check({ "Authorization": `Bearer ${token}` }), 200, { valid: true }); controller.hosts.delete(controller.hosts.get(hostId)); assert.equal((await check({ "Authorization": `Bearer ${token}` })).status, 401); }); - it("should respond with 401 for a token issued before tokenValidAfter", async function() { - const { hostId, token } = await controller.hostPairing.createHost("Host A"); - const record = controller.hosts.getMutable(hostId); - record.tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000); - controller.hosts.set(record); - assert.equal((await check({ "Authorization": `Bearer ${token}` })).status, 401); - }); - }); - - describe("pairing flow", function() { - it("should pair a host approved while the installer polls", async function() { - const { requestId, code } = await createRequest("Host A"); - await assertResponse(await post("/poll", { requestId }), 202, { status: "pending" }); - const hostId = await controller.hostPairing.approve(code.toLowerCase(), undefined, "test"); - const result = await (await post("/poll", { requestId })).json(); - assert.deepEqual(result, { hostId, token: result.token, name: "Host A" }); - await assertResponse( - await check({ "Authorization": `Bearer ${result.token}` }), 200, { valid: true } - ); - }); }); }); describe("/api/upload-save", function() { diff --git a/test/ctl/commands_host.test.js b/test/ctl/commands_host.test.js deleted file mode 100644 index 3390c59d..00000000 --- a/test/ctl/commands_host.test.js +++ /dev/null @@ -1,145 +0,0 @@ -import assert from "node:assert/strict"; -import * as lib from "@clusterio/lib"; -import { hostCommands } from "@clusterio/ctl/dist/node/src/commands_host.js"; - -/** Stand-in for Ctl that answers requests from a map of handlers and records what was sent */ -class FakeCtl { - constructor(handlers) { - this.handlers = new Map(handlers); - this.sent = []; - } - - async send(request) { - return await this.sendTo("controller", request); - } - - async sendTo(dst, request) { - this.sent.push(request); - const handler = this.handlers.get(request.constructor); - if (!handler) { - throw new Error(`Unexpected ${request.constructor.name}`); - } - return handler(request); - } -} - -async function run(command, args, ctl) { - const lines = []; - const originalLog = console.log; // eslint-disable-line no-console - console.log = (...content) => lines.push(content.join(" ")); // eslint-disable-line no-console - try { - await hostCommands.get(command).run(args, ctl); - } finally { - console.log = originalLog; // eslint-disable-line no-console - } - return lines; -} - -describe("ctl/commands_host", function() { - // Created when the request is handled, the ages printed depend on the current time. - const pending = () => [ - new lib.HostPairingDetails("ABC-DEF", "new-box", "192.0.2.10", "2.0.3", Date.now() - 65500), - new lib.HostPairingDetails("XYZ-234", "other-box", "2001:db8::1", "", Date.now() - 3720500), - ]; - const hosts = [{ id: 7, name: "basement-box" }, { id: 12, name: "attic-box" }]; - - it("no longer has the token based commands", function() { - for (const name of ["generate-token", "revoke-token", "create-config"]) { - assert.equal(hostCommands.get(name), undefined, `host ${name} still exists`); - } - }); - - describe("host pending", function() { - it("should list pending requests with their age", async function() { - const ctl = new FakeCtl([[lib.HostPairingListRequest, pending]]); - const lines = await run("pending", {}, ctl); - const output = lines.join("\n"); - assert.match(output, /code\s*\|\s*name\s*\|\s*address\s*\|\s*age/); - assert.match(output, /ABC-DEF\s*\|\s*new-box\s*\|\s*192\.0\.2\.10\s*\|\s*1m 5s/); - assert.match(output, /XYZ-234\s*\|\s*other-box\s*\|\s*2001:db8::1\s*\|\s*1h 2m/); - }); - - it("should say so when nothing is pending", async function() { - const ctl = new FakeCtl([[lib.HostPairingListRequest, () => []]]); - assert.deepEqual(await run("pending", {}, ctl), ["No pending host requests"]); - }); - }); - - describe("host approve", function() { - it("should approve with the requested name and print it", async function() { - const ctl = new FakeCtl([ - [lib.HostPairingListRequest, pending], - [lib.HostPairingApproveRequest, () => 42], - ]); - const lines = await run("approve", { code: "abcdef" }, ctl); - const approve = ctl.sent.find(request => request instanceof lib.HostPairingApproveRequest); - assert.equal(approve.code, "abcdef"); - assert.equal(approve.name, undefined); - assert.deepEqual(lines, ["Approved host new-box with id 42"]); - }); - - it("should pass --name and print it without listing requests", async function() { - const ctl = new FakeCtl([[lib.HostPairingApproveRequest, () => 43]]); - const lines = await run("approve", { code: "ABC-DEF", name: " renamed " }, ctl); - assert.equal(ctl.sent.length, 1); - assert.equal(ctl.sent[0].code, "ABC-DEF"); - assert.equal(ctl.sent[0].name, " renamed "); - assert.deepEqual(lines, ["Approved host renamed with id 43"]); - }); - - it("should pass on errors from the controller", async function() { - const ctl = new FakeCtl([ - [lib.HostPairingListRequest, () => []], - [lib.HostPairingApproveRequest, () => { - throw new lib.NotFoundError("No pending host request with code AAA-AAA"); - }], - ]); - await assert.rejects( - run("approve", { code: "AAA-AAA" }, ctl), - new lib.NotFoundError("No pending host request with code AAA-AAA"), - ); - }); - }); - - describe("host reject", function() { - it("should send the code to the controller", async function() { - const ctl = new FakeCtl([[lib.HostPairingRejectRequest, () => undefined]]); - assert.deepEqual(await run("reject", { code: "abc-def" }, ctl), []); - assert.equal(ctl.sent.length, 1); - assert(ctl.sent[0] instanceof lib.HostPairingRejectRequest); - assert.equal(ctl.sent[0].code, "abc-def"); - }); - }); - - describe("host revoke", function() { - function revokeCtl() { - return new FakeCtl([ - [lib.HostListRequest, () => hosts], - [lib.HostRevokeRequest, () => undefined], - ]); - } - - it("should revoke a host given by name", async function() { - const ctl = revokeCtl(); - await run("revoke", { host: "basement-box" }, ctl); - const revoke = ctl.sent.at(-1); - assert(revoke instanceof lib.HostRevokeRequest); - assert.equal(revoke.hostId, 7); - }); - - it("should revoke a host given by id", async function() { - const ctl = revokeCtl(); - await run("revoke", { host: "12" }, ctl); - assert.equal(ctl.sent.at(-1).hostId, 12); - }); - - it("should fail for an unknown host", async function() { - const ctl = revokeCtl(); - await assert.rejects( - run("revoke", { host: "missing" }, ctl), - new lib.CommandError("No host named missing"), - ); - assert(!ctl.sent.some(request => request instanceof lib.HostRevokeRequest)); - }); - }); -}); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index b3b18d24..67d99bf9 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -915,15 +915,6 @@ describe("Integration of Clusterio", function() { }); }); - describe("removed host token commands", function() { - it("should no longer exist", async function() { - slowTest(this); - for (const command of ["generate-token", "revoke-token", "create-config"]) { - await assert.rejects(execCtlProcess(`host ${command}`), new RegExp(`Unknown argument: ${command}`)); - } - }); - }); - describe("instance list", function() { it("runs", async function() { await execCtl("instance list"); diff --git a/test/lib/data/HostPairingDetails.test.js b/test/lib/data/HostPairingDetails.test.js index fa0b31be..5064c901 100644 --- a/test/lib/data/HostPairingDetails.test.js +++ b/test/lib/data/HostPairingDetails.test.js @@ -1,4 +1,3 @@ -import assert from "node:assert/strict"; import { HostPairingDetails } from "@clusterio/lib"; import { testMatrix, testRoundTripJsonSerialisable } from "../../common.js"; @@ -16,70 +15,4 @@ describe("lib/data/HostPairingDetails", function () { [undefined, true], // isDeleted )); }); - - describe("constructor", function () { - it("should set defaults correctly", function () { - const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1"); - - assert.equal(details.code, "ABC-DEF"); - assert.equal(details.name, "New Host"); - assert.equal(details.remoteAddress, "10.0.0.1"); - assert.equal(details.version, ""); - assert.equal(details.createdAtMs, 0); - assert.equal(details.updatedAtMs, 0); - assert.equal(details.isDeleted, false); - }); - - it("should use the code as the id", function () { - const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1"); - assert.equal(details.id, "ABC-DEF"); - }); - }); - - describe(".toJSON()", function () { - it("should only include required fields when optional fields are defaults", function () { - const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1", "", 0, 0, false); - - assert.deepEqual(details.toJSON(), { - code: "ABC-DEF", - name: "New Host", - remoteAddress: "10.0.0.1", - }); - }); - - it("should include all optional fields when set", function () { - const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1", "2.0.0", 100, 200, true); - - assert.deepEqual(details.toJSON(), { - code: "ABC-DEF", - name: "New Host", - remoteAddress: "10.0.0.1", - version: "2.0.0", - createdAtMs: 100, - updatedAtMs: 200, - isDeleted: true, - }); - }); - - it("should not include the id getter", function () { - const details = new HostPairingDetails("ABC-DEF", "New Host", "10.0.0.1"); - assert.equal(Object.hasOwn(JSON.parse(JSON.stringify(details)), "id"), false); - }); - }); - - describe("static fromJSON()", function () { - it("should construct minimal object", function () { - const details = HostPairingDetails.fromJSON({ - code: "ABC-DEF", - name: "New Host", - remoteAddress: "10.0.0.1", - }); - - assert.equal(details.code, "ABC-DEF"); - assert.equal(details.version, ""); - assert.equal(details.createdAtMs, 0); - assert.equal(details.updatedAtMs, 0); - assert.equal(details.isDeleted, false); - }); - }); }); diff --git a/test/messages/controller.test.js b/test/messages/controller.test.js index 4bf3e9ab..570a995f 100644 --- a/test/messages/controller.test.js +++ b/test/messages/controller.test.js @@ -192,39 +192,6 @@ describe("messages/controller", function() { new lib.HostPairingDetails("GHJ-KMN", "Host B", "2001:db8::1", "", 0, 300, true), ]; - it("defines the pairing messages for control links approving hosts", function() { - for (const Message of [ - lib.HostPairingListRequest, - lib.HostPairingApproveRequest, - lib.HostPairingRejectRequest, - ]) { - assert.equal(Message.type, "request", Message.name); - assert.equal(Message.src, "control", Message.name); - assert.equal(Message.dst, "controller", Message.name); - assert.equal(Message.permission, "core.host.approve", Message.name); - assert(lib.Link._requestsByClass.has(Message), `${Message.name} is not registered`); - } - assert.equal(lib.HostPairingUpdatesEvent.type, "event"); - assert.equal(lib.HostPairingUpdatesEvent.src, "controller"); - assert.equal(lib.HostPairingUpdatesEvent.dst, "control"); - assert.equal(lib.HostPairingUpdatesEvent.permission, "core.host.approve"); - assert(lib.Link._eventsByClass.has(lib.HostPairingUpdatesEvent)); - }); - it("removes the host token messages", function() { - for (const name of ["HostGenerateTokenRequest", "HostConfigCreateRequest", "HostRevokeTokensRequest"]) { - assert.equal(lib[name], undefined, `${name} is still exported`); - assert(!lib.Link._requestsByName.has(name), `${name} is still registered`); - } - }); - it("defines the host permissions", function() { - for (const name of ["core.host.approve", "core.host.revoke"]) { - assert(lib.permissions.has(name), `${name} is not defined`); - assert.equal(lib.permissions.get(name).grantByDefault, false, `${name} is granted by default`); - } - for (const name of ["core.host.generate_token", "core.host.revoke_token", "core.host.create_config"]) { - assert.equal(lib.permissions.has(name), false, `${name} is still defined`); - } - }); it("round trips HostPairingListRequest responses", function() { const validate = lib.compile(lib.HostPairingListRequest.Response.jsonSchema); const json = JSON.parse(JSON.stringify(details)); @@ -245,22 +212,6 @@ describe("messages/controller", function() { it("round trips HostPairingRejectRequest", function() { testRoundTripJsonSerialisable(lib.HostPairingRejectRequest, [["ABC-DEF"], ["abcdef"]]); }); - it("approves a pending request", async function() { - controller.config.set("controller.auth_secret", Buffer.from("TestSecretDoNotUse").toString("base64")); - const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); - const list = await controlConnection.handleHostPairingListRequest(new lib.HostPairingListRequest()); - assert.deepEqual(list.map(entry => entry.code), [code]); - const hostId = await controlConnection.handleHostPairingApproveRequest( - new lib.HostPairingApproveRequest(code) - ); - assert.equal(controller.hosts.get(hostId).name, "Host A"); - assert.equal(controller.hostPairing.poll(requestId).hostId, hostId); - }); - it("rejects a pending request", async function() { - const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); - await controlConnection.handleHostPairingRejectRequest(new lib.HostPairingRejectRequest(code)); - assert.throws(() => controller.hostPairing.poll(requestId), /Request was rejected/); - }); }); describe("InstanceConfigGetRequest", function() { diff --git a/test/messages/host.test.js b/test/messages/host.test.js index c90fd145..16d8dae5 100644 --- a/test/messages/host.test.js +++ b/test/messages/host.test.js @@ -22,13 +22,6 @@ describe("messages/host", function() { it("is round trip json serialisable", function() { testRoundTripJsonSerialisable(lib.HostRevokeRequest, [[1], [2 ** 31 - 1]]); }); - it("is a control request requiring core.host.revoke", function() { - assert.equal(lib.HostRevokeRequest.type, "request"); - assert.equal(lib.HostRevokeRequest.src, "control"); - assert.equal(lib.HostRevokeRequest.dst, "controller"); - assert.equal(lib.HostRevokeRequest.permission, "core.host.revoke"); - assert(lib.Link._requestsByClass.has(lib.HostRevokeRequest)); - }); }); describe("HostUpdateRequest", function() { diff --git a/test/web_ui/host_pairing.js b/test/web_ui/host_pairing.js index 0aff5115..5aa64a70 100644 --- a/test/web_ui/host_pairing.js +++ b/test/web_ui/host_pairing.js @@ -55,18 +55,6 @@ describe("web_ui/util/host_pairing", function() { )); }); - it("should check the permission after the connect handlers have run", async function() { - subscriber.subscribe(() => {}); - await events.once(mockControl.connector, "send"); - mockControl.connector.sentMessages.length = 0; - - permitted = false; - mockControl.connector.emit("connect"); - permitted = true; - await settle(); - assert.equal(sentRequests().length, 1); - }); - it("should resubscribe on account updates with permission", async function() { subscriber.subscribe(() => {}); await events.once(mockControl.connector, "send"); @@ -128,15 +116,5 @@ describe("web_ui/util/host_pairing", function() { assert.equal(subscriber.values.size, 1); assert.equal(subscriber.lastUpdatedMs, 10); }); - - it("should set synced false on drop and close", async function() { - subscriber.synced = true; - subscriber.handleConnectionEvent("drop"); - assert.equal(subscriber.synced, false); - - subscriber.synced = true; - subscriber.handleConnectionEvent("close"); - assert.equal(subscriber.synced, false); - }); }); }); From 4a49a198f0bbf5cfd0412f1393e8f7418abc56eb Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 21:13:14 +0100 Subject: [PATCH 04/44] Pair hosts when they start instead of in the installer The installer only writes the controller URL and host name. Running the host requests to join the cluster if it has not been approved yet, or if the controller no longer accepts its token, and waits until an admin approves it. Once approved it saves its id and token and starts as normal. A rejected host exits with code 8. clusteriohost run --user-token approves the host right away with the token of a user allowed to approve hosts. The token is not saved. This moves the pairing client into the host package, where it uses the same certificate setup as the host's connection to the controller, and removes --no-pair and the rerun handling from the installer. --- README.md | 11 +- docs/configuration.md | 7 +- docs/devs/plugin-migration-to-2.1.md | 4 +- docs/devs/protocol.md | 13 +- docs/managing-a-cluster.md | 12 +- packages/controller/src/HostPairingManager.ts | 2 +- packages/controller/src/WsServer.ts | 2 +- packages/create/README.md | 59 ++- packages/create/create.js | 240 ++---------- packages/create/local_host.js | 68 ++++ packages/create/pair.js | 275 -------------- packages/host/host.ts | 46 ++- packages/host/index.ts | 1 + packages/host/src/pairing.ts | 293 +++++++++++++++ .../web_ui/src/components/HostViewPage.tsx | 2 +- packages/web_ui/src/components/HostsPage.tsx | 10 +- scripts/version-compat-test.js | 24 +- test/controller/WsServer.js | 2 +- test/create.js | 211 +---------- test/create/local_host.js | 98 +++++ test/create/pair.js | 343 ------------------ test/host/pairing.js | 295 +++++++++++++++ test/integration/clusterio.js | 55 ++- test/integration/index.js | 4 +- test/integration/link.js | 2 +- 25 files changed, 948 insertions(+), 1131 deletions(-) create mode 100644 packages/create/local_host.js delete mode 100644 packages/create/pair.js create mode 100644 packages/host/src/pairing.ts create mode 100644 test/create/local_host.js delete mode 100644 test/create/pair.js create mode 100644 test/host/pairing.js diff --git a/README.md b/README.md index e40a34c4..7f1b47db 100644 --- a/README.md +++ b/README.md @@ -274,20 +274,19 @@ If you have copied these files over to `/etc/systemd/system/` then you can start Run the installer as described in the installation section and choose "Host only" as the operating mode to install, it'll ask for a name for the host and a controller URL. The URL is the same as what is needed to connect to the web interface. -Once it has installed everything the installer prints a code and waits for the host to be approved: +Then start the host, the first time it starts it prints a code and waits for the host to be approved: Waiting for approval. Approve "my-host" with code ABC-DEF on the Hosts page of https://example.com/, or run: npx clusterioctl host approve ABC-DEF Approve it in the web interface by clicking Add host on the Hosts page and approving the request with the same code, or with `npx clusterioctl host approve `. -The installer then saves the connection details for the host and exits. -Only approve a request if its code matches the one shown by the installer, approving gives the host full access to the cluster. +The host then saves its connection details and starts up, it should show up in the Hosts list and be available for assigning and running instances on. +Only approve a request if its code matches the one shown by the host, approving gives the host full access to the cluster. Approving requires the `core.host.approve` permission, which the Cluster Admin role has. -If you have that permission you can skip the wait by passing your own authentication token to the installer with `--controller-token `, it is only used to approve the host and is not saved. -Once you start up the host it should show up in the Hosts list and be available for assigning and running instances on. +If you have that permission you can skip the wait by starting the host with `npx clusteriohost run --user-token ` and your own authentication token, it is only used to approve the host and is not saved. To remove a host from the cluster, unassign its instances and click Revoke host access on the page for the host in the web interface, or run `npx clusterioctl host revoke `. This disconnects the host and its saved connection details stop working. -To add the computer back to the cluster run the installer again in the same directory, it reuses the existing installation and asks for approval again. +To add the computer back to the cluster start the host again, it asks for approval again. ## Setting up clusterioctl diff --git a/docs/configuration.md b/docs/configuration.md index 2ea6bf06..ea25d482 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -327,7 +327,7 @@ Defaults to "New Host". ### host.id Immutable numeric id of the host which uniquely identifies this host in the cluster. -The controller picks the id when the host is approved and the installer writes it here, the controller does not accept hosts with an id it has not approved. +The controller picks the id when the host is approved and the host writes it here, the controller does not accept hosts with an id it has not approved. Hidden in the web interface. Defaults to a random 31 bit number. @@ -367,8 +367,9 @@ Defaults to "http://localhost:8080/". ### host.controller_token Access token used for authenticating with the controller. -The installer writes it together with host.id when the host is approved, see [Setting up remote hosts](/README.md#setting-up-remote-hosts). -The token only works while the host exists on the controller, if the host is revoked run the installer again in the host directory to get it approved again. +The host saves it together with host.id when it is approved, see [Setting up remote hosts](/README.md#setting-up-remote-hosts). +The token only works while the host exists on the controller, if the host is revoked it asks to be approved again the next time it starts. +Set it to `enter token here` to make the host ask to join again, for example after changing host.controller_url to another controller. The value can only be read and changed from the host itself. Defaults to "enter token here". diff --git a/docs/devs/plugin-migration-to-2.1.md b/docs/devs/plugin-migration-to-2.1.md index 2161e4a9..f53f53d2 100644 --- a/docs/devs/plugin-migration-to-2.1.md +++ b/docs/devs/plugin-migration-to-2.1.md @@ -64,7 +64,7 @@ The `container` property of `WebPluginContext` is now the plugin's ES module nam ## Host pairing Host tokens are no longer generated by users. -A host joins the cluster by running the installer and having the code it shows approved, see [Host pairing](protocol.md#host-pairing) for the HTTP endpoints it uses. +A host joins the cluster by having the code it shows when it first starts approved, see [Host pairing](protocol.md#host-pairing) for the HTTP endpoints it uses. - `HostGenerateTokenRequest` and `HostConfigCreateRequest` are removed. Pending requests are listed with `HostPairingListRequest`, approved with `HostPairingApproveRequest` and rejected with `HostPairingRejectRequest`, and changes to them are sent to subscribers of `HostPairingUpdatesEvent`. @@ -72,7 +72,7 @@ A host joins the cluster by running the installer and having the code it shows a It throws `ConflictError` while instances are assigned to the host. - The permissions `core.host.generate_token`, `core.host.revoke_token` and `core.host.create_config` are removed. Check `core.host.approve` and `core.host.revoke` instead. -- `clusteriocontroller bootstrap generate-host-token` is removed, scripts that set up hosts should run the installer with `--controller-token` to approve the host with a user token. +- `clusteriocontroller bootstrap generate-host-token` is removed, scripts that set up hosts can start the host with `--user-token` to approve it with a user token. Host records are no longer created when a host connects, only by approving it, and the controller rejects hosts it has no record of. Revoking a host deletes its record and note. diff --git a/docs/devs/protocol.md b/docs/devs/protocol.md index 8408d2cb..ab028efa 100644 --- a/docs/devs/protocol.md +++ b/docs/devs/protocol.md @@ -34,9 +34,9 @@ The [send-buffer setting](../configuration.md#controllermax_send_buffer_size) co ## Host pairing -The host id and token sent in `registerHost` come from the installer, which gets them from the controller over HTTP before the host is started. +The host id and token sent in `registerHost` are requested by the host over HTTP the first time it starts, before it connects. Hosts are only added to the controller by approving a pairing request, and `registerHost` is rejected with close code 3000 (Unauthorized) for a host id the controller has no record of. -Revoking a host deletes its record, so its token stops working and the installer has to pair it again, which gives it a new id. +Revoking a host deletes its record, so its token stops working and the host has to be approved again the next time it starts, which gives it a new id. See [Setting up remote hosts](/README.md#setting-up-remote-hosts) for how this looks to the user. The endpoints below take and return JSON bodies and are sent with `Cache-Control: no-store`. @@ -50,7 +50,7 @@ A user or host token is sent in either an `Authorization: Bearer ` header Creates a request for a host to join the cluster. - `name: string` - Name of the host, at most 100 characters after trimming whitespace, without control characters. -- `version: string` (optional) - Version of the installer. +- `version: string` (optional) - Version of the host. Responds with 202 and `{"requestId": string, "code": string}`. The `code` is shown to the user to approve the request with, formatted as six characters like `ABC-DEF`. @@ -73,20 +73,20 @@ Rejected requests give 403, and unknown or expired requests give 404. The result of an approved or rejected request is only returned once, after which the request is gone. A pending request expires 30 seconds after it was last polled, and after an hour of waiting. -The installer polls every 2 seconds. +The host polls every 2 seconds, and makes a new request when the previous one has expired. ### `POST /api/host-pairing/cancel` - `requestId: string` - Id returned when the request was created. -Removes a pending request, sent by the installer when it stops waiting. +Removes a pending request, sent by the host when it stops waiting. Responds with 204 whether or not the request existed. ### `GET /api/host-pairing/check` Checks a host token. Responds with 200 and `{"valid": true}` if the token is valid and the host it was issued to still exists, and with 401 otherwise. -The installer uses this to tell whether an existing host config still works when it is run again. +A host with a saved token checks it with this when it starts, and requests to join again if the token is not accepted. ## Connector flow @@ -178,6 +178,7 @@ Host handshake for establishing a new connection session. The controller closes the connection with code 3000 (Unauthorized) if the token is invalid, was issued for another id, or if there is no host with this id, see [Host pairing](#host-pairing). If this happens when the host starts it exits with code 8, and the run scripts created by the installer do not restart it. +A host that has been rejected also exits with code 8. A host that is revoked while connected is disconnected, is rejected when it tries to connect again, and shuts down. #### `registerControl` diff --git a/docs/managing-a-cluster.md b/docs/managing-a-cluster.md index 20ff255a..c7101959 100644 --- a/docs/managing-a-cluster.md +++ b/docs/managing-a-cluster.md @@ -23,23 +23,23 @@ Listing, approving and rejecting installers waiting for approval needs the `core ctl> host pending -Lists installers waiting for approval with the code they show, the name they asked for, the address they connected from and how long they have been waiting. -A request goes away when its installer stops waiting, and after waiting for an hour. +Lists hosts waiting for approval with the code they show, the name they asked for, the address they connected from and how long they have been waiting. +A request goes away when its host stops waiting, and after waiting for an hour, after which the host asks again with a new code. ### Approve host ctl> host approve [--name ] -Approves the installer showing the code and prints the id of the new host. +Approves the host showing the code and prints the id of the new host. The code is not case sensitive and the dash can be left out. -`--name` gives the host a different name than the one entered in the installer. +`--name` gives the host a different name than the one it asked for. Approving gives the host full access to the cluster, check with `host pending` that the name and address match the computer you are setting up. ### Reject host ctl> host reject -Rejects the installer showing the code, the installer then exits with an error. +Rejects the host showing the code, the host then exits with an error. ### Revoke host @@ -49,7 +49,7 @@ Specify the host by name or ID. Removes the host from the cluster, disconnects it and makes its token stop working. Instances assigned to the host have to be unassigned first. Requires the `core.host.revoke` permission. -To add the computer back to the cluster, run the installer again in the directory the host is installed in. +To add the computer back to the cluster, start the host again and approve it. ### Update Clusterio diff --git a/packages/controller/src/HostPairingManager.ts b/packages/controller/src/HostPairingManager.ts index b420ca67..55293dbf 100644 --- a/packages/controller/src/HostPairingManager.ts +++ b/packages/controller/src/HostPairingManager.ts @@ -258,7 +258,7 @@ export default class HostPairingManager { cancel(secret: string) { const request = this._bySecret.get(secret); if (request && request.state === "pending") { - logger.verbose(`Host pairing request ${request.details.code} cancelled by the installer`); + logger.verbose(`Host pairing request ${request.details.code} cancelled by the host`); this._remove(request); } } diff --git a/packages/controller/src/WsServer.ts b/packages/controller/src/WsServer.ts index 6320a338..fea731a6 100644 --- a/packages/controller/src/WsServer.ts +++ b/packages/controller/src/WsServer.ts @@ -265,7 +265,7 @@ ${err.stack}` let host = this.controller.hosts.get(data.id); if (!host) { - throw new Error("host is not registered, run the installer again in the host directory"); + throw new Error("host is not registered, start the host again to request to join the cluster"); } if (!tokenPayload.iat || tokenPayload.iat < host.tokenValidAfter) { throw new Error("invalid token"); diff --git a/packages/create/README.md b/packages/create/README.md index 166e449b..afbb84b2 100644 --- a/packages/create/README.md +++ b/packages/create/README.md @@ -37,11 +37,10 @@ The installer runs without prompts when all the questions for the mode are answe | `--http-port ` | standalone, controller | Port the controller web interface listens on, defaults to 8080. | | `--host-name ` | host | Name to request for the host, defaults to the name of the computer. | | `--controller-url ` | host, ctl | URL of the controller, the same as for its web interface. | -| `--controller-token ` | host, ctl | ctl: authentication token saved for clusterioctl. host: authentication token of a user that approves the host without waiting, it is not saved. | -| `--no-pair` | host | Install the host without pairing it with the controller. | +| `--controller-token ` | ctl | Authentication token saved for clusterioctl. | | `--public-address
` | standalone, host | DNS name or IP address players connect to this computer with. | | `--factorio-dir ` | standalone, host | Path to the Factorio installation. | -| `--self-signed-cert ` | standalone, host | Certificate to trust for a controller using a self-signed certificate. It is used for pairing and added to the run scripts. | +| `--self-signed-cert ` | standalone, host | Certificate to trust for a controller using a self-signed certificate. It is added to the run scripts. | | `--remote-npm`, `--no-remote-npm` | standalone, controller, host | Allow or deny remote updates, plugin updates and plugin installs from the web interface. | | `--plugins ` | all | Plugins to install, pass `--plugins` without packages to install none. | | `--plugin-template `, `--plugin-name ` | plugin-template | Templates to use and name of the new plugin. | @@ -57,47 +56,39 @@ A new host has to be approved before it can join the cluster: 1. Run the installer with the host mode on the computer for the host. It asks for a name for the host, the controller URL, the public address, the Factorio installation, whether to allow remote updates and which plugins to install. -2. Once it has installed everything it prints a code and waits for the host to be approved: +2. Start the host with `run-host.cmd`, `run-host.sh` or the systemd service in the `systemd` folder. + The first time it starts it prints a code and waits for the host to be approved: Waiting for approval. Approve "my-host" with code ABC-DEF on the Hosts page of https://example.com/, or run: npx clusterioctl host approve ABC-DEF 3. Approve the request by clicking Add host on the Hosts page of the web interface, or run `npx clusterioctl host approve ABC-DEF`. - Only approve a request if its code matches the one shown by the installer. + Only approve a request if its code matches the one shown by the host. Approving requires the `core.host.approve` permission. -4. The installer saves `host.id`, `host.name`, `host.controller_url` and `host.controller_token` to `config-host.json` and exits. - Start the host with `run-host.cmd`, `run-host.sh` or the systemd service in the `systemd` folder. +4. The host saves `host.id`, `host.name` and `host.controller_token` to `config-host.json` and starts up. + Later starts connect to the cluster straight away. -Press Ctrl+C while the installer waits to cancel the request. -A request that is not approved within an hour expires. +The host keeps waiting until the request is approved, also if the controller is restarted or can't be reached for a while. +Press Ctrl+C to stop waiting. +If the request is rejected the host exits with code 8, which the run scripts do not restart on. -Pass `--controller-token ` with the token of a user that has the `core.host.approve` permission to approve the host without waiting. +Start the host with `npx clusteriohost run --user-token ` and the token of a user that has the `core.host.approve` permission to approve the host without waiting. The token is only used for approving the host and is not saved. If the controller uses a self-signed certificate pass it with `--self-signed-cert `, see [setting up TLS](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate). +The installer adds it to the run scripts. When a host is revoked in the web interface or with `npx clusterioctl host revoke ` its token stops working and a running host shuts down. -Starting it again fails with exit code 8, which the run scripts do not restart on. +Starting it again requests to join the cluster again, and the host gets a new id once approved. - -## Running the installer again - -Running the installer with the host mode in a directory with `config-host.json` in it does not install anything, it only pairs the host again if needed: - -- If the controller accepts the saved token it prints "Host is already set up" and exits with code 0. -- If it does not, for example because the host was revoked or never approved, it asks for the host name and controller URL, defaulting to the saved ones, and pairs the host again. - The host gets a new id, other settings in `config-host.json` are kept. -- If the controller cannot be reached it prints a warning, keeps the saved config and exits with code 0. - -The token is only checked with the controller in `config-host.json`. -Passing a different `--controller-url` pairs the host with that controller instead, without sending it the saved token, and the host gets a new id. -If the same controller only moved to a new address, change the address with `npx clusteriohost config set host.controller_url ` instead, which keeps the host. +To move a host to another controller, change `host.controller_url` with `npx clusteriohost config set host.controller_url ` and set `host.controller_token` back to `enter token here`. +The host then requests to join that controller the next time it starts. ## Docker -To run a host in a container install it when building the image with all answers given as options and `--no-pair`. -Then run the installer again with the host mode when the container starts, it pairs the host on the first start and exits straight away on later starts. +To run a host in a container install it when building the image with all answers given as options. +The host requests to join the cluster the first time the container starts. ```dockerfile FROM node:22 @@ -105,18 +96,16 @@ FROM node:22 RUN mkdir /clusterio && chown node:node /clusterio USER node WORKDIR /clusterio -RUN npm init "@clusterio" -- --mode host --no-pair --public-address factorio.example.com \ - --factorio-dir /opt/factorio --no-remote-npm --plugins +RUN npm init "@clusterio" -- --mode host --host-name my-host --controller-url https://example.com/ \ + --public-address factorio.example.com --factorio-dir /opt/factorio --no-remote-npm --plugins VOLUME /clusterio -CMD npm init "@clusterio" -- --mode host --controller-url "$CONTROLLER_URL" --host-name "$HOST_NAME" \ - ${CONTROLLER_TOKEN:+--controller-token "$CONTROLLER_TOKEN"} && exec ./run-host.sh +CMD exec ./node_modules/.bin/clusteriohost run ${USER_TOKEN:+--user-token "$USER_TOKEN"} ``` - docker run -d --name my-host -v my-host:/clusterio -p 34100-34199:34100-34199/udp \ - -e CONTROLLER_URL=https://example.com/ -e HOST_NAME=my-host my-host-image + docker run -d --name my-host -v my-host:/clusterio -p 34100-34199:34100-34199/udp my-host-image docker logs my-host -- The volume holds `config-host.json`, without it every new container pairs as a new host. +- The volume holds `config-host.json`, without it every new container requests to join as a new host. Docker copies the installation from the image into a new empty volume, later image updates do not change the installation in an existing volume. -- The code to approve is printed to the container log, or set `CONTROLLER_TOKEN` to approve the host without waiting. -- Stop the container after revoking its host, otherwise a restart runs the installer again and the host asks to join again. +- The code to approve is printed to the container log, or set `USER_TOKEN` to approve the host without waiting. +- Stop the container after revoking its host, otherwise a restart requests to join again. diff --git a/packages/create/create.js b/packages/create/create.js index 8a8aac9a..5a0c5663 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -10,7 +10,7 @@ import util from "node:util"; import yargs from "yargs"; import { levels, logger, setLogLevel } from "./logging.js"; -import { checkHostToken, createLocalHost, pairHost, PairingError } from "./pair.js"; +import { createLocalHost } from "./local_host.js"; import { copyPluginTemplates } from "./template.js"; let dev = false; const scriptExt = process.platform === "win32" ? ".cmd" : ""; @@ -200,21 +200,6 @@ function normaliseControllerUrl(input) { return url.href; } -async function readSelfSignedCert(args) { - if (!args.selfSignedCert) { - return undefined; - } - try { - return await fs.readFile(args.selfSignedCert); - } catch (err) { - throw new InstallError(`Failed to read --self-signed-cert ${args.selfSignedCert}: ${err.message}`); - } -} - -async function installerVersion() { - return JSON.parse(await fs.readFile(path.join(import.meta.dirname, "package.json"))).version; -} - // Remote updates, plugin updates and plugin installs are allowed or denied together. async function setRemoteUpdates(exec, component, allow) { for (let field of ["allow_remote_updates", "allow_plugin_updates", "allow_plugin_install"]) { @@ -222,135 +207,6 @@ async function setRemoteUpdates(exec, component, allow) { } } -// Turn an error from pairing into a message for the user. -function pairingFailed(err) { - const retry = "Run the installer again with --mode host in this directory to retry."; - if (err.name === "AbortError") { - return new InstallError(`Pairing cancelled. ${retry}`); - } - if (err instanceof PairingError) { - return new InstallError(`Pairing failed: ${err.message}. ${retry}`); - } - if (typeof err.code === "string") { - const hint = /CERT|SELF_SIGNED|UNABLE_TO_VERIFY/.test(err.code) - ? " Pass the certificate with --self-signed-cert if the controller uses a self-signed one." - : ""; - return new InstallError(`Failed to connect to the controller: ${err.message}.${hint} ${retry}`); - } - return err; -} - -// Request to join the cluster and save the id and token of the host once approved. -async function pairWithController(args, { hostName, controllerUrl }) { - const ca = await readSelfSignedCert(args); - await execHost(["config", "set", "host.name", hostName]); - await execHost(["config", "set", "host.controller_url", controllerUrl]); - - // Ctrl-C may arrive twice under npm init, as npm forwards it to the installer, so keep - // listening until the request has been cancelled. - const abort = new AbortController(); - const onSigint = () => { abort.abort(); }; - process.on("SIGINT", onSigint); - // The close patch for inquirer on Windows leaves the console in raw mode, where Ctrl-C is - // read as input instead of raising SIGINT. No prompts follow, so it can be turned off. - if (process.platform === "win32" && process.stdin.isTTY && process.stdin.isRaw) { - process.stdin.setRawMode(false); - } - let result; - try { - logger.info(`Requesting to join the cluster at ${controllerUrl}`); - result = await pairHost({ - controllerUrl, - name: hostName, - version: await installerVersion(), - token: args.controllerToken, - ca, - signal: abort.signal, - onPending: ({ code }) => { - // eslint-disable-next-line no-console - console.log( - `Waiting for approval. Approve "${hostName}" with code ${code} on the Hosts page of ` + - `${controllerUrl}, or run: npx clusterioctl host approve ${code}` - ); - }, - }); - } catch (err) { - throw pairingFailed(err); - } finally { - process.off("SIGINT", onSigint); - } - - logger.info(`Host approved as "${result.name}" with id ${result.hostId}`); - await execHost(["config", "set", "host.id", result.hostId]); - if (result.name !== hostName) { - await execHost(["config", "set", "host.name", result.name]); - } - // Saved last so that an interrupted install is paired again when the installer is run again. - await execHost(["config", "set", "host.controller_token", result.token]); -} - -// Pair a host that is already installed again, unless the controller still accepts it. -async function rerunHost(args) { - let config; - try { - config = JSON.parse(await fs.readFile("config-host.json")); - } catch (err) { - throw new InstallError(`Failed to read config-host.json: ${err.message}`); - } - // Let a rerun fix run scripts written before the certificate was passed. - if (!dev && args.selfSignedCert) { - logger.info("Writing run scripts"); - await writeScripts("host", args.selfSignedCert); - } - if (!args.pair) { - // eslint-disable-next-line no-console - console.log("Host is already installed"); - return; - } - - const token = config["host.controller_token"]; - const controllerUrl = config["host.controller_url"]; - let otherController = false; - if (args.controllerUrl !== undefined) { - const result = validateControllerUrl(args.controllerUrl); - if (result !== true) { - throw new InstallError(result); - } - const savedUrl = validateControllerUrl(controllerUrl) === true ? normaliseControllerUrl(controllerUrl) : ""; - otherController = normaliseControllerUrl(args.controllerUrl) !== savedUrl; - } - // The token is never sent to another controller, it would give that controller access to this cluster. - if (otherController) { - logger.info(`Pairing with ${args.controllerUrl} instead of the controller in config-host.json`); - } else if (token && token !== "enter token here") { - let valid; - try { - valid = await checkHostToken({ controllerUrl, token, ca: await readSelfSignedCert(args) }); - } catch (err) { - if (err instanceof InstallError) { - throw err; - } - logger.warn(`Could not check if ${controllerUrl} accepts this host: ${err.message}`); - logger.warn("Keeping the existing host config, pass --controller-url to pair with another controller"); - return; - } - if (valid) { - // eslint-disable-next-line no-console - console.log("Host is already set up"); - return; - } - logger.info("The controller no longer accepts this host, pairing it again"); - } - - const answers = await inquirerPairingArgs(args, { - hostName: config["host.name"], - controllerUrl: config["host.controller_url"], - }); - await pairWithController(args, answers); - // eslint-disable-next-line no-console - console.log("Successfully set up host"); -} - async function validateInstallDir() { let entries = new Set(await fs.readdir(".")); if (entries.size) { @@ -681,37 +537,6 @@ WantedBy=multi-user.target } } -async function inquirerPairingArgs(args, defaults = {}) { - let answers = {}; - if (args.hostName) { answers.hostName = args.hostName; } - if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; } - answers = await inquirer.prompt([ - { - type: "input", - name: "hostName", - message: "Name of host", - default: defaults.hostName ?? os.hostname(), - validate: validateHostName, - }, - { - type: "input", - name: "controllerUrl", - message: "Controller URL", - default: defaults.controllerUrl, - validate: validateControllerUrl, - }, - ], answers); - - // Inquirer does not validate answers given as arguments - for (let result of [validateHostName(answers.hostName), validateControllerUrl(answers.controllerUrl)]) { - if (result !== true) { - throw new InstallError(result); - } - } - answers.controllerUrl = normaliseControllerUrl(answers.controllerUrl); - return answers; -} - // eslint-disable-next-line complexity async function inquirerMissingArgs(args) { let answers = {}; @@ -771,13 +596,31 @@ async function inquirerMissingArgs(args) { } if (answers.mode === "host") { - if (await pathExists("config-host.json")) { - // An installed host is only paired again, see rerunHost - return { mode: "host", rerun: true }; - } - if (args.pair) { - answers = { ...answers, ...await inquirerPairingArgs(args) }; + if (args.hostName) { answers.hostName = args.hostName; } + if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; } + answers = await inquirer.prompt([ + { + type: "input", + name: "hostName", + message: "Name of host", + default: os.hostname(), + validate: validateHostName, + }, + { + type: "input", + name: "controllerUrl", + message: "Controller URL", + validate: validateControllerUrl, + }, + ], answers); + + // Inquirer does not validate answers given as arguments + for (let result of [validateHostName(answers.hostName), validateControllerUrl(answers.controllerUrl)]) { + if (result !== true) { + throw new InstallError(result); + } } + answers.controllerUrl = normaliseControllerUrl(answers.controllerUrl); } if (answers.mode === "ctl") { @@ -947,12 +790,7 @@ async function main() { nargs: 1, describe: "Controller URL [host/ctl]", type: "string", }) .option("controller-token", { - nargs: 1, type: "string", describe: - "User token for ctl, or of a user that approves the host without waiting (not saved) [host/ctl]", - }) - .option("pair", { - describe: "Pair the host with the controller, use --no-pair to only install it [host]", - default: true, type: "boolean", + nargs: 1, describe: "Controller authentication token [ctl]", type: "string", }) .option("public-address", { nargs: 1, describe: "DNS/IP Address to connect to this server [standalone/host]", type: "string", @@ -1005,11 +843,6 @@ async function main() { let answers = await inquirerMissingArgs(args); logger.verbose(JSON.stringify(answers)); - if (answers.rerun) { - await rerunHost(args); - return; - } - if (answers.pluginTemplate) { await copyPluginTemplates(answers.pluginName, answers.pluginTemplate); return; @@ -1048,13 +881,18 @@ async function main() { await execHost(["config", "set", "host.controller_token", host.token]); } + if (answers.mode === "host") { + // The host requests to join the cluster when it is started. + await execHost(["config", "set", "host.name", answers.hostName]); + await execHost(["config", "set", "host.controller_url", answers.controllerUrl]); + } + if (["standalone", "host"].includes(answers.mode)) { await execHost(["config", "set", "host.public_address", answers.publicAddress]); await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]); await setRemoteUpdates(execHost, "host", answers.remoteNpm); } - // Written before pairing as running the installer again on an installed host only pairs it. if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) { logger.info("Writing run scripts"); await writeScripts(answers.mode, args.selfSignedCert); @@ -1066,16 +904,14 @@ async function main() { await execCtl(["config", "set", "ctl.controller_token", answers.controllerToken]); } - if (answers.mode === "host") { - if (args.pair) { - await pairWithController(args, answers); - } else { - logger.info("Run the installer again with --mode host in this directory to pair the host"); - } - } - /* eslint-disable no-console */ console.log(`Successfully installed ${answers.mode}`); + if (answers.mode === "host") { + console.log( + `Start the host with run-host${process.platform === "win32" ? ".cmd" : ".sh"}, then approve it on the ` + + `Hosts page of ${answers.controllerUrl}` + ); + } if (adminToken) { console.log(`Admin authentication token: ${adminToken}`); } diff --git a/packages/create/local_host.js b/packages/create/local_host.js new file mode 100644 index 00000000..bc534f14 --- /dev/null +++ b/packages/create/local_host.js @@ -0,0 +1,68 @@ +// Adds the host of a standalone install to the controller database, used by the installer. +// Only uses Node.js builtins as it runs before Clusterio is installed. +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; + +function signHostToken(payload, secret) { + const encode = value => Buffer.from(JSON.stringify(value)).toString("base64url"); + const unsigned = `${encode({ alg: "HS256", typ: "JWT" })}.${encode(payload)}`; + const signature = crypto.createHmac("sha256", secret).update(unsigned).digest("base64url"); + return `${unsigned}.${signature}`; +} + +/** + * Add a host to the database of a controller installed in the same directory + * + * The controller must not be running, as it would overwrite the database with + * what it has in memory. + * + * @param {object} options - Host options. + * @param {string=} options.directory - Directory with config-controller.json in it. + * @param {string} options.name - Name of the host. + * @param {number=} options.id - + * Id to give the host, picks a random unused id if not given. If a host with + * this id exists, a new token is signed for it and its name is kept. + * @param {string=} options.version - Version to record for the host. + * @returns {Promise<{ hostId: number, token: string, name: string }>} id, token and name of the host. + */ +export async function createLocalHost({ directory = ".", name, id, version = "" }) { + const configPath = path.join(directory, "config-controller.json"); + const config = JSON.parse(await fs.readFile(configPath, "utf8")); + const secret = config["controller.auth_secret"]; + if (typeof secret !== "string" || !secret) { + throw new Error(`${configPath} has no controller.auth_secret`); + } + + const databaseDirectory = path.resolve(directory, config["controller.database_directory"] ?? "database"); + const hostsPath = path.join(databaseDirectory, "hosts.json"); + let hosts = []; + try { + hosts = JSON.parse(await fs.readFile(hostsPath, "utf8")); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + const now = Math.floor(Date.now() / 1000); + const existing = id === undefined ? undefined : hosts.find(host => host.id === id); + let hostId = id; + if (existing) { + // Running the installer again signs a new token for the same host. + name = existing.name; + } else { + const ids = new Set(hosts.map(host => host.id)); + while (hostId === undefined || ids.has(hostId)) { + hostId = crypto.randomInt(1, 2 ** 31); + } + hosts.push({ id: hostId, name, version, plugins: {}, token_valid_after: now }); + await fs.mkdir(databaseDirectory, { recursive: true }); + const temporary = path.join(databaseDirectory, "hosts.tmp.json"); + await fs.writeFile(temporary, JSON.stringify(hosts, null, "\t"), { flush: true }); + await fs.rename(temporary, hostsPath); + } + + const token = signHostToken({ aud: "host", host: hostId, iat: now }, Buffer.from(secret, "base64")); + return { hostId, token, name }; +} diff --git a/packages/create/pair.js b/packages/create/pair.js deleted file mode 100644 index c290c0ad..00000000 --- a/packages/create/pair.js +++ /dev/null @@ -1,275 +0,0 @@ -// Pairing of hosts with the controller, used by the installer. -// Only uses Node.js builtins as it runs before or without Clusterio being installed. -import crypto from "node:crypto"; -import fs from "node:fs/promises"; -import http from "node:http"; -import https from "node:https"; -import path from "node:path"; -import { setTimeout as sleep } from "node:timers/promises"; - -/** Time to keep polling through network errors, the controller drops requests not polled for 30 seconds */ -const pollRetryMs = 30e3; -/** Time to wait for a response from the controller */ -const requestTimeoutMs = 30e3; -/** Largest response accepted from the controller */ -const maxResponseSize = 64 * 1024; - -/** Error responded by the controller, or for an invalid controller URL */ -export class PairingError extends Error { - /** - * @param {string} message - Description of the error. - * @param {number=} status - HTTP status the controller responded with. - */ - constructor(message, status) { - super(message); - this.name = "PairingError"; - /** @type {number | undefined} */ - this.status = status; - } -} - -function apiUrl(controllerUrl, route) { - let url; - try { - url = new URL(route, controllerUrl.endsWith("/") ? controllerUrl : `${controllerUrl}/`); - } catch (err) { - throw new PairingError(`Invalid controller URL ${controllerUrl}`); - } - if (!["http:", "https:"].includes(url.protocol)) { - throw new PairingError(`Controller URL ${controllerUrl} must start with http:// or https://`); - } - return url; -} - -function request(url, { method = "POST", body, token, ca, signal, timeoutMs = requestTimeoutMs }) { - return new Promise((resolve, reject) => { - const data = body === undefined ? undefined : JSON.stringify(body); - const headers = { "Accept": "application/json" }; - if (data !== undefined) { - headers["Content-Type"] = "application/json"; - headers["Content-Length"] = Buffer.byteLength(data); - } - if (token) { - headers["Authorization"] = `Bearer ${token}`; - } - const options = { method, headers, signal, timeout: timeoutMs }; - if (url.protocol === "https:" && ca) { - options.ca = ca; - } - - const req = (url.protocol === "https:" ? https : http).request(url, options, res => { - const chunks = []; - let size = 0; - res.on("data", chunk => { - size += chunk.length; - if (size > maxResponseSize) { - req.destroy(new PairingError("Response from controller is too large", res.statusCode)); - return; - } - chunks.push(chunk); - }); - res.on("error", reject); - res.on("end", () => { - let json; - try { - json = JSON.parse(Buffer.concat(chunks).toString()); - } catch (err) { - json = undefined; - } - resolve({ status: res.statusCode, headers: res.headers, json }); - }); - }); - req.on("timeout", () => { - const err = new Error(`Request to ${url} timed out`); - err.code = "ETIMEDOUT"; - req.destroy(err); - }); - req.on("error", reject); - req.end(data); - }); -} - -function responseError(response) { - let message = typeof response.json?.error === "string" - ? response.json.error - : `Unexpected response from controller: ${response.status} ${http.STATUS_CODES[response.status] ?? ""}`.trim(); - const retryAfter = response.headers["retry-after"]; - if (response.status === 429 && retryAfter) { - message += `, try again in ${retryAfter} seconds`; - } - return new PairingError(message, response.status); -} - -function pairingResult(response) { - const { hostId, token, name } = response.json ?? {}; - if (!Number.isInteger(hostId) || typeof token !== "string" || typeof name !== "string") { - throw new PairingError("Invalid response from controller", response.status); - } - return { hostId, token, name }; -} - -async function pollUntilDone(controllerUrl, requestId, ca, pollIntervalMs, signal) { - const url = apiUrl(controllerUrl, "api/host-pairing/poll"); - let lastResponseMs = Date.now(); - while (true) { - await sleep(pollIntervalMs, undefined, { signal }); - let response; - try { - response = await request(url, { body: { requestId }, ca, signal }); - } catch (err) { - if (signal?.aborted || Date.now() - lastResponseMs > pollRetryMs) { - throw err; - } - continue; - } - lastResponseMs = Date.now(); - if (response.status === 200) { - return pairingResult(response); - } - if (response.status !== 202) { - throw responseError(response); - } - } -} - -/** - * Request to join the cluster and wait until the request is approved - * - * @param {object} options - Pairing options. - * @param {string} options.controllerUrl - URL of the controller, with or without a trailing slash. - * @param {string} options.name - Name to request for the host. - * @param {string=} options.version - Version of the installer, shown with the request. - * @param {string=} options.token - - * Token of a user allowed to approve hosts, approves the host without waiting. - * @param {(string|Buffer)=} options.ca - Certificate to trust for a controller using a self-signed certificate. - * @param {number=} options.pollIntervalMs - Interval to poll for the result at. - * @param {function({ code: string }): void=} options.onPending - - * Called once with the code to approve the request with if it has to wait for approval. - * @param {AbortSignal=} options.signal - Signal to cancel the request with. - * @returns {Promise<{ hostId: number, token: string, name: string }>} id, token and approved name of the host. - * @throws {PairingError} if the controller refuses or rejects the request. - * @throws {DOMException} AbortError if the signal is aborted. - */ -export async function pairHost({ - controllerUrl, name, version = "", token, ca, pollIntervalMs = 2000, onPending, signal, -}) { - let requestId; - try { - signal?.throwIfAborted(); - const response = await request(apiUrl(controllerUrl, "api/host-pairing"), { - body: { name, version }, token, ca, signal, - }); - if (response.status === 200) { - return pairingResult(response); - } - if (response.status === 404) { - throw new PairingError("The controller must be updated to a version that supports host pairing", 404); - } - if (response.status !== 202) { - throw responseError(response); - } - if (typeof response.json?.requestId !== "string" || typeof response.json.code !== "string") { - throw new PairingError("Invalid response from controller", response.status); - } - requestId = response.json.requestId; - onPending?.({ code: response.json.code }); - return await pollUntilDone(controllerUrl, requestId, ca, pollIntervalMs, signal); - - } catch (err) { - if (!signal?.aborted) { - throw err; - } - if (requestId !== undefined) { - await request(apiUrl(controllerUrl, "api/host-pairing/cancel"), { - body: { requestId }, ca, timeoutMs: 5e3, - }).catch(() => {}); - } - throw new DOMException("Pairing was cancelled", "AbortError"); - } -} - -/** - * Check if the controller accepts a host token - * - * @param {object} options - Check options. - * @param {string} options.controllerUrl - URL of the controller, with or without a trailing slash. - * @param {string} options.token - Host token to check. - * @param {(string|Buffer)=} options.ca - Certificate to trust for a controller using a self-signed certificate. - * @returns {Promise} true if the token is valid and its host is registered. - * @throws {PairingError} if the controller gives an unexpected response. - */ -export async function checkHostToken({ controllerUrl, token, ca }) { - const response = await request(apiUrl(controllerUrl, "api/host-pairing/check"), { - method: "GET", token, ca, - }); - if (response.status === 200) { - return true; - } - if (response.status === 401) { - return false; - } - throw responseError(response); -} - -function signHostToken(payload, secret) { - const encode = value => Buffer.from(JSON.stringify(value)).toString("base64url"); - const unsigned = `${encode({ alg: "HS256", typ: "JWT" })}.${encode(payload)}`; - const signature = crypto.createHmac("sha256", secret).update(unsigned).digest("base64url"); - return `${unsigned}.${signature}`; -} - -/** - * Add a host to the database of a controller installed in the same directory - * - * The controller must not be running, as it would overwrite the database with - * what it has in memory. - * - * @param {object} options - Host options. - * @param {string=} options.directory - Directory with config-controller.json in it. - * @param {string} options.name - Name of the host. - * @param {number=} options.id - - * Id to give the host, picks a random unused id if not given. If a host with - * this id exists, a new token is signed for it and its name is kept. - * @param {string=} options.version - Version to record for the host. - * @returns {Promise<{ hostId: number, token: string, name: string }>} id, token and name of the host. - */ -export async function createLocalHost({ directory = ".", name, id, version = "" }) { - const configPath = path.join(directory, "config-controller.json"); - const config = JSON.parse(await fs.readFile(configPath, "utf8")); - const secret = config["controller.auth_secret"]; - if (typeof secret !== "string" || !secret) { - throw new Error(`${configPath} has no controller.auth_secret`); - } - - const databaseDirectory = path.resolve(directory, config["controller.database_directory"] ?? "database"); - const hostsPath = path.join(databaseDirectory, "hosts.json"); - let hosts = []; - try { - hosts = JSON.parse(await fs.readFile(hostsPath, "utf8")); - } catch (err) { - if (err.code !== "ENOENT") { - throw err; - } - } - - const now = Math.floor(Date.now() / 1000); - const existing = id === undefined ? undefined : hosts.find(host => host.id === id); - let hostId = id; - if (existing) { - // Running the installer again signs a new token for the same host. - name = existing.name; - } else { - const ids = new Set(hosts.map(host => host.id)); - while (hostId === undefined || ids.has(hostId)) { - hostId = crypto.randomInt(1, 2 ** 31); - } - hosts.push({ id: hostId, name, version, plugins: {}, token_valid_after: now }); - await fs.mkdir(databaseDirectory, { recursive: true }); - const temporary = path.join(databaseDirectory, "hosts.tmp.json"); - await fs.writeFile(temporary, JSON.stringify(hosts, null, "\t"), { flush: true }); - await fs.rename(temporary, hostsPath); - } - - const token = signHostToken({ aud: "host", host: hostId, iat: now }, Buffer.from(secret, "base64")); - return { hostId, token, name }; -} diff --git a/packages/host/host.ts b/packages/host/host.ts index d3deccd7..a718fc42 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -25,6 +25,7 @@ import { ConsoleTransport, levels, logger } from "@clusterio/lib"; import Host from "./src/Host.js"; import { createScenarioCommand, handleCreateScenarioCommand } from "./src/create_scenario.js"; +import { PairingError, ensurePaired } from "./src/pairing.js"; let host: Host | undefined; @@ -134,6 +135,10 @@ async function startHost() { type: "boolean", nargs: 0, default: false, describe: "Start the host in recovery mode with all plugins disabled and controller disconnected", }); + yargs.option("user-token", { + type: "string", nargs: 1, + describe: "Token of a user allowed to approve hosts, approves this host without waiting. Not saved", + }); }) .demandCommand(1, "You need to specify a command to run") .strict() @@ -212,18 +217,6 @@ async function startHost() { // and as the process name in ps/top on linux. process.title = "clusteriohost"; - // make sure we have the controller access token - if (hostConfig.get("host.controller_token") === "enter token here") { - logger.fatal("ERROR invalid config!"); - logger.fatal( - "This host has not been set up to connect to a controller. Run the installer\n"+ - "again in this directory with npm init @clusterio and approve the host in the\n"+ - "web interface." - ); - process.exitCode = 1; - return; - } - // make sure url ends with / if (!hostConfig.get("host.controller_url").endsWith("/")) { logger.fatal("ERROR invalid config!"); @@ -232,6 +225,35 @@ async function startHost() { return; } + if (!args.recovery) { + const abort = new AbortController(); + const onSignal = () => { abort.abort(); }; + process.on("SIGINT", onSignal); + process.on("SIGTERM", onSignal); + try { + await ensurePaired(hostConfig, { + version: packageConfig.version, + userToken: args.userToken as string | undefined, + signal: abort.signal, + }); + } catch (err: any) { + if (abort.signal.aborted) { + logger.info("Stopped waiting for approval"); + return; + } + if (err instanceof PairingError) { + logger.fatal(`Unable to join the cluster: ${err.message}`); + // Exit code 8 stops the run scripts from restarting the host. + process.exitCode = err.status === 403 ? 8 : 1; + return; + } + throw err; + } finally { + process.off("SIGINT", onSignal); + process.off("SIGTERM", onSignal); + } + } + let hostConnector = new HostConnector(hostConfig, pluginInfos); host = await Host.fromConfig( hostConfig, diff --git a/packages/host/index.ts b/packages/host/index.ts index c9d81c49..4420639d 100644 --- a/packages/host/index.ts +++ b/packages/host/index.ts @@ -4,6 +4,7 @@ export { BaseInstancePlugin, InstanceHooks, type InstancePluginContext } from ". export { default as Host, HostRouter } from "./src/Host.js"; export { default as Instance } from "./src/Instance.js"; export { default as InstanceConnection } from "./src/InstanceConnection.js"; +export { PairingError, checkHostToken, pairHost } from "./src/pairing.js"; if (import.meta.main) { bootstrap(); diff --git a/packages/host/src/pairing.ts b/packages/host/src/pairing.ts new file mode 100644 index 00000000..6d320f4c --- /dev/null +++ b/packages/host/src/pairing.ts @@ -0,0 +1,293 @@ +import { setTimeout as sleep } from "node:timers/promises"; + +import * as lib from "@clusterio/lib"; +import { logger } from "@clusterio/lib"; + +/** Time to wait for a response from the controller */ +const requestTimeoutMs = 30e3; +/** Placeholder token of a host that has not been paired */ +const unpairedToken = "enter token here"; + +/** Error responded by the controller */ +export class PairingError extends Error { + constructor( + message: string, + /** HTTP status the controller responded with */ + public status?: number, + ) { + super(message); + this.name = "PairingError"; + } +} + +export interface PairingResult { + hostId: number; + token: string; + name: string; +} + +export interface PairHostOptions { + /** URL of the controller, ending with a slash */ + controllerUrl: string; + /** Name to request for the host */ + name: string; + /** Version of the host, shown with the request */ + version?: string; + /** Token of a user allowed to approve hosts, approves the host without waiting */ + token?: string; + /** Interval to poll for the result at */ + pollIntervalMs?: number; + /** Delay before trying again when the controller can't be reached or is busy */ + retryDelayMs?: number; + /** Called with the code to approve the request with each time a request is made */ + onPending?: (code: string) => void; + /** Called when the controller can't be reached or is busy and the request is retried */ + onRetry?: (err: Error, delayMs: number) => void; + /** Signal to cancel pairing with */ + signal?: AbortSignal; +} + +interface Response { + status: number; + headers: Headers; + json: any; +} + +async function request( + controllerUrl: string, + route: string, + options: { body?: object, token?: string, signal?: AbortSignal, timeoutMs?: number }, +): Promise { + const headers: Record = { "Accept": "application/json" }; + if (options.body) { + headers["Content-Type"] = "application/json"; + } + if (options.token) { + headers["Authorization"] = `Bearer ${options.token}`; + } + const timeout = AbortSignal.timeout(options.timeoutMs ?? requestTimeoutMs); + const response = await fetch(new URL(route, controllerUrl), { + method: options.body ? "POST" : "GET", + headers, + body: options.body ? JSON.stringify(options.body) : undefined, + signal: options.signal ? AbortSignal.any([options.signal, timeout]) : timeout, + }); + let json; + try { + json = await response.json(); + } catch { + json = undefined; + } + return { status: response.status, headers: response.headers, json }; +} + +function responseError(response: Response) { + const message = typeof response.json?.error === "string" + ? response.json.error + : `Unexpected response from controller: ${response.status}`; + return new PairingError(message, response.status); +} + +function pairingResult(response: Response): PairingResult { + const { hostId, token, name } = response.json ?? {}; + if (!Number.isInteger(hostId) || typeof token !== "string" || typeof name !== "string") { + throw new PairingError("Invalid response from controller", response.status); + } + return { hostId, token, name }; +} + +/** Responses that are worth trying again later */ +function isTemporary(status: number) { + return status === 429 || status >= 500; +} + +function retryDelay(response: Response, defaultMs: number) { + const seconds = Number(response.headers.get("retry-after")); + return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : defaultMs; +} + +/** + * Poll a request until it is approved, rejected or no longer known by the controller + * + * @returns the result if approved, or null if the controller no longer knows the request. + */ +async function poll(requestId: string, options: Required> & PairHostOptions) { + let failing = false; + while (true) { + await sleep(options.pollIntervalMs, undefined, { signal: options.signal }); + let response; + try { + response = await request(options.controllerUrl, "api/host-pairing/poll", { + body: { requestId }, signal: options.signal, + }); + } catch (err: any) { + if (options.signal?.aborted) { + throw err; + } + // Keep polling through network errors, the request is made again if it expired meanwhile. + if (!failing) { + options.onRetry?.(err, options.pollIntervalMs); + failing = true; + } + continue; + } + failing = false; + if (response.status === 200) { + return pairingResult(response); + } + if (response.status === 404) { + return null; + } + if (response.status !== 202 && !isTemporary(response.status)) { + throw responseError(response); + } + } +} + +/** + * Request to join the cluster and wait until the request is approved + * + * Waits through network errors and controller restarts, making a new request + * if the controller no longer knows the previous one. + * + * @param options - Pairing options. + * @returns id, token and approved name of the host. + * @throws {PairingError} if the controller refuses or rejects the request. + * @throws {DOMException} AbortError if the signal is aborted. + */ +export async function pairHost(options: PairHostOptions): Promise { + const opts = { pollIntervalMs: 2000, retryDelayMs: 10e3, version: "", ...options }; + let requestId: string | undefined; + try { + while (true) { + opts.signal?.throwIfAborted(); + let response; + try { + response = await request(opts.controllerUrl, "api/host-pairing", { + body: { name: opts.name, version: opts.version }, token: opts.token, signal: opts.signal, + }); + } catch (err: any) { + if (opts.signal?.aborted) { + throw err; + } + opts.onRetry?.(err, opts.retryDelayMs); + await sleep(opts.retryDelayMs, undefined, { signal: opts.signal }); + continue; + } + + if (response.status === 200) { + return pairingResult(response); + } + if (isTemporary(response.status)) { + const delayMs = retryDelay(response, opts.retryDelayMs); + opts.onRetry?.(responseError(response), delayMs); + await sleep(delayMs, undefined, { signal: opts.signal }); + continue; + } + if (response.status === 404) { + throw new PairingError("The controller must be updated to a version that supports host pairing", 404); + } + if (response.status !== 202) { + throw responseError(response); + } + if (typeof response.json?.requestId !== "string" || typeof response.json.code !== "string") { + throw new PairingError("Invalid response from controller", response.status); + } + + requestId = response.json.requestId as string; + opts.onPending?.(response.json.code); + const result = await poll(requestId, opts); + requestId = undefined; + if (result) { + return result; + } + } + + } catch (err) { + if (!opts.signal?.aborted) { + throw err; + } + if (requestId !== undefined) { + await request(opts.controllerUrl, "api/host-pairing/cancel", { + body: { requestId }, timeoutMs: 5e3, + }).catch(() => {}); + } + throw new DOMException("Pairing was cancelled", "AbortError"); + } +} + +/** + * Check if the controller accepts a host token + * + * @param controllerUrl - URL of the controller, ending with a slash. + * @param token - Host token to check. + * @returns true if the token is valid and its host is registered. + * @throws {PairingError} if the controller gives an unexpected response. + */ +export async function checkHostToken(controllerUrl: string, token: string) { + const response = await request(controllerUrl, "api/host-pairing/check", { token }); + if (response.status === 200) { + return true; + } + if (response.status === 401) { + return false; + } + throw responseError(response); +} + +/** + * Pair the host with the controller if it has not been paired, or the controller no longer accepts it + * + * Waits until the request is approved and saves the id, name and token of the host to its config. + * + * @param config - Config of the host. + * @param options - Pairing options. + * @throws {PairingError} if the controller refuses or rejects the request. + * @throws {DOMException} AbortError if the signal is aborted. + */ +export async function ensurePaired( + config: lib.HostConfig, + options: { version: string, userToken?: string, signal?: AbortSignal }, +) { + const controllerUrl = config.get("host.controller_url"); + const token = config.get("host.controller_token"); + if (token && token !== unpairedToken) { + let valid; + try { + valid = await checkHostToken(controllerUrl, token); + } catch (err: any) { + // Connecting reports the problem, or authenticates if the controller predates pairing. + logger.verbose(`Unable to check if the controller accepts this host: ${err.message}`); + return; + } + if (valid) { + return; + } + logger.warn("The controller no longer accepts this host, requesting to join again"); + } + + const name = config.get("host.name"); + logger.info(`Requesting to join the cluster at ${controllerUrl} as "${name}"`); + const result = await pairHost({ + controllerUrl, + name, + version: options.version, + token: options.userToken, + signal: options.signal, + onPending: code => { + logger.info( + `Waiting for approval. Approve "${name}" with code ${code} on the Hosts page of ` + + `${controllerUrl}, or run: npx clusterioctl host approve ${code}` + ); + }, + onRetry: (err, delayMs) => { + logger.warn(`Requesting to join failed: ${err.message}, trying again in ${Math.ceil(delayMs / 1000)}s`); + }, + }); + + config.set("host.id", result.hostId); + config.set("host.name", result.name); + config.set("host.controller_token", result.token); + await config.save(); + logger.info(`Host approved as "${result.name}" with id ${result.hostId}`); +} diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index 6e21f95f..5691985d 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -114,7 +114,7 @@ function RevokeHostButton({ host, instances }: RevokeHostButtonProps) { title={`Revoke access for ${host.name}?`} description={<> The host is removed from the cluster and disconnected.
- To add it back, run the installer again in its directory. + To add it back, start the host again and approve it. } placement="bottomRight" okText="Revoke" diff --git a/packages/web_ui/src/components/HostsPage.tsx b/packages/web_ui/src/components/HostsPage.tsx index 2947e2e5..e31095a0 100644 --- a/packages/web_ui/src/components/HostsPage.tsx +++ b/packages/web_ui/src/components/HostsPage.tsx @@ -116,8 +116,8 @@ function AddHostModal(props: { open: boolean, onClose: () => void }) { width="800px" > - Run the installer in an empty directory on the machine the host will run on. - It asks for a name for the host, then shows a code and waits for the request to be approved here. + Run the installer in an empty directory on the machine the host will run on, then start the host + with the run-host script. It shows a code and waits for the request to be approved here.
void }) {

- Once approved the installer saves the connection details of the host and exits. - The host can then be started with the run-host script in that directory. + Once approved the host saves its connection details and starts up. + Later starts connect to the cluster straight away. Waiting for approval - Only approve a request if its code matches the code shown by the installer. + Only approve a request if its code matches the code shown by the host. An approved host has full access to the cluster.
`${hostBin} config set ${field} ${escapeArg(String(value))}`; - const approved = { hostId: 1234, token: "host-token", name: "Approved" }; - let server; - let controllerUrl; - let requests; - let handler; - - beforeEach(async function() { - requests = []; - handler = () => [500, { error: "No handler" }]; - server = http.createServer(async (req, res) => { - let body = ""; - for await (const chunk of req) { - body += chunk; - } - const entry = { url: req.url, headers: req.headers, body: body ? JSON.parse(body) : undefined }; - requests.push(entry); - const [status, json] = handler(entry); - res.writeHead(status, { "Content-Type": "application/json" }); - res.end(json === undefined ? undefined : JSON.stringify(json)); - }); - server.listen(0, "localhost"); - await events.once(server, "listening"); - controllerUrl = `http://localhost:${server.address().port}/`; - }); - - afterEach(function() { - server.closeAllConnections(); - server.close(); - }); - - // Stub of the controller pairing endpoints, approving after the given number of pending polls. - function pairingHandler(pendingPolls = 0, result = approved) { - let polls = 0; - return ({ url }) => { - if (url === "/api/host-pairing") { - return [202, { requestId: "secret", code: "ABC-DEF" }]; - } - if (url === "/api/host-pairing/poll") { - polls += 1; - return polls > pendingPolls ? [200, result] : [202, { status: "pending" }]; - } - if (url === "/api/host-pairing/cancel") { - return [204, undefined]; - } - return [404, { error: "Not found" }]; - }; - } + const controllerUrl = "http://controller.example.com:8080/"; function run(args, env = {}) { // Record the commands for npm and the clusterio CLIs instead of running them. @@ -203,20 +155,6 @@ describe("create installer CLI", function() { }); return { stdout, stderr }; }; - if (process.env.CREATE_TEST_SIGINT) { - // Stands in for Ctrl-C, which cannot be sent to a child process on Windows. - // Sent twice like when npm init forwards the one it gets to the installer. - const write = process.stdout.write.bind(process.stdout); - process.stdout.write = (chunk, ...rest) => { - if (String(chunk).includes("Waiting for approval")) { - setTimeout(() => { - process.emit("SIGINT", "SIGINT"); - process.emit("SIGINT", "SIGINT"); - }, 50); - } - return write(chunk, ...rest); - }; - } `; return execFileAsync(process.execPath, [ "--import", `data:text/javascript,${encodeURIComponent(mockExec)}`, @@ -231,171 +169,43 @@ describe("create installer CLI", function() { // Commands run by the installer, without the getent lookups made when writing systemd units. async function commands() { - let content; - try { - content = await fs.readFile(path.join(directory, "commands.jsonl"), "utf8"); - } catch (err) { - if (err.code === "ENOENT") { - return []; - } - throw err; - } + const content = await fs.readFile(path.join(directory, "commands.jsonl"), "utf8"); return content.split("\n").filter(Boolean).map(line => JSON.parse(line)) .filter(command => !command.startsWith("getent ")); } - function hostArgs(...extra) { + function hostArgs(url = controllerUrl) { return [ - "--mode", "host", "--host-name", "Test Host", "--controller-url", controllerUrl, - "--public-address", "host.example.com", "--factorio-dir", "factorio", "--no-remote-npm", - ...extra, "--plugins", + "--mode", "host", "--host-name", "Test Host", "--controller-url", url, + "--public-address", "host.example.com", "--factorio-dir", "factorio", "--no-remote-npm", "--plugins", ]; } - async function writeHostConfig(token) { - await fs.writeFile(path.join(directory, "config-host.json"), JSON.stringify({ - "host.name": "Test Host", - "host.controller_url": controllerUrl, - "host.controller_token": token, - })); - } - - it("pairs a new host once approved and saves its id and token", async function() { - handler = pairingHandler(0); + it("writes the host config and leaves joining the cluster to the host", async function() { const { stdout } = await run(hostArgs()); - assert(stdout.includes( - `Waiting for approval. Approve "Test Host" with code ABC-DEF on the Hosts page of ${controllerUrl}, ` + - "or run: npx clusterioctl host approve ABC-DEF" - ), stdout); assert.match(stdout, /Successfully installed host/); - - const { version } = JSON.parse(await fs.readFile(path.join(path.dirname(installer), "package.json"))); - assert.deepEqual(requests[0].body, { name: "Test Host", version }); - assert.equal(requests[0].headers.authorization, undefined); + assert.match(stdout, /Start the host with run-host/); const recorded = await commands(); assert.match(recorded[0], /^npm(?:\.cmd)? install @clusterio\/host$/); assert.deepEqual(recorded.slice(1), [ `${hostBin} config create`, + hostSet("host.name", "Test Host"), + hostSet("host.controller_url", controllerUrl), hostSet("host.public_address", "host.example.com"), hostSet("host.factorio_directory", "factorio"), hostSet("host.allow_remote_updates", false), hostSet("host.allow_plugin_updates", false), hostSet("host.allow_plugin_install", false), - hostSet("host.name", "Test Host"), - hostSet("host.controller_url", controllerUrl), - hostSet("host.id", 1234), - hostSet("host.name", "Approved"), - hostSet("host.controller_token", "host-token"), ]); await fs.access(path.join(directory, `run-host${process.platform === "win32" ? ".cmd" : ".sh"}`)); }); - it("uses --controller-token to approve the host without saving it", async function() { - handler = ({ headers }) => (headers.authorization === "Bearer user-token" - ? [200, { hostId: 99, token: "host-token", name: "Test Host" }] - : [401, { error: "Invalid user token" }] - ); - const { stdout } = await run(hostArgs("--controller-token", "user-token")); - assert.doesNotMatch(stdout, /Waiting for approval/); - const recorded = await commands(); - assert(!recorded.some(command => command.includes("user-token")), "user token must not be saved"); - assert.deepEqual(recorded.slice(-2), [ - hostSet("host.id", 99), - hostSet("host.controller_token", "host-token"), - ]); - }); - - it("cancels the request on Ctrl-C", async function() { - handler = pairingHandler(Infinity); - await assert.rejects(run(hostArgs(), { CREATE_TEST_SIGINT: "1" }), err => { - assert.equal(err.code, 1); - assert.match(err.stdout + err.stderr, /Pairing cancelled/); - return true; - }); - assert.deepEqual(requests.find(r => r.url === "/api/host-pairing/cancel")?.body, { requestId: "secret" }); - assert(!(await commands()).some(command => command.includes("host.controller_token"))); - }); - - it("installs without contacting the controller with --no-pair", async function() { - const { stdout } = await run([ - "--mode", "host", "--no-pair", "--public-address", "localhost", "--factorio-dir", "factorio", - "--remote-npm", "--plugins", - ]); - assert.match(stdout, /Successfully installed host/); - assert.equal(requests.length, 0); - const recorded = await commands(); - assert(recorded.includes(hostSet("host.allow_plugin_install", true))); - assert(!recorded.some(command => command.includes("host.controller_token"))); - }); - - it("exits when run again on a host the controller accepts", async function() { - await writeHostConfig("valid-token"); - handler = ({ url, headers }) => { - const valid = url === "/api/host-pairing/check" && headers.authorization === "Bearer valid-token"; - return valid ? [200, { valid: true }] : [401, undefined]; - }; - const { stdout } = await run(["--mode", "host"]); - assert.match(stdout, /Host is already set up/); - assert.deepEqual(requests.map(r => r.url), ["/api/host-pairing/check"]); - assert.deepEqual(await commands(), []); - }); - - it("pairs again when run again on a revoked host", async function() { - await writeHostConfig("revoked-token"); - const pairing = pairingHandler(0, { hostId: 5, token: "new-token", name: "Renamed" }); - handler = entry => (entry.url === "/api/host-pairing/check" ? [401, undefined] : pairing(entry)); - const { stdout } = await run([ - "--mode", "host", "--host-name", "Renamed", "--controller-url", controllerUrl, - ]); - assert.match(stdout, /Successfully set up host/); - assert.equal(requests[0].headers.authorization, "Bearer revoked-token"); - assert.equal(requests[1].body.name, "Renamed"); - assert.deepEqual(await commands(), [ - hostSet("host.name", "Renamed"), - hostSet("host.controller_url", controllerUrl), - hostSet("host.id", 5), - hostSet("host.controller_token", "new-token"), - ]); - }); - - it("does not send the saved token to another controller", async function() { - await writeHostConfig("old-token"); - // Same stub server under its IP address, which the installer sees as another controller. - const { address, port } = server.address(); - const otherUrl = `http://${address.includes(":") ? `[${address}]` : address}:${port}/`; - handler = pairingHandler(0, { hostId: 6, token: "other-token", name: "Moved" }); - const { stdout } = await run(["--mode", "host", "--host-name", "Moved", "--controller-url", otherUrl]); - assert.match(stdout, /Successfully set up host/); - assert(!requests.some(r => r.url === "/api/host-pairing/check"), "token checked with other controller"); - assert(!requests.some(r => String(r.headers.authorization).includes("old-token"))); - assert.deepEqual(await commands(), [ - hostSet("host.name", "Moved"), - hostSet("host.controller_url", otherUrl), - hostSet("host.id", 6), - hostSet("host.controller_token", "other-token"), - ]); - }); - it("adds the trailing slash the host requires to the controller URL", async function() { - handler = pairingHandler(0); - const args = hostArgs(); - args[args.indexOf(controllerUrl)] = controllerUrl.slice(0, -1); - await run(args); + await run(hostArgs(controllerUrl.slice(0, -1))); assert((await commands()).includes(hostSet("host.controller_url", controllerUrl))); }); - it("writes the certificate into the run scripts when run again with one", async function() { - await writeHostConfig("valid-token"); - handler = ({ url }) => (url === "/api/host-pairing/check" ? [200, { valid: true }] : [404, undefined]); - const cert = path.resolve(import.meta.dirname, "file", "tls", "cert.pem"); - const { stdout } = await run(["--mode", "host", "--self-signed-cert", cert]); - assert.match(stdout, /Host is already set up/); - const script = process.platform === "win32" ? "run-host.cmd" : "run-host.sh"; - const content = await fs.readFile(path.join(directory, script), "utf8"); - assert(content.includes(`NODE_EXTRA_CA_CERTS=${cert}`), content); - }); - it("adds the host of a standalone install to the controller database", async function() { const secret = crypto.randomBytes(256).toString("base64"); await fs.writeFile(path.join(directory, "config-controller.json"), JSON.stringify({ @@ -429,7 +239,6 @@ describe("create installer CLI", function() { ]) { assert(recorded.includes(expected), `missing ${expected}`); } - assert(!recorded.some(command => command.includes("generate-host-token"))); }); it("keeps the host of a standalone install when run again", async function() { diff --git a/test/create/local_host.js b/test/create/local_host.js new file mode 100644 index 00000000..3f32b748 --- /dev/null +++ b/test/create/local_host.js @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import jwt from "jsonwebtoken"; + +import * as lib from "@clusterio/lib"; +import HostRecord from "@clusterio/controller/dist/node/src/HostRecord.js"; +import { createLocalHost } from "../../packages/create/local_host.js"; + +describe("create/local_host createLocalHost()", function() { + let directory; + let secret; + beforeEach(async function() { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-local-host-")); + secret = crypto.randomBytes(256).toString("base64"); + }); + afterEach(async function() { + await fs.rm(directory, { recursive: true, force: true, maxRetries: 10 }); + }); + + // Written with the controller's own config class to match its file format. + async function writeControllerConfig(fields = {}) { + const configPath = path.join(directory, "config-controller.json"); + const config = new lib.ControllerConfig("controller", undefined, configPath); + config.set("controller.auth_secret", secret); + for (const [name, value] of Object.entries(fields)) { + config.set(name, value); + } + await config.save(true); + } + + async function loadHosts(databaseDirectory = "database") { + const provider = new lib.JsonIdDatastoreProvider( + path.join(directory, databaseDirectory, "hosts.json"), HostRecord.fromJSON.bind(HostRecord), + ); + return await provider.load(); + } + + it("adds a host the controller loads with a token it accepts", async function() { + await writeControllerConfig(); + const before = Math.floor(Date.now() / 1000); + const result = await createLocalHost({ directory, name: "local" }); + assert.equal(result.name, "local"); + assert(Number.isInteger(result.hostId) && result.hostId >= 1 && result.hostId < 2 ** 31); + + const hosts = await loadHosts(); + assert.deepEqual([...hosts.keys()], [result.hostId]); + const record = hosts.get(result.hostId); + assert.equal(record.name, "local"); + assert.equal(record.version, ""); + assert.deepEqual(record.plugins, new Map()); + assert(record.tokenValidAfter >= before); + + const payload = jwt.verify(result.token, Buffer.from(secret, "base64"), { audience: "host" }); + assert.equal(payload.host, result.hostId); + assert(payload.iat >= record.tokenValidAfter, "token must not be issued before token_valid_after"); + }); + + it("appends to an existing database in the configured directory", async function() { + await writeControllerConfig({ "controller.database_directory": "db" }); + await fs.mkdir(path.join(directory, "db")); + await fs.writeFile(path.join(directory, "db", "hosts.json"), JSON.stringify([ + { id: 1, name: "existing", version: "2.0.0", plugins: {} }, + ])); + const result = await createLocalHost({ directory, name: "local", id: 7, version: "2.0.3" }); + assert.equal(result.hostId, 7); + + const hosts = await loadHosts("db"); + assert.deepEqual([...hosts.keys()], [1, 7]); + assert.equal(hosts.get(1).name, "existing"); + assert.equal(hosts.get(7).version, "2.0.3"); + await assert.rejects(fs.access(path.join(directory, "database")), { code: "ENOENT" }); + }); + + it("signs a new token for an existing host with the same id", async function() { + await writeControllerConfig(); + const first = await createLocalHost({ directory, name: "first", id: 7 }); + const before = await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"); + const second = await createLocalHost({ directory, name: "second", id: 7 }); + assert.equal(second.hostId, 7); + assert.equal(second.name, "first", "the name of the existing host should be kept"); + assert.equal(await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"), before); + + const record = (await loadHosts()).get(7); + for (const token of [first.token, second.token]) { + const payload = jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }); + assert.equal(payload.host, 7); + assert(payload.iat >= record.tokenValidAfter); + } + }); + + it("throws if the controller config has no auth secret", async function() { + await fs.writeFile(path.join(directory, "config-controller.json"), "{}"); + await assert.rejects(createLocalHost({ directory, name: "local" }), /controller\.auth_secret/); + }); +}); diff --git a/test/create/pair.js b/test/create/pair.js deleted file mode 100644 index 96bf54a0..00000000 --- a/test/create/pair.js +++ /dev/null @@ -1,343 +0,0 @@ -import assert from "node:assert/strict"; -import crypto from "node:crypto"; -import events from "node:events"; -import fs from "node:fs/promises"; -import http from "node:http"; -import https from "node:https"; -import os from "node:os"; -import path from "node:path"; -import tls from "node:tls"; -import jwt from "jsonwebtoken"; - -import * as lib from "@clusterio/lib"; -import HostRecord from "@clusterio/controller/dist/node/src/HostRecord.js"; -import { - PairingError, checkHostToken, createLocalHost, pairHost, -} from "../../packages/create/pair.js"; - -import "../setup_logging.js"; - -const tlsDir = path.join(import.meta.dirname, "..", "file", "tls"); - -// Stub controller recording requests, responding with what the test's handler returns. -async function startStub(createServer = http.createServer) { - const stub = { - requests: [], - handler: () => [500, { error: "No handler" }], - }; - stub.server = createServer(async (req, res) => { - const chunks = []; - for await (const chunk of req) { - chunks.push(chunk); - } - const entry = { - method: req.method, - url: req.url, - headers: req.headers, - body: chunks.length ? JSON.parse(Buffer.concat(chunks)) : undefined, - }; - stub.requests.push(entry); - const [status, json, headers = {}] = await stub.handler(entry); - res.writeHead(status, { "Content-Type": "application/json", ...headers }); - res.end(json === undefined ? undefined : JSON.stringify(json)); - }); - stub.server.listen(0, "localhost"); - await events.once(stub.server, "listening"); - stub.url = `${stub.server instanceof https.Server ? "https" : "http"}://localhost:${stub.server.address().port}/`; - stub.close = async () => { - stub.server.closeAllConnections(); - stub.server.close(); - await events.once(stub.server, "close"); - }; - return stub; -} - -// Handler for a request that is approved after the given number of pending polls. -function approveAfter(polls, result = { hostId: 1234, token: "host-token", name: "Approved" }) { - let pending = 0; - return ({ url }) => { - if (url === "/api/host-pairing") { - return [202, { requestId: "secret", code: "ABC-DEF" }]; - } - if (url === "/api/host-pairing/poll") { - pending += 1; - return pending > polls ? [200, result] : [202, { status: "pending" }]; - } - return [404, { error: "Not found" }]; - }; -} - -describe("create/pair", function() { - let stub; - beforeEach(async function() { - stub = await startStub(); - }); - afterEach(async function() { - await stub.close(); - }); - - describe("pairHost()", function() { - it("returns the host when approved with a user token", async function() { - stub.handler = () => [200, { hostId: 5, token: "host-token", name: "Host" }]; - let pendingCalls = 0; - const result = await pairHost({ - controllerUrl: stub.url, name: "Host", version: "2.0.3", token: "user-token", - onPending: () => { pendingCalls += 1; }, - }); - assert.deepEqual(result, { hostId: 5, token: "host-token", name: "Host" }); - assert.equal(pendingCalls, 0); - assert.equal(stub.requests.length, 1); - const [created] = stub.requests; - assert.equal(created.method, "POST"); - assert.equal(created.url, "/api/host-pairing"); - assert.equal(created.headers.authorization, "Bearer user-token"); - assert.deepEqual(created.body, { name: "Host", version: "2.0.3" }); - }); - - it("polls until the request is approved", async function() { - stub.handler = approveAfter(2); - const codes = []; - const result = await pairHost({ - controllerUrl: stub.url, name: "Host", pollIntervalMs: 10, - onPending: ({ code }) => { codes.push(code); }, - }); - assert.deepEqual(result, { hostId: 1234, token: "host-token", name: "Approved" }); - assert.deepEqual(codes, ["ABC-DEF"]); - assert.deepEqual(stub.requests.map(r => r.url), [ - "/api/host-pairing", "/api/host-pairing/poll", "/api/host-pairing/poll", "/api/host-pairing/poll", - ]); - assert.equal(stub.requests[0].headers.authorization, undefined); - assert.deepEqual(stub.requests[0].body, { name: "Host", version: "" }); - for (const poll of stub.requests.slice(1)) { - assert.deepEqual(poll.body, { requestId: "secret" }); - } - }); - - ["sub", "sub/"].forEach(controllerPath => { - it(`resolves routes under a controller URL ending in ${controllerPath}`, async function() { - stub.handler = ({ url }) => (url === "/sub/api/host-pairing" - ? [200, { hostId: 5, token: "host-token", name: "Host" }] - : [404, {}] - ); - const result = await pairHost({ controllerUrl: `${stub.url}${controllerPath}`, name: "Host" }); - assert.equal(result.hostId, 5); - }); - }); - - it("throws the error from the controller when the request is rejected", async function() { - const handler = approveAfter(1); - stub.handler = entry => { - if (entry.url === "/api/host-pairing/poll" && stub.requests.length > 2) { - return [403, { error: "Request was rejected" }]; - } - return handler(entry); - }; - await assert.rejects( - pairHost({ controllerUrl: stub.url, name: "Host", pollIntervalMs: 10 }), - err => err instanceof PairingError && err.status === 403 && err.message === "Request was rejected", - ); - }); - - it("throws if the controller does not support pairing", async function() { - stub.handler = () => [404, undefined]; - await assert.rejects( - pairHost({ controllerUrl: stub.url, name: "Host" }), - err => err instanceof PairingError && err.status === 404 && /must be updated/.test(err.message), - ); - }); - - it("throws on an invalid user token", async function() { - stub.handler = () => [401, { error: "Invalid user token" }]; - await assert.rejects( - pairHost({ controllerUrl: stub.url, name: "Host", token: "bad", onPending: () => assert.fail() }), - err => err instanceof PairingError && err.status === 401 && err.message === "Invalid user token", - ); - assert.equal(stub.requests.length, 1); - }); - - it("includes the retry time when rate limited", async function() { - stub.handler = () => [429, { error: "Too many requests from this address" }, { "Retry-After": "42" }]; - await assert.rejects( - pairHost({ controllerUrl: stub.url, name: "Host" }), - err => err instanceof PairingError && err.status === 429 && /try again in 42 seconds/.test(err.message), - ); - }); - - it("throws if the request expired", async function() { - const handler = approveAfter(0); - stub.handler = entry => (entry.url === "/api/host-pairing/poll" - ? [404, { error: "Request not found" }] - : handler(entry) - ); - await assert.rejects( - pairHost({ controllerUrl: stub.url, name: "Host", pollIntervalMs: 10 }), - err => err instanceof PairingError && err.status === 404 && err.message === "Request not found", - ); - }); - - it("cancels the request when aborted", async function() { - const controller = new AbortController(); - stub.handler = entry => { - if (entry.url === "/api/host-pairing/poll") { - controller.abort(); - } - if (entry.url === "/api/host-pairing/cancel") { - return [204, undefined]; - } - return approveAfter(Infinity)(entry); - }; - await assert.rejects( - pairHost({ controllerUrl: stub.url, name: "Host", pollIntervalMs: 10, signal: controller.signal }), - { name: "AbortError" }, - ); - const cancel = stub.requests.find(r => r.url === "/api/host-pairing/cancel"); - assert.deepEqual(cancel?.body, { requestId: "secret" }); - }); - - it("does not send anything if already aborted", async function() { - await assert.rejects( - pairHost({ controllerUrl: stub.url, name: "Host", signal: AbortSignal.abort() }), - { name: "AbortError" }, - ); - assert.equal(stub.requests.length, 0); - }); - - it("throws on invalid controller URLs", async function() { - for (const controllerUrl of ["localhost:8080", "ftp://localhost/", "not a url"]) { - await assert.rejects(pairHost({ controllerUrl, name: "Host" }), PairingError); - } - }); - - it("trusts the certificate passed as ca", async function() { - const [key, cert] = await Promise.all([ - fs.readFile(path.join(tlsDir, "key.pem")), - fs.readFile(path.join(tlsDir, "cert.pem")), - ]); - const httpsStub = await startStub(handler => https.createServer({ key, cert }, handler)); - try { - httpsStub.handler = () => [200, { hostId: 5, token: "host-token", name: "Host" }]; - const result = await pairHost({ controllerUrl: httpsStub.url, name: "Host", ca: cert }); - assert.equal(result.hostId, 5); - - // Passing a different certificate replaces the trusted ones, proving ca is used. - await assert.rejects( - pairHost({ controllerUrl: httpsStub.url, name: "Host", ca: tls.rootCertificates[0] }), - err => typeof err.code === "string" && !(err instanceof PairingError), - ); - } finally { - await httpsStub.close(); - } - }); - }); - - describe("checkHostToken()", function() { - it("returns true if the controller accepts the token", async function() { - stub.handler = () => [200, { valid: true }]; - assert.equal(await checkHostToken({ controllerUrl: stub.url, token: "host-token" }), true); - assert.equal(stub.requests[0].method, "GET"); - assert.equal(stub.requests[0].url, "/api/host-pairing/check"); - assert.equal(stub.requests[0].headers.authorization, "Bearer host-token"); - }); - - it("returns false if the controller does not accept the token", async function() { - stub.handler = () => [401, undefined]; - assert.equal(await checkHostToken({ controllerUrl: stub.url, token: "host-token" }), false); - }); - - it("throws on other responses", async function() { - stub.handler = () => [503, { error: "Controller is in recovery mode" }]; - await assert.rejects( - checkHostToken({ controllerUrl: stub.url, token: "host-token" }), - err => err instanceof PairingError && err.status === 503, - ); - }); - }); -}); - -describe("create/pair createLocalHost()", function() { - let directory; - let secret; - beforeEach(async function() { - directory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-pair-")); - secret = crypto.randomBytes(256).toString("base64"); - }); - afterEach(async function() { - await fs.rm(directory, { recursive: true, force: true, maxRetries: 10 }); - }); - - // Written with the controller's own config class to match its file format. - async function writeControllerConfig(fields = {}) { - const configPath = path.join(directory, "config-controller.json"); - const config = new lib.ControllerConfig("controller", undefined, configPath); - config.set("controller.auth_secret", secret); - for (const [name, value] of Object.entries(fields)) { - config.set(name, value); - } - await config.save(true); - } - - async function loadHosts(databaseDirectory = "database") { - const provider = new lib.JsonIdDatastoreProvider( - path.join(directory, databaseDirectory, "hosts.json"), HostRecord.fromJSON.bind(HostRecord), - ); - return await provider.load(); - } - - it("adds a host the controller loads with a token it accepts", async function() { - await writeControllerConfig(); - const before = Math.floor(Date.now() / 1000); - const result = await createLocalHost({ directory, name: "local" }); - assert.equal(result.name, "local"); - assert(Number.isInteger(result.hostId) && result.hostId >= 1 && result.hostId < 2 ** 31); - - const hosts = await loadHosts(); - assert.deepEqual([...hosts.keys()], [result.hostId]); - const record = hosts.get(result.hostId); - assert.equal(record.name, "local"); - assert.equal(record.version, ""); - assert.deepEqual(record.plugins, new Map()); - assert(record.tokenValidAfter >= before); - - const payload = jwt.verify(result.token, Buffer.from(secret, "base64"), { audience: "host" }); - assert.equal(payload.host, result.hostId); - assert(payload.iat >= record.tokenValidAfter, "token must not be issued before token_valid_after"); - }); - - it("appends to an existing database in the configured directory", async function() { - await writeControllerConfig({ "controller.database_directory": "db" }); - await fs.mkdir(path.join(directory, "db")); - await fs.writeFile(path.join(directory, "db", "hosts.json"), JSON.stringify([ - { id: 1, name: "existing", version: "2.0.0", plugins: {} }, - ])); - const result = await createLocalHost({ directory, name: "local", id: 7, version: "2.0.3" }); - assert.equal(result.hostId, 7); - - const hosts = await loadHosts("db"); - assert.deepEqual([...hosts.keys()], [1, 7]); - assert.equal(hosts.get(1).name, "existing"); - assert.equal(hosts.get(7).version, "2.0.3"); - await assert.rejects(fs.access(path.join(directory, "database")), { code: "ENOENT" }); - }); - - it("signs a new token for an existing host with the same id", async function() { - await writeControllerConfig(); - const first = await createLocalHost({ directory, name: "first", id: 7 }); - const before = await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"); - const second = await createLocalHost({ directory, name: "second", id: 7 }); - assert.equal(second.hostId, 7); - assert.equal(second.name, "first", "the name of the existing host should be kept"); - assert.equal(await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"), before); - - const record = (await loadHosts()).get(7); - for (const token of [first.token, second.token]) { - const payload = jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }); - assert.equal(payload.host, 7); - assert(payload.iat >= record.tokenValidAfter); - } - }); - - it("throws if the controller config has no auth secret", async function() { - await fs.writeFile(path.join(directory, "config-controller.json"), "{}"); - await assert.rejects(createLocalHost({ directory, name: "local" }), /controller\.auth_secret/); - }); -}); diff --git a/test/host/pairing.js b/test/host/pairing.js new file mode 100644 index 00000000..7bbede3a --- /dev/null +++ b/test/host/pairing.js @@ -0,0 +1,295 @@ +import assert from "node:assert/strict"; +import events from "node:events"; +import fs from "node:fs/promises"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; + +import * as lib from "@clusterio/lib"; +import { PairingError, checkHostToken, pairHost } from "@clusterio/host"; +import { ensurePaired } from "@clusterio/host/dist/node/src/pairing.js"; + +import "../setup_logging.js"; + +// Stub controller recording requests, responding with what the test's handler returns. +async function startStub() { + const stub = { + requests: [], + handler: () => [500, { error: "No handler" }], + }; + stub.server = http.createServer(async (req, res) => { + const chunks = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const entry = { + method: req.method, + url: req.url, + headers: req.headers, + body: chunks.length ? JSON.parse(Buffer.concat(chunks)) : undefined, + }; + stub.requests.push(entry); + const [status, json, headers = {}] = await stub.handler(entry); + res.writeHead(status, { "Content-Type": "application/json", ...headers }); + res.end(json === undefined ? undefined : JSON.stringify(json)); + }); + stub.server.listen(0, "127.0.0.1"); + await events.once(stub.server, "listening"); + stub.url = `http://127.0.0.1:${stub.server.address().port}/`; + stub.close = async () => { + stub.server.closeAllConnections(); + stub.server.close(); + await events.once(stub.server, "close"); + }; + return stub; +} + +const approved = { hostId: 1234, token: "host-token", name: "Approved" }; + +// Handler for requests that are approved after the given number of pending polls. +function approveAfter(polls, result = approved) { + let pending = 0; + let requests = 0; + return ({ url }) => { + if (url === "/api/host-pairing") { + requests += 1; + return [202, { requestId: `secret-${requests}`, code: `ABC-DE${requests}` }]; + } + if (url === "/api/host-pairing/poll") { + pending += 1; + return pending > polls ? [200, result] : [202, { status: "pending" }]; + } + if (url === "/api/host-pairing/cancel") { + return [204, undefined]; + } + return [404, { error: "Not found" }]; + }; +} + +describe("host/src/pairing", function() { + let stub; + beforeEach(async function() { + stub = await startStub(); + }); + afterEach(async function() { + await stub.close(); + }); + + function pair(options = {}) { + return pairHost({ controllerUrl: stub.url, name: "Host", pollIntervalMs: 10, retryDelayMs: 10, ...options }); + } + + describe("pairHost()", function() { + it("returns the host when approved with a user token", async function() { + stub.handler = ({ headers }) => (headers.authorization === "Bearer user-token" + ? [200, approved] + : [401, { error: "Invalid user token" }] + ); + const result = await pair({ token: "user-token", onPending: () => assert.fail("should not wait") }); + assert.deepEqual(result, approved); + assert.equal(stub.requests.length, 1); + }); + + it("polls until the request is approved", async function() { + stub.handler = approveAfter(2); + const codes = []; + const result = await pair({ version: "2.1.0", onPending: code => { codes.push(code); } }); + assert.deepEqual(result, approved); + assert.deepEqual(codes, ["ABC-DE1"]); + assert.deepEqual(stub.requests[0].body, { name: "Host", version: "2.1.0" }); + assert.equal(stub.requests[0].headers.authorization, undefined); + for (const poll of stub.requests.slice(1)) { + assert.deepEqual(poll.body, { requestId: "secret-1" }); + } + }); + + it("resolves routes under the path of the controller URL", async function() { + stub.handler = ({ url }) => (url === "/sub/api/host-pairing" ? [200, approved] : [404, {}]); + const result = await pair({ controllerUrl: `${stub.url}sub/` }); + assert.equal(result.hostId, 1234); + }); + + it("makes a new request when the previous one expired", async function() { + const handler = approveAfter(0); + let expired = false; + stub.handler = entry => { + if (entry.url === "/api/host-pairing/poll" && !expired) { + expired = true; + return [404, { error: "Request not found" }]; + } + return handler(entry); + }; + const codes = []; + const result = await pair({ onPending: code => { codes.push(code); } }); + assert.deepEqual(result, approved); + assert.deepEqual(codes, ["ABC-DE1", "ABC-DE2"]); + }); + + it("waits and tries again when the controller is busy", async function() { + const handler = approveAfter(0); + let busy = 2; + stub.handler = entry => { + if (entry.url === "/api/host-pairing" && busy > 0) { + busy -= 1; + return busy ? [429, { error: "Too many requests" }] : [503, { error: "Recovery mode" }]; + } + return handler(entry); + }; + const retries = []; + const result = await pair({ onRetry: err => { retries.push(err.message); } }); + assert.deepEqual(result, approved); + assert.deepEqual(retries, ["Too many requests", "Recovery mode"]); + }); + + it("keeps trying while the controller can't be reached", async function() { + const { url } = stub; + await stub.close(); + const abort = new AbortController(); + const errors = []; + await assert.rejects( + pairHost({ + controllerUrl: url, name: "Host", retryDelayMs: 10, signal: abort.signal, + onRetry: err => { + errors.push(err); + if (errors.length === 3) { + abort.abort(); + } + }, + }), + { name: "AbortError" }, + ); + assert.equal(errors.length, 3); + stub = await startStub(); + }); + + it("throws when the request is rejected", async function() { + const handler = approveAfter(Infinity); + stub.handler = entry => (entry.url === "/api/host-pairing/poll" && stub.requests.length > 2 + ? [403, { error: "Request was rejected" }] + : handler(entry) + ); + await assert.rejects( + pair(), + err => err instanceof PairingError && err.status === 403 && err.message === "Request was rejected", + ); + }); + + it("throws if the controller does not support pairing", async function() { + stub.handler = () => [404, undefined]; + await assert.rejects( + pair(), + err => err instanceof PairingError && err.status === 404 && /must be updated/.test(err.message), + ); + }); + + it("throws on an invalid user token without waiting", async function() { + stub.handler = () => [401, { error: "Invalid user token" }]; + await assert.rejects( + pair({ token: "bad" }), + err => err instanceof PairingError && err.status === 401 && err.message === "Invalid user token", + ); + assert.equal(stub.requests.length, 1); + }); + + it("cancels the request when aborted", async function() { + const abort = new AbortController(); + const handler = approveAfter(Infinity); + stub.handler = entry => { + if (entry.url === "/api/host-pairing/poll") { + abort.abort(); + } + return handler(entry); + }; + await assert.rejects(pair({ signal: abort.signal }), { name: "AbortError" }); + const cancel = stub.requests.find(r => r.url === "/api/host-pairing/cancel"); + assert.deepEqual(cancel?.body, { requestId: "secret-1" }); + }); + + it("does not send anything if already aborted", async function() { + await assert.rejects(pair({ signal: AbortSignal.abort() }), { name: "AbortError" }); + assert.equal(stub.requests.length, 0); + }); + }); + + describe("checkHostToken()", function() { + it("returns true if the controller accepts the token", async function() { + stub.handler = ({ url, headers }) => ( + url === "/api/host-pairing/check" && headers.authorization === "Bearer host-token" + ? [200, { valid: true }] + : [401, undefined] + ); + assert.equal(await checkHostToken(stub.url, "host-token"), true); + assert.equal(await checkHostToken(stub.url, "other-token"), false); + }); + + it("throws on other responses", async function() { + stub.handler = () => [404, undefined]; + await assert.rejects(checkHostToken(stub.url, "host-token"), PairingError); + }); + }); + + describe("ensurePaired()", function() { + let directory; + let config; + beforeEach(async function() { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-pairing-")); + config = new lib.HostConfig("host", undefined, path.join(directory, "config-host.json")); + config.set("host.name", "My Host"); + config.set("host.controller_url", stub.url); + }); + afterEach(async function() { + await fs.rm(directory, { recursive: true, force: true, maxRetries: 10 }); + }); + + async function savedConfig() { + return JSON.parse(await fs.readFile(path.join(directory, "config-host.json"), "utf8")); + } + + it("waits for approval and saves the host when not paired", async function() { + this.timeout(5000); // Polls at the default interval of 2 seconds + stub.handler = approveAfter(0); + await ensurePaired(config, { version: "2.1.0" }); + assert.deepEqual(stub.requests[0].body, { name: "My Host", version: "2.1.0" }); + const saved = await savedConfig(); + assert.equal(saved["host.id"], 1234); + assert.equal(saved["host.name"], "Approved"); + assert.equal(saved["host.controller_token"], "host-token"); + }); + + it("does nothing when the controller accepts the saved token", async function() { + config.set("host.controller_token", "valid-token"); + stub.handler = ({ headers }) => (headers.authorization === "Bearer valid-token" + ? [200, { valid: true }] + : [401, undefined] + ); + await ensurePaired(config, { version: "2.1.0" }); + assert.deepEqual(stub.requests.map(r => r.url), ["/api/host-pairing/check"]); + assert.equal(config.get("host.controller_token"), "valid-token"); + }); + + it("asks to join again with the user token when the saved token is not accepted", async function() { + config.set("host.controller_token", "revoked-token"); + stub.handler = ({ url, headers }) => { + if (url === "/api/host-pairing/check") { + return [401, undefined]; + } + return headers.authorization === "Bearer user-token" + ? [200, { hostId: 5, token: "new-token", name: "My Host" }] + : [401, { error: "Invalid user token" }]; + }; + await ensurePaired(config, { version: "2.1.0", userToken: "user-token" }); + const saved = await savedConfig(); + assert.equal(saved["host.id"], 5); + assert.equal(saved["host.controller_token"], "new-token"); + assert(!JSON.stringify(saved).includes("user-token"), "user token must not be saved"); + }); + + it("keeps the saved token when it can't be checked", async function() { + config.set("host.controller_token", "some-token"); + stub.handler = () => [404, undefined]; + await ensurePaired(config, { version: "2.1.0" }); + assert.equal(config.get("host.controller_token"), "some-token"); + assert.equal(stub.requests.length, 1); + }); + }); +}); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 67d99bf9..9eeaf741 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -15,7 +15,7 @@ import { spawnNode, instancesDir, factorioDir, databaseDir, controllerConfigPath, requiresFactorio, hasFactorio, pairTestHost, writeHostConfig, } from "./index.js"; -import { pairHost, checkHostToken, PairingError } from "../../packages/create/pair.js"; +import { pairHost, checkHostToken, PairingError } from "@clusterio/host"; import { externalTest, slowTest } from "../common.js"; import { getTestRconPort } from "./rcon_port.js"; import { ControllerEcho, HostEchoReceived, InstanceEcho } from "../file/test_plugin/messages.js"; @@ -718,7 +718,7 @@ describe("Integration of Clusterio", function() { describe("host pending, approve and reject", function() { let pairings = []; afterEach(async function() { - // Stop installers left waiting by a failed test + // Stop hosts left waiting by a failed test for (const pairing of pairings) { pairing.abort.abort(); } @@ -726,7 +726,7 @@ describe("Integration of Clusterio", function() { pairings = []; }); - // Start pairing like an installer without a token, returns once the request is pending. + // Start pairing like a host without a user token, returns once the request is pending. async function requestPairing(name) { const abort = new AbortController(); let onPending; @@ -740,7 +740,7 @@ describe("Integration of Clusterio", function() { throw new Error("Pairing finished before the request was pending"); }); ended.catch(() => {}); // Only of interest while racing the pending request - ({ code: pairing.code } = await Promise.race([pending, ended])); + pairing.code = await Promise.race([pending, ended]); return pairing; } @@ -776,12 +776,12 @@ describe("Integration of Clusterio", function() { assert(!requests.some(request => request.code === pairing.code), "Approved request is still pending"); const hosts = await control.send(new lib.HostListRequest()); assert.equal(hosts.find(host => host.id === approved.hostId)?.name, "approved-host"); - assert.equal(await checkHostToken({ controllerUrl: url, token: approved.token }), true); + assert.equal(await checkHostToken(url, approved.token), true); await execCtl(`host revoke ${approved.hostId}`); }); - it("should make the installer fail when the request is rejected", async function() { + it("should make the host fail when the request is rejected", async function() { const pairing = await requestPairing("rejected-host"); await execCtl(`host reject ${pairing.code}`); await assert.rejects(pairing.result, pairingError(403)); @@ -792,7 +792,7 @@ describe("Integration of Clusterio", function() { assert(!hosts.some(host => host.name === "rejected-host"), "Rejected host was added"); }); - it("should remove the request when the installer stops waiting", async function() { + it("should remove the request when the host stops waiting", async function() { const pairing = await requestPairing("cancelled-host"); pairing.abort.abort(); await assert.rejects(pairing.result, err => err.name === "AbortError"); @@ -836,7 +836,7 @@ describe("Integration of Clusterio", function() { describe("host revoke", function() { it("should remove the host and invalidate its token", async function() { const pairing = await pairTestHost("revokee-host"); - assert.equal(await checkHostToken({ controllerUrl: url, token: pairing.token }), true); + assert.equal(await checkHostToken(url, pairing.token), true); await execCtl(`note set host ${pairing.hostId} short-lived`); const control = getControl(); @@ -848,7 +848,7 @@ describe("Integration of Clusterio", function() { ); const hosts = await control.send(new lib.HostListRequest()); assert(!hosts.some(host => host.id === pairing.hostId), "Host was not removed"); - assert.equal(await checkHostToken({ controllerUrl: url, token: pairing.token }), false); + assert.equal(await checkHostToken(url, pairing.token), false); const notes = await control.send(new lib.NoteListRequest()); assert(!notes.some(note => note.id === `host/${pairing.hostId}`), "Note of the host was not removed"); }); @@ -880,7 +880,7 @@ describe("Integration of Clusterio", function() { assert(!hosts.some(host => host.id === pairing.hostId), "Host was not removed"); }); - it("should stop a running host and prevent it from starting again", async function() { + it("should stop a running host and have it ask to join again when started", async function() { slowTest(this); this.timeout(30000); const config = "revoke-host-config.json"; @@ -905,13 +905,34 @@ describe("Integration of Clusterio", function() { } } - // Exit code 8 tells the run scripts not to restart the host - await assert.rejects(execHost(`--config ${config} run`), err => { - const output = `${err.stdout}${err.stderr}`; - assert.equal(err.code, 8, `Unexpected exit code ${err.code}:\n${output}`); - assert.match(output, /host is not registered/); - return true; - }); + // Starting it again asks to join the cluster again, and it starts up once approved + const rejoining = await spawnNode( + "revoke-host:", `../../packages/host run --config ${config}`, /Waiting for approval/ + ); + const exited = events.once(rejoining, "exit"); + try { + const started = new Promise(resolve => { + rejoining.stdout.on("data", chunk => { + if (String(chunk).includes("Started host")) { + resolve(); + } + }); + }); + const requests = await getControl().send(new lib.HostPairingListRequest()); + const request = requests.find(entry => entry.name === "running-revokee"); + assert(request, "Revoked host did not ask to join again"); + const newHostId = await getControl().send(new lib.HostPairingApproveRequest(request.code)); + assert.notEqual(newHostId, pairing.hostId); + assert.notEqual( + await lib.timeout(started, 10e3, "timeout"), "timeout", "Host did not start once approved" + ); + await execCtl(`host revoke ${newHostId}`); + await lib.timeout(exited, 10e3, "timeout"); + } finally { + if (rejoining.exitCode === null && rejoining.signalCode === null) { + rejoining.kill("SIGINT"); + } + } }); }); diff --git a/test/integration/index.js b/test/integration/index.js index d6aa7943..c28b9a7e 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -12,7 +12,7 @@ import { LineSplitter, logger } from "@clusterio/lib"; import { selectTargetCommand, initialize as initializeCtl } from "@clusterio/ctl"; import { _listFactorioVersions } from "@clusterio/host/dist/node/src/server.js"; -import { pairHost } from "../../packages/create/pair.js"; +import { pairHost } from "@clusterio/host"; import "../setup_logging.js"; @@ -276,7 +276,7 @@ export async function pairTestHost(name) { } /** - * Write a new host config for a paired host, with the fields the installer sets + * Write a new host config for a paired host, with the fields the installer and pairing set * * @param {string} configPath - Path to write the config to. * @param {{ hostId: number, token: string, name: string }} pairing - Host returned by pairing. diff --git a/test/integration/link.js b/test/integration/link.js index 0580af8a..012d545d 100644 --- a/test/integration/link.js +++ b/test/integration/link.js @@ -162,7 +162,7 @@ describe("Integration of lib/link", function() { await assert.rejects( hostConnector.connect(), new Error( - "Authentication failed: host is not registered, run the installer again in the host directory" + "Authentication failed: host is not registered, start the host again to request to join the cluster" ) ); }); From 13c75b9e35d7cd2ec32995317a1f3cd34341c759 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 21:23:03 +0100 Subject: [PATCH 05/44] Link to an approval dialog from the waiting host A host waiting for approval logs a link to the Hosts page with the code in it. Opening it shows a dialog with the code, the address of the host, a name prefilled with the requested one and a large Approve button. A code that is no longer waiting shows a message saying so. --- README.md | 6 +- packages/create/README.md | 7 +- packages/host/src/pairing.ts | 9 +- packages/web_ui/src/components/HostsPage.tsx | 130 ++++++++++++++++++- 4 files changed, 143 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7f1b47db..9b5bb0c5 100644 --- a/README.md +++ b/README.md @@ -276,9 +276,11 @@ Run the installer as described in the installation section and choose "Host only The URL is the same as what is needed to connect to the web interface. Then start the host, the first time it starts it prints a code and waits for the host to be approved: - Waiting for approval. Approve "my-host" with code ABC-DEF on the Hosts page of https://example.com/, or run: npx clusterioctl host approve ABC-DEF + Waiting for approval of "my-host" with code ABC-DEF, open this link to approve it: + https://example.com/hosts?approve=ABC-DEF + Or run: npx clusterioctl host approve ABC-DEF -Approve it in the web interface by clicking Add host on the Hosts page and approving the request with the same code, or with `npx clusterioctl host approve `. +Open the link to approve it in the web interface, where you can also change its name, or click Add host on the Hosts page and approve the request with the same code, or run `npx clusterioctl host approve `. The host then saves its connection details and starts up, it should show up in the Hosts list and be available for assigning and running instances on. Only approve a request if its code matches the one shown by the host, approving gives the host full access to the cluster. Approving requires the `core.host.approve` permission, which the Cluster Admin role has. diff --git a/packages/create/README.md b/packages/create/README.md index afbb84b2..7d8ed1d1 100644 --- a/packages/create/README.md +++ b/packages/create/README.md @@ -59,9 +59,12 @@ A new host has to be approved before it can join the cluster: 2. Start the host with `run-host.cmd`, `run-host.sh` or the systemd service in the `systemd` folder. The first time it starts it prints a code and waits for the host to be approved: - Waiting for approval. Approve "my-host" with code ABC-DEF on the Hosts page of https://example.com/, or run: npx clusterioctl host approve ABC-DEF + Waiting for approval of "my-host" with code ABC-DEF, open this link to approve it: + https://example.com/hosts?approve=ABC-DEF + Or run: npx clusterioctl host approve ABC-DEF -3. Approve the request by clicking Add host on the Hosts page of the web interface, or run `npx clusterioctl host approve ABC-DEF`. +3. Open the link to approve the request in the web interface, where its name can be changed first. + It can also be approved under Add host on the Hosts page, or with `npx clusterioctl host approve ABC-DEF`. Only approve a request if its code matches the one shown by the host. Approving requires the `core.host.approve` permission. diff --git a/packages/host/src/pairing.ts b/packages/host/src/pairing.ts index 6d320f4c..8bd41bc2 100644 --- a/packages/host/src/pairing.ts +++ b/packages/host/src/pairing.ts @@ -275,10 +275,11 @@ export async function ensurePaired( token: options.userToken, signal: options.signal, onPending: code => { - logger.info( - `Waiting for approval. Approve "${name}" with code ${code} on the Hosts page of ` + - `${controllerUrl}, or run: npx clusterioctl host approve ${code}` - ); + const approveUrl = new URL(`hosts?approve=${code}`, controllerUrl).href; + // The link is logged on its own so terminals can make it clickable without it wrapping. + logger.info(`Waiting for approval of "${name}" with code ${code}, open this link to approve it:`); + logger.info(approveUrl); + logger.info(`Or run: npx clusterioctl host approve ${code}`); }, onRetry: (err, delayMs) => { logger.warn(`Requesting to join failed: ${err.message}, trying again in ${Math.ceil(delayMs / 1000)}s`); diff --git a/packages/web_ui/src/components/HostsPage.tsx b/packages/web_ui/src/components/HostsPage.tsx index e31095a0..9be610d8 100644 --- a/packages/web_ui/src/components/HostsPage.tsx +++ b/packages/web_ui/src/components/HostsPage.tsx @@ -1,5 +1,8 @@ import React, { useContext, useState, useEffect } from "react"; -import { Badge, Button, Flex, Input, Modal, Space, Switch, Table, Tooltip, Typography } from "antd"; +import { useSearchParams } from "react-router-dom"; +import { + Badge, Button, Descriptions, Flex, Input, Modal, Result, Space, Spin, Switch, Table, Tooltip, Typography, +} from "antd"; import { ExclamationCircleOutlined, CopyOutlined, PlusOutlined } from "@ant-design/icons"; import * as lib from "@clusterio/lib"; @@ -200,6 +203,120 @@ function AddHostModal(props: { open: boolean, onClose: () => void }) { ; } +/** Format a code like the controller does, matching any case and without the dash */ +function normaliseCode(code: string) { + const plain = code.toUpperCase().replace(/[^0-9A-Z]/g, ""); + return `${plain.slice(0, 3)}-${plain.slice(3)}`; +} + +/** Dialog opened by the link a waiting host prints, for approving that host */ +function ApproveHostModal(props: { code: string, onClose: () => void }) { + const control = useContext(ControlContext); + const account = useAccount(); + const [requests, synced] = useHostPairings(); + const code = normaliseCode(props.code); + const request = requests.get(code); + const [name, setName] = useState(); + const [busy, setBusy] = useState(false); + const now = useNow(true); + const permitted = account.hasPermission("core.host.approve"); + const hostName = (name ?? request?.name ?? "").trim(); + + function approve() { + if (!request) { + return; + } + setBusy(true); + control.send( + new lib.HostPairingApproveRequest(request.code, hostName !== request.name ? hostName : undefined) + ).then(() => { + notify(`Approved host ${hostName}`, "success"); + props.onClose(); + }).catch( + notifyErrorHandler(`Error approving host ${request.name}`) + ).finally(() => { + setBusy(false); + }); + } + + function reject() { + if (!request) { + return; + } + setBusy(true); + control.send(new lib.HostPairingRejectRequest(request.code)).then(() => { + notify(`Rejected host ${request.name}`); + props.onClose(); + }).catch( + notifyErrorHandler(`Error rejecting host ${request.name}`) + ).finally(() => { + setBusy(false); + }); + } + + let content; + if (!permitted) { + content = ; + } else if (!request && !synced) { + content = ; + } else if (!request) { + content = No host is waiting with code {code}} + subTitle={ + "It may have been approved or rejected already, or stopped waiting. " + + "A host that is still waiting shows a new code." + } + />; + } else { + content = <> + + A host is asking to join the cluster. + Only approve it if the code below matches the code shown by the host, + an approved host has full access to the cluster. + + + {request.code} + + + {request.remoteAddress} + {request.version && {request.version}} + + {formatDuration(Math.max(0, now - request.createdAtMs))} + + + Name + { setName(event.target.value); }} + onPressEnter={approve} + style={{ margin: "4px 0 24px" }} + /> + + + + + ; + } + + return + {content} + ; +} + function AddHostButton() { const [open, setOpen] = useState(false); const [requests] = useHostPairings(); @@ -266,8 +383,19 @@ export default function HostsPage() { }); const nameSearch = useColumnSearch(tableState, "name", host => host.name, "Search hosts"); const rowNav = useRowNavigation(); + const [params, setParams] = useSearchParams(); + const approveCode = params.get("approve"); + + function closeApprove() { + setParams(current => { + const next = new URLSearchParams(current); + next.delete("approve"); + return next; + }, { replace: true }); + } return + {approveCode !== null && } : undefined} From 1ed46dff9c30868f9f84a70e224f2299e496fcec Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 21:49:53 +0100 Subject: [PATCH 06/44] Fix issues found in review of host pairing - Saves of a datastore run one at a time, and changes made while saving are saved by the next save. Overlapping saves from approving and revoking hosts could write an older snapshot after a newer one. - Map subscribers fetch everything again when a new session starts, as the controller only sends values that exist. A host revoked while a browser was disconnected stayed in its list. - The host token check accepts the same pre-rename tokens as the WebSocket, so upgrading a host with such a token does not make it ask to join again. --- packages/controller/src/Controller.ts | 24 ++++++++++++++++++ packages/controller/src/WsServer.ts | 17 +------------ packages/controller/src/routes.ts | 12 +-------- packages/lib/src/datastore.ts | 19 ++++++++++++--- packages/lib/src/subscriptions.ts | 10 ++++++++ test/controller/routes.js | 6 +++++ test/lib/datastore.js | 35 +++++++++++++++++++++++++++ test/lib/subscriptions.test.js | 28 +++++++++++++++++++++ 8 files changed, 121 insertions(+), 30 deletions(-) diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 5908c62e..b2341cc2 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -1025,6 +1025,30 @@ export default class Controller { return jwt.sign({ aud: "host", host: hostId }, this.authSecret); } + /** + * Check a host token + * + * @param token - Host token to check. + * @returns ID of the host the token was issued to. + * @throws if the token is invalid, its host is not registered or it was issued before tokenValidAfter. + */ + verifyHostToken(token: string): number { + // migrate: allow pre-rename tokens issued to hosts before alpha-14 + const payload = jwt.verify(token, this.authSecret, { audience: ["host", "slave"] }); + if (typeof payload === "string") { + throw new Error("unexpected JsonWebToken type"); + } + const hostId = payload.host !== undefined ? payload.host : payload.slave; + const host = this.hosts.get(hostId); + if (!host) { + throw new Error("host is not registered, start the host again to request to join the cluster"); + } + if (!payload.iat || payload.iat < host.tokenValidAfter) { + throw new Error("invalid token"); + } + return hostId; + } + hostsUpdated(hosts: HostRecord[]) { let updates = hosts.map(host => host.toHostDetails()); this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates)); diff --git a/packages/controller/src/WsServer.ts b/packages/controller/src/WsServer.ts index fea731a6..2d88b91c 100644 --- a/packages/controller/src/WsServer.ts +++ b/packages/controller/src/WsServer.ts @@ -252,24 +252,9 @@ ${err.stack}` } try { - // migrate: allow pre-rename tokens issued to hosts before alpha-14 - const tokenPayload = jwt.verify(data.token, this.controller.authSecret, { audience: ["host", "slave"] }); - if (typeof tokenPayload === "string") { - throw new Error("unexpected JsonWebToken type"); - } - - // migrate: allow pre-rename tokens issued to hosts before alpha-14 - if ((tokenPayload.host !== undefined ? tokenPayload.host : tokenPayload.slave) !== data.id) { + if (this.controller.verifyHostToken(data.token) !== data.id) { throw new Error("missmatched host id"); } - - let host = this.controller.hosts.get(data.id); - if (!host) { - throw new Error("host is not registered, start the host again to request to join the cluster"); - } - if (!tokenPayload.iat || tokenPayload.iat < host.tokenValidAfter) { - throw new Error("invalid token"); - } } catch (err: any) { logger.verbose(`WsServer | authentication failed for ${this.remoteAddr(req)}`); wsRejectedConnectionsCounter.inc(); diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index b9ce7803..fa967fe2 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -189,17 +189,7 @@ function validateHostToken(req: Request, res: Response, next: any) { } try { - const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "host" }); - if (typeof tokenPayload === "string") { - throw new Error("unexpected JsonWebToken type"); - } - let host = req.app.locals.controller.hosts.get(tokenPayload.host); - if (!host) { - throw new Error("invalid host"); - } - if (!tokenPayload.iat || tokenPayload.iat < (host.tokenValidAfter ?? 0)) { - throw new Error("invalid token"); - } + req.app.locals.controller.verifyHostToken(token); } catch (err) { res.sendStatus(401); return; diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 212a7bbb..59be8d96 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -166,6 +166,7 @@ export abstract class Datastore< "update": [ updates: U[] ], }> { private dirty = false; + private saving: Promise = Promise.resolve(); constructor( private provider: DatastoreProvider = new MemoryDatastoreProvider(), @@ -181,10 +182,22 @@ export abstract class Datastore< // Save the datastore to the provider async save() { - if (this.dirty) { - await this.provider.save(this.data); + // Saves run one at a time so an older snapshot can't be written after a newer one. + const save = this.saving.then(async () => { + if (!this.dirty) { + return; + } + // Cleared before writing so changes made while saving are saved by the next save. this.dirty = false; - } + try { + await this.provider.save(this.data); + } catch (err) { + this.dirty = true; + throw err; + } + }); + this.saving = save.catch(() => {}); + await save; } // Load the datastore from the provider diff --git a/packages/lib/src/subscriptions.ts b/packages/lib/src/subscriptions.ts index cc66781e..9bb97d99 100644 --- a/packages/lib/src/subscriptions.ts +++ b/packages/lib/src/subscriptions.ts @@ -725,6 +725,16 @@ export class MapSubscriber< /** Current value of the subscribed resource */ values = new Map(); + override handleConnectionEvent(event: "connect" | "drop" | "resume" | "close") { + if (event === "connect") { + // A new session is only sent values that exist, not ones removed while disconnected, + // so everything is fetched again. + this.values.clear(); + this.lastUpdatedMs = -1; + } + super.handleConnectionEvent(event); + } + protected override getLastUpdatedTimeMs(event: E) { let maxTime = this.lastUpdatedMs; for (const update of event.updates) { diff --git a/test/controller/routes.js b/test/controller/routes.js index 32f427e9..b0b88cd2 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -221,6 +221,7 @@ describe("controller/src/routes", function() { controller.hosts = new SubscribableDatastore(); controller.hostPairing = new HostPairingManager(controller); controller.generateHostToken = Controller.prototype.generateHostToken; + controller.verifyHostToken = Controller.prototype.verifyHostToken; controller.wsServer.remoteAddr = req => req.socket.remoteAddress; adminToken = controller.users.signUserToken(controller.users.getByName("test")); playerToken = controller.users.signUserToken(controller.users.getByName("player")); @@ -478,6 +479,11 @@ describe("controller/src/routes", function() { } assert(controller.hosts.has(hostId)); }); + it("should respond with 200 for a host token from before the slave rename", async function() { + const { hostId } = await controller.hostPairing.createHost("Host A"); + const token = jwt.sign({ aud: "slave", slave: hostId }, controller.authSecret); + await assertResponse(await check({ "Authorization": `Bearer ${token}` }), 200, { valid: true }); + }); it("should respond with 401 without a valid host token", async function() { await controller.hostPairing.createHost("Host A"); assert.equal((await check({})).status, 401); diff --git a/test/lib/datastore.js b/test/lib/datastore.js index 4c829e76..5959bf0b 100644 --- a/test/lib/datastore.js +++ b/test/lib/datastore.js @@ -346,6 +346,41 @@ describe("lib/datastore", function() { await datastore.save(); assert.equal(datastoreProvider.methodCalls.length, 2); }); + it("should write overlapping saves in order with the latest data", async function() { + const written = []; + let release; + datastoreProvider.save = async data => { + const keys = [...data.keys()]; + if (!written.length) { + await new Promise(resolve => { release = resolve; }); + } + written.push(keys); + }; + datastore = new MockDatastore(datastoreProvider, new Map([["a", 1]])); + datastore.callTouch(); + const first = datastore.save(); + await new Promise(resolve => setImmediate(resolve)); // First write in progress + datastore.getData().set("b", 2); + datastore.callTouch(); + const second = datastore.save(); + release(); + await Promise.all([first, second]); + assert.deepEqual(written, [["a"], ["a", "b"]]); + }); + it("should save again after a failed save", async function() { + let fail = true; + datastoreProvider.save = async () => { + if (fail) { + fail = false; + throw new Error("disk full"); + } + datastoreProvider.methodCalls.push(["save"]); + }; + datastore.callTouch(); + await assert.rejects(datastore.save(), /disk full/); + await datastore.save(); + assert.deepEqual(datastoreProvider.methodCalls.at(-1), ["save"]); + }); }); describe("load", function() { diff --git a/test/lib/subscriptions.test.js b/test/lib/subscriptions.test.js index 461ca0ee..c71923f8 100644 --- a/test/lib/subscriptions.test.js +++ b/test/lib/subscriptions.test.js @@ -1584,6 +1584,34 @@ describe("lib/subscriptions", function() { assert.equal(snap2.has(2), true); }); + it("should fetch everything again when a new session starts", async function() { + subscriber.subscribe(() => {}); + await events.once(mockControl.connector, "send"); + await subscriber._handleEvent(new MapEvent([ + { id: 1, updatedAtMs: 10, isDeleted: false }, + ])); + const [before] = subscriber.getSnapshot(); + mockControl.connector.sentMessages.length = 0; + + mockControl.connector.emit("connect"); + assert.equal(subscriber.values.size, 0); + assert.notStrictEqual(subscriber.getSnapshot()[0], before); + await new Promise(resolve => setImmediate(resolve)); + const requests = mockControl.connector.sentMessages.filter(m => m.name === "SubscriptionRequest"); + assert.equal(requests.length, 1); + assert.equal(requests[0].data.lastRequestTimeMs, -1); + }); + + it("should keep values when a session is resumed", async function() { + await subscriber._handleEvent(new MapEvent([ + { id: 1, updatedAtMs: 10, isDeleted: false }, + ])); + subscriber.handleConnectionEvent("drop"); + subscriber.handleConnectionEvent("resume"); + assert.equal(subscriber.values.size, 1); + assert.equal(subscriber.lastUpdatedMs, 10); + }); + it("should handle multiple updates correctly", async function() { await subscriber._handleEvent(new MapEvent([ { id: 1, updatedAtMs: 10, isDeleted: false }, From a4894aae9dfad98cb525f85d3f9b77e978cfffdb Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 22:59:32 +0100 Subject: [PATCH 07/44] Simplify approving hosts The list of waiting hosts opens the approval dialog instead of having its own name input and buttons, so hosts are approved in one place. Removes the limit on requests per minute, the limits on pending requests already cap what an address can do. Also updates text that still said the installer makes the requests. --- docs/devs/protocol.md | 2 +- packages/controller/src/Controller.ts | 2 +- packages/controller/src/HostPairingManager.ts | 41 +++----- packages/controller/src/routes.ts | 6 +- packages/lib/src/data/HostPairingDetails.ts | 10 +- packages/lib/src/permissions.ts | 4 +- packages/web_ui/src/components/HostsPage.tsx | 96 +++---------------- test/controller/HostPairingManager.test.js | 44 +-------- 8 files changed, 37 insertions(+), 168 deletions(-) diff --git a/docs/devs/protocol.md b/docs/devs/protocol.md index ab028efa..747137d0 100644 --- a/docs/devs/protocol.md +++ b/docs/devs/protocol.md @@ -61,7 +61,7 @@ If a user token is sent and the user has the `core.host.approve` permission the An invalid token gives 401 and a user without the permission gives 403, a token is never ignored to create a pending request instead. A missing or too long name gives 400. -Too many requests from the same address or in total gives 429 with a `Retry-After` header in seconds. +Too many pending requests from the same address or in total gives 429 with a `Retry-After` header in seconds. ### `POST /api/host-pairing/poll` diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index b2341cc2..3bf2bdf4 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -92,7 +92,7 @@ export default class Controller { users: UserManager; /** Instance manager for the cluster, also contains access to the user records */ instances: InstanceManager; - /** Pending requests from host installers to join the cluster */ + /** Pending requests from hosts to join the cluster */ hostPairing: HostPairingManager; // Possible states are new, starting, running, stopping, stopped diff --git a/packages/controller/src/HostPairingManager.ts b/packages/controller/src/HostPairingManager.ts index 55293dbf..8dc3ad99 100644 --- a/packages/controller/src/HostPairingManager.ts +++ b/packages/controller/src/HostPairingManager.ts @@ -12,7 +12,7 @@ import HostRecord from "./HostRecord.js"; const codeAlphabet = "23456789ABCDEFGHJKMNPQRSTUVWXYZ"; const codeLength = 6; -/** Time a request is kept after the last poll from the installer */ +/** Time a request is kept after the last poll from the host */ export const pollTimeoutMs = 30e3; /** Maximum time a request can stay pending */ export const maxPendingMs = 60 * 60e3; @@ -20,21 +20,19 @@ export const maxPendingMs = 60 * 60e3; export const maxPending = 50; /** Maximum number of pending requests from one address */ export const maxPendingPerAddress = 3; -/** Maximum number of requests created from one address per minute */ -export const maxCreatesPerMinute = 10; /** Maximum length of a host name */ export const maxNameLength = 100; /** Control and bidirectional override characters refused in host names */ const nameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u; -/** Result of an approved request, sent to the installer */ +/** Result of an approved request, sent to the host */ export interface HostPairingResult { hostId: number, token: string, name: string, } -/** Error with an HTTP status code to respond to the installer with */ +/** Error with an HTTP status code to respond to the host with */ export class HostPairingError extends Error { constructor( message: string, @@ -47,7 +45,7 @@ export class HostPairingError extends Error { } interface PairingRequest { - /** Secret the installer polls with, never shown to users */ + /** Secret the host polls with, never shown to users */ secret: string, details: lib.HostPairingDetails, /** Address with IPv6 addresses grouped by /64, used for limits */ @@ -84,7 +82,7 @@ export function addressKey(address: string) { * Normalise a pairing code typed by a user * * @param code - Code as typed, may contain dashes and lower case letters. - * @returns code formatted as shown by the installer. + * @returns code formatted as shown by the host. */ export function normaliseCode(code: string) { const plain = code.toUpperCase().replace(/[^0-9A-Z]/g, ""); @@ -100,9 +98,9 @@ function generateCode() { } /** - * Keeps track of requests from host installers to join the cluster + * Keeps track of requests from hosts to join the cluster * - * Requests only live in memory, and only while the installer keeps polling + * Requests only live in memory, and only while the host keeps polling * for the result. */ export default class HostPairingManager { @@ -110,8 +108,6 @@ export default class HostPairingManager { requests = new lib.SubscribableDatastore(); private _bySecret = new Map(); private _byCode = new Map(); - /** Creation times of recent requests by address key */ - private _recentCreates = new Map(); private _pruneInterval?: ReturnType; constructor( @@ -129,7 +125,7 @@ export default class HostPairingManager { } /** - * Check and trim a host name given by an installer or a user + * Check and trim a host name given by a host or a user * * @param name - Name to check. * @returns the trimmed name. @@ -180,8 +176,8 @@ export default class HostPairingManager { /** * Create a pending request * - * @param name - Name the installer asked for. - * @param version - Version of the installer. + * @param name - Name the host asked for. + * @param version - Version of the host. * @param remoteAddress - Address the request came from. * @returns the secret to poll with and the code to show the user. */ @@ -189,12 +185,6 @@ export default class HostPairingManager { this.prune(); const key = addressKey(remoteAddress); const now = Date.now(); - const recent = (this._recentCreates.get(key) ?? []).filter(ms => ms > now - 60e3); - if (recent.length >= maxCreatesPerMinute) { - throw new HostPairingError( - "Too many requests from this address", 429, Math.ceil((recent[0] + 60e3 - now) / 1000), - ); - } const pendingFromAddress = [...this._bySecret.values()].filter( request => request.addressKey === key && request.state === "pending" ).length; @@ -204,8 +194,6 @@ export default class HostPairingManager { if (this.requests.size >= maxPending) { throw new HostPairingError("Too many pending requests", 429, 30); } - recent.push(now); - this._recentCreates.set(key, recent); let code; do { @@ -332,7 +320,7 @@ export default class HostPairingManager { } } - /** Remove requests the installer stopped polling for or that waited too long */ + /** Remove requests the host stopped polling for or that waited too long */ prune() { const now = Date.now(); for (const request of this._bySecret.values()) { @@ -347,15 +335,10 @@ export default class HostPairingManager { if (request.state === "approved") { logger.warn( `Approved host ${request.result!.name} (${request.result!.hostId}) was never picked up ` + - "by its installer, revoke it on the hosts page if it is not needed" + "by the host, revoke it on the hosts page if it is not needed" ); } this._remove(request); } - for (const [key, times] of this._recentCreates) { - if (!times.some(ms => ms > now - 60e3)) { - this._recentCreates.delete(key); - } - } } } diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index fa967fe2..fa1d7b24 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -255,7 +255,7 @@ function checkHostPairingRequest(req: Request, res: Response, needsRequestId: bo return true; } -// Request from an installer for a host to join the cluster. +// Request from a host for a host to join the cluster. async function createHostPairing(req: Request, res: Response) { if (!checkHostPairingRequest(req, res, false)) { return; @@ -299,7 +299,7 @@ async function createHostPairing(req: Request, res: Response) { } } -// Poll from an installer for the result of its request. +// Poll from a host for the result of its request. function pollHostPairing(req: Request, res: Response) { if (!checkHostPairingRequest(req, res, true)) { return; @@ -318,7 +318,7 @@ function pollHostPairing(req: Request, res: Response) { } } -// Cancel from an installer that stopped waiting. +// Cancel from a host that stopped waiting. function cancelHostPairing(req: Request, res: Response) { if (!checkHostPairingRequest(req, res, true)) { return; diff --git a/packages/lib/src/data/HostPairingDetails.ts b/packages/lib/src/data/HostPairingDetails.ts index 69524efd..7ef079fc 100644 --- a/packages/lib/src/data/HostPairingDetails.ts +++ b/packages/lib/src/data/HostPairingDetails.ts @@ -1,20 +1,20 @@ import { Type, Static } from "@sinclair/typebox"; /** - * A pending request from a host installer to join the cluster + * A pending request from a host to join the cluster * - * Requests are kept in memory on the controller only while the installer + * Requests are kept in memory on the controller only while the host * waits for them to be approved or rejected. */ export default class HostPairingDetails { constructor( - /** Code shown by the installer, used to identify the request. */ + /** Code shown by the host, used to identify the request. */ public code: string, - /** Name the installer asked for the host to have. */ + /** Name the host asked to have. */ public name: string, /** Address the request was made from. */ public remoteAddress: string, - /** Version of the installer that made the request. */ + /** Version of the host that made the request. */ public version = "", /** Millisecond Unix timestamp the request was made at. */ public createdAtMs = 0, diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index aeadc779..9a5e6acb 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -199,8 +199,8 @@ export const corePermissions = [ name: "core.host.approve", title: "Approve hosts", description: - "List, approve and reject requests from host installers to join the cluster, and approve hosts " + - "directly by passing a token to the installer. Approving gives the new host full access to the cluster.", + "List, approve and reject requests from hosts to join the cluster, and approve hosts " + + "directly by starting them with a user token. Approving gives the new host full access to the cluster.", }, { name: "core.host.revoke", diff --git a/packages/web_ui/src/components/HostsPage.tsx b/packages/web_ui/src/components/HostsPage.tsx index 9be610d8..654ae644 100644 --- a/packages/web_ui/src/components/HostsPage.tsx +++ b/packages/web_ui/src/components/HostsPage.tsx @@ -46,69 +46,19 @@ function useNow(active: boolean) { } function AddHostModal(props: { open: boolean, onClose: () => void }) { - const control = useContext(ControlContext); const [requests, synced] = useHostPairings(); - const [names, setNames] = useState>(new Map()); - const [busyCodes, setBusyCodes] = useState>(new Set()); + const [, setParams] = useSearchParams(); const now = useNow(props.open); const controllerUrl = new URL(webRoot, document.location.href).href; const installCommand = `npm init "@clusterio" -- --mode host --controller-url ${controllerUrl}`; - function setName(code: string, name?: string) { - setNames(current => { - const next = new Map(current); - if (name === undefined) { - next.delete(code); - } else { - next.set(code, name); - } + function review(code: string) { + setParams(current => { + const next = new URLSearchParams(current); + next.set("approve", code); return next; - }); - } - - function setBusy(code: string, busy: boolean) { - setBusyCodes(current => { - const next = new Set(current); - if (busy) { - next.add(code); - } else { - next.delete(code); - } - return next; - }); - } - - function nameOf(request: Readonly) { - return (names.get(request.code) ?? request.name).trim(); - } - - function approve(request: Readonly) { - const name = nameOf(request); - setBusy(request.code, true); - control.send( - new lib.HostPairingApproveRequest(request.code, name !== request.name ? name : undefined) - ).then(() => { - setName(request.code); - notify(`Approved host ${name}`, "success"); - }).catch( - notifyErrorHandler(`Error approving host ${request.name}`) - ).finally(() => { - setBusy(request.code, false); - }); - } - - function reject(request: Readonly) { - setBusy(request.code, true); - control.send( - new lib.HostPairingRejectRequest(request.code) - ).then(() => { - setName(request.code); - }).catch( - notifyErrorHandler(`Error rejecting host ${request.name}`) - ).finally(() => { - setBusy(request.code, false); - }); + }, { replace: true }); } return void }) { Later starts connect to the cluster straight away. Waiting for approval - - Only approve a request if its code matches the code shown by the host. - An approved host has full access to the cluster. -
void }) { columns={[ { title: "Name", - key: "name", - render: (_, request) => { setName(request.code, event.target.value); }} - />, + dataIndex: "name", }, { title: "Code", @@ -180,23 +120,11 @@ function AddHostModal(props: { open: boolean, onClose: () => void }) { { key: "action", align: "right", - render: (_, request) => { - const busy = busyCodes.has(request.code); - return - - - ; - }, + render: (_, request) => , }, ]} /> diff --git a/test/controller/HostPairingManager.test.js b/test/controller/HostPairingManager.test.js index ce6c8084..c47f20fe 100644 --- a/test/controller/HostPairingManager.test.js +++ b/test/controller/HostPairingManager.test.js @@ -6,7 +6,7 @@ import * as lib from "@clusterio/lib"; import { Controller, HostPairingManager, HostPairingError, HostRecord } from "@clusterio/controller"; import { pollTimeoutMs, maxPendingMs, maxPending, maxPendingPerAddress, - maxCreatesPerMinute, maxNameLength, + maxNameLength, } from "@clusterio/controller/dist/node/src/HostPairingManager.js"; import "../setup_logging.js"; @@ -277,48 +277,6 @@ describe("controller/src/HostPairingManager", function() { }); assert.equal(manager.requests.size, maxPending); }); - it("should limit the number of requests created per minute from an address", function() { - assert.equal(maxCreatesPerMinute, 10); - for (let i = 0; i < maxCreatesPerMinute; i++) { - manager.cancel(manager.create("Host", "", "10.0.0.1").requestId); - } - assert.throws(() => manager.create("Host", "", "10.0.0.1"), err => { - assert(err instanceof HostPairingError); - assert.equal(err.status, 429); - assert.equal(err.message, "Too many requests from this address"); - assert(err.retryAfter >= 1 && err.retryAfter <= 60, `bad retryAfter ${err.retryAfter}`); - return true; - }); - // IPv6 addresses in the same /64 share the limit - for (let i = 0; i < maxCreatesPerMinute; i++) { - manager.cancel(manager.create("Host", "", `2001:db8:0:1::${i + 1}`).requestId); - } - assertPairingError( - () => manager.create("Host", "", "2001:db8:0:1:ffff::1"), 429, "Too many requests from this address" - ); - manager.create("Host", "", "10.0.0.2"); - }); - it("should allow creating requests again after a minute", function() { - for (let i = 0; i < maxCreatesPerMinute; i++) { - manager.cancel(manager.create("Host", "", "10.0.0.1").requestId); - } - assertPairingError(() => manager.create("Host", "", "10.0.0.1"), 429); - for (const times of manager._recentCreates.values()) { - times.forEach((ms, index) => { times[index] = ms - 60e3 - 1; }); - } - manager.create("Host", "", "10.0.0.1"); - }); - it("should not count refused requests towards the rate limit", function() { - const first = manager.create("Host", "", "10.0.0.1"); - manager.create("Host", "", "10.0.0.1"); - manager.create("Host", "", "10.0.0.1"); - const createRefused = () => manager.create("Host", "", "10.0.0.1"); - for (let i = 0; i < maxCreatesPerMinute; i++) { - assertPairingError(createRefused, 429); - } - manager.cancel(first.requestId); - manager.create("Host", "", "10.0.0.1"); - }); }); describe(".poll()", function() { From e5701a8dae52119f213ae6bf8800a6cf955fc6da Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 23:15:22 +0100 Subject: [PATCH 08/44] Approve standalone hosts like any other host A standalone install no longer adds its host to the controller database by signing a token and writing hosts.json itself. The host asks to join when it first starts and is approved once with the link it prints. --- README.md | 1 + docs/contributing.md | 4 +- packages/create/README.md | 2 +- packages/create/create.js | 23 +++----- packages/create/local_host.js | 68 ------------------------ test/create.js | 47 ++--------------- test/create/local_host.js | 98 ----------------------------------- 7 files changed, 16 insertions(+), 227 deletions(-) delete mode 100644 packages/create/local_host.js delete mode 100644 test/create/local_host.js diff --git a/README.md b/README.md index 9b5bb0c5..3046df2b 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,7 @@ Do not install the [Clusterio mod](https://mods.factorio.com/mod/clusterio), thi After completing the installation and creating your admin user, start up the controller and at least one host separately. The installer provides the `run-controller` and `run-host` scripts to make this simple. Once the controller process is running you can log into the web interface which is hosted by default on http://localhost:8080/ (adjust the port number if you changed it), use the admin authentication token provided from the installation to log in. +The first time a host starts it prints a link for approving it, open the link and click Approve. The basics of setting up a Factorio server from the web interface is to create an instance, assign it to a host and then click start. You can also use the "Connect via Steam" button to launch Factorio directly and connect to the server using Steam's protocol handler. diff --git a/docs/contributing.md b/docs/contributing.md index a206de81..a09ede7d 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -87,10 +87,12 @@ Install dependencies with: This installs dependencies and builds the packages. `pnpm-lock.yaml` is intentionally ignored; do not force-add it or assume a frozen-lockfile install is supported. -For a fresh development cluster, run the installer and select standalone mode to create `config-controller.json`, `config-host.json`, and the credentials needed to connect them: +For a fresh development cluster, run the installer and select standalone mode to create `config-controller.json` and `config-host.json`: node packages/create --dev +The first time the host starts it prints a link for approving it in the web interface. + You can skip this step if you already have a working controller and host configuration. To add plugins to use in the development environment use diff --git a/packages/create/README.md b/packages/create/README.md index 7d8ed1d1..df4fbd61 100644 --- a/packages/create/README.md +++ b/packages/create/README.md @@ -19,7 +19,7 @@ The installer runs without prompts when all the questions for the mode are answe ## Modes -- `standalone`: controller and host on this computer. +- `standalone`: controller and host on this computer. The host is approved like any other host the first time it starts. The host is added to the controller directly and does not need to be approved. - `controller`: controller only. - `host`: host only, joins the cluster of an existing controller, see [Setting up a host](#setting-up-a-host). diff --git a/packages/create/create.js b/packages/create/create.js index 5a0c5663..6cea81ac 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -10,7 +10,6 @@ import util from "node:util"; import yargs from "yargs"; import { levels, logger, setLogLevel } from "./logging.js"; -import { createLocalHost } from "./local_host.js"; import { copyPluginTemplates } from "./template.js"; let dev = false; const scriptExt = process.platform === "win32" ? ".cmd" : ""; @@ -869,25 +868,15 @@ async function main() { } if (answers.mode === "standalone") { - // The controller is not running, so the host is added to its database directly. The id - // from the host config is kept so running the installer again reuses the same host. - let result = await execHost(["config", "show", "host.id"]); - let hostId = Number.parseInt(result.stdout.split("\n").slice(-2)[0], 10); - let host = await createLocalHost({ name: "local", id: Number.isInteger(hostId) ? hostId : undefined }); - await execHost(["config", "set", "host.id", host.hostId]); - await execHost(["config", "set", "host.name", host.name]); // Default to localhost on correct port for host in standalone mode - await execHost(["config", "set", "host.controller_url", `http://localhost:${answers.httpPort}/`]); - await execHost(["config", "set", "host.controller_token", host.token]); - } - - if (answers.mode === "host") { - // The host requests to join the cluster when it is started. - await execHost(["config", "set", "host.name", answers.hostName]); - await execHost(["config", "set", "host.controller_url", answers.controllerUrl]); + answers.hostName = "local"; + answers.controllerUrl = `http://localhost:${answers.httpPort}/`; } if (["standalone", "host"].includes(answers.mode)) { + // The host requests to join the cluster when it is started. + await execHost(["config", "set", "host.name", answers.hostName]); + await execHost(["config", "set", "host.controller_url", answers.controllerUrl]); await execHost(["config", "set", "host.public_address", answers.publicAddress]); await execHost(["config", "set", "host.factorio_directory", answers.factorioDir]); await setRemoteUpdates(execHost, "host", answers.remoteNpm); @@ -906,7 +895,7 @@ async function main() { /* eslint-disable no-console */ console.log(`Successfully installed ${answers.mode}`); - if (answers.mode === "host") { + if (["standalone", "host"].includes(answers.mode)) { console.log( `Start the host with run-host${process.platform === "win32" ? ".cmd" : ".sh"}, then approve it on the ` + `Hosts page of ${answers.controllerUrl}` diff --git a/packages/create/local_host.js b/packages/create/local_host.js deleted file mode 100644 index bc534f14..00000000 --- a/packages/create/local_host.js +++ /dev/null @@ -1,68 +0,0 @@ -// Adds the host of a standalone install to the controller database, used by the installer. -// Only uses Node.js builtins as it runs before Clusterio is installed. -import crypto from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; - -function signHostToken(payload, secret) { - const encode = value => Buffer.from(JSON.stringify(value)).toString("base64url"); - const unsigned = `${encode({ alg: "HS256", typ: "JWT" })}.${encode(payload)}`; - const signature = crypto.createHmac("sha256", secret).update(unsigned).digest("base64url"); - return `${unsigned}.${signature}`; -} - -/** - * Add a host to the database of a controller installed in the same directory - * - * The controller must not be running, as it would overwrite the database with - * what it has in memory. - * - * @param {object} options - Host options. - * @param {string=} options.directory - Directory with config-controller.json in it. - * @param {string} options.name - Name of the host. - * @param {number=} options.id - - * Id to give the host, picks a random unused id if not given. If a host with - * this id exists, a new token is signed for it and its name is kept. - * @param {string=} options.version - Version to record for the host. - * @returns {Promise<{ hostId: number, token: string, name: string }>} id, token and name of the host. - */ -export async function createLocalHost({ directory = ".", name, id, version = "" }) { - const configPath = path.join(directory, "config-controller.json"); - const config = JSON.parse(await fs.readFile(configPath, "utf8")); - const secret = config["controller.auth_secret"]; - if (typeof secret !== "string" || !secret) { - throw new Error(`${configPath} has no controller.auth_secret`); - } - - const databaseDirectory = path.resolve(directory, config["controller.database_directory"] ?? "database"); - const hostsPath = path.join(databaseDirectory, "hosts.json"); - let hosts = []; - try { - hosts = JSON.parse(await fs.readFile(hostsPath, "utf8")); - } catch (err) { - if (err.code !== "ENOENT") { - throw err; - } - } - - const now = Math.floor(Date.now() / 1000); - const existing = id === undefined ? undefined : hosts.find(host => host.id === id); - let hostId = id; - if (existing) { - // Running the installer again signs a new token for the same host. - name = existing.name; - } else { - const ids = new Set(hosts.map(host => host.id)); - while (hostId === undefined || ids.has(hostId)) { - hostId = crypto.randomInt(1, 2 ** 31); - } - hosts.push({ id: hostId, name, version, plugins: {}, token_valid_after: now }); - await fs.mkdir(databaseDirectory, { recursive: true }); - const temporary = path.join(databaseDirectory, "hosts.tmp.json"); - await fs.writeFile(temporary, JSON.stringify(hosts, null, "\t"), { flush: true }); - await fs.rename(temporary, hostsPath); - } - - const token = signHostToken({ aud: "host", host: hostId, iat: now }, Buffer.from(secret, "base64")); - return { hostId, token, name }; -} diff --git a/test/create.js b/test/create.js index e1c9263b..af67b8fb 100644 --- a/test/create.js +++ b/test/create.js @@ -1,11 +1,9 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import crypto from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; -import jwt from "jsonwebtoken"; import { escapeArg } from "../packages/create/escape_arg.js"; @@ -145,13 +143,10 @@ describe("create installer CLI", function() { fs.appendFileSync(process.env.CREATE_TEST_COMMANDS, JSON.stringify(command) + "\\n"); const stdout = new PassThrough(); const stderr = new PassThrough(); - // Lets tests give the host id a config show prints. - const output = process.env.CREATE_TEST_HOST_ID && command.includes("config show host.id") - ? process.env.CREATE_TEST_HOST_ID + "\\n" : ""; queueMicrotask(() => { stdout.end(); stderr.end(); - callback(null, output, ""); + callback(null, "", ""); }); return { stdout, stderr }; }; @@ -206,28 +201,17 @@ describe("create installer CLI", function() { assert((await commands()).includes(hostSet("host.controller_url", controllerUrl))); }); - it("adds the host of a standalone install to the controller database", async function() { - const secret = crypto.randomBytes(256).toString("base64"); - await fs.writeFile(path.join(directory, "config-controller.json"), JSON.stringify({ - "controller.auth_secret": secret, - })); + it("leaves joining the cluster to the host of a standalone install", async function() { + await fs.writeFile(path.join(directory, "config-controller.json"), "{}"); const { stdout } = await run([ "--mode", "standalone", "--admin", "admin", "--http-port", "8123", "--public-address", "localhost", "--factorio-dir", "factorio", "--remote-npm", "--plugins", ]); assert.match(stdout, /Successfully installed standalone/); - - const hosts = JSON.parse(await fs.readFile(path.join(directory, "database", "hosts.json"))); - assert.equal(hosts.length, 1); - assert.equal(hosts[0].name, "local"); + assert.match(stdout, /Start the host with run-host/); const recorded = await commands(); - const tokenSet = `${hostBin} config set host.controller_token `; - const token = recorded.find(command => command.startsWith(tokenSet)).slice(tokenSet.length); - const payload = jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }); - assert.equal(payload.host, hosts[0].id); for (const expected of [ - hostSet("host.id", hosts[0].id), hostSet("host.name", "local"), hostSet("host.controller_url", "http://localhost:8123/"), `${controllerBin} config set controller.allow_remote_updates true`, @@ -239,28 +223,7 @@ describe("create installer CLI", function() { ]) { assert(recorded.includes(expected), `missing ${expected}`); } - }); - - it("keeps the host of a standalone install when run again", async function() { - const secret = crypto.randomBytes(256).toString("base64"); - await fs.writeFile(path.join(directory, "config-controller.json"), JSON.stringify({ - "controller.auth_secret": secret, - })); - await fs.mkdir(path.join(directory, "database")); - const existing = [{ id: 1234, name: "local", version: "2.0.3", plugins: {}, token_valid_after: 1 }]; - await fs.writeFile(path.join(directory, "database", "hosts.json"), JSON.stringify(existing)); - const { stdout } = await run([ - "--mode", "standalone", "--admin", "admin", "--http-port", "8123", "--public-address", "localhost", - "--factorio-dir", "factorio", "--remote-npm", "--plugins", - ], { CREATE_TEST_HOST_ID: "1234" }); - assert.match(stdout, /Successfully installed standalone/); - - assert.deepEqual(JSON.parse(await fs.readFile(path.join(directory, "database", "hosts.json"))), existing); - const recorded = await commands(); - assert(recorded.includes(hostSet("host.id", 1234)), "host id was changed"); - const tokenSet = `${hostBin} config set host.controller_token `; - const token = recorded.find(command => command.startsWith(tokenSet)).slice(tokenSet.length); - assert.equal(jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }).host, 1234); + assert(!recorded.some(command => command.includes("host.controller_token"))); }); }); }); diff --git a/test/create/local_host.js b/test/create/local_host.js deleted file mode 100644 index 3f32b748..00000000 --- a/test/create/local_host.js +++ /dev/null @@ -1,98 +0,0 @@ -import assert from "node:assert/strict"; -import crypto from "node:crypto"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import jwt from "jsonwebtoken"; - -import * as lib from "@clusterio/lib"; -import HostRecord from "@clusterio/controller/dist/node/src/HostRecord.js"; -import { createLocalHost } from "../../packages/create/local_host.js"; - -describe("create/local_host createLocalHost()", function() { - let directory; - let secret; - beforeEach(async function() { - directory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-local-host-")); - secret = crypto.randomBytes(256).toString("base64"); - }); - afterEach(async function() { - await fs.rm(directory, { recursive: true, force: true, maxRetries: 10 }); - }); - - // Written with the controller's own config class to match its file format. - async function writeControllerConfig(fields = {}) { - const configPath = path.join(directory, "config-controller.json"); - const config = new lib.ControllerConfig("controller", undefined, configPath); - config.set("controller.auth_secret", secret); - for (const [name, value] of Object.entries(fields)) { - config.set(name, value); - } - await config.save(true); - } - - async function loadHosts(databaseDirectory = "database") { - const provider = new lib.JsonIdDatastoreProvider( - path.join(directory, databaseDirectory, "hosts.json"), HostRecord.fromJSON.bind(HostRecord), - ); - return await provider.load(); - } - - it("adds a host the controller loads with a token it accepts", async function() { - await writeControllerConfig(); - const before = Math.floor(Date.now() / 1000); - const result = await createLocalHost({ directory, name: "local" }); - assert.equal(result.name, "local"); - assert(Number.isInteger(result.hostId) && result.hostId >= 1 && result.hostId < 2 ** 31); - - const hosts = await loadHosts(); - assert.deepEqual([...hosts.keys()], [result.hostId]); - const record = hosts.get(result.hostId); - assert.equal(record.name, "local"); - assert.equal(record.version, ""); - assert.deepEqual(record.plugins, new Map()); - assert(record.tokenValidAfter >= before); - - const payload = jwt.verify(result.token, Buffer.from(secret, "base64"), { audience: "host" }); - assert.equal(payload.host, result.hostId); - assert(payload.iat >= record.tokenValidAfter, "token must not be issued before token_valid_after"); - }); - - it("appends to an existing database in the configured directory", async function() { - await writeControllerConfig({ "controller.database_directory": "db" }); - await fs.mkdir(path.join(directory, "db")); - await fs.writeFile(path.join(directory, "db", "hosts.json"), JSON.stringify([ - { id: 1, name: "existing", version: "2.0.0", plugins: {} }, - ])); - const result = await createLocalHost({ directory, name: "local", id: 7, version: "2.0.3" }); - assert.equal(result.hostId, 7); - - const hosts = await loadHosts("db"); - assert.deepEqual([...hosts.keys()], [1, 7]); - assert.equal(hosts.get(1).name, "existing"); - assert.equal(hosts.get(7).version, "2.0.3"); - await assert.rejects(fs.access(path.join(directory, "database")), { code: "ENOENT" }); - }); - - it("signs a new token for an existing host with the same id", async function() { - await writeControllerConfig(); - const first = await createLocalHost({ directory, name: "first", id: 7 }); - const before = await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"); - const second = await createLocalHost({ directory, name: "second", id: 7 }); - assert.equal(second.hostId, 7); - assert.equal(second.name, "first", "the name of the existing host should be kept"); - assert.equal(await fs.readFile(path.join(directory, "database", "hosts.json"), "utf8"), before); - - const record = (await loadHosts()).get(7); - for (const token of [first.token, second.token]) { - const payload = jwt.verify(token, Buffer.from(secret, "base64"), { audience: "host" }); - assert.equal(payload.host, 7); - assert(payload.iat >= record.tokenValidAfter); - } - }); - - it("throws if the controller config has no auth secret", async function() { - await fs.writeFile(path.join(directory, "config-controller.json"), "{}"); - await assert.rejects(createLocalHost({ directory, name: "local" }), /controller\.auth_secret/); - }); -}); From 49a0b27d2998d8664195fd2e5111e5ab592394ae Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 31 Oct 2026 16:31:45 +0100 Subject: [PATCH 09/44] Address review of host pairing Revoking a host now unassigns its instances instead of being refused while it has any, and removes its system metrics. Approving and revoking no longer force a save. Hosts are saved by autosave and on exit like everything else, which also removes the approving state from pairing requests. The approval code and link are logged as warnings so they show up in the systemd service, which only prints warnings and above. The pairing limits are now the controller.host_pairing_timeout and controller.host_pairing_max_pending config values, 0 removes the limit. Subscribers take an optional permitted function, used for the pairing requests instead of overrides in a subclass. --- README.md | 4 +- docs/configuration.md | 19 +- docs/devs/plugin-migration-to-2.1.md | 4 +- docs/devs/protocol.md | 8 +- docs/managing-a-cluster.md | 6 +- packages/controller/src/ControlConnection.ts | 27 ++- packages/controller/src/Controller.ts | 6 +- packages/controller/src/HostPairingManager.ts | 71 +++---- packages/controller/src/routes.ts | 12 +- packages/create/README.md | 20 +- packages/create/create.js | 23 +-- packages/host/README.md | 5 +- packages/host/src/pairing.ts | 41 ++-- packages/lib/src/config/definitions.ts | 16 ++ packages/lib/src/datastore.ts | 4 +- packages/lib/src/subscriptions.ts | 4 +- .../web_ui/src/components/HostViewPage.tsx | 7 +- packages/web_ui/src/model/host.ts | 10 +- packages/web_ui/src/util/host_pairing.ts | 40 +--- test/controller/ControlConnection.test.js | 56 +++--- test/controller/HostPairingManager.test.js | 185 +++++++----------- test/controller/routes.js | 26 +-- test/create.js | 11 +- test/host/pairing.js | 41 +++- test/integration/clusterio.js | 20 +- test/integration/index.js | 13 +- test/lib/subscriptions.test.js | 30 +++ test/mock.js | 2 + test/web_ui/host_pairing.js | 63 +----- 29 files changed, 356 insertions(+), 418 deletions(-) diff --git a/README.md b/README.md index 3046df2b..4427194f 100644 --- a/README.md +++ b/README.md @@ -287,8 +287,8 @@ Only approve a request if its code matches the one shown by the host, approving Approving requires the `core.host.approve` permission, which the Cluster Admin role has. If you have that permission you can skip the wait by starting the host with `npx clusteriohost run --user-token ` and your own authentication token, it is only used to approve the host and is not saved. -To remove a host from the cluster, unassign its instances and click Revoke host access on the page for the host in the web interface, or run `npx clusterioctl host revoke `. -This disconnects the host and its saved connection details stop working. +To remove a host from the cluster, click Revoke host access on the page for the host in the web interface, or run `npx clusterioctl host revoke `. +This disconnects the host, its instances become unassigned and its saved connection details stop working. To add the computer back to the cluster start the host again, it asks for approval again. diff --git a/docs/configuration.md b/docs/configuration.md index ea25d482..e8c24900 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -48,6 +48,23 @@ Must be greater than 0. Defaults to 60. +### controller.host_pairing_timeout + +Time in seconds a host can wait for approval before its request expires, after which the host requests again with a new code. +Set to 0 for no limit. + +Defaults to 3600. + + +### controller.host_pairing_max_pending + +Maximum number of hosts that can wait for approval at the same time. +Hosts over the limit keep trying until there is room. +Set to 0 for no limit. + +Defaults to 50. + + ### controller.http_port Port to host HTTP server on. @@ -369,7 +386,7 @@ Defaults to "http://localhost:8080/". Access token used for authenticating with the controller. The host saves it together with host.id when it is approved, see [Setting up remote hosts](/README.md#setting-up-remote-hosts). The token only works while the host exists on the controller, if the host is revoked it asks to be approved again the next time it starts. -Set it to `enter token here` to make the host ask to join again, for example after changing host.controller_url to another controller. +Set it to `enter token here` to make the host ask to join again. The value can only be read and changed from the host itself. Defaults to "enter token here". diff --git a/docs/devs/plugin-migration-to-2.1.md b/docs/devs/plugin-migration-to-2.1.md index f53f53d2..3a6a440f 100644 --- a/docs/devs/plugin-migration-to-2.1.md +++ b/docs/devs/plugin-migration-to-2.1.md @@ -69,7 +69,7 @@ A host joins the cluster by having the code it shows when it first starts approv - `HostGenerateTokenRequest` and `HostConfigCreateRequest` are removed. Pending requests are listed with `HostPairingListRequest`, approved with `HostPairingApproveRequest` and rejected with `HostPairingRejectRequest`, and changes to them are sent to subscribers of `HostPairingUpdatesEvent`. - `HostRevokeTokensRequest` is replaced by `HostRevokeRequest`, which deletes the host instead of only invalidating its tokens. - It throws `ConflictError` while instances are assigned to the host. + Instances assigned to the host become unassigned. - The permissions `core.host.generate_token`, `core.host.revoke_token` and `core.host.create_config` are removed. Check `core.host.approve` and `core.host.revoke` instead. - `clusteriocontroller bootstrap generate-host-token` is removed, scripts that set up hosts can start the host with `--user-token` to approve it with a user token. @@ -78,4 +78,4 @@ Host records are no longer created when a host connects, only by approving it, a Revoking a host deletes its record and note. Plugins that keep data per host should remove it when the host is deleted. On the controller the `update` event of `controller.hosts` gives the record with `isDeleted` set to true, and subscribers of `HostUpdatesEvent` receive the host with `isDeleted` set to true. -A host that is revoked and installed again gets a new id. +A host that is revoked and approved again gets a new id. diff --git a/docs/devs/protocol.md b/docs/devs/protocol.md index 747137d0..07c71a6c 100644 --- a/docs/devs/protocol.md +++ b/docs/devs/protocol.md @@ -61,7 +61,7 @@ If a user token is sent and the user has the `core.host.approve` permission the An invalid token gives 401 and a user without the permission gives 403, a token is never ignored to create a pending request instead. A missing or too long name gives 400. -Too many pending requests from the same address or in total gives 429 with a `Retry-After` header in seconds. +Too many pending requests from the same address, or in total as set by [controller.host_pairing_max_pending](/docs/configuration.md#controllerhost_pairing_max_pending), gives 429 with a `Retry-After` header in seconds. ### `POST /api/host-pairing/poll` @@ -72,7 +72,7 @@ Once approved it responds with 200 and `{"hostId": integer, "token": string, "na Rejected requests give 403, and unknown or expired requests give 404. The result of an approved or rejected request is only returned once, after which the request is gone. -A pending request expires 30 seconds after it was last polled, and after an hour of waiting. +A pending request expires 30 seconds after it was last polled, and after waiting for the time set by [controller.host_pairing_timeout](/docs/configuration.md#controllerhost_pairing_timeout). The host polls every 2 seconds, and makes a new request when the previous one has expired. ### `POST /api/host-pairing/cancel` @@ -177,8 +177,8 @@ Host handshake for establishing a new connection session. Object mapping plugin names to plugin version for plugins that are available on the host. The controller closes the connection with code 3000 (Unauthorized) if the token is invalid, was issued for another id, or if there is no host with this id, see [Host pairing](#host-pairing). -If this happens when the host starts it exits with code 8, and the run scripts created by the installer do not restart it. -A host that has been rejected also exits with code 8. +The host checks its token before connecting and requests to join again if it is not accepted, so this normally only happens to a host that is already running. +A host that has been rejected exits with code 8, and the run scripts created by the installer do not restart it. A host that is revoked while connected is disconnected, is rejected when it tries to connect again, and shuts down. #### `registerControl` diff --git a/docs/managing-a-cluster.md b/docs/managing-a-cluster.md index c7101959..51982131 100644 --- a/docs/managing-a-cluster.md +++ b/docs/managing-a-cluster.md @@ -17,14 +17,14 @@ The easiest way to do this is to run `npx clusteriocontroller bootstrap create-c Lists all hosts. Hosts are added to the cluster by running the installer on the computer the host runs on and approving the code it shows, see [Setting up remote hosts](/README.md#setting-up-remote-hosts). -Listing, approving and rejecting installers waiting for approval needs the `core.host.approve` permission. +Listing, approving and rejecting hosts waiting for approval needs the `core.host.approve` permission. ### List pending hosts ctl> host pending Lists hosts waiting for approval with the code they show, the name they asked for, the address they connected from and how long they have been waiting. -A request goes away when its host stops waiting, and after waiting for an hour, after which the host asks again with a new code. +A request goes away when its host stops waiting, and after the time set by `controller.host_pairing_timeout`, an hour by default, after which the host asks again with a new code. ### Approve host @@ -47,7 +47,7 @@ Rejects the host showing the code, the host then exits with an error. Specify the host by name or ID. Removes the host from the cluster, disconnects it and makes its token stop working. -Instances assigned to the host have to be unassigned first. +Instances assigned to the host become unassigned, their files stay on the computer. Requires the `core.host.revoke` permission. To add the computer back to the cluster, start the host again and approve it. diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 81849632..43af122d 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -255,26 +255,25 @@ export default class ControlConnection extends BaseConnection { throw new lib.NotFoundError(`Unknown host id (${request.hostId})`); } - const assigned = [...this._controller.instances.values()].filter( - instance => instance.config.get("instance.assigned_host") === request.hostId - ); - if (assigned.length) { - const names = assigned.map(instance => instance.config.get("instance.name")).join(", "); - throw new lib.ConflictError( - `Cannot revoke host ${host.name} while instances are assigned to it: ${names}` - ); - } - this._controller.hosts.delete(host); this._controller.deleteNote("host", request.hostId, this.user.name); + const system = this._controller.systems.getMutable(request.hostId); + if (system) { + this._controller.systems.delete(system); + } - // Disconnect before saving so the host is cut off even if the save fails. + // Disconnected first so unassigning the instances does not depend on the host responding. const hostConnection = this._controller.wsServer.hostConnections.get(request.hostId); if (hostConnection) { hostConnection.connector.terminate(); } + const assigned = [...this._controller.instances.values()].filter( + instance => instance.config.get("instance.assigned_host") === request.hostId + ); + for (const instance of assigned) { + await this._controller.instances.unassignInstance(instance.id); + } logger.audit(`User ${this.user.name} revoked host ${host.name} (${host.id})`); - await this._controller.hosts.save(); } async handleHostListRequest(): Promise { @@ -283,11 +282,11 @@ export default class ControlConnection extends BaseConnection { async handleHostPairingListRequest(): Promise { this._controller.hostPairing.prune(); - return [...this._controller.hostPairing.requests.values()]; + return [...this._controller.hostPairing.records.values()]; } async handleHostPairingApproveRequest(request: lib.HostPairingApproveRequest) { - return await this._controller.hostPairing.approve(request.code, request.name, this.user.name); + return this._controller.hostPairing.approve(request.code, request.name, this.user.name); } async handleHostPairingRejectRequest(request: lib.HostPairingRejectRequest) { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 3bf2bdf4..1d006df9 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -323,7 +323,7 @@ export default class Controller { // Handle updates for datastores this.systems.on("update", this.systemsUpdated.bind(this)); this.hosts.on("update", this.hostsUpdated.bind(this)); - this.hostPairing.requests.on("update", this.hostPairingRequestsUpdated.bind(this)); + this.hostPairing.records.on("update", this.hostPairingRequestsUpdated.bind(this)); this.instances.records.on("update", this.instanceUpdated.bind(this)); this.saves.on("update", this.savesUpdated.bind(this)); this.modPacks.on("update", this.modPacksUpdated.bind(this)); @@ -373,7 +373,6 @@ export default class Controller { async _startInternal(args: ControllerArgs) { this.logDirectory = args.logDirectory; - this.hostPairing.start(); this.clusterLogIndex = await lib.LogIndex.load(path.join(this.logDirectory, "cluster")); this.clusterLogBuildInterval = setInterval(() => { @@ -496,6 +495,7 @@ export default class Controller { logger.info(`Listening for HTTPS on port ${(this.httpsServer.address() as AddressInfo).port}`); } + this.hostPairing.start(); this.onAutosaveIntervalChanged(); this.onSystemMetricsIntervalChanged(); @@ -1062,7 +1062,7 @@ export default class Controller { if (request.action === "unsubscribe") { return null; } - const requests = [...this.hostPairing.requests.values()].filter( + const requests = [...this.hostPairing.records.values()].filter( pairing => pairing.updatedAtMs > request.lastRequestTimeMs, ); return requests.length ? new lib.HostPairingUpdatesEvent(requests) : null; diff --git a/packages/controller/src/HostPairingManager.ts b/packages/controller/src/HostPairingManager.ts index 8dc3ad99..32daf21f 100644 --- a/packages/controller/src/HostPairingManager.ts +++ b/packages/controller/src/HostPairingManager.ts @@ -14,10 +14,6 @@ const codeLength = 6; /** Time a request is kept after the last poll from the host */ export const pollTimeoutMs = 30e3; -/** Maximum time a request can stay pending */ -export const maxPendingMs = 60 * 60e3; -/** Maximum number of pending requests in total */ -export const maxPending = 50; /** Maximum number of pending requests from one address */ export const maxPendingPerAddress = 3; /** Maximum length of a host name */ @@ -44,16 +40,16 @@ export class HostPairingError extends Error { } } -interface PairingRequest { +type PairingRequest = { /** Secret the host polls with, never shown to users */ secret: string, details: lib.HostPairingDetails, /** Address with IPv6 addresses grouped by /64, used for limits */ addressKey: string, lastPollMs: number, - state: "pending" | "approving" | "approved" | "rejected", - result?: HostPairingResult, -} +} & ( + { state: "pending" | "rejected" } | { state: "approved", result: HostPairingResult } +); /** * Strips the IPv4 mapped prefix and groups IPv6 addresses by their /64 @@ -105,7 +101,7 @@ function generateCode() { */ export default class HostPairingManager { /** Pending requests by code, broadcast to subscribed control links */ - requests = new lib.SubscribableDatastore(); + records = new lib.SubscribableDatastore(); private _bySecret = new Map(); private _byCode = new Map(); private _pruneInterval?: ReturnType; @@ -148,28 +144,19 @@ export default class HostPairingManager { /** * Create an approved host with a new id and token * - * The host record is saved to disk before this returns. - * * @param name - Name of the host. * @returns the id, token and name of the new host. */ - async createHost(name: string): Promise { + createHost(name: string): HostPairingResult { const hosts = this._controller.hosts; let hostId; do { hostId = crypto.randomInt(1, 2 ** 31); } while (hosts.has(hostId)); - const record = new HostRecord( + hosts.set(new HostRecord( hostId, name, "", new Map(), false, "", "", Math.floor(Date.now() / 1000), - ); - hosts.set(record); - try { - await hosts.save(); - } catch (err) { - hosts.delete(record); - throw err; - } + )); return { hostId, token: this._controller.generateHostToken(hostId), name }; } @@ -191,7 +178,8 @@ export default class HostPairingManager { if (pendingFromAddress >= maxPendingPerAddress) { throw new HostPairingError("Too many pending requests from this address", 429, 30); } - if (this.requests.size >= maxPending) { + const maxPending = this._controller.config.get("controller.host_pairing_max_pending"); + if (maxPending && this.records.size >= maxPending) { throw new HostPairingError("Too many pending requests", 429, 30); } @@ -209,7 +197,7 @@ export default class HostPairingManager { }; this._bySecret.set(request.secret, request); this._byCode.set(code, request); - this.requests.set(request.details); + this.records.set(request.details); logger.verbose(`Host pairing request ${code} for "${name}" from ${remoteAddress}`); return { requestId: request.secret, code }; } @@ -229,7 +217,7 @@ export default class HostPairingManager { request.lastPollMs = Date.now(); if (request.state === "approved") { this._remove(request); - return request.result!; + return request.result; } if (request.state === "rejected") { this._remove(request); @@ -268,7 +256,7 @@ export default class HostPairingManager { * @param approvedBy - Name of the user approving it, for logging. * @returns the id of the new host. */ - async approve(code: string, name: string | undefined, approvedBy: string) { + approve(code: string, name: string | undefined, approvedBy: string) { const request = this._pending(code); let hostName; try { @@ -277,18 +265,9 @@ export default class HostPairingManager { throw new lib.ValidationError(err.message); } - request.state = "approving"; - let result; - try { - result = await this.createHost(hostName); - } catch (err) { - request.state = "pending"; - throw err; - } - - request.state = "approved"; - request.result = result; - this.requests.delete(request.details); + const result = this.createHost(hostName); + Object.assign(request, { state: "approved", result }); + this.records.delete(request.details); logger.audit( `User ${approvedBy} approved host ${result.name} (${result.hostId}) with code ` + `${request.details.code} from ${request.details.remoteAddress}` @@ -305,7 +284,7 @@ export default class HostPairingManager { reject(code: string, rejectedBy: string) { const request = this._pending(code); request.state = "rejected"; - this.requests.delete(request.details); + this.records.delete(request.details); logger.audit( `User ${rejectedBy} rejected host ${request.details.name} with code ` + `${request.details.code} from ${request.details.remoteAddress}` @@ -315,26 +294,22 @@ export default class HostPairingManager { private _remove(request: PairingRequest) { this._bySecret.delete(request.secret); this._byCode.delete(request.details.code); - if (this.requests.has(request.details.code)) { - this.requests.delete(request.details); - } + this.records.delete(request.details); } /** Remove requests the host stopped polling for or that waited too long */ prune() { const now = Date.now(); + const timeoutMs = this._controller.config.get("controller.host_pairing_timeout") * 1000; for (const request of this._bySecret.values()) { - if (request.state === "approving") { - continue; - } - const expired = now - request.lastPollMs > pollTimeoutMs - || (request.state === "pending" && now - request.details.createdAtMs > maxPendingMs); - if (!expired) { + const timedOut = request.state === "pending" && timeoutMs > 0 + && now - request.details.createdAtMs > timeoutMs; + if (!timedOut && now - request.lastPollMs <= pollTimeoutMs) { continue; } if (request.state === "approved") { logger.warn( - `Approved host ${request.result!.name} (${request.result!.hostId}) was never picked up ` + + `Approved host ${request.result.name} (${request.result.hostId}) was never picked up ` + "by the host, revoke it on the hosts page if it is not needed" ); } diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index fa1d7b24..36e1cde9 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -256,7 +256,7 @@ function checkHostPairingRequest(req: Request, res: Response, needsRequestId: bo } // Request from a host for a host to join the cluster. -async function createHostPairing(req: Request, res: Response) { +function createHostPairing(req: Request, res: Response) { if (!checkHostPairingRequest(req, res, false)) { return; } @@ -279,10 +279,13 @@ async function createHostPairing(req: Request, res: Response) { try { user.checkPermission("core.host.approve"); } catch (err) { + logger.audit( + `User ${user.name} tried to approve host ${name} from ${remoteAddress} without permission` + ); res.status(403).json({ error: `User ${user.name} is not allowed to approve hosts` }); return; } - const result = await controller.hostPairing.createHost(name); + const result = controller.hostPairing.createHost(name); logger.audit( `User ${user.name} approved host ${result.name} (${result.hostId}) from ${remoteAddress} ` + "with their token" @@ -783,10 +786,7 @@ export function addRouteHandlers(app: Application) { next(); }; const pairingJson = express.json({ limit: "4kb" }); - app.post("/api/host-pairing", - noStore, pairingJson, - (req:Request, res:Response, next:any) => createHostPairing(req, res).catch(next) - ); + app.post("/api/host-pairing", noStore, pairingJson, createHostPairing); app.post("/api/host-pairing/poll", noStore, pairingJson, pollHostPairing); app.post("/api/host-pairing/cancel", noStore, pairingJson, cancelHostPairing); app.get("/api/host-pairing/check", noStore, validateHostToken, (req: Request, res: Response) => { diff --git a/packages/create/README.md b/packages/create/README.md index df4fbd61..38cc24cd 100644 --- a/packages/create/README.md +++ b/packages/create/README.md @@ -20,7 +20,6 @@ The installer runs without prompts when all the questions for the mode are answe ## Modes - `standalone`: controller and host on this computer. The host is approved like any other host the first time it starts. - The host is added to the controller directly and does not need to be approved. - `controller`: controller only. - `host`: host only, joins the cluster of an existing controller, see [Setting up a host](#setting-up-a-host). - `ctl`: the clusterioctl command line interface only. @@ -33,21 +32,24 @@ The installer runs without prompts when all the questions for the mode are answe | Option | Modes | Description | | --- | --- | --- | | `--mode ` | all | Operating mode to install. | -| `--admin ` | standalone, controller | Name of the admin account to create. | -| `--http-port ` | standalone, controller | Port the controller web interface listens on, defaults to 8080. | +| `--admin ` | controller | Name of the admin account to create. | +| `--http-port ` | controller | Port the controller web interface listens on, defaults to 8080. | | `--host-name ` | host | Name to request for the host, defaults to the name of the computer. | | `--controller-url ` | host, ctl | URL of the controller, the same as for its web interface. | | `--controller-token ` | ctl | Authentication token saved for clusterioctl. | -| `--public-address
` | standalone, host | DNS name or IP address players connect to this computer with. | -| `--factorio-dir ` | standalone, host | Path to the Factorio installation. | -| `--self-signed-cert ` | standalone, host | Certificate to trust for a controller using a self-signed certificate. It is added to the run scripts. | -| `--remote-npm`, `--no-remote-npm` | standalone, controller, host | Allow or deny remote updates, plugin updates and plugin installs from the web interface. | +| `--public-address
` | host | DNS name or IP address players connect to this computer with. | +| `--factorio-dir ` | host | Path to the Factorio installation. | +| `--self-signed-cert ` | host | Certificate to trust for a controller using a self-signed certificate. It is added to the run scripts. | +| `--remote-npm`, `--no-remote-npm` | controller, host | Allow or deny remote updates, plugin updates and plugin installs from the web interface. | | `--plugins ` | all | Plugins to install, pass `--plugins` without packages to install none. | | `--plugin-template `, `--plugin-name ` | plugin-template | Templates to use and name of the new plugin. | | `--log-level ` | all | Log level to print, defaults to info. | | `--allow-install-as-root` | all | Linux only, allow installing as root (not recommended). | | `--migrate-rename` | | Migrate an installation from before the slave/master rename. | +Options for the `controller` and `host` modes also work in the `standalone` mode, except for `--controller-url` as the host connects to the controller on the same computer. +`--host-name` defaults to `local` in the `standalone` mode. + ## Setting up a host @@ -84,8 +86,8 @@ The installer adds it to the run scripts. When a host is revoked in the web interface or with `npx clusterioctl host revoke ` its token stops working and a running host shuts down. Starting it again requests to join the cluster again, and the host gets a new id once approved. -To move a host to another controller, change `host.controller_url` with `npx clusteriohost config set host.controller_url ` and set `host.controller_token` back to `enter token here`. -The host then requests to join that controller the next time it starts. +To move a host to another controller, change `host.controller_url` with `npx clusteriohost config set host.controller_url `. +The new controller does not accept the token of the host, so the host requests to join it the next time it starts. ## Docker diff --git a/packages/create/create.js b/packages/create/create.js index 6cea81ac..587a9672 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -161,6 +161,9 @@ async function execCtl(args) { return await execFile(path.join("node_modules", ".bin", `clusterioctl${scriptExt}`), args); } +// Same characters as the controller refuses in host names. +const hostNameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u; + function validateHostName(input) { if (!input || !input.trim()) { return "Host name may not be empty"; @@ -174,9 +177,6 @@ function validateHostName(input) { return true; } -// Same characters as the controller refuses in host names. -const hostNameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u; - function validateControllerUrl(input) { let url; try { @@ -594,6 +594,14 @@ async function inquirerMissingArgs(args) { ], answers); } + if (answers.mode === "standalone") { + answers.hostName = args.hostName ?? "local"; + const result = validateHostName(answers.hostName); + if (result !== true) { + throw new InstallError(result); + } + } + if (answers.mode === "host") { if (args.hostName) { answers.hostName = args.hostName; } if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; } @@ -783,7 +791,7 @@ async function main() { nargs: 1, describe: "HTTP port to listen on [standalone/controller]", type: "number", }) .option("host-name", { - nargs: 1, describe: "Host name [host]", type: "string", + nargs: 1, describe: "Host name [standalone/host]", type: "string", }) .option("controller-url", { nargs: 1, describe: "Controller URL [host/ctl]", type: "string", @@ -869,7 +877,6 @@ async function main() { if (answers.mode === "standalone") { // Default to localhost on correct port for host in standalone mode - answers.hostName = "local"; answers.controllerUrl = `http://localhost:${answers.httpPort}/`; } @@ -895,12 +902,6 @@ async function main() { /* eslint-disable no-console */ console.log(`Successfully installed ${answers.mode}`); - if (["standalone", "host"].includes(answers.mode)) { - console.log( - `Start the host with run-host${process.platform === "win32" ? ".cmd" : ".sh"}, then approve it on the ` + - `Hosts page of ${answers.controllerUrl}` - ); - } if (adminToken) { console.log(`Admin authentication token: ${adminToken}`); } diff --git a/packages/host/README.md b/packages/host/README.md index 95fe9ac8..b719227b 100644 --- a/packages/host/README.md +++ b/packages/host/README.md @@ -132,8 +132,9 @@ Lists up all configuration entries with their currently configured values. ### `run` Runs the host. -The host has to be approved on the controller before it can connect, which is done by running the installer in the host directory, see Setting up remote hosts in the readme of the main repository. -If the controller rejects the token of the host when it starts, for example because the host was revoked, it exits with code 8 and the run scripts created by the installer do not restart it. +A host that has not been approved prints a code and a link and waits for approval before it connects, see [Setting up a host](https://github.com/clusterio/clusterio/blob/master/packages/create/README.md#setting-up-a-host). +If the controller no longer accepts the token of the host when it starts, for example because the host was revoked, it asks to be approved again. +If its request is rejected it exits with code 8 and the run scripts created by the installer do not restart it. A host that is revoked while it runs shuts down. diff --git a/packages/host/src/pairing.ts b/packages/host/src/pairing.ts index 8bd41bc2..ab1ceda5 100644 --- a/packages/host/src/pairing.ts +++ b/packages/host/src/pairing.ts @@ -111,10 +111,14 @@ function retryDelay(response: Response, defaultMs: number) { * * @returns the result if approved, or null if the controller no longer knows the request. */ -async function poll(requestId: string, options: Required> & PairHostOptions) { - let failing = false; +async function poll( + requestId: string, + options: Required> & PairHostOptions, +) { + let delayMs = options.pollIntervalMs; while (true) { - await sleep(options.pollIntervalMs, undefined, { signal: options.signal }); + await sleep(delayMs, undefined, { signal: options.signal }); + delayMs = options.pollIntervalMs; let response; try { response = await request(options.controllerUrl, "api/host-pairing/poll", { @@ -125,20 +129,22 @@ async function poll(requestId: string, options: Required * * @param controllerUrl - URL of the controller, ending with a slash. * @param token - Host token to check. + * @param signal - Signal to cancel the check with. * @returns true if the token is valid and its host is registered. * @throws {PairingError} if the controller gives an unexpected response. */ -export async function checkHostToken(controllerUrl: string, token: string) { - const response = await request(controllerUrl, "api/host-pairing/check", { token }); +export async function checkHostToken(controllerUrl: string, token: string, signal?: AbortSignal) { + const response = await request(controllerUrl, "api/host-pairing/check", { token, signal }); if (response.status === 200) { return true; } @@ -254,8 +261,11 @@ export async function ensurePaired( if (token && token !== unpairedToken) { let valid; try { - valid = await checkHostToken(controllerUrl, token); + valid = await checkHostToken(controllerUrl, token, options.signal); } catch (err: any) { + if (options.signal?.aborted) { + throw err; + } // Connecting reports the problem, or authenticates if the controller predates pairing. logger.verbose(`Unable to check if the controller accepts this host: ${err.message}`); return; @@ -276,10 +286,11 @@ export async function ensurePaired( signal: options.signal, onPending: code => { const approveUrl = new URL(`hosts?approve=${code}`, controllerUrl).href; + // Logged as warnings for the systemd service, which only prints warnings and above. // The link is logged on its own so terminals can make it clickable without it wrapping. - logger.info(`Waiting for approval of "${name}" with code ${code}, open this link to approve it:`); - logger.info(approveUrl); - logger.info(`Or run: npx clusterioctl host approve ${code}`); + logger.warn(`Waiting for approval of "${name}" with code ${code}, open this link to approve it:`); + logger.warn(approveUrl); + logger.warn(`Or run: npx clusterioctl host approve ${code}`); }, onRetry: (err, delayMs) => { logger.warn(`Requesting to join failed: ${err.message}, trying again in ${Math.ceil(delayMs / 1000)}s`); diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 5d19897a..b28db160 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -37,6 +37,8 @@ export interface ControllerConfigFields { "controller.default_mod_pack_id": number | null; "controller.default_role_id": number | null; "controller.autosave_interval": number; + "controller.host_pairing_timeout": number; + "controller.host_pairing_max_pending": number; "controller.mod_portal_cache_duration_minutes": number; "controller.mod_portal_page_size": number; "controller.allow_remote_updates": boolean; @@ -98,6 +100,20 @@ export class ControllerConfig extends classes.Config { initialValue: 60, validator: validators.greaterThanZero, }, + "controller.host_pairing_timeout": { + title: "Host Pairing Timeout", + description: "Time in seconds a host can wait for approval before it has to request again, 0 for no limit.", + type: "number", + initialValue: 3600, + validator: validators.greaterThanEqualZero, + }, + "controller.host_pairing_max_pending": { + title: "Host Pairing Max Pending", + description: "Maximum number of hosts that can wait for approval at the same time, 0 for no limit.", + type: "number", + initialValue: 50, + validator: validators.all(validators.integer, validators.greaterThanEqualZero), + }, "controller.http_port": { title: "HTTP Port", description: "Port to listen for HTTP connections on, set to null to not listen for HTTP connections.", diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 59be8d96..1db3518e 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -183,7 +183,7 @@ export abstract class Datastore< // Save the datastore to the provider async save() { // Saves run one at a time so an older snapshot can't be written after a newer one. - const save = this.saving.then(async () => { + const save = this.saving.catch(() => {}).then(async () => { if (!this.dirty) { return; } @@ -196,7 +196,7 @@ export abstract class Datastore< throw err; } }); - this.saving = save.catch(() => {}); + this.saving = save; await save; } diff --git a/packages/lib/src/subscriptions.ts b/packages/lib/src/subscriptions.ts index 9bb97d99..e5deea6c 100644 --- a/packages/lib/src/subscriptions.ts +++ b/packages/lib/src/subscriptions.ts @@ -480,6 +480,8 @@ export class EventSubscriber { protected Event: EventClass, public control: Link, protected filteredHandler?: EventSubscriberCallback, + /** Nothing is sent to the controller while this returns false, for accounts without permission */ + protected permitted: () => boolean = () => true, ) { control.handle(Event, this._handleEvent.bind(this)); // The events below exist only on websocket connectors, we can't early return because of test mocking @@ -670,7 +672,7 @@ export class EventSubscriber { * Update the subscription with the controller based on current handler counts */ private async _sendRequest(action: SubscriptionAction, filters: SubscriptionFilters) { - if (!this.control.connector.valid) { + if (!this.control.connector.valid || !this.permitted()) { return; } diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index 5691985d..a7b1d6fc 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -104,16 +104,11 @@ function RevokeHostButton({ host, instances }: RevokeHostButtonProps) { const navigate = useNavigate(); const assigned = [...instances.values()].map(instance => instance.name); - if (assigned.length) { - return - - ; - } - return The host is removed from the cluster and disconnected.
+ {assigned.length > 0 && <>Its instances become unassigned: {assigned.join(", ")}
} To add it back, start the host again and approve it. } placement="bottomRight" diff --git a/packages/web_ui/src/model/host.ts b/packages/web_ui/src/model/host.ts index 1e6ffe7c..3d300a46 100644 --- a/packages/web_ui/src/model/host.ts +++ b/packages/web_ui/src/model/host.ts @@ -1,6 +1,5 @@ import { useCallback, useContext, useSyncExternalStore } from "react"; import ControlContext from "../components/ControlContext"; -import { useAccount } from "./account"; export function useHost(id?: number) { const [hosts, synced] = useHosts(); @@ -13,14 +12,9 @@ export function useHosts() { return useSyncExternalStore(subscribe, () => control.hosts.getSnapshot()); } -/** Requests from host installers waiting to be approved, empty if the account may not approve hosts */ +/** Requests from hosts waiting to be approved, empty if the account may not approve hosts */ export function useHostPairings() { const control = useContext(ControlContext); - const account = useAccount(); - const permitted = Boolean(account.hasPermission("core.host.approve")); - const subscribe = useCallback( - (callback: () => void) => (permitted ? control.hostPairings.subscribe(callback) : () => {}), - [control, permitted], - ); + const subscribe = useCallback((callback: () => void) => control.hostPairings.subscribe(callback), [control]); return useSyncExternalStore(subscribe, () => control.hostPairings.getSnapshot()); } diff --git a/packages/web_ui/src/util/host_pairing.ts b/packages/web_ui/src/util/host_pairing.ts index c8cb79b0..480a8455 100644 --- a/packages/web_ui/src/util/host_pairing.ts +++ b/packages/web_ui/src/util/host_pairing.ts @@ -1,19 +1,13 @@ import * as lib from "@clusterio/lib"; /** - * Subscriber for the requests from host installers waiting to be approved + * Subscriber for the requests from hosts waiting to be approved * - * Only accounts that may approve hosts can subscribe to the requests, so - * this does not resubscribe on connect or account changes for other - * accounts, which would fail with a permission error. The requests only - * live in the controller's memory and it does not tell a new session about - * requests removed before it started, so the requests are cleared when a - * new session starts and fetched again. The same applies to requests removed - * while nothing was subscribed, so they are also cleared when subscribing - * again after the last handler unsubscribed. + * The requests only live in the controller's memory and it does not tell + * subscribers about requests removed while nothing was subscribed, so they + * are cleared when subscribing again after the last handler unsubscribed. */ export class HostPairingSubscriber extends lib.MapSubscriber { - private _permitted: () => boolean; private _handlers = 0; /** @@ -21,31 +15,7 @@ export class HostPairingSubscriber extends lib.MapSubscriber boolean) { - super(lib.HostPairingUpdatesEvent, control); - this._permitted = permitted; - } - - override handleConnectionEvent(event: "connect" | "drop" | "resume" | "close") { - if (event === "connect") { - this.values.clear(); - this.lastUpdatedMs = -1; - } - if (event === "connect" || event === "resume") { - // The account of a new session is reported after the connect event. - queueMicrotask(() => { - if (this._permitted()) { - super.handleConnectionEvent(event); - } - }); - return; - } - super.handleConnectionEvent(event); - } - - override handleAccountEvent() { - if (this._permitted()) { - super.handleAccountEvent(); - } + super(lib.HostPairingUpdatesEvent, control, undefined, permitted); } override subscribe(handler: lib.EventSubscriberCallback) { diff --git a/test/controller/ControlConnection.test.js b/test/controller/ControlConnection.test.js index 39c8ee95..6311c2b1 100644 --- a/test/controller/ControlConnection.test.js +++ b/test/controller/ControlConnection.test.js @@ -326,16 +326,9 @@ describe("controller/src/ControlConnection", function() { await connection.handleHostRevokeRequest(new lib.HostRevokeRequest(hostId)); } - it("deletes and saves the host record", async function() { - let savedWithoutHost; - const originalSave = controller.hosts.save; - controller.hosts.save = async function() { - savedWithoutHost = !this.has(10); - return await originalSave.call(this); - }; + it("deletes the host record", async function() { await revoke(); assert.equal(controller.hosts.has(10), false); - assert.equal(savedWithoutHost, true, "hosts were not saved after deleting the record"); assert.deepEqual(broadcasts, [{ id: 10, isDeleted: true }]); assert.equal(audits.length, 1); assert.match(audits[0], /User admin revoked host host-10 \(10\)/); @@ -356,15 +349,6 @@ describe("controller/src/ControlConnection", function() { await revoke(); assert.equal(terminated, true); }); - it("terminates the connection of the host when saving fails", async function() { - let terminated = false; - controller.wsServer.hostConnections.set(10, { connector: { terminate() { terminated = true; } } }); - controller.hosts.save = async () => { throw new Error("EPERM: operation not permitted"); }; - await assert.rejects(revoke(), /EPERM/); - assert.equal(terminated, true, "revoked host kept its connection"); - assert.equal(controller.hosts.has(10), false); - assert.equal(controller.notes.get("host/10"), undefined); - }); it("does not leave a host record behind when the connection closes", async function() { const connector = new MockConnector(addr("controller"), addr({ hostId: 10 })); connector.terminate = () => { connector.emit("close"); }; @@ -378,27 +362,31 @@ describe("controller/src/ControlConnection", function() { assert.equal(controller.hosts.has(10), false, "host record was recreated"); assert.deepEqual(broadcasts.at(-1), { id: 10, isDeleted: true }); }); - it("refuses to revoke a host with assigned instances", async function() { - controller.setNote("host", "10", "rack 3", "someone"); - let terminated = false; - controller.wsServer.hostConnections.set(10, { connector: { terminate() { terminated = true; } } }); + it("unassigns the instances of the host without waiting for it", async function() { + let closing = false; + controller.wsServer.hostConnections.set(10, { + connector: { terminate() { closing = true; }, get closing() { return closing; } }, + send: () => new Promise(() => {}), + }); assignInstance(101, "alpha", 10); assignInstance(102, "beta", 10); assignInstance(201, "gamma", 20); - await assert.rejects( - revoke(), - new lib.ConflictError("Cannot revoke host host-10 while instances are assigned to it: alpha, beta"), - ); - assert.equal(controller.hosts.get(10), record); - assert.equal(controller.notes.get("host/10").isDeleted, false); - assert.equal(terminated, false); - assert.deepEqual(broadcasts, []); - }); - it("ignores instances assigned to other hosts", async function() { - assignInstance(201, "gamma", 20); await revoke(); assert.equal(controller.hosts.has(10), false); + for (const id of [101, 102]) { + const instance = controller.instances.get(id); + assert.equal(instance.config.get("instance.assigned_host"), null); + assert.equal(instance.status, "unassigned"); + } + assert.equal(controller.instances.get(201).config.get("instance.assigned_host"), 20); + }); + it("removes the system metrics of the host", async function() { + controller.systems.set(await lib.gatherSystemInfo(10, false, false, 0)); + controller.systems.set(await lib.gatherSystemInfo("controller", false, false, 0)); + await revoke(); + assert.equal(controller.systems.has(10), false); + assert.equal(controller.systems.has("controller"), true); }); it("throws NotFoundError for an unknown host", async function() { await assert.rejects(revoke(99), new lib.NotFoundError("Unknown host id (99)")); @@ -421,7 +409,7 @@ describe("controller/src/ControlConnection", function() { const rejected = controller.hostPairing.create("Host C", "", "10.0.0.3"); const pending = controller.hostPairing.create("Host D", "", "10.0.0.4"); controller.hostPairing._bySecret.get(expired.requestId).lastPollMs -= 60e3; - await controller.hostPairing.approve(approved.code, undefined, "admin"); + controller.hostPairing.approve(approved.code, undefined, "admin"); controller.hostPairing.reject(rejected.code, "admin"); const list = await connection.handleHostPairingListRequest(new lib.HostPairingListRequest()); @@ -440,7 +428,7 @@ describe("controller/src/ControlConnection", function() { it("rejects a request", async function() { const { requestId, code } = controller.hostPairing.create("Host A", "", "10.0.0.1"); await connection.handleHostPairingRejectRequest(new lib.HostPairingRejectRequest(code.toLowerCase())); - assert.equal(controller.hostPairing.requests.has(code), false); + assert.equal(controller.hostPairing.records.has(code), false); assert.throws(() => controller.hostPairing.poll(requestId), /Request was rejected/); assert.equal(controller.hosts.size, 0); assert(audits.some(msg => msg.startsWith(`User admin rejected host Host A with code ${code}`))); diff --git a/test/controller/HostPairingManager.test.js b/test/controller/HostPairingManager.test.js index c47f20fe..a1db9b2a 100644 --- a/test/controller/HostPairingManager.test.js +++ b/test/controller/HostPairingManager.test.js @@ -5,8 +5,7 @@ import jwt from "jsonwebtoken"; import * as lib from "@clusterio/lib"; import { Controller, HostPairingManager, HostPairingError, HostRecord } from "@clusterio/controller"; import { - pollTimeoutMs, maxPendingMs, maxPending, maxPendingPerAddress, - maxNameLength, + pollTimeoutMs, maxPendingPerAddress, maxNameLength, } from "@clusterio/controller/dist/node/src/HostPairingManager.js"; import "../setup_logging.js"; @@ -31,12 +30,6 @@ function stubRandomInt(values) { return () => { crypto.randomInt = original; }; } -function deferred() { - let resolve, reject; - const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); - return { promise, resolve, reject }; -} - function assertPairingError(fn, status, message) { assert.throws(fn, err => { assert(err instanceof HostPairingError, `expected HostPairingError, got ${err}`); @@ -95,7 +88,7 @@ describe("controller/src/HostPairingManager", function() { controller = makeController(); manager = controller.hostPairing; updates = []; - manager.requests.on("update", changed => { + manager.records.on("update", changed => { updates.push(...changed.map(details => ({ code: details.code, isDeleted: details.isDeleted }))); }); restoreRandomInt = undefined; @@ -111,15 +104,15 @@ describe("controller/src/HostPairingManager", function() { return manager._bySecret.get(requestId); } - // Pretend the installer last polled the request ms milliseconds earlier. + // Pretend the host last polled the request ms milliseconds earlier. function idle(requestId, ms) { internal(requestId).lastPollMs -= ms; } describe(".createHost()", function() { - it("should create a host record with a valid token", async function() { + it("should create a host record with a valid token", function() { const before = Math.floor(Date.now() / 1000); - const result = await manager.createHost("Host A"); + const result = manager.createHost("Host A"); assert.equal(result.name, "Host A"); assert(Number.isInteger(result.hostId), "hostId is not an integer"); @@ -136,31 +129,10 @@ describe("controller/src/HostPairingManager", function() { assert.equal(payload.host, result.hostId); assert(payload.iat >= record.tokenValidAfter, "token is not valid for the new record"); }); - it("should save the hosts before returning", async function() { - const save = deferred(); - let saveCalls = 0; - controller.hosts.save = () => { saveCalls += 1; return save.promise; }; - - let done = false; - const promise = manager.createHost("Host A").then(result => { done = true; return result; }); - await new Promise(resolve => setImmediate(resolve)); - assert.equal(saveCalls, 1); - assert.equal(done, false, "createHost returned before the hosts were saved"); - save.resolve(); - const result = await promise; - assert(controller.hosts.has(result.hostId)); - }); - it("should remove the record if saving fails", async function() { - restoreRandomInt = stubRandomInt([1234]); - controller.hosts.save = async () => { throw new Error("disk full"); }; - await assert.rejects(manager.createHost("Host A"), /disk full/); - assert.equal(controller.hosts.has(1234), false); - assert.equal(controller.hosts.size, 0); - }); - it("should not reuse the id of an existing host", async function() { + it("should not reuse the id of an existing host", function() { controller.hosts.set(new HostRecord(5, "existing", "", new Map())); restoreRandomInt = stubRandomInt([5, 5, 6]); - const result = await manager.createHost("Host A"); + const result = manager.createHost("Host A"); assert.equal(result.hostId, 6); assert.equal(controller.hosts.get(5).name, "existing"); assert.equal(controller.hosts.get(6).name, "Host A"); @@ -177,7 +149,7 @@ describe("controller/src/HostPairingManager", function() { assert.match(code, codePattern); assert.notEqual(requestId, code); - const details = manager.requests.get(code); + const details = manager.records.get(code); assert(details instanceof lib.HostPairingDetails); assert.equal(details.code, code); assert.equal(details.name, "Host A"); @@ -191,23 +163,23 @@ describe("controller/src/HostPairingManager", function() { }); it("should create unique codes", function() { const codes = new Set(); - for (let i = 0; i < maxPending; i++) { + for (let i = 0; i < 50; i++) { const { code } = manager.create(`Host ${i}`, "", `10.1.${i}.1`); assert.match(code, codePattern); codes.add(code); } - assert.equal(codes.size, maxPending); + assert.equal(codes.size, 50); }); it("should not reuse the code of a pending request", function() { restoreRandomInt = stubRandomInt([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]); assert.equal(manager.create("Host A", "", "10.0.0.1").code, "222-222"); assert.equal(manager.create("Host B", "", "10.0.0.2").code, "333-333"); }); - it("should not reuse the code of an approved request not yet picked up", async function() { + it("should not reuse the code of an approved request not yet picked up", function() { restoreRandomInt = stubRandomInt([0, 0, 0, 0, 0, 0, 1234, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]); const { code } = manager.create("Host A", "", "10.0.0.1"); assert.equal(code, "222-222"); - assert.equal(await manager.approve(code, undefined, "admin"), 1234); + assert.equal(manager.approve(code, undefined, "admin"), 1234); assert.equal(manager.create("Host B", "", "10.0.0.2").code, "333-333"); }); }); @@ -227,7 +199,7 @@ describe("controller/src/HostPairingManager", function() { }); // Other addresses are not affected manager.create("Host", "", "10.0.0.2"); - assert.equal(manager.requests.size, maxPendingPerAddress + 1); + assert.equal(manager.records.size, maxPendingPerAddress + 1); }); it("should count IPv4 mapped addresses as the IPv4 address", function() { manager.create("Host", "", "10.0.0.1"); @@ -245,7 +217,7 @@ describe("controller/src/HostPairingManager", function() { ); manager.create("Host", "", "2001:db8:0:2::1"); }); - it("should free the slot when a request is no longer pending", async function() { + it("should free the slot when a request is no longer pending", function() { const first = manager.create("Host", "", "10.0.0.1"); const second = manager.create("Host", "", "10.0.0.1"); const third = manager.create("Host", "", "10.0.0.1"); @@ -256,14 +228,15 @@ describe("controller/src/HostPairingManager", function() { assertPairingError(() => manager.create("Host", "", "10.0.0.1"), 429); // Approved and rejected results waiting to be picked up do not count - await manager.approve(second.code, undefined, "admin"); + manager.approve(second.code, undefined, "admin"); manager.reject(third.code, "admin"); manager.create("Host", "", "10.0.0.1"); manager.create("Host", "", "10.0.0.1"); assertPairingError(() => manager.create("Host", "", "10.0.0.1"), 429); - assert(manager.requests.has(fourth.code)); + assert(manager.records.has(fourth.code)); }); it("should limit the total number of pending requests", function() { + const maxPending = controller.config.get("controller.host_pairing_max_pending"); assert.equal(maxPending, 50); for (let i = 0; i < maxPending; i++) { manager.create(`Host ${i}`, "", `10.1.${i}.1`); @@ -275,7 +248,14 @@ describe("controller/src/HostPairingManager", function() { assert.equal(err.message, "Too many pending requests"); return true; }); - assert.equal(manager.requests.size, maxPending); + assert.equal(manager.records.size, maxPending); + }); + it("should not limit the total number of pending requests when set to 0", function() { + controller.config.set("controller.host_pairing_max_pending", 0); + for (let i = 0; i < 60; i++) { + manager.create(`Host ${i}`, "", `10.1.${i}.1`); + } + assert.equal(manager.records.size, 60); }); }); @@ -292,9 +272,9 @@ describe("controller/src/HostPairingManager", function() { const { code } = manager.create("Host A", "", "10.0.0.1"); assertPairingError(() => manager.poll(code), 404); }); - it("should hand over an approved result once", async function() { + it("should hand over an approved result once", function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - const hostId = await manager.approve(code, undefined, "admin"); + const hostId = manager.approve(code, undefined, "admin"); const result = manager.poll(requestId); assert.deepEqual(Object.keys(result).sort(), ["hostId", "name", "token"]); assert.equal(result.hostId, hostId); @@ -317,7 +297,7 @@ describe("controller/src/HostPairingManager", function() { it("should remove a pending request", function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); manager.cancel(requestId); - assert.equal(manager.requests.has(code), false); + assert.equal(manager.records.has(code), false); assertPairingError(() => manager.poll(requestId), 404); assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); @@ -325,9 +305,9 @@ describe("controller/src/HostPairingManager", function() { it("should ignore unknown requests", function() { manager.cancel("invalid"); }); - it("should not throw away an approved result", async function() { + it("should not throw away an approved result", function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - const hostId = await manager.approve(code, undefined, "admin"); + const hostId = manager.approve(code, undefined, "admin"); manager.cancel(requestId); assert.equal(manager.poll(requestId).hostId, hostId); }); @@ -340,92 +320,58 @@ describe("controller/src/HostPairingManager", function() { }); describe(".approve()", function() { - it("should create the host with the requested name", async function() { + it("should create the host with the requested name", function() { const { code } = manager.create("Host A", "", "10.0.0.1"); - const hostId = await manager.approve(code, undefined, "admin"); + const hostId = manager.approve(code, undefined, "admin"); assert.equal(controller.hosts.get(hostId).name, "Host A"); - assert.equal(manager.requests.has(code), false); + assert.equal(manager.records.has(code), false); assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); }); - it("should accept codes in any case and without the dash", async function() { + it("should accept codes in any case and without the dash", function() { const { code } = manager.create("Host A", "", "10.0.0.1"); - const hostId = await manager.approve(code.toLowerCase().replace("-", ""), undefined, "admin"); + const hostId = manager.approve(code.toLowerCase().replace("-", ""), undefined, "admin"); assert(controller.hosts.has(hostId)); }); - it("should use and trim the name given", async function() { + it("should use and trim the name given", function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - const hostId = await manager.approve(code, " Renamed ", "admin"); + const hostId = manager.approve(code, " Renamed ", "admin"); assert.equal(controller.hosts.get(hostId).name, "Renamed"); assert.equal(manager.poll(requestId).name, "Renamed"); }); - it("should reject an invalid name and keep the request pending", async function() { + it("should reject an invalid name and keep the request pending", function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - await assert.rejects( - manager.approve(code, "", "admin"), + assert.throws( + () => manager.approve(code, "", "admin"), new lib.ValidationError("Host name is required"), ); - await assert.rejects( - manager.approve(code, "x".repeat(maxNameLength + 1), "admin"), + assert.throws( + () => manager.approve(code, "x".repeat(maxNameLength + 1), "admin"), new lib.ValidationError("Host name must be at most 100 characters"), ); assert.equal(controller.hosts.size, 0); assert.equal(manager.poll(requestId), null); - await manager.approve(code, undefined, "admin"); + manager.approve(code, undefined, "admin"); assert.equal(controller.hosts.size, 1); }); - it("should throw NotFoundError for an unknown code", async function() { - await assert.rejects( - manager.approve("ZZZ-ZZZ", undefined, "admin"), + it("should throw NotFoundError for an unknown code", function() { + assert.throws( + () => manager.approve("ZZZ-ZZZ", undefined, "admin"), new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), ); }); - it("should only approve a request once", async function() { + it("should only approve a request once", function() { const { code } = manager.create("Host A", "", "10.0.0.1"); - await manager.approve(code, undefined, "admin"); - await assert.rejects(manager.approve(code, undefined, "admin"), lib.NotFoundError); + manager.approve(code, undefined, "admin"); + assert.throws(() => manager.approve(code, undefined, "admin"), lib.NotFoundError); assert.equal(controller.hosts.size, 1); }); - it("should not create two hosts when approved twice at the same time", async function() { - const save = deferred(); - controller.hosts.save = () => save.promise; - const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - - const first = manager.approve(code, undefined, "admin"); - await assert.rejects(manager.approve(code, undefined, "admin"), lib.NotFoundError); - assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); - - // The request stays while approving, the installer sees it as pending - manager.cancel(requestId); - idle(requestId, pollTimeoutMs + 1); - manager.prune(); - assert.equal(manager.poll(requestId), null); - assert(manager.requests.has(code), "request removed while approving"); - - save.resolve(); - const hostId = await first; - assert.equal(controller.hosts.size, 1); - assert.equal(manager.poll(requestId).hostId, hostId); - }); - it("should keep the request pending if creating the host fails", async function() { - const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - const originalSave = controller.hosts.save; - controller.hosts.save = async () => { throw new Error("disk full"); }; - await assert.rejects(manager.approve(code, undefined, "admin"), /disk full/); - assert.equal(controller.hosts.size, 0); - assert(manager.requests.has(code)); - assert.equal(manager.poll(requestId), null); - - controller.hosts.save = originalSave; - const hostId = await manager.approve(code, undefined, "admin"); - assert.equal(manager.poll(requestId).hostId, hostId); - }); }); describe(".reject()", function() { it("should remove the request from the pending list", function() { const { code } = manager.create("Host A", "", "10.0.0.1"); manager.reject(code.toLowerCase(), "admin"); - assert.equal(manager.requests.has(code), false); + assert.equal(manager.records.has(code), false); assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); assert.equal(controller.hosts.size, 0); }); @@ -435,9 +381,9 @@ describe("controller/src/HostPairingManager", function() { new lib.NotFoundError("No pending host request with code ZZZ-ZZZ"), ); }); - it("should not reject an approved request", async function() { + it("should not reject an approved request", function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - const hostId = await manager.approve(code, undefined, "admin"); + const hostId = manager.approve(code, undefined, "admin"); assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); assert.equal(manager.poll(requestId).hostId, hostId); }); @@ -455,11 +401,11 @@ describe("controller/src/HostPairingManager", function() { lib.logger.warn = originalWarn; }); - it("should expire a request the installer stopped polling", function() { + it("should expire a request the host stopped polling", function() { const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); idle(requestId, pollTimeoutMs + 1); manager.prune(); - assert.equal(manager.requests.has(code), false); + assert.equal(manager.records.has(code), false); assert.deepEqual(updates, [{ code, isDeleted: false }, { code, isDeleted: true }]); assertPairingError(() => manager.poll(requestId), 404); assert.throws(() => manager.reject(code, "admin"), lib.NotFoundError); @@ -472,22 +418,31 @@ describe("controller/src/HostPairingManager", function() { assert.equal(manager.poll(requestId), null); idle(requestId, pollTimeoutMs - 1000); manager.prune(); - assert(manager.requests.has(code)); + assert(manager.records.has(code)); }); it("should expire a request that has been pending for too long", function() { - assert.equal(maxPendingMs, 60 * 60e3); + const timeoutMs = controller.config.get("controller.host_pairing_timeout") * 1000; + assert.equal(timeoutMs, 60 * 60e3); const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - internal(requestId).details.createdAtMs -= maxPendingMs - 1000; + internal(requestId).details.createdAtMs -= timeoutMs - 1000; manager.prune(); assert.equal(manager.poll(requestId), null); internal(requestId).details.createdAtMs -= 1001; manager.prune(); - assert.equal(manager.requests.has(code), false); + assert.equal(manager.records.has(code), false); assertPairingError(() => manager.poll(requestId), 404); }); - it("should drop and warn about an approved host never picked up", async function() { + it("should not limit how long a request is pending when the timeout is 0", function() { + controller.config.set("controller.host_pairing_timeout", 0); const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); - const hostId = await manager.approve(code, undefined, "admin"); + internal(requestId).details.createdAtMs -= 365 * 24 * 60 * 60e3; + manager.prune(); + assert(manager.records.has(code)); + assert.equal(manager.poll(requestId), null); + }); + it("should drop and warn about an approved host never picked up", function() { + const { requestId, code } = manager.create("Host A", "", "10.0.0.1"); + const hostId = manager.approve(code, undefined, "admin"); idle(requestId, pollTimeoutMs + 1); manager.prune(); assertPairingError(() => manager.poll(requestId), 404); @@ -526,7 +481,7 @@ describe("controller/src/HostPairingManager", function() { assert(event instanceof lib.HostPairingUpdatesEvent); assert.deepEqual(event.updates.map(details => details.code), [code]); - const lastUpdate = manager.requests.get(code).updatedAtMs; + const lastUpdate = manager.records.get(code).updatedAtMs; assert.equal(await controller.handleHostPairingSubscription( new lib.SubscriptionRequest(lib.HostPairingUpdatesEvent.name, true, lastUpdate) ), null); diff --git a/test/controller/routes.js b/test/controller/routes.js index b0b88cd2..bd516690 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -254,7 +254,7 @@ describe("controller/src/routes", function() { } function assertNothingCreated() { - assert.equal(controller.hostPairing.requests.size, 0, "a pending request was created"); + assert.equal(controller.hostPairing.records.size, 0, "a pending request was created"); assert.equal(controller.hosts.size, 0, "a host was created"); } @@ -267,7 +267,7 @@ describe("controller/src/routes", function() { assert.deepEqual(Object.keys(body).sort(), ["code", "requestId"]); assert.match(body.code, /^[2-9A-HJKMNP-Z]{3}-[2-9A-HJKMNP-Z]{3}$/); - const details = controller.hostPairing.requests.get(body.code); + const details = controller.hostPairing.records.get(body.code); assert.equal(details.name, "Host A"); assert.equal(details.version, "2.0.0"); assert.match(details.remoteAddress, /127\.0\.0\.1|::1/); @@ -275,12 +275,12 @@ describe("controller/src/routes", function() { }); it("should trim the name and limit the version", async function() { let { code } = await (await post("", { name: " Host A ", version: "x".repeat(100) })).json(); - let details = controller.hostPairing.requests.get(code); + let details = controller.hostPairing.records.get(code); assert.equal(details.name, "Host A"); assert.equal(details.version, "x".repeat(64)); ({ code } = await (await post("", { name: "Host B", version: 5 })).json()); - details = controller.hostPairing.requests.get(code); + details = controller.hostPairing.records.get(code); assert.equal(details.version, ""); }); it("should respond with 400 for a missing or invalid name", async function() { @@ -316,7 +316,7 @@ describe("controller/src/routes", function() { const response = await post("", { name: "Host A" }); assert.equal(response.headers.get("retry-after"), "30"); await assertResponse(response, 429, { error: "Too many pending requests from this address" }); - assert.equal(controller.hostPairing.requests.size, 3); + assert.equal(controller.hostPairing.records.size, 3); }); it("should respond with 503 in recovery mode", async function() { controller.recoveryMode = true; @@ -344,7 +344,7 @@ describe("controller/src/routes", function() { assert.equal(payload.host, body.hostId); } assert.equal(controller.hosts.size, 3); - assert.equal(controller.hostPairing.requests.size, 0, "a pending request was created"); + assert.equal(controller.hostPairing.records.size, 0, "a pending request was created"); }); it("should validate the name before approving directly", async function() { await assertResponse( @@ -393,7 +393,7 @@ describe("controller/src/routes", function() { }); it("should hand over the result of an approved request once", async function() { const { requestId, code } = await createRequest(); - const hostId = await controller.hostPairing.approve(code, "Renamed", "test"); + const hostId = controller.hostPairing.approve(code, "Renamed", "test"); const response = await post("/poll", { requestId }); assert.equal(response.status, 200); @@ -438,14 +438,14 @@ describe("controller/src/routes", function() { it("should cancel a pending request", async function() { const { requestId, code } = await createRequest(); await assertResponse(await post("/cancel", { requestId }), 204); - assert.equal(controller.hostPairing.requests.has(code), false); + assert.equal(controller.hostPairing.records.has(code), false); await assertResponse(await post("/poll", { requestId }), 404, { error: "Request not found" }); }); it("should respond with 204 for an unknown request", async function() { const { code } = await createRequest(); await assertResponse(await post("/cancel", { requestId: "invalid" }), 204); await assertResponse(await post("/cancel", { requestId: code }), 204); - assert(controller.hostPairing.requests.has(code), "cancelled by the code"); + assert(controller.hostPairing.records.has(code), "cancelled by the code"); }); }); @@ -470,7 +470,7 @@ describe("controller/src/routes", function() { describe("GET /api/host-pairing/check", function() { it("should respond with 200 for a valid host token", async function() { - const { hostId, token } = await controller.hostPairing.createHost("Host A"); + const { hostId, token } = controller.hostPairing.createHost("Host A"); for (const headers of [ { "Authorization": `Bearer ${token}` }, { "X-Access-Token": token }, @@ -480,18 +480,18 @@ describe("controller/src/routes", function() { assert(controller.hosts.has(hostId)); }); it("should respond with 200 for a host token from before the slave rename", async function() { - const { hostId } = await controller.hostPairing.createHost("Host A"); + const { hostId } = controller.hostPairing.createHost("Host A"); const token = jwt.sign({ aud: "slave", slave: hostId }, controller.authSecret); await assertResponse(await check({ "Authorization": `Bearer ${token}` }), 200, { valid: true }); }); it("should respond with 401 without a valid host token", async function() { - await controller.hostPairing.createHost("Host A"); + controller.hostPairing.createHost("Host A"); assert.equal((await check({})).status, 401); assert.equal((await check({ "Authorization": "Bearer invalid" })).status, 401); assert.equal((await check({ "Authorization": `Bearer ${adminToken}` })).status, 401); }); it("should respond with 401 once the host is revoked", async function() { - const { hostId, token } = await controller.hostPairing.createHost("Host A"); + const { hostId, token } = controller.hostPairing.createHost("Host A"); await assertResponse(await check({ "Authorization": `Bearer ${token}` }), 200, { valid: true }); controller.hosts.delete(controller.hosts.get(hostId)); assert.equal((await check({ "Authorization": `Bearer ${token}` })).status, 401); diff --git a/test/create.js b/test/create.js index af67b8fb..36f0559c 100644 --- a/test/create.js +++ b/test/create.js @@ -179,7 +179,6 @@ describe("create installer CLI", function() { it("writes the host config and leaves joining the cluster to the host", async function() { const { stdout } = await run(hostArgs()); assert.match(stdout, /Successfully installed host/); - assert.match(stdout, /Start the host with run-host/); const recorded = await commands(); assert.match(recorded[0], /^npm(?:\.cmd)? install @clusterio\/host$/); @@ -208,7 +207,6 @@ describe("create installer CLI", function() { "--factorio-dir", "factorio", "--remote-npm", "--plugins", ]); assert.match(stdout, /Successfully installed standalone/); - assert.match(stdout, /Start the host with run-host/); const recorded = await commands(); for (const expected of [ @@ -225,5 +223,14 @@ describe("create installer CLI", function() { } assert(!recorded.some(command => command.includes("host.controller_token"))); }); + + it("uses the host name given for a standalone install", async function() { + await fs.writeFile(path.join(directory, "config-controller.json"), "{}"); + await run([ + "--mode", "standalone", "--admin", "admin", "--http-port", "8123", "--public-address", "localhost", + "--factorio-dir", "factorio", "--remote-npm", "--plugins", "--host-name", "Test Host", + ]); + assert((await commands()).includes(hostSet("host.name", "Test Host"))); + }); }); }); diff --git a/test/host/pairing.js b/test/host/pairing.js index 7bbede3a..bb5e98c6 100644 --- a/test/host/pairing.js +++ b/test/host/pairing.js @@ -162,6 +162,22 @@ describe("host/src/pairing", function() { stub = await startStub(); }); + it("keeps polling while the controller is busy", async function() { + const handler = approveAfter(0); + let busy = 2; + stub.handler = entry => { + if (entry.url === "/api/host-pairing/poll" && busy > 0) { + busy -= 1; + return [503, { error: "Recovery mode" }]; + } + return handler(entry); + }; + const retries = []; + const result = await pair({ onRetry: err => { retries.push(err.message); } }); + assert.deepEqual(result, approved); + assert.deepEqual(retries, ["Recovery mode", "Recovery mode"]); + }); + it("throws when the request is rejected", async function() { const handler = approveAfter(Infinity); stub.handler = entry => (entry.url === "/api/host-pairing/poll" && stub.requests.length > 2 @@ -248,8 +264,17 @@ describe("host/src/pairing", function() { it("waits for approval and saves the host when not paired", async function() { this.timeout(5000); // Polls at the default interval of 2 seconds stub.handler = approveAfter(0); - await ensurePaired(config, { version: "2.1.0" }); + const warnings = []; + const originalWarn = lib.logger.warn; + lib.logger.warn = message => { warnings.push(message); }; + try { + await ensurePaired(config, { version: "2.1.0" }); + } finally { + lib.logger.warn = originalWarn; + } assert.deepEqual(stub.requests[0].body, { name: "My Host", version: "2.1.0" }); + // The systemd service created by the installer only prints warnings and above. + assert(warnings.includes(`${stub.url}hosts?approve=ABC-DE1`), "approval link is not a warning"); const saved = await savedConfig(); assert.equal(saved["host.id"], 1234); assert.equal(saved["host.name"], "Approved"); @@ -284,6 +309,20 @@ describe("host/src/pairing", function() { assert(!JSON.stringify(saved).includes("user-token"), "user token must not be saved"); }); + it("stops when aborted while checking the saved token", async function() { + config.set("host.controller_token", "some-token"); + const abort = new AbortController(); + stub.handler = () => { + abort.abort(); + return new Promise(() => {}); + }; + await assert.rejects( + ensurePaired(config, { version: "2.1.0", signal: abort.signal }), + { name: "AbortError" }, + ); + assert.equal(config.get("host.controller_token"), "some-token"); + }); + it("keeps the saved token when it can't be checked", async function() { config.set("host.controller_token", "some-token"); stub.handler = () => [404, undefined]; diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 6183856f..54dcd29b 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -851,7 +851,7 @@ describe("Integration of Clusterio", function() { assert(!notes.some(note => note.id === `host/${pairing.hostId}`), "Note of the host was not removed"); }); - it("should refuse while instances are assigned to the host", async function() { + it("should unassign the instances of the host", async function() { const pairing = await pairTestHost("assigned-host"); // Instances can only be assigned to connected hosts, a link that accepts the assignment is enough. const connector = new TestHostConnector(url, 0.2); @@ -863,19 +863,17 @@ describe("Integration of Clusterio", function() { await execCtl("instance create revoke-assigned --id 45"); try { await execCtl(`instance assign revoke-assigned ${pairing.hostId}`); - await assert.rejects( - execCtl(`host revoke ${pairing.hostId}`), - /Cannot revoke host assigned-host while instances are assigned to it: revoke-assigned/, - ); + connector.setClosing(); // Stops it from reconnecting after being revoked + await execCtl(`host revoke ${pairing.hostId}`); + + const hosts = await getControl().send(new lib.HostListRequest()); + assert(!hosts.some(host => host.id === pairing.hostId), "Host was not removed"); + const instance = await getControl().send(new lib.InstanceDetailsGetRequest(45)); + assert.equal(instance.assignedHost, undefined); + assert.equal(instance.status, "unassigned"); } finally { - await connector.disconnect(); - await execCtl("instance assign revoke-assigned"); await execCtl("instance delete revoke-assigned"); } - - await execCtl(`host revoke ${pairing.hostId}`); - const hosts = await getControl().send(new lib.HostListRequest()); - assert(!hosts.some(host => host.id === pairing.hostId), "Host was not removed"); }); it("should stop a running host and have it ask to join again when started", async function() { diff --git a/test/integration/index.js b/test/integration/index.js index e29d96a2..781f5cf1 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -311,22 +311,19 @@ export function spawn(name, cmd, waitFor, options = {}) { let process = child_process.spawn(parts[0], parts.slice(1), childOptions(options)); let stdout = new LineSplitter({ readableObjectMode: true }); let stderr = new LineSplitter({ readableObjectMode: true }); - let onDataOut = line => { + let onData = line => { line = line.toString("utf8"); if (waitFor.test(line)) { if (silent) { - stdout.off("data", onDataOut); - stderr.off("data", onDataErr); + stdout.off("data", onData); + stderr.off("data", onData); } resolve(process); } log(name, line); }; - let onDataErr = line => { - log(name, line.toString("utf8")); - }; - stdout.on("data", onDataOut); - stderr.on("data", onDataErr); + stdout.on("data", onData); + stderr.on("data", onData); process.stdout.pipe(stdout); process.stderr.pipe(stderr); }); diff --git a/test/lib/subscriptions.test.js b/test/lib/subscriptions.test.js index c71923f8..1d4c535d 100644 --- a/test/lib/subscriptions.test.js +++ b/test/lib/subscriptions.test.js @@ -1250,6 +1250,36 @@ describe("lib/subscriptions", function() { }); }); + describe("permitted", function() { + let permitted; + let subscriber; + + beforeEach(function() { + mockControl = new MockControl(new MockConnector(addr({ controlId: 0 }), addr("controller"))); + permitted = false; + subscriber = new lib.EventSubscriber(RegisteredEvent, mockControl, undefined, () => permitted); + }); + + it("should send nothing while not permitted", async function() { + subscriber.subscribe(() => {}); + mockControl.connector.emit("connect"); + subscriber.handleConnectionEvent("resume"); + subscriber.handleAccountEvent(); + await new Promise(r => setImmediate(r)); + assertNoMessageSent(); + }); + + it("should subscribe when the account becomes permitted", async function() { + subscriber.subscribe(() => {}); + permitted = true; + subscriber.handleAccountEvent(); + await onceConnectorSend(); + assertLastMessage(new lib.SubscriptionRequest( + RegisteredEvent.name, "replace", -1, lib.SubscriptionFilters.all() + )); + }); + }); + describe(".handleAccountEvent()", function() { it("should send replace(all) if subscribed", async function() { eventSubscriber.subscribe(() => {}); diff --git a/test/mock.js b/test/mock.js index 035a54fb..337e8d7e 100644 --- a/test/mock.js +++ b/test/mock.js @@ -176,6 +176,8 @@ export class MockController { ["controller.auth_secret", "TestSecretDoNotUse"], ["controller.proxy_stream_timeout", 1], ["controller.metrics_timeout", 1], + ["controller.host_pairing_timeout", 3600], + ["controller.host_pairing_max_pending", 50], ]); this.config = { get: (name) => { diff --git a/test/web_ui/host_pairing.js b/test/web_ui/host_pairing.js index 5aa64a70..41078578 100644 --- a/test/web_ui/host_pairing.js +++ b/test/web_ui/host_pairing.js @@ -12,13 +12,11 @@ const addr = lib.Address.fromShorthand; describe("web_ui/util/host_pairing", function() { describe("class HostPairingSubscriber", function() { let mockControl; - let permitted; let subscriber; beforeEach(function() { mockControl = new MockControl(new MockConnector(addr({ controlId: 0 }), addr("controller"))); - permitted = true; - subscriber = new HostPairingSubscriber(mockControl, () => permitted); + subscriber = new HostPairingSubscriber(mockControl, () => true); }); function pairing(code, updatedAtMs) { @@ -33,65 +31,6 @@ describe("web_ui/util/host_pairing", function() { return mockControl.connector.sentMessages.filter(message => message.name === "SubscriptionRequest"); } - it("should not send requests on connect or account updates without permission", async function() { - permitted = false; - mockControl.connector.emit("connect"); - subscriber.handleConnectionEvent("resume"); - subscriber.handleAccountEvent(); - await settle(); - assert.deepEqual(sentRequests(), []); - }); - - it("should resubscribe on connect with permission", async function() { - subscriber.subscribe(() => {}); - await events.once(mockControl.connector, "send"); - mockControl.connector.sentMessages.length = 0; - - mockControl.connector.emit("connect"); - await settle(); - assert.equal(sentRequests().length, 1); - assert.deepEqual(sentRequests()[0].data, new lib.SubscriptionRequest( - lib.HostPairingUpdatesEvent.name, "replace", -1, lib.SubscriptionFilters.all() - )); - }); - - it("should resubscribe on account updates with permission", async function() { - subscriber.subscribe(() => {}); - await events.once(mockControl.connector, "send"); - mockControl.connector.sentMessages.length = 0; - - permitted = false; - subscriber.handleAccountEvent(); - await settle(); - assert.deepEqual(sentRequests(), []); - - permitted = true; - subscriber.handleAccountEvent(); - await settle(); - assert.equal(sentRequests().length, 1); - }); - - it("should clear requests when a new session starts", async function() { - await subscriber._handleEvent(new lib.HostPairingUpdatesEvent([pairing("ABC-DEF", 10)])); - assert.equal(subscriber.values.size, 1); - const [before] = subscriber.getSnapshot(); - - mockControl.connector.emit("connect"); - assert.equal(subscriber.values.size, 0); - assert.equal(subscriber.lastUpdatedMs, -1); - const [after] = subscriber.getSnapshot(); - assert.notEqual(after, before); - assert.equal(after.size, 0); - }); - - it("should keep requests when a session is resumed", async function() { - await subscriber._handleEvent(new lib.HostPairingUpdatesEvent([pairing("ABC-DEF", 10)])); - subscriber.handleConnectionEvent("drop"); - subscriber.handleConnectionEvent("resume"); - assert.equal(subscriber.values.size, 1); - assert.equal(subscriber.lastUpdatedMs, 10); - }); - it("should clear requests when subscribing again after the last unsubscribe", async function() { const unsubscribe = subscriber.subscribe(() => {}); await events.once(mockControl.connector, "send"); From 119c81c770d58cca0a810b41ec26a12de1dc68a2 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 31 Oct 2026 23:23:32 +0100 Subject: [PATCH 10/44] Serve the web interface from source with --dev-plugin With --dev-plugin and no --dev the page kept the production import map, while the development server gave the plugin its own copies of the shared modules. The plugin then failed to load with "Attempt to import duplicate copy of @clusterio/lib". Use the development server's web interface and import map whenever it runs, so the plugin and the web interface share the same modules. --- docs/contributing.md | 1 + packages/controller/src/Controller.ts | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index a206de81..5a7b718a 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -123,6 +123,7 @@ For web development on the controller there are also the following flags: `--dev` - Serve the `web_ui` package from source through Vite's development server with hot module replacement. `--dev-plugin [name]` - Serve the web module for the given plugin from source the same way. +This also serves the `web_ui` package from source, as the plugin has to share its modules with it. These options can be combined and multiple plugins can be actived for dev mode. Keep in mind if you're working on external plugins the `--dev-plugin` option is likely to only work if the plugin is placed inside the `external_plugins` directory and its dependencies are installed with pnpm as part of the clusterio repository workspace. diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index bbd2286d..914f2930 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -420,7 +420,7 @@ export default class Controller { this.app.use(this.devServer.middlewares); } - if (!args.dev) { + if (!this.devServer) { let manifestPath = path.join(import.meta.dirname, "..", "..", "web", "manifest.json"); let manifest = await Controller.loadJsonObject(manifestPath); @@ -527,14 +527,14 @@ export default class Controller { }, plugins: [webUiVite.clusterioDevPlugins(pluginDirs)], }); - if (args.dev) { - this.app.locals.webBundle = { - main: this.devServer.config.plugins.find(p => p.name === "clusterio:app")!.api.entry, - imports: Object.fromEntries(webUiVite.sharedModules.map( - name => [name, webUiVite.devSharedUrl(name)] - )), - }; - } + // Plugins served from source import the shared modules from the development server, the web + // interface has to use the same copies, so it is served from source as well with --dev-plugin. + this.app.locals.webBundle = { + main: this.devServer.config.plugins.find(p => p.name === "clusterio:app")!.api.entry, + imports: Object.fromEntries(webUiVite.sharedModules.map( + name => [name, webUiVite.devSharedUrl(name)] + )), + }; this.app.locals.devServer = this.devServer; } From 3c51d3b1eab7425f48495b7869c5f8ab83f875fa Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 31 Oct 2026 23:24:40 +0100 Subject: [PATCH 11/44] Test starting the controller with an existing database No test started a controller with instances, hosts, mod packs and users already on disk, which is how #510 got through with all tests passing. --- test/controller/Controller.js | 69 +++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/controller/Controller.js b/test/controller/Controller.js index 68f535bd..7f672eef 100644 --- a/test/controller/Controller.js +++ b/test/controller/Controller.js @@ -89,6 +89,75 @@ describe("controller/src/Controller", function() { assert(loaded.modPacks.size > 0, "default mod packs were not added"); await fs.access(modsDirectory); }); + it("should start and stop with an existing database", async function() { + this.timeout(10000); + const databaseDirectory = path.join(dir, "database"); + await fs.mkdir(databaseDirectory); + const records = { + "hosts.json": [new HostRecord(5, "existing-host", controllerVersion, new Map(), true)], + "instances.json": [ + new InstanceRecord(new InstanceConfig("controller", { + "instance.id": 100, "instance.assigned_host": 5, + }), "running"), + new InstanceRecord(new InstanceConfig("controller", { "instance.id": 101 }), "unassigned"), + ], + "mod-packs.json": [ModPack.fromJSON({ id: 7, name: "existing-pack" })], + "users.json": [new UserRecord(0, "admin", new Set([0]), new Set(), true)], + }; + for (const [file, content] of Object.entries(records)) { + await fs.writeFile(path.join(databaseDirectory, file), JSON.stringify(content)); + } + + // Port 0 means not configured, so ask the OS for a free one. + const portProbe = http.createServer(); + portProbe.listen(0, "127.0.0.1"); + await events.once(portProbe, "listening"); + const port = portProbe.address().port; + portProbe.close(); + await events.once(portProbe, "close"); + + const config = new ControllerConfig("controller", { + "controller.version": controllerVersion, + "controller.database_directory": databaseDirectory, + "controller.mods_directory": path.join(dir, "mods"), + "controller.http_port": port, + "controller.bind_address": "127.0.0.1", + }); + + const errors = []; + const errorCapture = new lib.ConsoleTransport({ + level: "error", + filter: info => { + // pnpm build does not produce the production web bundle + if (!info.message.includes("Missing main entry")) { + errors.push(info.message); + } + return false; + }, + }); + lib.logger.add(errorCapture); + let loaded; + try { + loaded = await Controller.fromConfig(config, [], lib.logger); + await loaded.start({ logDirectory: path.join(dir, "logs"), _: ["run"] }); + assert.deepEqual( + [...loaded.instances.values()].map(instance => [instance.id, instance.status]), + [[100, "unknown"], [101, "unassigned"]], + ); + assert.equal(loaded.hosts.get(5).connected, false); + assert.equal(loaded.modPacks.get(7).name, "existing-pack"); + assert(loaded.users.getByName("admin"), "existing user was not loaded"); + const response = await fetch(`http://127.0.0.1:${port}/`); + assert.equal(response.status, 200); + await response.text(); + } finally { + await loaded?.stop(); + lib.logger.remove(errorCapture); + } + assert.deepEqual(errors, []); + const saved = JSON.parse(await fs.readFile(path.join(databaseDirectory, "instances.json"), "utf8")); + assert.deepEqual(saved.map(instance => instance.config["instance.id"]), [100, 101]); + }); describe("with a plugin version change", function() { before(function() { addPluginConfigFields([{ name: "from_config_plugin", version: "1.2.0", instanceEntrypoint: "x" }]); From 7ada2e8975b71526ccbe2499c02e8d91e35148b8 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 31 Oct 2026 23:26:39 +0100 Subject: [PATCH 12/44] Skip icons that fail to export instead of failing the export An icon that could not be read or decoded, such as a truncated PNG, a JPEG or a layered icon with no layers, threw out of exportIcons and failed the whole data export. Log a warning naming the prototype and export it without an icon instead. Also warn when a mod's zip is missing while exporting the locale, which was skipped silently. --- packages/host/src/export.ts | 11 +++++++++- test/host/export.js | 44 ++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index 5df9d1df..ad9aaaa3 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -413,7 +413,13 @@ async function exportIcons( state.metadata[baseName] ??= metadata; let count = 0; for (let prototype of prototypes) { - const icon = await exportIcon(server, modVersions, zipCache, state, prototype); + let icon: lib.ExportMetadataEntry["icon"]; + try { + icon = await exportIcon(server, modVersions, zipCache, state, prototype); + } catch (err: any) { + // One broken icon should not stop the rest of the export + server._logger.warn(`Skipping icon of ${prototype.type} ${prototype.name}: ${err.message}`); + } if (icon) { count += 1; } @@ -484,6 +490,7 @@ async function exportLocale( modZip = await loadZip(server, modVersions, zipCache, mod); } catch (err: any) { if (err.code === "ENOENT") { + server._logger.warn(`Skipping locale of ${mod}, its mod zip was not found`); continue; } throw err; @@ -635,6 +642,8 @@ async function buildExport(server: FactorioServer, zipCache: ZipCache) { // For testing only +export const _createSheetState = createSheetState; +export const _exportIcons = exportIcons; export const _exportLocale = exportLocale; export const _loadSimpleIcon = loadSimpleIcon; export const _loadLayeredIcon = loadLayeredIcon; diff --git a/test/host/export.js b/test/host/export.js index 43109532..4d85dd6e 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -2,7 +2,9 @@ import assert from "node:assert/strict"; import path from "node:path"; import { FactorioServer } from "@clusterio/host/dist/node/src/server.js"; -import { _exportLocale, _loadLayeredIcon, _loadSimpleIcon } from "@clusterio/host/dist/node/src/export.js"; +import { + _createSheetState, _exportIcons, _exportLocale, _loadLayeredIcon, _loadSimpleIcon, +} from "@clusterio/host/dist/node/src/export.js"; describe("host/src/export", function() { @@ -12,7 +14,25 @@ describe("host/src/export", function() { testServer = await FactorioServer.fromOptions(path.join("test", "file", "factorio"), writePath, {}); }); + let warnings; + let originalLogger; + beforeEach(function() { + warnings = []; + originalLogger = testServer._logger; + testServer._logger = Object.create(originalLogger, { warn: { value: message => warnings.push(message) } }); + }); + afterEach(function() { + testServer._logger = originalLogger; + }); + describe("exportLocale()", function() { + it("warns about mods without a zip and exports the rest", async function() { + const locale = await _exportLocale( + testServer, new Map([["missing_mod", "1.0.0"]]), new Map(), ["base", "missing_mod"], "en" + ); + assert.equal(locale.size > 0, true); + assert.deepEqual(warnings, ["Skipping locale of missing_mod, its mod zip was not found"]); + }); it("returns a flattened mapping with locale definitions", async function() { let locale = await _exportLocale(testServer, new Map(), new Map(), ["base"], "en"); assert.deepEqual(locale, new Map([ @@ -30,6 +50,28 @@ describe("host/src/export", function() { // test-icon.png is a red 64px icon followed by a blue 32px mipmap const red = 0xff0000ff; + describe("exportIcons()", function() { + it("skips icons that fail to load and exports the rest", async function() { + const state = await _createSheetState(); + const icon = "__base__/graphics/icons/test-icon.png"; + const prototypes = [ + { type: "item", name: "first", icon, icon_size: 64 }, + { type: "item", name: "no-layers", icons: [] }, + { type: "item", name: "unknown-mod", icon: "__not-a-mod__/graphics/icons/x.png", icon_size: 64 }, + { type: "item", name: "last", icon, icon_size: 64 }, + ]; + const count = await _exportIcons(testServer, new Map(), new Map(), prototypes, "item", state); + assert.equal(count, 2); + assert.deepEqual( + state.metadata.item.map(entry => [entry.name, Boolean(entry.icon)]), + [["first", true], ["no-layers", false], ["unknown-mod", false], ["last", true]], + ); + assert.equal(warnings.length, 2); + assert.match(warnings[0], /^Skipping icon of item no-layers: /); + assert.match(warnings[1], /^Skipping icon of item unknown-mod: Got path for unknown mod not-a-mod/); + }); + }); + describe("loadSimpleIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadSimpleIcon( From 495831da29fe33ac86dfa89780f685953954e240 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 1 Nov 2026 12:07:34 +0100 Subject: [PATCH 13/44] Stop waiting for approval when the terminal closes SIGHUP while waiting for approval released the config lock but left the host polling, as only SIGINT and SIGTERM stopped the wait. --- packages/host/host.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/host/host.ts b/packages/host/host.ts index d72d692b..a5302b70 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -232,6 +232,7 @@ async function startHost() { const onSignal = () => { abort.abort(); }; process.on("SIGINT", onSignal); process.on("SIGTERM", onSignal); + process.on("SIGHUP", onSignal); try { await ensurePaired(hostConfig, { version: packageConfig.version, @@ -253,6 +254,7 @@ async function startHost() { } finally { process.off("SIGINT", onSignal); process.off("SIGTERM", onSignal); + process.off("SIGHUP", onSignal); } } From 928e03ce02a31f38347563f2e156ee158b760002 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 1 Nov 2026 17:40:19 +0100 Subject: [PATCH 14/44] Merge pull request #1161 from clusterio/host-pairing Replace host tokens with host pairing From 2e1710d3a96fcf72b2054855f76bf10b43d9069a Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 30 Oct 2026 22:39:47 +0100 Subject: [PATCH 15/44] Show the same page for missing, deleted and unloadable resources Every view page handled a resource that can't be shown its own way, and users without the subscribe permission were left on a spinner forever. Adds useResourceState and a ResourceState component, used by the host, instance, mod pack, icon reference, user, role and plugin pages and exported for plugin pages. The single item hooks now also return the subscription error. Resolves #1165 --- docs/writing-plugins.md | 29 +++++++++++ .../web_ui/src/components/HostViewPage.tsx | 16 +++--- .../src/components/IconReferencePage.tsx | 9 +++- .../src/components/InstanceViewPage.tsx | 25 +++------- .../web_ui/src/components/ModPackViewPage.tsx | 19 +++---- .../src/components/PluginClusterStatus.tsx | 14 ++++-- .../web_ui/src/components/PluginViewPage.tsx | 39 +++++++-------- .../web_ui/src/components/ResourceState.tsx | 49 +++++++++++++++++++ .../web_ui/src/components/RoleViewPage.tsx | 30 +++--------- .../web_ui/src/components/UserViewPage.tsx | 19 +++---- packages/web_ui/src/index.ts | 2 + packages/web_ui/src/model/host.ts | 4 +- packages/web_ui/src/model/instance.ts | 4 +- packages/web_ui/src/model/mod_pack.ts | 4 +- packages/web_ui/src/model/roles.ts | 4 +- packages/web_ui/src/model/user.tsx | 4 +- packages/web_ui/src/util/useResourceState.ts | 37 ++++++++++++++ test/web_ui/useResourceState.js | 37 ++++++++++++++ 18 files changed, 233 insertions(+), 112 deletions(-) create mode 100644 packages/web_ui/src/components/ResourceState.tsx create mode 100644 packages/web_ui/src/util/useResourceState.ts create mode 100644 test/web_ui/useResourceState.js diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index d2a2a82f..fd571c0e 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -16,6 +16,7 @@ Each entrypoint of a plugin exports a function that attaches handlers to hooks c - [Defining Link Messages](#defining-link-messages) - [Sending Link Messages](#sending-link-messages) - [Handling connection events](#handling-connection-events) +- [Web Pages for a Single Resource](#web-pages-for-a-single-resource) - [Collecting Statistics](#collecting-statistics) - [Adding Custom Commands to clusterioctl](#adding-custom-commands-to-clusterioctl) @@ -842,6 +843,34 @@ This means that if your plugin sends a lot of events or requests, they can end u To avoid this you should be throtteling and/or stopping your requests/events after `drop` has been raised, and continue back as normal when `resume` is raised. +## Web Pages for a Single Resource + +A web page showing one resource from a subscription has more to handle than the resource itself. +It can still be loading, it might not exist, it can be deleted while the page is open, and the subscription is rejected if the user lacks the permission for it. +`useResourceState` and `ResourceState` from `@clusterio/web_ui` show these the same way as the pages for hosts, instances and the other built in resources. + +```tsx +function FrobberPage() { + const id = Number(useParams().id); + const [frobbers, synced, error] = useSyncExternalStore(subscribe, () => frobberSubscriber.getSnapshot()); + const frobber = frobbers.get(id); + const state = useResourceState(frobber, synced, error, id); + + const nav = [{ name: "Frobbers", path: "/frobbers" }, { name: frobber?.name ?? String(id) }]; + if (!frobber) { + return ; + } + + return {frobber.name}; +} +``` + +`useResourceState` returns `"ok"` when there is a value and otherwise `"loading"`, `"missing"`, `"deleted"` or `"error"`. +It tells deleted from missing by remembering if it has seen a value for that id. +`ResourceState` renders the whole page for the states without a value, with a link back to the page at `listPath`. +The `useHost`, `useInstance`, `useUser`, `useRole` and `useModPack` hooks return `[value, synced, error]` so they can be used the same way. + + ## Collecting Statistics Clusterio comes with its own Prometheus client implementation, one part due to Not Invented Here and another part due to collectors in prom-client being difficult to get to work nicely with collecting data from plugins optionally loaded at runtime on different computers. diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index a7b1d6fc..2328fce7 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -1,5 +1,5 @@ import React, { useContext, useState } from "react"; -import { Descriptions, Spin, Typography, Button, Space, Popconfirm, Flex, Tooltip } from "antd"; +import { Descriptions, Typography, Button, Space, Popconfirm, Flex, Tooltip } from "antd"; import { ExclamationCircleOutlined } from "@ant-design/icons"; import { useNavigate, useParams } from "react-router-dom"; @@ -17,11 +17,13 @@ import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; import ResourceNotes from "./ResourceNotes"; +import ResourceState from "./ResourceState"; import { MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, MetricDiskUsed, MetricDiskRatio, MetricRelativeDate, } from "./system_metrics"; import { useSystems } from "../model/system"; +import useResourceState from "../util/useResourceState"; import { hasNpmButtonPermission, NpmButton } from "./NpmButton"; import VariableDropdownButton, { VariableDropdownButtonProps } from "./VariableDropdownButton"; @@ -132,21 +134,15 @@ export default function HostViewPage() { let [instances] = useInstances(); const [systems] = useSystems(); const system = systems.get(hostId); - const [host, synced] = useHost(hostId); + const [host, synced, error] = useHost(hostId); + const state = useResourceState(host, synced, error, hostId); const [maxLevel, setMaxLevel] = useState("info"); const canShowNpm = hasNpmButtonPermission(false); const hostInstances = new Map([...instances].filter(([_id, instance]) => instance.assignedHost === hostId)); let nav = [{ name: "Hosts", path: "/hosts" }, { name: host?.name ?? String(hostId) }]; if (!host) { - if (!synced) { - return ; - } - - return - -

Host with id {hostId} was not found on the controller.

- ; + return ; } let hostButtons = diff --git a/packages/web_ui/src/components/IconReferencePage.tsx b/packages/web_ui/src/components/IconReferencePage.tsx index de5263b0..2a8f58dc 100644 --- a/packages/web_ui/src/components/IconReferencePage.tsx +++ b/packages/web_ui/src/components/IconReferencePage.tsx @@ -8,6 +8,8 @@ import FactorioIcon from "./FactorioIcon"; import { useParams } from "react-router-dom"; import { useModPack } from "../model/mod_pack"; import { PrototypeMetadataEntry } from "../store/export_prototype_metadata_store"; +import useResourceState from "../util/useResourceState"; +import ResourceState from "./ResourceState"; const { Text } = Typography; @@ -39,7 +41,8 @@ export default function IconReferencePage() { const [filter, setFilter] = useState(""); const [expandedGroups, setExpandedGroups] = useState>(new Set()); - const [modPack] = useModPack(modPackId); + const [modPack, synced, error] = useModPack(modPackId); + const state = useResourceState(modPack, synced, error, modPackId); const manifest = modPack?.exportManifest; const prototypes = useExportPrototypeMetadata(modPack); const filtered = useMemo(() => filterPrototypes(prototypes, filter), [prototypes, filter]); @@ -100,6 +103,10 @@ export default function IconReferencePage() { { name: modPack?.name ?? String(modPackId), path: `/mods/mod-packs/${modPackId}/view` }, { name: "Icon Reference" }, ]; + if (!modPack) { + return ; + } + return ( diff --git a/packages/web_ui/src/components/InstanceViewPage.tsx b/packages/web_ui/src/components/InstanceViewPage.tsx index ab52b69a..ce12ef8d 100644 --- a/packages/web_ui/src/components/InstanceViewPage.tsx +++ b/packages/web_ui/src/components/InstanceViewPage.tsx @@ -1,7 +1,7 @@ import React, { useContext, useState } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { - Alert, Button, Descriptions, Dropdown, Flex, MenuProps, Modal, Segmented, Space, Spin, Tooltip, Typography, + Button, Descriptions, Dropdown, Flex, MenuProps, Modal, Segmented, Space, Tooltip, Typography, message, } from "antd"; import CopyOutlined from "@ant-design/icons/CopyOutlined"; @@ -16,6 +16,7 @@ import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; import ResourceNotes from "./ResourceNotes"; +import ResourceState from "./ResourceState"; import InstanceConfigTree from "./InstanceConfigTree"; import LogConsole, { SelectMaxLogLevel } from "./LogConsole"; import InstanceRcon from "./InstanceRcon"; @@ -31,6 +32,7 @@ import InstanceStatusTag from "./InstanceStatusTag"; import Link from "./Link"; import { instancePublicAddress } from "../util/instance"; import useLocalStorage from "../util/useLocalStorage"; +import useResourceState from "../util/useResourceState"; import { MetricRelativeDate } from "./system_metrics"; type MenuItem = Required["items"][number]; @@ -208,26 +210,15 @@ export default function InstanceViewPage() { let navigate = useNavigate(); let account = useAccount(); - let [instance, synced] = useInstance(instanceId); + let [instance, synced, error] = useInstance(instanceId); + const state = useResourceState(instance, synced, error, instanceId); let [host] = useHost(instance?.assignedHost); let nav = [{ name: "Instances", path: "/instances" }, { name: instance?.name ?? String(instanceId) }]; if (!instance) { - if (!synced) { - return ; - } - - return - Instance with id {instanceId} was not found on the controller.} - type="warning" - action={ - Go back to instances list - } - /> - ; + return ; } return diff --git a/packages/web_ui/src/components/ModPackViewPage.tsx b/packages/web_ui/src/components/ModPackViewPage.tsx index 575b3361..f0216254 100644 --- a/packages/web_ui/src/components/ModPackViewPage.tsx +++ b/packages/web_ui/src/components/ModPackViewPage.tsx @@ -3,7 +3,7 @@ import { useNavigate, useParams } from "react-router-dom"; import { Button, Card, ColorPicker, Checkbox, Col, ConfigProvider, Descriptions, Form, Input, Pagination, - Popconfirm, Row, Table, Tag, Typography, Select, Skeleton, Space, Spin, Switch, Modal, Tooltip, + Popconfirm, Row, Table, Tag, Typography, Select, Skeleton, Space, Switch, Modal, Tooltip, TablePaginationConfig, Alert, notification, FormProps, } from "antd"; @@ -22,10 +22,12 @@ import { useModPack } from "../model/mod_pack"; import { useMods } from "../model/mods"; import { useClipboard } from "../util/clipboard"; import notify, { notifyErrorHandler } from "../util/notify"; +import useResourceState from "../util/useResourceState"; import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import SectionHeader from "./SectionHeader"; import ResourceNotes from "./ResourceNotes"; +import ResourceState from "./ResourceState"; import ModDetails from "./ModDetails"; import { InputPartialVersion } from "./InputVersion"; import Link from "./Link"; @@ -1183,7 +1185,8 @@ export default function ModPackViewPage() { let modPackId = Number(params.id); const [form] = Form.useForm(); - const [modPack, synced] = useModPack(modPackId); + const [modPack, synced, error] = useModPack(modPackId); + const state = useResourceState(modPack, synced, error, modPackId); let prototypes: any = useExportedAsset(modPack, "settings"); let locale: Map = useExportedAsset(modPack, "locale"); let [changes, setChanges] = useState([]); @@ -1253,17 +1256,7 @@ export default function ModPackViewPage() { let nav = [{ name: "Mods", path: "/mods" }, { name: "Mod Packs" }, { name: modPack?.name ?? String(modPackId) }]; if (!modifiedModPack || !modPack) { - if (!synced) { - return - - - ; - } - - return - -

Mod pack with id {modPackId} was not found on the controller.

- ; + return ; } return diff --git a/packages/web_ui/src/components/PluginClusterStatus.tsx b/packages/web_ui/src/components/PluginClusterStatus.tsx index 8c17234f..570c1f1b 100644 --- a/packages/web_ui/src/components/PluginClusterStatus.tsx +++ b/packages/web_ui/src/components/PluginClusterStatus.tsx @@ -35,6 +35,8 @@ type MachinePluginsState = { unknown: Machine[], /** Fetch the plugin lists of connected machines now, instead of waiting for the next system metrics */ refresh: (addresses?: lib.AddressShorthand[]) => Promise, + /** Why machines is not available, if it's not just still loading */ + error: Error | null, }; /** @@ -44,7 +46,7 @@ type MachinePluginsState = { export function useMachinePlugins(): MachinePluginsState { const control = useContext(ControlContext); const account = useAccount(); - const [hosts, hostsSynced] = useHosts(); + const [hosts, hostsSynced, hostsError] = useHosts(); const [systems, systemsSynced] = useSystems(); // Lists fetched directly, used until the system metrics of the machine update const [listed, setListed] = useState(new Map()); @@ -88,8 +90,12 @@ export function useMachinePlugins(): MachinePluginsState { } }, [canList, hostsSynced, connectedKey]); - if (!canList || !hostsSynced || (!systemsSynced && !listed.size)) { - return { machines: undefined, unknown: [], refresh }; + if (!canList) { + const error = new lib.PermissionError("Missing permission core.plugin.list"); + return { machines: undefined, unknown: [], refresh, error }; + } + if (!hostsSynced || (!systemsSynced && !listed.size)) { + return { machines: undefined, unknown: [], refresh, error: hostsError }; } const machines: MachinePlugins[] = []; const unknown: Machine[] = []; @@ -105,7 +111,7 @@ export function useMachinePlugins(): MachinePluginsState { unknown.push(machine); } } - return { machines, unknown, refresh }; + return { machines, unknown, refresh, error: null }; } /** Latest version published on npm of each package */ diff --git a/packages/web_ui/src/components/PluginViewPage.tsx b/packages/web_ui/src/components/PluginViewPage.tsx index 2a7e6c8e..6903f258 100644 --- a/packages/web_ui/src/components/PluginViewPage.tsx +++ b/packages/web_ui/src/components/PluginViewPage.tsx @@ -1,15 +1,15 @@ import React, { useEffect, useContext, useState } from "react"; import { useParams } from "react-router-dom"; -import { Alert, Descriptions, Spin } from "antd"; +import { Alert, Descriptions } from "antd"; import * as lib from "@clusterio/lib"; import type { PluginWebApi } from "@clusterio/lib"; import ControlContext from "./ControlContext"; import PageLayout from "./PageLayout"; -import PageHeader from "./PageHeader"; +import ResourceState from "./ResourceState"; import SectionHeader from "./SectionHeader"; -import notify from "../util/notify"; +import useResourceState from "../util/useResourceState"; import { usePluginInstaller } from "./PluginInstall"; import { PluginMachinesTable, PluginUpdateButton, useLatestVersions, useMachinePlugins } from "./PluginClusterStatus"; import { PluginStatus, computePluginStatuses, pluginStatusKey } from "../util/plugin_status"; @@ -89,18 +89,18 @@ export default function PluginViewPage() { let params = useParams(); const control = useContext(ControlContext); let [pluginList, setPluginList] = useState(); + const [pluginListError, setPluginListError] = useState(null); let pluginName = String(params.name); - const { machines, unknown, refresh } = useMachinePlugins(); + const { machines, unknown, refresh, error: machinesError } = useMachinePlugins(); useEffect(() => { (async () => { let response = await fetch(`${webRoot}api/plugins`); - if (response.ok) { - setPluginList(await response.json()); - } else { - notify("Failed to load plugin list"); + if (!response.ok) { + throw new Error(`Failed to load plugin list: ${response.status} ${response.statusText}`); } - })(); + setPluginList(await response.json()); + })().catch(setPluginListError); }, []); const pluginMeta = pluginList?.find(p => p.name === pluginName); @@ -110,19 +110,16 @@ export default function PluginViewPage() { : undefined; const pluginInfo = control.pluginInfos.get(pluginName); const pluginTitle = pluginInfo?.title ?? status?.title ?? pluginName; + // Machines are only needed for plugins not on the controller, which isn't known until the list has loaded + const error = pluginListError ?? (pluginList ? machinesError : null); + const state = useResourceState( + pluginList && (pluginMeta ?? status), Boolean(pluginList && machines), error, pluginName + ); let nav = [{ name: "Plugins", path: "/plugins" }, { name: pluginTitle }]; - if (!pluginList || (!pluginMeta && !machines)) { - return - - - ; - } - - if (!pluginMeta && !status) { - return - -

Plugin with name {pluginName} was not found on the controller or any host.

- ; + if (state !== "ok") { + return ; } return diff --git a/packages/web_ui/src/components/ResourceState.tsx b/packages/web_ui/src/components/ResourceState.tsx new file mode 100644 index 00000000..d2587704 --- /dev/null +++ b/packages/web_ui/src/components/ResourceState.tsx @@ -0,0 +1,49 @@ +import React from "react"; +import { Result, ResultProps, Spin } from "antd"; + +import Link from "./Link"; +import PageHeader from "./PageHeader"; +import PageLayout from "./PageLayout"; +import type { ResourceStatus } from "../util/useResourceState"; + +type ResourceStateProps = { + /** Status from useResourceState */ + state: ResourceStatus; + /** Capitalised name of the kind of resource, e.g. "Mod pack" */ + type: string; + /** Id or name of the resource */ + id: number | string; + nav: React.ComponentProps["nav"]; + /** Page listing this kind of resource */ + listPath: string; + /** Error from the subscription */ + error: Error | null; +}; + +/** Whole page to render in place of a view page when its resource can't be shown */ +export default function ResourceState(props: ResourceStateProps) { + const { state, type, id, nav } = props; + if (state === "ok") { + return null; + } + + if (state === "loading") { + return + + + ; + } + + const results: Record = { + missing: { status: "warning", title: `${type} not found`, subTitle: `${type} ${id} was not found.` }, + deleted: { status: "warning", title: `${type} ${id} has been deleted` }, + error: { status: "error", title: `Could not load ${type.toLowerCase()}`, subTitle: props.error?.message }, + }; + + return + Back to {type.toLowerCase()} list} + /> + ; +} diff --git a/packages/web_ui/src/components/RoleViewPage.tsx b/packages/web_ui/src/components/RoleViewPage.tsx index 5e61df85..5a7d8ba4 100644 --- a/packages/web_ui/src/components/RoleViewPage.tsx +++ b/packages/web_ui/src/components/RoleViewPage.tsx @@ -1,6 +1,6 @@ -import React, { useEffect, useContext, useState, useMemo, useRef } from "react"; +import React, { useEffect, useContext, useState, useMemo } from "react"; import { useNavigate, useParams } from "react-router-dom"; -import { Button, Checkbox, Col, Grid, Input, Popconfirm, Row, Space, Spin, Tooltip } from "antd"; +import { Button, Checkbox, Col, Grid, Input, Popconfirm, Row, Space, Tooltip } from "antd"; import { ExclamationCircleOutlined, StarOutlined, DeleteOutlined} from "@ant-design/icons"; import * as lib from "@clusterio/lib"; @@ -12,7 +12,9 @@ import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; import ResourceNotes from "./ResourceNotes"; +import ResourceState from "./ResourceState"; import { notifyErrorHandler } from "../util/notify"; +import useResourceState from "../util/useResourceState"; /** Splits a permission name into parts, removing the last part */ @@ -162,14 +164,14 @@ export default function RoleViewPage() { const navigate = useNavigate(); const account = useAccount(); const control = useContext(ControlContext); - const [role, synced] = useRole(roleId); + const [role, synced, error] = useRole(roleId); + const state = useResourceState(role, synced, error, roleId); // Local edits on top of the live role const [nameEdit, setNameEdit] = useState(); const [descriptionEdit, setDescriptionEdit] = useState(); const [permissionEdits, setPermissionEdits] = useState>({}); const [search, setSearch] = useState(""); - const loadedRoleId = useRef(undefined); // Hidden below lg, where the site sider leaves too little room; undefined until measured const screens = Grid.useBreakpoint(); const showGroupList = screens.lg !== false; @@ -203,27 +205,9 @@ export default function RoleViewPage() { setPermissionEdits({}); }, [roleId]); - if (role) { - loadedRoleId.current = roleId; - } - - // If no role then display either the loading screen or the not found screen if (!role) { const nav = [{ name: "Roles", path: "/roles" }, { name: String(roleId) }]; - - if (!synced) { - return - - - ; - } - - return - -

Role with id {roleId} { - loadedRoleId.current === roleId ? "has been deleted" : "was not found on the controller" - }.

- ; + return ; } const name = nameEdit ?? role.name; diff --git a/packages/web_ui/src/components/UserViewPage.tsx b/packages/web_ui/src/components/UserViewPage.tsx index 02e9733b..5e3581c0 100644 --- a/packages/web_ui/src/components/UserViewPage.tsx +++ b/packages/web_ui/src/components/UserViewPage.tsx @@ -2,7 +2,7 @@ import React, { useEffect, useContext, useState } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { Button, Col, Descriptions, Form, Input, Popconfirm, - Row, Table, Tag, Select, Space, Spin, Switch, + Row, Table, Tag, Select, Space, Switch, } from "antd"; import DeleteOutlined from "@ant-design/icons/DeleteOutlined"; @@ -18,7 +18,9 @@ import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; import SectionHeader from "./SectionHeader"; import ResourceNotes from "./ResourceNotes"; +import ResourceState from "./ResourceState"; import useRowNavigation from "../util/useRowNavigation"; +import useResourceState from "../util/useResourceState"; import Link from "./Link"; import notify, { notifyErrorHandler } from "../util/notify"; import { formatDuration } from "../util/time_format"; @@ -40,7 +42,8 @@ export default function UserViewPage() { let account = useAccount(); let control = useContext(ControlContext); let [instances] = useInstances(); - const [user, synced] = useUser(userName); + const [user, synced, error] = useUser(userName); + const state = useResourceState(user, synced, error, userName); let [roles, rolesSynced] = useRoles(); let [form] = Form.useForm(); const statsTable = useTableQueryState<[number, lib.PlayerStats]>({ @@ -78,17 +81,7 @@ export default function UserViewPage() { let nav = [{ name: "Users", path: "/users" }, { name: userName }]; if (!user) { - if (!synced) { - return - - - ; - } - - return - -

User with name {userName} was not found on the controller.

- ; + return ; } let roleSelector = (undefined); + const exists = value !== undefined && value !== null; + if (exists) { + seenId.current = id; + } + return getResourceStatus(exists, seenId.current === id, synced, error); +} diff --git a/test/web_ui/useResourceState.js b/test/web_ui/useResourceState.js new file mode 100644 index 00000000..50666001 --- /dev/null +++ b/test/web_ui/useResourceState.js @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; + +import { getResourceStatus } from "../../packages/web_ui/src/util/useResourceState.ts"; + +describe("web_ui/util/useResourceState", function() { + describe("getResourceStatus()", function() { + const error = new Error("Permission denied"); + + it("should be ok when the resource exists", function() { + assert.equal(getResourceStatus(true, true, true, null), "ok"); + }); + it("should keep showing a resource after the subscription stopped syncing", function() { + assert.equal(getResourceStatus(true, true, false, null), "ok"); + assert.equal(getResourceStatus(true, true, false, error), "ok"); + }); + it("should be loading until the subscription has synced", function() { + assert.equal(getResourceStatus(false, false, false, null), "loading"); + }); + it("should be loading while resyncing a resource that was seen", function() { + assert.equal(getResourceStatus(false, true, false, null), "loading"); + }); + it("should be missing when synced without having seen the resource", function() { + assert.equal(getResourceStatus(false, false, true, null), "missing"); + }); + it("should be deleted when synced after having seen the resource", function() { + assert.equal(getResourceStatus(false, true, true, null), "deleted"); + }); + it("should be error when the subscription failed", function() { + assert.equal(getResourceStatus(false, false, false, error), "error"); + assert.equal(getResourceStatus(false, true, false, error), "error"); + }); + it("should ignore the error of an earlier attempt once synced", function() { + assert.equal(getResourceStatus(false, false, true, error), "missing"); + assert.equal(getResourceStatus(false, true, true, error), "deleted"); + }); + }); +}); From d2a4e9a4a152d984f31e93150960955cc65c24a9 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:51:19 +0000 Subject: [PATCH 16/44] Replace inquirer 8 with the @inquirer prompt packages in create inquirer 8 pulls in 50 packages and 7.4 MB (rxjs, old chalk) for every user running npm init @clusterio. The standalone select, input, confirm and checkbox packages from the same project cover all prompts used and cut the create install from 70 packages / 8.1 MB to 31 / 0.94 MB. The Windows rl.close workaround now patches readline.Interface instead of inquirer's Prompt class, since the new prompts create their own readline interface. Ctrl+C at a prompt exits with code 130 instead of 0, and closed stdin no longer crashes with ERR_USE_AFTER_CLOSE. Co-Authored-By: Claude Opus 5.5 --- packages/create/create.js | 268 ++++++++++++++--------------------- packages/create/package.json | 5 +- 2 files changed, 111 insertions(+), 162 deletions(-) diff --git a/packages/create/create.js b/packages/create/create.js index 587a9672..48b81ac5 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -1,10 +1,14 @@ #!/usr/bin/env node +import checkbox from "@inquirer/checkbox"; +import confirm from "@inquirer/confirm"; +import input from "@inquirer/input"; +import select from "@inquirer/select"; import child_process from "node:child_process"; import fs from "node:fs/promises"; -import inquirer from "inquirer"; import os from "node:os"; import path from "node:path"; +import readline from "node:readline"; import stream from "node:stream"; import util from "node:util"; import yargs from "yargs"; @@ -16,15 +20,13 @@ const scriptExt = process.platform === "win32" ? ".cmd" : ""; import { escapeArg } from "./escape_arg.js"; // I hate this, there is a bug with rl.close on windows that was meant to have been fixed in Node14.4 -// See nodejs/node#21771 and nodejs/node#30701 for the apparent fix that was applied -// One option is to not call rl.close: SBoudrias/Inquirer.js#767 -// But this comes with its own issues: SBoudrias/Inquirer.js#899 +// See nodejs/node#21771, nodejs/node#30701 and nodejs/node#31762 // The solution below was adapted from aws-amplify/amplify-cli#12347 if (process.platform === "win32") { - const _inquirer_close = inquirer.ui.Prompt.prototype.close; - inquirer.ui.Prompt.prototype.close = function() { - this.rl.terminal = false; - _inquirer_close.call(this); + const close = readline.Interface.prototype.close; + readline.Interface.prototype.close = function() { + this.terminal = false; + close.call(this); }; } @@ -164,23 +166,23 @@ async function execCtl(args) { // Same characters as the controller refuses in host names. const hostNameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u; -function validateHostName(input) { - if (!input || !input.trim()) { +function validateHostName(value) { + if (!value || !value.trim()) { return "Host name may not be empty"; } - if (input.trim().length > 100) { + if (value.trim().length > 100) { return "Host name may be at most 100 characters"; } - if (hostNameControlCharacters.test(input)) { + if (hostNameControlCharacters.test(value)) { return "Host name may not contain control characters"; } return true; } -function validateControllerUrl(input) { +function validateControllerUrl(value) { let url; try { - url = new URL(input); + url = new URL(value); } catch (err) { return "Controller URL must be a URL like https://example.com/"; } @@ -191,8 +193,8 @@ function validateControllerUrl(input) { } // The host requires the controller URL to end with a slash. -function normaliseControllerUrl(input) { - const url = new URL(input); +function normaliseControllerUrl(value) { + const url = new URL(value); if (!url.pathname.endsWith("/")) { url.pathname += "/"; } @@ -537,61 +539,49 @@ WantedBy=multi-user.target } // eslint-disable-next-line complexity -async function inquirerMissingArgs(args) { +async function promptMissingArgs(args) { let answers = {}; if (args.mode) { answers.mode = args.mode; } if (args["plugin-template"]) { answers.mode = "plugin-template"; } - answers = await inquirer.prompt([ - { - type: "list", - name: "mode", - message: "Operating mode to install", - default: "standalone", - choices: [ - { name: "Standalone (install both controller and host on this computer)", value: "standalone" }, - { name: "Controller only", value: "controller" }, - { name: "Host only", value: "host" }, - { name: "Ctl only", value: "ctl" }, - { name: "Plugins only", value: "plugins" }, - { name: "Plugin template", value: "plugin-template" }, - ], - }, - ], answers); + answers.mode ??= await select({ + message: "Operating mode to install", + default: "standalone", + choices: [ + { name: "Standalone (install both controller and host on this computer)", value: "standalone" }, + { name: "Controller only", value: "controller" }, + { name: "Host only", value: "host" }, + { name: "Ctl only", value: "ctl" }, + { name: "Plugins only", value: "plugins" }, + { name: "Plugin template", value: "plugin-template" }, + ], + }); if (["standalone", "controller"].includes(answers.mode)) { if (args.admin) { answers.admin = args.admin; } - answers = await inquirer.prompt([ - { - type: "input", - name: "admin", - message: "Admin account name", - validate: input => { - if (!input) { - return "May not be empty"; - } - return true; - }, + answers.admin ??= await input({ + message: "Admin account name", + validate: value => { + if (!value) { + return "May not be empty"; + } + return true; }, - ], answers); + }); if (args.httpPort) { answers.httpPort = args.httpPort; } - answers = await inquirer.prompt([ - { - type: "input", - name: "httpPort", - message: "HTTP port to listen on", - default: 8080, - validate: input => { - if (!input) { - return "May not be empty"; - } - if (!Number.isInteger(Number(input))) { - return "Must be a number"; - } - return true; - }, + answers.httpPort ??= await input({ + message: "HTTP port to listen on", + default: "8080", + validate: value => { + if (!value) { + return "May not be empty"; + } + if (!Number.isInteger(Number(value))) { + return "Must be a number"; + } + return true; }, - ], answers); + }); } if (answers.mode === "standalone") { @@ -605,23 +595,17 @@ async function inquirerMissingArgs(args) { if (answers.mode === "host") { if (args.hostName) { answers.hostName = args.hostName; } if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; } - answers = await inquirer.prompt([ - { - type: "input", - name: "hostName", - message: "Name of host", - default: os.hostname(), - validate: validateHostName, - }, - { - type: "input", - name: "controllerUrl", - message: "Controller URL", - validate: validateControllerUrl, - }, - ], answers); + answers.hostName ??= await input({ + message: "Name of host", + default: os.hostname(), + validate: validateHostName, + }); + answers.controllerUrl ??= await input({ + message: "Controller URL", + validate: validateControllerUrl, + }); - // Inquirer does not validate answers given as arguments + // The prompts do not validate answers given as arguments for (let result of [validateHostName(answers.hostName), validateControllerUrl(answers.controllerUrl)]) { if (result !== true) { throw new InstallError(result); @@ -632,22 +616,10 @@ async function inquirerMissingArgs(args) { if (answers.mode === "ctl") { if (args.controllerUrl) { answers.controllerUrl = args.controllerUrl; } - answers = await inquirer.prompt([ - { - type: "input", - name: "controllerUrl", - message: "Controller URL", - }, - ], answers); + answers.controllerUrl ??= await input({ message: "Controller URL" }); if (args.controllerToken) { answers.controllerToken = args.controllerToken; } - answers = await inquirer.prompt([ - { - type: "input", - name: "controllerToken", - message: "Controller authentication Token", - }, - ], answers); + answers.controllerToken ??= await input({ message: "Controller authentication Token" }); } if (["standalone", "host"].includes(answers.mode)) { @@ -660,14 +632,10 @@ async function inquirerMissingArgs(args) { myIp = await result.text(); } catch (err) { /* ignore */ } } - answers = await inquirer.prompt([ - { - type: "input", - name: "publicAddress", - message: "Public DNS/IP address of this server", - default: myIp, - }, - ], answers); + answers.publicAddress ??= await input({ + message: "Public DNS/IP address of this server", + default: myIp, + }); if (args.factorioDir) { answers.factorioDir = args.factorioDir; } let locations = factorioLocations[process.platform] || []; @@ -678,28 +646,17 @@ async function inquirerMissingArgs(args) { } } - answers = await inquirer.prompt([ - { - type: "list", - name: "factorioDir", - message: "Path to Factorio installation", - choices: [ - ...foundLocations.map(location => ({ name: `${location} (auto detected)`, value: location })), - { name: "Use local factorio directory, you must copy an installation to it", value: "factorio" }, - { name: "Provide path manually", value: null }, - ], - }, - ], answers); + answers.factorioDir ??= await select({ + message: "Path to Factorio installation", + choices: [ + ...foundLocations.map(location => ({ name: `${location} (auto detected)`, value: location })), + { name: "Use local factorio directory, you must copy an installation to it", value: "factorio" }, + { name: "Provide path manually", value: null }, + ], + }); if (answers.factorioDir === null) { - answers = await inquirer.prompt([ - { - type: "input", - name: "factorioDir", - message: "Path to Factorio installation", - askAnswered: true, - }, - ], answers); + answers.factorioDir = await input({ message: "Path to Factorio installation" }); } else if (answers.factorioDir === "factorio") { await fs.mkdir("factorio", { recursive: true }); } @@ -707,14 +664,10 @@ async function inquirerMissingArgs(args) { if (["standalone", "controller", "host"].includes(answers.mode)) { if (args.hasOwnProperty("remoteNpm")) { answers.remoteNpm = args.remoteNpm; } - answers = await inquirer.prompt([ - { - type: "confirm", - name: "remoteNpm", - message: "Allow remote updates and plugin installs from the web interface?", - default: true, - }, - ], answers); + answers.remoteNpm ??= await confirm({ + message: "Allow remote updates and plugin installs from the web interface?", + default: true, + }); } if (answers.mode === "plugin-template") { @@ -725,29 +678,23 @@ async function inquirerMissingArgs(args) { answers.pluginName = args["plugin-name"]; } answers.plugins = []; - answers = await inquirer.prompt([ - { - type: "checkbox", - name: "pluginTemplate", - message: "Plugin templates to use", - choices: [ - { name: "Controller", value: "controller" }, - { name: "Host", value: "host" }, - { name: "Instance", value: "instance" }, - { name: "Lua Module", value: "module" }, - { name: "Command Line", value: "ctl" }, - { name: "Web UI", value: "web" }, - { name: "No Typescript", value: "js" }, - { name: "No Config", value: "no_config" }, - ], - }, - { - type: "input", - name: "pluginName", - message: "Name of your new plugin", - default: path.basename(path.resolve()), - }, - ], answers); + answers.pluginTemplate ??= await checkbox({ + message: "Plugin templates to use", + choices: [ + { name: "Controller", value: "controller" }, + { name: "Host", value: "host" }, + { name: "Instance", value: "instance" }, + { name: "Lua Module", value: "module" }, + { name: "Command Line", value: "ctl" }, + { name: "Web UI", value: "web" }, + { name: "No Typescript", value: "js" }, + { name: "No Config", value: "no_config" }, + ], + }); + answers.pluginName ??= await input({ + message: "Name of your new plugin", + default: path.basename(path.resolve()), + }); } if (dev) { @@ -755,15 +702,11 @@ async function inquirerMissingArgs(args) { } else if (args.plugins) { answers.plugins = args.plugins; } - answers = await inquirer.prompt([ - { - type: "checkbox", - name: "plugins", - message: "Plugins to install", - choices: availablePlugins, - pageSize: 20, - }, - ], answers); + answers.plugins ??= await checkbox({ + message: "Plugins to install", + choices: availablePlugins, + pageSize: 20, + }); return answers; } @@ -847,7 +790,7 @@ async function main() { return; } - let answers = await inquirerMissingArgs(args); + let answers = await promptMissingArgs(args); logger.verbose(JSON.stringify(answers)); if (answers.pluginTemplate) { @@ -911,7 +854,10 @@ async function main() { if (import.meta.main) { main().catch(err => { process.exitCode = 1; - if (err instanceof InstallError) { + if (err.name === "ExitPromptError") { + // Ctrl+C at a prompt + process.exitCode = 130; + } else if (err instanceof InstallError) { logger.error(err.message); } else { logger.fatal(` diff --git a/packages/create/package.json b/packages/create/package.json index ba34a393..e76b220b 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -32,8 +32,11 @@ }, "dependencies": { + "@inquirer/checkbox": "^5.2.8", + "@inquirer/confirm": "^6.3.5", + "@inquirer/input": "^5.1.9", + "@inquirer/select": "^5.2.8", "chalk": "catalog:", - "inquirer": "^8.2.7", "yargs": "catalog:" } } From 46c61532014ebaa39c5403b53814198526156964 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 1 Nov 2026 18:33:10 +0100 Subject: [PATCH 17/44] Export without the graphics directory when it has no matching install If host.factorio_graphics_directory had no install matching the major and minor version of the instance, or did not exist, findVersion threw and the whole export failed. Without the setting the same instance exports everything except the base game icons, so it now logs a warning that names the setting and exports without it. --- docs/configuration.md | 1 + packages/host/src/export.ts | 29 +++++++++++++++++++++++++++-- test/host/export.js | 22 +++++++++++++++++++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b6c46453..77516d26 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -363,6 +363,7 @@ Defaults to "factorio". Directory to read the base game's icons from when exporting data. The headless build of Factorio has no graphics, so instances running it export none of the base game's icons unless this points to a full install of Factorio. Like `host.factorio_directory` this can either point to a Factorio directory, or to a directory containing multiple versions, in which case the latest one matching the major and minor version of the instance is used. +If there is no install matching the instance, a warning is logged and the data is exported without it. Defaults to null. diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index 511ab231..cbc386db 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -523,6 +523,31 @@ async function exportLocale( return mergedLocales; } +/** + * Find the data dir to read builtin mod graphics from + * + * @param server - The server the export runs on. + * @param graphicsDirectory - Full Factorio install, or directory of installs. + * @returns + * Data dir of the install matching the major and minor version of the + * server, or undefined if there is none. + * @internal + */ +async function findGraphicsDataDir(server: FactorioServer, graphicsDirectory: string) { + const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion; + try { + const [dataDir] = await findVersion(graphicsDirectory, majorMinor); + return dataDir; + } catch (err: any) { + // Without it the export still has everything but the base game icons + server._logger.warn( + `Unable to use host.factorio_graphics_directory for Factorio ${majorMinor}, exporting without it: ` + + `${err.message}` + ); + return undefined; + } +} + /** * Export locale, icons and metadata for the given factorio server * @@ -535,8 +560,7 @@ async function exportLocale( export async function exportData(server: FactorioServer, graphicsDirectory?: string | null) { const files: ExportFiles = { zipCache: new Map() }; if (graphicsDirectory) { - const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion; - [files.graphicsDataDir] = await findVersion(graphicsDirectory, majorMinor); + files.graphicsDataDir = await findGraphicsDataDir(server, graphicsDirectory); } try { return await buildExport(server, files); @@ -700,6 +724,7 @@ async function buildExport(server: FactorioServer, files: ExportFiles) { export const _createSheetState = createSheetState; export const _exportIcons = exportIcons; export const _exportLocale = exportLocale; +export const _findGraphicsDataDir = findGraphicsDataDir; export const _loadSimpleIcon = loadSimpleIcon; export const _loadLayeredIcon = loadLayeredIcon; export const _warnMissingBuiltinIcons = warnMissingBuiltinIcons; diff --git a/test/host/export.js b/test/host/export.js index 1bc5a4a3..c2755c1b 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -3,7 +3,8 @@ import path from "node:path"; import { FactorioServer } from "@clusterio/host/dist/node/src/server.js"; import { - _createSheetState, _exportIcons, _exportLocale, _loadLayeredIcon, _loadSimpleIcon, _warnMissingBuiltinIcons, + _createSheetState, _exportIcons, _exportLocale, _findGraphicsDataDir, _loadLayeredIcon, _loadSimpleIcon, + _warnMissingBuiltinIcons, } from "@clusterio/host/dist/node/src/export.js"; @@ -121,6 +122,25 @@ describe("host/src/export", function() { assert.equal(icon, null); }); }); + describe("findGraphicsDataDir()", function() { + const installs = path.join("test", "file", "factorio"); + it("returns the data dir of the install matching the server version", async function() { + const dataDir = await _findGraphicsDataDir(testServer, installs); + assert.equal(dataDir, path.join(installs, "0.1.2", "data")); + assert.deepEqual(warnings, []); + }); + it("warns and returns undefined when no install matches the server version", async function() { + const server = Object.create(testServer, { version: { value: "0.2.5" } }); + assert.equal(await _findGraphicsDataDir(server, installs), undefined); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /^Unable to use host\.factorio_graphics_directory for Factorio 0\.2, /); + }); + it("warns and returns undefined when the directory does not exist", async function() { + assert.equal(await _findGraphicsDataDir(testServer, path.join("test", "file", "missing")), undefined); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /^Unable to use host\.factorio_graphics_directory for Factorio 0\.1, .*ENOENT/); + }); + }); describe("warnMissingBuiltinIcons()", function() { it("warns once with the number of missing builtin files", function() { _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ From 37d74ca2a6cd550fbf499037804c1b511cda36c1 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:53:05 +0000 Subject: [PATCH 18/44] Find a full install in host.factorio_directory instead of a setting Per review, drop host.factorio_graphics_directory. findVersion takes a headless option that filters installs by build, telling them apart by data/base/graphics/icons. When the instance runs the headless build the export looks for a full install of the same major.minor version in the host's factorio directory. Server startup is unchanged. If none is found the export still runs, and the reason is included in the missing icons warning. Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 11 +--- packages/host/src/Instance.ts | 2 +- packages/host/src/export.ts | 52 +++++++++--------- packages/host/src/server.ts | 44 ++++++++++++++- packages/lib/src/config/definitions.ts | 9 --- .../base/graphics/icons/graphics-icon.png | Bin .../factorio-mixed/full/data/changelog.txt | 3 + .../headless/data/changelog.txt | 3 + test/host/export.js | 46 +++++++++++----- test/host/server.js | 27 +++++++++ 10 files changed, 132 insertions(+), 65 deletions(-) rename test/file/{factorio-graphics => factorio-mixed/full/data}/base/graphics/icons/graphics-icon.png (100%) create mode 100644 test/file/factorio-mixed/full/data/changelog.txt create mode 100644 test/file/factorio-mixed/headless/data/changelog.txt diff --git a/docs/configuration.md b/docs/configuration.md index 77516d26..89650ca9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -354,20 +354,11 @@ Defaults to a random 31 bit number. Directory to look for the Factorio server(s) in. This can either point to a Factorio server directory, or to a directory containing multiple versions of the Factorio server. +The headless build has no graphics, so when exporting data from an instance running it the base game's icons are read from a full install of the same major and minor version in this directory instead, if there is one. Defaults to "factorio". -### host.factorio_graphics_directory - -Directory to read the base game's icons from when exporting data. -The headless build of Factorio has no graphics, so instances running it export none of the base game's icons unless this points to a full install of Factorio. -Like `host.factorio_directory` this can either point to a Factorio directory, or to a directory containing multiple versions, in which case the latest one matching the major and minor version of the instance is used. -If there is no install matching the instance, a warning is logged and the data is exported without it. - -Defaults to null. - - ### host.mods_directory Directory the host stores mods downloaded from the controller in. diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 3dc7f6e6..6cee6a7b 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -1479,7 +1479,7 @@ end`.replace(/\r?\n/g, " "); this.logger.info("Exporting data ....."); await this.syncMods(); - let zip = await exportData(this.server, this._host.config.get("host.factorio_graphics_directory")); + let zip = await exportData(this.server); let url = new URL(this._host.config.get("host.controller_url")); url.pathname += "api/upload-export"; diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index cbc386db..8be4843a 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -10,7 +10,7 @@ import yazl from "yazl"; import * as lib from "@clusterio/lib"; import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js"; -import { findVersion, type FactorioServer } from "./server.js"; +import { findVersion, isHeadless, type FactorioServer } from "./server.js"; import { definesPrototypesFallback } from "./export_fallback.js"; interface SimpleIconSpecification { @@ -100,6 +100,8 @@ interface ExportFiles { zipCache: ZipCache; /** Data dir of a full install to fall back to for builtin mod files. */ graphicsDataDir?: string; + /** Why no full install was found for a headless server. */ + graphicsError?: string; } function closeZipCache(zipCache: ZipCache) { @@ -524,43 +526,36 @@ async function exportLocale( } /** - * Find the data dir to read builtin mod graphics from + * Find a full install to read builtin mod graphics from * * @param server - The server the export runs on. - * @param graphicsDirectory - Full Factorio install, or directory of installs. - * @returns - * Data dir of the install matching the major and minor version of the - * server, or undefined if there is none. + * @param files - Updated with the data dir found, or why there is none. * @internal */ -async function findGraphicsDataDir(server: FactorioServer, graphicsDirectory: string) { +async function findGraphicsDataDir(server: FactorioServer, files: ExportFiles) { const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion; try { - const [dataDir] = await findVersion(graphicsDirectory, majorMinor); - return dataDir; + [files.graphicsDataDir] = await findVersion(server._factorioDir, majorMinor, { headless: false }); } catch (err: any) { // Without it the export still has everything but the base game icons - server._logger.warn( - `Unable to use host.factorio_graphics_directory for Factorio ${majorMinor}, exporting without it: ` + - `${err.message}` - ); - return undefined; + files.graphicsError = err.message; } } /** * Export locale, icons and metadata for the given factorio server * + * If the server runs the headless build, builtin mod graphics are read + * from a full install of the same major and minor version found next to + * it, if there is one. + * * @param server - The server to export the data from. - * @param graphicsDirectory - - * Full Factorio install, or directory of installs, to read builtin mod - * graphics from when the server's install lacks them. * @returns zip file with exported data. */ -export async function exportData(server: FactorioServer, graphicsDirectory?: string | null) { +export async function exportData(server: FactorioServer) { const files: ExportFiles = { zipCache: new Map() }; - if (graphicsDirectory) { - files.graphicsDataDir = await findGraphicsDataDir(server, graphicsDirectory); + if (await isHeadless(server.dataPath())) { + await findGraphicsDataDir(server, files); } try { return await buildExport(server, files); @@ -592,13 +587,16 @@ function warnMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, ico } const dirs = [server.dataPath(), files.graphicsDataDir].filter(Boolean).join(" or "); - const hint = files.graphicsDataDir - ? "Check that host.factorio_graphics_directory points to a full Factorio install" - : "Set host.factorio_graphics_directory to a full Factorio install to read them from there"; - server._logger.warn( - `${missing} icon files of the base game were not found in ${dirs}, icons using them were not exported. ` + - `The headless build of Factorio has no graphics. ${hint}, or export the data on a host with a full install.` - ); + let message = + `${missing} icon files of the base game were not found in ${dirs}, icons using them were not exported.`; + if (files.graphicsError !== undefined) { + const majorMinor = server.version!.replace(/\.\d+$/, ""); + message += + ` The headless build of Factorio has no graphics and no full install was found (${files.graphicsError}).` + + ` Add a full install of Factorio ${majorMinor} to host.factorio_directory to read them from there,` + + " or export the data on a host with a full install."; + } + server._logger.warn(message); } async function buildExport(server: FactorioServer, files: ExportFiles) { diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index df3e85f7..a3abe32b 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -262,15 +262,34 @@ export async function checkForUpdates( * @param targetVersion - * Version to look for, supports the special value "latest" for the * latest version available. + * @param options - Optional parameters. + * @param options.headless - + * Only consider headless installs if true, or only full installs if + * false. * @returns Array with path to data dir and version found. */ export async function findVersion( factorioDir: string, targetVersion: lib.TargetVersion, + options: { headless?: boolean } = {}, ): Promise<[string, lib.FullVersion]> { + let kind = ""; + if (options.headless !== undefined) { + kind = options.headless ? "headless " : "full "; + } + const wrongKind = async (dir: string) => ( + options.headless !== undefined && await isHeadless(path.join(dir, "data")) !== options.headless + ); + // Check if this is a direct installation dir const directVersion = await getFactorioVersion(factorioDir); if (directVersion !== null) { + if (await wrongKind(factorioDir)) { + throw new Error( + `Unable to find a ${kind}install of Factorio ${targetVersion}: ${factorioDir} is a direct ` + + `(single-version) install of the ${options.headless ? "full" : "headless"} build.` + ); + } if (targetVersion === "latest" || directVersion === targetVersion || directVersion.startsWith(targetVersion)) { return [path.join(factorioDir, "data"), directVersion]; } @@ -287,7 +306,7 @@ export async function findVersion( // Avoid any fancy directory detection in order to follow symlinks const fullPath = path.join(factorioDir, name); const version = await getFactorioVersion(fullPath); - if (version === null) { + if (version === null || await wrongKind(fullPath)) { continue; } @@ -299,7 +318,7 @@ export async function findVersion( } if (!versions.size) { - throw new Error(`Unable to find any Factorio install in ${factorioDir}`); + throw new Error(`Unable to find any ${kind}Factorio install in ${factorioDir}`); } if (lib.isPartialVersion(targetVersion)) { @@ -315,7 +334,26 @@ export async function findVersion( return [path.join(factorioDir, versions.get(latest)!, "data"), latest]; } - throw new Error(`Unable to find Factorio version ${targetVersion}`); + throw new Error(`Unable to find ${kind}Factorio version ${targetVersion}`); +} + +/** + * Check if a Factorio install is the headless build + * + * The headless build has no icons, which is what tells it apart. + * + * @param dataDir - Path to the data dir of the install. + */ +export async function isHeadless(dataDir: string) { + try { + await fs.access(path.join(dataDir, "base", "graphics", "icons")); + return false; + } catch (err: any) { + if (err.code === "ENOENT" || err.code === "ENOTDIR") { + return true; + } + throw err; + } } /** diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 4296f31d..b28db160 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -340,7 +340,6 @@ export interface HostConfigFields { "host.name": string; "host.id": number; "host.factorio_directory": string; - "host.factorio_graphics_directory": string | null; "host.mods_directory": string; "host.instances_directory": string; "host.controller_url": string; @@ -389,14 +388,6 @@ export class HostConfig extends classes.Config { initialValue: "factorio", validator: validators.filePath, }, - "host.factorio_graphics_directory": { - description: - "Path to a full Factorio install, or directory of installs, to read base game icons from when " + - "exporting data. Needed when running the headless build, which has no graphics.", - type: "string", - optional: true, - validator: validators.optional(validators.filePath), - }, "host.mods_directory": { title: "Mods Directory", description: "Path to directory where mods for instances are cached.", diff --git a/test/file/factorio-graphics/base/graphics/icons/graphics-icon.png b/test/file/factorio-mixed/full/data/base/graphics/icons/graphics-icon.png similarity index 100% rename from test/file/factorio-graphics/base/graphics/icons/graphics-icon.png rename to test/file/factorio-mixed/full/data/base/graphics/icons/graphics-icon.png diff --git a/test/file/factorio-mixed/full/data/changelog.txt b/test/file/factorio-mixed/full/data/changelog.txt new file mode 100644 index 00000000..2b12ff4b --- /dev/null +++ b/test/file/factorio-mixed/full/data/changelog.txt @@ -0,0 +1,3 @@ +--------------------------------------------------------------------------------------------------- +Version: 0.1.2 +About: Dummy changelog.txt for FactorioServer tests diff --git a/test/file/factorio-mixed/headless/data/changelog.txt b/test/file/factorio-mixed/headless/data/changelog.txt new file mode 100644 index 00000000..2b12ff4b --- /dev/null +++ b/test/file/factorio-mixed/headless/data/changelog.txt @@ -0,0 +1,3 @@ +--------------------------------------------------------------------------------------------------- +Version: 0.1.2 +About: Dummy changelog.txt for FactorioServer tests diff --git a/test/host/export.js b/test/host/export.js index c2755c1b..0d7a2806 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -74,7 +74,7 @@ describe("host/src/export", function() { }); // Stands in for the data dir of a full install - const graphicsDataDir = path.join("test", "file", "factorio-graphics"); + const graphicsDataDir = path.join("test", "file", "factorio-mixed", "full", "data"); describe("loadSimpleIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadSimpleIcon( @@ -124,21 +124,29 @@ describe("host/src/export", function() { }); describe("findGraphicsDataDir()", function() { const installs = path.join("test", "file", "factorio"); - it("returns the data dir of the install matching the server version", async function() { - const dataDir = await _findGraphicsDataDir(testServer, installs); - assert.equal(dataDir, path.join(installs, "0.1.2", "data")); - assert.deepEqual(warnings, []); + const withInstalls = (factorioDir, version = testServer.version) => Object.create(testServer, { + _factorioDir: { value: factorioDir }, version: { value: version }, }); - it("warns and returns undefined when no install matches the server version", async function() { - const server = Object.create(testServer, { version: { value: "0.2.5" } }); - assert.equal(await _findGraphicsDataDir(server, installs), undefined); - assert.equal(warnings.length, 1); - assert.match(warnings[0], /^Unable to use host\.factorio_graphics_directory for Factorio 0\.2, /); + it("finds a full install matching the major and minor version of the server", async function() { + const files = { zipCache: new Map() }; + await _findGraphicsDataDir(withInstalls(path.join("test", "file", "factorio"), "0.1.1"), files); + assert.deepEqual(files, { zipCache: new Map(), graphicsDataDir: path.join(installs, "0.1.2", "data") }); }); - it("warns and returns undefined when the directory does not exist", async function() { - assert.equal(await _findGraphicsDataDir(testServer, path.join("test", "file", "missing")), undefined); - assert.equal(warnings.length, 1); - assert.match(warnings[0], /^Unable to use host\.factorio_graphics_directory for Factorio 0\.1, .*ENOENT/); + it("skips headless installs of the same version", async function() { + const files = { zipCache: new Map() }; + await _findGraphicsDataDir(withInstalls(path.join("test", "file", "factorio-mixed")), files); + assert.equal(files.graphicsDataDir, graphicsDataDir); + }); + it("records why when there is no full install", async function() { + const files = { zipCache: new Map() }; + await _findGraphicsDataDir(withInstalls(installs, "0.2.5"), files); + assert.equal(files.graphicsDataDir, undefined); + assert.equal(files.graphicsError, "Unable to find full Factorio version 0.2"); + }); + it("records why when the directory does not exist", async function() { + const files = { zipCache: new Map() }; + await _findGraphicsDataDir(withInstalls(path.join("test", "file", "missing")), files); + assert.match(files.graphicsError, /ENOENT/); }); }); describe("warnMissingBuiltinIcons()", function() { @@ -151,7 +159,15 @@ describe("host/src/export", function() { ])); assert.equal(warnings.length, 1); assert.match(warnings[0], /^2 icon files of the base game were not found/); - assert.match(warnings[0], /headless/); + assert.doesNotMatch(warnings[0], /headless/); + }); + it("explains the headless build when no full install was found", function() { + _warnMissingBuiltinIcons(testServer, { zipCache: new Map(), graphicsError: "Not found" }, new Map([ + ["__base__/graphics/icons/a.png", null], + ])); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /has no graphics and no full install was found \(Not found\)/); + assert.match(warnings[0], /Add a full install of Factorio 0\.1 to host\.factorio_directory/); }); it("does not warn when no builtin files are missing", function() { _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ diff --git a/test/host/server.js b/test/host/server.js index da6169a9..a1cc1c33 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -105,6 +105,33 @@ describe("host/server", function() { ); }); }); + describe("headless option", function() { + const mixedDir = path.join("test", "file", "factorio-mixed"); + it("should find only full installs when false", async function() { + const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", { headless: false }); + assert.equal(dir, path.join(mixedDir, "full", "data")); + }); + it("should find only headless installs when true", async function() { + const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", { headless: true }); + assert.equal(dir, path.join(mixedDir, "headless", "data")); + }); + it("should reject a direct install of the other build", async function() { + const installDir = path.join(mixedDir, "headless"); + await assert.rejects( + hostServer.findVersion(installDir, "0.1.2", { headless: false }), + new Error( + `Unable to find a full install of Factorio 0.1.2: ${installDir} is a direct ` + + "(single-version) install of the headless build." + ) + ); + }); + it("should reject if no install of that build has the version", async function() { + await assert.rejects( + hostServer.findVersion(path.join("test", "file", "factorio"), "0.1.2", { headless: true }), + new Error("Unable to find headless Factorio version 0.1.2") + ); + }); + }); }); describe("_listFactorioVersions()", function() { From f8aafe9265e4240b5e5bf2b365ae5991407ee67d Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:44:43 +0000 Subject: [PATCH 19/44] Address review on findVersion and the export graphics lookup findVersion takes headless as a plain optional argument, the helper is checkInstallType, the warning is checkMissingBuiltinIcons and the full install lookup is inlined into exportData. Co-Authored-By: Claude Opus 5.5 --- packages/host/src/export.ts | 35 ++++++++++++---------------------- packages/host/src/server.ts | 19 +++++++++---------- test/host/export.js | 38 +++++-------------------------------- test/host/server.js | 15 ++++++++++----- 4 files changed, 36 insertions(+), 71 deletions(-) diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index 8be4843a..e77ff46e 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -97,6 +97,7 @@ async function loadZip(server: FactorioServer, modVersions: Map, /** Where icon files are read from during an export. */ interface ExportFiles { + /** Mod zips opened during the export, closed when it ends. */ zipCache: ZipCache; /** Data dir of a full install to fall back to for builtin mod files. */ graphicsDataDir?: string; @@ -525,23 +526,6 @@ async function exportLocale( return mergedLocales; } -/** - * Find a full install to read builtin mod graphics from - * - * @param server - The server the export runs on. - * @param files - Updated with the data dir found, or why there is none. - * @internal - */ -async function findGraphicsDataDir(server: FactorioServer, files: ExportFiles) { - const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion; - try { - [files.graphicsDataDir] = await findVersion(server._factorioDir, majorMinor, { headless: false }); - } catch (err: any) { - // Without it the export still has everything but the base game icons - files.graphicsError = err.message; - } -} - /** * Export locale, icons and metadata for the given factorio server * @@ -555,7 +539,13 @@ async function findGraphicsDataDir(server: FactorioServer, files: ExportFiles) { export async function exportData(server: FactorioServer) { const files: ExportFiles = { zipCache: new Map() }; if (await isHeadless(server.dataPath())) { - await findGraphicsDataDir(server, files); + const majorMinor = server.version!.replace(/\.\d+$/, "") as lib.MajorMinorVersion; + try { + [files.graphicsDataDir] = await findVersion(server._factorioDir, majorMinor, false); + } catch (err: any) { + // Without it the export still has everything but the base game icons + files.graphicsError = err.message; + } } try { return await buildExport(server, files); @@ -566,14 +556,14 @@ export async function exportData(server: FactorioServer) { } /** - * Warn about builtin mod icon files that could not be found + * Warn if any builtin mod icon files could not be found * * @param server - The server the export ran on. * @param files - Where icon files were read from. * @param iconCache - Icons loaded by the export, null for missing files. * @internal */ -function warnMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, iconCache: IconCache) { +function checkMissingBuiltinIcons(server: FactorioServer, files: ExportFiles, iconCache: IconCache) { const builtinModNames = ["core", ...lib.ModPack.getBuiltinModNames(server.version!)]; let missing = 0; for (const [iconPath, icon] of iconCache) { @@ -703,7 +693,7 @@ async function buildExport(server: FactorioServer, files: ExportFiles) { const count = await exportIcons(server, modVersions, files, prototypesWithIcons, baseName, state); server._logger.info(`Exported ${count} ${baseName} icons`); } - warnMissingBuiltinIcons(server, files, state.iconCache); + checkMissingBuiltinIcons(server, files, state.iconCache); // Crop sheet to actual used height and write single spritesheet + metadata. const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE)); @@ -722,7 +712,6 @@ async function buildExport(server: FactorioServer, files: ExportFiles) { export const _createSheetState = createSheetState; export const _exportIcons = exportIcons; export const _exportLocale = exportLocale; -export const _findGraphicsDataDir = findGraphicsDataDir; export const _loadSimpleIcon = loadSimpleIcon; export const _loadLayeredIcon = loadLayeredIcon; -export const _warnMissingBuiltinIcons = warnMissingBuiltinIcons; +export const _checkMissingBuiltinIcons = checkMissingBuiltinIcons; diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index a3abe32b..1e1526a5 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -262,8 +262,7 @@ export async function checkForUpdates( * @param targetVersion - * Version to look for, supports the special value "latest" for the * latest version available. - * @param options - Optional parameters. - * @param options.headless - + * @param headless - * Only consider headless installs if true, or only full installs if * false. * @returns Array with path to data dir and version found. @@ -271,23 +270,23 @@ export async function checkForUpdates( export async function findVersion( factorioDir: string, targetVersion: lib.TargetVersion, - options: { headless?: boolean } = {}, + headless?: boolean, ): Promise<[string, lib.FullVersion]> { let kind = ""; - if (options.headless !== undefined) { - kind = options.headless ? "headless " : "full "; + if (headless !== undefined) { + kind = headless ? "headless " : "full "; } - const wrongKind = async (dir: string) => ( - options.headless !== undefined && await isHeadless(path.join(dir, "data")) !== options.headless + const checkInstallType = async (dir: string) => ( + headless === undefined || await isHeadless(path.join(dir, "data")) === headless ); // Check if this is a direct installation dir const directVersion = await getFactorioVersion(factorioDir); if (directVersion !== null) { - if (await wrongKind(factorioDir)) { + if (!await checkInstallType(factorioDir)) { throw new Error( `Unable to find a ${kind}install of Factorio ${targetVersion}: ${factorioDir} is a direct ` + - `(single-version) install of the ${options.headless ? "full" : "headless"} build.` + `(single-version) install of the ${headless ? "full" : "headless"} build.` ); } if (targetVersion === "latest" || directVersion === targetVersion || directVersion.startsWith(targetVersion)) { @@ -306,7 +305,7 @@ export async function findVersion( // Avoid any fancy directory detection in order to follow symlinks const fullPath = path.join(factorioDir, name); const version = await getFactorioVersion(fullPath); - if (version === null || await wrongKind(fullPath)) { + if (version === null || !await checkInstallType(fullPath)) { continue; } diff --git a/test/host/export.js b/test/host/export.js index 0d7a2806..b5f12579 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -3,8 +3,7 @@ import path from "node:path"; import { FactorioServer } from "@clusterio/host/dist/node/src/server.js"; import { - _createSheetState, _exportIcons, _exportLocale, _findGraphicsDataDir, _loadLayeredIcon, _loadSimpleIcon, - _warnMissingBuiltinIcons, + _checkMissingBuiltinIcons, _createSheetState, _exportIcons, _exportLocale, _loadLayeredIcon, _loadSimpleIcon, } from "@clusterio/host/dist/node/src/export.js"; @@ -122,36 +121,9 @@ describe("host/src/export", function() { assert.equal(icon, null); }); }); - describe("findGraphicsDataDir()", function() { - const installs = path.join("test", "file", "factorio"); - const withInstalls = (factorioDir, version = testServer.version) => Object.create(testServer, { - _factorioDir: { value: factorioDir }, version: { value: version }, - }); - it("finds a full install matching the major and minor version of the server", async function() { - const files = { zipCache: new Map() }; - await _findGraphicsDataDir(withInstalls(path.join("test", "file", "factorio"), "0.1.1"), files); - assert.deepEqual(files, { zipCache: new Map(), graphicsDataDir: path.join(installs, "0.1.2", "data") }); - }); - it("skips headless installs of the same version", async function() { - const files = { zipCache: new Map() }; - await _findGraphicsDataDir(withInstalls(path.join("test", "file", "factorio-mixed")), files); - assert.equal(files.graphicsDataDir, graphicsDataDir); - }); - it("records why when there is no full install", async function() { - const files = { zipCache: new Map() }; - await _findGraphicsDataDir(withInstalls(installs, "0.2.5"), files); - assert.equal(files.graphicsDataDir, undefined); - assert.equal(files.graphicsError, "Unable to find full Factorio version 0.2"); - }); - it("records why when the directory does not exist", async function() { - const files = { zipCache: new Map() }; - await _findGraphicsDataDir(withInstalls(path.join("test", "file", "missing")), files); - assert.match(files.graphicsError, /ENOENT/); - }); - }); - describe("warnMissingBuiltinIcons()", function() { + describe("checkMissingBuiltinIcons()", function() { it("warns once with the number of missing builtin files", function() { - _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ + _checkMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ ["__base__/graphics/icons/a.png", null], ["__core__/graphics/icons/b.png", null], ["__base__/graphics/icons/c.png", {}], @@ -162,7 +134,7 @@ describe("host/src/export", function() { assert.doesNotMatch(warnings[0], /headless/); }); it("explains the headless build when no full install was found", function() { - _warnMissingBuiltinIcons(testServer, { zipCache: new Map(), graphicsError: "Not found" }, new Map([ + _checkMissingBuiltinIcons(testServer, { zipCache: new Map(), graphicsError: "Not found" }, new Map([ ["__base__/graphics/icons/a.png", null], ])); assert.equal(warnings.length, 1); @@ -170,7 +142,7 @@ describe("host/src/export", function() { assert.match(warnings[0], /Add a full install of Factorio 0\.1 to host\.factorio_directory/); }); it("does not warn when no builtin files are missing", function() { - _warnMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ + _checkMissingBuiltinIcons(testServer, { zipCache: new Map() }, new Map([ ["__base__/graphics/icons/c.png", {}], ["__some_mod__/graphics/icons/d.png", null], ])); diff --git a/test/host/server.js b/test/host/server.js index a1cc1c33..043aae0c 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -105,20 +105,25 @@ describe("host/server", function() { ); }); }); - describe("headless option", function() { + describe("headless argument", function() { const mixedDir = path.join("test", "file", "factorio-mixed"); it("should find only full installs when false", async function() { - const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", { headless: false }); + const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", false); assert.equal(dir, path.join(mixedDir, "full", "data")); }); it("should find only headless installs when true", async function() { - const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", { headless: true }); + const [dir] = await hostServer.findVersion(mixedDir, "0.1.2", true); assert.equal(dir, path.join(mixedDir, "headless", "data")); }); + it("should skip installs of the other build when matching a partial version", async function() { + const installDir = path.join("test", "file", "factorio"); + const [dir, version] = await hostServer.findVersion(installDir, "0.1", false); + assert.deepEqual([dir, version], [path.join(installDir, "0.1.2", "data"), "0.1.2"]); + }); it("should reject a direct install of the other build", async function() { const installDir = path.join(mixedDir, "headless"); await assert.rejects( - hostServer.findVersion(installDir, "0.1.2", { headless: false }), + hostServer.findVersion(installDir, "0.1.2", false), new Error( `Unable to find a full install of Factorio 0.1.2: ${installDir} is a direct ` + "(single-version) install of the headless build." @@ -127,7 +132,7 @@ describe("host/server", function() { }); it("should reject if no install of that build has the version", async function() { await assert.rejects( - hostServer.findVersion(path.join("test", "file", "factorio"), "0.1.2", { headless: true }), + hostServer.findVersion(path.join("test", "file", "factorio"), "0.1.2", true), new Error("Unable to find headless Factorio version 0.1.2") ); }); From 03ea6accc1c1cdfd740a2e72b813072de906a86c Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 2 Nov 2026 00:06:02 +0100 Subject: [PATCH 20/44] Show defaults for mod settings added while the page is open (#1172) The mod pack view gave the settings form its values through the Form's initialValues, which rc-field-form only applies when the form is created. Settings added by a data export while the page was open were shown empty, and color settings black, until the page was reloaded. Each setting field now gets its saved value as its own initialValue, which is applied when the field is created and when the form is reset. --- packages/web_ui/src/components/ModPackViewPage.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/web_ui/src/components/ModPackViewPage.tsx b/packages/web_ui/src/components/ModPackViewPage.tsx index f0216254..39ecfafc 100644 --- a/packages/web_ui/src/components/ModPackViewPage.tsx +++ b/packages/web_ui/src/components/ModPackViewPage.tsx @@ -907,6 +907,8 @@ type SettingsTableFieldProps = { allowedValues: any; locale: Map; changed: boolean; + /** Saved value of the setting, used when the field is created and when the form is reset */ + initialValue: lib.ModSetting["value"] | undefined; }; const SettingsTableField = memo((props: SettingsTableFieldProps) => { const name = props.name; @@ -932,6 +934,7 @@ const SettingsTableField = memo((props: SettingsTableFieldProps) => { return { type SettingsTableProps = { modPack: lib.ModPack; + /** Mod pack without unsaved changes, reverting the form goes back to its values */ + savedModPack: lib.ModPack; prototypes: any; locale: any; changes: ModChange[]; @@ -1014,6 +1019,7 @@ function SettingsTable(props: SettingsTableProps) { allowedValues={prototype.allowed_values} locale={props.locale} changed={hasChange(props.changes, { type: "settings.set", scope, name: prototype.name })} + initialValue={props.savedModPack.settings[scope].get(prototype.name)?.value} /> )} ); @@ -1294,13 +1300,12 @@ export default function ModPackViewPage() { syncTimeout.current = setTimeout(syncPack.bind(undefined, allFields), 500); } }} + // Mod settings get their value on each field instead, the form only applies initialValues when it is + // created, so settings added by an export while the page is open would show up empty. initialValues={{ name: modPack.name, description: modPack.description, factorioVersion: modPack.factorioVersion, - "startup": Object.fromEntries(pickValue(modPack.settings["startup"])), - "runtime-global": Object.fromEntries(pickValue(modPack.settings["runtime-global"])), - "runtime-per-user": Object.fromEntries(pickValue(modPack.settings["runtime-per-user"])), }} > @@ -1349,6 +1354,7 @@ export default function ModPackViewPage() { /> Date: Mon, 2 Nov 2026 00:06:16 +0100 Subject: [PATCH 21/44] Keep recipe notifications when a recipe list is empty (#1170) Factorio encodes an empty Lua table as {}, so a player with no cleared recipes sent {} with the download request. new Set({}) threw, the instance dropped the recipe notifications from the download, and the next upload replaced the stored list on the controller. An empty stored list was dropped the same way. Decode both lists treating {} as empty. --- plugins/player_sync/instance.ts | 20 ++++++++++++++++++-- plugins/player_sync/test/instance.js | 26 +++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/plugins/player_sync/instance.ts b/plugins/player_sync/instance.ts index 30ed0223..4c185527 100644 --- a/plugins/player_sync/instance.ts +++ b/plugins/player_sync/instance.ts @@ -87,6 +87,22 @@ export function recipeNotificationDelta(stored: string[], current: string[]): Re return delta; } +/** + * Decode a list of cleared recipe names, Factorio serializes an empty list as {} + * @param encoded - Encoded JSON list of recipe names + * @returns the recipe names + */ +async function decodeRecipeNames(encoded: string): Promise { + const names = JSON.parse(await decodeLuaString(encoded)); + if (Array.isArray(names)) { + return names; + } + if (names !== null && typeof names === "object" && Object.keys(names).length === 0) { + return []; + } + throw new Error(`Expected a list of recipe names, got ${JSON.stringify(names)}`); +} + /** * Replace the stored recipe notifications with the difference from the * current state on the instance to reduce the amount of data sent to Lua. @@ -101,9 +117,9 @@ export async function applyRecipeNotificationDelta( return; } try { - const stored = JSON.parse(await decodeLuaString(playerData.recipe_notifications)); + const stored = await decodeRecipeNames(playerData.recipe_notifications); const delta: RecipeNotificationDelta = current - ? recipeNotificationDelta(stored, JSON.parse(await decodeLuaString(current))) + ? recipeNotificationDelta(stored, await decodeRecipeNames(current)) : { ...recipeNotificationDelta(stored, []), full: true }; playerData.recipe_notifications = await encodeLuaString(JSON.stringify(delta)); } catch (err: any) { diff --git a/plugins/player_sync/test/instance.js b/plugins/player_sync/test/instance.js index 12209271..67132ce6 100644 --- a/plugins/player_sync/test/instance.js +++ b/plugins/player_sync/test/instance.js @@ -64,6 +64,29 @@ describe("player_sync", function() { await apply(playerData, undefined); assert.deepEqual(await decode(playerData.recipe_notifications), { full: true }); }); + // helpers.table_to_json in Factorio encodes an empty list as {} + it("should treat an empty current list encoded as {} as empty", async function() { + const playerData = { name: "test", recipe_notifications: await encode(["a", "b"]) }; + await apply(playerData, await encode({})); + assert.deepEqual(await decode(playerData.recipe_notifications), { add: ["a", "b"] }); + }); + it("should treat an empty stored list encoded as {} as empty", async function() { + const playerData = { name: "test", recipe_notifications: await encode({}) }; + await apply(playerData, await encode(["a"])); + assert.deepEqual(await decode(playerData.recipe_notifications), { remove: ["a"] }); + }); + it("should mark an empty stored list encoded as {} full without a current list", async function() { + const playerData = { name: "test", recipe_notifications: await encode({}) }; + await apply(playerData, undefined); + assert.deepEqual(await decode(playerData.recipe_notifications), { full: true }); + }); + it("should drop the field when the list is a non-empty object", async function() { + const playerData = { name: "test", recipe_notifications: await encode({ a: 1 }) }; + const count = warnings.length; + await apply(playerData, await encode(["a"])); + assert.equal(playerData.recipe_notifications, undefined); + assert.equal(warnings.length, count + 1); + }); it("should do nothing without stored notifications", async function() { const playerData = { name: "test" }; await apply(playerData, await encode(["a"])); @@ -71,9 +94,10 @@ describe("player_sync", function() { }); it("should drop the field on invalid input", async function() { const playerData = { name: "test", recipe_notifications: "not base64 zlib" }; + const count = warnings.length; await apply(playerData, await encode(["a"])); assert.equal(playerData.recipe_notifications, undefined); - assert.equal(warnings.length, 1); + assert.equal(warnings.length, count + 1); }); }); From 1b87497e154e2daa07571e5e9aca33c4841616cd Mon Sep 17 00:00:00 2001 From: Bastiaan <17990055+bbassie@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:10:27 +0100 Subject: [PATCH 22/44] Merge pull request #1154 from bbassie/fix/deleted-exports Handle deleted export files --- packages/controller/src/Controller.ts | 34 ++++++++++ packages/controller/src/routes.ts | 24 +++++++ .../web_ui/src/components/ModPackViewPage.tsx | 11 +++- .../web_ui/src/store/export_locale_stores.ts | 21 +++--- .../store/export_prototype_metadata_store.ts | 12 ++-- test/controller/Controller.js | 37 +++++++++++ test/controller/routes.js | 64 +++++++++++++++---- 7 files changed, 178 insertions(+), 25 deletions(-) diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 6bf88250..c5c838f8 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -226,6 +226,8 @@ export default class Controller { await fs.mkdir(modsDirectory, { recursive: true }); const modStore = await lib.ModStore.fromDirectory(modsDirectory); + await this.validateExports(modPacks, "static"); + // Add default mod packs if (modPacks.size === 0) { modPacks.setMany(lib.ModPack.defaultModPacks); @@ -876,6 +878,37 @@ export default class Controller { }); } + /** Remove export manifests that reference files missing from staticDirectory */ + static async validateExports(modPacks: lib.SubscribableDatastore, staticDirectory: string) { + const invalid: lib.ModPack[] = []; + for (const modPack of modPacks.valuesMutable()) { + if (!modPack.exportManifest) { + continue; + } + const missing: string[] = []; + for (const file of Object.values(modPack.exportManifest.assets)) { + const valid = path.basename(file) === file + && await fs.stat(path.join(staticDirectory, file)).then(s => s.isFile(), () => false); + if (!valid) { + missing.push(file); + } + } + if (missing.length) { + Controller.dropExport(modPack, `missing ${missing.join(", ")}`); + invalid.push(modPack); + } + } + if (invalid.length) { + modPacks.setMany(invalid); + } + } + + /** Clear the export manifest of a mutable mod pack, caller must save it */ + static dropExport(modPack: lib.ModPack, reason: string) { + logger.warn(`Removing export data of mod pack ${modPack.name} (id ${modPack.id}): ${reason}`); + modPack.exportManifest = undefined; + } + static migrateRoles(rawJson: unknown[]): Static[] { const serialized = rawJson as Static[]; // migrate: host token permissions were removed with host pairing in 2.1 @@ -984,6 +1017,7 @@ export default class Controller { express.static(path.join(import.meta.dirname, "..", "..", "web", "static"), staticOptions) ); app.use("/static", express.static("static", staticOptions)); // Used for data export files + app.use("/static", routes.missingExportAsset); // Add API routes routes.addRouteHandlers(app); diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 36e1cde9..0499d22a 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -394,6 +394,30 @@ async function uploadExport(req: Request, res: Response) { res.sendStatus(200); } +// Reached when no static file matched, drops exports whose files were removed +export function missingExportAsset(req: Request, res: Response, next: any) { + const file = req.path.slice(1); + if (!["GET", "HEAD"].includes(req.method) || !file) { + next(); + return; + } + const modPacks = res.app.locals.controller.modPacks; + const affected = [...modPacks.valuesMutable()].filter( + modPack => modPack.exportManifest && Object.values(modPack.exportManifest.assets).includes(file) + ); + if (!affected.length) { + next(); + return; + } + for (const modPack of affected) { + Controller.dropExport(modPack, `${file} was removed from disk`); + } + modPacks.setMany(affected); + res.status(404).type("text/plain").send( + `Export data file ${file} is missing, run export-data on an instance to regenerate it` + ); +} + type ProxyStreamEvents = { "error": [ err: any ], "close": [], diff --git a/packages/web_ui/src/components/ModPackViewPage.tsx b/packages/web_ui/src/components/ModPackViewPage.tsx index 39ecfafc..cbb4e592 100644 --- a/packages/web_ui/src/components/ModPackViewPage.tsx +++ b/packages/web_ui/src/components/ModPackViewPage.tsx @@ -1108,7 +1108,9 @@ function useExportedAsset(modPack: lib.ModPack | undefined, asset: "settings"|"l } useEffect(() => { async function load() { + const empty = asset === "locale" ? new Map() : {}; if (!assetFilename) { + setAssetData(empty); return; } @@ -1117,7 +1119,14 @@ function useExportedAsset(modPack: lib.ModPack | undefined, asset: "settings"|"l let data = await response.json(); setAssetData(asset === "locale" ? new Map(data) : data); } else { - logger.error(`Error loading mod pack asset "${asset}" (${response.status}): ${await response.text()}`); + if (response.status === 404) { + logger.warn(`Mod pack asset "${asset}" is missing on the controller`); + } else { + logger.error( + `Error loading mod pack asset "${asset}" (${response.status}): ${await response.text()}` + ); + } + setAssetData(empty); } } diff --git a/packages/web_ui/src/store/export_locale_stores.ts b/packages/web_ui/src/store/export_locale_stores.ts index e15c6aa8..800c130e 100644 --- a/packages/web_ui/src/store/export_locale_stores.ts +++ b/packages/web_ui/src/store/export_locale_stores.ts @@ -24,19 +24,24 @@ class ExportLocaleStore { let response = await fetch(`${staticRoot}${cacheEntry.localePath}`); if (response.ok) { cacheEntry.data = new Map(await response.json() as ExportLocale); - for (const callback of cacheEntry.callbacks) { - callback(); - } } else { - notify( - `Failed to locale for mod pack ${modPackId}, server returned: ` + - `${response.status} ${response.statusText}` - ); + if (response.status === 404) { + logger.warn(`Locale for mod pack ${modPackId} is missing on the controller`); + } else { + notify( + `Failed to locale for mod pack ${modPackId}, server returned: ` + + `${response.status} ${response.statusText}` + ); + } + cacheEntry.data = new Map(); + } + for (const callback of cacheEntry.callbacks) { + callback(); } } subscribe(modPackId: number | undefined, localePath: string | undefined, callback: () => void) { - if (modPackId === undefined || !localePath) { + if (modPackId === undefined) { return () => {}; } diff --git a/packages/web_ui/src/store/export_prototype_metadata_store.ts b/packages/web_ui/src/store/export_prototype_metadata_store.ts index cd919658..1023363e 100644 --- a/packages/web_ui/src/store/export_prototype_metadata_store.ts +++ b/packages/web_ui/src/store/export_prototype_metadata_store.ts @@ -28,10 +28,14 @@ class ExportPrototypeMetadataStore { const response = await fetch(`${staticRoot}${cacheEntry.metadataPath}`); if (!response.ok) { - notify( - `Failed to fetch prototype metadata for mod pack ${modPackId}, server returned: ` + - `${response.status} ${response.statusText}` - ); + if (response.status === 404) { + logger.warn(`Prototype metadata for mod pack ${modPackId} is missing on the controller`); + } else { + notify( + `Failed to fetch prototype metadata for mod pack ${modPackId}, server returned: ` + + `${response.status} ${response.statusText}` + ); + } // eslint-disable-next-line node/no-sync cacheEntry.styleSheet.replaceSync(""); cacheEntry.metadata = new Map(); diff --git a/test/controller/Controller.js b/test/controller/Controller.js index 7f672eef..732feceb 100644 --- a/test/controller/Controller.js +++ b/test/controller/Controller.js @@ -198,6 +198,43 @@ describe("controller/src/Controller", function() { }); }); }); + describe(".validateExports()", function() { + let dir; + before(async function() { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-exports-")); + await fs.writeFile(path.join(dir, "locale.abc.json"), "[]"); + await fs.writeFile(path.join(dir, "metadata.abc.json"), "{}"); + }); + after(async function() { + await fs.rm(dir, { recursive: true, force: true }); + }); + it("should drop export manifests with missing or invalid files", async function() { + const provider = new lib.MemoryDatastoreProvider(); + const modPacks = new lib.SubscribableDatastore(provider, new Map()); + const packs = [ + { id: 1, assets: { locale: "locale.abc.json", metadata: "metadata.abc.json" } }, + { id: 2, assets: { locale: "locale.abc.json", metadata: "metadata.def.json" } }, + { id: 3, assets: { locale: "../locale.abc.json" } }, + { id: 4 }, + ].map(({ id, assets }) => { + const modPack = ModPack.fromJSON({ id, name: `pack ${id}` }); + if (assets) { + modPack.exportManifest = new lib.ExportManifest(assets); + } + return modPack; + }); + modPacks.setMany(packs); + const updates = []; + modPacks.on("update", updated => updates.push(...updated)); + + await Controller.validateExports(modPacks, dir); + assert.deepEqual(modPacks.get(1).exportManifest.assets, packs[0].exportManifest.assets); + assert.equal(modPacks.get(2).exportManifest, undefined); + assert.equal(modPacks.get(3).exportManifest, undefined); + assert.equal(modPacks.get(4).exportManifest, undefined); + assert.deepEqual(updates.map(modPack => modPack.id), [2, 3]); + }); + }); describe(".loadJsonObject()", function() { let dir; before(async function() { diff --git a/test/controller/routes.js b/test/controller/routes.js index bd516690..53d5d083 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -1,11 +1,15 @@ import assert from "node:assert/strict"; import events from "node:events"; +import fs from "node:fs/promises"; import http from "node:http"; - +import os from "node:os"; +import path from "node:path"; import jwt from "jsonwebtoken"; +import express from "express"; -import { SubscribableDatastore, wait, expositionContentType } from "@clusterio/lib"; +import * as lib from "@clusterio/lib"; import { Controller, HostPairingManager, HostRecord } from "@clusterio/controller"; + import * as routes from "@clusterio/controller/dist/node/src/routes.js"; import * as mock from "../mock.js"; @@ -51,7 +55,7 @@ describe("controller/src/routes", function() { stream = await routes.createProxyStream(controller.app); responses = await Promise.all([ (async function() { - await wait(100); + await lib.wait(100); return await fetch(`${endpoint}/${stream.id}`, { method: "PUT", body: "test content", }); @@ -67,7 +71,7 @@ describe("controller/src/routes", function() { method: "PUT", body: "test content", }), (async function() { - await wait(100); + await lib.wait(100); return await fetch(`${endpoint}/${stream.id}`); }()), ]); @@ -114,7 +118,7 @@ describe("controller/src/routes", function() { let response = await fetch(endpoint, { headers }); assert.equal(response.status, 200); const contentType = response.headers.get("content-type"); - for (const part of expositionContentType.split("; ")) { + for (const part of lib.expositionContentType.split("; ")) { assert(contentType.includes(part), `expected ${part} in content type ${contentType}`); } assert(/^# HELP /m.test(await response.text()), "expected exposition text"); @@ -124,7 +128,7 @@ describe("controller/src/routes", function() { let calls = 0; controller.hooks.metrics.attach("test", async () => { calls += 1; - await wait(100); + await lib.wait(100); }); const headers = { "X-Access-Token": testToken }; let responses = await Promise.all([ @@ -141,6 +145,42 @@ describe("controller/src/routes", function() { assert.equal(calls, 2); }); }); + describe("missingExportAsset()", function() { + let dir; + beforeEach(async function() { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-static-")); + await fs.writeFile(path.join(dir, "locale.abc.json"), "[]"); + const provider = new lib.MemoryDatastoreProvider(); + controller.modPacks = new lib.SubscribableDatastore(provider, new Map()); + const modPack = lib.ModPack.fromJSON({ id: 1, name: "test" }); + modPack.exportManifest = new lib.ExportManifest({ + locale: "locale.abc.json", metadata: "metadata.abc.json", + }); + controller.modPacks.set(modPack); + controller.app.use("/static", express.static(dir)); + controller.app.use("/static", routes.missingExportAsset); + }); + afterEach(async function() { + await fs.rm(dir, { recursive: true, force: true }); + }); + it("should serve export files that exist", async function() { + const response = await fetch(`http://localhost:${port}/static/locale.abc.json`); + assert.equal(response.status, 200); + assert(controller.modPacks.get(1).exportManifest); + }); + it("should drop the export and explain when a file is missing", async function() { + const response = await fetch(`http://localhost:${port}/static/metadata.abc.json`); + assert.equal(response.status, 404); + assert.match(await response.text(), /metadata\.abc\.json is missing/); + assert.equal(controller.modPacks.get(1).exportManifest, undefined); + }); + it("should pass through files not in an export", async function() { + const response = await fetch(`http://localhost:${port}/static/other.js`); + assert.equal(response.status, 404); + assert.doesNotMatch(await response.text(), /export-data/); + assert(controller.modPacks.get(1).exportManifest); + }); + }); describe("stripStaticPrefix()", function() { it("should remove a leading static/", function() { assert.equal(routes.stripStaticPrefix("static/main.abc.js"), "main.abc.js"); @@ -218,7 +258,7 @@ describe("controller/src/routes", function() { beforeEach(function() { endpoint = `http://localhost:${port}/api/host-pairing`; // The mock controller lacks what pairing needs, use the real implementations. - controller.hosts = new SubscribableDatastore(); + controller.hosts = new lib.SubscribableDatastore(); controller.hostPairing = new HostPairingManager(controller); controller.generateHostToken = Controller.prototype.generateHostToken; controller.verifyHostToken = Controller.prototype.verifyHostToken; @@ -227,8 +267,8 @@ describe("controller/src/routes", function() { playerToken = controller.users.signUserToken(controller.users.getByName("player")); }); - function post(path, body, headers = {}) { - return fetch(`${endpoint}${path}`, { + function post(urlPath, body, headers = {}) { + return fetch(`${endpoint}${urlPath}`, { method: "POST", headers: { "Content-Type": "application/json", ...headers }, body: typeof body === "string" || body === undefined ? body : JSON.stringify(body), @@ -453,7 +493,7 @@ describe("controller/src/routes", function() { it("should not be accepted in the URL", async function() { const { requestId } = await createRequest(); const id = encodeURIComponent(requestId); - for (const [method, path] of [ + for (const [method, urlPath] of [ ["GET", `/poll?requestId=${id}`], ["GET", `/poll/${id}`], ["POST", `/poll/${id}`], @@ -461,8 +501,8 @@ describe("controller/src/routes", function() { ["GET", `/${id}`], ["POST", `/cancel/${id}`], ]) { - const response = await fetch(`${endpoint}${path}`, { method }); - assert.equal(response.status, 404, `${method} ${path}`); + const response = await fetch(`${endpoint}${urlPath}`, { method }); + assert.equal(response.status, 404, `${method} ${urlPath}`); } await assertResponse(await post("/poll", { requestId }), 202, { status: "pending" }); }); From 024d7b427ac1962711d77b9b38c5d2f257bb6cb0 Mon Sep 17 00:00:00 2001 From: Bastiaan <17990055+bbassie@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:14:31 +0100 Subject: [PATCH 23/44] Rename safeOutputFile to atomicOverwriteFile (#1167) "safe" suggested the data being written survives a crash, it only guarantees the old file is either kept or fully replaced. The new name says what it does. No alias, plugin API breaks are allowed in minor versions and the 2.1 migration guide lists the rename. Co-authored-by: Claude Opus 5.5 Co-authored-by: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> --- docs/devs/plugin-migration-to-2.1.md | 6 ++++++ packages/controller/controller.ts | 2 +- packages/controller/src/Controller.ts | 2 +- packages/controller/src/UserManager.ts | 4 ++-- packages/controller/src/routes.ts | 2 +- packages/create/create.js | 22 +++++++++++----------- packages/host/src/Instance.ts | 10 +++++----- packages/lib/src/ModStore.ts | 4 ++-- packages/lib/src/config/classes.ts | 4 ++-- packages/lib/src/database.ts | 4 ++-- packages/lib/src/datastore.ts | 10 +++++----- packages/lib/src/file_ops.ts | 4 ++-- packages/lib/src/load_plugin_list.ts | 2 +- packages/lib/src/logging_utils.ts | 2 +- packages/lib/src/shared_commands.ts | 4 ++-- plugins/player_sync/controller.ts | 2 +- plugins/research_sync/controller.ts | 4 ++-- plugins/subspace_storage/controller.ts | 2 +- test/lib/file_ops.js | 14 +++++++------- 19 files changed, 55 insertions(+), 49 deletions(-) diff --git a/docs/devs/plugin-migration-to-2.1.md b/docs/devs/plugin-migration-to-2.1.md index 3a6a440f..56b96d6a 100644 --- a/docs/devs/plugin-migration-to-2.1.md +++ b/docs/devs/plugin-migration-to-2.1.md @@ -61,6 +61,12 @@ The `container` property of `WebPluginContext` is now the plugin's ES module nam `lib.PluginWebpackEnvInfo` is renamed to `lib.PluginWebEnvInfo`. +## Renamed lib functions + +`lib.safeOutputFile` is renamed to `lib.atomicOverwriteFile`. +It takes the same arguments. + + ## Host pairing Host tokens are no longer generated by users. diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index e2cdb06a..29334ed4 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -209,7 +209,7 @@ async function handleBootstrapCommand( console.log(content); } else { logger.info(`Writing ${args.output}`); - await lib.safeOutputFile(args.output, content, { mode: 0o600 }); + await lib.atomicOverwriteFile(args.output, content, { mode: 0o600 }); } } } diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index c5c838f8..e51d78fa 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -138,7 +138,7 @@ export default class Controller { const cachePath = path.join(this.config.get("controller.database_directory"), "latest-releases.json"); try { const releases = await lib.fetchLatestReleases(); - await lib.safeOutputFile(cachePath, JSON.stringify(releases, null, "\t")); + await lib.atomicOverwriteFile(cachePath, JSON.stringify(releases, null, "\t")); return releases; } catch (err: any) { logger.warn(`Failed to retrieve Factorio releases ${err.message}`); diff --git a/packages/controller/src/UserManager.ts b/packages/controller/src/UserManager.ts index 9ef56d7c..4e19bc28 100644 --- a/packages/controller/src/UserManager.ts +++ b/packages/controller/src/UserManager.ts @@ -40,8 +40,8 @@ export default class UserManager { } // We intentionally write the roles first because if it fails then the old user file will still contain them - await lib.safeOutputFile(rolesFilePath, JSON.stringify(combinedJson.roles)); - await lib.safeOutputFile(usersFilePath, JSON.stringify(combinedJson.users)); + await lib.atomicOverwriteFile(rolesFilePath, JSON.stringify(combinedJson.roles)); + await lib.atomicOverwriteFile(usersFilePath, JSON.stringify(combinedJson.users)); } catch (err: any) { if (err.code !== "ENOENT") { throw err; diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 0499d22a..785283be 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -384,7 +384,7 @@ async function uploadExport(req: Request, res: Response) { let { name, ext } = path.posix.parse(filePath); let hash = await lib.hashStream(await zip.openStream(file)); assets[name] = `${name}.${hash}${ext}`; - await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), content); + await lib.atomicOverwriteFile(path.join("static", `${name}.${hash}${ext}`), content); } modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString()); diff --git a/packages/create/create.js b/packages/create/create.js index 48b81ac5..d5d91505 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -104,7 +104,7 @@ async function pathExists(filePath) { return true; } -async function safeOutputFile(file, data, options={}) { +async function atomicOverwriteFile(file, data, options={}) { let { dir, name, ext } = path.parse(file); let temporary = path.join(dir, `${name}.tmp${ext}`); if (dir) { @@ -257,7 +257,7 @@ async function migrateRename(args) { async function migrateConfig(source, destination) { if (await pathExists(source) && !await pathExists(destination)) { - await safeOutputFile(destination, JSON.stringify( + await atomicOverwriteFile(destination, JSON.stringify( renameConfig(JSON.parse(await fs.readFile(source))), null, "\t" ), { mode: 0o600 }); logger.info(`Migrated ${source} to ${destination}`); @@ -269,7 +269,7 @@ async function migrateRename(args) { let instancesFile = path.join("database", "instances.json"); if (await pathExists(instancesFile)) { - await safeOutputFile(instancesFile, JSON.stringify( + await atomicOverwriteFile(instancesFile, JSON.stringify( JSON.parse(await fs.readFile(instancesFile)).map(renameConfig), null, "\t" )); logger.info(`Migrated ${instancesFile}`); @@ -281,7 +281,7 @@ async function migrateRename(args) { for (let role of users["roles"]) { role["permissions"] = role["permissions"].map(rename); } - await safeOutputFile(usersFile, JSON.stringify(users, null, "\t")); + await atomicOverwriteFile(usersFile, JSON.stringify(users, null, "\t")); logger.info(`Migrated ${usersFile}`); } @@ -309,7 +309,7 @@ async function migrateRename(args) { lines.push(inputLine); } } - safeOutputFile(outputFile, Buffer.concat(lines)); + atomicOverwriteFile(outputFile, Buffer.concat(lines)); } async function migrateLogsDir(inputLogs, outputLogs) { @@ -434,7 +434,7 @@ async function groupIdToName(gid) { async function writeScripts(mode, selfSignedCert) { if (["standalone", "controller"].includes(mode)) { if (process.platform === "win32") { - await safeOutputFile( + await atomicOverwriteFile( "run-controller.cmd", `\ @echo off @@ -447,7 +447,7 @@ goto restart ` ); } else { - await safeOutputFile( + await atomicOverwriteFile( "run-controller.sh", `\ #!/bin/bash @@ -459,7 +459,7 @@ done `, { mode: 0o755 }, ); - await safeOutputFile( + await atomicOverwriteFile( "systemd/clusteriocontroller.service", `\ [Unit] @@ -486,7 +486,7 @@ WantedBy=multi-user.target if (["standalone", "host"].includes(mode)) { if (process.platform === "win32") { let extraCaCertsLine = selfSignedCert ? `set "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : ""; - await safeOutputFile( + await atomicOverwriteFile( "run-host.cmd", `\ @echo off @@ -500,7 +500,7 @@ goto restart ); } else { let extraCaCertsLine = selfSignedCert ? `export "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : ""; - await safeOutputFile( + await atomicOverwriteFile( "run-host.sh", `\ #!/bin/bash @@ -513,7 +513,7 @@ done { mode: 0o755 }, ); extraCaCertsLine = selfSignedCert ? `Environment=NODE_EXTRA_CA_CERTS=${selfSignedCert}\n` : ""; - await safeOutputFile( + await atomicOverwriteFile( "systemd/clusteriohost.service", `\ [Unit] diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 6cee6a7b..a6a6c02c 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -693,7 +693,7 @@ end`.replace(/\r?\n/g, " "); players: [...this.playerStats], player_autosave_slot: this._playerAutosaveSlot, }, null, "\t"); - await lib.safeOutputFile(this.path("instance-stats.json"), content); + await lib.atomicOverwriteFile(this.path("instance-stats.json"), content); } /** @@ -870,7 +870,7 @@ end`.replace(/\r?\n/g, " "); "_comment_warning": warning, ...await this.resolveServerSettings(this.config.get("factorio.settings"), includeCredentials), }; - await lib.safeOutputFile( + await lib.atomicOverwriteFile( this.server.writePath("server-settings.json"), JSON.stringify(serverSettings, null, "\t") ); @@ -996,7 +996,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_adminlist") !== "disabled") { this.logger.verbose("Writing server-adminlist.json"); - await lib.safeOutputFile( + await lib.atomicOverwriteFile( this.server.writePath("server-adminlist.json"), JSON.stringify([...this._host.adminlist], null, "\t") ); @@ -1004,7 +1004,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_banlist") !== "disabled") { this.logger.verbose("Writing server-banlist.json"); - await lib.safeOutputFile( + await lib.atomicOverwriteFile( this.server.writePath("server-banlist.json"), JSON.stringify([...this._host.banlist].map( ([username, reason]) => ({ username, reason }) @@ -1014,7 +1014,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_whitelist") !== "disabled") { this.logger.verbose("Writing server-whitelist.json"); - await lib.safeOutputFile( + await lib.atomicOverwriteFile( this.server.writePath("server-whitelist.json"), JSON.stringify([...this._host.whitelist], null, "\t") ); diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 999157f3..3efb7a5b 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -3,7 +3,7 @@ import path from "path"; import events from "events"; import { Static } from "@sinclair/typebox"; import { logger } from "./logging.js"; -import { downloadFile, safeOutputFile } from "./file_ops.js"; +import { downloadFile, atomicOverwriteFile } from "./file_ops.js"; import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod.js"; import { ModInfo, ModPack, MajorMinorVersion, ModVersionEquality, SourceVersion } from "./data/index.js"; @@ -148,7 +148,7 @@ export default class ModStore extends events.EventEmitter { if (files.size) { logger.info(`Loaded info for ${files.size} mods`); const content: ModInfoCache = [...files.values()].map(mod => mod.toJSON()); - await safeOutputFile(cacheFilename, JSON.stringify(content)); + await atomicOverwriteFile(cacheFilename, JSON.stringify(content)); } return new this(modsDirectory, files); diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index 19a7dbe0..84367ad6 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -7,7 +7,7 @@ import isDeepStrictEqual from "#is_deep_strict_equal"; import { basicType } from "../helpers.js"; import * as libSchema from "../schema.js"; import { StringEnum } from "../data/composites.js"; -import { safeOutputFile } from "../file_ops.js"; +import { atomicOverwriteFile } from "../file_ops.js"; import * as validators from "./validators.js"; import { InvalidAccessError, InvalidFieldError, InvalidValueError } from "../errors.js"; import type { SyncHook } from "../SyncHook.js"; @@ -475,7 +475,7 @@ export class Config< if (!this.filepath) { throw new Error("Cannot save config which has no filepath"); } - await safeOutputFile(this.filepath, JSON.stringify(this, null, "\t"), { mode: 0o600 }); + await atomicOverwriteFile(this.filepath, JSON.stringify(this, null, "\t"), { mode: 0o600 }); this._dirty = false; } diff --git a/packages/lib/src/database.ts b/packages/lib/src/database.ts index 1e94d298..2b833e5f 100644 --- a/packages/lib/src/database.ts +++ b/packages/lib/src/database.ts @@ -79,7 +79,7 @@ export async function loadJsonAsMap(filePath: string): Promise) { let obj = mapToObject(map); - await libFileOps.safeOutputFile(filePath, JSON.stringify(obj, null, "\t")); + await libFileOps.atomicOverwriteFile(filePath, JSON.stringify(obj, null, "\t")); } /** @@ -139,7 +139,7 @@ export async function loadJsonArrayAsMap(filePath: string): Promise) { - await libFileOps.safeOutputFile(filePath, JSON.stringify([...map.values()], null, "\t")); + await libFileOps.atomicOverwriteFile(filePath, JSON.stringify([...map.values()], null, "\t")); } diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 1db3518e..2b39d94b 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -2,7 +2,7 @@ import path from "path"; import fs from "node:fs/promises"; import { EventEmitter } from "events"; -import { safeOutputFile } from "./file_ops.js"; +import { atomicOverwriteFile } from "./file_ops.js"; import { SubscribableValue } from "./subscriptions.js"; import { logger } from "./logging.js"; import { ControllerConfig } from "./config/index.js"; @@ -56,7 +56,7 @@ export class JsonDatastoreProvider< // Save the data to the json file async save(data: Map) { logger.verbose(`Saving ${this.filePath}`); - await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t")); + await atomicOverwriteFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t")); } // Load the data from the json file @@ -91,7 +91,7 @@ export class JsonDatastoreProvider< logger.warn( `Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}` ); - await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t")); + await atomicOverwriteFile(backupPath, JSON.stringify(rawJson, null, "\t")); } return result; @@ -115,7 +115,7 @@ export class JsonIdDatastoreProvider< // Save the data to the json file async save(data: Map) { logger.verbose(`Saving ${this.filePath}`); - await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t")); + await atomicOverwriteFile(this.filePath, JSON.stringify([...data.values()], null, "\t")); } // Load the data from the json file @@ -150,7 +150,7 @@ export class JsonIdDatastoreProvider< logger.warn( `Duplicates detected in ${this.filePath}, a backup was written to: ${backupPath}` ); - await safeOutputFile(backupPath, JSON.stringify(rawJson, null, "\t")); + await atomicOverwriteFile(backupPath, JSON.stringify(rawJson, null, "\t")); } return result; diff --git a/packages/lib/src/file_ops.ts b/packages/lib/src/file_ops.ts index de8b30c4..16701808 100644 --- a/packages/lib/src/file_ops.ts +++ b/packages/lib/src/file_ops.ts @@ -89,7 +89,7 @@ export async function getTempFile(prefix = "tmp.", suffix = "", tmpdir = "./") { } /** - * Safely overwrite file data + * Atomically overwrite file data * * Writes the passed data to a temporary file and rename it over the target * file. If the operation fails because the parent directory does not exist, it @@ -105,7 +105,7 @@ export async function getTempFile(prefix = "tmp.", suffix = "", tmpdir = "./") { * @param data - Content to write. * @param options - see fs.writeFile, `flag` and `flush` may not be used. */ -export async function safeOutputFile( +export async function atomicOverwriteFile( file: string, data: string | Buffer, options?: Omit< diff --git a/packages/lib/src/load_plugin_list.ts b/packages/lib/src/load_plugin_list.ts index d6838206..65f1e2fd 100644 --- a/packages/lib/src/load_plugin_list.ts +++ b/packages/lib/src/load_plugin_list.ts @@ -328,7 +328,7 @@ export async function loadPluginList( hasChanges = await findNpmPlugins(pluginList, skippedNpmPlugins) || hasChanges; if (hasChanges) { - await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t")); + await libFileOps.atomicOverwriteFile(pluginListPath, JSON.stringify([...pluginList], null, "\t")); } return pluginList; diff --git a/packages/lib/src/logging_utils.ts b/packages/lib/src/logging_utils.ts index 2a02f57d..ab235182 100644 --- a/packages/lib/src/logging_utils.ts +++ b/packages/lib/src/logging_utils.ts @@ -312,7 +312,7 @@ export class LogIndex { * was loaded/created from. */ async save() { - await libFileOps.safeOutputFile( + await libFileOps.atomicOverwriteFile( path.join(this.logDirectory, "index.json"), JSON.stringify(this.serialize(), null, "\t") ); diff --git a/packages/lib/src/shared_commands.ts b/packages/lib/src/shared_commands.ts index 7767ce8e..155be8f3 100644 --- a/packages/lib/src/shared_commands.ts +++ b/packages/lib/src/shared_commands.ts @@ -199,7 +199,7 @@ export async function handlePluginCommand( } pluginList.set(pluginInfo.name, pluginPath); - await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t")); + await libFileOps.atomicOverwriteFile(pluginListPath, JSON.stringify([...pluginList], null, "\t")); print(`Added ${pluginInfo.name}`); } else if (command === "remove") { @@ -209,7 +209,7 @@ export async function handlePluginCommand( return; } - await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t")); + await libFileOps.atomicOverwriteFile(pluginListPath, JSON.stringify([...pluginList], null, "\t")); print(`Removed ${args.name}`); } else if (command === "list") { diff --git a/plugins/player_sync/controller.ts b/plugins/player_sync/controller.ts index 0df18cb1..a94800cc 100644 --- a/plugins/player_sync/controller.ts +++ b/plugins/player_sync/controller.ts @@ -63,7 +63,7 @@ async function saveDatabase( let file = path.resolve(controllerConfig.get("controller.database_directory"), "player_sync.json"); logger.verbose(`writing ${file}`); let content = JSON.stringify([...playerDatastore, ...tombstones]); - await lib.safeOutputFile(file, content); + await lib.atomicOverwriteFile(file, content); } export default async function(context: ControllerPluginContext) { diff --git a/plugins/research_sync/controller.ts b/plugins/research_sync/controller.ts index 3f29a3a3..a4e02a47 100644 --- a/plugins/research_sync/controller.ts +++ b/plugins/research_sync/controller.ts @@ -49,7 +49,7 @@ async function saveTechnologies( ) { let filePath = path.join(controllerConfig.get("controller.database_directory"), "technologies.json"); logger.verbose(`writing ${filePath}`); - await lib.safeOutputFile(filePath, JSON.stringify([...technologies.entries()], null, "\t")); + await lib.atomicOverwriteFile(filePath, JSON.stringify([...technologies.entries()], null, "\t")); } const MAX_QUEUE = 7; @@ -71,7 +71,7 @@ async function loadQueue(controllerConfig: lib.ControllerConfig, logger: lib.Log async function saveQueue(controllerConfig: lib.ControllerConfig, queue: string[], logger: lib.Logger) { let filePath = path.join(controllerConfig.get("controller.database_directory"), "research_queue.json"); logger.verbose(`writing ${filePath}`); - await lib.safeOutputFile(filePath, JSON.stringify(queue, null, "\t")); + await lib.atomicOverwriteFile(filePath, JSON.stringify(queue, null, "\t")); } export default async function(context: ControllerPluginContext) { diff --git a/plugins/subspace_storage/controller.ts b/plugins/subspace_storage/controller.ts index cad2ec52..8f28b801 100644 --- a/plugins/subspace_storage/controller.ts +++ b/plugins/subspace_storage/controller.ts @@ -64,7 +64,7 @@ async function saveDatabase( let file = path.resolve(controllerConfig.get("controller.database_directory"), "items.json"); logger.verbose(`writing ${file}`); let content = JSON.stringify(items.serialize()); - await lib.safeOutputFile(file, content); + await lib.atomicOverwriteFile(file, content); } else if (items) { logger.error(`Item database too large, not saving (${items.size})`); } diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index 3e8dd21f..a44e069f 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -51,7 +51,7 @@ describe("lib/file_ops", function() { }); }); - describe("safeOutputFile()", function() { + describe("atomicOverwriteFile()", function() { async function assertNoTemporary(target) { let { dir, name } = path.parse(target); let entries = (await fs.readdir(dir)).filter( @@ -61,14 +61,14 @@ describe("lib/file_ops", function() { } it("should write new target file", async function() { let target = path.join(baseDir, "safe", "simple.txt"); - await lib.safeOutputFile(target, "a text file", "utf8"); + await lib.atomicOverwriteFile(target, "a text file", "utf8"); await assertNoTemporary(target); assert.equal(await fs.readFile(target, "utf8"), "a text file"); }); it("should overwrite existing target file", async function() { let target = path.join(baseDir, "safe", "exists.txt"); await fs.writeFile(target, "previous", "utf8"); - await lib.safeOutputFile(target, "current", "utf8"); + await lib.atomicOverwriteFile(target, "current", "utf8"); await assertNoTemporary(target); assert.equal(await fs.readFile(target, "utf8"), "current"); }); @@ -77,7 +77,7 @@ describe("lib/file_ops", function() { this.skip(); } let target = path.join(baseDir, "safe", "mode.txt"); - await lib.safeOutputFile(target, "private", { mode: 0o600 }); + await lib.atomicOverwriteFile(target, "private", { mode: 0o600 }); assert.equal((await fs.stat(target)).mode & 0o777, 0o600); assert.equal(await fs.readFile(target, "utf8"), "private"); }); @@ -85,7 +85,7 @@ describe("lib/file_ops", function() { let target = path.join(baseDir, "safe", "concurrent.txt"); let writes = []; for (let i = 0; i < 20; i++) { - writes.push(lib.safeOutputFile(target, `write ${i}`, "utf8")); + writes.push(lib.atomicOverwriteFile(target, `write ${i}`, "utf8")); } // Ignore errors from concurrently renaming onto the same file. await Promise.allSettled(writes); @@ -95,13 +95,13 @@ describe("lib/file_ops", function() { it("should remove the temporary file when the rename fails", async function() { let target = path.join(baseDir, "safe", "rename-fails"); await fs.mkdir(target, { recursive: true }); - await assert.rejects(lib.safeOutputFile(target, "data", "utf8")); + await assert.rejects(lib.atomicOverwriteFile(target, "data", "utf8")); await assertNoTemporary(target); }); it("should handle creating file in current working directory", async function() { let target = "temporary-file-made-to-test-cwd.txt"; try { - await lib.safeOutputFile(target, "a text file", "utf8"); + await lib.atomicOverwriteFile(target, "a text file", "utf8"); } finally { try { await fs.unlink(target); From 01cf789f5d6d96ea417265012fc2b32a33573c7a Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:09:06 +0000 Subject: [PATCH 24/44] Add base instances for instance config inheritance Instances can be marked as a base with instance.is_base and other instances inherit config values from it through instance.base_id. The controller copies base values into inheriting configs and keeps track of inherited fields in the hidden instance.inherited_fields, so hosts receive resolved configs and need no changes. Setting a field stops inheriting it, `instance config inherit` undoes that. Bases cannot be assigned to a host, are hidden from `instance list` unless --bases is given, and cannot be deleted or unmarked while other instances inherit from them. Chains of bases are allowed and cycles are rejected. Part of #200 Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 22 ++ docs/managing-a-cluster.md | 17 +- packages/controller/src/ControlConnection.ts | 38 +++- packages/controller/src/InstanceManager.ts | 207 ++++++++++++++++++- packages/controller/src/InstanceRecord.ts | 1 + packages/ctl/src/commands_instance.ts | 53 ++++- packages/lib/src/config/definitions.ts | 44 ++++ packages/lib/src/data/InstanceDetails.ts | 10 +- packages/lib/src/data/messages_instance.ts | 22 ++ packages/lib/src/link/messages.ts | 1 + test/controller/ControlConnection.test.js | 99 +++++++++ test/controller/Controller.js | 4 + test/controller/InstanceManager.test.js | 175 ++++++++++++++++ test/integration/clusterio.js | 57 +++++ test/integration/routing.js | 3 +- test/lib/config/definitions.js | 42 ++++ 16 files changed, 769 insertions(+), 26 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 89650ca9..54e1f015 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -521,6 +521,28 @@ This allows you to prevent certain instances from being started when using the b Defaults to false. +### instance.is_base + +If enabled this instance is a base other instances can inherit config values from. +Base instances cannot be assigned to a host and therefore never start. +Hidden from `instance list` in clusterioctl unless `--bases` is passed. +An instance cannot stop being a base while other instances inherit from it. + +Defaults to false. + + +### instance.base_id + +ID of the base instance to inherit config values from. +Inherited fields take their value from the base and follow it when it changes, setting a field on this instance stops inheriting it. +When set on an existing instance only fields with the same value as the base become inherited. +`instance.name`, `instance.is_base`, `instance.base_id` and hidden fields are never inherited. +A base may itself inherit from another base, but not from itself through a chain of bases. +The fields currently inherited are stored in the hidden `instance.inherited_fields` field. + +Defaults to null meaning no base. + + ### factorio.version Version of Factorio to use for this instance. diff --git a/docs/managing-a-cluster.md b/docs/managing-a-cluster.md index 51982131..14073d9d 100644 --- a/docs/managing-a-cluster.md +++ b/docs/managing-a-cluster.md @@ -79,15 +79,16 @@ Marks the host as requiring a restart, the same way a plugin update does. Use it ### List Instances - ctl> instance list + ctl> instance list [--bases] List instances. +Base instances are only listed if `--bases` is passed. ### Create instance - ctl> instance create [--id ] + ctl> instance create [--id ] [--from ] [--base ] [--is-base] -Create a new instance. `name` is the instance name. `--id` is the optional id. +Create a new instance. `name` is the instance name. `--id` is the optional id. `--from` copies the config of an existing instance. `--base` makes the new instance inherit all config values from the given base instance. `--is-base` creates the instance as a base, see `instance.is_base` in the [configuration docs](configuration.md). ### Kill instance @@ -120,7 +121,7 @@ Most notably Clusterio records wall clock time while the save time is recorded i ctl> instance config list -List config values for an instance. `instance` is the instance whose config values to list. +List config values for an instance. `instance` is the instance whose config values to list. Values inherited from a base instance are marked with `(inherited)`. #### Set config value @@ -132,11 +133,17 @@ Set field in instance config. `instance` is instance to set. `field` is field to To be written. +#### Inherit config value + + ctl> instance config inherit + +Make a field in the instance config take its value from the base instance again, undoing a previous `set`. `instance` is the instance to update, `field` is the field to inherit. + #### Edit instance config ctl> instance config edit [editor] -Edit instance config in a text editor. Once file is saved and editor has been exited, config values will be read back in. `instance` is the instance to edit, `editor` is the editor to use. If `editor` is not given, clusterioctl will read from the `EDITOR` or `VISUAL` enviroment variables. +Edit instance config in a text editor. Once file is saved and editor has been exited, changed config values will be read back in. `instance` is the instance to edit, `editor` is the editor to use. If `editor` is not given, clusterioctl will read from the `EDITOR` or `VISUAL` enviroment variables. ### Assign instance to host diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 43af122d..72b8ae88 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -8,6 +8,7 @@ import { logFilter, logger } from "@clusterio/lib"; import BaseConnection from "./BaseConnection.js"; import User from "./User.js"; import UserRecord from "./UserRecord.js"; +import type InstanceRecord from "./InstanceRecord.js"; import * as routes from "./routes.js"; import Controller from "./Controller.js"; @@ -94,6 +95,7 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.InstanceConfigSetRequest, this.handleInstanceConfigSetRequest.bind(this)); this.handle(lib.InstanceConfigSetFieldRequest, this.handleInstanceConfigSetFieldRequest.bind(this)); this.handle(lib.InstanceConfigSetPropRequest, this.handleInstanceConfigSetPropRequest.bind(this)); + this.handle(lib.InstanceConfigInheritFieldRequest, this.handleInstanceConfigInheritFieldRequest.bind(this)); this.handle(lib.InstanceAssignRequest, this.handleInstanceAssignRequest.bind(this)); this.handle(lib.InstanceSaveDetailsListRequest, this.handleInstanceSaveDetailsListRequest.bind(this)); this.handle(lib.InstanceRenameSaveRequest, controller.sendRequestToHostByInstanceId.bind(controller)); @@ -315,6 +317,12 @@ export default class ControlConnection extends BaseConnection { instanceConfig.set("instance.assigned_host", null); // New instances are unassigned } instanceConfig.update(request.config, false, "control"); + if (Object.hasOwn(request.config, "instance.base_id")) { + this._controller.instances.checkInheritance( + instanceConfig, instanceConfig.get("instance.is_base"), instanceConfig.get("instance.base_id") + ); + this._controller.instances.applyBase(instanceConfig, Object.keys(request.config)); + } await this._controller.instances.createInstance(instanceConfig); } @@ -355,17 +363,36 @@ export default class ControlConnection extends BaseConnection { } } + this.checkStagedInheritance(instance); instance.config.commitStaging(); + this._controller.instances.overrideFields(instance, Object.keys(request.fields)); await this._controller.instanceConfigUpdated(instance); } finally { instance.config.revertStaging(); } } + private checkStagedInheritance(instance: InstanceRecord) { + const staged = instance.config.staging; + this._controller.instances.checkInheritance( + instance.config, staged.get("instance.is_base"), staged.get("instance.base_id") + ); + } + async handleInstanceConfigSetFieldRequest(request: lib.InstanceConfigSetFieldRequest) { const instance = this._controller.instances.getForRequest(request.instanceId); - ControlConnection.validateInstanceConfigSetField(request.field); - instance.config.set(request.field as keyof lib.InstanceConfigFields, request.value, "control"); + const field = request.field as keyof lib.InstanceConfigFields; + ControlConnection.validateInstanceConfigSetField(field); + if (["instance.is_base", "instance.base_id"].includes(field)) { + try { + instance.config.stage(field, request.value, "control"); + this.checkStagedInheritance(instance); + } finally { + instance.config.revertStaging(); + } + } + instance.config.set(field, request.value, "control"); + this._controller.instances.overrideFields(instance, [field]); await this._controller.instanceConfigUpdated(instance); } @@ -374,9 +401,16 @@ export default class ControlConnection extends BaseConnection { let { field, prop, value } = request; ControlConnection.validateInstanceConfigSetField(field); instance.config.setProp(field as keyof lib.InstanceConfigFields, prop, value, "control"); + this._controller.instances.overrideFields(instance, [field]); await this._controller.instanceConfigUpdated(instance); } + async handleInstanceConfigInheritFieldRequest(request: lib.InstanceConfigInheritFieldRequest) { + ControlConnection.validateInstanceConfigSetField(request.field); + this._controller.instances.inheritField(request.instanceId, request.field); + await this._controller.instanceConfigUpdated(this._controller.instances.getForRequest(request.instanceId)); + } + async handleInstanceAssignRequest(request: lib.InstanceAssignRequest) { await this._controller.instances.assignInstance(request.instanceId, request.hostId ?? null); } diff --git a/packages/controller/src/InstanceManager.ts b/packages/controller/src/InstanceManager.ts index 4323d59a..38802a96 100644 --- a/packages/controller/src/InstanceManager.ts +++ b/packages/controller/src/InstanceManager.ts @@ -1,4 +1,6 @@ +import { isDeepStrictEqual } from "node:util"; import * as lib from "@clusterio/lib"; +import { logger } from "@clusterio/lib"; import type Controller from "./Controller.js"; import type HostConnection from "./HostConnection.js"; @@ -24,6 +26,9 @@ export default class InstanceManager { non_blocking_saving: false, } as const; + /** Non-zero while base values are being copied into inheriting instances */ + private _inheriting = 0; + constructor( public readonly records: lib.SubscribableDatastore, private readonly _controller: Controller, @@ -106,11 +111,190 @@ export default class InstanceManager { */ addInstanceHooks(instance: InstanceRecord): void { instance.config.on("fieldChanged", (field: string, curr: unknown, prev: unknown) => { + if (field === "instance.base_id") { + this._rebase(instance); + } else if (!this._inheriting && instance.config.isInherited(field)) { + this._setInherited(instance, [field], false); + } this.records.set(instance); this._notifyConfigFieldChanged(instance, field, curr, prev); + this._propagate(instance, field, curr); }); } + /* ------------------------------------------------------------------------- */ + /* Inheritance */ + /* ------------------------------------------------------------------------- */ + + /** + * Get the instances that inherit directly from the given base. + * + * @param baseId - ID of the base instance. + * @returns Instances with `instance.base_id` set to baseId. + */ + inheritorsOf(baseId: number): InstanceRecord[] { + return [...this.records.values()].filter( + instance => instance.config.get("instance.base_id") === baseId + ); + } + + /** + * Check that new values for `instance.is_base` and `instance.base_id` are allowed. + * + * @param config - Current config of the instance. + * @param isBase - New value for `instance.is_base`. + * @param baseId - New value for `instance.base_id`. + * @throws {module:lib.ValidationError} if the base is not a base or would create a cycle. + * @throws {module:lib.ConflictError} if the instance is assigned or still has inheritors. + */ + checkInheritance(config: lib.InstanceConfig, isBase: boolean, baseId: number | null): void { + const instanceId = config.get("instance.id"); + if (isBase && config.get("instance.assigned_host") !== null) { + throw new lib.ConflictError("Instance must be unassigned before it can be made a base"); + } + if (!isBase && config.get("instance.is_base")) { + const count = this.inheritorsOf(instanceId).length; + if (count) { + throw new lib.ConflictError(`Instance is the base of ${count} other instance(s)`); + } + } + if (baseId === null) { + return; + } + const base = this.records.get(baseId); + if (!base) { + throw new lib.NotFoundError(`Instance with ID ${baseId} does not exist`); + } + if (!base.config.get("instance.is_base")) { + throw new lib.ValidationError(`Instance ${base.config.get("instance.name")} is not a base`); + } + const visited = new Set(); + for (let id: number | null = baseId; id !== null && !visited.has(id);) { + if (id === instanceId) { + throw new lib.ValidationError("Base instances cannot inherit from themselves"); + } + visited.add(id); + id = this.records.get(id)?.config.get("instance.base_id") ?? null; + } + } + + /** + * Copy all inheritable values from the base set in `instance.base_id` + * into a config that is not yet part of the cluster and mark them inherited. + * + * @param config - Config of the instance to be created. + * @param keep - Fields to keep the current value of. + */ + applyBase(config: lib.InstanceConfig, keep: string[] = []): void { + const base = this.records.get(config.get("instance.base_id")!); + if (!base) { + return; + } + const values: Record = {}; + const inherited: Record = {}; + for (const name of Object.keys(lib.InstanceConfig.fieldDefinitions)) { + if (lib.InstanceConfig.isInheritable(name) && !keep.includes(name)) { + values[name] = structuredClone(base.config.get(name as keyof lib.InstanceConfigFields)); + inherited[name] = true; + } + } + config.update(values, false); + config.set("instance.inherited_fields", inherited); + } + + /** + * Make a field take its value from the base instance again. + * + * @param instanceId - ID of the instance to update. + * @param field - Name of the field to inherit. + * @throws {module:lib.ValidationError} if the instance has no base or the field cannot be inherited. + */ + inheritField(instanceId: number, field: string): void { + const instance = this.getForRequest(instanceId); + const baseId = instance.config.get("instance.base_id"); + const base = baseId !== null ? this.records.get(baseId) : undefined; + if (!base) { + throw new lib.ValidationError("Instance does not have a base"); + } + if (!lib.InstanceConfig.isInheritable(field)) { + throw new lib.ValidationError(`Field ${field} cannot be inherited`); + } + this._setInherited(instance, [field], true); + this._inheriting += 1; + try { + instance.config.update({ + [field]: structuredClone(base.config.get(field as keyof lib.InstanceConfigFields)), + }, true); + } finally { + this._inheriting -= 1; + } + } + + /** + * Stop inheriting the given fields, used when a value is explicitly set. + * + * @param instance - Instance the fields were set on. + * @param fields - Names of fields set. + */ + overrideFields(instance: InstanceRecord, fields: string[]): void { + const inherited = fields.filter(field => instance.config.isInherited(field)); + if (inherited.length) { + this._setInherited(instance, inherited, false); + } + } + + private _setInherited(instance: InstanceRecord, fields: string[], inherited: boolean): void { + const value = { ...instance.config.get("instance.inherited_fields") }; + for (const field of fields) { + if (inherited) { + value[field] = true; + } else { + delete value[field]; + } + } + instance.config.set("instance.inherited_fields", value); + } + + /** Recompute inherited fields after base_id changed, values equal to the base are inherited */ + private _rebase(instance: InstanceRecord): void { + const baseId = instance.config.get("instance.base_id"); + const base = baseId !== null ? this.records.get(baseId) : undefined; + const inherited: Record = {}; + if (base) { + for (const name of Object.keys(lib.InstanceConfig.fieldDefinitions)) { + const field = name as keyof lib.InstanceConfigFields; + if ( + lib.InstanceConfig.isInheritable(name) + && isDeepStrictEqual(instance.config.get(field), base.config.get(field)) + ) { + inherited[name] = true; + } + } + } + instance.config.set("instance.inherited_fields", inherited); + } + + /** Copy a changed value from a base to the instances inheriting it */ + private _propagate(base: InstanceRecord, field: string, value: unknown): void { + if (!lib.InstanceConfig.isInheritable(field)) { + return; + } + for (const instance of this.inheritorsOf(base.id)) { + if (!instance.config.isInherited(field)) { + continue; + } + this._inheriting += 1; + try { + instance.config.update({ [field]: structuredClone(value) as any }, true); + } finally { + this._inheriting -= 1; + } + this._controller.instanceConfigUpdated(instance).catch(err => { + logger.error(`Failed to send config of instance ${instance.id} to its host:\n${err.stack}`); + }); + } + } + /* ------------------------------------------------------------------------- */ /* Lifecycle */ /* ------------------------------------------------------------------------- */ @@ -136,13 +320,15 @@ export default class InstanceManager { throw new lib.ConflictError(`Instance with ID ${instanceId} already exists`); } - const controllerName = this._controller.config.get("controller.name"); - instanceConfig.set("factorio.settings", { - name: `${controllerName} - ${instanceConfig.get("instance.name")}`, - description: `Clusterio instance for ${controllerName}`, - ...InstanceManager.DefaultFactorioSettings, - ...instanceConfig.get("factorio.settings"), - }); + if (!instanceConfig.isInherited("factorio.settings")) { + const controllerName = this._controller.config.get("controller.name"); + instanceConfig.set("factorio.settings", { + name: `${controllerName} - ${instanceConfig.get("instance.name")}`, + description: `Clusterio instance for ${controllerName}`, + ...InstanceManager.DefaultFactorioSettings, + ...instanceConfig.get("factorio.settings"), + }); + } const instance = new InstanceRecord(instanceConfig, "unassigned"); this.addInstanceHooks(instance); @@ -186,6 +372,9 @@ export default class InstanceManager { if (currentAssignedHost === hostId) { return; } + if (hostId !== null && instance.config.get("instance.is_base")) { + throw new lib.ValidationError("Base instances cannot be assigned to a host"); + } let newHostConnection: HostConnection | undefined; if (hostId !== null) { @@ -252,6 +441,10 @@ export default class InstanceManager { async deleteInstance(instanceId: number): Promise { const instance = this.getForRequest(instanceId); const hostId = instance.config.get("instance.assigned_host"); + const inheritors = this.inheritorsOf(instanceId).length; + if (inheritors) { + throw new lib.ConflictError(`Instance is the base of ${inheritors} other instance(s)`); + } if (hostId !== null) { await this._controller.sendTo( diff --git a/packages/controller/src/InstanceRecord.ts b/packages/controller/src/InstanceRecord.ts index 9ad0db58..a441a88a 100644 --- a/packages/controller/src/InstanceRecord.ts +++ b/packages/controller/src/InstanceRecord.ts @@ -83,6 +83,7 @@ export default class InstanceRecord { this.startedAtMs, this.updatedAtMs, this.config.get("instance.exclude_from_start_all"), + this.config.get("instance.is_base"), ); } diff --git a/packages/ctl/src/commands_instance.ts b/packages/ctl/src/commands_instance.ts index 63440875..c575827b 100644 --- a/packages/ctl/src/commands_instance.ts +++ b/packages/ctl/src/commands_instance.ts @@ -20,10 +20,14 @@ export const instanceCommands = new lib.CommandTree({ name: "instance", alias: ["i"], description: "Instance management", }); instanceCommands.add(new lib.Command({ - definition: [["list", "l"], "List instances known to the controller"], - handler: async function(args: object, ctl: Ctl) { + definition: [["list", "l"], "List instances known to the controller", (yargs) => { + yargs.options({ + "bases": { describe: "Include base instances", nargs: 0, type: "boolean", default: false }, + }); + }], + handler: async function(args: { bases: boolean }, ctl: Ctl) { let list = await ctl.send(new lib.InstanceDetailsListRequest()); - print(asTable(list)); + print(asTable(list.filter(instance => args.bases || !instance.isBase))); }, })); @@ -34,18 +38,30 @@ instanceCommands.add(new lib.Command({ yargs.options({ "id": { type: "number", nargs: 1, describe: "Instance id" }, "from": { type: "number", nargs: 1, describe: "Clone config from instance id" }, + "base": { type: "string", nargs: 1, describe: "Base instance to inherit config from" }, + "is-base": { type: "boolean", nargs: 0, describe: "Create the instance as a base" }, }); }], - handler: async function(args: { name: string, id?: number, from?: number, }, ctl: Ctl) { + handler: async function( + args: { name: string, id?: number, from?: number, base?: string, isBase?: boolean }, + ctl: Ctl, + ) { let instanceConfig = new lib.InstanceConfig("control"); + let fields: (keyof lib.InstanceConfigFields)[] = ["instance.id", "instance.name"]; if (args.id !== undefined) { instanceConfig.set("instance.id", args.id); } instanceConfig.set("instance.name", args.name); + if (args.base !== undefined) { + instanceConfig.set("instance.base_id", await lib.resolveInstance(ctl, args.base)); + fields.push("instance.base_id"); + } + if (args.isBase) { + instanceConfig.set("instance.is_base", true); + fields.push("instance.is_base"); + } await ctl.send(new lib.InstanceCreateRequest( - instanceConfig.toRemote("controller", [ - "instance.id", "instance.name", - ]), + instanceConfig.toRemote("controller", fields), args.from )); }, @@ -61,13 +77,30 @@ instanceConfigCommands.add(new lib.Command({ handler: async function(args: { instance: string }, ctl: Ctl) { let instanceId = await lib.resolveInstance(ctl, args.instance); let config = await ctl.send(new lib.InstanceConfigGetRequest(instanceId)); + const inherited = config["instance.base_id"] !== null + ? config["instance.inherited_fields"] as Record ?? {} + : {}; for (let [name, value] of Object.entries(config)) { - print(`${name} ${JSON.stringify(value)}`); + if (name === "instance.inherited_fields") { + continue; + } + print(`${name} ${JSON.stringify(value)}${inherited[name] ? " (inherited)" : ""}`); } }, })); +instanceConfigCommands.add(new lib.Command({ + definition: ["inherit ", "Make field in instance config use the value of its base", (yargs) => { + yargs.positional("instance", { describe: "Instance to update config on", type: "string" }); + yargs.positional("field", { describe: "Field to inherit", type: "string" }); + }], + handler: async function(args: { instance: string, field: string }, ctl: Ctl) { + let instanceId = await lib.resolveInstance(ctl, args.instance); + await ctl.send(new lib.InstanceConfigInheritFieldRequest(instanceId, args.field)); + }, +})); + instanceConfigCommands.add(new lib.Command({ definition: ["set [value]", "Set field in instance config", (yargs) => { yargs.positional("instance", { describe: "Instance to set config on", type: "string" }); @@ -170,8 +203,10 @@ instanceConfigCommands.add(new lib.Command({ await events.once(editorSpawn, "exit"); const data = await fs.readFile(tmpFile, "utf8"); const final = await configToKeyVal(data); + const original = await configToKeyVal(allConfigElements); for (let index in final) { - if (index in final) { + // Unchanged fields are skipped so they keep inheriting from the base + if (index in final && final[index] !== original[index]) { try { await ctl.send(new lib.InstanceConfigSetFieldRequest( instanceId, diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index b28db160..061d24dd 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -488,6 +488,9 @@ export interface InstanceConfigFields { "instance.assigned_host": number | null; "instance.auto_start": boolean; "instance.exclude_from_start_all": boolean; + "instance.is_base": boolean; + "instance.base_id": number | null; + "instance.inherited_fields": Record; [key: `${string}.load_plugin`]: boolean; [key: `${string}.version`]: string; @@ -539,6 +542,23 @@ export class InstanceConfig extends classes.Config { return config; } + /** Fields that are never inherited from a base instance */ + static uninheritableFields = ["instance.name", "instance.is_base", "instance.base_id"]; + + /** True if the field can take its value from a base instance */ + static isInheritable(name: string) { + const def = this.fieldDefinitions[name as keyof InstanceConfigFields]; + return Boolean(def) + && !def.hidden + && (def.access ?? this.defaultAccess).includes("controller") + && !this.uninheritableFields.includes(name); + } + + /** True if the field currently takes its value from the base instance */ + isInherited(name: string) { + return this.get("instance.base_id") !== null && Boolean(this.get("instance.inherited_fields")[name]); + } + static fieldDefinitions: classes.ConfigDefs = { "instance.name": { type: "string", @@ -566,6 +586,30 @@ export class InstanceConfig extends classes.Config { type: "boolean", initialValue: false, }, + "instance.is_base": { + description: + "Mark this instance as a base other instances can inherit config from. " + + "Bases cannot be assigned to a host or started.", + access: ["controller", "control"], + type: "boolean", + initialValue: false, + }, + "instance.base_id": { + title: "Base Instance", + description: "ID of the base instance to inherit config values from", + access: ["controller", "control"], + type: "number", + optional: true, + // Checked by the controller + }, + "instance.inherited_fields": { + description: "Fields whose value comes from the base instance", + access: ["controller", "control"], + readonly: ["controller"], + type: "object", + initialValue: {}, + hidden: true, + }, "factorio.version": { description: "Version of the game to run. Use 'latest' for the latest installed version, a release " + diff --git a/packages/lib/src/data/InstanceDetails.ts b/packages/lib/src/data/InstanceDetails.ts index adf0b9d8..ce008c9b 100644 --- a/packages/lib/src/data/InstanceDetails.ts +++ b/packages/lib/src/data/InstanceDetails.ts @@ -41,7 +41,8 @@ export default class InstanceDetails { public startedAtMs = 0, /** Millisecond Unix timestamp this entry was last updated at */ public updatedAtMs = 0, - public excludeFromStartAll = false + public excludeFromStartAll = false, + public isBase = false, ) {} get isDeleted() { @@ -61,6 +62,7 @@ export default class InstanceDetails { "startedAtMs": Type.Optional(Type.Number()), "updatedAtMs": Type.Optional(Type.Number()), "excludeFromStartAll": Type.Optional(Type.Boolean()), + "isBase": Type.Optional(Type.Boolean()), }); static fromJSON(json: Static) { @@ -73,7 +75,8 @@ export default class InstanceDetails { json.factorioVersion, json.startedAtMs, json.updatedAtMs, - json.excludeFromStartAll + json.excludeFromStartAll, + json.isBase, ); } @@ -102,6 +105,9 @@ export default class InstanceDetails { if (this.excludeFromStartAll !== false) { json.excludeFromStartAll = this.excludeFromStartAll; } + if (this.isBase !== false) { + json.isBase = this.isBase; + } return json; } diff --git a/packages/lib/src/data/messages_instance.ts b/packages/lib/src/data/messages_instance.ts index d107632c..5cde39b7 100644 --- a/packages/lib/src/data/messages_instance.ts +++ b/packages/lib/src/data/messages_instance.ts @@ -176,6 +176,28 @@ export class InstanceConfigSetPropRequest { } } +export class InstanceConfigInheritFieldRequest { + declare ["constructor"]: typeof InstanceConfigInheritFieldRequest; + static type = "request" as const; + static src = "control" as const; + static dst = "controller" as const; + static permission = "core.instance.update_config" as const; + + constructor( + public instanceId: number, + public field: string, + ) { } + + static jsonSchema = Type.Object({ + "instanceId": Type.Integer(), + "field": Type.String(), + }); + + static fromJSON(json: Static) { + return new this(json.instanceId, json.field); + } +} + export class InstanceAssignRequest { declare ["constructor"]: typeof InstanceAssignRequest; static type = "request" as const; diff --git a/packages/lib/src/link/messages.ts b/packages/lib/src/link/messages.ts index f4e15976..c0ea4940 100644 --- a/packages/lib/src/link/messages.ts +++ b/packages/lib/src/link/messages.ts @@ -71,6 +71,7 @@ export const dataClasses: (RequestClass | EventClass) instance.InstanceConfigSetRequest, instance.InstanceConfigSetFieldRequest, instance.InstanceConfigSetPropRequest, + instance.InstanceConfigInheritFieldRequest, instance.InstanceAssignRequest, instance.InstanceMetricsRequest, instance.InstanceStartRequest, diff --git a/test/controller/ControlConnection.test.js b/test/controller/ControlConnection.test.js index 6311c2b1..e4d9efde 100644 --- a/test/controller/ControlConnection.test.js +++ b/test/controller/ControlConnection.test.js @@ -156,6 +156,105 @@ describe("controller/src/ControlConnection", function() { assert.equal(clone.config.get("instance.name"), "clone"); assert.equal(clone.config.get("instance.assigned_host"), null); }); + + it("creates an instance inheriting from a base", async function() { + await create({ "instance.id": 4001, "instance.name": "base", "instance.is_base": true }); + mockController.instances.getMutable(4001).config.set("factorio.version", "2.0"); + + await create({ "instance.id": 4002, "instance.name": "child", "instance.base_id": 4001 }); + const child = mockController.instances.get(4002); + assert.equal(child.config.get("factorio.version"), "2.0"); + assert.equal(child.config.isInherited("factorio.version"), true); + assert.equal(child.config.isInherited("instance.name"), false); + }); + + it("rejects a base that is not a base", async function() { + await create({ "instance.id": 4001, "instance.name": "plain" }); + await assert.rejects( + create({ "instance.id": 4002, "instance.name": "child", "instance.base_id": 4001 }), + new lib.ValidationError("Instance plain is not a base"), + ); + assert.equal(mockController.instances.has(4002), false); + }); + }); + + describe("instance config set handlers", function() { + let mockController; + let connection; + + beforeEach(async function() { + mockController = new MockController(); + mockController.mockConfigEntries.set("controller.name", "Test"); + mockController.instances = new InstanceManager(new lib.SubscribableDatastore(), mockController); + mockController.instanceConfigUpdated = async () => {}; + connection = { + _controller: mockController, + checkStagedInheritance: ControlConnection.prototype.checkStagedInheritance, + }; + for (const [id, fields] of [ + [1, { "instance.is_base": true, "factorio.version": "2.0" }], + [2, { "instance.base_id": 1 }], + ]) { + const config = new lib.InstanceConfig("controller", { + "instance.id": id, "instance.name": `i${id}`, ...fields, + }); + if (fields["instance.base_id"]) { + mockController.instances.applyBase(config, ["instance.id", "instance.name", "instance.base_id"]); + } + await mockController.instances.createInstance(config); + } + }); + + function call(handler, request) { + return ControlConnection.prototype[handler].call(connection, request); + } + const setField = (...args) => new lib.InstanceConfigSetFieldRequest(...args); + const setFields = (...args) => new lib.InstanceConfigSetRequest(...args); + + it("stops inheriting a field set to the inherited value", async function() { + await call("handleInstanceConfigSetFieldRequest", setField(2, "factorio.version", "2.0")); + assert.equal(mockController.instances.get(2).config.isInherited("factorio.version"), false); + }); + + it("stops inheriting a field set through set request", async function() { + await call("handleInstanceConfigSetRequest", setFields(2, { "factorio.version": "2.0" })); + assert.equal(mockController.instances.get(2).config.isInherited("factorio.version"), false); + }); + + it("stops inheriting a field changed through set-prop", async function() { + await call( + "handleInstanceConfigSetPropRequest", + new lib.InstanceConfigSetPropRequest(2, "factorio.settings", "max_players", 0), + ); + assert.equal(mockController.instances.get(2).config.isInherited("factorio.settings"), false); + }); + + it("rejects base_id pointing at a non-base", async function() { + await assert.rejects( + call("handleInstanceConfigSetFieldRequest", setField(1, "instance.base_id", "2")), + new lib.ValidationError("Instance i2 is not a base"), + ); + assert.equal(mockController.instances.get(1).config.get("instance.base_id"), null); + }); + + it("rejects unmarking a base with inheritors through set request", async function() { + await assert.rejects( + call("handleInstanceConfigSetRequest", setFields(1, { "instance.is_base": "false" })), + new lib.ConflictError("Instance is the base of 1 other instance(s)"), + ); + assert.equal(mockController.instances.get(1).config.get("instance.is_base"), true); + }); + + it("inherits a field again", async function() { + await call("handleInstanceConfigSetFieldRequest", setField(2, "factorio.version", "2.1")); + await call( + "handleInstanceConfigInheritFieldRequest", + new lib.InstanceConfigInheritFieldRequest(2, "factorio.version"), + ); + const config = mockController.instances.get(2).config; + assert.equal(config.get("factorio.version"), "2.0"); + assert.equal(config.isInherited("factorio.version"), true); + }); }); describe("close cleanup", function() { diff --git a/test/controller/Controller.js b/test/controller/Controller.js index 732feceb..9a5908c0 100644 --- a/test/controller/Controller.js +++ b/test/controller/Controller.js @@ -34,6 +34,10 @@ class MockInstanceConfig extends EventEmitter { this._data.set(field, value); this.emit("fieldChanged", field, value, prev); } + + isInherited() { + return false; + } } describe("controller/src/Controller", function() { diff --git a/test/controller/InstanceManager.test.js b/test/controller/InstanceManager.test.js index dc254629..fc6e262b 100644 --- a/test/controller/InstanceManager.test.js +++ b/test/controller/InstanceManager.test.js @@ -393,4 +393,179 @@ describe("controller/InstanceManager", function () { assert.equal(sentRequest.instanceId, 1); }); }); + + describe("inheritance", function () { + /** @type {InstanceRecord} */ + let base; + /** @type {InstanceRecord} */ + let child; + let updated; + + async function createInheritor(id, baseId, fields = {}) { + const config = new lib.InstanceConfig("controller", { + "instance.id": id, "instance.name": `Instance ${id}`, "instance.base_id": baseId, ...fields, + }); + instances.applyBase(config, ["instance.id", "instance.name", "instance.base_id", ...Object.keys(fields)]); + return await instances.createInstance(config); + } + + beforeEach(async function () { + base = await instances.createInstance(new lib.InstanceConfig("controller", { + "instance.id": 10, + "instance.name": "Base", + "instance.is_base": true, + "factorio.version": "2.0", + "factorio.settings": { name: "Base server" }, + })); + child = await createInheritor(11, 10, { "factorio.game_port": 34197 }); + updated = []; + controller.instanceConfigUpdated = async (inst) => { updated.push(inst.id); }; + }); + + describe(".applyBase()", function () { + it("should copy base values and mark them inherited", function () { + assert.equal(child.config.get("factorio.version"), "2.0"); + assert.equal(child.config.isInherited("factorio.version"), true); + assert.equal(child.config.get("factorio.game_port"), 34197); + assert.equal(child.config.isInherited("factorio.game_port"), false); + assert.equal(child.config.isInherited("instance.name"), false); + }); + it("should keep inherited factorio.settings in createInstance", function () { + assert.deepEqual(child.config.get("factorio.settings"), base.config.get("factorio.settings")); + assert.notEqual(child.config.get("factorio.settings"), base.config.get("factorio.settings")); + }); + }); + + describe("base changes", function () { + it("should propagate to inherited fields and send the config to the host", function () { + base.config.set("factorio.version", "2.1"); + assert.equal(child.config.get("factorio.version"), "2.1"); + assert.equal(child.config.isInherited("factorio.version"), true); + assert.deepEqual(updated, [11]); + }); + it("should mark the inheritor as requiring restart", function () { + child.config.restartRequired = false; + base.config.set("factorio.version", "2.1"); + assert.equal(child.config.restartRequired, true); + }); + it("should not touch fields the inheritor has set", function () { + base.config.set("factorio.game_port", 1234); + assert.equal(child.config.get("factorio.game_port"), 34197); + assert.deepEqual(updated, []); + }); + it("should propagate through several levels", async function () { + child.config.set("instance.is_base", true); + const grandchild = await createInheritor(12, 11); + base.config.setProp("factorio.settings", "max_players", 10); + assert.equal(grandchild.config.get("factorio.settings").max_players, 10); + assert.deepEqual(updated.sort(), [11, 12]); + }); + }); + + describe("inheritor changes", function () { + it("should stop inheriting a field that is changed", function () { + child.config.set("factorio.version", "2.1"); + assert.equal(child.config.isInherited("factorio.version"), false); + base.config.set("factorio.version", "2.0.1"); + assert.equal(child.config.get("factorio.version"), "2.1"); + }); + it("should stop inheriting with overrideFields() even if the value is unchanged", function () { + instances.overrideFields(child, ["factorio.version", "factorio.game_port"]); + assert.equal(child.config.isInherited("factorio.version"), false); + }); + }); + + describe(".inheritField()", function () { + it("should restore the base value", function () { + child.config.set("factorio.version", "2.1"); + instances.inheritField(11, "factorio.version"); + assert.equal(child.config.get("factorio.version"), "2.0"); + assert.equal(child.config.isInherited("factorio.version"), true); + }); + it("should throw if the instance has no base", function () { + assert.throws( + () => instances.inheritField(1, "factorio.version"), + new lib.ValidationError("Instance does not have a base"), + ); + }); + it("should throw for fields that cannot be inherited", function () { + assert.throws( + () => instances.inheritField(11, "instance.name"), + new lib.ValidationError("Field instance.name cannot be inherited"), + ); + }); + }); + + describe("base_id changes", function () { + it("should inherit fields equal to the new base", function () { + instance.config.set("factorio.version", "2.0"); + instance.config.set("instance.base_id", 10); + assert.equal(instance.config.isInherited("factorio.version"), true); + assert.equal(instance.config.isInherited("factorio.settings"), false); + }); + it("should clear inherited fields when the base is removed", function () { + child.config.set("instance.base_id", null); + assert.deepEqual(child.config.get("instance.inherited_fields"), {}); + assert.equal(child.config.get("factorio.version"), "2.0"); + }); + }); + + describe(".checkInheritance()", function () { + it("should accept a valid base", function () { + instances.checkInheritance(instance.config, false, 10); + }); + it("should reject a missing base", function () { + assert.throws( + () => instances.checkInheritance(instance.config, false, 99), + new lib.NotFoundError("Instance with ID 99 does not exist"), + ); + }); + it("should reject an instance that is not a base", function () { + assert.throws( + () => instances.checkInheritance(instance.config, false, 11), + new lib.ValidationError("Instance Instance 11 is not a base"), + ); + }); + it("should reject cycles", function () { + child.config.set("instance.is_base", true); + assert.throws( + () => instances.checkInheritance(base.config, true, 11), + new lib.ValidationError("Base instances cannot inherit from themselves"), + ); + }); + it("should reject making an assigned instance a base", function () { + instance.config.set("instance.assigned_host", 5); + assert.throws( + () => instances.checkInheritance(instance.config, true, null), + new lib.ConflictError("Instance must be unassigned before it can be made a base"), + ); + }); + it("should reject unmarking a base that has inheritors", function () { + assert.throws( + () => instances.checkInheritance(base.config, false, null), + new lib.ConflictError("Instance is the base of 1 other instance(s)"), + ); + }); + }); + + it("should not assign a base to a host", async function () { + controller.wsServer.hostConnections.set(5, { send: async () => {}, connector: { closing: false } }); + await assert.rejects( + instances.assignInstance(10, 5), + new lib.ValidationError("Base instances cannot be assigned to a host"), + ); + }); + + it("should not delete a base that has inheritors", async function () { + await assert.rejects( + instances.deleteInstance(10), + new lib.ConflictError("Instance is the base of 1 other instance(s)"), + ); + }); + + it("should report bases in instance details", function () { + assert.equal(base.toInstanceDetails().isBase, true); + assert.equal(child.toInstanceDetails().isBase, false); + }); + }); }); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 54dcd29b..d25fc29b 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -971,6 +971,63 @@ describe("Integration of Clusterio", function() { }); }); + describe("instance base inheritance", function() { + after(async function() { + for (const id of [4402, 4401, 4400]) { + await getControl().send(new lib.InstanceDeleteRequest(id)).catch(() => {}); + } + }); + it("creates a base and an instance inheriting from it", async function() { + await execCtl("instance create inheritBase --id 4400 --is-base"); + await execCtl("instance config set inheritBase factorio.version 2.0"); + await execCtl("instance create inheritChild --id 4401 --base inheritBase"); + const config = await getControl().send(new lib.InstanceConfigGetRequest(4401)); + assert.equal(config["instance.base_id"], 4400); + assert.equal(config["factorio.version"], "2.0"); + assert.equal(config["instance.inherited_fields"]["factorio.version"], true); + assert.equal((await getInstances()).get(4400).isBase, true); + }); + it("propagates base changes", async function() { + await execCtl("instance config set inheritBase factorio.version 2.1"); + const config = await getControl().send(new lib.InstanceConfigGetRequest(4401)); + assert.equal(config["factorio.version"], "2.1"); + }); + it("rejects inheriting from an instance that is not a base", async function() { + await assert.rejects( + execCtl("instance create inheritBad --id 4402 --base inheritChild"), + new lib.ValidationError("Instance inheritChild is not a base"), + ); + }); + it("rejects assigning a base to a host", async function() { + await assert.rejects( + execCtl("instance assign inheritBase 4"), + new lib.ValidationError("Base instances cannot be assigned to a host"), + ); + }); + describe("instance list --bases", function() { + it("runs", async function() { + await execCtl("instance list --bases"); + }); + }); + describe("instance config list", function() { + it("runs on an inheriting instance", async function() { + await execCtl("instance config list inheritChild"); + }); + }); + describe("instance config inherit", function() { + it("runs", async function() { + await execCtl("instance config set inheritChild factorio.version 2.0"); + let config = await getControl().send(new lib.InstanceConfigGetRequest(4401)); + assert.equal(config["instance.inherited_fields"]["factorio.version"], undefined); + + await execCtl("instance config inherit inheritChild factorio.version"); + config = await getControl().send(new lib.InstanceConfigGetRequest(4401)); + assert.equal(config["factorio.version"], "2.1"); + assert.equal(config["instance.inherited_fields"]["factorio.version"], true); + }); + }); + }); + describe("instance assign", function() { it("creates the instance files", async function() { await execCtl(`instance assign test ${hostId}`); diff --git a/test/integration/routing.js b/test/integration/routing.js index 19c73dfa..4bfc7834 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -41,7 +41,8 @@ function connectInstance(controller, host, instanceId) { instanceConfig.set("instance.id", instanceId); instanceConfig.set("instance.assigned_host", host.config.get("host.id")); host.assignedInstances.set(instanceId, { path: "invalid", config: instanceConfig }); - controller.instances.records.set(new InstanceRecord(instanceConfig, "stopped")); + const controllerConfig = new lib.InstanceConfig("controller", instanceConfig.toRemote("controller")); + controller.instances.records.set(new InstanceRecord(controllerConfig, "stopped")); const instance = new Instance(host, instanceSide, "invalid", instanceConfig, new MockServer()); const instanceConnection = new InstanceConnection(hostSide, host, instance); host.instanceConnections.set(instanceId, instanceConnection); diff --git a/test/lib/config/definitions.js b/test/lib/config/definitions.js index a406f579..33cc097e 100644 --- a/test/lib/config/definitions.js +++ b/test/lib/config/definitions.js @@ -4,6 +4,48 @@ import * as lib from "@clusterio/lib"; describe("lib/config/definitions", function() { + describe("InstanceConfig", function() { + describe("isInheritable()", function() { + it("should allow regular fields", function() { + assert.equal(lib.InstanceConfig.isInheritable("factorio.version"), true); + assert.equal(lib.InstanceConfig.isInheritable("instance.auto_start"), true); + }); + it("should reject identity, hidden, host only and unknown fields", function() { + for (const name of [ + "instance.name", "instance.id", "instance.assigned_host", "instance.is_base", + "instance.base_id", "instance.inherited_fields", "factorio.host_assigned_game_port", "no.field", + ]) { + assert.equal(lib.InstanceConfig.isInheritable(name), false, name); + } + }); + }); + describe(".isInherited()", function() { + it("should only report fields as inherited when a base is set", function() { + const config = new lib.InstanceConfig("controller", { + "instance.inherited_fields": { "factorio.version": true }, + }); + assert.equal(config.isInherited("factorio.version"), false); + config.set("instance.base_id", 1); + assert.equal(config.isInherited("factorio.version"), true); + assert.equal(config.isInherited("factorio.game_port"), false); + }); + }); + it("should not send inheritance fields to the host", function() { + const config = new lib.InstanceConfig("controller", { "instance.base_id": 1 }); + const remote = config.toRemote("host"); + for (const name of ["instance.is_base", "instance.base_id", "instance.inherited_fields"]) { + assert(!Object.hasOwn(remote, name), name); + } + }); + it("should not let control write inherited_fields", function() { + const config = new lib.InstanceConfig("controller"); + assert.throws( + () => config.set("instance.inherited_fields", { "factorio.version": true }, "control"), + new lib.InvalidAccessError("Field 'instance.inherited_fields' is not accessible from control"), + ); + }); + }); + describe("addPluginConfigFields()", function() { after(function() { delete lib.CtlConfig.fieldDefinitions["ctl_test.field"]; From b5c372f109f874614bea8a6bc16fdc92d1aeb6c7 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:19:08 +0000 Subject: [PATCH 25/44] Rename bases to templates and add per property inheritance Review changes for #1142: - base becomes template everywhere (is_template, template_id, --template, --is-template, --templates, InstanceDetails.isTemplate). - Object fields inherit per property. inherited_fields lists the overridden properties for object fields and the template object is merged with them. - Linking a template also inherits fields that hold the default value. The server name stays per instance. - Ports are never inherited. - InstanceDetails carries templateId. Web UI: inherited values are locked until unlocked, overridden values can be inherited again, templates are listed separately, the create modal can create templates and instances from templates, and the instance page has "Create template" which can link the source. Part of #200 Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 26 +- docs/managing-a-cluster.md | 14 +- packages/controller/src/ControlConnection.ts | 29 +- packages/controller/src/InstanceManager.ts | 287 ++++++++++++------ packages/controller/src/InstanceRecord.ts | 3 +- packages/ctl/src/commands_instance.ts | 56 ++-- packages/lib/src/config/definitions.ts | 49 ++- packages/lib/src/data/InstanceDetails.ts | 16 +- packages/lib/src/data/messages_instance.ts | 4 +- packages/web_ui/src/bootstrap.tsx | 2 + .../web_ui/src/components/BaseConfigTree.tsx | 94 +++++- .../web_ui/src/components/InputInstance.tsx | 9 +- .../src/components/InstanceConfigTree.tsx | 35 ++- .../web_ui/src/components/InstanceList.tsx | 3 +- .../src/components/InstanceViewPage.tsx | 92 ++++-- .../web_ui/src/components/InstancesPage.tsx | 131 ++++++-- test/controller/ControlConnection.test.js | 77 +++-- test/controller/InstanceManager.test.js | 177 +++++++---- test/integration/clusterio.js | 49 +-- test/lib/config/definitions.js | 22 +- test/lib/data/InstanceDetails.test.js | 2 + 21 files changed, 865 insertions(+), 312 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 54e1f015..d6398ba7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -521,26 +521,28 @@ This allows you to prevent certain instances from being started when using the b Defaults to false. -### instance.is_base +### instance.is_template -If enabled this instance is a base other instances can inherit config values from. -Base instances cannot be assigned to a host and therefore never start. -Hidden from `instance list` in clusterioctl unless `--bases` is passed. -An instance cannot stop being a base while other instances inherit from it. +If enabled this instance is a template other instances can inherit config values from. +Templates cannot be assigned to a host and therefore never start. +Hidden from `instance list` in clusterioctl unless `--templates` is passed, and listed separately in the web interface. +An instance cannot stop being a template while other instances inherit from it. Defaults to false. -### instance.base_id +### instance.template_id -ID of the base instance to inherit config values from. -Inherited fields take their value from the base and follow it when it changes, setting a field on this instance stops inheriting it. -When set on an existing instance only fields with the same value as the base become inherited. -`instance.name`, `instance.is_base`, `instance.base_id` and hidden fields are never inherited. -A base may itself inherit from another base, but not from itself through a chain of bases. +ID of the template to inherit config values from. +Inherited fields take their value from the template and follow it when it changes, setting a field on this instance stops inheriting it. +Object fields like `factorio.settings` are inherited per property, properties set on this instance override the ones from the template. +When set on an existing instance fields with the same value as the template or the default value become inherited, except the server name in `factorio.settings`. +`instance.name`, `instance.is_template`, `instance.template_id`, the port fields and hidden fields are never inherited. +A template may itself inherit from another template, but not from itself through a chain of templates. The fields currently inherited are stored in the hidden `instance.inherited_fields` field. +In the web interface inherited values are locked and must be unlocked before they can be changed. -Defaults to null meaning no base. +Defaults to null meaning no template. ### factorio.version diff --git a/docs/managing-a-cluster.md b/docs/managing-a-cluster.md index 14073d9d..67cd5a13 100644 --- a/docs/managing-a-cluster.md +++ b/docs/managing-a-cluster.md @@ -79,16 +79,16 @@ Marks the host as requiring a restart, the same way a plugin update does. Use it ### List Instances - ctl> instance list [--bases] + ctl> instance list [--templates] List instances. -Base instances are only listed if `--bases` is passed. +Templates are only listed if `--templates` is passed. ### Create instance - ctl> instance create [--id ] [--from ] [--base ] [--is-base] + ctl> instance create [--id ] [--from ] [--template ] [--is-template] -Create a new instance. `name` is the instance name. `--id` is the optional id. `--from` copies the config of an existing instance. `--base` makes the new instance inherit all config values from the given base instance. `--is-base` creates the instance as a base, see `instance.is_base` in the [configuration docs](configuration.md). +Create a new instance. `name` is the instance name. `--id` is the optional id. `--from` copies the config of an existing instance. `--template` makes the new instance inherit all config values from the given template. `--is-template` creates the instance as a template, see `instance.is_template` in the [configuration docs](configuration.md). Use `--is-template --from ` to make a template out of an existing instance. ### Kill instance @@ -121,7 +121,7 @@ Most notably Clusterio records wall clock time while the save time is recorded i ctl> instance config list -List config values for an instance. `instance` is the instance whose config values to list. Values inherited from a base instance are marked with `(inherited)`. +List config values for an instance. `instance` is the instance whose config values to list. Values inherited from a template are marked with `(inherited)`, object fields also list the properties set on the instance. #### Set config value @@ -135,9 +135,9 @@ To be written. #### Inherit config value - ctl> instance config inherit + ctl> instance config inherit [prop] -Make a field in the instance config take its value from the base instance again, undoing a previous `set`. `instance` is the instance to update, `field` is the field to inherit. +Make a field in the instance config take its value from the template again, undoing a previous `set`. `instance` is the instance to update, `field` is the field to inherit, `prop` limits it to one property of an object field. #### Edit instance config diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 72b8ae88..aa618603 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -317,11 +317,11 @@ export default class ControlConnection extends BaseConnection { instanceConfig.set("instance.assigned_host", null); // New instances are unassigned } instanceConfig.update(request.config, false, "control"); - if (Object.hasOwn(request.config, "instance.base_id")) { - this._controller.instances.checkInheritance( - instanceConfig, instanceConfig.get("instance.is_base"), instanceConfig.get("instance.base_id") - ); - this._controller.instances.applyBase(instanceConfig, Object.keys(request.config)); + this._controller.instances.checkInheritance( + instanceConfig, instanceConfig.get("instance.is_template"), instanceConfig.get("instance.template_id") + ); + if (Object.hasOwn(request.config, "instance.template_id")) { + this._controller.instances.applyTemplate(instanceConfig, Object.keys(request.config)); } await this._controller.instances.createInstance(instanceConfig); } @@ -365,7 +365,12 @@ export default class ControlConnection extends BaseConnection { this.checkStagedInheritance(instance); instance.config.commitStaging(); - this._controller.instances.overrideFields(instance, Object.keys(request.fields)); + for (const [fieldName, fieldValue] of Object.entries(request.fields)) { + const props = typeof fieldValue === "object" + ? Object.keys(fieldValue).filter(prop => fieldValue[prop] !== undefined) + : undefined; + this._controller.instances.overrideField(instance, fieldName, props); + } await this._controller.instanceConfigUpdated(instance); } finally { instance.config.revertStaging(); @@ -375,7 +380,7 @@ export default class ControlConnection extends BaseConnection { private checkStagedInheritance(instance: InstanceRecord) { const staged = instance.config.staging; this._controller.instances.checkInheritance( - instance.config, staged.get("instance.is_base"), staged.get("instance.base_id") + instance.config, staged.get("instance.is_template"), staged.get("instance.template_id") ); } @@ -383,7 +388,7 @@ export default class ControlConnection extends BaseConnection { const instance = this._controller.instances.getForRequest(request.instanceId); const field = request.field as keyof lib.InstanceConfigFields; ControlConnection.validateInstanceConfigSetField(field); - if (["instance.is_base", "instance.base_id"].includes(field)) { + if (["instance.is_template", "instance.template_id"].includes(field)) { try { instance.config.stage(field, request.value, "control"); this.checkStagedInheritance(instance); @@ -392,7 +397,7 @@ export default class ControlConnection extends BaseConnection { } } instance.config.set(field, request.value, "control"); - this._controller.instances.overrideFields(instance, [field]); + this._controller.instances.overrideField(instance, field); await this._controller.instanceConfigUpdated(instance); } @@ -401,13 +406,15 @@ export default class ControlConnection extends BaseConnection { let { field, prop, value } = request; ControlConnection.validateInstanceConfigSetField(field); instance.config.setProp(field as keyof lib.InstanceConfigFields, prop, value, "control"); - this._controller.instances.overrideFields(instance, [field]); + if (value !== undefined) { + this._controller.instances.overrideField(instance, field, [prop]); + } await this._controller.instanceConfigUpdated(instance); } async handleInstanceConfigInheritFieldRequest(request: lib.InstanceConfigInheritFieldRequest) { ControlConnection.validateInstanceConfigSetField(request.field); - this._controller.instances.inheritField(request.instanceId, request.field); + this._controller.instances.inheritField(request.instanceId, request.field, request.prop); await this._controller.instanceConfigUpdated(this._controller.instances.getForRequest(request.instanceId)); } diff --git a/packages/controller/src/InstanceManager.ts b/packages/controller/src/InstanceManager.ts index 38802a96..ffbeddc9 100644 --- a/packages/controller/src/InstanceManager.ts +++ b/packages/controller/src/InstanceManager.ts @@ -6,6 +6,9 @@ import type Controller from "./Controller.js"; import type HostConnection from "./HostConnection.js"; import InstanceRecord from "./InstanceRecord.js"; +type InheritedFields = lib.InstanceConfigFields["instance.inherited_fields"]; +type InheritableField = keyof lib.InstanceConfigFields & string; + export default class InstanceManager { private static readonly DefaultFactorioSettings = { tags: ["clusterio"], @@ -26,7 +29,7 @@ export default class InstanceManager { non_blocking_saving: false, } as const; - /** Non-zero while base values are being copied into inheriting instances */ + /** Non-zero while template values are being copied into inheriting instances */ private _inheriting = 0; constructor( @@ -111,10 +114,10 @@ export default class InstanceManager { */ addInstanceHooks(instance: InstanceRecord): void { instance.config.on("fieldChanged", (field: string, curr: unknown, prev: unknown) => { - if (field === "instance.base_id") { - this._rebase(instance); + if (field === "instance.template_id") { + this._link(instance); } else if (!this._inheriting && instance.config.isInherited(field)) { - this._setInherited(instance, [field], false); + this._override(instance, field, curr, prev); } this.records.set(instance); this._notifyConfigFieldChanged(instance, field, curr, prev); @@ -127,168 +130,249 @@ export default class InstanceManager { /* ------------------------------------------------------------------------- */ /** - * Get the instances that inherit directly from the given base. + * Get the instances that inherit directly from the given template. * - * @param baseId - ID of the base instance. - * @returns Instances with `instance.base_id` set to baseId. + * @param templateId - ID of the template. + * @returns Instances with `instance.template_id` set to templateId. */ - inheritorsOf(baseId: number): InstanceRecord[] { + inheritorsOf(templateId: number): InstanceRecord[] { return [...this.records.values()].filter( - instance => instance.config.get("instance.base_id") === baseId + instance => instance.config.get("instance.template_id") === templateId ); } /** - * Check that new values for `instance.is_base` and `instance.base_id` are allowed. + * Check that new values for `instance.is_template` and `instance.template_id` are allowed. * * @param config - Current config of the instance. - * @param isBase - New value for `instance.is_base`. - * @param baseId - New value for `instance.base_id`. - * @throws {module:lib.ValidationError} if the base is not a base or would create a cycle. + * @param isTemplate - New value for `instance.is_template`. + * @param templateId - New value for `instance.template_id`. + * @throws {module:lib.ValidationError} if the template is not a template or would create a cycle. * @throws {module:lib.ConflictError} if the instance is assigned or still has inheritors. */ - checkInheritance(config: lib.InstanceConfig, isBase: boolean, baseId: number | null): void { + checkInheritance(config: lib.InstanceConfig, isTemplate: boolean, templateId: number | null): void { const instanceId = config.get("instance.id"); - if (isBase && config.get("instance.assigned_host") !== null) { - throw new lib.ConflictError("Instance must be unassigned before it can be made a base"); + if (isTemplate && config.get("instance.assigned_host") !== null) { + throw new lib.ConflictError("Instance must be unassigned before it can be made a template"); } - if (!isBase && config.get("instance.is_base")) { + if (!isTemplate && config.get("instance.is_template")) { const count = this.inheritorsOf(instanceId).length; if (count) { - throw new lib.ConflictError(`Instance is the base of ${count} other instance(s)`); + throw new lib.ConflictError(`Instance is the template of ${count} other instance(s)`); } } - if (baseId === null) { + if (templateId === null) { return; } - const base = this.records.get(baseId); - if (!base) { - throw new lib.NotFoundError(`Instance with ID ${baseId} does not exist`); + const template = this.records.get(templateId); + if (!template) { + throw new lib.NotFoundError(`Instance with ID ${templateId} does not exist`); } - if (!base.config.get("instance.is_base")) { - throw new lib.ValidationError(`Instance ${base.config.get("instance.name")} is not a base`); + if (!template.config.get("instance.is_template")) { + throw new lib.ValidationError(`Instance ${template.config.get("instance.name")} is not a template`); } const visited = new Set(); - for (let id: number | null = baseId; id !== null && !visited.has(id);) { + for (let id: number | null = templateId; id !== null && !visited.has(id);) { if (id === instanceId) { - throw new lib.ValidationError("Base instances cannot inherit from themselves"); + throw new lib.ValidationError("Templates cannot inherit from themselves"); } visited.add(id); - id = this.records.get(id)?.config.get("instance.base_id") ?? null; + id = this.records.get(id)?.config.get("instance.template_id") ?? null; } } /** - * Copy all inheritable values from the base set in `instance.base_id` + * Copy the inheritable values from the template set in `instance.template_id` * into a config that is not yet part of the cluster and mark them inherited. * * @param config - Config of the instance to be created. - * @param keep - Fields to keep the current value of. + * @param own - Fields set by the creator, object fields keep their properties as overrides. */ - applyBase(config: lib.InstanceConfig, keep: string[] = []): void { - const base = this.records.get(config.get("instance.base_id")!); - if (!base) { + applyTemplate(config: lib.InstanceConfig, own: string[] = []): void { + const template = this._templateOf(config); + if (!template) { return; } - const values: Record = {}; - const inherited: Record = {}; - for (const name of Object.keys(lib.InstanceConfig.fieldDefinitions)) { - if (lib.InstanceConfig.isInheritable(name) && !keep.includes(name)) { - values[name] = structuredClone(base.config.get(name as keyof lib.InstanceConfigFields)); + const inherited: InheritedFields = {}; + for (const name of InstanceManager._inheritableFields()) { + const def = lib.InstanceConfig.fieldDefinitions[name]; + if (def.type === "object") { + inherited[name] = own.includes(name) ? Object.keys(config.get(name) as object ?? {}) : []; + } else if (!own.includes(name)) { inherited[name] = true; } } - config.update(values, false); config.set("instance.inherited_fields", inherited); + config.update(InstanceManager._resolve(config, template.config, Object.keys(inherited)), false); } /** - * Make a field take its value from the base instance again. + * Make a field, or a property of an object field, take its value from the template again. * * @param instanceId - ID of the instance to update. * @param field - Name of the field to inherit. - * @throws {module:lib.ValidationError} if the instance has no base or the field cannot be inherited. + * @param prop - Property of an object field to inherit, inherits all properties if not given. + * @throws {module:lib.ValidationError} if the instance has no template or the field cannot be inherited. */ - inheritField(instanceId: number, field: string): void { + inheritField(instanceId: number, field: string, prop?: string): void { const instance = this.getForRequest(instanceId); - const baseId = instance.config.get("instance.base_id"); - const base = baseId !== null ? this.records.get(baseId) : undefined; - if (!base) { - throw new lib.ValidationError("Instance does not have a base"); + if (!this._templateOf(instance.config)) { + throw new lib.ValidationError("Instance does not have a template"); } if (!lib.InstanceConfig.isInheritable(field)) { throw new lib.ValidationError(`Field ${field} cannot be inherited`); } - this._setInherited(instance, [field], true); + const name = field as InheritableField; + const inherited = { ...instance.config.get("instance.inherited_fields") }; + if (lib.InstanceConfig.fieldDefinitions[name].type !== "object") { + inherited[field] = true; + } else if (prop === undefined) { + inherited[field] = []; + } else { + const entry = inherited[field]; + const overrides = entry instanceof Array ? entry : Object.keys(instance.config.get(name) as object ?? {}); + inherited[field] = overrides.filter(p => p !== prop); + } + instance.config.set("instance.inherited_fields", inherited); + this._resolveInto(instance, [field]); + } + + /** + * Stop inheriting the given field or properties, used when a value is explicitly set. + * + * @param instance - Instance the field was set on. + * @param field - Name of the field set. + * @param props - Properties set on an object field. + */ + overrideField(instance: InstanceRecord, field: string, props?: string[]): void { + const entry = instance.config.get("instance.inherited_fields")[field]; + if (!instance.config.isInherited(field)) { + return; + } + const inherited = { ...instance.config.get("instance.inherited_fields") }; + if (entry instanceof Array && props) { + inherited[field] = [...new Set([...entry, ...props])]; + } else { + delete inherited[field]; + } + instance.config.set("instance.inherited_fields", inherited); + } + + private _templateOf(config: lib.InstanceConfig) { + const templateId = config.get("instance.template_id"); + return templateId !== null ? this.records.get(templateId) : undefined; + } + + private static _inheritableFields() { + return Object.keys(lib.InstanceConfig.fieldDefinitions).filter( + name => lib.InstanceConfig.isInheritable(name) + ) as InheritableField[]; + } + + /** Default value of a field, or undefined if it is generated */ + private static _defaultValue(name: InheritableField): unknown { + const def = lib.InstanceConfig.fieldDefinitions[name]; + if (name === "factorio.settings") { + return InstanceManager.DefaultFactorioSettings; + } + return typeof def.initialValue === "function" ? undefined : def.initialValue ?? null; + } + + /** Values the given inherited fields take from the template, with object overrides applied */ + private static _resolve(config: lib.InstanceConfig, template: lib.InstanceConfig, fields: string[]) { + const inherited = config.get("instance.inherited_fields"); + const values: Record = {}; + for (const field of fields) { + const name = field as InheritableField; + const entry = inherited[field]; + if (!entry) { + continue; + } + const value = structuredClone(template.get(name)) as any; + if (entry instanceof Array) { + const own = config.get(name) as Record ?? {}; + for (const prop of entry) { + if (Object.hasOwn(own, prop)) { + value[prop] = structuredClone(own[prop]); + } + } + } + values[field] = value; + } + return values; + } + + /** Update the given inherited fields of a live instance from its template */ + private _resolveInto(instance: InstanceRecord, fields: string[]): void { + const template = this._templateOf(instance.config); + if (!template) { + return; + } + const values = InstanceManager._resolve(instance.config, template.config, fields); this._inheriting += 1; try { - instance.config.update({ - [field]: structuredClone(base.config.get(field as keyof lib.InstanceConfigFields)), - }, true); + instance.config.update(values, true); } finally { this._inheriting -= 1; } } - /** - * Stop inheriting the given fields, used when a value is explicitly set. - * - * @param instance - Instance the fields were set on. - * @param fields - Names of fields set. - */ - overrideFields(instance: InstanceRecord, fields: string[]): void { - const inherited = fields.filter(field => instance.config.isInherited(field)); - if (inherited.length) { - this._setInherited(instance, inherited, false); + /** Track a change to an inherited field made outside of inheritance */ + private _override(instance: InstanceRecord, field: string, curr: unknown, prev: unknown): void { + const entry = instance.config.get("instance.inherited_fields")[field]; + if (!(entry instanceof Array)) { + this.overrideField(instance, field); + return; + } + const currObj = curr as Record ?? {}; + const prevObj = prev as Record ?? {}; + const changed = Object.keys(currObj).filter(p => !isDeepStrictEqual(currObj[p], prevObj[p])); + const removed = Object.keys(prevObj).filter(p => !Object.hasOwn(currObj, p)); + const overrides = [...new Set([...entry, ...changed])].filter(p => !removed.includes(p)); + const inherited = { ...instance.config.get("instance.inherited_fields"), [field]: overrides }; + instance.config.set("instance.inherited_fields", inherited); + if (removed.length) { + // Removed properties fall back to the template value + this._resolveInto(instance, [field]); } } - private _setInherited(instance: InstanceRecord, fields: string[], inherited: boolean): void { - const value = { ...instance.config.get("instance.inherited_fields") }; - for (const field of fields) { - if (inherited) { - value[field] = true; - } else { - delete value[field]; - } - } - instance.config.set("instance.inherited_fields", value); - } - - /** Recompute inherited fields after base_id changed, values equal to the base are inherited */ - private _rebase(instance: InstanceRecord): void { - const baseId = instance.config.get("instance.base_id"); - const base = baseId !== null ? this.records.get(baseId) : undefined; - const inherited: Record = {}; - if (base) { - for (const name of Object.keys(lib.InstanceConfig.fieldDefinitions)) { - const field = name as keyof lib.InstanceConfigFields; - if ( - lib.InstanceConfig.isInheritable(name) - && isDeepStrictEqual(instance.config.get(field), base.config.get(field)) + /** Recompute inherited fields after template_id changed, values equal to the template or default are inherited */ + private _link(instance: InstanceRecord): void { + const template = this._templateOf(instance.config); + const inherited: InheritedFields = {}; + if (template) { + for (const name of InstanceManager._inheritableFields()) { + const value = instance.config.get(name) as any; + const templateValue = template.config.get(name) as any; + const defaultValue = InstanceManager._defaultValue(name) as any; + if (lib.InstanceConfig.fieldDefinitions[name].type === "object") { + inherited[name] = Object.keys(value ?? {}).filter(prop => !( + isDeepStrictEqual(value[prop], templateValue?.[prop]) + || defaultValue && isDeepStrictEqual(value[prop], defaultValue[prop]) + ) || name === "factorio.settings" && prop === "name"); + } else if ( + isDeepStrictEqual(value, templateValue) + || defaultValue !== undefined && isDeepStrictEqual(value, defaultValue) ) { inherited[name] = true; } } } instance.config.set("instance.inherited_fields", inherited); + this._resolveInto(instance, Object.keys(inherited)); } - /** Copy a changed value from a base to the instances inheriting it */ - private _propagate(base: InstanceRecord, field: string, value: unknown): void { + /** Copy a changed value from a template to the instances inheriting it */ + private _propagate(template: InstanceRecord, field: string, value: unknown): void { if (!lib.InstanceConfig.isInheritable(field)) { return; } - for (const instance of this.inheritorsOf(base.id)) { + for (const instance of this.inheritorsOf(template.id)) { if (!instance.config.isInherited(field)) { continue; } - this._inheriting += 1; - try { - instance.config.update({ [field]: structuredClone(value) as any }, true); - } finally { - this._inheriting -= 1; - } + this._resolveInto(instance, [field]); this._controller.instanceConfigUpdated(instance).catch(err => { logger.error(`Failed to send config of instance ${instance.id} to its host:\n${err.stack}`); }); @@ -320,13 +404,22 @@ export default class InstanceManager { throw new lib.ConflictError(`Instance with ID ${instanceId} already exists`); } + const controllerName = this._controller.config.get("controller.name"); + const serverName = `${controllerName} - ${instanceConfig.get("instance.name")}`; + const settings = instanceConfig.get("factorio.settings"); if (!instanceConfig.isInherited("factorio.settings")) { - const controllerName = this._controller.config.get("controller.name"); instanceConfig.set("factorio.settings", { - name: `${controllerName} - ${instanceConfig.get("instance.name")}`, + name: serverName, description: `Clusterio instance for ${controllerName}`, ...InstanceManager.DefaultFactorioSettings, - ...instanceConfig.get("factorio.settings"), + ...settings, + }); + } else if (instanceConfig.isInherited("factorio.settings", "name")) { + // Each instance gets its own server name + instanceConfig.setProp("factorio.settings", "name", serverName); + const inherited = instanceConfig.get("instance.inherited_fields"); + instanceConfig.set("instance.inherited_fields", { + ...inherited, "factorio.settings": [...inherited["factorio.settings"] as string[], "name"], }); } @@ -372,8 +465,8 @@ export default class InstanceManager { if (currentAssignedHost === hostId) { return; } - if (hostId !== null && instance.config.get("instance.is_base")) { - throw new lib.ValidationError("Base instances cannot be assigned to a host"); + if (hostId !== null && instance.config.get("instance.is_template")) { + throw new lib.ValidationError("Templates cannot be assigned to a host"); } let newHostConnection: HostConnection | undefined; @@ -443,7 +536,7 @@ export default class InstanceManager { const hostId = instance.config.get("instance.assigned_host"); const inheritors = this.inheritorsOf(instanceId).length; if (inheritors) { - throw new lib.ConflictError(`Instance is the base of ${inheritors} other instance(s)`); + throw new lib.ConflictError(`Instance is the template of ${inheritors} other instance(s)`); } if (hostId !== null) { diff --git a/packages/controller/src/InstanceRecord.ts b/packages/controller/src/InstanceRecord.ts index a441a88a..02f3aa4e 100644 --- a/packages/controller/src/InstanceRecord.ts +++ b/packages/controller/src/InstanceRecord.ts @@ -83,7 +83,8 @@ export default class InstanceRecord { this.startedAtMs, this.updatedAtMs, this.config.get("instance.exclude_from_start_all"), - this.config.get("instance.is_base"), + this.config.get("instance.is_template"), + this.config.get("instance.template_id") ?? undefined, ); } diff --git a/packages/ctl/src/commands_instance.ts b/packages/ctl/src/commands_instance.ts index c575827b..c52610c6 100644 --- a/packages/ctl/src/commands_instance.ts +++ b/packages/ctl/src/commands_instance.ts @@ -22,12 +22,12 @@ export const instanceCommands = new lib.CommandTree({ instanceCommands.add(new lib.Command({ definition: [["list", "l"], "List instances known to the controller", (yargs) => { yargs.options({ - "bases": { describe: "Include base instances", nargs: 0, type: "boolean", default: false }, + "templates": { describe: "Include templates", nargs: 0, type: "boolean", default: false }, }); }], - handler: async function(args: { bases: boolean }, ctl: Ctl) { + handler: async function(args: { templates: boolean }, ctl: Ctl) { let list = await ctl.send(new lib.InstanceDetailsListRequest()); - print(asTable(list.filter(instance => args.bases || !instance.isBase))); + print(asTable(list.filter(instance => args.templates || !instance.isTemplate))); }, })); @@ -38,12 +38,12 @@ instanceCommands.add(new lib.Command({ yargs.options({ "id": { type: "number", nargs: 1, describe: "Instance id" }, "from": { type: "number", nargs: 1, describe: "Clone config from instance id" }, - "base": { type: "string", nargs: 1, describe: "Base instance to inherit config from" }, - "is-base": { type: "boolean", nargs: 0, describe: "Create the instance as a base" }, + "template": { type: "string", nargs: 1, describe: "Template to inherit config from" }, + "is-template": { type: "boolean", nargs: 0, describe: "Create the instance as a template" }, }); }], handler: async function( - args: { name: string, id?: number, from?: number, base?: string, isBase?: boolean }, + args: { name: string, id?: number, from?: number, template?: string, isTemplate?: boolean }, ctl: Ctl, ) { let instanceConfig = new lib.InstanceConfig("control"); @@ -52,13 +52,13 @@ instanceCommands.add(new lib.Command({ instanceConfig.set("instance.id", args.id); } instanceConfig.set("instance.name", args.name); - if (args.base !== undefined) { - instanceConfig.set("instance.base_id", await lib.resolveInstance(ctl, args.base)); - fields.push("instance.base_id"); + if (args.template !== undefined) { + instanceConfig.set("instance.template_id", await lib.resolveInstance(ctl, args.template)); + fields.push("instance.template_id"); } - if (args.isBase) { - instanceConfig.set("instance.is_base", true); - fields.push("instance.is_base"); + if (args.isTemplate) { + instanceConfig.set("instance.is_template", true); + fields.push("instance.is_template"); } await ctl.send(new lib.InstanceCreateRequest( instanceConfig.toRemote("controller", fields), @@ -77,27 +77,39 @@ instanceConfigCommands.add(new lib.Command({ handler: async function(args: { instance: string }, ctl: Ctl) { let instanceId = await lib.resolveInstance(ctl, args.instance); let config = await ctl.send(new lib.InstanceConfigGetRequest(instanceId)); - const inherited = config["instance.base_id"] !== null - ? config["instance.inherited_fields"] as Record ?? {} + const inherited = config["instance.template_id"] !== null + ? config["instance.inherited_fields"] as Record ?? {} : {}; for (let [name, value] of Object.entries(config)) { if (name === "instance.inherited_fields") { continue; } - print(`${name} ${JSON.stringify(value)}${inherited[name] ? " (inherited)" : ""}`); + const entry = inherited[name]; + let note = ""; + if (entry instanceof Array && entry.length) { + note = ` (inherited except ${entry.join(", ")})`; + } else if (entry) { + note = " (inherited)"; + } + print(`${name} ${JSON.stringify(value)}${note}`); } }, })); instanceConfigCommands.add(new lib.Command({ - definition: ["inherit ", "Make field in instance config use the value of its base", (yargs) => { - yargs.positional("instance", { describe: "Instance to update config on", type: "string" }); - yargs.positional("field", { describe: "Field to inherit", type: "string" }); - }], - handler: async function(args: { instance: string, field: string }, ctl: Ctl) { + definition: [ + "inherit [prop]", + "Make field or property in instance config use the value of its template", + (yargs) => { + yargs.positional("instance", { describe: "Instance to update config on", type: "string" }); + yargs.positional("field", { describe: "Field to inherit", type: "string" }); + yargs.positional("prop", { describe: "Property of object field to inherit", type: "string" }); + }, + ], + handler: async function(args: { instance: string, field: string, prop?: string }, ctl: Ctl) { let instanceId = await lib.resolveInstance(ctl, args.instance); - await ctl.send(new lib.InstanceConfigInheritFieldRequest(instanceId, args.field)); + await ctl.send(new lib.InstanceConfigInheritFieldRequest(instanceId, args.field, args.prop)); }, })); @@ -205,7 +217,7 @@ instanceConfigCommands.add(new lib.Command({ const final = await configToKeyVal(data); const original = await configToKeyVal(allConfigElements); for (let index in final) { - // Unchanged fields are skipped so they keep inheriting from the base + // Unchanged fields are skipped so they keep inheriting from the template if (index in final && final[index] !== original[index]) { try { await ctl.send(new lib.InstanceConfigSetFieldRequest( diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 061d24dd..a76698f0 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -488,9 +488,9 @@ export interface InstanceConfigFields { "instance.assigned_host": number | null; "instance.auto_start": boolean; "instance.exclude_from_start_all": boolean; - "instance.is_base": boolean; - "instance.base_id": number | null; - "instance.inherited_fields": Record; + "instance.is_template": boolean; + "instance.template_id": number | null; + "instance.inherited_fields": Record; [key: `${string}.load_plugin`]: boolean; [key: `${string}.version`]: string; @@ -542,10 +542,13 @@ export class InstanceConfig extends classes.Config { return config; } - /** Fields that are never inherited from a base instance */ - static uninheritableFields = ["instance.name", "instance.is_base", "instance.base_id"]; + /** Fields that are never inherited from a template, ports would clash on a shared host */ + static uninheritableFields = [ + "instance.name", "instance.is_template", "instance.template_id", + "factorio.game_port", "factorio.rcon_port", "factorio.lua_udp_port", + ]; - /** True if the field can take its value from a base instance */ + /** True if the field can take its value from a template */ static isInheritable(name: string) { const def = this.fieldDefinitions[name as keyof InstanceConfigFields]; return Boolean(def) @@ -554,9 +557,21 @@ export class InstanceConfig extends classes.Config { && !this.uninheritableFields.includes(name); } - /** True if the field currently takes its value from the base instance */ - isInherited(name: string) { - return this.get("instance.base_id") !== null && Boolean(this.get("instance.inherited_fields")[name]); + /** + * True if the field, or the given property of an object field, takes its value from the template + * + * @param name - Name of field to check. + * @param prop - Property of an object field to check. + */ + isInherited(name: string, prop?: string) { + if (this.get("instance.template_id") === null) { + return false; + } + const entry = this.get("instance.inherited_fields")[name]; + if (entry instanceof Array) { + return prop === undefined || !entry.includes(prop); + } + return Boolean(entry); } static fieldDefinitions: classes.ConfigDefs = { @@ -586,24 +601,26 @@ export class InstanceConfig extends classes.Config { type: "boolean", initialValue: false, }, - "instance.is_base": { + "instance.is_template": { description: - "Mark this instance as a base other instances can inherit config from. " + - "Bases cannot be assigned to a host or started.", + "Mark this instance as a template other instances can inherit config from. " + + "Templates cannot be assigned to a host or started.", access: ["controller", "control"], type: "boolean", initialValue: false, }, - "instance.base_id": { - title: "Base Instance", - description: "ID of the base instance to inherit config values from", + "instance.template_id": { + title: "Template", + description: "ID of the template to inherit config values from", access: ["controller", "control"], + inputComponent: "instance_template", type: "number", optional: true, // Checked by the controller }, "instance.inherited_fields": { - description: "Fields whose value comes from the base instance", + description: + "Fields whose value comes from the template, for object fields the properties set on this instance", access: ["controller", "control"], readonly: ["controller"], type: "object", diff --git a/packages/lib/src/data/InstanceDetails.ts b/packages/lib/src/data/InstanceDetails.ts index ce008c9b..cad0761d 100644 --- a/packages/lib/src/data/InstanceDetails.ts +++ b/packages/lib/src/data/InstanceDetails.ts @@ -42,7 +42,8 @@ export default class InstanceDetails { /** Millisecond Unix timestamp this entry was last updated at */ public updatedAtMs = 0, public excludeFromStartAll = false, - public isBase = false, + public isTemplate = false, + public templateId?: number, ) {} get isDeleted() { @@ -62,7 +63,8 @@ export default class InstanceDetails { "startedAtMs": Type.Optional(Type.Number()), "updatedAtMs": Type.Optional(Type.Number()), "excludeFromStartAll": Type.Optional(Type.Boolean()), - "isBase": Type.Optional(Type.Boolean()), + "isTemplate": Type.Optional(Type.Boolean()), + "templateId": Type.Optional(Type.Integer()), }); static fromJSON(json: Static) { @@ -76,7 +78,8 @@ export default class InstanceDetails { json.startedAtMs, json.updatedAtMs, json.excludeFromStartAll, - json.isBase, + json.isTemplate, + json.templateId, ); } @@ -105,8 +108,11 @@ export default class InstanceDetails { if (this.excludeFromStartAll !== false) { json.excludeFromStartAll = this.excludeFromStartAll; } - if (this.isBase !== false) { - json.isBase = this.isBase; + if (this.isTemplate !== false) { + json.isTemplate = this.isTemplate; + } + if (this.templateId !== undefined) { + json.templateId = this.templateId; } return json; diff --git a/packages/lib/src/data/messages_instance.ts b/packages/lib/src/data/messages_instance.ts index 5cde39b7..77500fab 100644 --- a/packages/lib/src/data/messages_instance.ts +++ b/packages/lib/src/data/messages_instance.ts @@ -186,15 +186,17 @@ export class InstanceConfigInheritFieldRequest { constructor( public instanceId: number, public field: string, + public prop?: string, ) { } static jsonSchema = Type.Object({ "instanceId": Type.Integer(), "field": Type.String(), + "prop": Type.Optional(Type.String()), }); static fromJSON(json: Static) { - return new this(json.instanceId, json.field); + return new this(json.instanceId, json.field, json.prop); } } diff --git a/packages/web_ui/src/bootstrap.tsx b/packages/web_ui/src/bootstrap.tsx index d3e9f739..b87d4852 100644 --- a/packages/web_ui/src/bootstrap.tsx +++ b/packages/web_ui/src/bootstrap.tsx @@ -8,6 +8,7 @@ import * as lib from "@clusterio/lib"; import App from "./components/App"; import InputRole from "./components/InputRole"; import InputModPack from "./components/InputModPack"; +import { InputInstanceTemplate } from "./components/InputInstance"; import { InputTargetVersion, InputPartialVersion, InputFullVersion } from "./components/InputVersion"; import { Control, ControlConnector } from "./util/websocket"; import { loadedPluginSetKey } from "./util/pluginSet"; @@ -148,6 +149,7 @@ async function inputComponentsFromHooks(control: Control) { "target_version": InputTargetVersion, "mod_pack": InputModPack, "role": InputRole, + "instance_template": InputInstanceTemplate, }]); const iterable = result.map( diff --git a/packages/web_ui/src/components/BaseConfigTree.tsx b/packages/web_ui/src/components/BaseConfigTree.tsx index 058487ce..489bdf7e 100644 --- a/packages/web_ui/src/components/BaseConfigTree.tsx +++ b/packages/web_ui/src/components/BaseConfigTree.tsx @@ -1,8 +1,11 @@ import React, { useContext, useEffect, useState } from "react"; import { - Button, Card, Checkbox, Form, FormInstance, Input, InputNumber, Select, Space, Spin, Tooltip, Tree, Typography, + Button, Card, Checkbox, Form, FormInstance, Input, InputNumber, Popconfirm, Select, Space, Spin, Tooltip, Tree, + Typography, } from "antd"; +import LockOutlined from "@ant-design/icons/LockOutlined"; import ReloadOutlined from "@ant-design/icons/ReloadOutlined"; +import UnlockOutlined from "@ant-design/icons/UnlockOutlined"; import * as lib from "@clusterio/lib"; @@ -65,8 +68,24 @@ function renderInput(inputComponents: Map, def: lib.Fiel return `Unknown type ${def.type}`; } +/** Lets fields follow the value of another config, inherited values are locked until unlocked */ +export type ConfigInheritance = { + canInherit: (config: lib.Config, field: string) => boolean; + isInherited: (config: lib.Config, field: string, prop?: string) => boolean; + inherit: (field: string, prop?: string) => Promise; + notice?: (config: lib.Config) => React.ReactNode; +}; + +type LockState = { + inheritance?: ConfigInheritance; + unlocked: Set; + setUnlocked: (unlocked: Set) => void; + inherit: (field: string, prop?: string) => void; +}; + type BaseConfigTreeProps = { ConfigClass: typeof lib.ControllerConfig | typeof lib.HostConfig | typeof lib.InstanceConfig; + inheritance?: ConfigInheritance; retrieveConfig: () => Promise; setConfig: (fields: Record>) => Promise; id?: number; @@ -79,6 +98,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { let [changedFields, setChangedFields] = useState>(new Set()); let [errorFields, setErrorFields] = useState(new Map()); let [applying, setApplying] = useState(false); + let [unlocked, setUnlocked] = useState>(new Set()); const available = props.available ?? true; const control = useContext(ControlContext); @@ -136,6 +156,17 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { } } + function inherit(field: string, prop?: string) { + props.inheritance!.inherit(field, prop).then(async () => { + const newConfig = await updateConfig(); + const values = getInitialValues(newConfig, props); + const keys = Object.keys(values).filter(key => key === field || key.startsWith(`${field}.`)); + form.resetFields(keys); + form.setFieldsValue(Object.fromEntries(keys.map(key => [key, values[key]]))); + setChangedFields(fields => new Set([...fields].filter(key => !keys.includes(key)))); + }).catch(err => notify(err.message, "error")); + } + const [propsMap, treeData] = computeTreeData( control, form, @@ -145,6 +176,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { setConfig, props.ConfigClass, onValuesChange, + { inheritance: props.inheritance, unlocked, setUnlocked, inherit }, ); return <> @@ -179,14 +211,21 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { } } + let applied = false; try { await props.setConfig(fields); setChangedFields(new Set()); + setUnlocked(new Set()); + applied = true; } catch (err: any) { notify(err.message, "error"); } - await updateConfig(); + const newConfig = await updateConfig(); + if (applied) { + // Linking a template can change other values + form.setFieldsValue(getInitialValues(newConfig, props)); + } setApplying(false); if (props.onApply) { @@ -202,6 +241,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { onClick={() => { form.resetFields(); setChangedFields(new Set()); + setUnlocked(new Set()); setErrorFields(new Map()); }} > @@ -210,6 +250,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { + {props.inheritance?.notice?.(config)}
) => void, ConfigClass: BaseConfigTreeProps["ConfigClass"], onValuesChange: (changedValues: object) => void, + lock: LockState, ) { let restartTip = + } + ; + } + return { account.hasAnyPermission(...InstanceControlButtonPermissions) @@ -190,6 +236,12 @@ function InstanceButtons(props: { instance: lib.InstanceDetails }) { Connect via Steam {account.hasPermission("core.instance.load_scenario") && } + {canCreate && } {account.hasAnyPermission( "core.instance.export_data", "core.instance.extract_players", @@ -224,11 +276,16 @@ export default function InstanceViewPage() { return } + status={instance.isTemplate + ? Template + : } extra={} /> - + {instance.isTemplate + ? + : } + {!instance.isTemplate && <> {account.hasPermission("core.user.list") && } @@ -275,6 +332,7 @@ export default function InstanceViewPage() { account.hasPermission("core.instance.send_rcon") && } + } {account.hasPermission("core.instance.get_config") && } diff --git a/packages/web_ui/src/components/InstancesPage.tsx b/packages/web_ui/src/components/InstancesPage.tsx index ed1498a2..9e6a7d54 100644 --- a/packages/web_ui/src/components/InstancesPage.tsx +++ b/packages/web_ui/src/components/InstancesPage.tsx @@ -14,12 +14,85 @@ import { useInstanceConfig, useInstances } from "../model/instance"; import { useSaves } from "../model/saves"; import InstanceList from "./InstanceList"; import { notifyErrorHandler } from "../util/notify"; +import type { Control } from "../util/websocket"; type SaveMode = "none" | "copy" | "save_and_copy"; const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare; -function CreateInstanceButton(props: { instances: ReturnType[0] }) { +function ConfigSourceFields(props: { + instances: ReturnType[0]; + templateId?: number | null; + source?: Readonly; + asTemplate: boolean; +}) { + const templates = [...props.instances.values()].filter(i => i.isTemplate); + const hasTemplate = typeof props.templateId === "number"; + return <> + {(templates.length > 0 || hasTemplate) && + + !i.isTemplate), + ].map(i => ({ value: i.id, label: i.name }))} + /> + + + {props.asTemplate && props.source && + Make {props.source.name} inherit from the new template + } + ; +} + +async function linkToTemplate(control: Control, source: lib.InstanceDetails | undefined, templateId: number) { + if (source) { + await control.send(new lib.InstanceConfigSetFieldRequest( + source.id, "instance.template_id", String(templateId), + )); + } +} + +function setTemplateFields( + config: lib.InstanceConfig, + fields: (keyof lib.InstanceConfigFields)[], + templateId: unknown, + asTemplate: boolean, +) { + if (typeof templateId === "number") { + config.set("instance.template_id", templateId); + fields.push("instance.template_id"); + } + if (asTemplate) { + config.set("instance.is_template", true); + fields.push("instance.is_template"); + } +} + +type CreateInstanceInitial = { + instanceClone?: number; + template?: number; + asTemplate?: boolean; + linkSource?: boolean; +}; + +export function CreateInstanceButton({ buttonText = "Create", buttonType = "primary", ...props }: { + instances: ReturnType[0]; + initial?: CreateInstanceInitial; + buttonText?: string; + buttonType?: "primary" | "default"; +}) { let control = useContext(ControlContext); let account = useAccount(); let navigate = useNavigate(); @@ -31,9 +104,11 @@ function CreateInstanceButton(props: { instances: ReturnType= 0 ? props.instances.get(cloneId) : undefined; const sourceRunning = source?.status === "running"; const activeSave = source ? [...saves.values()].find( @@ -42,6 +117,7 @@ function CreateInstanceButton(props: { instances: ReturnType | undefined; const sourceServerName = typeof sourceSettings?.name === "string" ? sourceSettings.name : undefined; + const copySettings = sourceSettings !== undefined && !asTemplate; useEffect(() => { if (open && sourceServerName !== undefined) { @@ -68,7 +144,8 @@ function CreateInstanceButton(props: { instances: ReturnType + >{buttonText} - {canAssign && + + + Create as template + + + {canAssign && !asTemplate && } - - - @@ -198,6 +285,8 @@ export default function InstancesPage() { let control = useContext(ControlContext); let account = useAccount(); let [instances] = useInstances(); + const plainInstances = new Map([...instances].filter(([, instance]) => !instance.isTemplate)); + const templates = new Map([...instances].filter(([, instance]) => instance.isTemplate)); return } /> - + + {templates.size > 0 && <> + Templates + + } ; } diff --git a/test/controller/ControlConnection.test.js b/test/controller/ControlConnection.test.js index e4d9efde..845c56fb 100644 --- a/test/controller/ControlConnection.test.js +++ b/test/controller/ControlConnection.test.js @@ -157,22 +157,22 @@ describe("controller/src/ControlConnection", function() { assert.equal(clone.config.get("instance.assigned_host"), null); }); - it("creates an instance inheriting from a base", async function() { - await create({ "instance.id": 4001, "instance.name": "base", "instance.is_base": true }); + it("creates an instance inheriting from a template", async function() { + await create({ "instance.id": 4001, "instance.name": "template", "instance.is_template": true }); mockController.instances.getMutable(4001).config.set("factorio.version", "2.0"); - await create({ "instance.id": 4002, "instance.name": "child", "instance.base_id": 4001 }); + await create({ "instance.id": 4002, "instance.name": "child", "instance.template_id": 4001 }); const child = mockController.instances.get(4002); assert.equal(child.config.get("factorio.version"), "2.0"); assert.equal(child.config.isInherited("factorio.version"), true); assert.equal(child.config.isInherited("instance.name"), false); }); - it("rejects a base that is not a base", async function() { + it("rejects a template that is not a template", async function() { await create({ "instance.id": 4001, "instance.name": "plain" }); await assert.rejects( - create({ "instance.id": 4002, "instance.name": "child", "instance.base_id": 4001 }), - new lib.ValidationError("Instance plain is not a base"), + create({ "instance.id": 4002, "instance.name": "child", "instance.template_id": 4001 }), + new lib.ValidationError("Instance plain is not a template"), ); assert.equal(mockController.instances.has(4002), false); }); @@ -192,14 +192,16 @@ describe("controller/src/ControlConnection", function() { checkStagedInheritance: ControlConnection.prototype.checkStagedInheritance, }; for (const [id, fields] of [ - [1, { "instance.is_base": true, "factorio.version": "2.0" }], - [2, { "instance.base_id": 1 }], + [1, { "instance.is_template": true, "factorio.version": "2.0" }], + [2, { "instance.template_id": 1 }], ]) { const config = new lib.InstanceConfig("controller", { "instance.id": id, "instance.name": `i${id}`, ...fields, }); - if (fields["instance.base_id"]) { - mockController.instances.applyBase(config, ["instance.id", "instance.name", "instance.base_id"]); + if (fields["instance.template_id"]) { + mockController.instances.applyTemplate( + config, ["instance.id", "instance.name", "instance.template_id"], + ); } await mockController.instances.createInstance(config); } @@ -208,6 +210,11 @@ describe("controller/src/ControlConnection", function() { function call(handler, request) { return ControlConnection.prototype[handler].call(connection, request); } + async function create(config, cloneFromId) { + await ControlConnection.prototype.handleInstanceCreateRequest.call( + connection, new lib.InstanceCreateRequest(config, cloneFromId), + ); + } const setField = (...args) => new lib.InstanceConfigSetFieldRequest(...args); const setFields = (...args) => new lib.InstanceConfigSetRequest(...args); @@ -221,28 +228,58 @@ describe("controller/src/ControlConnection", function() { assert.equal(mockController.instances.get(2).config.isInherited("factorio.version"), false); }); - it("stops inheriting a field changed through set-prop", async function() { + it("stops inheriting a property set through set-prop", async function() { await call( "handleInstanceConfigSetPropRequest", new lib.InstanceConfigSetPropRequest(2, "factorio.settings", "max_players", 0), ); - assert.equal(mockController.instances.get(2).config.isInherited("factorio.settings"), false); + const config = mockController.instances.get(2).config; + assert.equal(config.isInherited("factorio.settings", "max_players"), false); + assert.equal(config.isInherited("factorio.settings", "auto_pause"), true); }); - it("rejects base_id pointing at a non-base", async function() { + it("rejects template_id pointing at a non-template", async function() { await assert.rejects( - call("handleInstanceConfigSetFieldRequest", setField(1, "instance.base_id", "2")), - new lib.ValidationError("Instance i2 is not a base"), + call("handleInstanceConfigSetFieldRequest", setField(1, "instance.template_id", "2")), + new lib.ValidationError("Instance i2 is not a template"), ); - assert.equal(mockController.instances.get(1).config.get("instance.base_id"), null); + assert.equal(mockController.instances.get(1).config.get("instance.template_id"), null); }); - it("rejects unmarking a base with inheritors through set request", async function() { + it("rejects unmarking a template with inheritors through set request", async function() { await assert.rejects( - call("handleInstanceConfigSetRequest", setFields(1, { "instance.is_base": "false" })), - new lib.ConflictError("Instance is the base of 1 other instance(s)"), + call("handleInstanceConfigSetRequest", setFields(1, { "instance.is_template": "false" })), + new lib.ConflictError("Instance is the template of 1 other instance(s)"), ); - assert.equal(mockController.instances.get(1).config.get("instance.is_base"), true); + assert.equal(mockController.instances.get(1).config.get("instance.is_template"), true); + }); + + it("stops inheriting only the properties set through set request", async function() { + await call("handleInstanceConfigSetRequest", setFields(2, { "factorio.settings": { max_players: 3 } })); + const config = mockController.instances.get(2).config; + assert.equal(config.isInherited("factorio.settings", "max_players"), false); + assert.equal(config.isInherited("factorio.settings", "auto_pause"), true); + }); + + it("inherits a property again", async function() { + await call("handleInstanceConfigSetPropRequest", new lib.InstanceConfigSetPropRequest( + 2, "factorio.settings", "max_players", 3, + )); + await call( + "handleInstanceConfigInheritFieldRequest", + new lib.InstanceConfigInheritFieldRequest(2, "factorio.settings", "max_players"), + ); + const config = mockController.instances.get(2).config; + assert.equal(config.get("factorio.settings").max_players, 0); + assert.equal(config.isInherited("factorio.settings", "max_players"), true); + }); + + it("creates a template from an existing instance", async function() { + await create({ "instance.id": 3, "instance.name": "copy", "instance.is_template": true }, 2); + const config = mockController.instances.get(3).config; + assert.equal(config.get("instance.is_template"), true); + assert.equal(config.get("instance.template_id"), 1); + assert.equal(config.get("factorio.version"), "2.0"); }); it("inherits a field again", async function() { diff --git a/test/controller/InstanceManager.test.js b/test/controller/InstanceManager.test.js index fc6e262b..26e26385 100644 --- a/test/controller/InstanceManager.test.js +++ b/test/controller/InstanceManager.test.js @@ -396,67 +396,94 @@ describe("controller/InstanceManager", function () { describe("inheritance", function () { /** @type {InstanceRecord} */ - let base; + let template; /** @type {InstanceRecord} */ let child; let updated; - async function createInheritor(id, baseId, fields = {}) { + async function createInheritor(id, templateId, fields = {}) { const config = new lib.InstanceConfig("controller", { - "instance.id": id, "instance.name": `Instance ${id}`, "instance.base_id": baseId, ...fields, + "instance.id": id, "instance.name": `Instance ${id}`, "instance.template_id": templateId, ...fields, }); - instances.applyBase(config, ["instance.id", "instance.name", "instance.base_id", ...Object.keys(fields)]); + instances.applyTemplate( + config, ["instance.id", "instance.name", "instance.template_id", ...Object.keys(fields)] + ); return await instances.createInstance(config); } beforeEach(async function () { - base = await instances.createInstance(new lib.InstanceConfig("controller", { + template = await instances.createInstance(new lib.InstanceConfig("controller", { "instance.id": 10, - "instance.name": "Base", - "instance.is_base": true, + "instance.name": "Template", + "instance.is_template": true, "factorio.version": "2.0", - "factorio.settings": { name: "Base server" }, + "factorio.game_port": 34190, + "factorio.settings": { name: "Template server", max_players: 5 }, })); child = await createInheritor(11, 10, { "factorio.game_port": 34197 }); updated = []; controller.instanceConfigUpdated = async (inst) => { updated.push(inst.id); }; }); - describe(".applyBase()", function () { - it("should copy base values and mark them inherited", function () { + describe(".applyTemplate()", function () { + it("should copy template values and mark them inherited", function () { assert.equal(child.config.get("factorio.version"), "2.0"); assert.equal(child.config.isInherited("factorio.version"), true); assert.equal(child.config.get("factorio.game_port"), 34197); assert.equal(child.config.isInherited("factorio.game_port"), false); assert.equal(child.config.isInherited("instance.name"), false); }); - it("should keep inherited factorio.settings in createInstance", function () { - assert.deepEqual(child.config.get("factorio.settings"), base.config.get("factorio.settings")); - assert.notEqual(child.config.get("factorio.settings"), base.config.get("factorio.settings")); + it("should never inherit ports", function () { + const config = new lib.InstanceConfig("controller", { "instance.id": 12, "instance.template_id": 10 }); + instances.applyTemplate(config, ["instance.id", "instance.template_id"]); + assert.equal(config.get("factorio.game_port"), null); + assert.equal(config.isInherited("factorio.game_port"), false); + }); + it("should inherit factorio.settings per property with its own server name", function () { + const settings = child.config.get("factorio.settings"); + assert.equal(settings.max_players, 5); + assert.equal(settings.name, "TestController - Instance 11"); + assert.equal(child.config.isInherited("factorio.settings", "max_players"), true); + assert.equal(child.config.isInherited("factorio.settings", "name"), false); + }); + it("should keep properties given by the creator as overrides", async function () { + const other = await createInheritor(12, 10, { "factorio.settings": { max_players: 8 } }); + assert.equal(other.config.get("factorio.settings").max_players, 8); + assert.equal(other.config.isInherited("factorio.settings", "max_players"), false); + assert.equal(other.config.isInherited("factorio.settings", "auto_pause"), true); }); }); - describe("base changes", function () { + describe("template changes", function () { it("should propagate to inherited fields and send the config to the host", function () { - base.config.set("factorio.version", "2.1"); + template.config.set("factorio.version", "2.1"); assert.equal(child.config.get("factorio.version"), "2.1"); assert.equal(child.config.isInherited("factorio.version"), true); assert.deepEqual(updated, [11]); }); it("should mark the inheritor as requiring restart", function () { child.config.restartRequired = false; - base.config.set("factorio.version", "2.1"); + template.config.set("factorio.version", "2.1"); assert.equal(child.config.restartRequired, true); }); it("should not touch fields the inheritor has set", function () { - base.config.set("factorio.game_port", 1234); - assert.equal(child.config.get("factorio.game_port"), 34197); - assert.deepEqual(updated, []); + child.config.set("factorio.version", "2.1"); + template.config.set("factorio.version", "2.2"); + assert.equal(child.config.get("factorio.version"), "2.1"); + }); + it("should merge object properties and keep overrides", function () { + template.config.setProp("factorio.settings", "max_players", 10); + template.config.setProp("factorio.settings", "name", "Renamed"); + template.config.setProp("factorio.settings", "auto_pause", true); + const settings = child.config.get("factorio.settings"); + assert.equal(settings.max_players, 10); + assert.equal(settings.auto_pause, true); + assert.equal(settings.name, "TestController - Instance 11"); }); it("should propagate through several levels", async function () { - child.config.set("instance.is_base", true); + child.config.set("instance.is_template", true); const grandchild = await createInheritor(12, 11); - base.config.setProp("factorio.settings", "max_players", 10); + template.config.setProp("factorio.settings", "max_players", 10); assert.equal(grandchild.config.get("factorio.settings").max_players, 10); assert.deepEqual(updated.sort(), [11, 12]); }); @@ -466,26 +493,47 @@ describe("controller/InstanceManager", function () { it("should stop inheriting a field that is changed", function () { child.config.set("factorio.version", "2.1"); assert.equal(child.config.isInherited("factorio.version"), false); - base.config.set("factorio.version", "2.0.1"); - assert.equal(child.config.get("factorio.version"), "2.1"); }); - it("should stop inheriting with overrideFields() even if the value is unchanged", function () { - instances.overrideFields(child, ["factorio.version", "factorio.game_port"]); + it("should only override the properties changed in an object", function () { + child.config.setProp("factorio.settings", "max_players", 3); + assert.equal(child.config.isInherited("factorio.settings", "max_players"), false); + assert.equal(child.config.isInherited("factorio.settings", "auto_pause"), true); + }); + it("should restore the template value of a removed property", function () { + child.config.setProp("factorio.settings", "max_players", 3); + child.config.setProp("factorio.settings", "max_players"); + assert.equal(child.config.get("factorio.settings").max_players, 5); + assert.equal(child.config.isInherited("factorio.settings", "max_players"), true); + }); + it("should stop inheriting with overrideField() even if the value is unchanged", function () { + instances.overrideField(child, "factorio.version"); + instances.overrideField(child, "factorio.settings", ["auto_pause"]); assert.equal(child.config.isInherited("factorio.version"), false); + assert.equal(child.config.isInherited("factorio.settings", "auto_pause"), false); }); }); describe(".inheritField()", function () { - it("should restore the base value", function () { + it("should restore the template value", function () { child.config.set("factorio.version", "2.1"); instances.inheritField(11, "factorio.version"); assert.equal(child.config.get("factorio.version"), "2.0"); assert.equal(child.config.isInherited("factorio.version"), true); }); - it("should throw if the instance has no base", function () { + it("should restore a single property", function () { + child.config.setProp("factorio.settings", "max_players", 3); + instances.inheritField(11, "factorio.settings", "max_players"); + assert.equal(child.config.get("factorio.settings").max_players, 5); + assert.equal(child.config.get("factorio.settings").name, "TestController - Instance 11"); + }); + it("should restore all properties", function () { + instances.inheritField(11, "factorio.settings"); + assert.equal(child.config.get("factorio.settings").name, "Template server"); + }); + it("should throw if the instance has no template", function () { assert.throws( () => instances.inheritField(1, "factorio.version"), - new lib.ValidationError("Instance does not have a base"), + new lib.ValidationError("Instance does not have a template"), ); }); it("should throw for fields that cannot be inherited", function () { @@ -493,79 +541,102 @@ describe("controller/InstanceManager", function () { () => instances.inheritField(11, "instance.name"), new lib.ValidationError("Field instance.name cannot be inherited"), ); + assert.throws( + () => instances.inheritField(11, "factorio.rcon_port"), + new lib.ValidationError("Field factorio.rcon_port cannot be inherited"), + ); }); }); - describe("base_id changes", function () { - it("should inherit fields equal to the new base", function () { + describe("template_id changes", function () { + it("should inherit fields equal to the template or the default", function () { instance.config.set("factorio.version", "2.0"); - instance.config.set("instance.base_id", 10); + instance.config.set("factorio.verbose_logging", true); + template.config.set("instance.auto_start", true); + instance.config.set("instance.template_id", 10); assert.equal(instance.config.isInherited("factorio.version"), true); - assert.equal(instance.config.isInherited("factorio.settings"), false); + assert.equal(instance.config.isInherited("instance.auto_start"), true); + assert.equal(instance.config.get("instance.auto_start"), true); + assert.equal(instance.config.isInherited("factorio.verbose_logging"), false); }); - it("should clear inherited fields when the base is removed", function () { - child.config.set("instance.base_id", null); + it("should inherit object properties equal to the template or the default", function () { + instance.config.setProp("factorio.settings", "max_players", 0); + instance.config.setProp("factorio.settings", "game_password", "secret"); + instance.config.set("instance.template_id", 10); + const settings = instance.config.get("factorio.settings"); + assert.equal(settings.max_players, 5); + assert.equal(settings.game_password, "secret"); + assert.equal(instance.config.isInherited("factorio.settings", "game_password"), false); + assert.equal(instance.config.isInherited("factorio.settings", "max_players"), true); + }); + it("should keep the server name of the instance", function () { + instance.config.setProp("factorio.settings", "name", "Template server"); + instance.config.set("instance.template_id", 10); + assert.equal(instance.config.isInherited("factorio.settings", "name"), false); + }); + it("should clear inherited fields when the template is removed", function () { + child.config.set("instance.template_id", null); assert.deepEqual(child.config.get("instance.inherited_fields"), {}); assert.equal(child.config.get("factorio.version"), "2.0"); }); }); describe(".checkInheritance()", function () { - it("should accept a valid base", function () { + it("should accept a valid template", function () { instances.checkInheritance(instance.config, false, 10); }); - it("should reject a missing base", function () { + it("should reject a missing template", function () { assert.throws( () => instances.checkInheritance(instance.config, false, 99), new lib.NotFoundError("Instance with ID 99 does not exist"), ); }); - it("should reject an instance that is not a base", function () { + it("should reject an instance that is not a template", function () { assert.throws( () => instances.checkInheritance(instance.config, false, 11), - new lib.ValidationError("Instance Instance 11 is not a base"), + new lib.ValidationError("Instance Instance 11 is not a template"), ); }); it("should reject cycles", function () { - child.config.set("instance.is_base", true); + child.config.set("instance.is_template", true); assert.throws( - () => instances.checkInheritance(base.config, true, 11), - new lib.ValidationError("Base instances cannot inherit from themselves"), + () => instances.checkInheritance(template.config, true, 11), + new lib.ValidationError("Templates cannot inherit from themselves"), ); }); - it("should reject making an assigned instance a base", function () { + it("should reject making an assigned instance a template", function () { instance.config.set("instance.assigned_host", 5); assert.throws( () => instances.checkInheritance(instance.config, true, null), - new lib.ConflictError("Instance must be unassigned before it can be made a base"), + new lib.ConflictError("Instance must be unassigned before it can be made a template"), ); }); - it("should reject unmarking a base that has inheritors", function () { + it("should reject unmarking a template that has inheritors", function () { assert.throws( - () => instances.checkInheritance(base.config, false, null), - new lib.ConflictError("Instance is the base of 1 other instance(s)"), + () => instances.checkInheritance(template.config, false, null), + new lib.ConflictError("Instance is the template of 1 other instance(s)"), ); }); }); - it("should not assign a base to a host", async function () { + it("should not assign a template to a host", async function () { controller.wsServer.hostConnections.set(5, { send: async () => {}, connector: { closing: false } }); await assert.rejects( instances.assignInstance(10, 5), - new lib.ValidationError("Base instances cannot be assigned to a host"), + new lib.ValidationError("Templates cannot be assigned to a host"), ); }); - it("should not delete a base that has inheritors", async function () { + it("should not delete a template that has inheritors", async function () { await assert.rejects( instances.deleteInstance(10), - new lib.ConflictError("Instance is the base of 1 other instance(s)"), + new lib.ConflictError("Instance is the template of 1 other instance(s)"), ); }); - it("should report bases in instance details", function () { - assert.equal(base.toInstanceDetails().isBase, true); - assert.equal(child.toInstanceDetails().isBase, false); + it("should report templates in instance details", function () { + assert.equal(template.toInstanceDetails().isTemplate, true); + assert.equal(child.toInstanceDetails().isTemplate, false); }); }); }); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index d25fc29b..3f0752df 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -971,42 +971,53 @@ describe("Integration of Clusterio", function() { }); }); - describe("instance base inheritance", function() { + describe("instance template inheritance", function() { after(async function() { - for (const id of [4402, 4401, 4400]) { + for (const id of [4403, 4402, 4401, 4400]) { await getControl().send(new lib.InstanceDeleteRequest(id)).catch(() => {}); } }); - it("creates a base and an instance inheriting from it", async function() { - await execCtl("instance create inheritBase --id 4400 --is-base"); - await execCtl("instance config set inheritBase factorio.version 2.0"); - await execCtl("instance create inheritChild --id 4401 --base inheritBase"); + it("creates a template and an instance inheriting from it", async function() { + await execCtl("instance create inheritTemplate --id 4400 --is-template"); + await execCtl("instance config set inheritTemplate factorio.version 2.0"); + await execCtl("instance create inheritChild --id 4401 --template inheritTemplate"); const config = await getControl().send(new lib.InstanceConfigGetRequest(4401)); - assert.equal(config["instance.base_id"], 4400); + assert.equal(config["instance.template_id"], 4400); assert.equal(config["factorio.version"], "2.0"); assert.equal(config["instance.inherited_fields"]["factorio.version"], true); - assert.equal((await getInstances()).get(4400).isBase, true); + assert.equal((await getInstances()).get(4400).isTemplate, true); }); - it("propagates base changes", async function() { - await execCtl("instance config set inheritBase factorio.version 2.1"); + it("propagates template changes per property", async function() { + await execCtl("instance config set inheritTemplate factorio.version 2.1"); + await execCtl("instance config set-prop inheritChild factorio.settings max_players 3"); + await execCtl("instance config set-prop inheritTemplate factorio.settings max_players 7"); + await execCtl("instance config set-prop inheritTemplate factorio.settings afk_autokick_interval 9"); const config = await getControl().send(new lib.InstanceConfigGetRequest(4401)); assert.equal(config["factorio.version"], "2.1"); + assert.equal(config["factorio.settings"].max_players, 3); + assert.equal(config["factorio.settings"].afk_autokick_interval, 9); }); - it("rejects inheriting from an instance that is not a base", async function() { + it("creates a template from an existing instance", async function() { + await execCtl("instance create inheritCopy --id 4403 --is-template --from 4401"); + const config = await getControl().send(new lib.InstanceConfigGetRequest(4403)); + assert.equal(config["instance.is_template"], true); + assert.equal(config["factorio.settings"].max_players, 3); + }); + it("rejects inheriting from an instance that is not a template", async function() { await assert.rejects( - execCtl("instance create inheritBad --id 4402 --base inheritChild"), - new lib.ValidationError("Instance inheritChild is not a base"), + execCtl("instance create inheritBad --id 4402 --template inheritChild"), + new lib.ValidationError("Instance inheritChild is not a template"), ); }); - it("rejects assigning a base to a host", async function() { + it("rejects assigning a template to a host", async function() { await assert.rejects( - execCtl("instance assign inheritBase 4"), - new lib.ValidationError("Base instances cannot be assigned to a host"), + execCtl("instance assign inheritTemplate 4"), + new lib.ValidationError("Templates cannot be assigned to a host"), ); }); - describe("instance list --bases", function() { + describe("instance list --templates", function() { it("runs", async function() { - await execCtl("instance list --bases"); + await execCtl("instance list --templates"); }); }); describe("instance config list", function() { @@ -1021,9 +1032,11 @@ describe("Integration of Clusterio", function() { assert.equal(config["instance.inherited_fields"]["factorio.version"], undefined); await execCtl("instance config inherit inheritChild factorio.version"); + await execCtl("instance config inherit inheritChild factorio.settings max_players"); config = await getControl().send(new lib.InstanceConfigGetRequest(4401)); assert.equal(config["factorio.version"], "2.1"); assert.equal(config["instance.inherited_fields"]["factorio.version"], true); + assert.equal(config["factorio.settings"].max_players, 7); }); }); }); diff --git a/test/lib/config/definitions.js b/test/lib/config/definitions.js index 33cc097e..632e10b1 100644 --- a/test/lib/config/definitions.js +++ b/test/lib/config/definitions.js @@ -12,28 +12,38 @@ describe("lib/config/definitions", function() { }); it("should reject identity, hidden, host only and unknown fields", function() { for (const name of [ - "instance.name", "instance.id", "instance.assigned_host", "instance.is_base", - "instance.base_id", "instance.inherited_fields", "factorio.host_assigned_game_port", "no.field", + "instance.name", "instance.id", "instance.assigned_host", "instance.is_template", + "instance.template_id", "instance.inherited_fields", "factorio.host_assigned_game_port", + "factorio.game_port", "factorio.rcon_port", "factorio.lua_udp_port", "no.field", ]) { assert.equal(lib.InstanceConfig.isInheritable(name), false, name); } }); }); describe(".isInherited()", function() { - it("should only report fields as inherited when a base is set", function() { + it("should only report fields as inherited when a template is set", function() { const config = new lib.InstanceConfig("controller", { "instance.inherited_fields": { "factorio.version": true }, }); assert.equal(config.isInherited("factorio.version"), false); - config.set("instance.base_id", 1); + config.set("instance.template_id", 1); assert.equal(config.isInherited("factorio.version"), true); assert.equal(config.isInherited("factorio.game_port"), false); }); + it("should report object properties not overridden as inherited", function() { + const config = new lib.InstanceConfig("controller", { + "instance.template_id": 1, + "instance.inherited_fields": { "factorio.settings": ["name"] }, + }); + assert.equal(config.isInherited("factorio.settings"), true); + assert.equal(config.isInherited("factorio.settings", "max_players"), true); + assert.equal(config.isInherited("factorio.settings", "name"), false); + }); }); it("should not send inheritance fields to the host", function() { - const config = new lib.InstanceConfig("controller", { "instance.base_id": 1 }); + const config = new lib.InstanceConfig("controller", { "instance.template_id": 1 }); const remote = config.toRemote("host"); - for (const name of ["instance.is_base", "instance.base_id", "instance.inherited_fields"]) { + for (const name of ["instance.is_template", "instance.template_id", "instance.inherited_fields"]) { assert(!Object.hasOwn(remote, name), name); } }); diff --git a/test/lib/data/InstanceDetails.test.js b/test/lib/data/InstanceDetails.test.js index 95058653..2cec180c 100644 --- a/test/lib/data/InstanceDetails.test.js +++ b/test/lib/data/InstanceDetails.test.js @@ -16,6 +16,8 @@ describe("lib/data/InstanceDetails", function () { [undefined, 1234], // startedAtMs [undefined, 9999], // updatedAtMs [undefined, true], // excludeFromStartAll + [undefined, true], // isTemplate + [undefined, 7], // templateId )); }); From b584991f5c83cb52896d014fa78082926444a855 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:22:15 +0000 Subject: [PATCH 26/44] Give templates their own table and keep rcon_password per instance The templates table shows name, inheritor count and a create instance action instead of the instance status and start/stop columns. factorio.rcon_password joins the ports as never inherited. Part of #200 Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 2 +- packages/lib/src/config/definitions.ts | 4 +- .../web_ui/src/components/InstanceList.tsx | 3 +- .../web_ui/src/components/InstancesPage.tsx | 60 +++++++++++++++++-- test/controller/InstanceManager.test.js | 5 +- test/lib/config/definitions.js | 3 +- 6 files changed, 66 insertions(+), 11 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d6398ba7..49685ef8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -537,7 +537,7 @@ ID of the template to inherit config values from. Inherited fields take their value from the template and follow it when it changes, setting a field on this instance stops inheriting it. Object fields like `factorio.settings` are inherited per property, properties set on this instance override the ones from the template. When set on an existing instance fields with the same value as the template or the default value become inherited, except the server name in `factorio.settings`. -`instance.name`, `instance.is_template`, `instance.template_id`, the port fields and hidden fields are never inherited. +`instance.name`, `instance.is_template`, `instance.template_id`, the port fields, `factorio.rcon_password` and hidden fields are never inherited. A template may itself inherit from another template, but not from itself through a chain of templates. The fields currently inherited are stored in the hidden `instance.inherited_fields` field. In the web interface inherited values are locked and must be unlocked before they can be changed. diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index a76698f0..03c602f5 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -542,10 +542,10 @@ export class InstanceConfig extends classes.Config { return config; } - /** Fields that are never inherited from a template, ports would clash on a shared host */ + /** Fields that are never inherited from a template, ports clash on a shared host and secrets stay per instance */ static uninheritableFields = [ "instance.name", "instance.is_template", "instance.template_id", - "factorio.game_port", "factorio.rcon_port", "factorio.lua_udp_port", + "factorio.game_port", "factorio.rcon_port", "factorio.rcon_password", "factorio.lua_udp_port", ]; /** True if the field can take its value from a template */ diff --git a/packages/web_ui/src/components/InstanceList.tsx b/packages/web_ui/src/components/InstanceList.tsx index 8d69f044..02315695 100644 --- a/packages/web_ui/src/components/InstanceList.tsx +++ b/packages/web_ui/src/components/InstanceList.tsx @@ -23,7 +23,6 @@ type InstanceListProps = { instances: ReadonlyMap>; size?: SizeType; hideAssignedHost?: boolean; - tableNamespace?: string; }; export default function InstanceList(props: InstanceListProps) { @@ -31,7 +30,7 @@ export default function InstanceList(props: InstanceListProps) { let [hosts] = useHosts(); const [systems] = useSystems(); const tableState = useTableQueryState({ - namespace: props.tableNamespace ?? "instance", defaultSortKey: "name", + namespace: "instance", defaultSortKey: "name", }); const nameSearch = useColumnSearch( tableState, "name", instance => instance.name, "Search instances" diff --git a/packages/web_ui/src/components/InstancesPage.tsx b/packages/web_ui/src/components/InstancesPage.tsx index 9e6a7d54..7fbbd3b8 100644 --- a/packages/web_ui/src/components/InstancesPage.tsx +++ b/packages/web_ui/src/components/InstancesPage.tsx @@ -1,6 +1,7 @@ import React, { useContext, useEffect, useState } from "react"; import { useNavigate } from "react-router-dom"; -import { Button, Checkbox, Form, Input, Modal, Select, Space, Tooltip, Typography } from "antd"; +import { Button, Checkbox, Form, Input, Modal, Select, Space, Table, Tooltip, Typography } from "antd"; +import type { ColumnType } from "antd/es/table/interface"; import * as lib from "@clusterio/lib"; @@ -13,6 +14,8 @@ import { useHosts } from "../model/host"; import { useInstanceConfig, useInstances } from "../model/instance"; import { useSaves } from "../model/saves"; import InstanceList from "./InstanceList"; +import Link from "./Link"; +import useRowNavigation from "../util/useRowNavigation"; import { notifyErrorHandler } from "../util/notify"; import type { Control } from "../util/websocket"; @@ -92,6 +95,7 @@ export function CreateInstanceButton({ buttonText = "Create", buttonType = "prim initial?: CreateInstanceInitial; buttonText?: string; buttonType?: "primary" | "default"; + buttonSize?: "small"; }) { let control = useContext(ControlContext); let account = useAccount(); @@ -195,6 +199,7 @@ export function CreateInstanceButton({ buttonText = "Create", buttonType = "prim return <>
instance.isTemplate)} + rowKey={template => template.id} + pagination={false} + onRow={record => rowNav(`/instances/${record.id}/view`)} + />; +} + export default function InstancesPage() { let control = useContext(ControlContext); let account = useAccount(); let [instances] = useInstances(); const plainInstances = new Map([...instances].filter(([, instance]) => !instance.isTemplate)); - const templates = new Map([...instances].filter(([, instance]) => instance.isTemplate)); + const hasTemplates = plainInstances.size !== instances.size; return - {templates.size > 0 && <> + {hasTemplates && <> Templates - + } ; diff --git a/test/controller/InstanceManager.test.js b/test/controller/InstanceManager.test.js index 26e26385..25b9e839 100644 --- a/test/controller/InstanceManager.test.js +++ b/test/controller/InstanceManager.test.js @@ -433,11 +433,14 @@ describe("controller/InstanceManager", function () { assert.equal(child.config.isInherited("factorio.game_port"), false); assert.equal(child.config.isInherited("instance.name"), false); }); - it("should never inherit ports", function () { + it("should never inherit ports or the rcon password", function () { + template.config.set("factorio.rcon_password", "secret"); const config = new lib.InstanceConfig("controller", { "instance.id": 12, "instance.template_id": 10 }); instances.applyTemplate(config, ["instance.id", "instance.template_id"]); assert.equal(config.get("factorio.game_port"), null); assert.equal(config.isInherited("factorio.game_port"), false); + assert.equal(config.get("factorio.rcon_password"), null); + assert.equal(config.isInherited("factorio.rcon_password"), false); }); it("should inherit factorio.settings per property with its own server name", function () { const settings = child.config.get("factorio.settings"); diff --git a/test/lib/config/definitions.js b/test/lib/config/definitions.js index 632e10b1..a20f4354 100644 --- a/test/lib/config/definitions.js +++ b/test/lib/config/definitions.js @@ -14,7 +14,8 @@ describe("lib/config/definitions", function() { for (const name of [ "instance.name", "instance.id", "instance.assigned_host", "instance.is_template", "instance.template_id", "instance.inherited_fields", "factorio.host_assigned_game_port", - "factorio.game_port", "factorio.rcon_port", "factorio.lua_udp_port", "no.field", + "factorio.game_port", "factorio.rcon_port", "factorio.rcon_password", "factorio.lua_udp_port", + "no.field", ]) { assert.equal(lib.InstanceConfig.isInheritable(name), false, name); } From cd04995b6563249c9e7a0225888a42f72c365341 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:40:41 +0000 Subject: [PATCH 27/44] Use link and disconnect icons for template inheritance Overridden values show a disconnect icon. Clicking it stages a link, shown with a link icon, which is applied together with the other changes instead of right away. Part of #200 Co-Authored-By: Claude Opus 5.5 --- .../web_ui/src/components/BaseConfigTree.tsx | 98 ++++++++++--------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/packages/web_ui/src/components/BaseConfigTree.tsx b/packages/web_ui/src/components/BaseConfigTree.tsx index 489bdf7e..1d808916 100644 --- a/packages/web_ui/src/components/BaseConfigTree.tsx +++ b/packages/web_ui/src/components/BaseConfigTree.tsx @@ -1,8 +1,9 @@ import React, { useContext, useEffect, useState } from "react"; import { - Button, Card, Checkbox, Form, FormInstance, Input, InputNumber, Popconfirm, Select, Space, Spin, Tooltip, Tree, - Typography, + Button, Card, Checkbox, Form, FormInstance, Input, InputNumber, Select, Space, Spin, Tooltip, Tree, Typography, } from "antd"; +import DisconnectOutlined from "@ant-design/icons/DisconnectOutlined"; +import LinkOutlined from "@ant-design/icons/LinkOutlined"; import LockOutlined from "@ant-design/icons/LockOutlined"; import ReloadOutlined from "@ant-design/icons/ReloadOutlined"; import UnlockOutlined from "@ant-design/icons/UnlockOutlined"; @@ -76,11 +77,14 @@ export type ConfigInheritance = { notice?: (config: lib.Config) => React.ReactNode; }; +type Linking = Map; + type LockState = { inheritance?: ConfigInheritance; unlocked: Set; setUnlocked: (unlocked: Set) => void; - inherit: (field: string, prop?: string) => void; + linking: Linking; + setLinking: (linking: Linking) => void; }; type BaseConfigTreeProps = { @@ -99,6 +103,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { let [errorFields, setErrorFields] = useState(new Map()); let [applying, setApplying] = useState(false); let [unlocked, setUnlocked] = useState>(new Set()); + let [linking, setLinking] = useState(new Map()); const available = props.available ?? true; const control = useContext(ControlContext); @@ -156,15 +161,16 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { } } - function inherit(field: string, prop?: string) { - props.inheritance!.inherit(field, prop).then(async () => { - const newConfig = await updateConfig(); - const values = getInitialValues(newConfig, props); - const keys = Object.keys(values).filter(key => key === field || key.startsWith(`${field}.`)); - form.resetFields(keys); - form.setFieldsValue(Object.fromEntries(keys.map(key => [key, values[key]]))); - setChangedFields(fields => new Set([...fields].filter(key => !keys.includes(key)))); - }).catch(err => notify(err.message, "error")); + async function applyChanges(fields: Record>) { + if (Object.keys(fields).length) { + await props.setConfig(fields); + } + for (const [field, prop] of linking.values()) { + await props.inheritance!.inherit(field, prop); + } + setChangedFields(new Set()); + setUnlocked(new Set()); + setLinking(new Map()); } const [propsMap, treeData] = computeTreeData( @@ -176,7 +182,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { setConfig, props.ConfigClass, onValuesChange, - { inheritance: props.inheritance, unlocked, setUnlocked, inherit }, + { inheritance: props.inheritance, unlocked, setUnlocked, linking, setLinking }, ); return <> @@ -213,9 +219,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { let applied = false; try { - await props.setConfig(fields); - setChangedFields(new Set()); - setUnlocked(new Set()); + await applyChanges(fields); applied = true; } catch (err: any) { notify(err.message, "error"); @@ -223,7 +227,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { const newConfig = await updateConfig(); if (applied) { - // Linking a template can change other values + // Linking can change other values form.setFieldsValue(getInitialValues(newConfig, props)); } setApplying(false); @@ -232,7 +236,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { props.onApply(); } }} - disabled={changedFields.size === 0} + disabled={changedFields.size === 0 && linking.size === 0} > Apply @@ -242,6 +246,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { form.resetFields(); setChangedFields(new Set()); setUnlocked(new Set()); + setLinking(new Map()); setErrorFields(new Map()); }} > @@ -297,42 +302,45 @@ function computeTreeData( }; function lockButton(key: string, fieldName: string, prop?: string) { - const { inheritance, unlocked, setUnlocked } = lock; + const { inheritance, unlocked, setUnlocked, linking, setLinking } = lock; if (!inheritance?.canInherit(config, fieldName) || !config.canAccess(fieldName, lib.ConfigAccess.write)) { return null; } + const button = (title: string, icon: React.ReactNode, label: string, onClick: () => void) => + @@ -246,7 +245,7 @@ export default function BaseConfigTree(props: BaseConfigTreeProps) { onClick={() => { form.resetFields(); setChangedFields(new Set()); - setUnlocked(new Set()); + setUnlinking(new Set()); setLinking(new Map()); setErrorFields(new Map()); }} @@ -284,7 +283,7 @@ function computeTreeData( setConfig: (newConfig: lib.Config) => void, ConfigClass: BaseConfigTreeProps["ConfigClass"], onValuesChange: (changedValues: object) => void, - lock: LockState, + link: LinkState, ) { let restartTip = void) => -