Merge remote-tracking branch 'origin/master' into fix/stream-reject-closes-connection

packages/controller/src/ControlConnection.ts

@@ -8,6 +8,7 @@ import { logFilter, logger } from "@clusterio/lib";
 import BaseConnection from "./BaseConnection.js";
 import User from "./User.js";
 import UserRecord from "./UserRecord.js";
+import type InstanceRecord from "./InstanceRecord.js";
 import * as routes from "./routes.js";
 import Controller from "./Controller.js";
 
@@ -81,10 +82,11 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.ControllerConfigSetRequest, this.handleControllerConfigSetRequest.bind(this));
 		this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this));
 		this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this));
-		this.handle(lib.HostRevokeTokensRequest, this.handleHostRevokeTokensRequest.bind(this));
+		this.handle(lib.HostRevokeRequest, this.handleHostRevokeRequest.bind(this));
 		this.handle(lib.HostListRequest, this.handleHostListRequest.bind(this));
-		this.handle(lib.HostGenerateTokenRequest, this.handleHostGenerateTokenRequest.bind(this));
-		this.handle(lib.HostConfigCreateRequest, this.handleHostConfigCreateRequest.bind(this));
+		this.handle(lib.HostPairingListRequest, this.handleHostPairingListRequest.bind(this));
+		this.handle(lib.HostPairingApproveRequest, this.handleHostPairingApproveRequest.bind(this));
+		this.handle(lib.HostPairingRejectRequest, this.handleHostPairingRejectRequest.bind(this));
 		this.handle(lib.InstanceDetailsGetRequest, this.handleInstanceDetailsGetRequest.bind(this));
 		this.handle(lib.InstanceDetailsListRequest, this.handleInstanceDetailsListRequest.bind(this));
 		this.handle(lib.InstanceCreateRequest, this.handleInstanceCreateRequest.bind(this));
@@ -93,6 +95,7 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.InstanceConfigSetRequest, this.handleInstanceConfigSetRequest.bind(this));
 		this.handle(lib.InstanceConfigSetFieldRequest, this.handleInstanceConfigSetFieldRequest.bind(this));
 		this.handle(lib.InstanceConfigSetPropRequest, this.handleInstanceConfigSetPropRequest.bind(this));
+		this.handle(lib.InstanceConfigInheritFieldRequest, this.handleInstanceConfigInheritFieldRequest.bind(this));
 		this.handle(lib.InstanceAssignRequest, this.handleInstanceAssignRequest.bind(this));
 		this.handle(lib.InstanceSaveDetailsListRequest, this.handleInstanceSaveDetailsListRequest.bind(this));
 		this.handle(lib.InstanceRenameSaveRequest, controller.sendRequestToHostByInstanceId.bind(controller));
@@ -248,49 +251,25 @@ export default class ControlConnection extends BaseConnection {
 		this._controller.config.setProp(field as keyof lib.ControllerConfigFields, prop, value, "control");
 	}
 
-	async handleHostRevokeTokensRequest(request: lib.HostRevokeTokensRequest) {
-		const host = this._controller.hosts.getMutable(request.hostId);
-		if (!host) {
-			throw new Error(`Unknown host id (${request.hostId})`);
-		}
-
-		host.tokenValidAfter = Math.floor(Date.now() / 1000);
-
-		const hostConnection = this._controller.wsServer.hostConnections.get(request.hostId);
-		if (hostConnection) {
-			hostConnection.connector.terminate();
-		}
-
-		this._controller.hosts.set(host);
+	async handleHostRevokeRequest(request: lib.HostRevokeRequest) {
+		await this._controller.hosts.revokeHost(request.hostId, this.user.name);
 	}
 
 	async handleHostListRequest(): Promise<lib.HostDetails[]> {
 		return [...this._controller.hosts.values()].map(host => host.toHostDetails());
 	}
 
-	async handleHostGenerateTokenRequest(message: lib.HostGenerateTokenRequest) {
-		let hostId = message.hostId;
-		if (hostId === undefined) {
-			hostId = Math.random() * 2**31 | 0;
-		}
-		return this._controller.generateHostToken(hostId);
+	async handleHostPairingListRequest(): Promise<lib.HostPairingDetails[]> {
+		this._controller.hostPairing.prune();
+		return [...this._controller.hostPairing.records.values()];
 	}
 
-	async handleHostConfigCreateRequest(request: lib.HostConfigCreateRequest) {
-		const hostConfig = new lib.HostConfig("host");
+	async handleHostPairingApproveRequest(request: lib.HostPairingApproveRequest) {
+		return this._controller.hostPairing.approve(request.code, request.name, this.user.name);
+	}
 
-		hostConfig.set("host.controller_url", this._controller.getControllerUrl());
-		if (request.id !== undefined) {
-			hostConfig.set("host.id", request.id);
-		}
-		if (request.name !== undefined) {
-			hostConfig.set("host.name", request.name);
-		}
-		if (request.generateToken) {
-			this.user.checkPermission("core.host.generate_token");
-			hostConfig.set("host.controller_token", this._controller.generateHostToken(hostConfig.get("host.id")));
-		}
-		return hostConfig.toJSON();
+	async handleHostPairingRejectRequest(request: lib.HostPairingRejectRequest) {
+		this._controller.hostPairing.reject(request.code, this.user.name);
 	}
 
 	async handleInstanceDetailsGetRequest(request: lib.InstanceDetailsGetRequest) {
@@ -315,6 +294,12 @@ export default class ControlConnection extends BaseConnection {
 			instanceConfig.set("instance.assigned_host", null); // New instances are unassigned
 		}
 		instanceConfig.update(request.config, false, "control");
+		this._controller.instances.checkInheritance(
+			instanceConfig, instanceConfig.get("instance.is_template"), instanceConfig.get("instance.template_id")
+		);
+		if (Object.hasOwn(request.config, "instance.template_id")) {
+			this._controller.instances.applyTemplate(instanceConfig, Object.keys(request.config));
+		}
 		await this._controller.instances.createInstance(instanceConfig);
 	}
 
@@ -355,17 +340,41 @@ export default class ControlConnection extends BaseConnection {
 				}
 			}
 
+			this.checkStagedInheritance(instance);
 			instance.config.commitStaging();
+			for (const [fieldName, fieldValue] of Object.entries(request.fields)) {
+				const props = typeof fieldValue === "object"
+					? Object.keys(fieldValue).filter(prop => fieldValue[prop] !== undefined)
+					: undefined;
+				this._controller.instances.overrideField(instance, fieldName, props);
+			}
 			await this._controller.instanceConfigUpdated(instance);
 		} finally {
 			instance.config.revertStaging();
 		}
 	}
 
+	private checkStagedInheritance(instance: InstanceRecord) {
+		const staged = instance.config.staging;
+		this._controller.instances.checkInheritance(
+			instance.config, staged.get("instance.is_template"), staged.get("instance.template_id")
+		);
+	}
+
 	async handleInstanceConfigSetFieldRequest(request: lib.InstanceConfigSetFieldRequest) {
 		const instance = this._controller.instances.getForRequest(request.instanceId);
-		ControlConnection.validateInstanceConfigSetField(request.field);
-		instance.config.set(request.field as keyof lib.InstanceConfigFields, request.value, "control");
+		const field = request.field as keyof lib.InstanceConfigFields;
+		ControlConnection.validateInstanceConfigSetField(field);
+		if (["instance.is_template", "instance.template_id"].includes(field)) {
+			try {
+				instance.config.stage(field, request.value, "control");
+				this.checkStagedInheritance(instance);
+			} finally {
+				instance.config.revertStaging();
+			}
+		}
+		instance.config.set(field, request.value, "control");
+		this._controller.instances.overrideField(instance, field);
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
@@ -374,9 +383,18 @@ export default class ControlConnection extends BaseConnection {
 		let { field, prop, value } = request;
 		ControlConnection.validateInstanceConfigSetField(field);
 		instance.config.setProp(field as keyof lib.InstanceConfigFields, prop, value, "control");
+		if (value !== undefined) {
+			this._controller.instances.overrideField(instance, field, [prop]);
+		}
 		await this._controller.instanceConfigUpdated(instance);
 	}
 
+	async handleInstanceConfigInheritFieldRequest(request: lib.InstanceConfigInheritFieldRequest) {
+		ControlConnection.validateInstanceConfigSetField(request.field);
+		this._controller.instances.inheritField(request.instanceId, request.field, request.prop);
+		await this._controller.instanceConfigUpdated(this._controller.instances.getForRequest(request.instanceId));
+	}
+
 	async handleInstanceAssignRequest(request: lib.InstanceAssignRequest) {
 		await this._controller.instances.assignInstance(request.instanceId, request.hostId ?? null);
 	}

packages/controller/src/Controller.ts

@@ -11,7 +11,6 @@ import events from "events";
 import fs from "node:fs/promises";
 import http from "http";
 import https from "https";
-import jwt from "jsonwebtoken";
 import path from "path";
 import semver from "semver";
 import stream from "stream";
@@ -28,7 +27,9 @@ import User from "./User.js";
 import UserRecord from "./UserRecord.js";
 import UserManager from "./UserManager.js";
 import WsServer from "./WsServer.js";
+import HostManager from "./HostManager.js";
 import HostRecord from "./HostRecord.js";
+import HostPairingManager from "./HostPairingManager.js";
 import { BaseControllerPlugin, ControllerHooks } from "./BaseControllerPlugin.js";
 import ControllerRouter from "./ControllerRouter.js";
 import InstanceManager from "./InstanceManager.js";
@@ -87,10 +88,14 @@ export default class Controller {
 
 	/** Event subscription controller */
 	subscriptions: lib.SubscriptionController;
+	/** Host manager for the cluster, also contains access to the host records */
+	hosts: HostManager;
 	/** User manager for the cluster, also contains access to the user records */
 	users: UserManager;
 	/** Instance manager for the cluster, also contains access to the user records */
 	instances: InstanceManager;
+	/** Pending requests from hosts to join the cluster */
+	hostPairing: HostPairingManager;
 
 	// Possible states are new, starting, running, stopping, stopped
 	private _state: string = "new";
@@ -135,7 +140,7 @@ export default class Controller {
 		const cachePath = path.join(this.config.get("controller.database_directory"), "latest-releases.json");
 		try {
 			const releases = await lib.fetchLatestReleases();
-			await lib.safeOutputFile(cachePath, JSON.stringify(releases, null, "\t"));
+			await lib.atomicOverwriteFile(cachePath, JSON.stringify(releases, null, "\t"));
 			return releases;
 		} catch (err: any) {
 			logger.warn(`Failed to retrieve Factorio releases ${err.message}`);
@@ -223,6 +228,8 @@ export default class Controller {
 		await fs.mkdir(modsDirectory, { recursive: true });
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
+		await this.validateExports(modPacks, "static");
+
 		// Add default mod packs
 		if (modPacks.size === 0) {
 			modPacks.setMany(lib.ModPack.defaultModPacks);
@@ -267,7 +274,7 @@ export default class Controller {
 		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
 		/** Mapping of host id to host record */
-		public hosts = new lib.SubscribableDatastore<HostRecord>(),
+		hosts = new lib.SubscribableDatastore<HostRecord>(),
 		/** Mapping of instance id to instance record */
 		instances = new lib.SubscribableDatastore<InstanceRecord>(),
 		/** Mapping of save id to save details */
@@ -294,8 +301,10 @@ export default class Controller {
 		this.trustedProxies = this.parseTrustedProxies();
 		this.wsServer = new WsServer(this);
 
+		this.hosts = new HostManager(hosts, this);
 		this.users = new UserManager(users, roles, config);
 		this.instances = new InstanceManager(instances, this);
+		this.hostPairing = new HostPairingManager(this);
 
 		this.modStore.on("change", mod => {
 			this.modsUpdated([mod]);
@@ -305,6 +314,7 @@ export default class Controller {
 		this.subscriptions = new lib.SubscriptionController(config.get("controller.max_subscription_filters"));
 		this.subscriptions.handle(lib.SystemInfoUpdateEvent, this.handleSystemInfoSubscription.bind(this));
 		this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this));
+		this.subscriptions.handle(lib.HostPairingUpdatesEvent, this.handleHostPairingSubscription.bind(this));
 		this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this));
 		this.subscriptions.handle(
 			lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsSubscription.bind(this)
@@ -317,7 +327,8 @@ export default class Controller {
 
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
-		this.hosts.on("update", this.hostsUpdated.bind(this));
+		this.hosts.records.on("update", this.hostsUpdated.bind(this));
+		this.hostPairing.records.on("update", this.hostPairingRequestsUpdated.bind(this));
 		this.instances.records.on("update", this.instanceUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
@@ -420,7 +431,7 @@ export default class Controller {
 			this.app.use(this.devServer.middlewares);
 		}
 
-		if (!args.dev) {
+		if (!this.devServer) {
 			let manifestPath = path.join(import.meta.dirname, "..", "..", "web", "manifest.json");
 
 			let manifest = await Controller.loadJsonObject(manifestPath);
@@ -489,6 +500,7 @@ export default class Controller {
 			logger.info(`Listening for HTTPS on port ${(this.httpsServer.address() as AddressInfo).port}`);
 		}
 
+		this.hostPairing.start();
 		this.onAutosaveIntervalChanged();
 		this.onSystemMetricsIntervalChanged();
 
@@ -527,14 +539,14 @@ export default class Controller {
 			},
 			plugins: [webUiVite.clusterioDevPlugins(pluginDirs)],
 		});
-		if (args.dev) {
-			this.app.locals.webBundle = {
-				main: this.devServer.config.plugins.find(p => p.name === "clusterio:app")!.api.entry,
-				imports: Object.fromEntries(webUiVite.sharedModules.map(
-					name => [name, webUiVite.devSharedUrl(name)]
-				)),
-			};
-		}
+		// Plugins served from source import the shared modules from the development server, the web
+		// interface has to use the same copies, so it is served from source as well with --dev-plugin.
+		this.app.locals.webBundle = {
+			main: this.devServer.config.plugins.find(p => p.name === "clusterio:app")!.api.entry,
+			imports: Object.fromEntries(webUiVite.sharedModules.map(
+				name => [name, webUiVite.devSharedUrl(name)]
+			)),
+		};
 		this.app.locals.devServer = this.devServer;
 	}
 
@@ -582,6 +594,8 @@ export default class Controller {
 			this.autosaveInterval = undefined;
 		}
 
+		this.hostPairing.stop();
+
 		if (this.clusterLogIndex) {
 			await this.clusterLogIndex.save();
 		}
@@ -785,7 +799,7 @@ export default class Controller {
 		await Promise.all([
 			this.config.save(),
 			this.systems.save(),
-			this.hosts.save(),
+			this.hosts.records.save(),
 			this.instances.records.save(),
 			this.saves.save(),
 			this.modPacks.save(),
@@ -867,8 +881,43 @@ export default class Controller {
 		});
 	}
 
+	/** Remove export manifests that reference files missing from staticDirectory */
+	static async validateExports(modPacks: lib.SubscribableDatastore<lib.ModPack>, staticDirectory: string) {
+		const invalid: lib.ModPack[] = [];
+		for (const modPack of modPacks.valuesMutable()) {
+			if (!modPack.exportManifest) {
+				continue;
+			}
+			const missing: string[] = [];
+			for (const file of Object.values(modPack.exportManifest.assets)) {
+				const valid = path.basename(file) === file
+					&& await fs.stat(path.join(staticDirectory, file)).then(s => s.isFile(), () => false);
+				if (!valid) {
+					missing.push(file);
+				}
+			}
+			if (missing.length) {
+				Controller.dropExport(modPack, `missing ${missing.join(", ")}`);
+				invalid.push(modPack);
+			}
+		}
+		if (invalid.length) {
+			modPacks.setMany(invalid);
+		}
+	}
+
+	/** Clear the export manifest of a mutable mod pack, caller must save it */
+	static dropExport(modPack: lib.ModPack, reason: string) {
+		logger.warn(`Removing export data of mod pack ${modPack.name} (id ${modPack.id}): ${reason}`);
+		modPack.exportManifest = undefined;
+	}
+
 	static migrateRoles(rawJson: unknown[]): Static<typeof lib.Role.jsonSchema>[] {
 		const serialized = rawJson as Static<typeof lib.Role.jsonSchema>[];
+		// migrate: host token permissions were removed with host pairing in 2.1
+		const removedPermissions = new Set([
+			"core.host.generate_token", "core.host.revoke_token", "core.host.create_config",
+		]);
 		return serialized.map(json => {
 			json.permissions = json.permissions.map(permission => {
 				// migrate: core.instance.save.list.subscribe was renamed in alpha 17
@@ -876,7 +925,7 @@ export default class Controller {
 					return "core.instance.save.subscribe";
 				}
 				return permission;
-			});
+			}).filter(permission => !removedPermissions.has(permission));
 			return lib.Role.pluginMigrations.reduce(json);
 		});
 	}
@@ -971,6 +1020,7 @@ export default class Controller {
 			express.static(path.join(import.meta.dirname, "..", "..", "web", "static"), staticOptions)
 		);
 		app.use("/static", express.static("static", staticOptions)); // Used for data export files
+		app.use("/static", routes.missingExportAsset);
 
 		// Add API routes
 		routes.addRouteHandlers(app);
@@ -1002,21 +1052,25 @@ export default class Controller {
 		return systems.length ? new lib.SystemInfoUpdateEvent(systems) : null;
 	}
 
-	/**
-	 * Generate access token for host
-	 *
-	 * @returns access token for host
-	 * @param hostId - ID of host to generate a token for.
-	 */
-	generateHostToken(hostId: number): string {
-		return jwt.sign({ aud: "host", host: hostId }, this.authSecret);
-	}
-
 	hostsUpdated(hosts: HostRecord[]) {
 		let updates = hosts.map(host => host.toHostDetails());
 		this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates));
 	}
 
+	hostPairingRequestsUpdated(requests: lib.HostPairingDetails[]) {
+		this.subscriptions.broadcast(new lib.HostPairingUpdatesEvent(requests));
+	}
+
+	async handleHostPairingSubscription(request: lib.SubscriptionRequest) {
+		if (request.action === "unsubscribe") {
+			return null;
+		}
+		const requests = [...this.hostPairing.records.values()].filter(
+			pairing => pairing.updatedAtMs > request.lastRequestTimeMs,
+		);
+		return requests.length ? new lib.HostPairingUpdatesEvent(requests) : null;
+	}
+
 	async handleHostSubscription(request: lib.SubscriptionRequest) {
 		if (request.action === "unsubscribe") {
 			return null;

packages/controller/src/HostConnection.ts

@@ -34,20 +34,17 @@ export default class HostConnection extends BaseConnection {
 
 		this.plugins = new Map(Object.entries(registerData.plugins));
 
-		const previousHostInfo = this._controller.hosts.get(registerData.id);
-		this.info = new HostRecord(
-			registerData.id,
-			previousHostInfo?.name ?? "",
-			registerData.version,
-			this.plugins,
-			true,
-			remoteAddress,
-			previousHostInfo?.publicAddress,
-			previousHostInfo?.tokenValidAfter,
-			0,
-			false,
-		);
-		this._controller.hosts.set(this.info);
+		// Hosts are only created by approving them, WsServer rejects hosts without a record.
+		const record = this._controller.hosts.getMutable(registerData.id);
+		if (!record) {
+			throw new Error(`Host ${registerData.id} is not registered`);
+		}
+		record.version = registerData.version;
+		record.plugins = this.plugins;
+		record.connected = true;
+		record.remoteAddress = remoteAddress;
+		this.info = record;
+		this._controller.hosts.records.set(this.info);
 
 		this._checkPluginVersions();
 
@@ -71,7 +68,7 @@ export default class HostConnection extends BaseConnection {
 			this._controller.instances.records.setMany(instances);
 
 			this.info.connected = false;
-			this._controller.hosts.set(this.info);
+			this.saveRecord();
 		});
 
 		this.handle(lib.GetFactorioCredentialsRequest, this.handleGetFactorioCredentialsRequest.bind(this));
@@ -175,10 +172,17 @@ export default class HostConnection extends BaseConnection {
 		return {};
 	}
 
+	/** Store changes to the host record, unless the host was revoked */
+	saveRecord() {
+		if (this._controller.hosts.getMutable(this.id) === this.info) {
+			this._controller.hosts.records.set(this.info);
+		}
+	}
+
 	async handleHostInfoUpdateEvent(event: lib.HostInfoUpdateEvent) {
 		this.info.name = event.update.name;
 		this.info.publicAddress = event.update.publicAddress;
-		this._controller.hosts.set(this.info);
+		this.saveRecord();
 	}
 
 	async handleInstanceStatusChangedEvent(request: lib.InstanceStatusChangedEvent) {

packages/controller/src/HostManager.ts

@@ -0,0 +1,149 @@
+import crypto from "node:crypto";
+import jwt from "jsonwebtoken";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+
+import type Controller from "./Controller.js";
+import HostRecord from "./HostRecord.js";
+
+/**
+ * Access layer for hosts on the controller.
+ *
+ * @alias module:controller/src/HostManager
+ */
+export default class HostManager {
+	constructor(
+		public readonly records: lib.SubscribableDatastore<HostRecord>,
+		private readonly _controller: Controller,
+	) { }
+
+	/* ------------------------------------------------------------------------- */
+	/* Basic Accessors                                                           */
+	/* ------------------------------------------------------------------------- */
+
+	has(id: HostRecord["id"]): boolean {
+		return this.records.has(id);
+	}
+
+	getMutable(id: HostRecord["id"]): HostRecord | undefined {
+		return this.records.getMutable(id);
+	}
+
+	get(id: HostRecord["id"]): Readonly<HostRecord> | undefined {
+		return this.records.get(id);
+	}
+
+	valuesMutable(): IterableIterator<HostRecord> {
+		return this.records.values();
+	}
+
+	values(): IterableIterator<Readonly<HostRecord>> {
+		return this.valuesMutable();
+	}
+
+	/**
+	 * Get host by ID for a request.
+	 *
+	 * @param hostId - ID of host to get.
+	 * @returns The host record.
+	 * @throws {module:lib.NotFoundError} if the host does not exist.
+	 */
+	getForRequest(hostId: number): HostRecord {
+		const host = this.records.getMutable(hostId);
+		if (!host) {
+			throw new lib.NotFoundError(`Unknown host id (${hostId})`);
+		}
+		return host;
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Lifecycle                                                                 */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Create a host record with a new random id
+	 *
+	 * @param name - Name of the host.
+	 * @returns the created host record.
+	 */
+	createHost(name: string): HostRecord {
+		let hostId;
+		do {
+			hostId = crypto.randomInt(1, 2 ** 31);
+		} while (this.records.has(hostId));
+
+		const host = new HostRecord(
+			hostId, name, "", new Map(), false, "", "", Math.floor(Date.now() / 1000),
+		);
+		this.records.set(host);
+		return host;
+	}
+
+	/**
+	 * Delete a host, disconnect it and unassign its instances
+	 *
+	 * @param hostId - ID of host to revoke.
+	 * @param revokedBy - Name of the user revoking it, for logging.
+	 */
+	async revokeHost(hostId: number, revokedBy: string) {
+		const host = this.getForRequest(hostId);
+		this.records.delete(host);
+		this._controller.deleteNote("host", hostId, revokedBy);
+		const system = this._controller.systems.getMutable(hostId);
+		if (system) {
+			this._controller.systems.delete(system);
+		}
+
+		// Disconnected first so unassigning the instances does not depend on the host responding.
+		const hostConnection = this._controller.wsServer.hostConnections.get(hostId);
+		if (hostConnection) {
+			hostConnection.connector.terminate();
+		}
+		const assigned = [...this._controller.instances.values()].filter(
+			instance => instance.config.get("instance.assigned_host") === hostId
+		);
+		for (const instance of assigned) {
+			await this._controller.instances.unassignInstance(instance.id);
+		}
+		logger.audit(`User ${revokedBy} revoked host ${host.name} (${host.id})`);
+	}
+
+	/* ------------------------------------------------------------------------- */
+	/* Helpers                                                                   */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Sign access token for the given host
+	 *
+	 * @param hostId - ID of host to sign token for.
+	 * @returns JWT access token for the host.
+	 */
+	signHostToken(hostId: number): string {
+		return jwt.sign({ aud: "host", host: hostId }, this._controller.authSecret);
+	}
+
+	/**
+	 * Check a host token
+	 *
+	 * @param token - Host token to check.
+	 * @returns the host the token was issued to.
+	 * @throws if the token is invalid, its host is not registered or it was issued before tokenValidAfter.
+	 */
+	verifyHostToken(token: string): HostRecord {
+		// migrate: allow pre-rename tokens issued to hosts before alpha-14
+		const payload = jwt.verify(token, this._controller.authSecret, { audience: ["host", "slave"] });
+		if (typeof payload === "string") {
+			throw new Error("unexpected JsonWebToken type");
+		}
+		const hostId = payload.host !== undefined ? payload.host : payload.slave;
+		const host = this.records.getMutable(hostId);
+		if (!host) {
+			throw new Error("host is not registered, start the host again to request to join the cluster");
+		}
+		if (!payload.iat || payload.iat < host.tokenValidAfter) {
+			throw new Error("invalid token");
+		}
+		return host;
+	}
+}

packages/controller/src/HostPairingManager.ts

@@ -0,0 +1,299 @@
+import type Controller from "./Controller.js";
+
+import crypto from "crypto";
+import net from "net";
+
+import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
+
+/** Symbols used in pairing codes, avoids 0/O and 1/I/L */
+const codeAlphabet = "23456789ABCDEFGHJKMNPQRSTUVWXYZ";
+const codeLength = 6;
+
+/** Time a request is kept after the last poll from the host */
+export const pollTimeoutMs = 30e3;
+/** Maximum number of pending requests from one address */
+export const maxPendingPerAddress = 3;
+/** Maximum length of a host name */
+export const maxNameLength = 100;
+/** Control and bidirectional override characters refused in host names */
+const nameControlCharacters = /[\p{Cc}\u202a-\u202e\u2066-\u2069]/u;
+
+/** Result of an approved request, sent to the host */
+export interface HostPairingResult {
+	hostId: number,
+	token: string,
+	name: string,
+}
+
+/** Error with an HTTP status code to respond to the host with */
+export class HostPairingError extends Error {
+	constructor(
+		message: string,
+		public status: number,
+		/** Seconds to wait before trying again, for status 429 */
+		public retryAfter?: number,
+	) {
+		super(message);
+	}
+}
+
+type PairingRequest = {
+	/** Secret the host polls with, never shown to users */
+	secret: string,
+	details: lib.HostPairingDetails,
+	/** Address with IPv6 addresses grouped by /64, used for limits */
+	addressKey: string,
+	lastPollMs: number,
+} & (
+	{ state: "pending" | "rejected" } | { state: "approved", result: HostPairingResult }
+);
+
+/**
+ * Strips the IPv4 mapped prefix and groups IPv6 addresses by their /64
+ *
+ * @param address - Address to make a key for
+ * @returns key used to count requests from an address.
+ */
+export function addressKey(address: string) {
+	if (address.startsWith("::ffff:") && net.isIPv4(address.slice(7))) {
+		return address.slice(7);
+	}
+	if (!net.isIPv6(address)) {
+		return address;
+	}
+	const [head, tail = ""] = address.split("::");
+	const headParts = head ? head.split(":") : [];
+	const tailParts = tail ? tail.split(":") : [];
+	const missing = 8 - headParts.length - tailParts.length;
+	const parts = address.includes("::")
+		? [...headParts, ...new Array(missing).fill("0"), ...tailParts]
+		: headParts;
+	return `${parts.slice(0, 4).map(part => part.toLowerCase().replace(/^0+(?=.)/, "")).join(":")}::/64`;
+}
+
+/**
+ * Normalise a pairing code typed by a user
+ *
+ * @param code - Code as typed, may contain dashes and lower case letters.
+ * @returns code formatted as shown by the host.
+ */
+export function normaliseCode(code: string) {
+	const plain = code.toUpperCase().replace(/[^0-9A-Z]/g, "");
+	return `${plain.slice(0, 3)}-${plain.slice(3)}`;
+}
+
+function generateCode() {
+	let code = "";
+	for (let i = 0; i < codeLength; i++) {
+		code += codeAlphabet[crypto.randomInt(codeAlphabet.length)];
+	}
+	return normaliseCode(code);
+}
+
+/**
+ * Keeps track of requests from hosts to join the cluster
+ *
+ * Requests only live in memory, and only while the host keeps polling
+ * for the result.
+ */
+export default class HostPairingManager {
+	/** Pending requests by code, broadcast to subscribed control links */
+	records = new lib.SubscribableDatastore<lib.HostPairingDetails>();
+	private _bySecret = new Map<string, PairingRequest>();
+	private _byCode = new Map<string, PairingRequest>();
+	private _pruneInterval?: ReturnType<typeof setInterval>;
+
+	constructor(
+		private _controller: Controller,
+	) { }
+
+	start() {
+		this._pruneInterval = setInterval(() => this.prune(), 5e3);
+		this._pruneInterval.unref();
+	}
+
+	stop() {
+		clearInterval(this._pruneInterval);
+		this._pruneInterval = undefined;
+	}
+
+	/**
+	 * Check and trim a host name given by a host or a user
+	 *
+	 * @param name - Name to check.
+	 * @returns the trimmed name.
+	 */
+	static validateName(name: unknown) {
+		if (typeof name !== "string" || !name.trim()) {
+			throw new HostPairingError("Host name is required", 400);
+		}
+		const trimmed = name.trim();
+		if (trimmed.length > maxNameLength) {
+			throw new HostPairingError(`Host name must be at most ${maxNameLength} characters`, 400);
+		}
+		// Names come from unauthenticated requests and are printed to terminals and logs.
+		if (nameControlCharacters.test(trimmed)) {
+			throw new HostPairingError("Host name may not contain control characters", 400);
+		}
+		return trimmed;
+	}
+
+	/**
+	 * Create a pending request
+	 *
+	 * @param name - Name the host asked for.
+	 * @param version - Version of the host.
+	 * @param remoteAddress - Address the request came from.
+	 * @returns the secret to poll with and the code to show the user.
+	 */
+	create(name: string, version: string, remoteAddress: string) {
+		this.prune();
+		const key = addressKey(remoteAddress);
+		const now = Date.now();
+		const pendingFromAddress = [...this._bySecret.values()].filter(
+			request => request.addressKey === key && request.state === "pending"
+		).length;
+		if (pendingFromAddress >= maxPendingPerAddress) {
+			throw new HostPairingError("Too many pending requests from this address", 429, 30);
+		}
+		const maxPending = this._controller.config.get("controller.host_pairing_max_pending");
+		if (maxPending && this.records.size >= maxPending) {
+			throw new HostPairingError("Too many pending requests", 429, 30);
+		}
+
+		let code;
+		do {
+			code = generateCode();
+		} while (this._byCode.has(code));
+
+		const request: PairingRequest = {
+			secret: crypto.randomBytes(32).toString("base64url"),
+			details: new lib.HostPairingDetails(code, name, remoteAddress, version, now),
+			addressKey: key,
+			lastPollMs: now,
+			state: "pending",
+		};
+		this._bySecret.set(request.secret, request);
+		this._byCode.set(code, request);
+		this.records.set(request.details);
+		logger.verbose(`Host pairing request ${code} for "${name}" from ${remoteAddress}`);
+		return { requestId: request.secret, code };
+	}
+
+	/**
+	 * Poll for the result of a request
+	 *
+	 * @param secret - Request id returned when the request was created.
+	 * @returns the result if the request was approved, or null if still pending.
+	 */
+	poll(secret: string): HostPairingResult | null {
+		this.prune();
+		const request = this._bySecret.get(secret);
+		if (!request) {
+			throw new HostPairingError("Request not found", 404);
+		}
+		request.lastPollMs = Date.now();
+		if (request.state === "approved") {
+			this._remove(request);
+			return request.result;
+		}
+		if (request.state === "rejected") {
+			this._remove(request);
+			throw new HostPairingError("Request was rejected", 403);
+		}
+		return null;
+	}
+
+	/**
+	 * Cancel a pending request
+	 *
+	 * @param secret - Request id returned when the request was created.
+	 */
+	cancel(secret: string) {
+		const request = this._bySecret.get(secret);
+		if (request && request.state === "pending") {
+			logger.verbose(`Host pairing request ${request.details.code} cancelled by the host`);
+			this._remove(request);
+		}
+	}
+
+	private _pending(code: string) {
+		this.prune();
+		const request = this._byCode.get(normaliseCode(code));
+		if (!request || request.state !== "pending") {
+			throw new lib.NotFoundError(`No pending host request with code ${code}`);
+		}
+		return request;
+	}
+
+	/**
+	 * Approve a pending request, creating the host
+	 *
+	 * @param code - Code of the request.
+	 * @param name - Name to give the host, defaults to the requested name.
+	 * @param approvedBy - Name of the user approving it, for logging.
+	 * @returns the id of the new host.
+	 */
+	approve(code: string, name: string | undefined, approvedBy: string) {
+		const request = this._pending(code);
+		let hostName;
+		try {
+			hostName = name === undefined ? request.details.name : HostPairingManager.validateName(name);
+		} catch (err: any) {
+			throw new lib.ValidationError(err.message);
+		}
+
+		const host = this._controller.hosts.createHost(hostName);
+		const result = { hostId: host.id, token: this._controller.hosts.signHostToken(host.id), name: host.name };
+		Object.assign(request, { state: "approved", result });
+		this.records.delete(request.details);
+		logger.audit(
+			`User ${approvedBy} approved host ${result.name} (${result.hostId}) with code ` +
+			`${request.details.code} from ${request.details.remoteAddress}`
+		);
+		return result.hostId;
+	}
+
+	/**
+	 * Reject a pending request
+	 *
+	 * @param code - Code of the request.
+	 * @param rejectedBy - Name of the user rejecting it, for logging.
+	 */
+	reject(code: string, rejectedBy: string) {
+		const request = this._pending(code);
+		request.state = "rejected";
+		this.records.delete(request.details);
+		logger.audit(
+			`User ${rejectedBy} rejected host ${request.details.name} with code ` +
+			`${request.details.code} from ${request.details.remoteAddress}`
+		);
+	}
+
+	private _remove(request: PairingRequest) {
+		this._bySecret.delete(request.secret);
+		this._byCode.delete(request.details.code);
+		this.records.delete(request.details);
+	}
+
+	/** Remove requests the host stopped polling for or that waited too long */
+	prune() {
+		const now = Date.now();
+		const timeoutMs = this._controller.config.get("controller.host_pairing_timeout") * 1000;
+		for (const request of this._bySecret.values()) {
+			const timedOut = request.state === "pending" && timeoutMs > 0
+				&& now - request.details.createdAtMs > timeoutMs;
+			if (!timedOut && now - request.lastPollMs <= pollTimeoutMs) {
+				continue;
+			}
+			if (request.state === "approved") {
+				logger.warn(
+					`Approved host ${request.result.name} (${request.result.hostId}) was never picked up ` +
+					"by the host, revoke it on the hosts page if it is not needed"
+				);
+			}
+			this._remove(request);
+		}
+	}
+}

packages/controller/src/InstanceManager.ts

@@ -1,9 +1,14 @@
+import { isDeepStrictEqual } from "node:util";
 import * as lib from "@clusterio/lib";
+import { logger } from "@clusterio/lib";
 
 import type Controller from "./Controller.js";
 import type HostConnection from "./HostConnection.js";
 import InstanceRecord from "./InstanceRecord.js";
 
+type InheritedFields = lib.InstanceConfigFields["instance.inherited_fields"];
+type InheritableField = keyof lib.InstanceConfigFields & string;
+
 export default class InstanceManager {
 	private static readonly DefaultFactorioSettings = {
 		tags: ["clusterio"],
@@ -24,6 +29,9 @@ export default class InstanceManager {
 		non_blocking_saving: false,
 	} as const;
 
+	/** Non-zero while template values are being copied into inheriting instances */
+	private _inheriting = 0;
+
 	constructor(
 		public readonly records: lib.SubscribableDatastore<InstanceRecord>,
 		private readonly _controller: Controller,
@@ -106,11 +114,271 @@ export default class InstanceManager {
 	 */
 	addInstanceHooks(instance: InstanceRecord): void {
 		instance.config.on("fieldChanged", (field: string, curr: unknown, prev: unknown) => {
+			if (field === "instance.template_id") {
+				this._linkMatchingFields(instance);
+			} else if (!this._inheriting && instance.config.isInherited(field)) {
+				this._unlinkEditedField(instance, field, curr, prev);
+			}
 			this.records.set(instance);
 			this._notifyConfigFieldChanged(instance, field, curr, prev);
+			this._updateInheritors(instance, field, curr);
 		});
 	}
 
+	/* ------------------------------------------------------------------------- */
+	/* Inheritance                                                               */
+	/* ------------------------------------------------------------------------- */
+
+	/**
+	 * Get the instances that inherit directly from the given template.
+	 *
+	 * @param templateId - ID of the template.
+	 * @returns Instances with `instance.template_id` set to templateId.
+	 */
+	inheritorsOf(templateId: number): InstanceRecord[] {
+		return [...this.records.values()].filter(
+			instance => instance.config.get("instance.template_id") === templateId
+		);
+	}
+
+	/**
+	 * Check that new values for `instance.is_template` and `instance.template_id` are allowed.
+	 *
+	 * @param config - Current config of the instance.
+	 * @param isTemplate - New value for `instance.is_template`.
+	 * @param templateId - New value for `instance.template_id`.
+	 * @throws {module:lib.ValidationError} if the template is not a template or would create a cycle.
+	 * @throws {module:lib.ConflictError} if the instance is assigned or still has inheritors.
+	 */
+	checkInheritance(config: lib.InstanceConfig, isTemplate: boolean, templateId: number | null): void {
+		const instanceId = config.get("instance.id");
+		if (isTemplate && config.get("instance.assigned_host") !== null) {
+			throw new lib.ConflictError("Instance must be unassigned before it can be made a template");
+		}
+		if (!isTemplate && config.get("instance.is_template")) {
+			const count = this.inheritorsOf(instanceId).length;
+			if (count) {
+				throw new lib.ConflictError(`Instance is the template of ${count} other instance(s)`);
+			}
+		}
+		if (templateId === null) {
+			return;
+		}
+		const template = this.records.get(templateId);
+		if (!template) {
+			throw new lib.NotFoundError(`Instance with ID ${templateId} does not exist`);
+		}
+		if (!template.config.get("instance.is_template")) {
+			throw new lib.ValidationError(`Instance ${template.config.get("instance.name")} is not a template`);
+		}
+		const visited = new Set<number>();
+		for (let id: number | null = templateId; id !== null && !visited.has(id);) {
+			if (id === instanceId) {
+				throw new lib.ValidationError("Templates cannot inherit from themselves");
+			}
+			visited.add(id);
+			id = this.records.get(id)?.config.get("instance.template_id") ?? null;
+		}
+	}
+
+	/**
+	 * Copy the inheritable values from the template set in `instance.template_id`
+	 * into a config that is not yet part of the cluster and mark them inherited.
+	 *
+	 * @param config - Config of the instance to be created.
+	 * @param own - Fields set by the creator, object fields keep their properties as overrides.
+	 */
+	applyTemplate(config: lib.InstanceConfig, own: string[] = []): void {
+		const template = this._templateOf(config);
+		if (!template) {
+			return;
+		}
+		const inherited: InheritedFields = {};
+		for (const name of InstanceManager._inheritableFields()) {
+			const def = lib.InstanceConfig.fieldDefinitions[name];
+			if (def.type === "object") {
+				inherited[name] = own.includes(name) ? Object.keys(config.get(name) as object ?? {}) : [];
+			} else if (!own.includes(name)) {
+				inherited[name] = true;
+			}
+		}
+		config.set("instance.inherited_fields", inherited);
+		config.update(InstanceManager._inheritedValues(config, template.config, Object.keys(inherited)), false);
+	}
+
+	/**
+	 * Make a field, or a property of an object field, take its value from the template again.
+	 *
+	 * @param instanceId - ID of the instance to update.
+	 * @param field - Name of the field to inherit.
+	 * @param prop - Property of an object field to inherit, inherits all properties if not given.
+	 * @throws {module:lib.ValidationError} if the instance has no template or the field cannot be inherited.
+	 */
+	inheritField(instanceId: number, field: string, prop?: string): void {
+		const instance = this.getForRequest(instanceId);
+		if (!this._templateOf(instance.config)) {
+			throw new lib.ValidationError("Instance does not have a template");
+		}
+		if (!lib.InstanceConfig.isInheritable(field)) {
+			throw new lib.ValidationError(`Field ${field} cannot be inherited`);
+		}
+		const name = field as InheritableField;
+		const inherited = { ...instance.config.get("instance.inherited_fields") };
+		if (lib.InstanceConfig.fieldDefinitions[name].type !== "object") {
+			inherited[field] = true;
+		} else if (prop === undefined) {
+			inherited[field] = [];
+		} else {
+			const entry = inherited[field];
+			const overrides = entry instanceof Array ? entry : Object.keys(instance.config.get(name) as object ?? {});
+			inherited[field] = overrides.filter(p => p !== prop);
+		}
+		instance.config.set("instance.inherited_fields", inherited);
+		this._copyInheritedValues(instance, [field]);
+	}
+
+	/**
+	 * Stop inheriting the given field or properties, used when a value is explicitly set.
+	 *
+	 * @param instance - Instance the field was set on.
+	 * @param field - Name of the field set.
+	 * @param props - Properties set on an object field.
+	 */
+	overrideField(instance: InstanceRecord, field: string, props?: string[]): void {
+		const entry = instance.config.get("instance.inherited_fields")[field];
+		if (!instance.config.isInherited(field)) {
+			return;
+		}
+		const inherited = { ...instance.config.get("instance.inherited_fields") };
+		if (entry instanceof Array && props) {
+			inherited[field] = [...new Set([...entry, ...props])];
+		} else {
+			delete inherited[field];
+		}
+		instance.config.set("instance.inherited_fields", inherited);
+	}
+
+	private _templateOf(config: lib.InstanceConfig) {
+		const templateId = config.get("instance.template_id");
+		return templateId !== null ? this.records.get(templateId) : undefined;
+	}
+
+	private static _inheritableFields() {
+		return Object.keys(lib.InstanceConfig.fieldDefinitions).filter(
+			name => lib.InstanceConfig.isInheritable(name)
+		) as InheritableField[];
+	}
+
+	/** Default value of a field for the given config */
+	private static _defaultValue(name: InheritableField, config: lib.InstanceConfig): unknown {
+		const def = lib.InstanceConfig.fieldDefinitions[name];
+		if (name === "factorio.settings") {
+			return InstanceManager.DefaultFactorioSettings;
+		}
+		return typeof def.initialValue === "function" ? def.initialValue(config) : def.initialValue ?? null;
+	}
+
+	/** Values the given inherited fields take from the template, with object overrides applied */
+	private static _inheritedValues(config: lib.InstanceConfig, template: lib.InstanceConfig, fields: string[]) {
+		const inherited = config.get("instance.inherited_fields");
+		const values: Record<string, any> = {};
+		for (const field of fields) {
+			const name = field as InheritableField;
+			const entry = inherited[field];
+			if (!entry) {
+				continue;
+			}
+			const value = structuredClone(template.get(name)) as any;
+			if (entry instanceof Array) {
+				const own = config.get(name) as Record<string, unknown> ?? {};
+				for (const prop of entry) {
+					if (Object.hasOwn(own, prop)) {
+						value[prop] = structuredClone(own[prop]);
+					}
+				}
+			}
+			values[field] = value;
+		}
+		return values;
+	}
+
+	/** Update the given inherited fields of a live instance from its template */
+	private _copyInheritedValues(instance: InstanceRecord, fields: string[]): void {
+		const template = this._templateOf(instance.config);
+		if (!template) {
+			return;
+		}
+		const values = InstanceManager._inheritedValues(instance.config, template.config, fields);
+		this._inheriting += 1;
+		try {
+			instance.config.update(values, true);
+		} finally {
+			this._inheriting -= 1;
+		}
+	}
+
+	/** Unlink a field, or the changed properties of an object field, edited on the instance */
+	private _unlinkEditedField(instance: InstanceRecord, field: string, curr: unknown, prev: unknown): void {
+		const entry = instance.config.get("instance.inherited_fields")[field];
+		if (!(entry instanceof Array)) {
+			this.overrideField(instance, field);
+			return;
+		}
+		const currObj = curr as Record<string, unknown> ?? {};
+		const prevObj = prev as Record<string, unknown> ?? {};
+		const changed = Object.keys(currObj).filter(p => !isDeepStrictEqual(currObj[p], prevObj[p]));
+		const removed = Object.keys(prevObj).filter(p => !Object.hasOwn(currObj, p));
+		const overrides = [...new Set([...entry, ...changed])].filter(p => !removed.includes(p));
+		const inherited = { ...instance.config.get("instance.inherited_fields"), [field]: overrides };
+		instance.config.set("instance.inherited_fields", inherited);
+		if (removed.length) {
+			// Removed properties fall back to the template value
+			this._copyInheritedValues(instance, [field]);
+		}
+	}
+
+	/** Recompute inherited fields after template_id changed, values equal to the template or default are inherited */
+	private _linkMatchingFields(instance: InstanceRecord): void {
+		const template = this._templateOf(instance.config);
+		const inherited: InheritedFields = {};
+		if (template) {
+			for (const name of InstanceManager._inheritableFields()) {
+				const value = instance.config.get(name) as any;
+				const templateValue = template.config.get(name) as any;
+				const defaultValue = InstanceManager._defaultValue(name, instance.config) as any;
+				if (lib.InstanceConfig.fieldDefinitions[name].type === "object") {
+					inherited[name] = Object.keys(value ?? {}).filter(prop => !(
+						isDeepStrictEqual(value[prop], templateValue?.[prop])
+						|| defaultValue && isDeepStrictEqual(value[prop], defaultValue[prop])
+					) || name === "factorio.settings" && prop === "name");
+				} else if (
+					isDeepStrictEqual(value, templateValue)
+					|| defaultValue !== undefined && isDeepStrictEqual(value, defaultValue)
+				) {
+					inherited[name] = true;
+				}
+			}
+		}
+		instance.config.set("instance.inherited_fields", inherited);
+		this._copyInheritedValues(instance, Object.keys(inherited));
+	}
+
+	/** Copy a changed value from a template to the instances inheriting it */
+	private _updateInheritors(template: InstanceRecord, field: string, value: unknown): void {
+		if (!lib.InstanceConfig.isInheritable(field)) {
+			return;
+		}
+		for (const instance of this.inheritorsOf(template.id)) {
+			if (!instance.config.isInherited(field)) {
+				continue;
+			}
+			this._copyInheritedValues(instance, [field]);
+			this._controller.instanceConfigUpdated(instance).catch(err => {
+				logger.error(`Failed to send config of instance ${instance.id} to its host:\n${err.stack}`);
+			});
+		}
+	}
+
 	/* ------------------------------------------------------------------------- */
 	/* Lifecycle                                                                 */
 	/* ------------------------------------------------------------------------- */
@@ -137,12 +405,23 @@ export default class InstanceManager {
 		}
 
 		const controllerName = this._controller.config.get("controller.name");
-		instanceConfig.set("factorio.settings", {
-			name: `${controllerName} - ${instanceConfig.get("instance.name")}`,
-			description: `Clusterio instance for ${controllerName}`,
-			...InstanceManager.DefaultFactorioSettings,
-			...instanceConfig.get("factorio.settings"),
-		});
+		const serverName = `${controllerName} - ${instanceConfig.get("instance.name")}`;
+		const settings = instanceConfig.get("factorio.settings");
+		if (!instanceConfig.isInherited("factorio.settings")) {
+			instanceConfig.set("factorio.settings", {
+				name: serverName,
+				description: `Clusterio instance for ${controllerName}`,
+				...InstanceManager.DefaultFactorioSettings,
+				...settings,
+			});
+		} else if (instanceConfig.isInherited("factorio.settings", "name")) {
+			// Each instance gets its own server name
+			instanceConfig.setProp("factorio.settings", "name", serverName);
+			const inherited = instanceConfig.get("instance.inherited_fields");
+			instanceConfig.set("instance.inherited_fields", {
+				...inherited, "factorio.settings": [...inherited["factorio.settings"] as string[], "name"],
+			});
+		}
 
 		const instance = new InstanceRecord(instanceConfig, "unassigned");
 		this.addInstanceHooks(instance);
@@ -186,6 +465,9 @@ export default class InstanceManager {
 		if (currentAssignedHost === hostId) {
 			return;
 		}
+		if (hostId !== null && instance.config.get("instance.is_template")) {
+			throw new lib.ValidationError("Templates cannot be assigned to a host");
+		}
 
 		let newHostConnection: HostConnection | undefined;
 		if (hostId !== null) {
@@ -252,6 +534,10 @@ export default class InstanceManager {
 	async deleteInstance(instanceId: number): Promise<void> {
 		const instance = this.getForRequest(instanceId);
 		const hostId = instance.config.get("instance.assigned_host");
+		const inheritors = this.inheritorsOf(instanceId).length;
+		if (inheritors) {
+			throw new lib.ConflictError(`Instance is the template of ${inheritors} other instance(s)`);
+		}
 
 		if (hostId !== null) {
 			await this._controller.sendTo(

packages/controller/src/InstanceRecord.ts

@@ -83,6 +83,9 @@ export default class InstanceRecord {
 			this.startedAtMs,
 			this.updatedAtMs,
 			this.config.get("instance.exclude_from_start_all"),
+			this.config.get("instance.is_template"),
+			this.config.get("instance.template_id") ?? undefined,
+			this.config.get("factorio.mod_pack_id") ?? undefined,
 		);
 	}
 

packages/controller/src/UserManager.ts

@@ -40,8 +40,8 @@ export default class UserManager {
 			}
 
 			// We intentionally write the roles first because if it fails then the old user file will still contain them
-			await lib.safeOutputFile(rolesFilePath, JSON.stringify(combinedJson.roles));
-			await lib.safeOutputFile(usersFilePath, JSON.stringify(combinedJson.users));
+			await lib.atomicOverwriteFile(rolesFilePath, JSON.stringify(combinedJson.roles));
+			await lib.atomicOverwriteFile(usersFilePath, JSON.stringify(combinedJson.users));
 		} catch (err: any) {
 			if (err.code !== "ENOENT") {
 				throw err;
@@ -151,11 +151,34 @@ export default class UserManager {
 	 * @returns JWT access token for the user.
 	 */
 	signUserToken(user: UserRecord): string {
-		return jwt.sign(
-			{ aud: "user", user: user.id },
-			// auth secret is never null after startup
-			Buffer.from(this._controllerConfig.get("controller.auth_secret")!, "base64")
-		);
+		return jwt.sign({ aud: "user", user: user.id }, this._authSecret);
+	}
+
+	/**
+	 * Check a user token
+	 *
+	 * @param token - User token to check.
+	 * @returns the user the token was issued to.
+	 * @throws if the token is invalid, its user does not exist or it was issued before tokenValidAfter.
+	 */
+	verifyUserToken(token: string): User {
+		const payload = jwt.verify(token, this._authSecret, { audience: "user" });
+		if (typeof payload === "string") {
+			throw new Error("unexpected JsonWebToken type");
+		}
+		const user = this.getByNameMutable(payload.user);
+		if (!user) {
+			throw new Error("invalid user");
+		}
+		if (!payload.iat || payload.iat < user.tokenValidAfter) {
+			throw new Error("invalid token");
+		}
+		return user;
+	}
+
+	private get _authSecret() {
+		// auth secret is never null after startup
+		return Buffer.from(this._controllerConfig.get("controller.auth_secret")!, "base64");
 	}
 
 	/** Clear all player stats for specific instance */

packages/controller/src/WsServer.ts

@@ -252,21 +252,9 @@ ${err.stack}`
 		}
 
 		try {
-			// migrate: allow pre-rename tokens issued to hosts before alpha-14
-			const tokenPayload = jwt.verify(data.token, this.controller.authSecret, { audience: ["host", "slave"] });
-			if (typeof tokenPayload === "string") {
-				throw new Error("unexpected JsonWebToken type");
-			}
-
-			// migrate: allow pre-rename tokens issued to hosts before alpha-14
-			if ((tokenPayload.host !== undefined ? tokenPayload.host : tokenPayload.slave) !== data.id) {
+			if (this.controller.hosts.verifyHostToken(data.token).id !== data.id) {
 				throw new Error("missmatched host id");
 			}
-
-			let host = this.controller.hosts.get(data.id);
-			if (!tokenPayload.iat || tokenPayload.iat < (host?.tokenValidAfter ?? 0)) {
-				throw new Error("invalid token");
-			}
 		} catch (err: any) {
 			logger.verbose(`WsServer | authentication failed for ${this.remoteAddr(req)}`);
 			wsRejectedConnectionsCounter.inc();
@@ -303,6 +291,13 @@ ${err.stack}`
 			);
 		}
 
+		// The host may have been revoked while waiting for the old connection to close.
+		if (!this.controller.hosts.has(data.id)) {
+			wsRejectedConnectionsCounter.inc();
+			socket.close(lib.ConnectionClosed.Unauthorized, "Authentication failed: host was revoked");
+			return;
+		}
+
 		let [connector, sessionToken] = this.createSession(new lib.Address(lib.Address.host, data.id));
 
 		// Handle close before HostConnection does. This avoids a host
@@ -325,18 +320,7 @@ ${err.stack}`
 	async registerControl(data: lib.RegisterControlData, socket: WebSocket, req: IncomingMessage) {
 		let user;
 		try {
-			const tokenPayload = jwt.verify(data.token, this.controller.authSecret, { audience: "user" });
-			if (typeof tokenPayload === "string") {
-				throw new Error("unexpected JsonWebToken type");
-			}
-
-			user = this.controller.users.getByNameMutable(tokenPayload.user);
-			if (!user) {
-				throw new Error("invalid user");
-			}
-			if ((tokenPayload.iat || 0) < user.tokenValidAfter) {
-				throw new Error("invalid token");
-			}
+			user = this.controller.users.verifyUserToken(data.token);
 			user.checkPermission("core.control.connect");
 
 		} catch (err: any) {

packages/controller/src/routes.ts

@@ -1,11 +1,11 @@
-import type { Application, Request, Response } from "express";
+import express, { type Application, type Request, type Response } from "express";
 import Controller from "./Controller.js";
+import HostPairingManager, { HostPairingError } from "./HostPairingManager.js";
 
 import busboy from "busboy";
 import crypto from "crypto";
 import events from "events";
 import fs from "node:fs/promises";
-import jwt from "jsonwebtoken";
 import path from "path";
 import nodeStream from "stream";
 import util from "util";
@@ -188,17 +188,7 @@ function validateHostToken(req: Request, res: Response, next: any) {
 	}
 
 	try {
-		const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "host" });
-		if (typeof tokenPayload === "string") {
-			throw new Error("unexpected JsonWebToken type");
-		}
-		let host = req.app.locals.controller.hosts.get(tokenPayload.host);
-		if (!host) {
-			throw new Error("invalid host");
-		}
-		if (!tokenPayload.iat || tokenPayload.iat < (host.tokenValidAfter ?? 0)) {
-			throw new Error("invalid token");
-		}
+		req.app.locals.controller.hosts.verifyHostToken(token);
 	} catch (err) {
 		res.sendStatus(401);
 		return;
@@ -215,19 +205,7 @@ function validateUserToken(req: Request, res: Response, next: any) {
 	}
 
 	try {
-		const tokenPayload = jwt.verify(token, req.app.locals.controller.authSecret, { audience: "user" });
-		if (typeof tokenPayload === "string") {
-			throw new Error("unexpected JsonWebToken type");
-		}
-		let user = req.app.locals.controller.users.getByName(tokenPayload.user);
-		if (!user) {
-			throw new Error("invalid user");
-		}
-		if (!tokenPayload.iat || tokenPayload.iat < user.tokenValidAfter) {
-			throw new Error("invalid token");
-		}
-		res.locals.user = user;
-
+		res.locals.user = req.app.locals.controller.users.verifyUserToken(token);
 	} catch (err) {
 		res.sendStatus(401);
 		return;
@@ -236,6 +214,105 @@ function validateUserToken(req: Request, res: Response, next: any) {
 	next();
 }
 
+// Responds with the status of a pairing error, returns false for other errors.
+function sendHostPairingError(res: Response, err: unknown) {
+	if (!(err instanceof HostPairingError)) {
+		return false;
+	}
+	if (err.retryAfter !== undefined) {
+		res.set("Retry-After", String(err.retryAfter));
+	}
+	res.status(err.status).json({ error: err.message });
+	return true;
+}
+
+// Checks recovery mode and the request id for the pairing endpoints, returns false if responded to.
+function checkHostPairingRequest(req: Request, res: Response, needsRequestId: boolean) {
+	if (req.app.locals.controller.recoveryMode) {
+		res.status(503).json({ error: "Controller is in recovery mode" });
+		return false;
+	}
+	if (needsRequestId && typeof req.body?.requestId !== "string") {
+		res.status(400).json({ error: "requestId is required" });
+		return false;
+	}
+	return true;
+}
+
+// Request from a host for a host to join the cluster.
+function createHostPairing(req: Request, res: Response) {
+	if (!checkHostPairingRequest(req, res, false)) {
+		return;
+	}
+	const controller = req.app.locals.controller;
+	const remoteAddress = controller.wsServer.remoteAddr(req);
+	try {
+		const name = HostPairingManager.validateName(req.body?.name);
+		const version = typeof req.body?.version === "string" ? req.body.version.slice(0, 64) : "";
+
+		// A user allowed to approve hosts can skip the wait by passing their token.
+		const token = getAccessToken(req);
+		if (token !== undefined) {
+			let user;
+			try {
+				user = controller.users.verifyUserToken(token);
+			} catch (err) {
+				res.status(401).json({ error: "Invalid user token" });
+				return;
+			}
+			try {
+				user.checkPermission("core.host.approve");
+			} catch (err) {
+				logger.audit(
+					`User ${user.name} tried to approve host ${name} from ${remoteAddress} without permission`
+				);
+				res.status(403).json({ error: `User ${user.name} is not allowed to approve hosts` });
+				return;
+			}
+			const host = controller.hosts.createHost(name);
+			logger.audit(
+				`User ${user.name} approved host ${host.name} (${host.id}) from ${remoteAddress} with their token`
+			);
+			res.status(200).json({ hostId: host.id, token: controller.hosts.signHostToken(host.id), name: host.name });
+			return;
+		}
+
+		res.status(202).json(controller.hostPairing.create(name, version, remoteAddress));
+	} catch (err) {
+		if (!sendHostPairingError(res, err)) {
+			throw err;
+		}
+	}
+}
+
+// Poll from a host for the result of its request.
+function pollHostPairing(req: Request, res: Response) {
+	if (!checkHostPairingRequest(req, res, true)) {
+		return;
+	}
+	try {
+		const result = req.app.locals.controller.hostPairing.poll(req.body.requestId);
+		if (result) {
+			res.status(200).json(result);
+		} else {
+			res.status(202).json({ status: "pending" });
+		}
+	} catch (err) {
+		if (!sendHostPairingError(res, err)) {
+			throw err;
+		}
+	}
+}
+
+// Cancel from a host that stopped waiting.
+function cancelHostPairing(req: Request, res: Response) {
+	if (!checkHostPairingRequest(req, res, true)) {
+		return;
+	}
+	req.app.locals.controller.hostPairing.cancel(req.body.requestId);
+	res.sendStatus(204);
+}
+
 // Handle an uploaded export package.
 async function uploadExport(req: Request, res: Response) {
 	if (req.get("Content-Type") !== "application/zip") {
@@ -290,7 +367,7 @@ async function uploadExport(req: Request, res: Response) {
 		let { name, ext } = path.posix.parse(filePath);
 		let hash = await lib.hashStream(await zip.openStream(file));
 		assets[name] = `${name}.${hash}${ext}`;
-		await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), content);
+		await lib.atomicOverwriteFile(path.join("static", `${name}.${hash}${ext}`), content);
 	}
 
 	modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString());
@@ -300,6 +377,30 @@ async function uploadExport(req: Request, res: Response) {
 	res.sendStatus(200);
 }
 
+// Reached when no static file matched, drops exports whose files were removed
+export function missingExportAsset(req: Request, res: Response, next: any) {
+	const file = req.path.slice(1);
+	if (!["GET", "HEAD"].includes(req.method) || !file) {
+		next();
+		return;
+	}
+	const modPacks = res.app.locals.controller.modPacks;
+	const affected = [...modPacks.valuesMutable()].filter(
+		modPack => modPack.exportManifest && Object.values(modPack.exportManifest.assets).includes(file)
+	);
+	if (!affected.length) {
+		next();
+		return;
+	}
+	for (const modPack of affected) {
+		Controller.dropExport(modPack, `${file} was removed from disk`);
+	}
+	modPacks.setMany(affected);
+	res.status(404).type("text/plain").send(
+		`Export data file ${file} is missing, run export-data on an instance to regenerate it`
+	);
+}
+
 type ProxyStreamEvents = {
 	"error": [ err: any ],
 	"close": [],
@@ -716,6 +817,18 @@ export function addRouteHandlers(app: Application) {
 	);
 	app.get("/api/plugins", getPlugins);
 	app.get("/api/cluster-name", getClusterName);
+	// Set before parsing so errors from the body parser and token check are not cached either.
+	const noStore = (req: Request, res: Response, next: any) => {
+		res.set("Cache-Control", "no-store");
+		next();
+	};
+	const pairingJson = express.json({ limit: "4kb" });
+	app.post("/api/host-pairing", noStore, pairingJson, createHostPairing);
+	app.post("/api/host-pairing/poll", noStore, pairingJson, pollHostPairing);
+	app.post("/api/host-pairing/cancel", noStore, pairingJson, cancelHostPairing);
+	app.get("/api/host-pairing/check", noStore, validateHostToken, (req: Request, res: Response) => {
+		res.json({ valid: true });
+	});
 	app.put("/api/upload-export",
 		validateHostToken,
 		(req:Request, res:Response, next:any) => uploadExport(req, res).catch(next)