Merge pull request #477 from bbassie/feature/auto-restart

Add exp_auto_restart plugin

exp_auto_restart/README.md

@@ -0,0 +1,26 @@
+# ExpGaming - Auto Restart
+
+Restarts hosts, and optionally the controller, that need a restart once nobody is playing on them.
+
+A host needs a restart after a plugin or clusterio update, and after a config change that only applies on restart. The web UI shows this as "Restart Required". This plugin waits until the host has had no players online for a while and then sends it the same restart request the web UI button does.
+
+Restarting a host stops every instance on it. Once the host is back, the instances that were running are started again. Instances with auto start enabled are started by the host itself.
+
+## Requirements
+
+- Hosts must be run with `--can-restart` so their process monitor starts them again. The systemd units and run scripts written by the installer do this.
+- `controller.system_metrics_interval` must be above 0. That poll is how the controller learns a host needs a restart.
+- Installing a plugin on a host does not yet mark it as needing a restart, only updates and config changes do. Until clusterio does, restart the host by hand after an install.
+
+## Config
+
+All fields are on the controller.
+
+| Field | Default | Description |
+|---|---|---|
+| `exp_auto_restart.idle_seconds` | 300 | How long a host must have had no players online before it is restarted. |
+| `exp_auto_restart.scope` | host | With `cluster`, players on any host hold back every restart. |
+| `exp_auto_restart.start_instances` | true | Start the instances that were running again once the host is back. |
+| `exp_auto_restart.restart_controller` | false | Also restart the controller when it needs it and no players are online anywhere. |
+
+Hosts are checked every 10 seconds. A host that refuses the restart, for example because it was not run with `--can-restart`, is logged once and left alone until its process changes.

exp_auto_restart/controller.ts

@@ -0,0 +1,295 @@
+import type {
+	Controller, ControllerPluginContext, HostConnection, HostRecord, InstanceRecord,
+} from "@clusterio/controller";
+import * as lib from "@clusterio/lib";
+
+/** How often hosts are checked for a pending restart */
+export const CHECK_INTERVAL_MS = 10_000;
+/** How long to wait for a restarted host to report its instances back before giving up on starting them */
+export const PENDING_TIMEOUT_MS = 10 * 60_000;
+
+/** A host that was told to restart and has not reported all of its instances back yet */
+export interface PendingRestart {
+	requestedAtMs: number;
+	/** Instances that were running when the restart was requested */
+	instanceIds: Set<number>;
+}
+
+export class AutoRestart {
+	controller: Controller;
+	logger: lib.Logger;
+	name: string;
+
+	/** When each host, and the cluster as a whole, last became empty of players. Absent while players are online. */
+	idleSince = new Map<number | "cluster", number>();
+	/** Hosts that were told to restart and are waiting for their instances to come back */
+	pending = new Map<number, PendingRestart>();
+	/** Process start time of each host last asked to restart, so the same process is not asked twice */
+	requested = new Map<number, number>();
+	/** Hosts already warned about not being restartable */
+	warned = new Set<number | "controller">();
+	controllerRestarting = false;
+	private checkInterval?: ReturnType<typeof setInterval>;
+
+	constructor(context: ControllerPluginContext) {
+		this.controller = context.controller;
+		this.logger = context.logger;
+		this.name = context.plugin.name;
+	}
+
+	/** Attach to the controller hooks and start checking for restarts. */
+	start() {
+		const hooks = this.controller.hooks;
+		hooks.shutdown.attach(this.name, () => this.stop());
+		hooks.playerEvent.attach(this.name, (instance, event) => this.onPlayerEvent(instance, event));
+		hooks.instanceStatusChanged.attach(this.name, (instance, prev) => this.onInstanceStatusChanged(instance, prev));
+		hooks.hostConnectionEvent.attach(this.name, (connection, event) => this.onHostConnectionEvent(connection, event));
+
+		if (this.controller.config.get("controller.system_metrics_interval") <= 0) {
+			this.logger.warn(
+				"controller.system_metrics_interval is 0, so the controller never learns which hosts need a restart"
+			);
+		}
+		this.refreshIdle();
+		this.checkInterval = setInterval(() => {
+			this.check().catch((err: any) => {
+				this.logger.error(`Unexpected error checking for restarts:\n${err.stack ?? err.message}`);
+			});
+		}, CHECK_INTERVAL_MS);
+		this.checkInterval.unref();
+	}
+
+	stop() {
+		clearInterval(this.checkInterval);
+	}
+
+	onPlayerEvent(instance: InstanceRecord, event: lib.PlayerEvent) {
+		if (event.type === "join" || event.type === "leave") {
+			this.refreshIdle();
+		}
+	}
+
+	onInstanceStatusChanged(instance: InstanceRecord, prev?: lib.InstanceStatus) {
+		this.refreshIdle();
+		// A host reports every instance it has when it connects, which is how
+		// a restarted host announces it is back
+		if (prev === "unknown") {
+			this.instanceReported(instance);
+		}
+	}
+
+	onHostConnectionEvent(connection: HostConnection, event: "connect" | "drop" | "resume" | "close") {
+		if (event === "connect" || event === "close") {
+			this.refreshIdle();
+		}
+	}
+
+	/** Players online per host, counted from the users online on running instances. */
+	onlinePlayers() {
+		const counts = new Map<number, number>();
+		for (const user of this.controller.users.records.values()) {
+			if (user.isDeleted) {
+				continue;
+			}
+			for (const instanceId of user.instances) {
+				const instance = this.controller.instances.get(instanceId);
+				// Nobody is online on an instance that is not running, whatever
+				// the user records say after a host crashed
+				if (!instance || instance.status !== "running") {
+					continue;
+				}
+				const hostId = instance.config.get("instance.assigned_host");
+				if (hostId === null) {
+					continue;
+				}
+				counts.set(hostId, (counts.get(hostId) ?? 0) + 1);
+			}
+		}
+		return counts;
+	}
+
+	/** Record when connected hosts and the cluster became empty, from the players online now. */
+	refreshIdle(nowMs = Date.now()) {
+		const counts = this.onlinePlayers();
+		for (const host of this.controller.hosts.values()) {
+			if (host.isDeleted || !host.connected) {
+				this.idleSince.delete(host.id);
+				continue;
+			}
+			this.markIdle(host.id, !counts.get(host.id), nowMs);
+		}
+
+		let total = 0;
+		for (const count of counts.values()) {
+			total += count;
+		}
+		this.markIdle("cluster", total === 0, nowMs);
+	}
+
+	private markIdle(key: number | "cluster", idle: boolean, nowMs: number) {
+		if (!idle) {
+			this.idleSince.delete(key);
+		} else if (!this.idleSince.has(key)) {
+			this.idleSince.set(key, nowMs);
+		}
+	}
+
+	/** How long a host has been without players, or undefined while it has some. Uses the whole cluster with scope cluster. */
+	idleMs(hostId: number, nowMs: number) {
+		const scope = this.controller.config.get("exp_auto_restart.scope");
+		const since = this.idleSince.get(scope === "cluster" ? "cluster" : hostId);
+		return since === undefined ? undefined : nowMs - since;
+	}
+
+	/** Restart every host, and the controller if enabled, that needs it and has been empty for long enough. */
+	async check(nowMs = Date.now()) {
+		this.refreshIdle(nowMs);
+		this.expirePending(nowMs);
+		const waitMs = this.controller.config.get("exp_auto_restart.idle_seconds") * 1000;
+
+		for (const system of this.controller.systems.values()) {
+			if (system.id === "controller" || system.isDeleted || !system.restartRequired) {
+				continue;
+			}
+			const hostId = system.id;
+			const host = this.controller.hosts.get(hostId);
+			if (!host || host.isDeleted || !host.connected) {
+				continue;
+			}
+			// Already asked, either waiting for it to come back or it refused
+			if (this.pending.has(hostId) || this.requested.get(hostId) === system.processStartedAtMs) {
+				continue;
+			}
+			if (!system.canRestart) {
+				this.warnOnce(hostId, `Host ${host.name} needs a restart but is not run with --can-restart`);
+				continue;
+			}
+			const idleMs = this.idleMs(hostId, nowMs);
+			if (idleMs === undefined || idleMs < waitMs) {
+				continue;
+			}
+			await this.restartHost(host, system, idleMs, nowMs);
+		}
+
+		if (this.controller.config.get("exp_auto_restart.restart_controller")) {
+			await this.checkController(nowMs, waitMs);
+		}
+	}
+
+	async restartHost(host: Readonly<HostRecord>, system: lib.SystemInfo, idleMs: number, nowMs: number) {
+		const running = [...this.controller.instances.values()].filter(
+			instance => instance.status === "running" && instance.config.get("instance.assigned_host") === host.id
+		);
+		this.requested.set(host.id, system.processStartedAtMs);
+		if (running.length && this.controller.config.get("exp_auto_restart.start_instances")) {
+			this.pending.set(host.id, {
+				requestedAtMs: nowMs,
+				instanceIds: new Set(running.map(instance => instance.id)),
+			});
+		}
+		this.logger.info(
+			`Restarting host ${host.name} after ${Math.round(idleMs / 1000)} s without players, ` +
+			`${running.length} instance(s) running`
+		);
+
+		try {
+			await this.controller.sendTo({ hostId: host.id }, new lib.HostRestartRequest());
+		} catch (err: any) {
+			this.pending.delete(host.id);
+			// The host refusing is final for this process, anything else is retried
+			if (!(err instanceof lib.RequestError)) {
+				this.requested.delete(host.id);
+			}
+			this.logger.error(`Failed to restart host ${host.name}: ${err.message}`);
+		}
+	}
+
+	/** An instance on a host that reconnected has reported its status. */
+	instanceReported(instance: InstanceRecord) {
+		const hostId = instance.config.get("instance.assigned_host");
+		if (hostId === null) {
+			return;
+		}
+		const pending = this.pending.get(hostId);
+		if (!pending?.instanceIds.delete(instance.id)) {
+			return;
+		}
+		if (!pending.instanceIds.size) {
+			this.pending.delete(hostId);
+		}
+
+		// The host starts these itself right after reporting them
+		if (instance.status !== "stopped" || instance.config.get("instance.auto_start")) {
+			return;
+		}
+		this.startInstance(instance);
+	}
+
+	async startInstance(instance: InstanceRecord) {
+		const name = instance.config.get("instance.name");
+		this.logger.info(`Starting ${name} again after its host restarted`);
+		try {
+			await this.controller.sendTo({ instanceId: instance.id }, new lib.InstanceStartRequest());
+		} catch (err: any) {
+			this.logger.error(`Failed to start ${name} after its host restarted: ${err.message}`);
+		}
+	}
+
+	/** Forget hosts that were restarted but never reported their instances back. */
+	expirePending(nowMs: number) {
+		for (const [hostId, pending] of this.pending) {
+			if (nowMs - pending.requestedAtMs >= PENDING_TIMEOUT_MS) {
+				this.pending.delete(hostId);
+				this.logger.warn(
+					`Host ${hostId} has not reported back since it was restarted, giving up on starting its instances`
+				);
+			}
+		}
+	}
+
+	async checkController(nowMs: number, waitMs: number) {
+		if (this.controllerRestarting) {
+			return;
+		}
+		const system = this.controller.systems.get("controller");
+		if (!system?.restartRequired) {
+			return;
+		}
+		if (!system.canRestart) {
+			this.warnOnce("controller", "The controller needs a restart but is not run with --can-restart");
+			return;
+		}
+		const since = this.idleSince.get("cluster");
+		if (since === undefined || nowMs - since < waitMs) {
+			return;
+		}
+		const downgrade = await this.controller.checkRestartDowngrade();
+		if (downgrade) {
+			this.warnOnce(
+				"controller",
+				`Not restarting the controller because installed version ${downgrade.installedVersion} ` +
+				`is older than running version ${downgrade.runningVersion}`
+			);
+			return;
+		}
+
+		this.controllerRestarting = true;
+		this.logger.info(`Restarting controller after ${Math.round((nowMs - since) / 1000)} s without players`);
+		this.controller.shouldRestart = true;
+		this.controller.stop().catch((err: any) => {
+			this.logger.error(`Failed to restart the controller: ${err.message}`);
+		});
+	}
+
+	private warnOnce(key: number | "controller", message: string) {
+		if (this.warned.has(key)) {
+			return;
+		}
+		this.warned.add(key);
+		this.logger.warn(message);
+	}
+}
+
+export default async function (context: ControllerPluginContext) {
+	new AutoRestart(context).start();
+}

exp_auto_restart/index.ts

@@ -0,0 +1,52 @@
+import * as lib from "@clusterio/lib";
+
+declare module "@clusterio/lib" {
+	export interface ControllerConfigFields {
+		"exp_auto_restart.idle_seconds": number;
+		"exp_auto_restart.scope": "host" | "cluster";
+		"exp_auto_restart.start_instances": boolean;
+		"exp_auto_restart.restart_controller": boolean;
+	}
+}
+
+export const plugin: lib.PluginDeclaration = {
+	name: "exp_auto_restart",
+	title: "ExpGaming - Auto Restart",
+	description: "Clusterio plugin restarting hosts and the controller once no players are online",
+
+	controllerEntrypoint: "./dist/node/controller.js",
+	controllerConfigFields: {
+		"exp_auto_restart.idle_seconds": {
+			title: "Idle Seconds",
+			description: "How long a host must have had no players online before it is restarted",
+			type: "number",
+			initialValue: 300,
+			validator: (value: number) => {
+				if (value < 0) {
+					throw new Error("Idle seconds cannot be negative");
+				}
+			},
+		},
+		"exp_auto_restart.scope": {
+			title: "Scope",
+			description: "Whether players on other hosts also hold back a restart",
+			type: "string",
+			enum: ["host", "cluster"],
+			initialValue: "host",
+		},
+		"exp_auto_restart.start_instances": {
+			title: "Start Instances",
+			description: "Start the instances that were running again once the host is back",
+			type: "boolean",
+			initialValue: true,
+		},
+		"exp_auto_restart.restart_controller": {
+			title: "Restart Controller",
+			description: "Also restart the controller when it needs it and no players are online anywhere",
+			type: "boolean",
+			initialValue: false,
+		},
+	},
+
+	webEntrypoint: "./web",
+};

exp_auto_restart/package.json

@@ -0,0 +1,40 @@
+{
+	"name": "@expcluster/auto-restart",
+	"version": "7.0.1",
+	"description": "Clusterio plugin restarting hosts and the controller once no players are online.",
+	"author": "Cooldude2606 <https://github.com/Cooldude2606>",
+	"license": "MIT",
+	"repository": "explosivegaming/ExpCluster",
+	"type": "module",
+	"main": "dist/node/index.js",
+	"scripts": {
+		"prepare": "tsc --build && webpack-cli --env production",
+		"test": "tap --type-strip-only --disable-coverage --allow-empty-coverage test/*.test.js",
+		"coverage": "tap --coverage-report=text test/*.test.js"
+	},
+	"engines": {
+		"node": ">=18"
+	},
+	"peerDependencies": {
+		"@clusterio/controller": "workspace:^",
+		"@clusterio/lib": "workspace:^"
+	},
+	"devDependencies": {
+		"@clusterio/controller": "workspace:^",
+		"@clusterio/lib": "workspace:^",
+		"@types/node": "catalog:",
+		"tap": "^21.1.0",
+		"typescript": "catalog:",
+		"webpack": "catalog:",
+		"webpack-cli": "catalog:",
+		"webpack-merge": "catalog:"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"keywords": [
+		"clusterio",
+		"clusterio-plugin",
+		"factorio"
+	]
+}

exp_auto_restart/test/controller.test.js

@@ -0,0 +1,353 @@
+import t from "tap";
+import * as lib from "@clusterio/lib";
+import { Controller, HostRecord, InstanceRecord } from "@clusterio/controller";
+import entrypoint, { AutoRestart, PENDING_TIMEOUT_MS } from "../dist/node/controller.js";
+import { plugin as pluginDeclaration } from "../dist/node/index.js";
+
+// The plugin's config fields must be defined before a ControllerConfig can set them
+lib.addPluginConfigFields([pluginDeclaration]);
+
+// Silence the singleton logger to avoid polluting the test output
+lib.logger.silent = true;
+t.after(() => {
+	lib.logger.silent = false;
+});
+
+/** Build the plugin around a real controller, which is side effect free while not started. */
+async function startPlugin(t2, { config = {}, canRestart = false } = {}) {
+	const logs = [];
+	const logger = {
+		child: () => logger,
+		info: message => logs.push(["info", message]),
+		warn: message => logs.push(["warn", message]),
+		error: message => logs.push(["error", message]),
+		verbose: () => {},
+	};
+	const controllerConfig = new lib.ControllerConfig("controller", {
+		"controller.database_directory": t2.testdir(),
+		...config,
+	});
+	const controller = new Controller(logger, [], controllerConfig, canRestart);
+
+	// Record what would be sent to hosts and instances instead of sending it
+	const sent = [];
+	controller.sendTo = async (address, message) => {
+		sent.push({ address, message });
+	};
+
+	const plugin = new AutoRestart({ controller, logger, plugin: pluginDeclaration, metrics: undefined });
+	plugin.start();
+	t2.teardown(() => plugin.stop());
+	return { plugin, controller, sent, logs };
+}
+
+function addHost(controller, id, connected = true) {
+	controller.hosts.set(new HostRecord(id, `host-${id}`, "2.0.0", new Map(), connected));
+}
+
+function addInstance(controller, id, hostId, status = "running", fields = {}) {
+	const config = new lib.InstanceConfig("controller", {
+		"instance.id": id,
+		"instance.name": `instance-${id}`,
+		"instance.assigned_host": hostId,
+		...fields,
+	});
+	const record = new InstanceRecord(config, status);
+	controller.instances.records.set(record);
+	return record;
+}
+
+function addSystem(controller, id, { restartRequired = true, canRestart = true, processStartedAtMs = 1000 } = {}) {
+	controller.systems.set(new lib.SystemInfo(
+		id, "box", "v22", "Linux", "x64", "cpu", [], 0, 0, 0, 0,
+		canRestart, restartRequired, 0, processStartedAtMs, Date.now(), false,
+	));
+}
+
+const join = (controller, name, instanceId) => controller.users.getOrCreateUser(name).notifyJoin(instanceId);
+const leave = (controller, name, instanceId) => controller.users.getOrCreateUser(name).notifyLeave(instanceId);
+const restartsSent = sent => sent.filter(({ message }) => message instanceof lib.HostRestartRequest);
+const startsSent = sent => sent.filter(({ message }) => message instanceof lib.InstanceStartRequest);
+
+t.test("entrypoint", async t2 => {
+	const logger = { child: () => logger, info: () => {}, warn: () => {}, error: () => {}, verbose: () => {} };
+	const controllerConfig = new lib.ControllerConfig("controller", { "controller.database_directory": t2.testdir() });
+	const controller = new Controller(logger, [], controllerConfig);
+	await entrypoint({ controller, logger, plugin: pluginDeclaration, metrics: undefined });
+	for (const hook of ["shutdown", "playerEvent", "instanceStatusChanged", "hostConnectionEvent"]) {
+		t2.ok([...controller.hooks[hook].attached].includes("exp_auto_restart"), `attached to ${hook}`);
+	}
+	await controller.hooks.shutdown.invoke();
+});
+
+t.test("class AutoRestart", t2 => {
+	t2.test(".onlinePlayers() counts players on running instances per host", async t3 => {
+		const { plugin, controller } = await startPlugin(t3);
+		addHost(controller, 1);
+		addHost(controller, 2);
+		addInstance(controller, 10, 1);
+		addInstance(controller, 11, 1, "stopped");
+		addInstance(controller, 20, 2);
+		join(controller, "alice", 10);
+		join(controller, "bob", 10);
+		join(controller, "carol", 11);
+		join(controller, "dave", 20);
+
+		t3.strictSame(plugin.onlinePlayers(), new Map([[1, 2], [2, 1]]), "players on stopped instances are not counted");
+	});
+
+	t2.test(".refreshIdle() tracks when hosts and the cluster became empty", async t3 => {
+		const { plugin, controller } = await startPlugin(t3);
+		addHost(controller, 1);
+		addHost(controller, 2);
+		addHost(controller, 3, false);
+		addInstance(controller, 10, 1);
+		join(controller, "alice", 10);
+
+		plugin.refreshIdle(1000);
+		t3.notOk(plugin.idleSince.has(1), "a host with players is not idle");
+		t3.strictSame(plugin.idleSince.get(2), 1000, "an empty host is idle from now");
+		t3.notOk(plugin.idleSince.has(3), "a disconnected host is not tracked");
+		t3.notOk(plugin.idleSince.has("cluster"), "the cluster is not idle while anyone is online");
+
+		leave(controller, "alice", 10);
+		plugin.refreshIdle(2000);
+		t3.strictSame(plugin.idleSince.get(1), 2000, "a host becomes idle when its last player leaves");
+		t3.strictSame(plugin.idleSince.get(2), 1000, "an idle host keeps its original time");
+		t3.strictSame(plugin.idleSince.get("cluster"), 2000, "the cluster becomes idle with the last player");
+
+		join(controller, "alice", 10);
+		plugin.refreshIdle(3000);
+		t3.notOk(plugin.idleSince.has(1), "a join clears the idle time");
+		t3.notOk(plugin.idleSince.has("cluster"));
+	});
+
+	t2.test(".check() restarts a host once it has been empty for long enough", async t3 => {
+		const { plugin, controller, sent } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 60 },
+		});
+		const now = Date.now();
+		addHost(controller, 1);
+		addInstance(controller, 10, 1);
+		addInstance(controller, 11, 1, "stopped");
+		addSystem(controller, 1);
+		join(controller, "alice", 10);
+
+		await plugin.check(now);
+		t3.strictSame(restartsSent(sent).length, 0, "not restarted while a player is online");
+
+		leave(controller, "alice", 10);
+		await plugin.check(now + 10_000);
+		t3.strictSame(restartsSent(sent).length, 0, "not restarted before the idle time has passed");
+
+		await plugin.check(now + 70_000);
+		t3.strictSame(restartsSent(sent).length, 1, "restarted once the idle time has passed");
+		t3.strictSame(restartsSent(sent)[0].address, { hostId: 1 }, "the restart is sent to the host");
+		t3.strictSame([...plugin.pending.get(1).instanceIds], [10], "only the running instances are remembered");
+
+		await plugin.check(now + 80_000);
+		t3.strictSame(restartsSent(sent).length, 1, "not restarted again while waiting for it to come back");
+	});
+
+	t2.test(".check() asks each host process only once", async t3 => {
+		const { plugin, controller, sent } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0 },
+		});
+		const now = Date.now();
+		addHost(controller, 1);
+		addSystem(controller, 1, { processStartedAtMs: 1000 });
+
+		await plugin.check(now);
+		t3.strictSame(restartsSent(sent).length, 1, "restarted with no running instances");
+		t3.notOk(plugin.pending.has(1), "nothing to wait for without running instances");
+
+		await plugin.check(now + 10_000);
+		t3.strictSame(restartsSent(sent).length, 1, "the same process is not asked again");
+
+		addSystem(controller, 1, { restartRequired: false, processStartedAtMs: 2000 });
+		await plugin.check(now + 20_000);
+		t3.strictSame(restartsSent(sent).length, 1, "the new process does not need a restart");
+
+		addSystem(controller, 1, { processStartedAtMs: 2000 });
+		await plugin.check(now + 30_000);
+		t3.strictSame(restartsSent(sent).length, 2, "the new process is restarted when it needs it");
+	});
+
+	t2.test(".check() leaves hosts alone that cannot or need not restart", async t3 => {
+		const { plugin, controller, sent, logs } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0 },
+		});
+		const now = Date.now();
+		addHost(controller, 1);
+		addSystem(controller, 1, { canRestart: false });
+		addHost(controller, 2);
+		addSystem(controller, 2, { restartRequired: false });
+		addHost(controller, 3, false);
+		addSystem(controller, 3);
+
+		await plugin.check(now);
+		await plugin.check(now + 10_000);
+		t3.strictSame(restartsSent(sent).length, 0, "no host is restarted");
+		const warnings = logs.filter(([level, message]) => level === "warn" && message.includes("--can-restart"));
+		t3.strictSame(warnings.length, 1, "a host without a process monitor is warned about once");
+	});
+
+	t2.test(".check() with scope cluster waits for the whole cluster to be empty", async t3 => {
+		const { plugin, controller, sent } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.scope": "cluster" },
+		});
+		const now = Date.now();
+		addHost(controller, 1);
+		addSystem(controller, 1);
+		addHost(controller, 2);
+		addInstance(controller, 20, 2);
+		join(controller, "alice", 20);
+
+		await plugin.check(now);
+		t3.strictSame(restartsSent(sent).length, 0, "a player on another host holds back the restart");
+
+		leave(controller, "alice", 20);
+		await plugin.check(now + 10_000);
+		t3.strictSame(restartsSent(sent).length, 1, "restarted once the cluster is empty");
+	});
+
+	t2.test(".restartHost() retries transport errors but not refusals", async t3 => {
+		const { plugin, controller, sent, logs } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0 },
+		});
+		const now = Date.now();
+		addHost(controller, 1);
+		addInstance(controller, 10, 1);
+		addSystem(controller, 1);
+
+		controller.sendTo = async () => { throw new lib.RequestError("Cannot restart"); };
+		await plugin.check(now);
+		await plugin.check(now + 10_000);
+		t3.strictSame(logs.filter(([level]) => level === "error").length, 1, "a refusal is logged once");
+		t3.notOk(plugin.pending.has(1), "a refused restart is not waited on");
+
+		addSystem(controller, 1, { processStartedAtMs: 2000 });
+		let attempts = 0;
+		controller.sendTo = async () => { attempts += 1; throw new Error("Connection lost"); };
+		await plugin.check(now + 20_000);
+		await plugin.check(now + 30_000);
+		t3.strictSame(attempts, 2, "a transport error is retried on the next check");
+	});
+
+	t2.test(".instanceReported() starts the instances that were running", async t3 => {
+		const { plugin, controller, sent } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0 },
+		});
+		const now = Date.now();
+		addHost(controller, 1);
+		const manual = addInstance(controller, 10, 1);
+		const auto = addInstance(controller, 11, 1, "running", { "instance.auto_start": true });
+		const stopped = addInstance(controller, 12, 1, "stopped");
+		addSystem(controller, 1);
+		await plugin.check(now);
+		t3.strictSame([...plugin.pending.get(1).instanceIds], [10, 11]);
+
+		// The host comes back and reports every instance as stopped
+		for (const instance of [manual, auto, stopped]) {
+			instance.status = "stopped";
+			await plugin.onInstanceStatusChanged(instance, "unknown");
+		}
+		t3.strictSame(startsSent(sent).map(({ address }) => address), [{ instanceId: 10 }], "only the instance the host will not start itself is started");
+		t3.notOk(plugin.pending.has(1), "the host is no longer waited on");
+
+		// An instance reporting in without a pending restart is ignored
+		await plugin.onInstanceStatusChanged(manual, "unknown");
+		t3.strictSame(startsSent(sent).length, 1);
+	});
+
+	t2.test(".restartHost() does not wait on instances when they are not to be started", async t3 => {
+		const { plugin, controller, sent } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.start_instances": false },
+		});
+		addHost(controller, 1);
+		const instance = addInstance(controller, 10, 1);
+		addSystem(controller, 1);
+		await plugin.check(Date.now());
+		t3.strictSame(restartsSent(sent).length, 1);
+		t3.notOk(plugin.pending.has(1), "nothing is waited on");
+
+		instance.status = "stopped";
+		await plugin.onInstanceStatusChanged(instance, "unknown");
+		t3.strictSame(startsSent(sent).length, 0, "nothing is started");
+	});
+
+	t2.test(".expirePending() gives up on hosts that never come back", async t3 => {
+		const { plugin, controller, logs } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0 },
+		});
+		const now = Date.now();
+		addHost(controller, 1);
+		addInstance(controller, 10, 1);
+		addSystem(controller, 1);
+		await plugin.check(now);
+		t3.ok(plugin.pending.has(1));
+
+		await plugin.check(now + PENDING_TIMEOUT_MS - 1);
+		t3.ok(plugin.pending.has(1), "still waited on before the timeout");
+		await plugin.check(now + PENDING_TIMEOUT_MS);
+		t3.notOk(plugin.pending.has(1), "no longer waited on after the timeout");
+		t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("giving up")), "giving up is logged");
+	});
+
+	t2.test(".checkController() restarts the controller when the cluster is empty", async t3 => {
+		const { plugin, controller } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 60, "exp_auto_restart.restart_controller": true },
+			canRestart: true,
+		});
+		const now = Date.now();
+		let stopped = 0;
+		controller.stop = async () => { stopped += 1; };
+		controller.checkRestartDowngrade = async () => null;
+		addHost(controller, 1);
+		addInstance(controller, 10, 1);
+		addSystem(controller, "controller");
+		join(controller, "alice", 10);
+
+		await plugin.check(now);
+		t3.strictSame(stopped, 0, "not restarted while a player is online");
+
+		leave(controller, "alice", 10);
+		await plugin.check(now + 10_000);
+		t3.strictSame(stopped, 0, "not restarted before the idle time has passed");
+
+		await plugin.check(now + 70_000);
+		t3.strictSame(stopped, 1, "restarted once the cluster has been empty for long enough");
+		t3.ok(controller.shouldRestart, "stopped with the intent to restart");
+
+		await plugin.check(now + 80_000);
+		t3.strictSame(stopped, 1, "not restarted twice");
+	});
+
+	t2.test(".checkController() refuses downgrades and missing process monitors", async t3 => {
+		const { plugin, controller, logs } = await startPlugin(t3, {
+			config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.restart_controller": true },
+			canRestart: true,
+		});
+		let stopped = 0;
+		controller.stop = async () => { stopped += 1; };
+		controller.checkRestartDowngrade = async () => ({ installedVersion: "2.0.0", runningVersion: "2.0.1" });
+		addSystem(controller, "controller");
+
+		await plugin.check(Date.now());
+		t3.strictSame(stopped, 0, "not restarted into an older version");
+		t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("older")), "the downgrade is logged");
+
+		plugin.warned.clear();
+		addSystem(controller, "controller", { canRestart: false });
+		await plugin.check(Date.now());
+		t3.strictSame(stopped, 0, "not restarted without a process monitor");
+		t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("--can-restart")));
+	});
+
+	t2.test(".start() warns when system metrics are disabled", async t3 => {
+		const { logs } = await startPlugin(t3, { config: { "controller.system_metrics_interval": 0 } });
+		t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("system_metrics_interval")));
+	});
+
+	t2.end();
+});

exp_auto_restart/tsconfig.browser.json

@@ -0,0 +1,4 @@
+{
+	"extends": "../tsconfig.browser.json",
+	"include": [ "web/**/*.tsx", "web/**/*.ts", "package.json" ],
+}

exp_auto_restart/tsconfig.json

@@ -0,0 +1,7 @@
+{
+	"files": [],
+	"references": [
+		{ "path": "./tsconfig.node.json" },
+		{ "path": "./tsconfig.browser.json" }
+	]
+}

exp_auto_restart/tsconfig.node.json

@@ -0,0 +1,5 @@
+{
+	"extends": "../tsconfig.node.json",
+	"include": ["./**/*.ts"],
+	"exclude": ["test/*", "./dist/*"],
+}

exp_auto_restart/webpack.config.cjs

@@ -0,0 +1,29 @@
+"use strict";
+const path = require("path");
+const webpack = require("webpack");
+const { merge } = require("webpack-merge");
+
+const common = require("@clusterio/web_ui/webpack.common");
+
+module.exports = (env = {}) => merge(common(env), {
+	context: __dirname,
+	entry: "./web/index.tsx",
+	output: {
+		path: path.resolve(__dirname, "dist", "web"),
+	},
+	plugins: [
+		new webpack.container.ModuleFederationPlugin({
+			name: "exp_auto_restart",
+			library: { type: "window", name: "plugin_exp_auto_restart" },
+			exposes: {
+				"./": "./index.ts",
+				"./package.json": "./package.json",
+				"./web": "./web/index.tsx",
+			},
+			shared: {
+				"@clusterio/lib": { import: false },
+				"@clusterio/web_ui": { import: false },
+			},
+		}),
+	],
+});

tsconfig.json

@@ -1,6 +1,7 @@
 {
 	"files": [],
 	"references": [
+		{ "path": "./exp_auto_restart/" },
 		{ "path": "./exp_commands/" },
 		{ "path": "./exp_groups/" },
 		{ "path": "./exp_gui/" },