From 5a6f3b46fee664e60e201af4b537a2bc65577076 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:39:31 +0000 Subject: [PATCH] Add exp_auto_restart plugin Restarts hosts, and optionally the controller, that report a pending restart once no players have been online for a configurable time. Only loaded on the controller: it counts players from the users online on running instances, sends HostRestartRequest through the existing core path, and starts the instances that were running once the host reports back. Each host process is asked at most once, keyed on its process start time, so a stale system info poll cannot trigger a second restart. Uses the function-style entrypoint clusterio now documents, so it loads without the deprecated class export warning. Co-Authored-By: Claude Fable 5.1 --- exp_auto_restart/README.md | 26 ++ exp_auto_restart/controller.ts | 295 +++++++++++++++++++ exp_auto_restart/index.ts | 50 ++++ exp_auto_restart/package.json | 37 +++ exp_auto_restart/test/controller.test.js | 353 +++++++++++++++++++++++ exp_auto_restart/tsconfig.json | 6 + exp_auto_restart/tsconfig.node.json | 5 + tsconfig.json | 1 + 8 files changed, 773 insertions(+) create mode 100644 exp_auto_restart/README.md create mode 100644 exp_auto_restart/controller.ts create mode 100644 exp_auto_restart/index.ts create mode 100644 exp_auto_restart/package.json create mode 100644 exp_auto_restart/test/controller.test.js create mode 100644 exp_auto_restart/tsconfig.json create mode 100644 exp_auto_restart/tsconfig.node.json diff --git a/exp_auto_restart/README.md b/exp_auto_restart/README.md new file mode 100644 index 00000000..954b9b09 --- /dev/null +++ b/exp_auto_restart/README.md @@ -0,0 +1,26 @@ +# ExpGaming - Auto Restart + +Restarts hosts, and optionally the controller, that need a restart once nobody is playing on them. + +A host needs a restart after a plugin or clusterio update, and after a config change that only applies on restart. The web UI shows this as "Restart Required". This plugin waits until the host has had no players online for a while and then sends it the same restart request the web UI button does. + +Restarting a host stops every instance on it. Once the host is back, the instances that were running are started again. Instances with auto start enabled are started by the host itself. + +## Requirements + +- Hosts must be run with `--can-restart` so their process monitor starts them again. The systemd units and run scripts written by the installer do this. +- `controller.system_metrics_interval` must be above 0. That poll is how the controller learns a host needs a restart. +- Installing a plugin on a host does not yet mark it as needing a restart, only updates and config changes do. Until clusterio does, restart the host by hand after an install. + +## Config + +All fields are on the controller. + +| Field | Default | Description | +|---|---|---| +| `exp_auto_restart.idle_seconds` | 300 | How long a host must have had no players online before it is restarted. | +| `exp_auto_restart.scope` | host | With `cluster`, players on any host hold back every restart. | +| `exp_auto_restart.start_instances` | true | Start the instances that were running again once the host is back. | +| `exp_auto_restart.restart_controller` | false | Also restart the controller when it needs it and no players are online anywhere. | + +Hosts are checked every 10 seconds. A host that refuses the restart, for example because it was not run with `--can-restart`, is logged once and left alone until its process changes. diff --git a/exp_auto_restart/controller.ts b/exp_auto_restart/controller.ts new file mode 100644 index 00000000..130dd2b0 --- /dev/null +++ b/exp_auto_restart/controller.ts @@ -0,0 +1,295 @@ +import type { + Controller, ControllerPluginContext, HostConnection, HostRecord, InstanceRecord, +} from "@clusterio/controller"; +import * as lib from "@clusterio/lib"; + +/** How often hosts are checked for a pending restart */ +export const CHECK_INTERVAL_MS = 10_000; +/** How long to wait for a restarted host to report its instances back before giving up on starting them */ +export const PENDING_TIMEOUT_MS = 10 * 60_000; + +/** A host that was told to restart and has not reported all of its instances back yet */ +export interface PendingRestart { + requestedAtMs: number; + /** Instances that were running when the restart was requested */ + instanceIds: Set; +} + +export class AutoRestart { + controller: Controller; + logger: lib.Logger; + name: string; + + /** When each host, and the cluster as a whole, last became empty of players. Absent while players are online. */ + idleSince = new Map(); + /** Hosts that were told to restart and are waiting for their instances to come back */ + pending = new Map(); + /** Process start time of each host last asked to restart, so the same process is not asked twice */ + requested = new Map(); + /** Hosts already warned about not being restartable */ + warned = new Set(); + controllerRestarting = false; + private checkInterval?: ReturnType; + + constructor(context: ControllerPluginContext) { + this.controller = context.controller; + this.logger = context.logger; + this.name = context.plugin.name; + } + + /** Attach to the controller hooks and start checking for restarts. */ + start() { + const hooks = this.controller.hooks; + hooks.shutdown.attach(this.name, () => this.stop()); + hooks.playerEvent.attach(this.name, (instance, event) => this.onPlayerEvent(instance, event)); + hooks.instanceStatusChanged.attach(this.name, (instance, prev) => this.onInstanceStatusChanged(instance, prev)); + hooks.hostConnectionEvent.attach(this.name, (connection, event) => this.onHostConnectionEvent(connection, event)); + + if (this.controller.config.get("controller.system_metrics_interval") <= 0) { + this.logger.warn( + "controller.system_metrics_interval is 0, so the controller never learns which hosts need a restart" + ); + } + this.refreshIdle(); + this.checkInterval = setInterval(() => { + this.check().catch((err: any) => { + this.logger.error(`Unexpected error checking for restarts:\n${err.stack ?? err.message}`); + }); + }, CHECK_INTERVAL_MS); + this.checkInterval.unref(); + } + + stop() { + clearInterval(this.checkInterval); + } + + onPlayerEvent(instance: InstanceRecord, event: lib.PlayerEvent) { + if (event.type === "join" || event.type === "leave") { + this.refreshIdle(); + } + } + + onInstanceStatusChanged(instance: InstanceRecord, prev?: lib.InstanceStatus) { + this.refreshIdle(); + // A host reports every instance it has when it connects, which is how + // a restarted host announces it is back + if (prev === "unknown") { + this.instanceReported(instance); + } + } + + onHostConnectionEvent(connection: HostConnection, event: "connect" | "drop" | "resume" | "close") { + if (event === "connect" || event === "close") { + this.refreshIdle(); + } + } + + /** Players online per host, counted from the users online on running instances. */ + onlinePlayers() { + const counts = new Map(); + for (const user of this.controller.users.records.values()) { + if (user.isDeleted) { + continue; + } + for (const instanceId of user.instances) { + const instance = this.controller.instances.get(instanceId); + // Nobody is online on an instance that is not running, whatever + // the user records say after a host crashed + if (!instance || instance.status !== "running") { + continue; + } + const hostId = instance.config.get("instance.assigned_host"); + if (hostId === null) { + continue; + } + counts.set(hostId, (counts.get(hostId) ?? 0) + 1); + } + } + return counts; + } + + /** Record when connected hosts and the cluster became empty, from the players online now. */ + refreshIdle(nowMs = Date.now()) { + const counts = this.onlinePlayers(); + for (const host of this.controller.hosts.values()) { + if (host.isDeleted || !host.connected) { + this.idleSince.delete(host.id); + continue; + } + this.markIdle(host.id, !counts.get(host.id), nowMs); + } + + let total = 0; + for (const count of counts.values()) { + total += count; + } + this.markIdle("cluster", total === 0, nowMs); + } + + private markIdle(key: number | "cluster", idle: boolean, nowMs: number) { + if (!idle) { + this.idleSince.delete(key); + } else if (!this.idleSince.has(key)) { + this.idleSince.set(key, nowMs); + } + } + + /** How long a host has been without players, or undefined while it has some. Uses the whole cluster with scope cluster. */ + idleMs(hostId: number, nowMs: number) { + const scope = this.controller.config.get("exp_auto_restart.scope"); + const since = this.idleSince.get(scope === "cluster" ? "cluster" : hostId); + return since === undefined ? undefined : nowMs - since; + } + + /** Restart every host, and the controller if enabled, that needs it and has been empty for long enough. */ + async check(nowMs = Date.now()) { + this.refreshIdle(nowMs); + this.expirePending(nowMs); + const waitMs = this.controller.config.get("exp_auto_restart.idle_seconds") * 1000; + + for (const system of this.controller.systems.values()) { + if (system.id === "controller" || system.isDeleted || !system.restartRequired) { + continue; + } + const hostId = system.id; + const host = this.controller.hosts.get(hostId); + if (!host || host.isDeleted || !host.connected) { + continue; + } + // Already asked, either waiting for it to come back or it refused + if (this.pending.has(hostId) || this.requested.get(hostId) === system.processStartedAtMs) { + continue; + } + if (!system.canRestart) { + this.warnOnce(hostId, `Host ${host.name} needs a restart but is not run with --can-restart`); + continue; + } + const idleMs = this.idleMs(hostId, nowMs); + if (idleMs === undefined || idleMs < waitMs) { + continue; + } + await this.restartHost(host, system, idleMs, nowMs); + } + + if (this.controller.config.get("exp_auto_restart.restart_controller")) { + await this.checkController(nowMs, waitMs); + } + } + + async restartHost(host: Readonly, system: lib.SystemInfo, idleMs: number, nowMs: number) { + const running = [...this.controller.instances.values()].filter( + instance => instance.status === "running" && instance.config.get("instance.assigned_host") === host.id + ); + this.requested.set(host.id, system.processStartedAtMs); + if (running.length && this.controller.config.get("exp_auto_restart.start_instances")) { + this.pending.set(host.id, { + requestedAtMs: nowMs, + instanceIds: new Set(running.map(instance => instance.id)), + }); + } + this.logger.info( + `Restarting host ${host.name} after ${Math.round(idleMs / 1000)} s without players, ` + + `${running.length} instance(s) running` + ); + + try { + await this.controller.sendTo({ hostId: host.id }, new lib.HostRestartRequest()); + } catch (err: any) { + this.pending.delete(host.id); + // The host refusing is final for this process, anything else is retried + if (!(err instanceof lib.RequestError)) { + this.requested.delete(host.id); + } + this.logger.error(`Failed to restart host ${host.name}: ${err.message}`); + } + } + + /** An instance on a host that reconnected has reported its status. */ + instanceReported(instance: InstanceRecord) { + const hostId = instance.config.get("instance.assigned_host"); + if (hostId === null) { + return; + } + const pending = this.pending.get(hostId); + if (!pending?.instanceIds.delete(instance.id)) { + return; + } + if (!pending.instanceIds.size) { + this.pending.delete(hostId); + } + + // The host starts these itself right after reporting them + if (instance.status !== "stopped" || instance.config.get("instance.auto_start")) { + return; + } + this.startInstance(instance); + } + + async startInstance(instance: InstanceRecord) { + const name = instance.config.get("instance.name"); + this.logger.info(`Starting ${name} again after its host restarted`); + try { + await this.controller.sendTo({ instanceId: instance.id }, new lib.InstanceStartRequest()); + } catch (err: any) { + this.logger.error(`Failed to start ${name} after its host restarted: ${err.message}`); + } + } + + /** Forget hosts that were restarted but never reported their instances back. */ + expirePending(nowMs: number) { + for (const [hostId, pending] of this.pending) { + if (nowMs - pending.requestedAtMs >= PENDING_TIMEOUT_MS) { + this.pending.delete(hostId); + this.logger.warn( + `Host ${hostId} has not reported back since it was restarted, giving up on starting its instances` + ); + } + } + } + + async checkController(nowMs: number, waitMs: number) { + if (this.controllerRestarting) { + return; + } + const system = this.controller.systems.get("controller"); + if (!system?.restartRequired) { + return; + } + if (!system.canRestart) { + this.warnOnce("controller", "The controller needs a restart but is not run with --can-restart"); + return; + } + const since = this.idleSince.get("cluster"); + if (since === undefined || nowMs - since < waitMs) { + return; + } + const downgrade = await this.controller.checkRestartDowngrade(); + if (downgrade) { + this.warnOnce( + "controller", + `Not restarting the controller because installed version ${downgrade.installedVersion} ` + + `is older than running version ${downgrade.runningVersion}` + ); + return; + } + + this.controllerRestarting = true; + this.logger.info(`Restarting controller after ${Math.round((nowMs - since) / 1000)} s without players`); + this.controller.shouldRestart = true; + this.controller.stop().catch((err: any) => { + this.logger.error(`Failed to restart the controller: ${err.message}`); + }); + } + + private warnOnce(key: number | "controller", message: string) { + if (this.warned.has(key)) { + return; + } + this.warned.add(key); + this.logger.warn(message); + } +} + +export default async function (context: ControllerPluginContext) { + new AutoRestart(context).start(); +} diff --git a/exp_auto_restart/index.ts b/exp_auto_restart/index.ts new file mode 100644 index 00000000..27420411 --- /dev/null +++ b/exp_auto_restart/index.ts @@ -0,0 +1,50 @@ +import * as lib from "@clusterio/lib"; + +declare module "@clusterio/lib" { + export interface ControllerConfigFields { + "exp_auto_restart.idle_seconds": number; + "exp_auto_restart.scope": "host" | "cluster"; + "exp_auto_restart.start_instances": boolean; + "exp_auto_restart.restart_controller": boolean; + } +} + +export const plugin: lib.PluginDeclaration = { + name: "exp_auto_restart", + title: "ExpGaming - Auto Restart", + description: "Clusterio plugin restarting hosts and the controller once no players are online", + + controllerEntrypoint: "./dist/node/controller.js", + controllerConfigFields: { + "exp_auto_restart.idle_seconds": { + title: "Idle Seconds", + description: "How long a host must have had no players online before it is restarted", + type: "number", + initialValue: 300, + validator: (value: number) => { + if (value < 0) { + throw new Error("Idle seconds cannot be negative"); + } + }, + }, + "exp_auto_restart.scope": { + title: "Scope", + description: "Whether players on other hosts also hold back a restart", + type: "string", + enum: ["host", "cluster"], + initialValue: "host", + }, + "exp_auto_restart.start_instances": { + title: "Start Instances", + description: "Start the instances that were running again once the host is back", + type: "boolean", + initialValue: true, + }, + "exp_auto_restart.restart_controller": { + title: "Restart Controller", + description: "Also restart the controller when it needs it and no players are online anywhere", + type: "boolean", + initialValue: false, + }, + }, +}; diff --git a/exp_auto_restart/package.json b/exp_auto_restart/package.json new file mode 100644 index 00000000..57df55ac --- /dev/null +++ b/exp_auto_restart/package.json @@ -0,0 +1,37 @@ +{ + "name": "@expcluster/auto-restart", + "version": "7.0.1", + "description": "Clusterio plugin restarting hosts and the controller once no players are online.", + "author": "Cooldude2606 ", + "license": "MIT", + "repository": "explosivegaming/ExpCluster", + "type": "module", + "main": "dist/node/index.js", + "scripts": { + "prepare": "tsc --build", + "test": "tap --type-strip-only --disable-coverage --allow-empty-coverage test/*.test.js", + "coverage": "tap --coverage-report=text test/*.test.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@clusterio/controller": "workspace:^", + "@clusterio/lib": "workspace:^" + }, + "devDependencies": { + "@clusterio/controller": "workspace:^", + "@clusterio/lib": "workspace:^", + "@types/node": "catalog:", + "tap": "^21.1.0", + "typescript": "catalog:" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ] +} diff --git a/exp_auto_restart/test/controller.test.js b/exp_auto_restart/test/controller.test.js new file mode 100644 index 00000000..e7461f53 --- /dev/null +++ b/exp_auto_restart/test/controller.test.js @@ -0,0 +1,353 @@ +import t from "tap"; +import * as lib from "@clusterio/lib"; +import { Controller, HostRecord, InstanceRecord } from "@clusterio/controller"; +import entrypoint, { AutoRestart, PENDING_TIMEOUT_MS } from "../dist/node/controller.js"; +import { plugin as pluginDeclaration } from "../dist/node/index.js"; + +// The plugin's config fields must be defined before a ControllerConfig can set them +lib.addPluginConfigFields([pluginDeclaration]); + +// Silence the singleton logger to avoid polluting the test output +lib.logger.silent = true; +t.after(() => { + lib.logger.silent = false; +}); + +/** Build the plugin around a real controller, which is side effect free while not started. */ +async function startPlugin(t2, { config = {}, canRestart = false } = {}) { + const logs = []; + const logger = { + child: () => logger, + info: message => logs.push(["info", message]), + warn: message => logs.push(["warn", message]), + error: message => logs.push(["error", message]), + verbose: () => {}, + }; + const controllerConfig = new lib.ControllerConfig("controller", { + "controller.database_directory": t2.testdir(), + ...config, + }); + const controller = new Controller(logger, [], controllerConfig, canRestart); + + // Record what would be sent to hosts and instances instead of sending it + const sent = []; + controller.sendTo = async (address, message) => { + sent.push({ address, message }); + }; + + const plugin = new AutoRestart({ controller, logger, plugin: pluginDeclaration, metrics: undefined }); + plugin.start(); + t2.teardown(() => plugin.stop()); + return { plugin, controller, sent, logs }; +} + +function addHost(controller, id, connected = true) { + controller.hosts.set(new HostRecord(id, `host-${id}`, "2.0.0", new Map(), connected)); +} + +function addInstance(controller, id, hostId, status = "running", fields = {}) { + const config = new lib.InstanceConfig("controller", { + "instance.id": id, + "instance.name": `instance-${id}`, + "instance.assigned_host": hostId, + ...fields, + }); + const record = new InstanceRecord(config, status); + controller.instances.records.set(record); + return record; +} + +function addSystem(controller, id, { restartRequired = true, canRestart = true, processStartedAtMs = 1000 } = {}) { + controller.systems.set(new lib.SystemInfo( + id, "box", "v22", "Linux", "x64", "cpu", [], 0, 0, 0, 0, + canRestart, restartRequired, 0, processStartedAtMs, Date.now(), false, + )); +} + +const join = (controller, name, instanceId) => controller.users.getOrCreateUser(name).notifyJoin(instanceId); +const leave = (controller, name, instanceId) => controller.users.getOrCreateUser(name).notifyLeave(instanceId); +const restartsSent = sent => sent.filter(({ message }) => message instanceof lib.HostRestartRequest); +const startsSent = sent => sent.filter(({ message }) => message instanceof lib.InstanceStartRequest); + +t.test("entrypoint", async t2 => { + const logger = { child: () => logger, info: () => {}, warn: () => {}, error: () => {}, verbose: () => {} }; + const controllerConfig = new lib.ControllerConfig("controller", { "controller.database_directory": t2.testdir() }); + const controller = new Controller(logger, [], controllerConfig); + await entrypoint({ controller, logger, plugin: pluginDeclaration, metrics: undefined }); + for (const hook of ["shutdown", "playerEvent", "instanceStatusChanged", "hostConnectionEvent"]) { + t2.ok([...controller.hooks[hook].attached].includes("exp_auto_restart"), `attached to ${hook}`); + } + await controller.hooks.shutdown.invoke(); +}); + +t.test("class AutoRestart", t2 => { + t2.test(".onlinePlayers() counts players on running instances per host", async t3 => { + const { plugin, controller } = await startPlugin(t3); + addHost(controller, 1); + addHost(controller, 2); + addInstance(controller, 10, 1); + addInstance(controller, 11, 1, "stopped"); + addInstance(controller, 20, 2); + join(controller, "alice", 10); + join(controller, "bob", 10); + join(controller, "carol", 11); + join(controller, "dave", 20); + + t3.strictSame(plugin.onlinePlayers(), new Map([[1, 2], [2, 1]]), "players on stopped instances are not counted"); + }); + + t2.test(".refreshIdle() tracks when hosts and the cluster became empty", async t3 => { + const { plugin, controller } = await startPlugin(t3); + addHost(controller, 1); + addHost(controller, 2); + addHost(controller, 3, false); + addInstance(controller, 10, 1); + join(controller, "alice", 10); + + plugin.refreshIdle(1000); + t3.notOk(plugin.idleSince.has(1), "a host with players is not idle"); + t3.strictSame(plugin.idleSince.get(2), 1000, "an empty host is idle from now"); + t3.notOk(plugin.idleSince.has(3), "a disconnected host is not tracked"); + t3.notOk(plugin.idleSince.has("cluster"), "the cluster is not idle while anyone is online"); + + leave(controller, "alice", 10); + plugin.refreshIdle(2000); + t3.strictSame(plugin.idleSince.get(1), 2000, "a host becomes idle when its last player leaves"); + t3.strictSame(plugin.idleSince.get(2), 1000, "an idle host keeps its original time"); + t3.strictSame(plugin.idleSince.get("cluster"), 2000, "the cluster becomes idle with the last player"); + + join(controller, "alice", 10); + plugin.refreshIdle(3000); + t3.notOk(plugin.idleSince.has(1), "a join clears the idle time"); + t3.notOk(plugin.idleSince.has("cluster")); + }); + + t2.test(".check() restarts a host once it has been empty for long enough", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 60 }, + }); + const now = Date.now(); + addHost(controller, 1); + addInstance(controller, 10, 1); + addInstance(controller, 11, 1, "stopped"); + addSystem(controller, 1); + join(controller, "alice", 10); + + await plugin.check(now); + t3.strictSame(restartsSent(sent).length, 0, "not restarted while a player is online"); + + leave(controller, "alice", 10); + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 0, "not restarted before the idle time has passed"); + + await plugin.check(now + 70_000); + t3.strictSame(restartsSent(sent).length, 1, "restarted once the idle time has passed"); + t3.strictSame(restartsSent(sent)[0].address, { hostId: 1 }, "the restart is sent to the host"); + t3.strictSame([...plugin.pending.get(1).instanceIds], [10], "only the running instances are remembered"); + + await plugin.check(now + 80_000); + t3.strictSame(restartsSent(sent).length, 1, "not restarted again while waiting for it to come back"); + }); + + t2.test(".check() asks each host process only once", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addSystem(controller, 1, { processStartedAtMs: 1000 }); + + await plugin.check(now); + t3.strictSame(restartsSent(sent).length, 1, "restarted with no running instances"); + t3.notOk(plugin.pending.has(1), "nothing to wait for without running instances"); + + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 1, "the same process is not asked again"); + + addSystem(controller, 1, { restartRequired: false, processStartedAtMs: 2000 }); + await plugin.check(now + 20_000); + t3.strictSame(restartsSent(sent).length, 1, "the new process does not need a restart"); + + addSystem(controller, 1, { processStartedAtMs: 2000 }); + await plugin.check(now + 30_000); + t3.strictSame(restartsSent(sent).length, 2, "the new process is restarted when it needs it"); + }); + + t2.test(".check() leaves hosts alone that cannot or need not restart", async t3 => { + const { plugin, controller, sent, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addSystem(controller, 1, { canRestart: false }); + addHost(controller, 2); + addSystem(controller, 2, { restartRequired: false }); + addHost(controller, 3, false); + addSystem(controller, 3); + + await plugin.check(now); + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 0, "no host is restarted"); + const warnings = logs.filter(([level, message]) => level === "warn" && message.includes("--can-restart")); + t3.strictSame(warnings.length, 1, "a host without a process monitor is warned about once"); + }); + + t2.test(".check() with scope cluster waits for the whole cluster to be empty", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.scope": "cluster" }, + }); + const now = Date.now(); + addHost(controller, 1); + addSystem(controller, 1); + addHost(controller, 2); + addInstance(controller, 20, 2); + join(controller, "alice", 20); + + await plugin.check(now); + t3.strictSame(restartsSent(sent).length, 0, "a player on another host holds back the restart"); + + leave(controller, "alice", 20); + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 1, "restarted once the cluster is empty"); + }); + + t2.test(".restartHost() retries transport errors but not refusals", async t3 => { + const { plugin, controller, sent, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addInstance(controller, 10, 1); + addSystem(controller, 1); + + controller.sendTo = async () => { throw new lib.RequestError("Cannot restart"); }; + await plugin.check(now); + await plugin.check(now + 10_000); + t3.strictSame(logs.filter(([level]) => level === "error").length, 1, "a refusal is logged once"); + t3.notOk(plugin.pending.has(1), "a refused restart is not waited on"); + + addSystem(controller, 1, { processStartedAtMs: 2000 }); + let attempts = 0; + controller.sendTo = async () => { attempts += 1; throw new Error("Connection lost"); }; + await plugin.check(now + 20_000); + await plugin.check(now + 30_000); + t3.strictSame(attempts, 2, "a transport error is retried on the next check"); + }); + + t2.test(".instanceReported() starts the instances that were running", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + const manual = addInstance(controller, 10, 1); + const auto = addInstance(controller, 11, 1, "running", { "instance.auto_start": true }); + const stopped = addInstance(controller, 12, 1, "stopped"); + addSystem(controller, 1); + await plugin.check(now); + t3.strictSame([...plugin.pending.get(1).instanceIds], [10, 11]); + + // The host comes back and reports every instance as stopped + for (const instance of [manual, auto, stopped]) { + instance.status = "stopped"; + await plugin.onInstanceStatusChanged(instance, "unknown"); + } + t3.strictSame(startsSent(sent).map(({ address }) => address), [{ instanceId: 10 }], "only the instance the host will not start itself is started"); + t3.notOk(plugin.pending.has(1), "the host is no longer waited on"); + + // An instance reporting in without a pending restart is ignored + await plugin.onInstanceStatusChanged(manual, "unknown"); + t3.strictSame(startsSent(sent).length, 1); + }); + + t2.test(".restartHost() does not wait on instances when they are not to be started", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.start_instances": false }, + }); + addHost(controller, 1); + const instance = addInstance(controller, 10, 1); + addSystem(controller, 1); + await plugin.check(Date.now()); + t3.strictSame(restartsSent(sent).length, 1); + t3.notOk(plugin.pending.has(1), "nothing is waited on"); + + instance.status = "stopped"; + await plugin.onInstanceStatusChanged(instance, "unknown"); + t3.strictSame(startsSent(sent).length, 0, "nothing is started"); + }); + + t2.test(".expirePending() gives up on hosts that never come back", async t3 => { + const { plugin, controller, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addInstance(controller, 10, 1); + addSystem(controller, 1); + await plugin.check(now); + t3.ok(plugin.pending.has(1)); + + await plugin.check(now + PENDING_TIMEOUT_MS - 1); + t3.ok(plugin.pending.has(1), "still waited on before the timeout"); + await plugin.check(now + PENDING_TIMEOUT_MS); + t3.notOk(plugin.pending.has(1), "no longer waited on after the timeout"); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("giving up")), "giving up is logged"); + }); + + t2.test(".checkController() restarts the controller when the cluster is empty", async t3 => { + const { plugin, controller } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 60, "exp_auto_restart.restart_controller": true }, + canRestart: true, + }); + const now = Date.now(); + let stopped = 0; + controller.stop = async () => { stopped += 1; }; + controller.checkRestartDowngrade = async () => null; + addHost(controller, 1); + addInstance(controller, 10, 1); + addSystem(controller, "controller"); + join(controller, "alice", 10); + + await plugin.check(now); + t3.strictSame(stopped, 0, "not restarted while a player is online"); + + leave(controller, "alice", 10); + await plugin.check(now + 10_000); + t3.strictSame(stopped, 0, "not restarted before the idle time has passed"); + + await plugin.check(now + 70_000); + t3.strictSame(stopped, 1, "restarted once the cluster has been empty for long enough"); + t3.ok(controller.shouldRestart, "stopped with the intent to restart"); + + await plugin.check(now + 80_000); + t3.strictSame(stopped, 1, "not restarted twice"); + }); + + t2.test(".checkController() refuses downgrades and missing process monitors", async t3 => { + const { plugin, controller, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.restart_controller": true }, + canRestart: true, + }); + let stopped = 0; + controller.stop = async () => { stopped += 1; }; + controller.checkRestartDowngrade = async () => ({ installedVersion: "2.0.0", runningVersion: "2.0.1" }); + addSystem(controller, "controller"); + + await plugin.check(Date.now()); + t3.strictSame(stopped, 0, "not restarted into an older version"); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("older")), "the downgrade is logged"); + + plugin.warned.clear(); + addSystem(controller, "controller", { canRestart: false }); + await plugin.check(Date.now()); + t3.strictSame(stopped, 0, "not restarted without a process monitor"); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("--can-restart"))); + }); + + t2.test(".start() warns when system metrics are disabled", async t3 => { + const { logs } = await startPlugin(t3, { config: { "controller.system_metrics_interval": 0 } }); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("system_metrics_interval"))); + }); + + t2.end(); +}); diff --git a/exp_auto_restart/tsconfig.json b/exp_auto_restart/tsconfig.json new file mode 100644 index 00000000..a0473e75 --- /dev/null +++ b/exp_auto_restart/tsconfig.json @@ -0,0 +1,6 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" } + ] +} diff --git a/exp_auto_restart/tsconfig.node.json b/exp_auto_restart/tsconfig.node.json new file mode 100644 index 00000000..3218f2e7 --- /dev/null +++ b/exp_auto_restart/tsconfig.node.json @@ -0,0 +1,5 @@ +{ + "extends": "../tsconfig.node.json", + "include": ["./**/*.ts"], + "exclude": ["test/*", "./dist/*"], +} diff --git a/tsconfig.json b/tsconfig.json index 8afb8e7c..94818dee 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,7 @@ { "files": [], "references": [ + { "path": "./exp_auto_restart/" }, { "path": "./exp_commands/" }, { "path": "./exp_groups/" }, { "path": "./exp_gui/" },