From be071be8f2f34ca994e26430e24f9364b099ed7f Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sat, 10 May 2025 21:33:45 +0100 Subject: [PATCH] Add recovery startup mode (#757) * Add recovery startup mode * Change error code to RecoveryMode --- packages/controller/controller.ts | 8 ++++++++ packages/controller/src/Controller.ts | 14 ++++++++++++++ packages/controller/src/WsServer.ts | 7 +++++++ packages/host/host.ts | 8 ++++++++ packages/host/src/Host.ts | 14 ++++++++++++++ packages/host/src/Instance.ts | 5 +++++ packages/lib/src/link/connectors.ts | 2 ++ 7 files changed, 58 insertions(+) diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 6587e01d..851a3bc7 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -270,6 +270,10 @@ async function initialize(): Promise { type: "boolean", nargs: 0, default: false, describe: "Indicate that a process monitor will restart the controller on failure", }); + yargs.option("recovery", { + type: "boolean", nargs: 0, default: false, + describe: "Start the controller in recovery mode with all plugins disabled and hosts disconnected", + }); yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 }); yargs.option("dev-plugin", { hidden: true, type: "array" }); }) @@ -313,6 +317,9 @@ async function initialize(): Promise { let shouldRun = false; if (command === "run") { logger.info(`Starting Clusterio controller ${version}`); + if (args.recovery) { + logger.warn("Controller recovery mode enabled. Some features will be disabled."); + } shouldRun = true; } @@ -406,6 +413,7 @@ async function startup() { controllerConfigPath, controllerConfig, Boolean(args.canRestart), + Boolean(args.recovery), ...await Controller.bootstrap(controllerConfig) ); diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 18a38751..8d42c2c0 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -163,6 +163,11 @@ export default class Controller { * will restart the controller on non-zero exit codes. */ public canRestart: boolean = false, + /** + * If true indicates the controller is in recovery mode and should + * disable certain actions such as loading plugins or connecting to hosts + */ + public recoveryMode: boolean = false, public systems = new lib.SubscribableDatastore(), /** Mapping of host id to host info */ public hosts = new lib.SubscribableDatastore(), @@ -538,6 +543,10 @@ export default class Controller { clearInterval(this.autosaveInterval); this.autosaveInterval = undefined; } + if (this.recoveryMode) { + logger.warn("Recovery | autosaving disabled"); + return; + } const autosaveIntervalSeconds = this.config.get("controller.autosave_interval"); if (autosaveIntervalSeconds > 0) { this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000); @@ -1058,6 +1067,11 @@ export default class Controller { continue; } + if (this.recoveryMode) { + logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`); + continue; + } + let ControllerPluginClass = BaseControllerPlugin; try { if (pluginInfo.controllerEntrypoint) { diff --git a/packages/controller/src/WsServer.ts b/packages/controller/src/WsServer.ts index c303eb10..c30b0f5c 100644 --- a/packages/controller/src/WsServer.ts +++ b/packages/controller/src/WsServer.ts @@ -239,6 +239,13 @@ ${err.stack}` } async registerHost(data: lib.RegisterHostData, socket: WebSocket, req: IncomingMessage) { + if (this.controller.recoveryMode) { + logger.warn(`Recovery | rejected host connection attempt for ${this.remoteAddr(req)}`); + wsRejectedConnectionsCounter.inc(); + socket.close(lib.ConnectionClosed.RecoveryMode, "Connection Refused: Controller in recovery mode"); + return; + } + try { let tokenPayload = jwt.verify( data.token, diff --git a/packages/host/host.ts b/packages/host/host.ts index 0247e9aa..f5ccd9b1 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -120,6 +120,10 @@ async function startHost() { type: "boolean", nargs: 0, default: false, describe: "Indicate that a process monitor will restart this host on failure", }); + yargs.option("recovery", { + type: "boolean", nargs: 0, default: false, + describe: "Start the host in recovery mode with all plugins disabled and controller disconnected", + }); }) .demandCommand(1, "You need to specify a command to run") .strict() @@ -144,6 +148,9 @@ async function startHost() { let command = args._[0]; if (command === "run") { logger.info(`Starting Clusterio host ${version}`); + if (args.recovery) { + logger.warn("Host recovery mode enabled. Some features will be disabled."); + } } logger.info(`Loading available plugins from ${args.pluginList}`); @@ -226,6 +233,7 @@ async function startHost() { tlsCa, pluginInfos, Boolean(args.canRestart), + Boolean(args.recovery), ...await Host.bootstrap(hostConfig) ); diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 0e60d2a1..9553932a 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -298,6 +298,11 @@ export default class Host extends lib.Link { * will restart this host on a non-zero exit codes. */ public canRestart = false, + /** + * If true indicates the host is in recovery mode and should + * disable certain actions such as loading plugins or instance autostart + */ + public recoveryMode = false, public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()), ) { super(connector); @@ -403,6 +408,11 @@ export default class Host extends lib.Link { continue; } + if (this.recoveryMode) { + logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`); + continue; + } + let HostPluginClass = BaseHostPlugin; try { if (pluginInfo.hostEntrypoint) { @@ -1080,6 +1090,10 @@ export default class Host extends lib.Link { for (let [instanceId, instanceInfo] of this.instanceInfos) { if (instanceInfo.config.get("instance.auto_start")) { + if (this.recoveryMode) { + logger.warn(`Recovery | skipping auto startup for ${instanceInfo.config.get("instance.name")}`); + continue; + } try { let instanceConnection = await this._connectInstance(instanceId); await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest()); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index d1026e49..1018bf0d 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -646,6 +646,11 @@ end`.replace(/\r?\n/g, " "); continue; } + if (this._host.recoveryMode) { + this.logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`); + continue; + } + try { await this._loadPlugin(pluginInfo, this._host); } catch (err) { diff --git a/packages/lib/src/link/connectors.ts b/packages/lib/src/link/connectors.ts index 9bbe2c2f..f5d04561 100644 --- a/packages/lib/src/link/connectors.ts +++ b/packages/lib/src/link/connectors.ts @@ -23,6 +23,7 @@ export enum ConnectionClosed { ProtocolError = 1002, // Protocol not followed PolicyViolation = 1008, // Generic code for any endpoint policy InternalError = 1011, // Endpoint failed to fulfil request + TryAgainLater = 1013, // Temporary server condition blocking requests // Codes after 3000 are available for frameworks // However they should be registered to IANA @@ -32,6 +33,7 @@ export enum ConnectionClosed { // Codes after 4000 are available for applications MalformedMessage = 4000, + RecoveryMode = 4001, }; /**