Add recovery startup mode (#757)

* Add recovery startup mode

* Change error code to RecoveryMode

packages/controller/controller.ts

@@ -270,6 +270,10 @@ async function initialize(): Promise<InitializeParameters> {
 				type: "boolean", nargs: 0, default: false,
 				describe: "Indicate that a process monitor will restart the controller on failure",
 			});
+			yargs.option("recovery", {
+				type: "boolean", nargs: 0, default: false,
+				describe: "Start the controller in recovery mode with all plugins disabled and hosts disconnected",
+			});
 			yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 });
 			yargs.option("dev-plugin", { hidden: true, type: "array" });
 		})
@@ -313,6 +317,9 @@ async function initialize(): Promise<InitializeParameters> {
 	let shouldRun = false;
 	if (command === "run") {
 		logger.info(`Starting Clusterio controller ${version}`);
+		if (args.recovery) {
+			logger.warn("Controller recovery mode enabled. Some features will be disabled.");
+		}
 		shouldRun = true;
 	}
 
@@ -406,6 +413,7 @@ async function startup() {
 		controllerConfigPath,
 		controllerConfig,
 		Boolean(args.canRestart),
+		Boolean(args.recovery),
 		...await Controller.bootstrap(controllerConfig)
 	);
 

packages/controller/src/Controller.ts

@@ -163,6 +163,11 @@ export default class Controller {
 		 * will restart the controller on non-zero exit codes.
 		 */
 		public canRestart: boolean = false,
+		/**
+		 * If true indicates the controller is in recovery mode and should
+		 * disable certain actions such as loading plugins or connecting to hosts
+		 */
+		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
 		/** Mapping of host id to host info */
 		public hosts = new lib.SubscribableDatastore<HostInfo>(),
@@ -538,6 +543,10 @@ export default class Controller {
 			clearInterval(this.autosaveInterval);
 			this.autosaveInterval = undefined;
 		}
+		if (this.recoveryMode) {
+			logger.warn("Recovery | autosaving disabled");
+			return;
+		}
 		const autosaveIntervalSeconds = this.config.get("controller.autosave_interval");
 		if (autosaveIntervalSeconds > 0) {
 			this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000);
@@ -1058,6 +1067,11 @@ export default class Controller {
 				continue;
 			}
 
+			if (this.recoveryMode) {
+				logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
+				continue;
+			}
+
 			let ControllerPluginClass = BaseControllerPlugin;
 			try {
 				if (pluginInfo.controllerEntrypoint) {

packages/controller/src/WsServer.ts

@@ -239,6 +239,13 @@ ${err.stack}`
 	}
 
 	async registerHost(data: lib.RegisterHostData, socket: WebSocket, req: IncomingMessage) {
+		if (this.controller.recoveryMode) {
+			logger.warn(`Recovery | rejected host connection attempt for ${this.remoteAddr(req)}`);
+			wsRejectedConnectionsCounter.inc();
+			socket.close(lib.ConnectionClosed.RecoveryMode, "Connection Refused: Controller in recovery mode");
+			return;
+		}
+
 		try {
 			let tokenPayload = jwt.verify(
 				data.token,

packages/host/host.ts

@@ -120,6 +120,10 @@ async function startHost() {
 				type: "boolean", nargs: 0, default: false,
 				describe: "Indicate that a process monitor will restart this host on failure",
 			});
+			yargs.option("recovery", {
+				type: "boolean", nargs: 0, default: false,
+				describe: "Start the host in recovery mode with all plugins disabled and controller disconnected",
+			});
 		})
 		.demandCommand(1, "You need to specify a command to run")
 		.strict()
@@ -144,6 +148,9 @@ async function startHost() {
 	let command = args._[0];
 	if (command === "run") {
 		logger.info(`Starting Clusterio host ${version}`);
+		if (args.recovery) {
+			logger.warn("Host recovery mode enabled. Some features will be disabled.");
+		}
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
@@ -226,6 +233,7 @@ async function startHost() {
 		tlsCa,
 		pluginInfos,
 		Boolean(args.canRestart),
+		Boolean(args.recovery),
 		...await Host.bootstrap(hostConfig)
 	);
 

packages/host/src/Host.ts

@@ -298,6 +298,11 @@ export default class Host extends lib.Link {
 		 * will restart this host on a non-zero exit codes.
 		 */
 		public canRestart = false,
+		/**
+		 * If true indicates the host is in recovery mode and should
+		 * disable certain actions such as loading plugins or instance autostart
+		 */
+		public recoveryMode = false,
 		public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
 	) {
 		super(connector);
@@ -403,6 +408,11 @@ export default class Host extends lib.Link {
 				continue;
 			}
 
+			if (this.recoveryMode) {
+				logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
+				continue;
+			}
+
 			let HostPluginClass = BaseHostPlugin;
 			try {
 				if (pluginInfo.hostEntrypoint) {
@@ -1080,6 +1090,10 @@ export default class Host extends lib.Link {
 
 			for (let [instanceId, instanceInfo] of this.instanceInfos) {
 				if (instanceInfo.config.get("instance.auto_start")) {
+					if (this.recoveryMode) {
+						logger.warn(`Recovery | skipping auto startup for ${instanceInfo.config.get("instance.name")}`);
+						continue;
+					}
 					try {
 						let instanceConnection = await this._connectInstance(instanceId);
 						await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest());

packages/host/src/Instance.ts

@@ -646,6 +646,11 @@ end`.replace(/\r?\n/g, " ");
 				continue;
 			}
 
+			if (this._host.recoveryMode) {
+				this.logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
+				continue;
+			}
+
 			try {
 				await this._loadPlugin(pluginInfo, this._host);
 			} catch (err) {

packages/lib/src/link/connectors.ts

@@ -23,6 +23,7 @@ export enum ConnectionClosed {
 	ProtocolError = 1002, // Protocol not followed
 	PolicyViolation = 1008, // Generic code for any endpoint policy
 	InternalError = 1011, // Endpoint failed to fulfil request
+	TryAgainLater = 1013, // Temporary server condition blocking requests
 
 	// Codes after 3000 are available for frameworks
 	// However they should be registered to IANA
@@ -32,6 +33,7 @@ export enum ConnectionClosed {
 
 	// Codes after 4000 are available for applications
 	MalformedMessage = 4000,
+	RecoveryMode = 4001,
 };
 
 /**