Add recovery startup mode (#757)

* Add recovery startup mode

* Change error code to RecoveryMode

packages/controller/src/Controller.ts

@@ -163,6 +163,11 @@ export default class Controller {
 		 * will restart the controller on non-zero exit codes.
 		 */
 		public canRestart: boolean = false,
+		/**
+		 * If true indicates the controller is in recovery mode and should
+		 * disable certain actions such as loading plugins or connecting to hosts
+		 */
+		public recoveryMode: boolean = false,
 		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
 		/** Mapping of host id to host info */
 		public hosts = new lib.SubscribableDatastore<HostInfo>(),
@@ -538,6 +543,10 @@ export default class Controller {
 			clearInterval(this.autosaveInterval);
 			this.autosaveInterval = undefined;
 		}
+		if (this.recoveryMode) {
+			logger.warn("Recovery | autosaving disabled");
+			return;
+		}
 		const autosaveIntervalSeconds = this.config.get("controller.autosave_interval");
 		if (autosaveIntervalSeconds > 0) {
 			this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000);
@@ -1058,6 +1067,11 @@ export default class Controller {
 				continue;
 			}
 
+			if (this.recoveryMode) {
+				logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
+				continue;
+			}
+
 			let ControllerPluginClass = BaseControllerPlugin;
 			try {
 				if (pluginInfo.controllerEntrypoint) {

packages/controller/src/WsServer.ts

@@ -239,6 +239,13 @@ ${err.stack}`
 	}
 
 	async registerHost(data: lib.RegisterHostData, socket: WebSocket, req: IncomingMessage) {
+		if (this.controller.recoveryMode) {
+			logger.warn(`Recovery | rejected host connection attempt for ${this.remoteAddr(req)}`);
+			wsRejectedConnectionsCounter.inc();
+			socket.close(lib.ConnectionClosed.RecoveryMode, "Connection Refused: Controller in recovery mode");
+			return;
+		}
+
 		try {
 			let tokenPayload = jwt.verify(
 				data.token,