Implement stop and restart of controller/hosts (#554)

Implement remotely initiated stopping and restarting of the controller
and hosts.  Restarting is implemented by exiting with a non-zero exit
code and relying on a process manager to start the procress again.  To
prevent accidentally trying to restart on a system that doesn't have a
process manager present the --can-restart command line option needs to
be passed to enable the restart operation.

packages/controller/src/ControlConnection.ts

@@ -68,6 +68,8 @@ export default class ControlConnection extends BaseConnection {
 			});
 		}
 
+		this.handle(lib.ControllerStopRequest, this.handleControllerStopRequest.bind(this));
+		this.handle(lib.ControllerRestartRequest, this.handleControllerRestartRequest.bind(this));
 		this.handle(lib.ControllerConfigGetRequest, this.handleControllerConfigGetRequest.bind(this));
 		this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this));
 		this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this));
@@ -167,6 +169,18 @@ export default class ControlConnection extends BaseConnection {
 		throw new Error("Should be unreachable");
 	}
 
+	async handleControllerStopRequest() {
+		this._controller.stop();
+	}
+
+	async handleControllerRestartRequest() {
+		if (!this._controller.canRestart) {
+			throw new lib.RequestError("Cannot restart, controller does not have a process monitor to restart it.");
+		}
+		this._controller.shouldRestart = true;
+		this._controller.stop();
+	}
+
 	async handleControllerConfigGetRequest() {
 		return this._controller.config.toRemote("control");
 	}

packages/controller/src/Controller.ts

@@ -84,6 +84,12 @@ export default class Controller {
 	// Possible states are new, starting, running, stopping, stopped
 	private _state: string = "new";
 	private _shouldStop: boolean = false;
+	/**
+	 * Set to true before calling stop to have the controller restart
+	 * instead of stopping. This only works if {@link Controller.canRestart}
+	 * is true.
+	 */
+	public shouldRestart: boolean = false;
 	_fallbackedRequests: Map<lib.RequestClass<unknown, unknown>, lib.RequestHandler<unknown, unknown>> = new Map();
 	_registeredRequests: Map<lib.RequestClass<unknown, unknown>, lib.RequestHandler<unknown, unknown>> = new Map();
 	_registeredEvents = new Map();
@@ -131,6 +137,11 @@ export default class Controller {
 		configPath: string,
 		config: lib.ControllerConfig,
 
+		/**
+		 * If true indicates that there is a process monitor present that
+		 * will restart the controller on non-zero exit codes.
+		 */
+		public canRestart: boolean = false,
 		public systems = new Map<lib.SystemInfo["id"], lib.SystemInfo>(),
 		/** Mapping of host id to host info */
 		public hosts = new Map<number, HostInfo>(),
@@ -365,7 +376,12 @@ export default class Controller {
 	async _stopInternal() {
 		// This function should never throw.
 		this._state = "stopping";
-		logger.info("Stopping controller");
+		if (this.shouldRestart) {
+			logger.info("Restarting controller");
+			process.exitCode = 1;
+		} else {
+			logger.info("Stopping controller");
+		}
 
 		if (this.clusterLogBuildInterval) {
 			clearInterval(this.clusterLogBuildInterval);
@@ -459,7 +475,7 @@ export default class Controller {
 				}
 				requests.push(hostConnection.send(new lib.SystemInfoRequest()));
 			}
-			requests.push(lib.gatherSystemInfo("controller"));
+			requests.push(lib.gatherSystemInfo("controller", this.canRestart));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
 				this.systems.set(metric.id, metric);