Implement stop and restart of controller/hosts (#554)

Implement remotely initiated stopping and restarting of the controller
and hosts.  Restarting is implemented by exiting with a non-zero exit
code and relying on a process manager to start the procress again.  To
prevent accidentally trying to restart on a system that doesn't have a
process manager present the --can-restart command line option needs to
be passed to enable the restart operation.

packages/controller/controller.ts

@@ -244,6 +244,10 @@ async function initialize(): Promise<InitializeParameters> {
 				.demandCommand(1, "You need to specify a command to run");
 		})
 		.command("run", "Run controller", yargs => {
+			yargs.option("can-restart", {
+				type: "boolean", nargs: 0, default: false,
+				describe: "Indicate that a process monitor will restart the controller on failure",
+			});
 			yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 });
 			yargs.option("dev-plugin", { hidden: true, type: "array" });
 		})
@@ -386,6 +390,7 @@ async function startup() {
 		pluginInfos,
 		controllerConfigPath,
 		controllerConfig,
+		Boolean(args.canRestart),
 		...await Controller.bootstrap(controllerConfig)
 	);
 

packages/controller/src/ControlConnection.ts

@@ -68,6 +68,8 @@ export default class ControlConnection extends BaseConnection {
 			});
 		}
 
+		this.handle(lib.ControllerStopRequest, this.handleControllerStopRequest.bind(this));
+		this.handle(lib.ControllerRestartRequest, this.handleControllerRestartRequest.bind(this));
 		this.handle(lib.ControllerConfigGetRequest, this.handleControllerConfigGetRequest.bind(this));
 		this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this));
 		this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this));
@@ -167,6 +169,18 @@ export default class ControlConnection extends BaseConnection {
 		throw new Error("Should be unreachable");
 	}
 
+	async handleControllerStopRequest() {
+		this._controller.stop();
+	}
+
+	async handleControllerRestartRequest() {
+		if (!this._controller.canRestart) {
+			throw new lib.RequestError("Cannot restart, controller does not have a process monitor to restart it.");
+		}
+		this._controller.shouldRestart = true;
+		this._controller.stop();
+	}
+
 	async handleControllerConfigGetRequest() {
 		return this._controller.config.toRemote("control");
 	}

packages/controller/src/Controller.ts

@@ -84,6 +84,12 @@ export default class Controller {
 	// Possible states are new, starting, running, stopping, stopped
 	private _state: string = "new";
 	private _shouldStop: boolean = false;
+	/**
+	 * Set to true before calling stop to have the controller restart
+	 * instead of stopping. This only works if {@link Controller.canRestart}
+	 * is true.
+	 */
+	public shouldRestart: boolean = false;
 	_fallbackedRequests: Map<lib.RequestClass<unknown, unknown>, lib.RequestHandler<unknown, unknown>> = new Map();
 	_registeredRequests: Map<lib.RequestClass<unknown, unknown>, lib.RequestHandler<unknown, unknown>> = new Map();
 	_registeredEvents = new Map();
@@ -131,6 +137,11 @@ export default class Controller {
 		configPath: string,
 		config: lib.ControllerConfig,
 
+		/**
+		 * If true indicates that there is a process monitor present that
+		 * will restart the controller on non-zero exit codes.
+		 */
+		public canRestart: boolean = false,
 		public systems = new Map<lib.SystemInfo["id"], lib.SystemInfo>(),
 		/** Mapping of host id to host info */
 		public hosts = new Map<number, HostInfo>(),
@@ -365,7 +376,12 @@ export default class Controller {
 	async _stopInternal() {
 		// This function should never throw.
 		this._state = "stopping";
-		logger.info("Stopping controller");
+		if (this.shouldRestart) {
+			logger.info("Restarting controller");
+			process.exitCode = 1;
+		} else {
+			logger.info("Stopping controller");
+		}
 
 		if (this.clusterLogBuildInterval) {
 			clearInterval(this.clusterLogBuildInterval);
@@ -459,7 +475,7 @@ export default class Controller {
 				}
 				requests.push(hostConnection.send(new lib.SystemInfoRequest()));
 			}
-			requests.push(lib.gatherSystemInfo("controller"));
+			requests.push(lib.gatherSystemInfo("controller", this.canRestart));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
 				this.systems.set(metric.id, metric);

packages/create/create.js

@@ -87,7 +87,7 @@ class InstallError extends Error { }
 
 async function safeOutputFile(file, data, options={}) {
 	let { dir, name, ext } = path.parse(file);
-	let temporary = `${dir}${name}.tmp${ext}`;
+	let temporary = path.join(dir, `${name}.tmp${ext}`);
 	await fs.outputFile(temporary, data, options);
 	await fs.rename(temporary, file);
 }
@@ -377,18 +377,25 @@ async function writeScripts(mode) {
 			await safeOutputFile(
 				"run-controller.cmd",
 				`\
-@echo off.
+@echo off
 set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%"
-\\node_modules\\.bin\\clusteriocontroller.cmd run
+:restart
+call .\\node_modules\\.bin\\clusteriocontroller.cmd run --can-restart
+if %errorlevel% equ 0 exit /b
+if %errorlevel% equ 8 exit /b
+goto restart
 `
 			);
 		} else {
 			await safeOutputFile(
 				"run-controller.sh",
 				`\
-#!/bin/sh
+#!/bin/bash
 export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS"
-exec ./node_modules/.bin/clusteriocontroller run
+while true; do
+	./node_modules/.bin/clusteriocontroller run --can-restart
+	if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi
+done
 `,
 				{ mode: 0o755 },
 			);
@@ -405,11 +412,14 @@ WorkingDirectory=${process.cwd()}
 KillMode=mixed
 KillSignal=SIGINT
 Environment=NODE_OPTIONS=--enable-source-maps
-ExecStart=${process.cwd()}/node_modules/.bin/clusteriocontroller run --log-level=warn
+ExecStart=${process.cwd()}/node_modules/.bin/clusteriocontroller run --log-level=warn --can-restart
+Restart=on-failure
+RestartPreventExitStatus=8
 
 [Install]
 WantedBy=multi-user.target
-`);
+`
+			);
 		}
 	}
 
@@ -417,17 +427,33 @@ WantedBy=multi-user.target
 		if (process.platform === "win32") {
 			await safeOutputFile(
 				"run-host.cmd",
-				"@echo off\n.\\node_modules\\.bin\\clusteriohost.cmd run\n"
+				`\
+@echo off
+set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%"
+:restart
+call .\\node_modules\\.bin\\clusteriohost.cmd run --can-restart
+if %errorlevel% equ 0 exit /b
+if %errorlevel% equ 8 exit /b
+goto restart
+`
 			);
 		} else {
 			await safeOutputFile(
 				"run-host.sh",
-				"#!/bin/sh\nexec ./node_modules/.bin/clusteriohost run\n",
+				`\
+#!/bin/bash
+export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS"
+while true; do
+	./node_modules/.bin/clusteriohost run --can-restart
+	if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi
+done
+`,
 				{ mode: 0o755 },
 			);
 			await safeOutputFile(
 				"systemd/clusteriohost.service",
-				`[Unit]
+				`\
+[Unit]
 Description=Clusterio Host
 
 [Service]
@@ -436,11 +462,14 @@ Group=${await groupIdToName(os.userInfo().gid)}
 WorkingDirectory=${process.cwd()}
 KillMode=mixed
 KillSignal=SIGINT
-ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn
+ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart
+Restart=on-failure
+RestartPreventExitStatus=8
 
 [Install]
 WantedBy=multi-user.target
-`);
+`
+			);
 		}
 	}
 }

packages/ctl/src/commands.ts

@@ -82,6 +82,20 @@ function serializedConfigToString(
 }
 
 const controllerCommands = new lib.CommandTree({ name: "controller", description: "Controller management" });
+controllerCommands.add(new lib.Command({
+	definition: ["stop", "Stop controller"],
+	handler: async function(_args: object, control: Control) {
+		await control.send(new lib.ControllerStopRequest());
+	},
+}));
+
+controllerCommands.add(new lib.Command({
+	definition: ["restart", "Restart controller"],
+	handler: async function(_args: object, control: Control) {
+		await control.send(new lib.ControllerRestartRequest());
+	},
+}));
+
 const controllerConfigCommands = new lib.CommandTree({
 	name: "config", alias: ["c"], description: "controller config management",
 });
@@ -224,6 +238,26 @@ controllerCommands.add(controllerPluginCommands);
 
 
 const hostCommands = new lib.CommandTree({ name: "host", description: "Host management" });
+hostCommands.add(new lib.Command({
+	definition: ["stop <host>", "Stop the given host", (yargs) => {
+		yargs.positional("host", { describe: "Host to stop", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostStopRequest());
+	},
+}));
+
+hostCommands.add(new lib.Command({
+	definition: ["restart <host>", "Restart the given host", (yargs) => {
+		yargs.positional("host", { describe: "Host to restart", type: "string" });
+	}],
+	handler: async function(args: { host: string }, control: Control) {
+		let hostId = await lib.resolveHost(control, args.host);
+		await control.sendTo({ hostId }, new lib.HostRestartRequest());
+	},
+}));
+
 hostCommands.add(new lib.Command({
 	definition: [["list", "l"], "List hosts connected to the controller"],
 	handler: async function(args: object, control: Control) {

packages/host/host.ts

@@ -94,7 +94,12 @@ async function startHost() {
 		})
 		.command("plugin", "Manage available plugins", lib.pluginCommand)
 		.command("config", "Manage Host config", lib.configCommand)
-		.command("run", "Run host")
+		.command("run", "Run host", yargs => {
+			yargs.option("can-restart", {
+				type: "boolean", nargs: 0, default: false,
+				describe: "Indicate that a process monitor will restart this host on failure",
+			});
+		})
 		.demandCommand(1, "You need to specify a command to run")
 		.strict()
 		.parseSync()
@@ -206,6 +211,7 @@ async function startHost() {
 		hostConfig,
 		tlsCa,
 		pluginInfos,
+		Boolean(args.canRestart),
 		...await Host.bootstrap(hostConfig)
 	);
 
@@ -295,7 +301,11 @@ ${err.stack}`
 			);
 		}
 
-		process.exitCode = 1;
+		if (err instanceof lib.AuthenticationFailed) {
+			process.exitCode = 8;
+		} else {
+			process.exitCode = 1;
+		}
 	});
 }
 

packages/host/src/Host.ts

@@ -278,7 +278,12 @@ export default class Host extends lib.Link {
 		hostConfig: lib.HostConfig,
 		tlsCa: string | undefined,
 		pluginInfos: lib.PluginNodeEnvInfo[],
-		public modStore: lib.ModStore,
+		/**
+		 * If true indicates that there is a process monitor present that
+		 * will restart this host on a non-zero exit codes.
+		 */
+		public canRestart = false,
+		public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
 	) {
 		super(connector);
 		this.tlsCa = tlsCa;
@@ -345,6 +350,8 @@ export default class Host extends lib.Link {
 			});
 		}
 
+		this.handle(lib.HostStopRequest, this.handleHostStopRequest.bind(this));
+		this.handle(lib.HostRestartRequest, this.handleHostRestartRequest.bind(this));
 		this.handle(lib.HostConfigGetRequest, this.handleHostConfigGetRequest.bind(this));
 		this.handle(lib.HostConfigSetFieldRequest, this.handleHostConfigSetFieldRequest.bind(this));
 		this.handle(lib.HostConfigSetPropRequest, this.handleHostConfigSetPropRequest.bind(this));
@@ -430,6 +437,18 @@ export default class Host extends lib.Link {
 		}
 	}
 
+	async handleHostStopRequest() {
+		this.shutdown();
+	}
+
+	async handleHostRestartRequest() {
+		if (!this.canRestart) {
+			throw new lib.RequestError("Cannot restart, host does not have a process monitor to restart it.");
+		}
+		process.exitCode = 1;
+		this.shutdown();
+	}
+
 	async handleHostConfigGetRequest() {
 		return this.config.toRemote("control");
 	}
@@ -751,7 +770,7 @@ export default class Host extends lib.Link {
 	}
 
 	async handleSystemInfoRequest() {
-		return lib.gatherSystemInfo(this.config.get("host.id"));
+		return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart);
 	}
 
 	async handleHostMetricsRequest() {

packages/lib/src/data/messages_controller.ts

@@ -3,6 +3,22 @@ import { JsonString, StringEnum, plainJson } from "./composites";
 import { levels } from "../logging";
 import { ControllerConfig, HostConfig } from "../config";
 
+export class ControllerStopRequest {
+	declare ["constructor"]: typeof ControllerStopRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.controller.stop" as const;
+}
+
+export class ControllerRestartRequest {
+	declare ["constructor"]: typeof ControllerRestartRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.controller.restart" as const;
+}
+
 export class ControllerConfigGetRequest {
 	declare ["constructor"]: typeof ControllerConfigGetRequest;
 	static type = "request" as const;
@@ -221,6 +237,7 @@ export class SystemInfo {
 		public memoryAvailable: number,
 		public diskCapacity: number,
 		public diskAvailable: number,
+		public canRestart: boolean,
 		/** Millisecond Unix timestamp this entry was last updated at */
 		public updatedAt: number,
 		public isDeleted: boolean,
@@ -238,6 +255,7 @@ export class SystemInfo {
 		"memoryAvailable": Type.Number(),
 		"diskCapacity": Type.Number(),
 		"diskAvailable": Type.Number(),
+		"canRestart": Type.Boolean(),
 		"updatedAt": Type.Number(),
 		"isDeleted": Type.Boolean(),
 	});
@@ -255,6 +273,7 @@ export class SystemInfo {
 			json.memoryAvailable,
 			json.diskCapacity,
 			json.diskAvailable,
+			json.canRestart,
 			json.updatedAt,
 			json.isDeleted,
 		);

packages/lib/src/data/messages_host.ts

@@ -3,6 +3,22 @@ import { jsonArray, plainJson, StringEnum } from "./composites";
 import { CollectorResultSerialized } from "../prometheus";
 import { HostConfig } from "../config/definitions";
 
+export class HostStopRequest {
+	declare ["constructor"]: typeof HostStopRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.stop" as const;
+}
+
+export class HostRestartRequest {
+	declare ["constructor"]: typeof HostRestartRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.restart" as const;
+}
+
 export class HostConfigGetRequest {
 	declare ["constructor"]: typeof HostConfigGetRequest;
 	static type = "request" as const;

packages/lib/src/link/messages.ts

@@ -17,6 +17,8 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	core.PingRequest,
 	core.AccountUpdateEvent,
 
+	controller.ControllerStopRequest,
+	controller.ControllerRestartRequest,
 	controller.ControllerConfigGetRequest,
 	controller.ControllerConfigSetFieldRequest,
 	controller.ControllerConfigSetPropRequest,
@@ -32,6 +34,8 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 
 	subscriptions.SubscriptionRequest,
 
+	host.HostStopRequest,
+	host.HostRestartRequest,
 	host.HostConfigGetRequest,
 	host.HostConfigSetFieldRequest,
 	host.HostConfigSetPropRequest,

packages/lib/src/permissions.ts

@@ -119,6 +119,18 @@ definePermission({
 	grantByDefault: true,
 });
 
+definePermission({
+	name: "core.controller.stop",
+	title: "Stop controller",
+	description:
+		"Stop the Node.js controller process making the cluster inoperable until someone with access to the system " +
+		"it runs on manually starts it again.",
+});
+definePermission({
+	name: "core.controller.restart",
+	title: "Restart controller",
+	description: "Restart the Node.js controller process if the system is set up for restarting.",
+});
 definePermission({
 	name: "core.controller.get_config",
 	title: "Get controller config",
@@ -136,6 +148,18 @@ definePermission({
 		"Subscribe to be notified when the system info detailing system specific information along with metrics " +
 		"such as cpu, memory and disk usage for the controller and hosts are updated.",
 });
+definePermission({
+	name: "core.host.stop",
+	title: "Stop Hosts",
+	description:
+		"Stop Node.js host processes making the host inaccessible until someone with access to the system " +
+		"it runs on manually starts it again.",
+});
+definePermission({
+	name: "core.host.restart",
+	title: "Restart Hosts",
+	description: "Restart Node.js host processes if the system they run on are set up for restarting.",
+});
 definePermission({
 	name: "core.host.get_config",
 	title: "Get host configs",

packages/lib/src/system_collectors.ts

@@ -134,7 +134,7 @@ function minZip<T>(a: T[], b: T[]): [T, T][] {
 
 let previousTotalCpuMs: number[] = [];
 let previousIdleCpuMs: number[] = [];
-export async function gatherSystemInfo(id: SystemInfo["id"]) {
+export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean) {
 	const cpus = os.cpus();
 	const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
 	const currentIdleCpuMs = cpus.map(({ times }) => times.idle);
@@ -164,6 +164,7 @@ export async function gatherSystemInfo(id: SystemInfo["id"]) {
 		os.freemem(),
 		diskCapacity,
 		diskAvailable,
+		canRestart,
 		Date.now(),
 		false,
 	);

packages/web_ui/src/components/ControllerPage.tsx

@@ -1,5 +1,7 @@
-import React from "react";
-import { Descriptions, Typography } from "antd";
+import React, {useContext} from "react";
+import { Button, Descriptions, Popconfirm, Space, Typography } from "antd";
+
+import * as lib from "@clusterio/lib";
 
 import PluginExtra from "./PluginExtra";
 import LogConsole from "./LogConsole";
@@ -9,8 +11,11 @@ import {
 } from "./system_metrics";
 import { useAccount } from "../model/account";
 import { useSystems } from "../model/system";
+import ControlContext from "./ControlContext";
 import ControllerConfigTree from "./ControllerConfigTree";
+import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
+import { notifyErrorHandler } from "../util/notify";
 import webUiPackage from "../../package.json";
 
 const { Title } = Typography;
@@ -18,11 +23,50 @@ const { Title } = Typography;
 
 export default function ControllerPage() {
 	let account = useAccount();
+	const control = useContext(ControlContext);
 	const [systems] = useSystems();
 	const system = systems.get("controller");
 
 	return <PageLayout nav={[{ name: "Controller" }]}>
-		<h2>Controller</h2>
+		<PageHeader
+			title="Controller"
+			extra={<Space>
+				{
+					account.hasPermission("core.controller.stop")
+					&& <Popconfirm
+						title={<>
+							Stopping the controller will leave the cluster inoperable until someone with
+							access to the system it runs on manually starts it again.<br />
+							Are you sure you want to stop the controller?
+						</>}
+						placement="bottomRight"
+						okText="Stop"
+						okButtonProps={{ danger: true }}
+						onConfirm={() => {
+							control.send(
+								new lib.ControllerStopRequest()
+							).catch(notifyErrorHandler("Error stopping controller"));
+						}}
+					>
+						<Button danger>Stop</Button>
+					</Popconfirm>
+				}
+				{
+					account.hasPermission("core.controller.restart")
+					&& <Button
+						disabled={system?.canRestart === false}
+						onClick={() => {
+							control.send(
+								new lib.ControllerRestartRequest()
+							).catch(notifyErrorHandler("Error restarting controller"));
+						}}
+					>
+						Restart
+					</Button>
+				}
+			</Space>}
+		/>
+
 		<Descriptions bordered size="small" column={{ xs: 1, md: 2, lg: 2, xl: 2, xxl: 2 }}>
 			<Descriptions.Item label="Version">{webUiPackage.version}</Descriptions.Item>
 			<Descriptions.Item label="Node.js">{system?.node}</Descriptions.Item>

packages/web_ui/src/components/HostViewPage.tsx

@@ -46,8 +46,9 @@ export default function HostViewPage() {
 		</PageLayout>;
 	}
 
-	let hostButtons = <Space> {
-		account.hasPermission("core.host.revoke_access")
+	let hostButtons = <Space>
+		{
+			account.hasPermission("core.host.revoke_access")
 			&& <Popconfirm
 				title={`Revoke tokens of ${host.name}?`}
 				placement="bottomRight"
@@ -63,7 +64,42 @@ export default function HostViewPage() {
 					Revoke tokens
 				</Button>
 			</Popconfirm>
-	}
+		}
+		{
+			account.hasPermission("core.host.stop")
+			&& <Popconfirm
+				title={<>
+					Stopping this host will leave it inaccessible until someone with
+					access to the system it runs on manually starts it again.<br />
+					Are you sure you want to stop the host?
+				</>}
+				placement="bottomRight"
+				okText="Stop"
+				okButtonProps={{ danger: true }}
+				onConfirm={() => {
+					control.sendTo(
+						{ hostId },
+						new lib.HostStopRequest()
+					).catch(notifyErrorHandler("Error stopping host"));
+				}}
+			>
+				<Button danger>Stop</Button>
+			</Popconfirm>
+		}
+		{
+			account.hasPermission("core.host.restart")
+			&& <Button
+				disabled={system?.canRestart === false}
+				onClick={() => {
+					control.sendTo(
+						{ hostId },
+						new lib.HostRestartRequest()
+					).catch(notifyErrorHandler("Error restarting host"));
+				}}
+			>
+				Restart
+			</Button>
+		}
 	</Space>;
 
 	return <PageLayout nav={nav}>

test/lib/data/SystemInfo.js

@@ -20,6 +20,7 @@ describe("lib/data/SystemInfo", function() {
 				768,
 				2048,
 				512,
+				false,
 				0,
 				false
 			);