From 8884116ee92d4e2e358d680497dda520930c8e17 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 18 Jan 2024 16:10:42 +0100 Subject: [PATCH] Implement stop and restart of controller/hosts (#554) Implement remotely initiated stopping and restarting of the controller and hosts. Restarting is implemented by exiting with a non-zero exit code and relying on a process manager to start the procress again. To prevent accidentally trying to restart on a system that doesn't have a process manager present the --can-restart command line option needs to be passed to enable the restart operation. --- packages/controller/controller.ts | 5 ++ packages/controller/src/ControlConnection.ts | 14 +++++ packages/controller/src/Controller.ts | 20 ++++++- packages/create/create.js | 53 ++++++++++++++----- packages/ctl/src/commands.ts | 34 ++++++++++++ packages/host/host.ts | 14 ++++- packages/host/src/Host.ts | 23 +++++++- packages/lib/src/data/messages_controller.ts | 19 +++++++ packages/lib/src/data/messages_host.ts | 16 ++++++ packages/lib/src/link/messages.ts | 4 ++ packages/lib/src/permissions.ts | 24 +++++++++ packages/lib/src/system_collectors.ts | 3 +- .../web_ui/src/components/ControllerPage.tsx | 50 +++++++++++++++-- .../web_ui/src/components/HostViewPage.tsx | 42 +++++++++++++-- test/lib/data/SystemInfo.js | 1 + 15 files changed, 297 insertions(+), 25 deletions(-) diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 08f3a71f..14b18e46 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -244,6 +244,10 @@ async function initialize(): Promise { .demandCommand(1, "You need to specify a command to run"); }) .command("run", "Run controller", yargs => { + yargs.option("can-restart", { + type: "boolean", nargs: 0, default: false, + describe: "Indicate that a process monitor will restart the controller on failure", + }); yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 }); yargs.option("dev-plugin", { hidden: true, type: "array" }); }) @@ -386,6 +390,7 @@ async function startup() { pluginInfos, controllerConfigPath, controllerConfig, + Boolean(args.canRestart), ...await Controller.bootstrap(controllerConfig) ); diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 9284b97c..595f900b 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -68,6 +68,8 @@ export default class ControlConnection extends BaseConnection { }); } + this.handle(lib.ControllerStopRequest, this.handleControllerStopRequest.bind(this)); + this.handle(lib.ControllerRestartRequest, this.handleControllerRestartRequest.bind(this)); this.handle(lib.ControllerConfigGetRequest, this.handleControllerConfigGetRequest.bind(this)); this.handle(lib.ControllerConfigSetFieldRequest, this.handleControllerConfigSetFieldRequest.bind(this)); this.handle(lib.ControllerConfigSetPropRequest, this.handleControllerConfigSetPropRequest.bind(this)); @@ -167,6 +169,18 @@ export default class ControlConnection extends BaseConnection { throw new Error("Should be unreachable"); } + async handleControllerStopRequest() { + this._controller.stop(); + } + + async handleControllerRestartRequest() { + if (!this._controller.canRestart) { + throw new lib.RequestError("Cannot restart, controller does not have a process monitor to restart it."); + } + this._controller.shouldRestart = true; + this._controller.stop(); + } + async handleControllerConfigGetRequest() { return this._controller.config.toRemote("control"); } diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index d8686527..89214e0f 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -84,6 +84,12 @@ export default class Controller { // Possible states are new, starting, running, stopping, stopped private _state: string = "new"; private _shouldStop: boolean = false; + /** + * Set to true before calling stop to have the controller restart + * instead of stopping. This only works if {@link Controller.canRestart} + * is true. + */ + public shouldRestart: boolean = false; _fallbackedRequests: Map, lib.RequestHandler> = new Map(); _registeredRequests: Map, lib.RequestHandler> = new Map(); _registeredEvents = new Map(); @@ -131,6 +137,11 @@ export default class Controller { configPath: string, config: lib.ControllerConfig, + /** + * If true indicates that there is a process monitor present that + * will restart the controller on non-zero exit codes. + */ + public canRestart: boolean = false, public systems = new Map(), /** Mapping of host id to host info */ public hosts = new Map(), @@ -365,7 +376,12 @@ export default class Controller { async _stopInternal() { // This function should never throw. this._state = "stopping"; - logger.info("Stopping controller"); + if (this.shouldRestart) { + logger.info("Restarting controller"); + process.exitCode = 1; + } else { + logger.info("Stopping controller"); + } if (this.clusterLogBuildInterval) { clearInterval(this.clusterLogBuildInterval); @@ -459,7 +475,7 @@ export default class Controller { } requests.push(hostConnection.send(new lib.SystemInfoRequest())); } - requests.push(lib.gatherSystemInfo("controller")); + requests.push(lib.gatherSystemInfo("controller", this.canRestart)); const newMetrics = await Promise.all(requests); for (const metric of newMetrics) { this.systems.set(metric.id, metric); diff --git a/packages/create/create.js b/packages/create/create.js index 0a1cd5da..a50b5dbd 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -87,7 +87,7 @@ class InstallError extends Error { } async function safeOutputFile(file, data, options={}) { let { dir, name, ext } = path.parse(file); - let temporary = `${dir}${name}.tmp${ext}`; + let temporary = path.join(dir, `${name}.tmp${ext}`); await fs.outputFile(temporary, data, options); await fs.rename(temporary, file); } @@ -377,18 +377,25 @@ async function writeScripts(mode) { await safeOutputFile( "run-controller.cmd", `\ -@echo off. +@echo off set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%" -\\node_modules\\.bin\\clusteriocontroller.cmd run +:restart +call .\\node_modules\\.bin\\clusteriocontroller.cmd run --can-restart +if %errorlevel% equ 0 exit /b +if %errorlevel% equ 8 exit /b +goto restart ` ); } else { await safeOutputFile( "run-controller.sh", `\ -#!/bin/sh +#!/bin/bash export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS" -exec ./node_modules/.bin/clusteriocontroller run +while true; do + ./node_modules/.bin/clusteriocontroller run --can-restart + if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi +done `, { mode: 0o755 }, ); @@ -405,11 +412,14 @@ WorkingDirectory=${process.cwd()} KillMode=mixed KillSignal=SIGINT Environment=NODE_OPTIONS=--enable-source-maps -ExecStart=${process.cwd()}/node_modules/.bin/clusteriocontroller run --log-level=warn +ExecStart=${process.cwd()}/node_modules/.bin/clusteriocontroller run --log-level=warn --can-restart +Restart=on-failure +RestartPreventExitStatus=8 [Install] WantedBy=multi-user.target -`); +` + ); } } @@ -417,17 +427,33 @@ WantedBy=multi-user.target if (process.platform === "win32") { await safeOutputFile( "run-host.cmd", - "@echo off\n.\\node_modules\\.bin\\clusteriohost.cmd run\n" + `\ +@echo off +set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%" +:restart +call .\\node_modules\\.bin\\clusteriohost.cmd run --can-restart +if %errorlevel% equ 0 exit /b +if %errorlevel% equ 8 exit /b +goto restart +` ); } else { await safeOutputFile( "run-host.sh", - "#!/bin/sh\nexec ./node_modules/.bin/clusteriohost run\n", + `\ +#!/bin/bash +export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS" +while true; do + ./node_modules/.bin/clusteriohost run --can-restart + if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi +done +`, { mode: 0o755 }, ); await safeOutputFile( "systemd/clusteriohost.service", - `[Unit] + `\ +[Unit] Description=Clusterio Host [Service] @@ -436,11 +462,14 @@ Group=${await groupIdToName(os.userInfo().gid)} WorkingDirectory=${process.cwd()} KillMode=mixed KillSignal=SIGINT -ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn +ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart +Restart=on-failure +RestartPreventExitStatus=8 [Install] WantedBy=multi-user.target -`); +` + ); } } } diff --git a/packages/ctl/src/commands.ts b/packages/ctl/src/commands.ts index 683588e5..8e892a50 100644 --- a/packages/ctl/src/commands.ts +++ b/packages/ctl/src/commands.ts @@ -82,6 +82,20 @@ function serializedConfigToString( } const controllerCommands = new lib.CommandTree({ name: "controller", description: "Controller management" }); +controllerCommands.add(new lib.Command({ + definition: ["stop", "Stop controller"], + handler: async function(_args: object, control: Control) { + await control.send(new lib.ControllerStopRequest()); + }, +})); + +controllerCommands.add(new lib.Command({ + definition: ["restart", "Restart controller"], + handler: async function(_args: object, control: Control) { + await control.send(new lib.ControllerRestartRequest()); + }, +})); + const controllerConfigCommands = new lib.CommandTree({ name: "config", alias: ["c"], description: "controller config management", }); @@ -224,6 +238,26 @@ controllerCommands.add(controllerPluginCommands); const hostCommands = new lib.CommandTree({ name: "host", description: "Host management" }); +hostCommands.add(new lib.Command({ + definition: ["stop ", "Stop the given host", (yargs) => { + yargs.positional("host", { describe: "Host to stop", type: "string" }); + }], + handler: async function(args: { host: string }, control: Control) { + let hostId = await lib.resolveHost(control, args.host); + await control.sendTo({ hostId }, new lib.HostStopRequest()); + }, +})); + +hostCommands.add(new lib.Command({ + definition: ["restart ", "Restart the given host", (yargs) => { + yargs.positional("host", { describe: "Host to restart", type: "string" }); + }], + handler: async function(args: { host: string }, control: Control) { + let hostId = await lib.resolveHost(control, args.host); + await control.sendTo({ hostId }, new lib.HostRestartRequest()); + }, +})); + hostCommands.add(new lib.Command({ definition: [["list", "l"], "List hosts connected to the controller"], handler: async function(args: object, control: Control) { diff --git a/packages/host/host.ts b/packages/host/host.ts index 2026ebe4..46827d85 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -94,7 +94,12 @@ async function startHost() { }) .command("plugin", "Manage available plugins", lib.pluginCommand) .command("config", "Manage Host config", lib.configCommand) - .command("run", "Run host") + .command("run", "Run host", yargs => { + yargs.option("can-restart", { + type: "boolean", nargs: 0, default: false, + describe: "Indicate that a process monitor will restart this host on failure", + }); + }) .demandCommand(1, "You need to specify a command to run") .strict() .parseSync() @@ -206,6 +211,7 @@ async function startHost() { hostConfig, tlsCa, pluginInfos, + Boolean(args.canRestart), ...await Host.bootstrap(hostConfig) ); @@ -295,7 +301,11 @@ ${err.stack}` ); } - process.exitCode = 1; + if (err instanceof lib.AuthenticationFailed) { + process.exitCode = 8; + } else { + process.exitCode = 1; + } }); } diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 96b3161b..5ecfb5bd 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -278,7 +278,12 @@ export default class Host extends lib.Link { hostConfig: lib.HostConfig, tlsCa: string | undefined, pluginInfos: lib.PluginNodeEnvInfo[], - public modStore: lib.ModStore, + /** + * If true indicates that there is a process monitor present that + * will restart this host on a non-zero exit codes. + */ + public canRestart = false, + public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()), ) { super(connector); this.tlsCa = tlsCa; @@ -345,6 +350,8 @@ export default class Host extends lib.Link { }); } + this.handle(lib.HostStopRequest, this.handleHostStopRequest.bind(this)); + this.handle(lib.HostRestartRequest, this.handleHostRestartRequest.bind(this)); this.handle(lib.HostConfigGetRequest, this.handleHostConfigGetRequest.bind(this)); this.handle(lib.HostConfigSetFieldRequest, this.handleHostConfigSetFieldRequest.bind(this)); this.handle(lib.HostConfigSetPropRequest, this.handleHostConfigSetPropRequest.bind(this)); @@ -430,6 +437,18 @@ export default class Host extends lib.Link { } } + async handleHostStopRequest() { + this.shutdown(); + } + + async handleHostRestartRequest() { + if (!this.canRestart) { + throw new lib.RequestError("Cannot restart, host does not have a process monitor to restart it."); + } + process.exitCode = 1; + this.shutdown(); + } + async handleHostConfigGetRequest() { return this.config.toRemote("control"); } @@ -751,7 +770,7 @@ export default class Host extends lib.Link { } async handleSystemInfoRequest() { - return lib.gatherSystemInfo(this.config.get("host.id")); + return lib.gatherSystemInfo(this.config.get("host.id"), this.canRestart); } async handleHostMetricsRequest() { diff --git a/packages/lib/src/data/messages_controller.ts b/packages/lib/src/data/messages_controller.ts index 2a755b0e..11477a53 100644 --- a/packages/lib/src/data/messages_controller.ts +++ b/packages/lib/src/data/messages_controller.ts @@ -3,6 +3,22 @@ import { JsonString, StringEnum, plainJson } from "./composites"; import { levels } from "../logging"; import { ControllerConfig, HostConfig } from "../config"; +export class ControllerStopRequest { + declare ["constructor"]: typeof ControllerStopRequest; + static type = "request" as const; + static src = "control" as const; + static dst = "controller" as const; + static permission = "core.controller.stop" as const; +} + +export class ControllerRestartRequest { + declare ["constructor"]: typeof ControllerRestartRequest; + static type = "request" as const; + static src = "control" as const; + static dst = "controller" as const; + static permission = "core.controller.restart" as const; +} + export class ControllerConfigGetRequest { declare ["constructor"]: typeof ControllerConfigGetRequest; static type = "request" as const; @@ -221,6 +237,7 @@ export class SystemInfo { public memoryAvailable: number, public diskCapacity: number, public diskAvailable: number, + public canRestart: boolean, /** Millisecond Unix timestamp this entry was last updated at */ public updatedAt: number, public isDeleted: boolean, @@ -238,6 +255,7 @@ export class SystemInfo { "memoryAvailable": Type.Number(), "diskCapacity": Type.Number(), "diskAvailable": Type.Number(), + "canRestart": Type.Boolean(), "updatedAt": Type.Number(), "isDeleted": Type.Boolean(), }); @@ -255,6 +273,7 @@ export class SystemInfo { json.memoryAvailable, json.diskCapacity, json.diskAvailable, + json.canRestart, json.updatedAt, json.isDeleted, ); diff --git a/packages/lib/src/data/messages_host.ts b/packages/lib/src/data/messages_host.ts index df8d719f..62a95647 100644 --- a/packages/lib/src/data/messages_host.ts +++ b/packages/lib/src/data/messages_host.ts @@ -3,6 +3,22 @@ import { jsonArray, plainJson, StringEnum } from "./composites"; import { CollectorResultSerialized } from "../prometheus"; import { HostConfig } from "../config/definitions"; +export class HostStopRequest { + declare ["constructor"]: typeof HostStopRequest; + static type = "request" as const; + static src = ["control", "controller"] as const; + static dst = "host" as const; + static permission = "core.host.stop" as const; +} + +export class HostRestartRequest { + declare ["constructor"]: typeof HostRestartRequest; + static type = "request" as const; + static src = ["control", "controller"] as const; + static dst = "host" as const; + static permission = "core.host.restart" as const; +} + export class HostConfigGetRequest { declare ["constructor"]: typeof HostConfigGetRequest; static type = "request" as const; diff --git a/packages/lib/src/link/messages.ts b/packages/lib/src/link/messages.ts index 5431eb0c..7dd7ed99 100644 --- a/packages/lib/src/link/messages.ts +++ b/packages/lib/src/link/messages.ts @@ -17,6 +17,8 @@ export const dataClasses: (RequestClass | EventClass) core.PingRequest, core.AccountUpdateEvent, + controller.ControllerStopRequest, + controller.ControllerRestartRequest, controller.ControllerConfigGetRequest, controller.ControllerConfigSetFieldRequest, controller.ControllerConfigSetPropRequest, @@ -32,6 +34,8 @@ export const dataClasses: (RequestClass | EventClass) subscriptions.SubscriptionRequest, + host.HostStopRequest, + host.HostRestartRequest, host.HostConfigGetRequest, host.HostConfigSetFieldRequest, host.HostConfigSetPropRequest, diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index e2b577b9..cbaf9a93 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -119,6 +119,18 @@ definePermission({ grantByDefault: true, }); +definePermission({ + name: "core.controller.stop", + title: "Stop controller", + description: + "Stop the Node.js controller process making the cluster inoperable until someone with access to the system " + + "it runs on manually starts it again.", +}); +definePermission({ + name: "core.controller.restart", + title: "Restart controller", + description: "Restart the Node.js controller process if the system is set up for restarting.", +}); definePermission({ name: "core.controller.get_config", title: "Get controller config", @@ -136,6 +148,18 @@ definePermission({ "Subscribe to be notified when the system info detailing system specific information along with metrics " + "such as cpu, memory and disk usage for the controller and hosts are updated.", }); +definePermission({ + name: "core.host.stop", + title: "Stop Hosts", + description: + "Stop Node.js host processes making the host inaccessible until someone with access to the system " + + "it runs on manually starts it again.", +}); +definePermission({ + name: "core.host.restart", + title: "Restart Hosts", + description: "Restart Node.js host processes if the system they run on are set up for restarting.", +}); definePermission({ name: "core.host.get_config", title: "Get host configs", diff --git a/packages/lib/src/system_collectors.ts b/packages/lib/src/system_collectors.ts index 98d92397..6d8e6628 100644 --- a/packages/lib/src/system_collectors.ts +++ b/packages/lib/src/system_collectors.ts @@ -134,7 +134,7 @@ function minZip(a: T[], b: T[]): [T, T][] { let previousTotalCpuMs: number[] = []; let previousIdleCpuMs: number[] = []; -export async function gatherSystemInfo(id: SystemInfo["id"]) { +export async function gatherSystemInfo(id: SystemInfo["id"], canRestart: boolean) { const cpus = os.cpus(); const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys); const currentIdleCpuMs = cpus.map(({ times }) => times.idle); @@ -164,6 +164,7 @@ export async function gatherSystemInfo(id: SystemInfo["id"]) { os.freemem(), diskCapacity, diskAvailable, + canRestart, Date.now(), false, ); diff --git a/packages/web_ui/src/components/ControllerPage.tsx b/packages/web_ui/src/components/ControllerPage.tsx index 7e49d233..9452e2c1 100644 --- a/packages/web_ui/src/components/ControllerPage.tsx +++ b/packages/web_ui/src/components/ControllerPage.tsx @@ -1,5 +1,7 @@ -import React from "react"; -import { Descriptions, Typography } from "antd"; +import React, {useContext} from "react"; +import { Button, Descriptions, Popconfirm, Space, Typography } from "antd"; + +import * as lib from "@clusterio/lib"; import PluginExtra from "./PluginExtra"; import LogConsole from "./LogConsole"; @@ -9,8 +11,11 @@ import { } from "./system_metrics"; import { useAccount } from "../model/account"; import { useSystems } from "../model/system"; +import ControlContext from "./ControlContext"; import ControllerConfigTree from "./ControllerConfigTree"; +import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; +import { notifyErrorHandler } from "../util/notify"; import webUiPackage from "../../package.json"; const { Title } = Typography; @@ -18,11 +23,50 @@ const { Title } = Typography; export default function ControllerPage() { let account = useAccount(); + const control = useContext(ControlContext); const [systems] = useSystems(); const system = systems.get("controller"); return -

Controller

+ + { + account.hasPermission("core.controller.stop") + && + Stopping the controller will leave the cluster inoperable until someone with + access to the system it runs on manually starts it again.
+ Are you sure you want to stop the controller? + } + placement="bottomRight" + okText="Stop" + okButtonProps={{ danger: true }} + onConfirm={() => { + control.send( + new lib.ControllerStopRequest() + ).catch(notifyErrorHandler("Error stopping controller")); + }} + > + + + } + { + account.hasPermission("core.controller.restart") + && + } + } + /> + {webUiPackage.version} {system?.node} diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index efe128f6..fc71311d 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -46,8 +46,9 @@ export default function HostViewPage() { ; } - let hostButtons = { - account.hasPermission("core.host.revoke_access") + let hostButtons = + { + account.hasPermission("core.host.revoke_access") && - } + } + { + account.hasPermission("core.host.stop") + && + Stopping this host will leave it inaccessible until someone with + access to the system it runs on manually starts it again.
+ Are you sure you want to stop the host? + } + placement="bottomRight" + okText="Stop" + okButtonProps={{ danger: true }} + onConfirm={() => { + control.sendTo( + { hostId }, + new lib.HostStopRequest() + ).catch(notifyErrorHandler("Error stopping host")); + }} + > + + + } + { + account.hasPermission("core.host.restart") + && + } ; return diff --git a/test/lib/data/SystemInfo.js b/test/lib/data/SystemInfo.js index 8053e6e9..7b772e3d 100644 --- a/test/lib/data/SystemInfo.js +++ b/test/lib/data/SystemInfo.js @@ -20,6 +20,7 @@ describe("lib/data/SystemInfo", function() { 768, 2048, 512, + false, 0, false );