diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 015389f4..5a7160ee 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -53,6 +53,8 @@ export default class Controller { config: lib.ControllerConfig; app: Application; + /** True if (@link Controller.systemMetrics} has changed since last time it was saved */ + systemMetricsDirty = false; /** True if {@link Controller.hosts} has changed since last time it was saved */ hostsDirty = false; /** True if {@link Controller.instances} has changed since last time it was saved */ @@ -89,6 +91,7 @@ export default class Controller { devMiddleware: any | null = null; autosaveInterval?: ReturnType; + systemMetricsInterval?: ReturnType; logDirectory: string = ""; clusterLogIndex: lib.LogIndex | null = null; @@ -98,6 +101,7 @@ export default class Controller { let databaseDirectory = config.get("controller.database_directory"); await fs.ensureDir(databaseDirectory); + const systemMetrics = await Controller.loadSystemMetrics(path.join(databaseDirectory, "metrics.json")); const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json")); const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json")); const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json")); @@ -110,6 +114,7 @@ export default class Controller { const modStore = await lib.ModStore.fromDirectory(modsDirectory); return [ + systemMetrics, hosts, instances, saves, @@ -125,6 +130,7 @@ export default class Controller { configPath: string, config: lib.ControllerConfig, + public systemMetrics = new Map(), /** Mapping of host id to host info */ public hosts = new Map(), /** Mapping of instance id to instance info */ @@ -154,6 +160,7 @@ export default class Controller { }); // Handle subscriptions for all internal properties + this.subscriptions.handle(lib.SystemMetricsUpdateEvent, this.handleSystemMetricsSubscription.bind(this)); this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this)); this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this)); this.subscriptions.handle( @@ -219,6 +226,8 @@ export default class Controller { this.config.on("fieldChanged", (field, curr, prev) => { if (field === "controller.autosave_interval") { this.onAutosaveIntervalChanged(); + } else if (field === "controller.system_metrics_interval") { + this.onSystemMetricsIntervalChanged(); } lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev); }); @@ -292,6 +301,7 @@ export default class Controller { } this.onAutosaveIntervalChanged(); + this.onSystemMetricsIntervalChanged(); logger.info("Started controller"); this._state = "running"; @@ -357,6 +367,11 @@ export default class Controller { clearInterval(this.clusterLogBuildInterval); } + if (this.systemMetricsInterval) { + clearInterval(this.systemMetricsInterval); + this.systemMetricsInterval = undefined; + } + if (this.autosaveInterval) { clearInterval(this.autosaveInterval); this.autosaveInterval = undefined; @@ -389,6 +404,41 @@ export default class Controller { logger.info("Goodbye"); } + onSystemMetricsIntervalChanged() { + if (this.systemMetricsInterval) { + clearInterval(this.systemMetricsInterval); + this.systemMetricsInterval = undefined; + } + const systemMetricsIntervalSeconds = this.config.get("controller.system_metrics_interval"); + if (systemMetricsIntervalSeconds > 0) { + this.systemMetricsInterval = setInterval( + this.updateSystemMetrics.bind(this), + systemMetricsIntervalSeconds * 1000 + ); + } + } + + async updateSystemMetrics() { + try { + const requests: Promise[] = []; + for (let hostConnection of this.wsServer.hostConnections.values()) { + if (!hostConnection.connected) { + continue; + } + requests.push(hostConnection.send(new lib.SystemMetricsRequest())); + } + requests.push(lib.gatherSystemMetrics("controller")); + const newMetrics = await Promise.all(requests); + for (const metric of newMetrics) { + this.systemMetrics.set(metric.id, metric); + } + this.systemMetricsDirty = true; + this.subscriptions.broadcast(new lib.SystemMetricsUpdateEvent(newMetrics)); + } catch (err: any) { + logger.error(`Unexpected error updating system metrics:\n${err.stack ?? err.message}`); + } + } + onAutosaveIntervalChanged() { if (this.autosaveInterval) { clearInterval(this.autosaveInterval); @@ -425,6 +475,11 @@ export default class Controller { } let databaseDirectory = this.config.get("controller.database_directory"); + if (this.systemMetricsDirty) { + this.systemMetricsDirty = false; + await Controller.saveSystemMetrics(path.join(databaseDirectory, "metrics.json"), this.systemMetrics); + } + if (this.hostsDirty) { this.hostsDirty = false; await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); @@ -452,6 +507,25 @@ export default class Controller { await lib.invokeHook(this.plugins, "onSaveData"); } + static async loadSystemMetrics(filePath: string): Promise> { + let json: Static[]; + try { + json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); + + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + + return new Map(json.map(m => lib.SystemMetrics.fromJSON(m)).map(m => [m.id, m])); + } + + static async saveSystemMetrics(filePath: string, systemMetrics: Map) { + await lib.safeOutputFile(filePath, JSON.stringify([...systemMetrics.values()], null, "\t")); + } + static async loadHosts(filePath: string): Promise> { let serialized: [number, Static][]; try { @@ -635,6 +709,13 @@ export default class Controller { } } + async handleSystemMetricsSubscription(request: lib.SubscriptionRequest) { + const systemMetrics = [...this.systemMetrics.values()].filter( + metric => metric.updatedAt > request.lastRequestTime, + ); + return systemMetrics.length ? new lib.SystemMetricsUpdateEvent(systemMetrics) : null; + } + /** * Generate access token for host * diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 6abe417c..5e9297f9 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -347,6 +347,7 @@ export default class Host extends lib.Link { this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.handleWhitelistUpdateEvent.bind(this)); this.handle(lib.InstanceAssignInternalRequest, this.handleInstanceAssignInternalRequest.bind(this)); this.handle(lib.InstanceUnassignInternalRequest, this.handleInstanceUnassignInternalRequest.bind(this)); + this.handle(lib.SystemMetricsRequest, this.handleSystemMetricsRequest.bind(this)); this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this)); this.fallbackRequest( lib.InstanceSaveDetailsListRequest, this.fallbackInstanceSaveDetailsListRequest.bind(this), @@ -723,6 +724,10 @@ export default class Host extends lib.Link { return instanceConnection; } + async handleSystemMetricsRequest() { + return lib.gatherSystemMetrics(this.config.get("host.id")); + } + async handleHostMetricsRequest() { let requests: Promise>[] = []; for (let instanceConnection of this.instanceConnections.values()) { diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index fd691557..69f54d79 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -17,6 +17,7 @@ export interface ControllerConfigFields { "controller.heartbeat_interval": number; "controller.session_timeout": number; "controller.metrics_timeout": number; + "controller.system_metrics_interval": number; "controller.proxy_stream_timeout": number; "controller.default_mod_pack_id": number | null; "controller.default_role_id": number | null; @@ -126,6 +127,12 @@ export class ControllerConfig extends classes.Config { type: "number", initialValue: 8, }, + "controller.system_metrics_interval": { + title: "System Metrics Interval", + description: "Interval in seconds to collect and update system metrics for the Web UI", + type: "number", + initialValue: 10, + }, "controller.proxy_stream_timeout": { title: "Proxy Stream Timeout", description: "Timeout in seconds for proxy streams to start flowing.", diff --git a/packages/lib/src/data/messages_controller.ts b/packages/lib/src/data/messages_controller.ts index d4a7d6f5..6f547164 100644 --- a/packages/lib/src/data/messages_controller.ts +++ b/packages/lib/src/data/messages_controller.ts @@ -204,6 +204,108 @@ export class LogMessageEvent { } } +export class SystemMetrics { + constructor( + /** + * Id of the host these metrics originate from, or the string + * "controller" if these metrics are for the controller. + */ + public id: number | "controller", + public coreRatios: number[], + public memoryCapacity: number, + public memoryAvailable: number, + public diskCapacity: number, + public diskAvailable: number, + /** Millisecond Unix timestamp this entry was last updated at */ + public updatedAt: number, + public isDeleted: boolean, + ) { } + + static jsonSchema = Type.Object({ + "id": Type.Union([Type.Number(), Type.Literal("controller")]), + "coreRatios": Type.Array(Type.Number()), + "memoryCapacity": Type.Number(), + "memoryAvailable": Type.Number(), + "diskCapacity": Type.Number(), + "diskAvailable": Type.Number(), + "updatedAt": Type.Number(), + "isDeleted": Type.Boolean(), + }); + + static fromJSON(json: Static) { + return new this( + json.id, + json.coreRatios, + json.memoryCapacity, + json.memoryAvailable, + json.diskCapacity, + json.diskAvailable, + json.updatedAt, + json.isDeleted, + ); + } + + get cpuCapacity() { + return this.coreRatios.length; + } + + get cpuUsed() { + return this.coreRatios.reduce((a, b) => a + b, 0); + } + + get cpuAvailable() { + return this.cpuCapacity - this.cpuUsed; + } + + get cpuRatio() { + return this.cpuUsed / this.cpuCapacity; + } + + get memoryUsed() { + return this.memoryCapacity - this.memoryAvailable; + } + + get memoryRatio() { + return (this.memoryCapacity - this.memoryAvailable) / this.memoryCapacity; + } + + get diskUsed() { + return this.diskCapacity - this.diskAvailable; + } + + get diskRatio() { + return (this.diskCapacity - this.diskAvailable) / this.diskCapacity; + } +} + +export class SystemMetricsRequest { + declare ["constructor"]: typeof SystemMetricsRequest; + static type = "request" as const; + static src = "controller" as const; + static dst = "host" as const; + static Response = SystemMetrics; +} + +export class SystemMetricsUpdateEvent { + declare ["constructor"]: typeof SystemMetricsUpdateEvent; + static type = "event" as const; + static src = "controller" as const; + static dst = "control" as const; + static permission = "core.system_metrics.subscribe" as const; + + constructor( + public updates: SystemMetrics[], + ) { } + + static jsonSchema = Type.Object({ + "updates": Type.Array(SystemMetrics.jsonSchema), + }); + + static fromJSON(json: Static) { + return new this(json.updates.map(update => SystemMetrics.fromJSON(update))); + } +} + export class DebugDumpWsRequest { declare ["constructor"]: typeof DebugDumpWsRequest; static type = "request" as const; diff --git a/packages/lib/src/link/messages.ts b/packages/lib/src/link/messages.ts index 13743844..79d5ba03 100644 --- a/packages/lib/src/link/messages.ts +++ b/packages/lib/src/link/messages.ts @@ -25,6 +25,8 @@ export const dataClasses: (RequestClass | EventClass) controller.LogSetSubscriptionsRequest, controller.LogQueryRequest, controller.LogMessageEvent, + controller.SystemMetricsRequest, + controller.SystemMetricsUpdateEvent, controller.DebugDumpWsRequest, controller.DebugWsMessageEvent, diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index 55a2ca5b..3f9d5f44 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -129,6 +129,13 @@ definePermission({ title: "Modify controller config", description: "Modify the controller config or entries of the controller config.", }); +definePermission({ + name: "core.system_metrics.subscribe", + title: "Subscribe to system metrics updates", + description: + "Subscribe to be notified when the system metrics detailing cpu, memory and disk usage for the " + + "controller and hosts are updated.", +}); definePermission({ name: "core.host.list", title: "List hosts", diff --git a/packages/lib/src/system_collectors.ts b/packages/lib/src/system_collectors.ts index 0dbdbe72..f957f16b 100644 --- a/packages/lib/src/system_collectors.ts +++ b/packages/lib/src/system_collectors.ts @@ -2,6 +2,7 @@ import fs from "fs-extra"; import os from "os"; import util from "util"; import { Gauge } from "./prometheus"; +import { SystemMetrics } from "./data"; function cpuModel() { const model = os.cpus()[0].model; @@ -119,3 +120,45 @@ export const systemDiskAvailableBytes = new Gauge( }, }, ); + +function minZip(a: T[], b: T[]): [T, T][] { + const length = Math.min(a.length, b.length); + const result = []; + result.length = length; + for (let i = 0; i < length; i++) { + result[i] = [a[i], b[i]] as [T, T]; + } + return result; +} + +let previousTotalCpuMs: number[] = []; +let previousIdleCpuMs: number[] = []; +export async function gatherSystemMetrics(id: SystemMetrics["id"]) { + const cpus = os.cpus(); + const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys); + const currentIdleCpuMs = cpus.map(({ times }) => times.idle); + const deltaTotalCpuMs = minZip(previousTotalCpuMs, currentTotalCpuMs).map(([prev, curr]) => curr - prev); + const deltaIdleCpuMs = minZip(previousIdleCpuMs, currentIdleCpuMs).map(([prev, curr]) => curr - prev); + const cpuUsage = minZip(deltaTotalCpuMs, deltaIdleCpuMs).map(([total, idle]) => (total - idle) / total); + previousTotalCpuMs = currentTotalCpuMs; + previousIdleCpuMs = currentIdleCpuMs; + let diskCapacity = 0; + let diskAvailable = 0; + if (fs.statfs) { // TODO: remove this check once minimum supported Node.js >= v18.15.0. + const statFsAsync = util.promisify(fs.statfs); + const stats = await statFsAsync("."); + diskCapacity = stats.blocks * stats.bsize; + diskAvailable = stats.bavail * stats.bsize; + } + + return new SystemMetrics( + id, + cpuUsage, + os.totalmem(), + os.freemem(), + diskCapacity, + diskAvailable, + Date.now(), + false, + ); +} diff --git a/packages/web_ui/src/components/ControllerPage.tsx b/packages/web_ui/src/components/ControllerPage.tsx index 75f67097..e4c61ae9 100644 --- a/packages/web_ui/src/components/ControllerPage.tsx +++ b/packages/web_ui/src/components/ControllerPage.tsx @@ -1,9 +1,14 @@ import React from "react"; -import { Typography } from "antd"; +import { Descriptions, Typography } from "antd"; import PluginExtra from "./PluginExtra"; import LogConsole from "./LogConsole"; +import { + MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, + MetricDiskUsed, MetricDiskRatio, +} from "./system_metrics"; import { useAccount } from "../model/account"; +import { useSystemMetrics } from "../model/system_metrics"; import ControllerConfigTree from "./ControllerConfigTree"; import PageLayout from "./PageLayout"; @@ -12,9 +17,19 @@ const { Title } = Typography; export default function ControllerPage() { let account = useAccount(); + const [systemMetrics] = useSystemMetrics(); + const metrics = systemMetrics.get("controller"); return

Controller

+ + + + + + + + {account.hasPermission("core.log.follow") && <> Console diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index 6c7c341e..f9c9f74e 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -13,7 +13,12 @@ import { useHost } from "../model/host"; import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; +import { + MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, + MetricDiskUsed, MetricDiskRatio, +} from "./system_metrics"; import { formatTimestamp } from "../util/time_format"; +import { useSystemMetrics } from "../model/system_metrics"; const { Title } = Typography; @@ -23,6 +28,8 @@ export default function HostViewPage() { let control = useContext(ControlContext); let account = useAccount(); let [instances] = useInstances(); + const [systemMetrics] = useSystemMetrics(); + const metrics = systemMetrics.get(hostId); const [host, synced] = useHost(hostId); const hostInstances = new Map([...instances].filter(([_id, instance]) => instance.assignedHost === hostId)); @@ -63,20 +70,27 @@ export default function HostViewPage() { title={host.name || String(hostId)} extra={hostButtons} /> - + {host["name"]} {host["connected"] ? "Connected" : "Disconnected"} + + + + + + {host["agent"]} {host["version"]} { host.tokenValidAfter - && - {formatTimestamp(host.tokenValidAfter*1000)} - + ? + {formatTimestamp(host.tokenValidAfter*1000)} + + : null } {account.hasPermission("core.instance.list") && <> diff --git a/packages/web_ui/src/components/HostsPage.tsx b/packages/web_ui/src/components/HostsPage.tsx index d80a4119..6a6586fa 100644 --- a/packages/web_ui/src/components/HostsPage.tsx +++ b/packages/web_ui/src/components/HostsPage.tsx @@ -1,6 +1,6 @@ import React, { useContext, useRef, useState, useEffect } from "react"; import { useNavigate } from "react-router-dom"; -import { Button, Form, InputNumber, Modal, Table, Tag, Typography } from "antd"; +import { Button, Flex, Form, InputNumber, Modal, Progress, Switch, Table, Tag, Typography } from "antd"; import CopyOutlined from "@ant-design/icons/lib/icons/CopyOutlined"; import * as lib from "@clusterio/lib"; @@ -10,7 +10,12 @@ import ControlContext from "./ControlContext"; import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; +import { + MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, + MetricDiskUsed, MetricDiskRatio, +} from "./system_metrics"; import { useHosts } from "../model/host"; +import { useSystemMetrics } from "../model/system_metrics"; import notify, { notifyErrorHandler } from "../util/notify"; const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare; @@ -180,11 +185,13 @@ function CopyButton({ text, message }: { text:string, message:string }) { ; } - export default function HostsPage() { let account = useAccount(); let navigate = useNavigate(); let [hosts] = useHosts(); + const [metrics] = useSystemMetrics(); + const [showRatios, setShowRatios] = useState(true); + const [showNumbers, setShowNumbers] = useState(false); return : undefined} /> strcmp(a.name, b.name), }, + { + title: "CPU%", + sorter: (a, b) => (metrics.get(a.id)?.cpuRatio ?? 0) - (metrics.get(b.id)?.cpuRatio ?? 0), + render: (_, host) => , + hidden: !showRatios, + }, + { + title: "Cores", + sorter: (a, b) => ( + (metrics.get(a.id)?.cpuUsed ?? 0) - (metrics.get(b.id)?.cpuUsed ?? 0) + ), + render: (_, host) => , + hidden: !showNumbers, + }, + { + title: "Mem%", + sorter: (a, b) => ( + (metrics.get(a.id)?.memoryRatio ?? 0) - (metrics.get(b.id)?.memoryRatio ?? 0) + ), + render: (_, host) => , + hidden: !showRatios, + }, + { + title: "Memory", + sorter: (a, b) => ( + (metrics.get(a.id)?.memoryUsed ?? 0) - (metrics.get(b.id)?.memoryUsed ?? 0) + ), + render: (_, host) => , + hidden: !showNumbers, + }, + { + title: "Disk%", + sorter: (a, b) => (metrics.get(a.id)?.diskAvailable ?? 0) - (metrics.get(b.id)?.diskAvailable ?? 0), + render: (_, host) => , + hidden: !showRatios, + }, + { + title: "Disk", + sorter: (a, b) => ( + (metrics.get(a.id)?.diskUsed ?? 0) - (metrics.get(b.id)?.diskUsed ?? 0) + ), + render: (_, host) => , + hidden: !showNumbers, + }, { title: "Agent", dataIndex: "agent", @@ -235,6 +287,16 @@ export default function HostsPage() { }, })} /> + + + + ; }; diff --git a/packages/web_ui/src/components/system_metrics.tsx b/packages/web_ui/src/components/system_metrics.tsx new file mode 100644 index 00000000..aed2dd83 --- /dev/null +++ b/packages/web_ui/src/components/system_metrics.tsx @@ -0,0 +1,55 @@ +import React from "react"; + +import * as lib from "@clusterio/lib"; +import { Progress } from "antd"; + +export function MetricCpuRatio(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const ratio = props.metrics.cpuRatio; + return ; +} + +export function MetricCpuUsed(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const used = props.metrics.cpuUsed.toLocaleString(undefined, { maximumFractionDigits: 1 }); + const capacity = props.metrics.cpuCapacity; + return `${used} / ${capacity}`; +} + +export function MetricMemoryRatio(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const ratio = props.metrics.memoryRatio; + return ; +} + +export function MetricMemoryUsed(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const used = lib.formatBytes(props.metrics.memoryUsed); + const capacity = lib.formatBytes(props.metrics.memoryCapacity); + return `${used} / ${capacity}`; +} + +export function MetricDiskRatio(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const ratio = props.metrics.diskRatio; + return ; +} + +export function MetricDiskUsed(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const used = lib.formatBytes(props.metrics.diskUsed); + const capacity = lib.formatBytes(props.metrics.diskCapacity); + return `${used} / ${capacity}`; +} diff --git a/packages/web_ui/src/model/system_metrics.ts b/packages/web_ui/src/model/system_metrics.ts new file mode 100644 index 00000000..4f1f0e5a --- /dev/null +++ b/packages/web_ui/src/model/system_metrics.ts @@ -0,0 +1,8 @@ +import { useCallback, useContext, useSyncExternalStore } from "react"; +import ControlContext from "../components/ControlContext"; + +export function useSystemMetrics() { + const control = useContext(ControlContext); + const subscribe = useCallback((callback: () => void) => control.systemMetrics.subscribe(callback), [control]); + return useSyncExternalStore(subscribe, () => control.systemMetrics.getSnapshot()); +} diff --git a/packages/web_ui/src/util/websocket.ts b/packages/web_ui/src/util/websocket.ts index 06f796e9..6596725b 100644 --- a/packages/web_ui/src/util/websocket.ts +++ b/packages/web_ui/src/util/websocket.ts @@ -60,6 +60,7 @@ export class Control extends lib.Link { logHandlers: Map = new Map(); /** Updates handled by the subscription service */ + systemMetrics = new lib.EventSubscriber(lib.SystemMetricsUpdateEvent, this); hosts = new lib.EventSubscriber(lib.HostUpdatesEvent, this); instances = new lib.EventSubscriber(lib.InstanceDetailsUpdatesEvent, this); saves = new lib.EventSubscriber(lib.InstanceSaveDetailsUpdatesEvent, this);