From 32d70a83d1e36583b3469d11ad4880727ac002ec Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Jan 2024 23:38:10 +0100 Subject: [PATCH 1/7] Fix debug dump-ws crashing the controller When ws was updated to version 8 it came with a breaking change where message data was no longer utf8 decoded from binary to a string. The main codepath handling messages was updated to handle this but the ws dump path was not resulting in the controller crashing when receiving a message that is then forwarded to the dumper without decoding. Fix by decoding the message in the dump-ws path. --- packages/controller/src/WsServer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/controller/src/WsServer.ts b/packages/controller/src/WsServer.ts index fb10cc66..e4b86251 100644 --- a/packages/controller/src/WsServer.ts +++ b/packages/controller/src/WsServer.ts @@ -107,7 +107,7 @@ export default class WsServer { socket.on("message", (message: WebSocket.RawData) => { wsMessageCounter.labels("in").inc(); if (!socket.clusterio_ignore_dump) { - this.controller.debugEvents.emit("message", { direction: "in", content: message }); + this.controller.debugEvents.emit("message", { direction: "in", content: message.toString("utf8") }); } }); let originalSend = socket.send; From 81d6369a3444f2500f82470ff167e881083a5640 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Jan 2024 23:41:56 +0100 Subject: [PATCH 2/7] Fix incorrect merging of samples from hosts The Prometheus library has an optimisation for metrics that expose multiple time series such as Summary that provide two values in the time series consisting of the metric name suffixed with _sum and _count. These are provided as a series of sufixes with corresponding samples such that a serialised summary metric named some_value would have a samples mapping looking something like this: [ ["_sum", ["label=\"value\"", 3]], ["_count", ["lavel=\"value\"", 2]], ] If multiple hosts expose the same metric then these need to be merged, which is the job of mergeSamples, which merges the samples from each sufix into the destination. One corner case that may occur is if the source and destination metric that's being merged have different lists of suffixed samples. In this case the suffixes that are present in the source but not in the destination needs to be added verbatim to the destination. The code for handling this case incorrectly added these samples to the source metric instead of the destination metric. This corner case only occrs if a metric's type differs between two hosts connected to the same controller. For example if a Gauge metric is turned into a Summary metric. --- packages/controller/src/routes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 852f4b59..ab67aeaf 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -39,7 +39,7 @@ function mergeSamples(destinationResult: lib.CollectorResultSerialized, sourceRe } for (let entry of receivedSamples) { - sourceResult.samples.push(entry); + destinationResult.samples.push(entry); } } From b46efeb4e31814603208672ce0efa58e87468799 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Jan 2024 21:18:37 +0100 Subject: [PATCH 3/7] Add system info, CPU, memory and disk metrics Collect basic metrics about the system the controller and hosts run on including CPU, memory and disk capacity and usage. --- packages/host/src/Host.ts | 6 ++ packages/lib/index.ts | 1 + packages/lib/src/system_collectors.ts | 121 ++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 packages/lib/src/system_collectors.ts diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index a975fd09..6abe417c 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -748,6 +748,12 @@ export default class Host extends lib.Link { metricName: result.metric.name.replace("process_", "clusterio_host_"), })); + } else if (result.metric.name.startsWith("system_")) { + results.push(lib.serializeResult(result, { + addLabels: { "host_id": String(this.config.get("host.id")) }, + metricName: result.metric.name.replace("system_", "clusterio_host_system_"), + })); + } else { results.push(lib.serializeResult(result)); } diff --git a/packages/lib/index.ts b/packages/lib/index.ts index 508b900c..3117bcbc 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -25,6 +25,7 @@ export * from "./src/prometheus"; export * from "./src/schema"; export * from "./src/shared_commands"; export * from "./src/stream"; +export * from "./src/system_collectors"; export * from "./src/zip_ops"; export * from "./src/subscriptions"; diff --git a/packages/lib/src/system_collectors.ts b/packages/lib/src/system_collectors.ts new file mode 100644 index 00000000..0dbdbe72 --- /dev/null +++ b/packages/lib/src/system_collectors.ts @@ -0,0 +1,121 @@ +import fs from "fs-extra"; +import os from "os"; +import util from "util"; +import { Gauge } from "./prometheus"; + +function cpuModel() { + const model = os.cpus()[0].model; + // Remove irelevant clutter + return model + .replace(/\(R\)/gi, "®") // Intel CPUs use (R) instead of ® + .replace(/\(TM\)/gi, "™") // Intel and AMD CPUs use (TM) instead of ™ + .replace(/ CPU /, " ") // Intel CPUs add redundant "CPU" in the model. + .replace(/ @ [0-9]+\.[0-9]+GHz/, "") // Intel CPUs add GHz after the model. + .replace(/ (Dual|Quad|Six|[0-9]+)-Core Processor/, "") // AMD CPUs add cores after the model. + .trim() // AMD CPUs may have trailing spaces in the model. + ; +} + +export const systemInfo = new Gauge( + "system_info", + "Static info about the system.", + { + labels: ["kernel", "machine", "cpu_model", "hostname", "node"], + } +); +systemInfo.labels({ + "kernel": os.type(), + "machine": os.machine(), + "cpu_model": cpuModel(), + "hostname": os.hostname(), + "node": process.version, +}).set(1); + +export const systemCpuSecondsTotal = new Gauge( + "system_cpu_seconds_total", + "Total system CPU time spent in seconds.", + { + labels: ["core"], + callback: function(collector) { + const cpus = os.cpus(); + for (let i = 0; i < cpus.length; i++) { + const times = cpus[i].times; + collector.labels(String(i)).set( + (times.user + times.idle + times.irq + times.nice + times.sys) / 1000 + ); + } + }, + }, +); + +export const systemCpuIdleSecondsTotal = new Gauge( + "system_cpu_idle_seconds_total", + "Total system CPU time spent being idle in seconds.", + { + labels: ["core"], + callback: function(collector) { + const cpus = os.cpus(); + for (let i = 0; i < cpus.length; i++) { + const times = cpus[i].times; + collector.labels(String(i)).set( + times.idle / 1000 + ); + } + }, + }, +); + +export const systemMemoryCapasityBytes = new Gauge( + "system_memory_capasity_bytes", + "Total system memory installed", + { + callback: function(collector) { + collector.set(os.totalmem()); + }, + }, +); + +export const systemMemoryAvailableBytes = new Gauge( + "system_memory_available_bytes", + "Total system memory available for use", + { + callback: function(collector) { + collector.set(os.freemem()); + }, + }, +); + +export const systemDiskCapasityBytes = new Gauge( + "system_disk_capasity_bytes", + "Size of the filesystem of the current working directory Node.js runs on", + { + callback: async function(collector) { + // statfs was added in Node.js v18.15.0 and may not be present. + // TODO: remove this check once minimum supported Node.js >= v18.15.0. + if (!fs.statfs) { + collector.set(0); + return; + } + const statFsAsync = util.promisify(fs.statfs); + const stats = await statFsAsync("."); + collector.set(stats.blocks * stats.bsize); + }, + }, +); + +export const systemDiskAvailableBytes = new Gauge( + "system_disk_available_bytes", + "Available space on the filesystem of the current working directory Node.js runs on", + { + callback: async function(collector) { + // TODO: remove this check once minimum supported Node.js >= v18.15.0. + if (!fs.statfs) { + collector.set(0); + return; + } + const statFsAsync = util.promisify(fs.statfs); + const stats = await statFsAsync("."); + collector.set(stats.bavail * stats.bsize); + }, + }, +); From 69bdc8e88444799f2f36a5a926df8cdbad8a06ef Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 14 Jan 2024 00:40:07 +0100 Subject: [PATCH 4/7] Update antd Use the latest version of antd to get support for the hidden property on table columns. --- packages/web_ui/package.json | 2 +- plugins/inventory_sync/package.json | 2 +- plugins/player_auth/package.json | 2 +- plugins/subspace_storage/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 87f93b92..07cbe1f5 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -20,7 +20,7 @@ "@babel/core": "^7.22.9", "@babel/preset-react": "^7.22.5", "@clusterio/lib": "^2.0.0-alpha.13", - "antd": "^5.7.3", + "antd": "^5.13.0", "assert": "^2.0.0", "babel-loader": "^9.1.3", "browserify-zlib": "^0.2.0", diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index 28cc8055..3f2bba9b 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -30,7 +30,7 @@ "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", - "antd": "^5.7.3", + "antd": "^5.13.0", "react": "^18.2.0", "react-dom": "^18.2.0", "typescript": "^5.1.6", diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index 0a497811..7faf3f69 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -32,7 +32,7 @@ "@types/jsonwebtoken": "^9.0.2", "@types/node": "^20.4.5", "@types/react": "^18.2.21", - "antd": "^5.7.3", + "antd": "^5.13.0", "phin": "^3.7.0", "react": "^18.2.0", "react-dom": "^18.2.0", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index 258668c7..0049ebd9 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -31,7 +31,7 @@ "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", - "antd": "^5.7.3", + "antd": "^5.13.0", "react": "^18.2.0", "react-dom": "^18.2.0", "typescript": "^5.1.6", From d64bf6015e86681b74bd6990e7ef4d6bfd484060 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 12 Jan 2024 02:24:04 +0100 Subject: [PATCH 5/7] Show System metrics in hosts page Implement simple collection of basic real time system metrics of hosts and the controller for the purpose of displaying them in the Web UI. The metrics are collected by the controller at a configurable interval and pushed out to web clients currently subscribed to the metrics. --- packages/controller/src/Controller.ts | 81 ++++++++++++++ packages/host/src/Host.ts | 5 + packages/lib/src/config/definitions.ts | 7 ++ packages/lib/src/data/messages_controller.ts | 102 ++++++++++++++++++ packages/lib/src/link/messages.ts | 2 + packages/lib/src/permissions.ts | 7 ++ packages/lib/src/system_collectors.ts | 43 ++++++++ .../web_ui/src/components/ControllerPage.tsx | 17 ++- .../web_ui/src/components/HostViewPage.tsx | 22 +++- packages/web_ui/src/components/HostsPage.tsx | 66 +++++++++++- .../web_ui/src/components/system_metrics.tsx | 55 ++++++++++ packages/web_ui/src/model/system_metrics.ts | 8 ++ packages/web_ui/src/util/websocket.ts | 1 + 13 files changed, 409 insertions(+), 7 deletions(-) create mode 100644 packages/web_ui/src/components/system_metrics.tsx create mode 100644 packages/web_ui/src/model/system_metrics.ts diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 015389f4..5a7160ee 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -53,6 +53,8 @@ export default class Controller { config: lib.ControllerConfig; app: Application; + /** True if (@link Controller.systemMetrics} has changed since last time it was saved */ + systemMetricsDirty = false; /** True if {@link Controller.hosts} has changed since last time it was saved */ hostsDirty = false; /** True if {@link Controller.instances} has changed since last time it was saved */ @@ -89,6 +91,7 @@ export default class Controller { devMiddleware: any | null = null; autosaveInterval?: ReturnType; + systemMetricsInterval?: ReturnType; logDirectory: string = ""; clusterLogIndex: lib.LogIndex | null = null; @@ -98,6 +101,7 @@ export default class Controller { let databaseDirectory = config.get("controller.database_directory"); await fs.ensureDir(databaseDirectory); + const systemMetrics = await Controller.loadSystemMetrics(path.join(databaseDirectory, "metrics.json")); const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json")); const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json")); const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json")); @@ -110,6 +114,7 @@ export default class Controller { const modStore = await lib.ModStore.fromDirectory(modsDirectory); return [ + systemMetrics, hosts, instances, saves, @@ -125,6 +130,7 @@ export default class Controller { configPath: string, config: lib.ControllerConfig, + public systemMetrics = new Map(), /** Mapping of host id to host info */ public hosts = new Map(), /** Mapping of instance id to instance info */ @@ -154,6 +160,7 @@ export default class Controller { }); // Handle subscriptions for all internal properties + this.subscriptions.handle(lib.SystemMetricsUpdateEvent, this.handleSystemMetricsSubscription.bind(this)); this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this)); this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this)); this.subscriptions.handle( @@ -219,6 +226,8 @@ export default class Controller { this.config.on("fieldChanged", (field, curr, prev) => { if (field === "controller.autosave_interval") { this.onAutosaveIntervalChanged(); + } else if (field === "controller.system_metrics_interval") { + this.onSystemMetricsIntervalChanged(); } lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev); }); @@ -292,6 +301,7 @@ export default class Controller { } this.onAutosaveIntervalChanged(); + this.onSystemMetricsIntervalChanged(); logger.info("Started controller"); this._state = "running"; @@ -357,6 +367,11 @@ export default class Controller { clearInterval(this.clusterLogBuildInterval); } + if (this.systemMetricsInterval) { + clearInterval(this.systemMetricsInterval); + this.systemMetricsInterval = undefined; + } + if (this.autosaveInterval) { clearInterval(this.autosaveInterval); this.autosaveInterval = undefined; @@ -389,6 +404,41 @@ export default class Controller { logger.info("Goodbye"); } + onSystemMetricsIntervalChanged() { + if (this.systemMetricsInterval) { + clearInterval(this.systemMetricsInterval); + this.systemMetricsInterval = undefined; + } + const systemMetricsIntervalSeconds = this.config.get("controller.system_metrics_interval"); + if (systemMetricsIntervalSeconds > 0) { + this.systemMetricsInterval = setInterval( + this.updateSystemMetrics.bind(this), + systemMetricsIntervalSeconds * 1000 + ); + } + } + + async updateSystemMetrics() { + try { + const requests: Promise[] = []; + for (let hostConnection of this.wsServer.hostConnections.values()) { + if (!hostConnection.connected) { + continue; + } + requests.push(hostConnection.send(new lib.SystemMetricsRequest())); + } + requests.push(lib.gatherSystemMetrics("controller")); + const newMetrics = await Promise.all(requests); + for (const metric of newMetrics) { + this.systemMetrics.set(metric.id, metric); + } + this.systemMetricsDirty = true; + this.subscriptions.broadcast(new lib.SystemMetricsUpdateEvent(newMetrics)); + } catch (err: any) { + logger.error(`Unexpected error updating system metrics:\n${err.stack ?? err.message}`); + } + } + onAutosaveIntervalChanged() { if (this.autosaveInterval) { clearInterval(this.autosaveInterval); @@ -425,6 +475,11 @@ export default class Controller { } let databaseDirectory = this.config.get("controller.database_directory"); + if (this.systemMetricsDirty) { + this.systemMetricsDirty = false; + await Controller.saveSystemMetrics(path.join(databaseDirectory, "metrics.json"), this.systemMetrics); + } + if (this.hostsDirty) { this.hostsDirty = false; await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); @@ -452,6 +507,25 @@ export default class Controller { await lib.invokeHook(this.plugins, "onSaveData"); } + static async loadSystemMetrics(filePath: string): Promise> { + let json: Static[]; + try { + json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); + + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + + return new Map(json.map(m => lib.SystemMetrics.fromJSON(m)).map(m => [m.id, m])); + } + + static async saveSystemMetrics(filePath: string, systemMetrics: Map) { + await lib.safeOutputFile(filePath, JSON.stringify([...systemMetrics.values()], null, "\t")); + } + static async loadHosts(filePath: string): Promise> { let serialized: [number, Static][]; try { @@ -635,6 +709,13 @@ export default class Controller { } } + async handleSystemMetricsSubscription(request: lib.SubscriptionRequest) { + const systemMetrics = [...this.systemMetrics.values()].filter( + metric => metric.updatedAt > request.lastRequestTime, + ); + return systemMetrics.length ? new lib.SystemMetricsUpdateEvent(systemMetrics) : null; + } + /** * Generate access token for host * diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 6abe417c..5e9297f9 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -347,6 +347,7 @@ export default class Host extends lib.Link { this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.handleWhitelistUpdateEvent.bind(this)); this.handle(lib.InstanceAssignInternalRequest, this.handleInstanceAssignInternalRequest.bind(this)); this.handle(lib.InstanceUnassignInternalRequest, this.handleInstanceUnassignInternalRequest.bind(this)); + this.handle(lib.SystemMetricsRequest, this.handleSystemMetricsRequest.bind(this)); this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this)); this.fallbackRequest( lib.InstanceSaveDetailsListRequest, this.fallbackInstanceSaveDetailsListRequest.bind(this), @@ -723,6 +724,10 @@ export default class Host extends lib.Link { return instanceConnection; } + async handleSystemMetricsRequest() { + return lib.gatherSystemMetrics(this.config.get("host.id")); + } + async handleHostMetricsRequest() { let requests: Promise>[] = []; for (let instanceConnection of this.instanceConnections.values()) { diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index fd691557..69f54d79 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -17,6 +17,7 @@ export interface ControllerConfigFields { "controller.heartbeat_interval": number; "controller.session_timeout": number; "controller.metrics_timeout": number; + "controller.system_metrics_interval": number; "controller.proxy_stream_timeout": number; "controller.default_mod_pack_id": number | null; "controller.default_role_id": number | null; @@ -126,6 +127,12 @@ export class ControllerConfig extends classes.Config { type: "number", initialValue: 8, }, + "controller.system_metrics_interval": { + title: "System Metrics Interval", + description: "Interval in seconds to collect and update system metrics for the Web UI", + type: "number", + initialValue: 10, + }, "controller.proxy_stream_timeout": { title: "Proxy Stream Timeout", description: "Timeout in seconds for proxy streams to start flowing.", diff --git a/packages/lib/src/data/messages_controller.ts b/packages/lib/src/data/messages_controller.ts index d4a7d6f5..6f547164 100644 --- a/packages/lib/src/data/messages_controller.ts +++ b/packages/lib/src/data/messages_controller.ts @@ -204,6 +204,108 @@ export class LogMessageEvent { } } +export class SystemMetrics { + constructor( + /** + * Id of the host these metrics originate from, or the string + * "controller" if these metrics are for the controller. + */ + public id: number | "controller", + public coreRatios: number[], + public memoryCapacity: number, + public memoryAvailable: number, + public diskCapacity: number, + public diskAvailable: number, + /** Millisecond Unix timestamp this entry was last updated at */ + public updatedAt: number, + public isDeleted: boolean, + ) { } + + static jsonSchema = Type.Object({ + "id": Type.Union([Type.Number(), Type.Literal("controller")]), + "coreRatios": Type.Array(Type.Number()), + "memoryCapacity": Type.Number(), + "memoryAvailable": Type.Number(), + "diskCapacity": Type.Number(), + "diskAvailable": Type.Number(), + "updatedAt": Type.Number(), + "isDeleted": Type.Boolean(), + }); + + static fromJSON(json: Static) { + return new this( + json.id, + json.coreRatios, + json.memoryCapacity, + json.memoryAvailable, + json.diskCapacity, + json.diskAvailable, + json.updatedAt, + json.isDeleted, + ); + } + + get cpuCapacity() { + return this.coreRatios.length; + } + + get cpuUsed() { + return this.coreRatios.reduce((a, b) => a + b, 0); + } + + get cpuAvailable() { + return this.cpuCapacity - this.cpuUsed; + } + + get cpuRatio() { + return this.cpuUsed / this.cpuCapacity; + } + + get memoryUsed() { + return this.memoryCapacity - this.memoryAvailable; + } + + get memoryRatio() { + return (this.memoryCapacity - this.memoryAvailable) / this.memoryCapacity; + } + + get diskUsed() { + return this.diskCapacity - this.diskAvailable; + } + + get diskRatio() { + return (this.diskCapacity - this.diskAvailable) / this.diskCapacity; + } +} + +export class SystemMetricsRequest { + declare ["constructor"]: typeof SystemMetricsRequest; + static type = "request" as const; + static src = "controller" as const; + static dst = "host" as const; + static Response = SystemMetrics; +} + +export class SystemMetricsUpdateEvent { + declare ["constructor"]: typeof SystemMetricsUpdateEvent; + static type = "event" as const; + static src = "controller" as const; + static dst = "control" as const; + static permission = "core.system_metrics.subscribe" as const; + + constructor( + public updates: SystemMetrics[], + ) { } + + static jsonSchema = Type.Object({ + "updates": Type.Array(SystemMetrics.jsonSchema), + }); + + static fromJSON(json: Static) { + return new this(json.updates.map(update => SystemMetrics.fromJSON(update))); + } +} + export class DebugDumpWsRequest { declare ["constructor"]: typeof DebugDumpWsRequest; static type = "request" as const; diff --git a/packages/lib/src/link/messages.ts b/packages/lib/src/link/messages.ts index 13743844..79d5ba03 100644 --- a/packages/lib/src/link/messages.ts +++ b/packages/lib/src/link/messages.ts @@ -25,6 +25,8 @@ export const dataClasses: (RequestClass | EventClass) controller.LogSetSubscriptionsRequest, controller.LogQueryRequest, controller.LogMessageEvent, + controller.SystemMetricsRequest, + controller.SystemMetricsUpdateEvent, controller.DebugDumpWsRequest, controller.DebugWsMessageEvent, diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index 55a2ca5b..3f9d5f44 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -129,6 +129,13 @@ definePermission({ title: "Modify controller config", description: "Modify the controller config or entries of the controller config.", }); +definePermission({ + name: "core.system_metrics.subscribe", + title: "Subscribe to system metrics updates", + description: + "Subscribe to be notified when the system metrics detailing cpu, memory and disk usage for the " + + "controller and hosts are updated.", +}); definePermission({ name: "core.host.list", title: "List hosts", diff --git a/packages/lib/src/system_collectors.ts b/packages/lib/src/system_collectors.ts index 0dbdbe72..f957f16b 100644 --- a/packages/lib/src/system_collectors.ts +++ b/packages/lib/src/system_collectors.ts @@ -2,6 +2,7 @@ import fs from "fs-extra"; import os from "os"; import util from "util"; import { Gauge } from "./prometheus"; +import { SystemMetrics } from "./data"; function cpuModel() { const model = os.cpus()[0].model; @@ -119,3 +120,45 @@ export const systemDiskAvailableBytes = new Gauge( }, }, ); + +function minZip(a: T[], b: T[]): [T, T][] { + const length = Math.min(a.length, b.length); + const result = []; + result.length = length; + for (let i = 0; i < length; i++) { + result[i] = [a[i], b[i]] as [T, T]; + } + return result; +} + +let previousTotalCpuMs: number[] = []; +let previousIdleCpuMs: number[] = []; +export async function gatherSystemMetrics(id: SystemMetrics["id"]) { + const cpus = os.cpus(); + const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys); + const currentIdleCpuMs = cpus.map(({ times }) => times.idle); + const deltaTotalCpuMs = minZip(previousTotalCpuMs, currentTotalCpuMs).map(([prev, curr]) => curr - prev); + const deltaIdleCpuMs = minZip(previousIdleCpuMs, currentIdleCpuMs).map(([prev, curr]) => curr - prev); + const cpuUsage = minZip(deltaTotalCpuMs, deltaIdleCpuMs).map(([total, idle]) => (total - idle) / total); + previousTotalCpuMs = currentTotalCpuMs; + previousIdleCpuMs = currentIdleCpuMs; + let diskCapacity = 0; + let diskAvailable = 0; + if (fs.statfs) { // TODO: remove this check once minimum supported Node.js >= v18.15.0. + const statFsAsync = util.promisify(fs.statfs); + const stats = await statFsAsync("."); + diskCapacity = stats.blocks * stats.bsize; + diskAvailable = stats.bavail * stats.bsize; + } + + return new SystemMetrics( + id, + cpuUsage, + os.totalmem(), + os.freemem(), + diskCapacity, + diskAvailable, + Date.now(), + false, + ); +} diff --git a/packages/web_ui/src/components/ControllerPage.tsx b/packages/web_ui/src/components/ControllerPage.tsx index 75f67097..e4c61ae9 100644 --- a/packages/web_ui/src/components/ControllerPage.tsx +++ b/packages/web_ui/src/components/ControllerPage.tsx @@ -1,9 +1,14 @@ import React from "react"; -import { Typography } from "antd"; +import { Descriptions, Typography } from "antd"; import PluginExtra from "./PluginExtra"; import LogConsole from "./LogConsole"; +import { + MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, + MetricDiskUsed, MetricDiskRatio, +} from "./system_metrics"; import { useAccount } from "../model/account"; +import { useSystemMetrics } from "../model/system_metrics"; import ControllerConfigTree from "./ControllerConfigTree"; import PageLayout from "./PageLayout"; @@ -12,9 +17,19 @@ const { Title } = Typography; export default function ControllerPage() { let account = useAccount(); + const [systemMetrics] = useSystemMetrics(); + const metrics = systemMetrics.get("controller"); return

Controller

+ + + + + + + + {account.hasPermission("core.log.follow") && <> Console diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index 6c7c341e..f9c9f74e 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -13,7 +13,12 @@ import { useHost } from "../model/host"; import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; +import { + MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, + MetricDiskUsed, MetricDiskRatio, +} from "./system_metrics"; import { formatTimestamp } from "../util/time_format"; +import { useSystemMetrics } from "../model/system_metrics"; const { Title } = Typography; @@ -23,6 +28,8 @@ export default function HostViewPage() { let control = useContext(ControlContext); let account = useAccount(); let [instances] = useInstances(); + const [systemMetrics] = useSystemMetrics(); + const metrics = systemMetrics.get(hostId); const [host, synced] = useHost(hostId); const hostInstances = new Map([...instances].filter(([_id, instance]) => instance.assignedHost === hostId)); @@ -63,20 +70,27 @@ export default function HostViewPage() { title={host.name || String(hostId)} extra={hostButtons} /> - + {host["name"]} {host["connected"] ? "Connected" : "Disconnected"} + + + + + + {host["agent"]} {host["version"]} { host.tokenValidAfter - && - {formatTimestamp(host.tokenValidAfter*1000)} - + ? + {formatTimestamp(host.tokenValidAfter*1000)} + + : null } {account.hasPermission("core.instance.list") && <> diff --git a/packages/web_ui/src/components/HostsPage.tsx b/packages/web_ui/src/components/HostsPage.tsx index d80a4119..6a6586fa 100644 --- a/packages/web_ui/src/components/HostsPage.tsx +++ b/packages/web_ui/src/components/HostsPage.tsx @@ -1,6 +1,6 @@ import React, { useContext, useRef, useState, useEffect } from "react"; import { useNavigate } from "react-router-dom"; -import { Button, Form, InputNumber, Modal, Table, Tag, Typography } from "antd"; +import { Button, Flex, Form, InputNumber, Modal, Progress, Switch, Table, Tag, Typography } from "antd"; import CopyOutlined from "@ant-design/icons/lib/icons/CopyOutlined"; import * as lib from "@clusterio/lib"; @@ -10,7 +10,12 @@ import ControlContext from "./ControlContext"; import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; +import { + MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed, + MetricDiskUsed, MetricDiskRatio, +} from "./system_metrics"; import { useHosts } from "../model/host"; +import { useSystemMetrics } from "../model/system_metrics"; import notify, { notifyErrorHandler } from "../util/notify"; const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare; @@ -180,11 +185,13 @@ function CopyButton({ text, message }: { text:string, message:string }) { ; } - export default function HostsPage() { let account = useAccount(); let navigate = useNavigate(); let [hosts] = useHosts(); + const [metrics] = useSystemMetrics(); + const [showRatios, setShowRatios] = useState(true); + const [showNumbers, setShowNumbers] = useState(false); return : undefined} /> strcmp(a.name, b.name), }, + { + title: "CPU%", + sorter: (a, b) => (metrics.get(a.id)?.cpuRatio ?? 0) - (metrics.get(b.id)?.cpuRatio ?? 0), + render: (_, host) => , + hidden: !showRatios, + }, + { + title: "Cores", + sorter: (a, b) => ( + (metrics.get(a.id)?.cpuUsed ?? 0) - (metrics.get(b.id)?.cpuUsed ?? 0) + ), + render: (_, host) => , + hidden: !showNumbers, + }, + { + title: "Mem%", + sorter: (a, b) => ( + (metrics.get(a.id)?.memoryRatio ?? 0) - (metrics.get(b.id)?.memoryRatio ?? 0) + ), + render: (_, host) => , + hidden: !showRatios, + }, + { + title: "Memory", + sorter: (a, b) => ( + (metrics.get(a.id)?.memoryUsed ?? 0) - (metrics.get(b.id)?.memoryUsed ?? 0) + ), + render: (_, host) => , + hidden: !showNumbers, + }, + { + title: "Disk%", + sorter: (a, b) => (metrics.get(a.id)?.diskAvailable ?? 0) - (metrics.get(b.id)?.diskAvailable ?? 0), + render: (_, host) => , + hidden: !showRatios, + }, + { + title: "Disk", + sorter: (a, b) => ( + (metrics.get(a.id)?.diskUsed ?? 0) - (metrics.get(b.id)?.diskUsed ?? 0) + ), + render: (_, host) => , + hidden: !showNumbers, + }, { title: "Agent", dataIndex: "agent", @@ -235,6 +287,16 @@ export default function HostsPage() { }, })} /> + + + + ; }; diff --git a/packages/web_ui/src/components/system_metrics.tsx b/packages/web_ui/src/components/system_metrics.tsx new file mode 100644 index 00000000..aed2dd83 --- /dev/null +++ b/packages/web_ui/src/components/system_metrics.tsx @@ -0,0 +1,55 @@ +import React from "react"; + +import * as lib from "@clusterio/lib"; +import { Progress } from "antd"; + +export function MetricCpuRatio(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const ratio = props.metrics.cpuRatio; + return ; +} + +export function MetricCpuUsed(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const used = props.metrics.cpuUsed.toLocaleString(undefined, { maximumFractionDigits: 1 }); + const capacity = props.metrics.cpuCapacity; + return `${used} / ${capacity}`; +} + +export function MetricMemoryRatio(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const ratio = props.metrics.memoryRatio; + return ; +} + +export function MetricMemoryUsed(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const used = lib.formatBytes(props.metrics.memoryUsed); + const capacity = lib.formatBytes(props.metrics.memoryCapacity); + return `${used} / ${capacity}`; +} + +export function MetricDiskRatio(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const ratio = props.metrics.diskRatio; + return ; +} + +export function MetricDiskUsed(props: { metrics?: lib.SystemMetrics }) { + if (!props.metrics) { + return "N/A"; + } + const used = lib.formatBytes(props.metrics.diskUsed); + const capacity = lib.formatBytes(props.metrics.diskCapacity); + return `${used} / ${capacity}`; +} diff --git a/packages/web_ui/src/model/system_metrics.ts b/packages/web_ui/src/model/system_metrics.ts new file mode 100644 index 00000000..4f1f0e5a --- /dev/null +++ b/packages/web_ui/src/model/system_metrics.ts @@ -0,0 +1,8 @@ +import { useCallback, useContext, useSyncExternalStore } from "react"; +import ControlContext from "../components/ControlContext"; + +export function useSystemMetrics() { + const control = useContext(ControlContext); + const subscribe = useCallback((callback: () => void) => control.systemMetrics.subscribe(callback), [control]); + return useSyncExternalStore(subscribe, () => control.systemMetrics.getSnapshot()); +} diff --git a/packages/web_ui/src/util/websocket.ts b/packages/web_ui/src/util/websocket.ts index 06f796e9..6596725b 100644 --- a/packages/web_ui/src/util/websocket.ts +++ b/packages/web_ui/src/util/websocket.ts @@ -60,6 +60,7 @@ export class Control extends lib.Link { logHandlers: Map = new Map(); /** Updates handled by the subscription service */ + systemMetrics = new lib.EventSubscriber(lib.SystemMetricsUpdateEvent, this); hosts = new lib.EventSubscriber(lib.HostUpdatesEvent, this); instances = new lib.EventSubscriber(lib.InstanceDetailsUpdatesEvent, this); saves = new lib.EventSubscriber(lib.InstanceSaveDetailsUpdatesEvent, this); From ae2c5cf98ed5173906977f3b9af79ee15a0b4804 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 14 Jan 2024 03:50:08 +0100 Subject: [PATCH 6/7] Implement power of two size suffies for memory Memory sizes are usually written using the prefixes with a base of 1024 instead of 1000. Implment formatting of the power of two based prefixes for displaying memory sizes. --- packages/lib/src/helpers.ts | 19 +++++--- .../web_ui/src/components/system_metrics.tsx | 4 +- test/lib/helpers.js | 44 +++++++++++++++++++ 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/packages/lib/src/helpers.ts b/packages/lib/src/helpers.ts index 34d5ba28..24cb3477 100644 --- a/packages/lib/src/helpers.ts +++ b/packages/lib/src/helpers.ts @@ -217,17 +217,26 @@ export function escapeRegExp(text: string) { * * Shortens a large number of bytes using the kB/MB/GB/TB prefixes. * @param bytes - Count of bytes to format. + * @param prefixes - Whethere to use SI powers (1000) or the binary powers (1024). * @returns formatted text. */ -export function formatBytes(bytes: number) { +export function formatBytes(bytes: number, prefixes: "si" | "binary" = "si") { if (bytes === 0) { return "0\u{A0}Bytes"; // No-break space } - let units = ["\u{A0}Bytes", "\u{A0}kB", "\u{A0}MB", "\u{A0}GB", "\u{A0}TB"]; - let factor = 1000; - let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length); - return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; + let base, units; + if (prefixes === "si") { + base = 1000; + units = ["\u{A0}Bytes", "\u{A0}kB", "\u{A0}MB", "\u{A0}GB", "\u{A0}TB"]; + } else { + base = 1024; + units = ["\u{A0}Bytes", "\u{A0}kiB", "\u{A0}MiB", "\u{A0}GiB", "\u{A0}TiB"]; + } + const exponent = Math.min(Math.floor(Math.log(bytes) / Math.log(base)), units.length); + const significant = bytes / base ** exponent; + const fractionDigits = Number(significant < 99.95) + Number(significant < 9.995); + return significant.toFixed(fractionDigits) + units[exponent]; } function skipWhitespace(pos: number, input: string) { diff --git a/packages/web_ui/src/components/system_metrics.tsx b/packages/web_ui/src/components/system_metrics.tsx index aed2dd83..15ca5e5b 100644 --- a/packages/web_ui/src/components/system_metrics.tsx +++ b/packages/web_ui/src/components/system_metrics.tsx @@ -32,8 +32,8 @@ export function MetricMemoryUsed(props: { metrics?: lib.SystemMetrics }) { if (!props.metrics) { return "N/A"; } - const used = lib.formatBytes(props.metrics.memoryUsed); - const capacity = lib.formatBytes(props.metrics.memoryCapacity); + const used = lib.formatBytes(props.metrics.memoryUsed, "binary"); + const capacity = lib.formatBytes(props.metrics.memoryCapacity, "binary"); return `${used} / ${capacity}`; } diff --git a/test/lib/helpers.js b/test/lib/helpers.js index 7b898215..b9398dcb 100644 --- a/test/lib/helpers.js +++ b/test/lib/helpers.js @@ -192,6 +192,50 @@ describe("lib/helpers", function() { }); }); + describe("formatBytes()", function() { + it("should format whole number of base unit bytes", function() { + assert.equal(lib.formatBytes(100), "100\u{A0}Bytes"); + assert.equal(lib.formatBytes(100e3), "100\u{A0}kB"); + assert.equal(lib.formatBytes(100e6), "100\u{A0}MB"); + assert.equal(lib.formatBytes(100e9), "100\u{A0}GB"); + assert.equal(lib.formatBytes(100e12), "100\u{A0}TB"); + assert.equal(lib.formatBytes(100, false), "100\u{A0}Bytes"); + assert.equal(lib.formatBytes(100 * 2**10, false), "100\u{A0}kiB"); + assert.equal(lib.formatBytes(100 * 2**20, false), "100\u{A0}MiB"); + assert.equal(lib.formatBytes(100 * 2**30, false), "100\u{A0}GiB"); + assert.equal(lib.formatBytes(100 * 2**40, false), "100\u{A0}TiB"); + }); + it("should format with 3 significant digits fractional number of base unit bytes", function() { + // 1 digits base and 2 digit fraction + assert.equal(lib.formatBytes(1.234e3), "1.23\u{A0}kB"); + assert.equal(lib.formatBytes(1.234e6), "1.23\u{A0}MB"); + assert.equal(lib.formatBytes(1.234e9), "1.23\u{A0}GB"); + assert.equal(lib.formatBytes(1.234e12), "1.23\u{A0}TB"); + assert.equal(lib.formatBytes(1.234 * 2**10, "binary"), "1.23\u{A0}kiB"); + assert.equal(lib.formatBytes(1.234 * 2**20, "binary"), "1.23\u{A0}MiB"); + assert.equal(lib.formatBytes(1.234 * 2**30, "binary"), "1.23\u{A0}GiB"); + assert.equal(lib.formatBytes(1.234 * 2**40, "binary"), "1.23\u{A0}TiB"); + // 2 digits base and 1 digit fraction + assert.equal(lib.formatBytes(12.34e3), "12.3\u{A0}kB"); + assert.equal(lib.formatBytes(12.34e6), "12.3\u{A0}MB"); + assert.equal(lib.formatBytes(12.34e9), "12.3\u{A0}GB"); + assert.equal(lib.formatBytes(12.34e12), "12.3\u{A0}TB"); + assert.equal(lib.formatBytes(12.34 * 2**10, "binary"), "12.3\u{A0}kiB"); + assert.equal(lib.formatBytes(12.34 * 2**20, "binary"), "12.3\u{A0}MiB"); + assert.equal(lib.formatBytes(12.34 * 2**30, "binary"), "12.3\u{A0}GiB"); + assert.equal(lib.formatBytes(12.34 * 2**40, "binary"), "12.3\u{A0}TiB"); + // 3 digits base and no fraction + assert.equal(lib.formatBytes(123.4e3), "123\u{A0}kB"); + assert.equal(lib.formatBytes(123.4e6), "123\u{A0}MB"); + assert.equal(lib.formatBytes(123.4e9), "123\u{A0}GB"); + assert.equal(lib.formatBytes(123.4e12), "123\u{A0}TB"); + assert.equal(lib.formatBytes(123.4 * 2**10, "binary"), "123\u{A0}kiB"); + assert.equal(lib.formatBytes(123.4 * 2**20, "binary"), "123\u{A0}MiB"); + assert.equal(lib.formatBytes(123.4 * 2**30, "binary"), "123\u{A0}GiB"); + assert.equal(lib.formatBytes(123.4 * 2**40, "binary"), "123\u{A0}TiB"); + }); + }); + function parse(input, attributes) { const result = lib.parseSearchString(input, attributes); if (!result.issues.length) { From d2861c4457d34ac19f8875e9276e7a881ccd3e58 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 15 Jan 2024 03:26:32 +0100 Subject: [PATCH 7/7] Add tests for derived SystemMetrics --- test/lib/data/SystemMetrics.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 test/lib/data/SystemMetrics.js diff --git a/test/lib/data/SystemMetrics.js b/test/lib/data/SystemMetrics.js new file mode 100644 index 00000000..163abcd2 --- /dev/null +++ b/test/lib/data/SystemMetrics.js @@ -0,0 +1,22 @@ +"use strict"; +const assert = require("assert").strict; + +const lib = require("@clusterio/lib"); +const { SystemMetrics } = lib; + + +describe("lib/data/ModPack", function() { + describe("class SystemMetrics", function() { + it("correctly calculates derived metrics", function() { + const metrics = new SystemMetrics("controller", [0.5, 0.25, 0, 1], 1024, 768, 2048, 512, 0, false); + assert.equal(metrics.cpuCapacity, 4); + assert.equal(metrics.cpuUsed, 1.75); + assert.equal(metrics.cpuAvailable, 2.25); + assert.equal(metrics.cpuRatio, 0.4375); + assert.equal(metrics.memoryUsed, 256); + assert.equal(metrics.memoryRatio, 0.25); + assert.equal(metrics.diskUsed, 1536); + assert.equal(metrics.diskRatio, 0.75); + }); + }); +});