Merge pull request #541 from Hornwitser/system-metrics

Real-time metrics

packages/controller/src/Controller.ts

@@ -53,6 +53,8 @@ export default class Controller {
 	config: lib.ControllerConfig;
 	app: Application;
 
+	/** True if (@link Controller.systemMetrics} has changed since last time it was saved */
+	systemMetricsDirty = false;
 	/** True if {@link Controller.hosts} has changed since last time it was saved */
 	hostsDirty = false;
 	/** True if {@link Controller.instances} has changed since last time it was saved */
@@ -89,6 +91,7 @@ export default class Controller {
 	devMiddleware: any | null = null;
 
 	autosaveInterval?: ReturnType<typeof setInterval>;
+	systemMetricsInterval?: ReturnType<typeof setInterval>;
 
 	logDirectory: string = "";
 	clusterLogIndex: lib.LogIndex | null = null;
@@ -98,6 +101,7 @@ export default class Controller {
 		let databaseDirectory = config.get("controller.database_directory");
 		await fs.ensureDir(databaseDirectory);
 
+		const systemMetrics = await Controller.loadSystemMetrics(path.join(databaseDirectory, "metrics.json"));
 		const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json"));
 		const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json"));
 		const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json"));
@@ -110,6 +114,7 @@ export default class Controller {
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
 		return [
+			systemMetrics,
 			hosts,
 			instances,
 			saves,
@@ -125,6 +130,7 @@ export default class Controller {
 		configPath: string,
 		config: lib.ControllerConfig,
 
+		public systemMetrics = new Map<lib.SystemMetrics["id"], lib.SystemMetrics>(),
 		/** Mapping of host id to host info */
 		public hosts = new Map<number, HostInfo>(),
 		/** Mapping of instance id to instance info */
@@ -154,6 +160,7 @@ export default class Controller {
 		});
 
 		// Handle subscriptions for all internal properties
+		this.subscriptions.handle(lib.SystemMetricsUpdateEvent, this.handleSystemMetricsSubscription.bind(this));
 		this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this));
 		this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this));
 		this.subscriptions.handle(
@@ -219,6 +226,8 @@ export default class Controller {
 		this.config.on("fieldChanged", (field, curr, prev) => {
 			if (field === "controller.autosave_interval") {
 				this.onAutosaveIntervalChanged();
+			} else if (field === "controller.system_metrics_interval") {
+				this.onSystemMetricsIntervalChanged();
 			}
 			lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev);
 		});
@@ -292,6 +301,7 @@ export default class Controller {
 		}
 
 		this.onAutosaveIntervalChanged();
+		this.onSystemMetricsIntervalChanged();
 
 		logger.info("Started controller");
 		this._state = "running";
@@ -357,6 +367,11 @@ export default class Controller {
 			clearInterval(this.clusterLogBuildInterval);
 		}
 
+		if (this.systemMetricsInterval) {
+			clearInterval(this.systemMetricsInterval);
+			this.systemMetricsInterval = undefined;
+		}
+
 		if (this.autosaveInterval) {
 			clearInterval(this.autosaveInterval);
 			this.autosaveInterval = undefined;
@@ -389,6 +404,41 @@ export default class Controller {
 		logger.info("Goodbye");
 	}
 
+	onSystemMetricsIntervalChanged() {
+		if (this.systemMetricsInterval) {
+			clearInterval(this.systemMetricsInterval);
+			this.systemMetricsInterval = undefined;
+		}
+		const systemMetricsIntervalSeconds = this.config.get("controller.system_metrics_interval");
+		if (systemMetricsIntervalSeconds > 0) {
+			this.systemMetricsInterval = setInterval(
+				this.updateSystemMetrics.bind(this),
+				systemMetricsIntervalSeconds * 1000
+			);
+		}
+	}
+
+	async updateSystemMetrics() {
+		try {
+			const requests: Promise<lib.SystemMetrics>[] = [];
+			for (let hostConnection of this.wsServer.hostConnections.values()) {
+				if (!hostConnection.connected) {
+					continue;
+				}
+				requests.push(hostConnection.send(new lib.SystemMetricsRequest()));
+			}
+			requests.push(lib.gatherSystemMetrics("controller"));
+			const newMetrics = await Promise.all(requests);
+			for (const metric of newMetrics) {
+				this.systemMetrics.set(metric.id, metric);
+			}
+			this.systemMetricsDirty = true;
+			this.subscriptions.broadcast(new lib.SystemMetricsUpdateEvent(newMetrics));
+		} catch (err: any) {
+			logger.error(`Unexpected error updating system metrics:\n${err.stack ?? err.message}`);
+		}
+	}
+
 	onAutosaveIntervalChanged() {
 		if (this.autosaveInterval) {
 			clearInterval(this.autosaveInterval);
@@ -425,6 +475,11 @@ export default class Controller {
 		}
 
 		let databaseDirectory = this.config.get("controller.database_directory");
+		if (this.systemMetricsDirty) {
+			this.systemMetricsDirty = false;
+			await Controller.saveSystemMetrics(path.join(databaseDirectory, "metrics.json"), this.systemMetrics);
+		}
+
 		if (this.hostsDirty) {
 			this.hostsDirty = false;
 			await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
@@ -452,6 +507,25 @@ export default class Controller {
 		await lib.invokeHook(this.plugins, "onSaveData");
 	}
 
+	static async loadSystemMetrics(filePath: string): Promise<Map<lib.SystemMetrics["id"], lib.SystemMetrics>> {
+		let json: Static<typeof lib.SystemMetrics.jsonSchema>[];
+		try {
+			json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
+
+		} catch (err: any) {
+			if (err.code !== "ENOENT") {
+				throw err;
+			}
+			return new Map();
+		}
+
+		return new Map(json.map(m => lib.SystemMetrics.fromJSON(m)).map(m => [m.id, m]));
+	}
+
+	static async saveSystemMetrics(filePath: string, systemMetrics: Map<lib.SystemMetrics["id"], lib.SystemMetrics>) {
+		await lib.safeOutputFile(filePath, JSON.stringify([...systemMetrics.values()], null, "\t"));
+	}
+
 	static async loadHosts(filePath: string): Promise<Map<number, HostInfo>> {
 		let serialized: [number, Static<typeof HostInfo.jsonSchema>][];
 		try {
@@ -635,6 +709,13 @@ export default class Controller {
 		}
 	}
 
+	async handleSystemMetricsSubscription(request: lib.SubscriptionRequest) {
+		const systemMetrics = [...this.systemMetrics.values()].filter(
+			metric => metric.updatedAt > request.lastRequestTime,
+		);
+		return systemMetrics.length ? new lib.SystemMetricsUpdateEvent(systemMetrics) : null;
+	}
+
 	/**
 	 * Generate access token for host
 	 *

packages/controller/src/WsServer.ts

@@ -107,7 +107,7 @@ export default class WsServer {
 		socket.on("message", (message: WebSocket.RawData) => {
 			wsMessageCounter.labels("in").inc();
 			if (!socket.clusterio_ignore_dump) {
-				this.controller.debugEvents.emit("message", { direction: "in", content: message });
+				this.controller.debugEvents.emit("message", { direction: "in", content: message.toString("utf8") });
 			}
 		});
 		let originalSend = socket.send;

packages/controller/src/routes.ts

@@ -39,7 +39,7 @@ function mergeSamples(destinationResult: lib.CollectorResultSerialized, sourceRe
 	}
 
 	for (let entry of receivedSamples) {
-		sourceResult.samples.push(entry);
+		destinationResult.samples.push(entry);
 	}
 }
 

packages/host/src/Host.ts

@@ -347,6 +347,7 @@ export default class Host extends lib.Link {
 		this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.handleWhitelistUpdateEvent.bind(this));
 		this.handle(lib.InstanceAssignInternalRequest, this.handleInstanceAssignInternalRequest.bind(this));
 		this.handle(lib.InstanceUnassignInternalRequest, this.handleInstanceUnassignInternalRequest.bind(this));
+		this.handle(lib.SystemMetricsRequest, this.handleSystemMetricsRequest.bind(this));
 		this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this));
 		this.fallbackRequest(
 			lib.InstanceSaveDetailsListRequest, this.fallbackInstanceSaveDetailsListRequest.bind(this),
@@ -723,6 +724,10 @@ export default class Host extends lib.Link {
 		return instanceConnection;
 	}
 
+	async handleSystemMetricsRequest() {
+		return lib.gatherSystemMetrics(this.config.get("host.id"));
+	}
+
 	async handleHostMetricsRequest() {
 		let requests: Promise<InstanceType<typeof lib.InstanceMetricsRequest["Response"]>>[] = [];
 		for (let instanceConnection of this.instanceConnections.values()) {
@@ -748,6 +753,12 @@ export default class Host extends lib.Link {
 					metricName: result.metric.name.replace("process_", "clusterio_host_"),
 				}));
 
+			} else if (result.metric.name.startsWith("system_")) {
+				results.push(lib.serializeResult(result, {
+					addLabels: { "host_id": String(this.config.get("host.id")) },
+					metricName: result.metric.name.replace("system_", "clusterio_host_system_"),
+				}));
+
 			} else {
 				results.push(lib.serializeResult(result));
 			}

packages/lib/index.ts

@@ -25,6 +25,7 @@ export * from "./src/prometheus";
 export * from "./src/schema";
 export * from "./src/shared_commands";
 export * from "./src/stream";
+export * from "./src/system_collectors";
 export * from "./src/zip_ops";
 export * from "./src/subscriptions";
 

packages/lib/src/config/definitions.ts

@@ -17,6 +17,7 @@ export interface ControllerConfigFields {
 	"controller.heartbeat_interval": number;
 	"controller.session_timeout": number;
 	"controller.metrics_timeout": number;
+	"controller.system_metrics_interval": number;
 	"controller.proxy_stream_timeout": number;
 	"controller.default_mod_pack_id": number | null;
 	"controller.default_role_id": number | null;
@@ -126,6 +127,12 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			type: "number",
 			initialValue: 8,
 		},
+		"controller.system_metrics_interval": {
+			title: "System Metrics Interval",
+			description: "Interval in seconds to collect and update system metrics for the Web UI",
+			type: "number",
+			initialValue: 10,
+		},
 		"controller.proxy_stream_timeout": {
 			title: "Proxy Stream Timeout",
 			description: "Timeout in seconds for proxy streams to start flowing.",

packages/lib/src/data/messages_controller.ts

@@ -204,6 +204,108 @@ export class LogMessageEvent {
 	}
 }
 
+export class SystemMetrics {
+	constructor(
+		/**
+		 * Id of the host these metrics originate from, or the string
+		 * "controller" if these metrics are for the controller.
+		 */
+		public id: number | "controller",
+		public coreRatios: number[],
+		public memoryCapacity: number,
+		public memoryAvailable: number,
+		public diskCapacity: number,
+		public diskAvailable: number,
+		/** Millisecond Unix timestamp this entry was last updated at */
+		public updatedAt: number,
+		public isDeleted: boolean,
+	) { }
+
+	static jsonSchema = Type.Object({
+		"id": Type.Union([Type.Number(), Type.Literal("controller")]),
+		"coreRatios": Type.Array(Type.Number()),
+		"memoryCapacity": Type.Number(),
+		"memoryAvailable": Type.Number(),
+		"diskCapacity": Type.Number(),
+		"diskAvailable": Type.Number(),
+		"updatedAt": Type.Number(),
+		"isDeleted": Type.Boolean(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(
+			json.id,
+			json.coreRatios,
+			json.memoryCapacity,
+			json.memoryAvailable,
+			json.diskCapacity,
+			json.diskAvailable,
+			json.updatedAt,
+			json.isDeleted,
+		);
+	}
+
+	get cpuCapacity() {
+		return this.coreRatios.length;
+	}
+
+	get cpuUsed() {
+		return this.coreRatios.reduce((a, b) => a + b, 0);
+	}
+
+	get cpuAvailable() {
+		return this.cpuCapacity - this.cpuUsed;
+	}
+
+	get cpuRatio() {
+		return this.cpuUsed / this.cpuCapacity;
+	}
+
+	get memoryUsed() {
+		return this.memoryCapacity - this.memoryAvailable;
+	}
+
+	get memoryRatio() {
+		return (this.memoryCapacity - this.memoryAvailable) / this.memoryCapacity;
+	}
+
+	get diskUsed() {
+		return this.diskCapacity - this.diskAvailable;
+	}
+
+	get diskRatio() {
+		return (this.diskCapacity - this.diskAvailable) / this.diskCapacity;
+	}
+}
+
+export class SystemMetricsRequest {
+	declare ["constructor"]: typeof SystemMetricsRequest;
+	static type = "request" as const;
+	static src = "controller" as const;
+	static dst = "host" as const;
+	static Response = SystemMetrics;
+}
+
+export class SystemMetricsUpdateEvent {
+	declare ["constructor"]: typeof SystemMetricsUpdateEvent;
+	static type = "event" as const;
+	static src = "controller" as const;
+	static dst = "control" as const;
+	static permission = "core.system_metrics.subscribe" as const;
+
+	constructor(
+		public updates: SystemMetrics[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		"updates": Type.Array(SystemMetrics.jsonSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.updates.map(update => SystemMetrics.fromJSON(update)));
+	}
+}
+
 export class DebugDumpWsRequest {
 	declare ["constructor"]: typeof DebugDumpWsRequest;
 	static type = "request" as const;

packages/lib/src/helpers.ts

@@ -217,17 +217,26 @@ export function escapeRegExp(text: string) {
  *
  * Shortens a large number of bytes using the kB/MB/GB/TB prefixes.
  * @param bytes - Count of bytes to format.
+ * @param prefixes - Whethere to use SI powers (1000) or the binary powers (1024).
  * @returns formatted text.
  */
-export function formatBytes(bytes: number) {
+export function formatBytes(bytes: number, prefixes: "si" | "binary" = "si") {
 	if (bytes === 0) {
 		return "0\u{A0}Bytes"; // No-break space
 	}
 
-	let units = ["\u{A0}Bytes", "\u{A0}kB", "\u{A0}MB", "\u{A0}GB", "\u{A0}TB"];
-	let factor = 1000;
-	let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length);
-	return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power];
+	let base, units;
+	if (prefixes === "si") {
+		base = 1000;
+		units = ["\u{A0}Bytes", "\u{A0}kB", "\u{A0}MB", "\u{A0}GB", "\u{A0}TB"];
+	} else {
+		base = 1024;
+		units = ["\u{A0}Bytes", "\u{A0}kiB", "\u{A0}MiB", "\u{A0}GiB", "\u{A0}TiB"];
+	}
+	const exponent = Math.min(Math.floor(Math.log(bytes) / Math.log(base)), units.length);
+	const significant = bytes / base ** exponent;
+	const fractionDigits = Number(significant < 99.95) + Number(significant < 9.995);
+	return significant.toFixed(fractionDigits) + units[exponent];
 }
 
 function skipWhitespace(pos: number, input: string) {

packages/lib/src/link/messages.ts

@@ -25,6 +25,8 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	controller.LogSetSubscriptionsRequest,
 	controller.LogQueryRequest,
 	controller.LogMessageEvent,
+	controller.SystemMetricsRequest,
+	controller.SystemMetricsUpdateEvent,
 	controller.DebugDumpWsRequest,
 	controller.DebugWsMessageEvent,
 

packages/lib/src/permissions.ts

@@ -129,6 +129,13 @@ definePermission({
 	title: "Modify controller config",
 	description: "Modify the controller config or entries of the controller config.",
 });
+definePermission({
+	name: "core.system_metrics.subscribe",
+	title: "Subscribe to system metrics updates",
+	description:
+		"Subscribe to be notified when the system metrics detailing cpu, memory and disk usage for the " +
+		"controller and hosts are updated.",
+});
 definePermission({
 	name: "core.host.list",
 	title: "List hosts",

packages/lib/src/system_collectors.ts

@@ -0,0 +1,164 @@
+import fs from "fs-extra";
+import os from "os";
+import util from "util";
+import { Gauge } from "./prometheus";
+import { SystemMetrics } from "./data";
+
+function cpuModel() {
+	const model = os.cpus()[0].model;
+	// Remove irelevant clutter
+	return model
+		.replace(/\(R\)/gi, "®") // Intel CPUs use (R) instead of ®
+		.replace(/\(TM\)/gi, "™") // Intel and AMD CPUs use (TM) instead of ™
+		.replace(/ CPU /, " ") // Intel CPUs add redundant "CPU" in the model.
+		.replace(/ @ [0-9]+\.[0-9]+GHz/, "") // Intel CPUs add GHz after the model.
+		.replace(/ (Dual|Quad|Six|[0-9]+)-Core Processor/, "") // AMD CPUs add cores after the model.
+		.trim() // AMD CPUs may have trailing spaces in the model.
+	;
+}
+
+export const systemInfo = new Gauge(
+	"system_info",
+	"Static info about the system.",
+	{
+		labels: ["kernel", "machine", "cpu_model", "hostname", "node"],
+	}
+);
+systemInfo.labels({
+	"kernel": os.type(),
+	"machine": os.machine(),
+	"cpu_model": cpuModel(),
+	"hostname": os.hostname(),
+	"node": process.version,
+}).set(1);
+
+export const systemCpuSecondsTotal = new Gauge(
+	"system_cpu_seconds_total",
+	"Total system CPU time spent in seconds.",
+	{
+		labels: ["core"],
+		callback: function(collector) {
+			const cpus = os.cpus();
+			for (let i = 0; i < cpus.length; i++) {
+				const times = cpus[i].times;
+				collector.labels(String(i)).set(
+					(times.user + times.idle + times.irq + times.nice + times.sys) / 1000
+				);
+			}
+		},
+	},
+);
+
+export const systemCpuIdleSecondsTotal = new Gauge(
+	"system_cpu_idle_seconds_total",
+	"Total system CPU time spent being idle in seconds.",
+	{
+		labels: ["core"],
+		callback: function(collector) {
+			const cpus = os.cpus();
+			for (let i = 0; i < cpus.length; i++) {
+				const times = cpus[i].times;
+				collector.labels(String(i)).set(
+					times.idle / 1000
+				);
+			}
+		},
+	},
+);
+
+export const systemMemoryCapasityBytes = new Gauge(
+	"system_memory_capasity_bytes",
+	"Total system memory installed",
+	{
+		callback: function(collector) {
+			collector.set(os.totalmem());
+		},
+	},
+);
+
+export const systemMemoryAvailableBytes = new Gauge(
+	"system_memory_available_bytes",
+	"Total system memory available for use",
+	{
+		callback: function(collector) {
+			collector.set(os.freemem());
+		},
+	},
+);
+
+export const systemDiskCapasityBytes = new Gauge(
+	"system_disk_capasity_bytes",
+	"Size of the filesystem of the current working directory Node.js runs on",
+	{
+		callback: async function(collector) {
+			// statfs was added in Node.js v18.15.0 and may not be present.
+			// TODO: remove this check once minimum supported Node.js >= v18.15.0.
+			if (!fs.statfs) {
+				collector.set(0);
+				return;
+			}
+			const statFsAsync = util.promisify(fs.statfs);
+			const stats = await statFsAsync(".");
+			collector.set(stats.blocks * stats.bsize);
+		},
+	},
+);
+
+export const systemDiskAvailableBytes = new Gauge(
+	"system_disk_available_bytes",
+	"Available space on the filesystem of the current working directory Node.js runs on",
+	{
+		callback: async function(collector) {
+			// TODO: remove this check once minimum supported Node.js >= v18.15.0.
+			if (!fs.statfs) {
+				collector.set(0);
+				return;
+			}
+			const statFsAsync = util.promisify(fs.statfs);
+			const stats = await statFsAsync(".");
+			collector.set(stats.bavail * stats.bsize);
+		},
+	},
+);
+
+function minZip<T>(a: T[], b: T[]): [T, T][] {
+	const length = Math.min(a.length, b.length);
+	const result = [];
+	result.length = length;
+	for (let i = 0; i < length; i++) {
+		result[i] = [a[i], b[i]] as [T, T];
+	}
+	return result;
+}
+
+let previousTotalCpuMs: number[] = [];
+let previousIdleCpuMs: number[] = [];
+export async function gatherSystemMetrics(id: SystemMetrics["id"]) {
+	const cpus = os.cpus();
+	const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
+	const currentIdleCpuMs = cpus.map(({ times }) => times.idle);
+	const deltaTotalCpuMs = minZip(previousTotalCpuMs, currentTotalCpuMs).map(([prev, curr]) => curr - prev);
+	const deltaIdleCpuMs = minZip(previousIdleCpuMs, currentIdleCpuMs).map(([prev, curr]) => curr - prev);
+	const cpuUsage = minZip(deltaTotalCpuMs, deltaIdleCpuMs).map(([total, idle]) => (total - idle) / total);
+	previousTotalCpuMs = currentTotalCpuMs;
+	previousIdleCpuMs = currentIdleCpuMs;
+	let diskCapacity = 0;
+	let diskAvailable = 0;
+	if (fs.statfs) { // TODO: remove this check once minimum supported Node.js >= v18.15.0.
+		const statFsAsync = util.promisify(fs.statfs);
+		const stats = await statFsAsync(".");
+		diskCapacity = stats.blocks * stats.bsize;
+		diskAvailable = stats.bavail * stats.bsize;
+	}
+
+	return new SystemMetrics(
+		id,
+		cpuUsage,
+		os.totalmem(),
+		os.freemem(),
+		diskCapacity,
+		diskAvailable,
+		Date.now(),
+		false,
+	);
+}

packages/web_ui/package.json

@@ -20,7 +20,7 @@
 		"@babel/core": "^7.22.9",
 		"@babel/preset-react": "^7.22.5",
 		"@clusterio/lib": "^2.0.0-alpha.13",
-		"antd": "^5.7.3",
+		"antd": "^5.13.0",
 		"assert": "^2.0.0",
 		"babel-loader": "^9.1.3",
 		"browserify-zlib": "^0.2.0",

packages/web_ui/src/components/ControllerPage.tsx

@@ -1,9 +1,14 @@
 import React from "react";
-import { Typography } from "antd";
+import { Descriptions, Typography } from "antd";
 
 import PluginExtra from "./PluginExtra";
 import LogConsole from "./LogConsole";
+import {
+	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
+	MetricDiskUsed, MetricDiskRatio,
+} from "./system_metrics";
 import { useAccount } from "../model/account";
+import { useSystemMetrics } from "../model/system_metrics";
 import ControllerConfigTree from "./ControllerConfigTree";
 import PageLayout from "./PageLayout";
 
@@ -12,9 +17,19 @@ const { Title } = Typography;
 
 export default function ControllerPage() {
 	let account = useAccount();
+	const [systemMetrics] = useSystemMetrics();
+	const metrics = systemMetrics.get("controller");
 
 	return <PageLayout nav={[{ name: "Controller" }]}>
 		<h2>Controller</h2>
+		<Descriptions bordered size="small" column={{ xs: 1, md: 2, lg: 2, xl: 2, xxl: 2 }}>
+			<Descriptions.Item label="CPU Usage"><MetricCpuRatio metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Cores"><MetricCpuUsed metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Memory Usage"><MetricMemoryRatio metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Memory"><MetricMemoryUsed metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Disk Usage"><MetricDiskRatio metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Disk"><MetricDiskUsed metrics={metrics} /></Descriptions.Item>
+		</Descriptions>
 		{account.hasPermission("core.log.follow") && <>
 			<Title level={5} style={{ marginTop: 16 }}>Console</Title>
 			<LogConsole controller={true} />

packages/web_ui/src/components/HostViewPage.tsx

@@ -13,7 +13,12 @@ import { useHost } from "../model/host";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import {
+	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
+	MetricDiskUsed, MetricDiskRatio,
+} from "./system_metrics";
 import { formatTimestamp } from "../util/time_format";
+import { useSystemMetrics } from "../model/system_metrics";
 
 const { Title } = Typography;
 
@@ -23,6 +28,8 @@ export default function HostViewPage() {
 	let control = useContext(ControlContext);
 	let account = useAccount();
 	let [instances] = useInstances();
+	const [systemMetrics] = useSystemMetrics();
+	const metrics = systemMetrics.get(hostId);
 	const [host, synced] = useHost(hostId);
 	const hostInstances = new Map([...instances].filter(([_id, instance]) => instance.assignedHost === hostId));
 
@@ -63,20 +70,27 @@ export default function HostViewPage() {
 			title={host.name || String(hostId)}
 			extra={hostButtons}
 		/>
-		<Descriptions bordered size="small" column={{ xs: 1, sm: 2, xl: 4 }}>
+		<Descriptions bordered size="small" column={{ xs: 1, md: 2, lg: 2, xl: 2, xxl: 2 }}>
 			<Descriptions.Item label="Name">{host["name"]}</Descriptions.Item>
 			<Descriptions.Item label="Connected">
 				<Tag color={host["connected"] ? "#389e0d" : "#cf1322"}>
 					{host["connected"] ? "Connected" : "Disconnected"}
 				</Tag>
 			</Descriptions.Item>
+			<Descriptions.Item label="CPU Usage"><MetricCpuRatio metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Cores"><MetricCpuUsed metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Memory Usage"><MetricMemoryRatio metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Memory"><MetricMemoryUsed metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Disk Usage"><MetricDiskRatio metrics={metrics} /></Descriptions.Item>
+			<Descriptions.Item label="Disk"><MetricDiskUsed metrics={metrics} /></Descriptions.Item>
 			<Descriptions.Item label="Agent">{host["agent"]}</Descriptions.Item>
 			<Descriptions.Item label="Version">{host["version"]}</Descriptions.Item>
 			{
 				host.tokenValidAfter
-				&& <Descriptions.Item label="Tokens valid after:">
-					{formatTimestamp(host.tokenValidAfter*1000)}
-				</Descriptions.Item>
+					? <Descriptions.Item label="Tokens valid after:">
+						{formatTimestamp(host.tokenValidAfter*1000)}
+					</Descriptions.Item>
+					: null
 			}
 		</Descriptions>
 		{account.hasPermission("core.instance.list") && <>

packages/web_ui/src/components/HostsPage.tsx

@@ -1,6 +1,6 @@
 import React, { useContext, useRef, useState, useEffect } from "react";
 import { useNavigate } from "react-router-dom";
-import { Button, Form, InputNumber, Modal, Table, Tag, Typography } from "antd";
+import { Button, Flex, Form, InputNumber, Modal, Progress, Switch, Table, Tag, Typography } from "antd";
 import CopyOutlined from "@ant-design/icons/lib/icons/CopyOutlined";
 
 import * as lib from "@clusterio/lib";
@@ -10,7 +10,12 @@ import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import {
+	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
+	MetricDiskUsed, MetricDiskRatio,
+} from "./system_metrics";
 import { useHosts } from "../model/host";
+import { useSystemMetrics } from "../model/system_metrics";
 import notify, { notifyErrorHandler } from "../util/notify";
 
 const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare;
@@ -180,11 +185,13 @@ function CopyButton({ text, message }: { text:string, message:string }) {
 	</Button>;
 }
 
-
 export default function HostsPage() {
 	let account = useAccount();
 	let navigate = useNavigate();
 	let [hosts] = useHosts();
+	const [metrics] = useSystemMetrics();
+	const [showRatios, setShowRatios] = useState(true);
+	const [showNumbers, setShowNumbers] = useState(false);
 
 	return <PageLayout nav={[{ name: "Hosts" }]}>
 		<PageHeader
@@ -192,6 +199,7 @@ export default function HostsPage() {
 			extra={account.hasPermission("core.host.generate_token") ? <GenerateHostTokenButton /> : undefined}
 		/>
 		<Table
+			style={{ overflowX: "auto" }}
 			columns={[
 				{
 					title: "Name",
@@ -199,6 +207,50 @@ export default function HostsPage() {
 					defaultSortOrder: "ascend",
 					sorter: (a, b) => strcmp(a.name, b.name),
 				},
+				{
+					title: "CPU%",
+					sorter: (a, b) => (metrics.get(a.id)?.cpuRatio ?? 0) - (metrics.get(b.id)?.cpuRatio ?? 0),
+					render: (_, host) => <MetricCpuRatio metrics={metrics.get(host.id)} />,
+					hidden: !showRatios,
+				},
+				{
+					title: "Cores",
+					sorter: (a, b) => (
+						(metrics.get(a.id)?.cpuUsed ?? 0) - (metrics.get(b.id)?.cpuUsed ?? 0)
+					),
+					render: (_, host) => <MetricCpuUsed metrics={metrics.get(host.id)} />,
+					hidden: !showNumbers,
+				},
+				{
+					title: "Mem%",
+					sorter: (a, b) => (
+						(metrics.get(a.id)?.memoryRatio ?? 0) - (metrics.get(b.id)?.memoryRatio ?? 0)
+					),
+					render: (_, host) => <MetricMemoryRatio metrics={metrics.get(host.id)} />,
+					hidden: !showRatios,
+				},
+				{
+					title: "Memory",
+					sorter: (a, b) => (
+						(metrics.get(a.id)?.memoryUsed ?? 0) - (metrics.get(b.id)?.memoryUsed ?? 0)
+					),
+					render: (_, host) => <MetricMemoryUsed metrics={metrics.get(host.id)} />,
+					hidden: !showNumbers,
+				},
+				{
+					title: "Disk%",
+					sorter: (a, b) => (metrics.get(a.id)?.diskAvailable ?? 0) - (metrics.get(b.id)?.diskAvailable ?? 0),
+					render: (_, host) => <MetricDiskRatio metrics={metrics.get(host.id)} />,
+					hidden: !showRatios,
+				},
+				{
+					title: "Disk",
+					sorter: (a, b) => (
+						(metrics.get(a.id)?.diskUsed ?? 0) - (metrics.get(b.id)?.diskUsed ?? 0)
+					),
+					render: (_, host) => <MetricDiskUsed metrics={metrics.get(host.id)} />,
+					hidden: !showNumbers,
+				},
 				{
 					title: "Agent",
 					dataIndex: "agent",
@@ -235,6 +287,16 @@ export default function HostsPage() {
 				},
 			})}
 		/>
+		<Flex wrap="wrap" style={{ margin: "16px 0 0 0" }} gap="small">
+			<label style={{ width: "14em", display: "flex", justifyContent: "space-between", marginRight: 16 }}>
+				Show metric ratios:
+				<Switch checked={showRatios} onChange={checked => { setShowRatios(checked); }} />
+			</label>
+			<label style={{ width: "14em", display: "flex", justifyContent: "space-between", marginRight: 16 }}>
+				Show metric numbers:
+				<Switch checked={showNumbers} onChange={checked => { setShowNumbers(checked); }} />
+			</label>
+		</Flex>
 		<PluginExtra component="HostsPage" />
 	</PageLayout>;
 };

packages/web_ui/src/components/system_metrics.tsx

@@ -0,0 +1,55 @@
+import React from "react";
+
+import * as lib from "@clusterio/lib";
+import { Progress } from "antd";
+
+export function MetricCpuRatio(props: { metrics?: lib.SystemMetrics }) {
+	if (!props.metrics) {
+		return "N/A";
+	}
+	const ratio = props.metrics.cpuRatio;
+	return <Progress status="normal" percent={Math.ceil(ratio * 100)}/>;
+}
+
+export function MetricCpuUsed(props: { metrics?: lib.SystemMetrics }) {
+	if (!props.metrics) {
+		return "N/A";
+	}
+	const used = props.metrics.cpuUsed.toLocaleString(undefined, { maximumFractionDigits: 1 });
+	const capacity = props.metrics.cpuCapacity;
+	return `${used} / ${capacity}`;
+}
+
+export function MetricMemoryRatio(props: { metrics?: lib.SystemMetrics }) {
+	if (!props.metrics) {
+		return "N/A";
+	}
+	const ratio = props.metrics.memoryRatio;
+	return <Progress status="normal" percent={Math.ceil(ratio * 100)}/>;
+}
+
+export function MetricMemoryUsed(props: { metrics?: lib.SystemMetrics }) {
+	if (!props.metrics) {
+		return "N/A";
+	}
+	const used = lib.formatBytes(props.metrics.memoryUsed, "binary");
+	const capacity = lib.formatBytes(props.metrics.memoryCapacity, "binary");
+	return `${used} / ${capacity}`;
+}
+
+export function MetricDiskRatio(props: { metrics?: lib.SystemMetrics }) {
+	if (!props.metrics) {
+		return "N/A";
+	}
+	const ratio = props.metrics.diskRatio;
+	return <Progress status="normal" percent={Math.ceil(ratio * 100)}/>;
+}
+
+export function MetricDiskUsed(props: { metrics?: lib.SystemMetrics }) {
+	if (!props.metrics) {
+		return "N/A";
+	}
+	const used = lib.formatBytes(props.metrics.diskUsed);
+	const capacity = lib.formatBytes(props.metrics.diskCapacity);
+	return `${used} / ${capacity}`;
+}

packages/web_ui/src/model/system_metrics.ts

@@ -0,0 +1,8 @@
+import { useCallback, useContext, useSyncExternalStore } from "react";
+import ControlContext from "../components/ControlContext";
+
+export function useSystemMetrics() {
+	const control = useContext(ControlContext);
+	const subscribe = useCallback((callback: () => void) => control.systemMetrics.subscribe(callback), [control]);
+	return useSyncExternalStore(subscribe, () => control.systemMetrics.getSnapshot());
+}

packages/web_ui/src/util/websocket.ts

@@ -60,6 +60,7 @@ export class Control extends lib.Link {
 	logHandlers: Map<lib.LogFilter, logHandler[]> = new Map();
 
 	/** Updates handled by the subscription service */
+	systemMetrics = new lib.EventSubscriber(lib.SystemMetricsUpdateEvent, this);
 	hosts = new lib.EventSubscriber(lib.HostUpdatesEvent, this);
 	instances = new lib.EventSubscriber(lib.InstanceDetailsUpdatesEvent, this);
 	saves = new lib.EventSubscriber(lib.InstanceSaveDetailsUpdatesEvent, this);

plugins/inventory_sync/package.json

@@ -30,7 +30,7 @@
 		"@types/fs-extra": "^11.0.1",
 		"@types/node": "^20.4.5",
 		"@types/react": "^18.2.21",
-		"antd": "^5.7.3",
+		"antd": "^5.13.0",
 		"react": "^18.2.0",
 		"react-dom": "^18.2.0",
 		"typescript": "^5.1.6",

plugins/player_auth/package.json

@@ -32,7 +32,7 @@
 		"@types/jsonwebtoken": "^9.0.2",
 		"@types/node": "^20.4.5",
 		"@types/react": "^18.2.21",
-		"antd": "^5.7.3",
+		"antd": "^5.13.0",
 		"phin": "^3.7.0",
 		"react": "^18.2.0",
 		"react-dom": "^18.2.0",

plugins/subspace_storage/package.json

@@ -31,7 +31,7 @@
 		"@types/fs-extra": "^11.0.1",
 		"@types/node": "^20.4.5",
 		"@types/react": "^18.2.21",
-		"antd": "^5.7.3",
+		"antd": "^5.13.0",
 		"react": "^18.2.0",
 		"react-dom": "^18.2.0",
 		"typescript": "^5.1.6",

test/lib/data/SystemMetrics.js

@@ -0,0 +1,22 @@
+"use strict";
+const assert = require("assert").strict;
+
+const lib = require("@clusterio/lib");
+const { SystemMetrics } = lib;
+
+
+describe("lib/data/ModPack", function() {
+	describe("class SystemMetrics", function() {
+		it("correctly calculates derived metrics", function() {
+			const metrics = new SystemMetrics("controller", [0.5, 0.25, 0, 1], 1024, 768, 2048, 512, 0, false);
+			assert.equal(metrics.cpuCapacity, 4);
+			assert.equal(metrics.cpuUsed, 1.75);
+			assert.equal(metrics.cpuAvailable, 2.25);
+			assert.equal(metrics.cpuRatio, 0.4375);
+			assert.equal(metrics.memoryUsed, 256);
+			assert.equal(metrics.memoryRatio, 0.25);
+			assert.equal(metrics.diskUsed, 1536);
+			assert.equal(metrics.diskRatio, 0.75);
+		});
+	});
+});

test/lib/helpers.js

@@ -192,6 +192,50 @@ describe("lib/helpers", function() {
 		});
 	});
 
+	describe("formatBytes()", function() {
+		it("should format whole number of base unit bytes", function() {
+			assert.equal(lib.formatBytes(100), "100\u{A0}Bytes");
+			assert.equal(lib.formatBytes(100e3), "100\u{A0}kB");
+			assert.equal(lib.formatBytes(100e6), "100\u{A0}MB");
+			assert.equal(lib.formatBytes(100e9), "100\u{A0}GB");
+			assert.equal(lib.formatBytes(100e12), "100\u{A0}TB");
+			assert.equal(lib.formatBytes(100, false), "100\u{A0}Bytes");
+			assert.equal(lib.formatBytes(100 * 2**10, false), "100\u{A0}kiB");
+			assert.equal(lib.formatBytes(100 * 2**20, false), "100\u{A0}MiB");
+			assert.equal(lib.formatBytes(100 * 2**30, false), "100\u{A0}GiB");
+			assert.equal(lib.formatBytes(100 * 2**40, false), "100\u{A0}TiB");
+		});
+		it("should format with 3 significant digits fractional number of base unit bytes", function() {
+			// 1 digits base and 2 digit fraction
+			assert.equal(lib.formatBytes(1.234e3), "1.23\u{A0}kB");
+			assert.equal(lib.formatBytes(1.234e6), "1.23\u{A0}MB");
+			assert.equal(lib.formatBytes(1.234e9), "1.23\u{A0}GB");
+			assert.equal(lib.formatBytes(1.234e12), "1.23\u{A0}TB");
+			assert.equal(lib.formatBytes(1.234 * 2**10, "binary"), "1.23\u{A0}kiB");
+			assert.equal(lib.formatBytes(1.234 * 2**20, "binary"), "1.23\u{A0}MiB");
+			assert.equal(lib.formatBytes(1.234 * 2**30, "binary"), "1.23\u{A0}GiB");
+			assert.equal(lib.formatBytes(1.234 * 2**40, "binary"), "1.23\u{A0}TiB");
+			// 2 digits base and 1 digit fraction
+			assert.equal(lib.formatBytes(12.34e3), "12.3\u{A0}kB");
+			assert.equal(lib.formatBytes(12.34e6), "12.3\u{A0}MB");
+			assert.equal(lib.formatBytes(12.34e9), "12.3\u{A0}GB");
+			assert.equal(lib.formatBytes(12.34e12), "12.3\u{A0}TB");
+			assert.equal(lib.formatBytes(12.34 * 2**10, "binary"), "12.3\u{A0}kiB");
+			assert.equal(lib.formatBytes(12.34 * 2**20, "binary"), "12.3\u{A0}MiB");
+			assert.equal(lib.formatBytes(12.34 * 2**30, "binary"), "12.3\u{A0}GiB");
+			assert.equal(lib.formatBytes(12.34 * 2**40, "binary"), "12.3\u{A0}TiB");
+			// 3 digits base and no fraction
+			assert.equal(lib.formatBytes(123.4e3), "123\u{A0}kB");
+			assert.equal(lib.formatBytes(123.4e6), "123\u{A0}MB");
+			assert.equal(lib.formatBytes(123.4e9), "123\u{A0}GB");
+			assert.equal(lib.formatBytes(123.4e12), "123\u{A0}TB");
+			assert.equal(lib.formatBytes(123.4 * 2**10, "binary"), "123\u{A0}kiB");
+			assert.equal(lib.formatBytes(123.4 * 2**20, "binary"), "123\u{A0}MiB");
+			assert.equal(lib.formatBytes(123.4 * 2**30, "binary"), "123\u{A0}GiB");
+			assert.equal(lib.formatBytes(123.4 * 2**40, "binary"), "123\u{A0}TiB");
+		});
+	});
+
 	function parse(input, attributes) {
 		const result = lib.parseSearchString(input, attributes);
 		if (!result.issues.length) {