Add system info, CPU, memory and disk metrics

Collect basic metrics about the system the controller and hosts run on
including CPU, memory and disk capacity and usage.

packages/host/src/Host.ts

@@ -748,6 +748,12 @@ export default class Host extends lib.Link {
 					metricName: result.metric.name.replace("process_", "clusterio_host_"),
 				}));
 
+			} else if (result.metric.name.startsWith("system_")) {
+				results.push(lib.serializeResult(result, {
+					addLabels: { "host_id": String(this.config.get("host.id")) },
+					metricName: result.metric.name.replace("system_", "clusterio_host_system_"),
+				}));
+
 			} else {
 				results.push(lib.serializeResult(result));
 			}

packages/lib/index.ts

@@ -25,6 +25,7 @@ export * from "./src/prometheus";
 export * from "./src/schema";
 export * from "./src/shared_commands";
 export * from "./src/stream";
+export * from "./src/system_collectors";
 export * from "./src/zip_ops";
 export * from "./src/subscriptions";
 

packages/lib/src/system_collectors.ts

@@ -0,0 +1,121 @@
+import fs from "fs-extra";
+import os from "os";
+import util from "util";
+import { Gauge } from "./prometheus";
+
+function cpuModel() {
+	const model = os.cpus()[0].model;
+	// Remove irelevant clutter
+	return model
+		.replace(/\(R\)/gi, "®") // Intel CPUs use (R) instead of ®
+		.replace(/\(TM\)/gi, "™") // Intel and AMD CPUs use (TM) instead of ™
+		.replace(/ CPU /, " ") // Intel CPUs add redundant "CPU" in the model.
+		.replace(/ @ [0-9]+\.[0-9]+GHz/, "") // Intel CPUs add GHz after the model.
+		.replace(/ (Dual|Quad|Six|[0-9]+)-Core Processor/, "") // AMD CPUs add cores after the model.
+		.trim() // AMD CPUs may have trailing spaces in the model.
+	;
+}
+
+export const systemInfo = new Gauge(
+	"system_info",
+	"Static info about the system.",
+	{
+		labels: ["kernel", "machine", "cpu_model", "hostname", "node"],
+	}
+);
+systemInfo.labels({
+	"kernel": os.type(),
+	"machine": os.machine(),
+	"cpu_model": cpuModel(),
+	"hostname": os.hostname(),
+	"node": process.version,
+}).set(1);
+
+export const systemCpuSecondsTotal = new Gauge(
+	"system_cpu_seconds_total",
+	"Total system CPU time spent in seconds.",
+	{
+		labels: ["core"],
+		callback: function(collector) {
+			const cpus = os.cpus();
+			for (let i = 0; i < cpus.length; i++) {
+				const times = cpus[i].times;
+				collector.labels(String(i)).set(
+					(times.user + times.idle + times.irq + times.nice + times.sys) / 1000
+				);
+			}
+		},
+	},
+);
+
+export const systemCpuIdleSecondsTotal = new Gauge(
+	"system_cpu_idle_seconds_total",
+	"Total system CPU time spent being idle in seconds.",
+	{
+		labels: ["core"],
+		callback: function(collector) {
+			const cpus = os.cpus();
+			for (let i = 0; i < cpus.length; i++) {
+				const times = cpus[i].times;
+				collector.labels(String(i)).set(
+					times.idle / 1000
+				);
+			}
+		},
+	},
+);
+
+export const systemMemoryCapasityBytes = new Gauge(
+	"system_memory_capasity_bytes",
+	"Total system memory installed",
+	{
+		callback: function(collector) {
+			collector.set(os.totalmem());
+		},
+	},
+);
+
+export const systemMemoryAvailableBytes = new Gauge(
+	"system_memory_available_bytes",
+	"Total system memory available for use",
+	{
+		callback: function(collector) {
+			collector.set(os.freemem());
+		},
+	},
+);
+
+export const systemDiskCapasityBytes = new Gauge(
+	"system_disk_capasity_bytes",
+	"Size of the filesystem of the current working directory Node.js runs on",
+	{
+		callback: async function(collector) {
+			// statfs was added in Node.js v18.15.0 and may not be present.
+			// TODO: remove this check once minimum supported Node.js >= v18.15.0.
+			if (!fs.statfs) {
+				collector.set(0);
+				return;
+			}
+			const statFsAsync = util.promisify(fs.statfs);
+			const stats = await statFsAsync(".");
+			collector.set(stats.blocks * stats.bsize);
+		},
+	},
+);
+
+export const systemDiskAvailableBytes = new Gauge(
+	"system_disk_available_bytes",
+	"Available space on the filesystem of the current working directory Node.js runs on",
+	{
+		callback: async function(collector) {
+			// TODO: remove this check once minimum supported Node.js >= v18.15.0.
+			if (!fs.statfs) {
+				collector.set(0);
+				return;
+			}
+			const statFsAsync = util.promisify(fs.statfs);
+			const stats = await statFsAsync(".");
+			collector.set(stats.bavail * stats.bsize);
+		},
+	},
+);