mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-21 16:13:12 +09:00
Show System metrics in hosts page
Implement simple collection of basic real time system metrics of hosts and the controller for the purpose of displaying them in the Web UI. The metrics are collected by the controller at a configurable interval and pushed out to web clients currently subscribed to the metrics.
This commit is contained in:
1 parent
69bdc8e884
commit
d64bf6015e
13 files changed
+409
-7
No files matched your search
@@ -53,6 +53,8 @@ export default class Controller {
|
|||||||
config: lib.ControllerConfig;
|
config: lib.ControllerConfig;
|
||||||
app: Application;
|
app: Application;
|
||||||
|
|
||||||
|
/** True if (@link Controller.systemMetrics} has changed since last time it was saved */
|
||||||
|
systemMetricsDirty = false;
|
||||||
/** True if {@link Controller.hosts} has changed since last time it was saved */
|
/** True if {@link Controller.hosts} has changed since last time it was saved */
|
||||||
hostsDirty = false;
|
hostsDirty = false;
|
||||||
/** True if {@link Controller.instances} has changed since last time it was saved */
|
/** True if {@link Controller.instances} has changed since last time it was saved */
|
||||||
@@ -89,6 +91,7 @@ export default class Controller {
|
|||||||
devMiddleware: any | null = null;
|
devMiddleware: any | null = null;
|
||||||
|
|
||||||
autosaveInterval?: ReturnType<typeof setInterval>;
|
autosaveInterval?: ReturnType<typeof setInterval>;
|
||||||
|
systemMetricsInterval?: ReturnType<typeof setInterval>;
|
||||||
|
|
||||||
logDirectory: string = "";
|
logDirectory: string = "";
|
||||||
clusterLogIndex: lib.LogIndex | null = null;
|
clusterLogIndex: lib.LogIndex | null = null;
|
||||||
@@ -98,6 +101,7 @@ export default class Controller {
|
|||||||
let databaseDirectory = config.get("controller.database_directory");
|
let databaseDirectory = config.get("controller.database_directory");
|
||||||
await fs.ensureDir(databaseDirectory);
|
await fs.ensureDir(databaseDirectory);
|
||||||
|
|
||||||
|
const systemMetrics = await Controller.loadSystemMetrics(path.join(databaseDirectory, "metrics.json"));
|
||||||
const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json"));
|
const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json"));
|
||||||
const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json"));
|
const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json"));
|
||||||
const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json"));
|
const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json"));
|
||||||
@@ -110,6 +114,7 @@ export default class Controller {
|
|||||||
const modStore = await lib.ModStore.fromDirectory(modsDirectory);
|
const modStore = await lib.ModStore.fromDirectory(modsDirectory);
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
systemMetrics,
|
||||||
hosts,
|
hosts,
|
||||||
instances,
|
instances,
|
||||||
saves,
|
saves,
|
||||||
@@ -125,6 +130,7 @@ export default class Controller {
|
|||||||
configPath: string,
|
configPath: string,
|
||||||
config: lib.ControllerConfig,
|
config: lib.ControllerConfig,
|
||||||
|
|
||||||
|
public systemMetrics = new Map<lib.SystemMetrics["id"], lib.SystemMetrics>(),
|
||||||
/** Mapping of host id to host info */
|
/** Mapping of host id to host info */
|
||||||
public hosts = new Map<number, HostInfo>(),
|
public hosts = new Map<number, HostInfo>(),
|
||||||
/** Mapping of instance id to instance info */
|
/** Mapping of instance id to instance info */
|
||||||
@@ -154,6 +160,7 @@ export default class Controller {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Handle subscriptions for all internal properties
|
// Handle subscriptions for all internal properties
|
||||||
|
this.subscriptions.handle(lib.SystemMetricsUpdateEvent, this.handleSystemMetricsSubscription.bind(this));
|
||||||
this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this));
|
this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this));
|
||||||
this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this));
|
this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this));
|
||||||
this.subscriptions.handle(
|
this.subscriptions.handle(
|
||||||
@@ -219,6 +226,8 @@ export default class Controller {
|
|||||||
this.config.on("fieldChanged", (field, curr, prev) => {
|
this.config.on("fieldChanged", (field, curr, prev) => {
|
||||||
if (field === "controller.autosave_interval") {
|
if (field === "controller.autosave_interval") {
|
||||||
this.onAutosaveIntervalChanged();
|
this.onAutosaveIntervalChanged();
|
||||||
|
} else if (field === "controller.system_metrics_interval") {
|
||||||
|
this.onSystemMetricsIntervalChanged();
|
||||||
}
|
}
|
||||||
lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev);
|
lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev);
|
||||||
});
|
});
|
||||||
@@ -292,6 +301,7 @@ export default class Controller {
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.onAutosaveIntervalChanged();
|
this.onAutosaveIntervalChanged();
|
||||||
|
this.onSystemMetricsIntervalChanged();
|
||||||
|
|
||||||
logger.info("Started controller");
|
logger.info("Started controller");
|
||||||
this._state = "running";
|
this._state = "running";
|
||||||
@@ -357,6 +367,11 @@ export default class Controller {
|
|||||||
clearInterval(this.clusterLogBuildInterval);
|
clearInterval(this.clusterLogBuildInterval);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this.systemMetricsInterval) {
|
||||||
|
clearInterval(this.systemMetricsInterval);
|
||||||
|
this.systemMetricsInterval = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
if (this.autosaveInterval) {
|
if (this.autosaveInterval) {
|
||||||
clearInterval(this.autosaveInterval);
|
clearInterval(this.autosaveInterval);
|
||||||
this.autosaveInterval = undefined;
|
this.autosaveInterval = undefined;
|
||||||
@@ -389,6 +404,41 @@ export default class Controller {
|
|||||||
logger.info("Goodbye");
|
logger.info("Goodbye");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
onSystemMetricsIntervalChanged() {
|
||||||
|
if (this.systemMetricsInterval) {
|
||||||
|
clearInterval(this.systemMetricsInterval);
|
||||||
|
this.systemMetricsInterval = undefined;
|
||||||
|
}
|
||||||
|
const systemMetricsIntervalSeconds = this.config.get("controller.system_metrics_interval");
|
||||||
|
if (systemMetricsIntervalSeconds > 0) {
|
||||||
|
this.systemMetricsInterval = setInterval(
|
||||||
|
this.updateSystemMetrics.bind(this),
|
||||||
|
systemMetricsIntervalSeconds * 1000
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async updateSystemMetrics() {
|
||||||
|
try {
|
||||||
|
const requests: Promise<lib.SystemMetrics>[] = [];
|
||||||
|
for (let hostConnection of this.wsServer.hostConnections.values()) {
|
||||||
|
if (!hostConnection.connected) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
requests.push(hostConnection.send(new lib.SystemMetricsRequest()));
|
||||||
|
}
|
||||||
|
requests.push(lib.gatherSystemMetrics("controller"));
|
||||||
|
const newMetrics = await Promise.all(requests);
|
||||||
|
for (const metric of newMetrics) {
|
||||||
|
this.systemMetrics.set(metric.id, metric);
|
||||||
|
}
|
||||||
|
this.systemMetricsDirty = true;
|
||||||
|
this.subscriptions.broadcast(new lib.SystemMetricsUpdateEvent(newMetrics));
|
||||||
|
} catch (err: any) {
|
||||||
|
logger.error(`Unexpected error updating system metrics:\n${err.stack ?? err.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
onAutosaveIntervalChanged() {
|
onAutosaveIntervalChanged() {
|
||||||
if (this.autosaveInterval) {
|
if (this.autosaveInterval) {
|
||||||
clearInterval(this.autosaveInterval);
|
clearInterval(this.autosaveInterval);
|
||||||
@@ -425,6 +475,11 @@ export default class Controller {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let databaseDirectory = this.config.get("controller.database_directory");
|
let databaseDirectory = this.config.get("controller.database_directory");
|
||||||
|
if (this.systemMetricsDirty) {
|
||||||
|
this.systemMetricsDirty = false;
|
||||||
|
await Controller.saveSystemMetrics(path.join(databaseDirectory, "metrics.json"), this.systemMetrics);
|
||||||
|
}
|
||||||
|
|
||||||
if (this.hostsDirty) {
|
if (this.hostsDirty) {
|
||||||
this.hostsDirty = false;
|
this.hostsDirty = false;
|
||||||
await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
|
await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
|
||||||
@@ -452,6 +507,25 @@ export default class Controller {
|
|||||||
await lib.invokeHook(this.plugins, "onSaveData");
|
await lib.invokeHook(this.plugins, "onSaveData");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static async loadSystemMetrics(filePath: string): Promise<Map<lib.SystemMetrics["id"], lib.SystemMetrics>> {
|
||||||
|
let json: Static<typeof lib.SystemMetrics.jsonSchema>[];
|
||||||
|
try {
|
||||||
|
json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
|
||||||
|
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.code !== "ENOENT") {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
return new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Map(json.map(m => lib.SystemMetrics.fromJSON(m)).map(m => [m.id, m]));
|
||||||
|
}
|
||||||
|
|
||||||
|
static async saveSystemMetrics(filePath: string, systemMetrics: Map<lib.SystemMetrics["id"], lib.SystemMetrics>) {
|
||||||
|
await lib.safeOutputFile(filePath, JSON.stringify([...systemMetrics.values()], null, "\t"));
|
||||||
|
}
|
||||||
|
|
||||||
static async loadHosts(filePath: string): Promise<Map<number, HostInfo>> {
|
static async loadHosts(filePath: string): Promise<Map<number, HostInfo>> {
|
||||||
let serialized: [number, Static<typeof HostInfo.jsonSchema>][];
|
let serialized: [number, Static<typeof HostInfo.jsonSchema>][];
|
||||||
try {
|
try {
|
||||||
@@ -635,6 +709,13 @@ export default class Controller {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async handleSystemMetricsSubscription(request: lib.SubscriptionRequest) {
|
||||||
|
const systemMetrics = [...this.systemMetrics.values()].filter(
|
||||||
|
metric => metric.updatedAt > request.lastRequestTime,
|
||||||
|
);
|
||||||
|
return systemMetrics.length ? new lib.SystemMetricsUpdateEvent(systemMetrics) : null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate access token for host
|
* Generate access token for host
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -347,6 +347,7 @@ export default class Host extends lib.Link {
|
|||||||
this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.handleWhitelistUpdateEvent.bind(this));
|
this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.handleWhitelistUpdateEvent.bind(this));
|
||||||
this.handle(lib.InstanceAssignInternalRequest, this.handleInstanceAssignInternalRequest.bind(this));
|
this.handle(lib.InstanceAssignInternalRequest, this.handleInstanceAssignInternalRequest.bind(this));
|
||||||
this.handle(lib.InstanceUnassignInternalRequest, this.handleInstanceUnassignInternalRequest.bind(this));
|
this.handle(lib.InstanceUnassignInternalRequest, this.handleInstanceUnassignInternalRequest.bind(this));
|
||||||
|
this.handle(lib.SystemMetricsRequest, this.handleSystemMetricsRequest.bind(this));
|
||||||
this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this));
|
this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this));
|
||||||
this.fallbackRequest(
|
this.fallbackRequest(
|
||||||
lib.InstanceSaveDetailsListRequest, this.fallbackInstanceSaveDetailsListRequest.bind(this),
|
lib.InstanceSaveDetailsListRequest, this.fallbackInstanceSaveDetailsListRequest.bind(this),
|
||||||
@@ -723,6 +724,10 @@ export default class Host extends lib.Link {
|
|||||||
return instanceConnection;
|
return instanceConnection;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async handleSystemMetricsRequest() {
|
||||||
|
return lib.gatherSystemMetrics(this.config.get("host.id"));
|
||||||
|
}
|
||||||
|
|
||||||
async handleHostMetricsRequest() {
|
async handleHostMetricsRequest() {
|
||||||
let requests: Promise<InstanceType<typeof lib.InstanceMetricsRequest["Response"]>>[] = [];
|
let requests: Promise<InstanceType<typeof lib.InstanceMetricsRequest["Response"]>>[] = [];
|
||||||
for (let instanceConnection of this.instanceConnections.values()) {
|
for (let instanceConnection of this.instanceConnections.values()) {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ export interface ControllerConfigFields {
|
|||||||
"controller.heartbeat_interval": number;
|
"controller.heartbeat_interval": number;
|
||||||
"controller.session_timeout": number;
|
"controller.session_timeout": number;
|
||||||
"controller.metrics_timeout": number;
|
"controller.metrics_timeout": number;
|
||||||
|
"controller.system_metrics_interval": number;
|
||||||
"controller.proxy_stream_timeout": number;
|
"controller.proxy_stream_timeout": number;
|
||||||
"controller.default_mod_pack_id": number | null;
|
"controller.default_mod_pack_id": number | null;
|
||||||
"controller.default_role_id": number | null;
|
"controller.default_role_id": number | null;
|
||||||
@@ -126,6 +127,12 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
|
|||||||
type: "number",
|
type: "number",
|
||||||
initialValue: 8,
|
initialValue: 8,
|
||||||
},
|
},
|
||||||
|
"controller.system_metrics_interval": {
|
||||||
|
title: "System Metrics Interval",
|
||||||
|
description: "Interval in seconds to collect and update system metrics for the Web UI",
|
||||||
|
type: "number",
|
||||||
|
initialValue: 10,
|
||||||
|
},
|
||||||
"controller.proxy_stream_timeout": {
|
"controller.proxy_stream_timeout": {
|
||||||
title: "Proxy Stream Timeout",
|
title: "Proxy Stream Timeout",
|
||||||
description: "Timeout in seconds for proxy streams to start flowing.",
|
description: "Timeout in seconds for proxy streams to start flowing.",
|
||||||
|
|||||||
@@ -204,6 +204,108 @@ export class LogMessageEvent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class SystemMetrics {
|
||||||
|
constructor(
|
||||||
|
/**
|
||||||
|
* Id of the host these metrics originate from, or the string
|
||||||
|
* "controller" if these metrics are for the controller.
|
||||||
|
*/
|
||||||
|
public id: number | "controller",
|
||||||
|
public coreRatios: number[],
|
||||||
|
public memoryCapacity: number,
|
||||||
|
public memoryAvailable: number,
|
||||||
|
public diskCapacity: number,
|
||||||
|
public diskAvailable: number,
|
||||||
|
/** Millisecond Unix timestamp this entry was last updated at */
|
||||||
|
public updatedAt: number,
|
||||||
|
public isDeleted: boolean,
|
||||||
|
) { }
|
||||||
|
|
||||||
|
static jsonSchema = Type.Object({
|
||||||
|
"id": Type.Union([Type.Number(), Type.Literal("controller")]),
|
||||||
|
"coreRatios": Type.Array(Type.Number()),
|
||||||
|
"memoryCapacity": Type.Number(),
|
||||||
|
"memoryAvailable": Type.Number(),
|
||||||
|
"diskCapacity": Type.Number(),
|
||||||
|
"diskAvailable": Type.Number(),
|
||||||
|
"updatedAt": Type.Number(),
|
||||||
|
"isDeleted": Type.Boolean(),
|
||||||
|
});
|
||||||
|
|
||||||
|
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||||
|
return new this(
|
||||||
|
json.id,
|
||||||
|
json.coreRatios,
|
||||||
|
json.memoryCapacity,
|
||||||
|
json.memoryAvailable,
|
||||||
|
json.diskCapacity,
|
||||||
|
json.diskAvailable,
|
||||||
|
json.updatedAt,
|
||||||
|
json.isDeleted,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
get cpuCapacity() {
|
||||||
|
return this.coreRatios.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
get cpuUsed() {
|
||||||
|
return this.coreRatios.reduce((a, b) => a + b, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
get cpuAvailable() {
|
||||||
|
return this.cpuCapacity - this.cpuUsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
get cpuRatio() {
|
||||||
|
return this.cpuUsed / this.cpuCapacity;
|
||||||
|
}
|
||||||
|
|
||||||
|
get memoryUsed() {
|
||||||
|
return this.memoryCapacity - this.memoryAvailable;
|
||||||
|
}
|
||||||
|
|
||||||
|
get memoryRatio() {
|
||||||
|
return (this.memoryCapacity - this.memoryAvailable) / this.memoryCapacity;
|
||||||
|
}
|
||||||
|
|
||||||
|
get diskUsed() {
|
||||||
|
return this.diskCapacity - this.diskAvailable;
|
||||||
|
}
|
||||||
|
|
||||||
|
get diskRatio() {
|
||||||
|
return (this.diskCapacity - this.diskAvailable) / this.diskCapacity;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SystemMetricsRequest {
|
||||||
|
declare ["constructor"]: typeof SystemMetricsRequest;
|
||||||
|
static type = "request" as const;
|
||||||
|
static src = "controller" as const;
|
||||||
|
static dst = "host" as const;
|
||||||
|
static Response = SystemMetrics;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SystemMetricsUpdateEvent {
|
||||||
|
declare ["constructor"]: typeof SystemMetricsUpdateEvent;
|
||||||
|
static type = "event" as const;
|
||||||
|
static src = "controller" as const;
|
||||||
|
static dst = "control" as const;
|
||||||
|
static permission = "core.system_metrics.subscribe" as const;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
public updates: SystemMetrics[],
|
||||||
|
) { }
|
||||||
|
|
||||||
|
static jsonSchema = Type.Object({
|
||||||
|
"updates": Type.Array(SystemMetrics.jsonSchema),
|
||||||
|
});
|
||||||
|
|
||||||
|
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||||
|
return new this(json.updates.map(update => SystemMetrics.fromJSON(update)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class DebugDumpWsRequest {
|
export class DebugDumpWsRequest {
|
||||||
declare ["constructor"]: typeof DebugDumpWsRequest;
|
declare ["constructor"]: typeof DebugDumpWsRequest;
|
||||||
static type = "request" as const;
|
static type = "request" as const;
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
|
|||||||
controller.LogSetSubscriptionsRequest,
|
controller.LogSetSubscriptionsRequest,
|
||||||
controller.LogQueryRequest,
|
controller.LogQueryRequest,
|
||||||
controller.LogMessageEvent,
|
controller.LogMessageEvent,
|
||||||
|
controller.SystemMetricsRequest,
|
||||||
|
controller.SystemMetricsUpdateEvent,
|
||||||
controller.DebugDumpWsRequest,
|
controller.DebugDumpWsRequest,
|
||||||
controller.DebugWsMessageEvent,
|
controller.DebugWsMessageEvent,
|
||||||
|
|
||||||
|
|||||||
@@ -129,6 +129,13 @@ definePermission({
|
|||||||
title: "Modify controller config",
|
title: "Modify controller config",
|
||||||
description: "Modify the controller config or entries of the controller config.",
|
description: "Modify the controller config or entries of the controller config.",
|
||||||
});
|
});
|
||||||
|
definePermission({
|
||||||
|
name: "core.system_metrics.subscribe",
|
||||||
|
title: "Subscribe to system metrics updates",
|
||||||
|
description:
|
||||||
|
"Subscribe to be notified when the system metrics detailing cpu, memory and disk usage for the " +
|
||||||
|
"controller and hosts are updated.",
|
||||||
|
});
|
||||||
definePermission({
|
definePermission({
|
||||||
name: "core.host.list",
|
name: "core.host.list",
|
||||||
title: "List hosts",
|
title: "List hosts",
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import fs from "fs-extra";
|
|||||||
import os from "os";
|
import os from "os";
|
||||||
import util from "util";
|
import util from "util";
|
||||||
import { Gauge } from "./prometheus";
|
import { Gauge } from "./prometheus";
|
||||||
|
import { SystemMetrics } from "./data";
|
||||||
|
|
||||||
function cpuModel() {
|
function cpuModel() {
|
||||||
const model = os.cpus()[0].model;
|
const model = os.cpus()[0].model;
|
||||||
@@ -119,3 +120,45 @@ export const systemDiskAvailableBytes = new Gauge(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
function minZip<T>(a: T[], b: T[]): [T, T][] {
|
||||||
|
const length = Math.min(a.length, b.length);
|
||||||
|
const result = [];
|
||||||
|
result.length = length;
|
||||||
|
for (let i = 0; i < length; i++) {
|
||||||
|
result[i] = [a[i], b[i]] as [T, T];
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
let previousTotalCpuMs: number[] = [];
|
||||||
|
let previousIdleCpuMs: number[] = [];
|
||||||
|
export async function gatherSystemMetrics(id: SystemMetrics["id"]) {
|
||||||
|
const cpus = os.cpus();
|
||||||
|
const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
|
||||||
|
const currentIdleCpuMs = cpus.map(({ times }) => times.idle);
|
||||||
|
const deltaTotalCpuMs = minZip(previousTotalCpuMs, currentTotalCpuMs).map(([prev, curr]) => curr - prev);
|
||||||
|
const deltaIdleCpuMs = minZip(previousIdleCpuMs, currentIdleCpuMs).map(([prev, curr]) => curr - prev);
|
||||||
|
const cpuUsage = minZip(deltaTotalCpuMs, deltaIdleCpuMs).map(([total, idle]) => (total - idle) / total);
|
||||||
|
previousTotalCpuMs = currentTotalCpuMs;
|
||||||
|
previousIdleCpuMs = currentIdleCpuMs;
|
||||||
|
let diskCapacity = 0;
|
||||||
|
let diskAvailable = 0;
|
||||||
|
if (fs.statfs) { // TODO: remove this check once minimum supported Node.js >= v18.15.0.
|
||||||
|
const statFsAsync = util.promisify(fs.statfs);
|
||||||
|
const stats = await statFsAsync(".");
|
||||||
|
diskCapacity = stats.blocks * stats.bsize;
|
||||||
|
diskAvailable = stats.bavail * stats.bsize;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new SystemMetrics(
|
||||||
|
id,
|
||||||
|
cpuUsage,
|
||||||
|
os.totalmem(),
|
||||||
|
os.freemem(),
|
||||||
|
diskCapacity,
|
||||||
|
diskAvailable,
|
||||||
|
Date.now(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,9 +1,14 @@
|
|||||||
import React from "react";
|
import React from "react";
|
||||||
import { Typography } from "antd";
|
import { Descriptions, Typography } from "antd";
|
||||||
|
|
||||||
import PluginExtra from "./PluginExtra";
|
import PluginExtra from "./PluginExtra";
|
||||||
import LogConsole from "./LogConsole";
|
import LogConsole from "./LogConsole";
|
||||||
|
import {
|
||||||
|
MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
|
||||||
|
MetricDiskUsed, MetricDiskRatio,
|
||||||
|
} from "./system_metrics";
|
||||||
import { useAccount } from "../model/account";
|
import { useAccount } from "../model/account";
|
||||||
|
import { useSystemMetrics } from "../model/system_metrics";
|
||||||
import ControllerConfigTree from "./ControllerConfigTree";
|
import ControllerConfigTree from "./ControllerConfigTree";
|
||||||
import PageLayout from "./PageLayout";
|
import PageLayout from "./PageLayout";
|
||||||
|
|
||||||
@@ -12,9 +17,19 @@ const { Title } = Typography;
|
|||||||
|
|
||||||
export default function ControllerPage() {
|
export default function ControllerPage() {
|
||||||
let account = useAccount();
|
let account = useAccount();
|
||||||
|
const [systemMetrics] = useSystemMetrics();
|
||||||
|
const metrics = systemMetrics.get("controller");
|
||||||
|
|
||||||
return <PageLayout nav={[{ name: "Controller" }]}>
|
return <PageLayout nav={[{ name: "Controller" }]}>
|
||||||
<h2>Controller</h2>
|
<h2>Controller</h2>
|
||||||
|
<Descriptions bordered size="small" column={{ xs: 1, md: 2, lg: 2, xl: 2, xxl: 2 }}>
|
||||||
|
<Descriptions.Item label="CPU Usage"><MetricCpuRatio metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Cores"><MetricCpuUsed metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Memory Usage"><MetricMemoryRatio metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Memory"><MetricMemoryUsed metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Disk Usage"><MetricDiskRatio metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Disk"><MetricDiskUsed metrics={metrics} /></Descriptions.Item>
|
||||||
|
</Descriptions>
|
||||||
{account.hasPermission("core.log.follow") && <>
|
{account.hasPermission("core.log.follow") && <>
|
||||||
<Title level={5} style={{ marginTop: 16 }}>Console</Title>
|
<Title level={5} style={{ marginTop: 16 }}>Console</Title>
|
||||||
<LogConsole controller={true} />
|
<LogConsole controller={true} />
|
||||||
|
|||||||
@@ -13,7 +13,12 @@ import { useHost } from "../model/host";
|
|||||||
import PageHeader from "./PageHeader";
|
import PageHeader from "./PageHeader";
|
||||||
import PageLayout from "./PageLayout";
|
import PageLayout from "./PageLayout";
|
||||||
import PluginExtra from "./PluginExtra";
|
import PluginExtra from "./PluginExtra";
|
||||||
|
import {
|
||||||
|
MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
|
||||||
|
MetricDiskUsed, MetricDiskRatio,
|
||||||
|
} from "./system_metrics";
|
||||||
import { formatTimestamp } from "../util/time_format";
|
import { formatTimestamp } from "../util/time_format";
|
||||||
|
import { useSystemMetrics } from "../model/system_metrics";
|
||||||
|
|
||||||
const { Title } = Typography;
|
const { Title } = Typography;
|
||||||
|
|
||||||
@@ -23,6 +28,8 @@ export default function HostViewPage() {
|
|||||||
let control = useContext(ControlContext);
|
let control = useContext(ControlContext);
|
||||||
let account = useAccount();
|
let account = useAccount();
|
||||||
let [instances] = useInstances();
|
let [instances] = useInstances();
|
||||||
|
const [systemMetrics] = useSystemMetrics();
|
||||||
|
const metrics = systemMetrics.get(hostId);
|
||||||
const [host, synced] = useHost(hostId);
|
const [host, synced] = useHost(hostId);
|
||||||
const hostInstances = new Map([...instances].filter(([_id, instance]) => instance.assignedHost === hostId));
|
const hostInstances = new Map([...instances].filter(([_id, instance]) => instance.assignedHost === hostId));
|
||||||
|
|
||||||
@@ -63,20 +70,27 @@ export default function HostViewPage() {
|
|||||||
title={host.name || String(hostId)}
|
title={host.name || String(hostId)}
|
||||||
extra={hostButtons}
|
extra={hostButtons}
|
||||||
/>
|
/>
|
||||||
<Descriptions bordered size="small" column={{ xs: 1, sm: 2, xl: 4 }}>
|
<Descriptions bordered size="small" column={{ xs: 1, md: 2, lg: 2, xl: 2, xxl: 2 }}>
|
||||||
<Descriptions.Item label="Name">{host["name"]}</Descriptions.Item>
|
<Descriptions.Item label="Name">{host["name"]}</Descriptions.Item>
|
||||||
<Descriptions.Item label="Connected">
|
<Descriptions.Item label="Connected">
|
||||||
<Tag color={host["connected"] ? "#389e0d" : "#cf1322"}>
|
<Tag color={host["connected"] ? "#389e0d" : "#cf1322"}>
|
||||||
{host["connected"] ? "Connected" : "Disconnected"}
|
{host["connected"] ? "Connected" : "Disconnected"}
|
||||||
</Tag>
|
</Tag>
|
||||||
</Descriptions.Item>
|
</Descriptions.Item>
|
||||||
|
<Descriptions.Item label="CPU Usage"><MetricCpuRatio metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Cores"><MetricCpuUsed metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Memory Usage"><MetricMemoryRatio metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Memory"><MetricMemoryUsed metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Disk Usage"><MetricDiskRatio metrics={metrics} /></Descriptions.Item>
|
||||||
|
<Descriptions.Item label="Disk"><MetricDiskUsed metrics={metrics} /></Descriptions.Item>
|
||||||
<Descriptions.Item label="Agent">{host["agent"]}</Descriptions.Item>
|
<Descriptions.Item label="Agent">{host["agent"]}</Descriptions.Item>
|
||||||
<Descriptions.Item label="Version">{host["version"]}</Descriptions.Item>
|
<Descriptions.Item label="Version">{host["version"]}</Descriptions.Item>
|
||||||
{
|
{
|
||||||
host.tokenValidAfter
|
host.tokenValidAfter
|
||||||
&& <Descriptions.Item label="Tokens valid after:">
|
? <Descriptions.Item label="Tokens valid after:">
|
||||||
{formatTimestamp(host.tokenValidAfter*1000)}
|
{formatTimestamp(host.tokenValidAfter*1000)}
|
||||||
</Descriptions.Item>
|
</Descriptions.Item>
|
||||||
|
: null
|
||||||
}
|
}
|
||||||
</Descriptions>
|
</Descriptions>
|
||||||
{account.hasPermission("core.instance.list") && <>
|
{account.hasPermission("core.instance.list") && <>
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import React, { useContext, useRef, useState, useEffect } from "react";
|
import React, { useContext, useRef, useState, useEffect } from "react";
|
||||||
import { useNavigate } from "react-router-dom";
|
import { useNavigate } from "react-router-dom";
|
||||||
import { Button, Form, InputNumber, Modal, Table, Tag, Typography } from "antd";
|
import { Button, Flex, Form, InputNumber, Modal, Progress, Switch, Table, Tag, Typography } from "antd";
|
||||||
import CopyOutlined from "@ant-design/icons/lib/icons/CopyOutlined";
|
import CopyOutlined from "@ant-design/icons/lib/icons/CopyOutlined";
|
||||||
|
|
||||||
import * as lib from "@clusterio/lib";
|
import * as lib from "@clusterio/lib";
|
||||||
@@ -10,7 +10,12 @@ import ControlContext from "./ControlContext";
|
|||||||
import PageHeader from "./PageHeader";
|
import PageHeader from "./PageHeader";
|
||||||
import PageLayout from "./PageLayout";
|
import PageLayout from "./PageLayout";
|
||||||
import PluginExtra from "./PluginExtra";
|
import PluginExtra from "./PluginExtra";
|
||||||
|
import {
|
||||||
|
MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
|
||||||
|
MetricDiskUsed, MetricDiskRatio,
|
||||||
|
} from "./system_metrics";
|
||||||
import { useHosts } from "../model/host";
|
import { useHosts } from "../model/host";
|
||||||
|
import { useSystemMetrics } from "../model/system_metrics";
|
||||||
import notify, { notifyErrorHandler } from "../util/notify";
|
import notify, { notifyErrorHandler } from "../util/notify";
|
||||||
|
|
||||||
const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare;
|
const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare;
|
||||||
@@ -180,11 +185,13 @@ function CopyButton({ text, message }: { text:string, message:string }) {
|
|||||||
</Button>;
|
</Button>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export default function HostsPage() {
|
export default function HostsPage() {
|
||||||
let account = useAccount();
|
let account = useAccount();
|
||||||
let navigate = useNavigate();
|
let navigate = useNavigate();
|
||||||
let [hosts] = useHosts();
|
let [hosts] = useHosts();
|
||||||
|
const [metrics] = useSystemMetrics();
|
||||||
|
const [showRatios, setShowRatios] = useState(true);
|
||||||
|
const [showNumbers, setShowNumbers] = useState(false);
|
||||||
|
|
||||||
return <PageLayout nav={[{ name: "Hosts" }]}>
|
return <PageLayout nav={[{ name: "Hosts" }]}>
|
||||||
<PageHeader
|
<PageHeader
|
||||||
@@ -192,6 +199,7 @@ export default function HostsPage() {
|
|||||||
extra={account.hasPermission("core.host.generate_token") ? <GenerateHostTokenButton /> : undefined}
|
extra={account.hasPermission("core.host.generate_token") ? <GenerateHostTokenButton /> : undefined}
|
||||||
/>
|
/>
|
||||||
<Table
|
<Table
|
||||||
|
style={{ overflowX: "auto" }}
|
||||||
columns={[
|
columns={[
|
||||||
{
|
{
|
||||||
title: "Name",
|
title: "Name",
|
||||||
@@ -199,6 +207,50 @@ export default function HostsPage() {
|
|||||||
defaultSortOrder: "ascend",
|
defaultSortOrder: "ascend",
|
||||||
sorter: (a, b) => strcmp(a.name, b.name),
|
sorter: (a, b) => strcmp(a.name, b.name),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
title: "CPU%",
|
||||||
|
sorter: (a, b) => (metrics.get(a.id)?.cpuRatio ?? 0) - (metrics.get(b.id)?.cpuRatio ?? 0),
|
||||||
|
render: (_, host) => <MetricCpuRatio metrics={metrics.get(host.id)} />,
|
||||||
|
hidden: !showRatios,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Cores",
|
||||||
|
sorter: (a, b) => (
|
||||||
|
(metrics.get(a.id)?.cpuUsed ?? 0) - (metrics.get(b.id)?.cpuUsed ?? 0)
|
||||||
|
),
|
||||||
|
render: (_, host) => <MetricCpuUsed metrics={metrics.get(host.id)} />,
|
||||||
|
hidden: !showNumbers,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Mem%",
|
||||||
|
sorter: (a, b) => (
|
||||||
|
(metrics.get(a.id)?.memoryRatio ?? 0) - (metrics.get(b.id)?.memoryRatio ?? 0)
|
||||||
|
),
|
||||||
|
render: (_, host) => <MetricMemoryRatio metrics={metrics.get(host.id)} />,
|
||||||
|
hidden: !showRatios,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Memory",
|
||||||
|
sorter: (a, b) => (
|
||||||
|
(metrics.get(a.id)?.memoryUsed ?? 0) - (metrics.get(b.id)?.memoryUsed ?? 0)
|
||||||
|
),
|
||||||
|
render: (_, host) => <MetricMemoryUsed metrics={metrics.get(host.id)} />,
|
||||||
|
hidden: !showNumbers,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Disk%",
|
||||||
|
sorter: (a, b) => (metrics.get(a.id)?.diskAvailable ?? 0) - (metrics.get(b.id)?.diskAvailable ?? 0),
|
||||||
|
render: (_, host) => <MetricDiskRatio metrics={metrics.get(host.id)} />,
|
||||||
|
hidden: !showRatios,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Disk",
|
||||||
|
sorter: (a, b) => (
|
||||||
|
(metrics.get(a.id)?.diskUsed ?? 0) - (metrics.get(b.id)?.diskUsed ?? 0)
|
||||||
|
),
|
||||||
|
render: (_, host) => <MetricDiskUsed metrics={metrics.get(host.id)} />,
|
||||||
|
hidden: !showNumbers,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
title: "Agent",
|
title: "Agent",
|
||||||
dataIndex: "agent",
|
dataIndex: "agent",
|
||||||
@@ -235,6 +287,16 @@ export default function HostsPage() {
|
|||||||
},
|
},
|
||||||
})}
|
})}
|
||||||
/>
|
/>
|
||||||
|
<Flex wrap="wrap" style={{ margin: "16px 0 0 0" }} gap="small">
|
||||||
|
<label style={{ width: "14em", display: "flex", justifyContent: "space-between", marginRight: 16 }}>
|
||||||
|
Show metric ratios:
|
||||||
|
<Switch checked={showRatios} onChange={checked => { setShowRatios(checked); }} />
|
||||||
|
</label>
|
||||||
|
<label style={{ width: "14em", display: "flex", justifyContent: "space-between", marginRight: 16 }}>
|
||||||
|
Show metric numbers:
|
||||||
|
<Switch checked={showNumbers} onChange={checked => { setShowNumbers(checked); }} />
|
||||||
|
</label>
|
||||||
|
</Flex>
|
||||||
<PluginExtra component="HostsPage" />
|
<PluginExtra component="HostsPage" />
|
||||||
</PageLayout>;
|
</PageLayout>;
|
||||||
};
|
};
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import React from "react";
|
||||||
|
|
||||||
|
import * as lib from "@clusterio/lib";
|
||||||
|
import { Progress } from "antd";
|
||||||
|
|
||||||
|
export function MetricCpuRatio(props: { metrics?: lib.SystemMetrics }) {
|
||||||
|
if (!props.metrics) {
|
||||||
|
return "N/A";
|
||||||
|
}
|
||||||
|
const ratio = props.metrics.cpuRatio;
|
||||||
|
return <Progress status="normal" percent={Math.ceil(ratio * 100)}/>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function MetricCpuUsed(props: { metrics?: lib.SystemMetrics }) {
|
||||||
|
if (!props.metrics) {
|
||||||
|
return "N/A";
|
||||||
|
}
|
||||||
|
const used = props.metrics.cpuUsed.toLocaleString(undefined, { maximumFractionDigits: 1 });
|
||||||
|
const capacity = props.metrics.cpuCapacity;
|
||||||
|
return `${used} / ${capacity}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function MetricMemoryRatio(props: { metrics?: lib.SystemMetrics }) {
|
||||||
|
if (!props.metrics) {
|
||||||
|
return "N/A";
|
||||||
|
}
|
||||||
|
const ratio = props.metrics.memoryRatio;
|
||||||
|
return <Progress status="normal" percent={Math.ceil(ratio * 100)}/>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function MetricMemoryUsed(props: { metrics?: lib.SystemMetrics }) {
|
||||||
|
if (!props.metrics) {
|
||||||
|
return "N/A";
|
||||||
|
}
|
||||||
|
const used = lib.formatBytes(props.metrics.memoryUsed);
|
||||||
|
const capacity = lib.formatBytes(props.metrics.memoryCapacity);
|
||||||
|
return `${used} / ${capacity}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function MetricDiskRatio(props: { metrics?: lib.SystemMetrics }) {
|
||||||
|
if (!props.metrics) {
|
||||||
|
return "N/A";
|
||||||
|
}
|
||||||
|
const ratio = props.metrics.diskRatio;
|
||||||
|
return <Progress status="normal" percent={Math.ceil(ratio * 100)}/>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function MetricDiskUsed(props: { metrics?: lib.SystemMetrics }) {
|
||||||
|
if (!props.metrics) {
|
||||||
|
return "N/A";
|
||||||
|
}
|
||||||
|
const used = lib.formatBytes(props.metrics.diskUsed);
|
||||||
|
const capacity = lib.formatBytes(props.metrics.diskCapacity);
|
||||||
|
return `${used} / ${capacity}`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
import { useCallback, useContext, useSyncExternalStore } from "react";
|
||||||
|
import ControlContext from "../components/ControlContext";
|
||||||
|
|
||||||
|
export function useSystemMetrics() {
|
||||||
|
const control = useContext(ControlContext);
|
||||||
|
const subscribe = useCallback((callback: () => void) => control.systemMetrics.subscribe(callback), [control]);
|
||||||
|
return useSyncExternalStore(subscribe, () => control.systemMetrics.getSnapshot());
|
||||||
|
}
|
||||||
@@ -60,6 +60,7 @@ export class Control extends lib.Link {
|
|||||||
logHandlers: Map<lib.LogFilter, logHandler[]> = new Map();
|
logHandlers: Map<lib.LogFilter, logHandler[]> = new Map();
|
||||||
|
|
||||||
/** Updates handled by the subscription service */
|
/** Updates handled by the subscription service */
|
||||||
|
systemMetrics = new lib.EventSubscriber(lib.SystemMetricsUpdateEvent, this);
|
||||||
hosts = new lib.EventSubscriber(lib.HostUpdatesEvent, this);
|
hosts = new lib.EventSubscriber(lib.HostUpdatesEvent, this);
|
||||||
instances = new lib.EventSubscriber(lib.InstanceDetailsUpdatesEvent, this);
|
instances = new lib.EventSubscriber(lib.InstanceDetailsUpdatesEvent, this);
|
||||||
saves = new lib.EventSubscriber(lib.InstanceSaveDetailsUpdatesEvent, this);
|
saves = new lib.EventSubscriber(lib.InstanceSaveDetailsUpdatesEvent, this);
|
||||||
|
|||||||
Reference in new issue
Block a user