From 484cffc13656551df25561c42137ef517154a3b3 Mon Sep 17 00:00:00 2001 From: Bastiaan <17990055+bbassie@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:37:13 +0100 Subject: [PATCH] Add controller metrics for WebSocket session events (#1199) Counts ready, resume, drop and session timeout per connection type (host or control), plus resume attempts rejected for an unknown or expired session. Lets a dashboard show hosts or users reconnecting. Fixes #1192 Co-authored-by: Claude Opus 5.5 --- packages/controller/src/WsServer.ts | 43 ++++++++++++++++ packages/controller/src/WsServerConnector.ts | 7 ++- test/controller/WsServer.js | 52 ++++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/controller/src/WsServer.ts b/packages/controller/src/WsServer.ts index 7fcc85b0..3db50248 100644 --- a/packages/controller/src/WsServer.ts +++ b/packages/controller/src/WsServer.ts @@ -30,6 +30,42 @@ const wsRejectedConnectionsCounter = new lib.Counter( "How many WebSocket connections have been rejected during the handshake on the controller" ); +const wsReadyCounter = new lib.Counter( + "clusterio_controller_websocket_ready_total", + "How many WebSocket sessions have been established on the controller", + { labels: ["type"] } +); + +const wsResumeCounter = new lib.Counter( + "clusterio_controller_websocket_resume_total", + "How many WebSocket sessions have been resumed on the controller", + { labels: ["type"] } +); + +const wsDropCounter = new lib.Counter( + "clusterio_controller_websocket_drop_total", + "How many WebSocket sessions have lost their connection without closing on the controller", + { labels: ["type"] } +); + +const wsSessionTimeoutCounter = new lib.Counter( + "clusterio_controller_websocket_session_timeout_total", + "How many WebSocket sessions have timed out waiting for a resume on the controller", + { labels: ["type"] } +); + +const wsFailedResumeCounter = new lib.Counter( + "clusterio_controller_websocket_failed_resume_total", + "How many WebSocket resume attempts have been rejected due to an unknown or expired session on the controller" +); + +for (const type of ["host", "control"]) { + wsReadyCounter.labels(type); + wsResumeCounter.labels(type); + wsDropCounter.labels(type); + wsSessionTimeoutCounter.labels(type); +} + /** * WebSocket server * @alias module:controller/src/WsServer @@ -240,6 +276,12 @@ ${err.stack}` this.activeConnectors.delete(sessionId); }); + const type = dst.type === lib.Address.host ? "host" : "control"; + connector.on("connect", () => wsReadyCounter.labels(type).inc()); + connector.on("resume", () => wsResumeCounter.labels(type).inc()); + connector.on("drop", () => wsDropCounter.labels(type).inc()); + connector.on("timeout", () => wsSessionTimeoutCounter.labels(type).inc()); + return [connector, sessionToken]; } @@ -367,6 +409,7 @@ ${err.stack}` } } catch (err) { + wsFailedResumeCounter.inc(); socket.send(JSON.stringify(new lib.MessageInvalidate())); this.attachHandler(socket, req); return; diff --git a/packages/controller/src/WsServerConnector.ts b/packages/controller/src/WsServerConnector.ts index 3793a651..7b8fcff6 100644 --- a/packages/controller/src/WsServerConnector.ts +++ b/packages/controller/src/WsServerConnector.ts @@ -7,13 +7,17 @@ import * as lib from "@clusterio/lib"; import { logger } from "@clusterio/lib"; +type WsServerConnectorEvents = { + "timeout": [], +}; + /** * Connector for controller connections * * @extends module:lib.WebSocketBaseConnector * @alias module:controller/src/WsServerConnector */ -export default class WsServerConnector extends lib.WebSocketBaseConnector { +export default class WsServerConnector extends lib.WebSocketBaseConnector { _timeoutId: ReturnType | null = null; constructor( @@ -119,6 +123,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector { _timedOut() { logger.verbose(`Connector | Connection to ${this.dst} timed out`); + this.emit("timeout"); this._close(); } diff --git a/test/controller/WsServer.js b/test/controller/WsServer.js index f6fc513f..a7a16f98 100644 --- a/test/controller/WsServer.js +++ b/test/controller/WsServer.js @@ -113,6 +113,58 @@ describe("controller/src/WsServer", function() { }); }); }); + describe("connection metrics", function() { + const req = { socket: { remoteAddress: "10.0.0.1", remoteFamily: "IPv4" }, headers: {} }; + /** @type {WsServer} */ + let wsServer; + before(function() { + const config = new ControllerConfig("controller"); + config.set("controller.auth_secret", Buffer.from("TestSecretDoNotUse").toString("base64")); + wsServer = new Controller(lib.logger, [], config).wsServer; + }); + + function counter(name) { + return lib.defaultRegistry.collectors.find(collector => collector.metric.name === name); + } + function value(name, type) { + const collector = counter(`clusterio_controller_websocket_${name}_total`); + return type ? collector.labels(type).get() : collector.get(); + } + + [["host", lib.Address.host], ["control", lib.Address.control]].forEach(([type, addressType]) => { + it(`should count ready, drop, resume and timeout for ${type} sessions`, function() { + const before = Object.fromEntries( + ["ready", "drop", "resume", "session_timeout"].map(name => [name, value(name, type)]) + ); + const [connector, sessionToken] = wsServer.createSession(new lib.Address(addressType, 1)); + connector.ready(new MockWebSocket(), new lib.Address(addressType, 1), sessionToken, undefined); + assert.equal(value("ready", type), before.ready + 1); + + connector._socket.terminate(); + assert.equal(value("drop", type), before.drop + 1); + + wsServer.resumeSession(new lib.ResumeData(sessionToken, 0), new MockWebSocket(), req); + assert.equal(value("resume", type), before.resume + 1); + + connector._socket.terminate(); + connector._timedOut(); + assert.equal(value("session_timeout", type), before.session_timeout + 1); + assert.equal(value("drop", type), before.drop + 2); + assert.equal(value("ready", type), before.ready + 1); + }); + }); + it("should count resumes of unknown sessions as failed", function() { + const before = value("failed_resume"); + const socket = new MockWebSocket(); + wsServer.resumeSession(new lib.ResumeData("invalid", 0), socket, req); + try { + assert.equal(value("failed_resume"), before + 1); + assert.equal(socket.sent[0].type, "invalidate"); + } finally { + socket.emit("message", "not json"); // ends the handshake and its timeout + } + }); + }); describe(".registerHost()", function() { const secret = Buffer.from("TestSecretDoNotUse"); const req = { socket: { remoteAddress: "10.0.0.1", remoteFamily: "IPv4" }, headers: {} };