Add controller metrics for WebSocket session events (#1199)

Counts ready, resume, drop and session timeout per connection type
(host or control), plus resume attempts rejected for an unknown or
expired session. Lets a dashboard show hosts or users reconnecting.

Fixes #1192

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

packages/controller/src/WsServer.ts

@@ -30,6 +30,42 @@ const wsRejectedConnectionsCounter = new lib.Counter(
 	"How many WebSocket connections have been rejected during the handshake on the controller"
 );
 
+const wsReadyCounter = new lib.Counter(
+	"clusterio_controller_websocket_ready_total",
+	"How many WebSocket sessions have been established on the controller",
+	{ labels: ["type"] }
+);
+
+const wsResumeCounter = new lib.Counter(
+	"clusterio_controller_websocket_resume_total",
+	"How many WebSocket sessions have been resumed on the controller",
+	{ labels: ["type"] }
+);
+
+const wsDropCounter = new lib.Counter(
+	"clusterio_controller_websocket_drop_total",
+	"How many WebSocket sessions have lost their connection without closing on the controller",
+	{ labels: ["type"] }
+);
+
+const wsSessionTimeoutCounter = new lib.Counter(
+	"clusterio_controller_websocket_session_timeout_total",
+	"How many WebSocket sessions have timed out waiting for a resume on the controller",
+	{ labels: ["type"] }
+);
+
+const wsFailedResumeCounter = new lib.Counter(
+	"clusterio_controller_websocket_failed_resume_total",
+	"How many WebSocket resume attempts have been rejected due to an unknown or expired session on the controller"
+);
+
+for (const type of ["host", "control"]) {
+	wsReadyCounter.labels(type);
+	wsResumeCounter.labels(type);
+	wsDropCounter.labels(type);
+	wsSessionTimeoutCounter.labels(type);
+}
+
 /**
  * WebSocket server
  * @alias module:controller/src/WsServer
@@ -240,6 +276,12 @@ ${err.stack}`
 			this.activeConnectors.delete(sessionId);
 		});
 
+		const type = dst.type === lib.Address.host ? "host" : "control";
+		connector.on("connect", () => wsReadyCounter.labels(type).inc());
+		connector.on("resume", () => wsResumeCounter.labels(type).inc());
+		connector.on("drop", () => wsDropCounter.labels(type).inc());
+		connector.on("timeout", () => wsSessionTimeoutCounter.labels(type).inc());
+
 		return [connector, sessionToken];
 	}
 
@@ -367,6 +409,7 @@ ${err.stack}`
 			}
 
 		} catch (err) {
+			wsFailedResumeCounter.inc();
 			socket.send(JSON.stringify(new lib.MessageInvalidate()));
 			this.attachHandler(socket, req);
 			return;

packages/controller/src/WsServerConnector.ts

@@ -7,13 +7,17 @@ import * as lib from "@clusterio/lib";
 import { logger } from "@clusterio/lib";
 
 
+type WsServerConnectorEvents = {
+	"timeout": [],
+};
+
 /**
  * Connector for controller connections
  *
  * @extends module:lib.WebSocketBaseConnector
  * @alias module:controller/src/WsServerConnector
  */
-export default class WsServerConnector extends lib.WebSocketBaseConnector {
+export default class WsServerConnector extends lib.WebSocketBaseConnector<WsServerConnectorEvents> {
 	_timeoutId: ReturnType<typeof setTimeout> | null = null;
 
 	constructor(
@@ -119,6 +123,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 
 	_timedOut() {
 		logger.verbose(`Connector | Connection to ${this.dst} timed out`);
+		this.emit("timeout");
 		this._close();
 	}
 

test/controller/WsServer.js

@@ -113,6 +113,58 @@ describe("controller/src/WsServer", function() {
 			});
 		});
 	});
+	describe("connection metrics", function() {
+		const req = { socket: { remoteAddress: "10.0.0.1", remoteFamily: "IPv4" }, headers: {} };
+		/** @type {WsServer} */
+		let wsServer;
+		before(function() {
+			const config = new ControllerConfig("controller");
+			config.set("controller.auth_secret", Buffer.from("TestSecretDoNotUse").toString("base64"));
+			wsServer = new Controller(lib.logger, [], config).wsServer;
+		});
+
+		function counter(name) {
+			return lib.defaultRegistry.collectors.find(collector => collector.metric.name === name);
+		}
+		function value(name, type) {
+			const collector = counter(`clusterio_controller_websocket_${name}_total`);
+			return type ? collector.labels(type).get() : collector.get();
+		}
+
+		[["host", lib.Address.host], ["control", lib.Address.control]].forEach(([type, addressType]) => {
+			it(`should count ready, drop, resume and timeout for ${type} sessions`, function() {
+				const before = Object.fromEntries(
+					["ready", "drop", "resume", "session_timeout"].map(name => [name, value(name, type)])
+				);
+				const [connector, sessionToken] = wsServer.createSession(new lib.Address(addressType, 1));
+				connector.ready(new MockWebSocket(), new lib.Address(addressType, 1), sessionToken, undefined);
+				assert.equal(value("ready", type), before.ready + 1);
+
+				connector._socket.terminate();
+				assert.equal(value("drop", type), before.drop + 1);
+
+				wsServer.resumeSession(new lib.ResumeData(sessionToken, 0), new MockWebSocket(), req);
+				assert.equal(value("resume", type), before.resume + 1);
+
+				connector._socket.terminate();
+				connector._timedOut();
+				assert.equal(value("session_timeout", type), before.session_timeout + 1);
+				assert.equal(value("drop", type), before.drop + 2);
+				assert.equal(value("ready", type), before.ready + 1);
+			});
+		});
+		it("should count resumes of unknown sessions as failed", function() {
+			const before = value("failed_resume");
+			const socket = new MockWebSocket();
+			wsServer.resumeSession(new lib.ResumeData("invalid", 0), socket, req);
+			try {
+				assert.equal(value("failed_resume"), before + 1);
+				assert.equal(socket.sent[0].type, "invalidate");
+			} finally {
+				socket.emit("message", "not json"); // ends the handshake and its timeout
+			}
+		});
+	});
 	describe(".registerHost()", function() {
 		const secret = Buffer.from("TestSecretDoNotUse");
 		const req = { socket: { remoteAddress: "10.0.0.1", remoteFamily: "IPv4" }, headers: {} };