From d7edae99c5932b8f14c5fc210810efdc42138168 Mon Sep 17 00:00:00 2001
From: Hornwitser <github@hornwitser.no>
Date: Mon, 22 Jan 2024 19:29:36 +0100
Subject: [PATCH] Add metrics for pending request

Track the number of requests currently awaiting a response from a
request handler both from interally issued requests and requests
forwarded from other parties.
---
 CHANGELOG.md                      |  3 ++-
 packages/controller/controller.ts | 18 ++++++++++++++++++
 packages/host/host.ts             | 31 ++++++++++++++++++++++++++-----
 packages/lib/src/link/link.ts     |  7 +++++++
 4 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca5d4e2f..4dd4be7c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,8 @@
 
 ### Features
 
-- Added support for `color-settings` Mod settings (#568)[https://github.com/clusterio/clusterio/issues/568].
+- Added support for `color-settings` Mod settings [#568](https://github.com/clusterio/clusterio/issues/568).
+- Added `clusterio_controller_pending_requests` and `clusterio_host_pending_requests` metrics.
 
 ### Changes
 
diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts
index 14b18e46..b4161f65 100644
--- a/packages/controller/controller.ts
+++ b/packages/controller/controller.ts
@@ -37,6 +37,24 @@ import { version } from "./package.json";
 // globals
 let controller: Controller;
 
+void new lib.Gauge(
+	"clusterio_controller_pending_requests",
+	"Count of pending link requests currently waiting in memory on the controller.",
+	{
+		callback: (gauge) => {
+			let count = 0;
+			if (controller) {
+				for (const host of controller.wsServer.hostConnections.values()) {
+					count += host.pendingRequestCount;
+				}
+				for (const control of controller.wsServer.controlConnections.values()) {
+					count += control.pendingRequestCount;
+				}
+			}
+			gauge.set(count);
+		},
+	}
+);
 
 void new lib.Gauge(
 	"clusterio_controller_host_mapping",
diff --git a/packages/host/host.ts b/packages/host/host.ts
index 46827d85..fa195675 100755
--- a/packages/host/host.ts
+++ b/packages/host/host.ts
@@ -26,6 +26,27 @@ import { ConsoleTransport, levels, logger } from "@clusterio/lib";
 
 import Host from "./src/Host";
 
+let host: Host | undefined;
+
+void new lib.Gauge(
+	"clusterio_host_pending_requests",
+	"Count of pending link requests currently waiting in memory on the host.",
+	{
+		labels: ["host_id"],
+		callback: (gauge) => {
+			if (!host) {
+				return;
+			}
+
+			let count = host.pendingRequestCount;
+			for (const connection of host.instanceConnections.values()) {
+				count += connection.pendingRequestCount;
+				count += connection.instance.pendingRequestCount;
+			}
+			gauge.labels(String(host.config.get("host.id"))).set(count);
+		},
+	}
+);
 
 export class HostConnector extends lib.WebSocketClientConnector {
 	constructor(
@@ -205,7 +226,7 @@ async function startHost() {
 	}
 
 	let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos);
-	let host = new Host(
+	host = new Host(
 		hostConnector,
 		args.config,
 		hostConfig,
@@ -227,7 +248,7 @@ async function startHost() {
 
 		secondSigint = true;
 		logger.info("Caught interrupt signal, shutting down");
-		host.shutdown();
+		host!.shutdown();
 	});
 	let secondSigterm = false;
 	process.on("SIGTERM", () => {
@@ -240,11 +261,11 @@ async function startHost() {
 
 		secondSigterm = true;
 		logger.info("Caught termination signal, shutting down");
-		host.shutdown();
+		host!.shutdown();
 	});
 	process.on("SIGHUP", () => {
 		logger.info("Terminal closed, shutting down");
-		host.shutdown();
+		host!.shutdown();
 	});
 
 	try {
@@ -255,7 +276,7 @@ async function startHost() {
 	}
 
 	hostConnector.once("connect", () => {
-		logger.add(new lib.LinkTransport({ link: host }));
+		logger.add(new lib.LinkTransport({ link: host! }));
 	});
 
 	await hostConnector.connect();
diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts
index deabd028..c7365190 100644
--- a/packages/lib/src/link/link.ts
+++ b/packages/lib/src/link/link.ts
@@ -141,6 +141,13 @@ export class Link {
 		});
 	}
 
+	/**
+	 * Count of requests currently waiting for a response on this link
+	 */
+	get pendingRequestCount() {
+		return this._pendingRequests.size + this._forwardedRequests.size;
+	}
+
 	_clearPendingRequests(err: Error & { code?: string }) {
 		for (let pending of this._pendingRequests.values()) {
 			pending.reject(err);