Add timeout to slave metric request

When the slave metrics request takes more than 10 seconds the response
to the /metrics endpoint ends up being discarded by Prometheus which
defaults to a timeout of 10 seconds.  Fix by ignoring slaves that take
longer than 8 seconds to respond to the metrics request.

packages/lib/helpers.js

@@ -19,6 +19,41 @@ function basicType(value) {
 	return typeof value;
 }
 
+
+/**
+ * Asynchronously wait for the given duration
+ *
+ * @param {number} duration - Time to wait for in milliseconds.
+ */
+async function wait(duration) {
+	await new Promise(resolve => { setTimeout(resolve, duration); });
+}
+
+
+/**
+ * Resolve a promise with a timeout.
+ *
+ * @param {Promise} promise - Promise to wait for.
+ * @param {number} time - Maximum time im milliseconds to wait for.
+ * @param {*=} timeoutResult - Value to return if the operation timed out.
+ */
+async function timeout(promise, time, timeoutResult) {
+	let timer;
+	try {
+		return await Promise.race([
+			promise,
+			new Promise(resolve => {
+				timer = setTimeout(() => resolve(timeoutResult), time);
+			}),
+		]);
+
+	} finally {
+		clearTimeout(timer);
+	}
+}
+
 module.exports = {
 	basicType,
+	wait,
+	timeout,
 };

packages/master/master.js

@@ -54,6 +54,7 @@ const libPrometheus = require("@clusterio/lib/prometheus");
 const libConfig = require("@clusterio/lib/config");
 const libUsers = require("@clusterio/lib/users");
 const libSharedCommands = require("@clusterio/lib/shared_commands");
+const libHelpers = require("@clusterio/lib/helpers");
 
 const express = require("express");
 const compression = require("compression");
@@ -147,8 +148,7 @@ async function getMetrics(req, res, next) {
 
 	let requests = [];
 	for (let slaveConnection of slaveConnections.values()) {
-		// XXX this needs a timeout
-		requests.push(libLink.messages.getMetrics.send(slaveConnection));
+		requests.push(libHelpers.timeout(libLink.messages.getMetrics.send(slaveConnection), 8e3, null));
 	}
 
 	for await (let result of await libPrometheus.defaultRegistry.collect()) {
@@ -157,6 +157,11 @@ async function getMetrics(req, res, next) {
 
 	let resultMap = new Map();
 	for (let response of await Promise.all(requests)) {
+		if (!response) {
+			// TODO: Log timeout occured?
+			continue;
+		}
+
 		for (let result of response.results) {
 			if (!resultMap.has(result.metric.name)) {
 				resultMap.set(result.metric.name, result);

test/lib/helpers.js

@@ -1,5 +1,6 @@
 "use strict";
 const assert = require("assert").strict;
+const hrtime = process.hrtime.bigint;
 
 const libHelpers = require("@clusterio/lib/helpers");
 
@@ -15,4 +16,30 @@ describe("lib/helpers", function() {
 			assert.equal(libHelpers.basicType({ a: 1 }), "object");
 		});
 	});
+
+	describe("wait", function() {
+		it("should wait the approximate amount of time given", async function() {
+			let startNs = hrtime();
+			await libHelpers.wait(100);
+			let duration = Number(hrtime() - startNs) / 1e6;
+
+			assert(duration > 90 && duration < 110, `duration waited (${duration}ms) 10% of expected (100ms)`);
+		});
+	});
+
+	describe("timeout", function() {
+		it("should return the result of an already resolved promise", async function() {
+			assert.equal(await libHelpers.timeout(Promise.resolve("value"), 10, "timeout"), "value");
+		});
+		it("should return the result of the promise if received before timeout", async function() {
+			assert.equal(await libHelpers.timeout(
+				(async () => { await libHelpers.wait(10); return "value"; })(), 20, "timeout"
+			), "value");
+		});
+		it("should return the timeoutResult if not received before timeout", async function() {
+			assert.equal(await libHelpers.timeout(
+				(async () => { await libHelpers.wait(20); return "value"; })(), 10, "timeout"
+			), "timeout");
+		});
+	});
 });