Add timeout to slave metric request

When the slave metrics request takes more than 10 seconds the response
to the /metrics endpoint ends up being discarded by Prometheus which
defaults to a timeout of 10 seconds.  Fix by ignoring slaves that take
longer than 8 seconds to respond to the metrics request.
This commit is contained in:
Hornwitser committed 2020-11-17 04:19:05 +01:00
1 parent a06781ebc3
commit b742291131
3 files changed
+69 -2

No files matched your search

+35
View File
@@ -19,6 +19,41 @@ function basicType(value) {
return typeof value;
}
/**
* Asynchronously wait for the given duration
*
* @param {number} duration - Time to wait for in milliseconds.
*/
async function wait(duration) {
await new Promise(resolve => { setTimeout(resolve, duration); });
}
/**
* Resolve a promise with a timeout.
*
* @param {Promise} promise - Promise to wait for.
* @param {number} time - Maximum time im milliseconds to wait for.
* @param {*=} timeoutResult - Value to return if the operation timed out.
*/
async function timeout(promise, time, timeoutResult) {
let timer;
try {
return await Promise.race([
promise,
new Promise(resolve => {
timer = setTimeout(() => resolve(timeoutResult), time);
}),
]);
} finally {
clearTimeout(timer);
}
}
module.exports = {
basicType,
wait,
timeout,
};
+7 -2
View File
@@ -54,6 +54,7 @@ const libPrometheus = require("@clusterio/lib/prometheus");
const libConfig = require("@clusterio/lib/config");
const libUsers = require("@clusterio/lib/users");
const libSharedCommands = require("@clusterio/lib/shared_commands");
const libHelpers = require("@clusterio/lib/helpers");
const express = require("express");
const compression = require("compression");
@@ -147,8 +148,7 @@ async function getMetrics(req, res, next) {
let requests = [];
for (let slaveConnection of slaveConnections.values()) {
// XXX this needs a timeout
requests.push(libLink.messages.getMetrics.send(slaveConnection));
requests.push(libHelpers.timeout(libLink.messages.getMetrics.send(slaveConnection), 8e3, null));
}
for await (let result of await libPrometheus.defaultRegistry.collect()) {
@@ -157,6 +157,11 @@ async function getMetrics(req, res, next) {
let resultMap = new Map();
for (let response of await Promise.all(requests)) {
if (!response) {
// TODO: Log timeout occured?
continue;
}
for (let result of response.results) {
if (!resultMap.has(result.metric.name)) {
resultMap.set(result.metric.name, result);