Add timeout to slave metric request

When the slave metrics request takes more than 10 seconds the response
to the /metrics endpoint ends up being discarded by Prometheus which
defaults to a timeout of 10 seconds.  Fix by ignoring slaves that take
longer than 8 seconds to respond to the metrics request.
This commit is contained in:
Hornwitser committed 2020-11-17 04:19:05 +01:00
1 parent a06781ebc3
commit b742291131
3 files changed
+69 -2

No files matched your search

+7 -2
View File
@@ -54,6 +54,7 @@ const libPrometheus = require("@clusterio/lib/prometheus");
const libConfig = require("@clusterio/lib/config");
const libUsers = require("@clusterio/lib/users");
const libSharedCommands = require("@clusterio/lib/shared_commands");
const libHelpers = require("@clusterio/lib/helpers");
const express = require("express");
const compression = require("compression");
@@ -147,8 +148,7 @@ async function getMetrics(req, res, next) {
let requests = [];
for (let slaveConnection of slaveConnections.values()) {
// XXX this needs a timeout
requests.push(libLink.messages.getMetrics.send(slaveConnection));
requests.push(libHelpers.timeout(libLink.messages.getMetrics.send(slaveConnection), 8e3, null));
}
for await (let result of await libPrometheus.defaultRegistry.collect()) {
@@ -157,6 +157,11 @@ async function getMetrics(req, res, next) {
let resultMap = new Map();
for (let response of await Promise.all(requests)) {
if (!response) {
// TODO: Log timeout occured?
continue;
}
for (let result of response.results) {
if (!resultMap.has(result.metric.name)) {
resultMap.set(result.metric.name, result);