mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-23 00:13:13 +09:00
Add timeout to slave metric request
When the slave metrics request takes more than 10 seconds the response to the /metrics endpoint ends up being discarded by Prometheus which defaults to a timeout of 10 seconds. Fix by ignoring slaves that take longer than 8 seconds to respond to the metrics request.
This commit is contained in:
1 parent
a06781ebc3
commit
b742291131
3 files changed
+69
-2
No files matched your search
@@ -19,6 +19,41 @@ function basicType(value) {
|
|||||||
return typeof value;
|
return typeof value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Asynchronously wait for the given duration
|
||||||
|
*
|
||||||
|
* @param {number} duration - Time to wait for in milliseconds.
|
||||||
|
*/
|
||||||
|
async function wait(duration) {
|
||||||
|
await new Promise(resolve => { setTimeout(resolve, duration); });
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve a promise with a timeout.
|
||||||
|
*
|
||||||
|
* @param {Promise} promise - Promise to wait for.
|
||||||
|
* @param {number} time - Maximum time im milliseconds to wait for.
|
||||||
|
* @param {*=} timeoutResult - Value to return if the operation timed out.
|
||||||
|
*/
|
||||||
|
async function timeout(promise, time, timeoutResult) {
|
||||||
|
let timer;
|
||||||
|
try {
|
||||||
|
return await Promise.race([
|
||||||
|
promise,
|
||||||
|
new Promise(resolve => {
|
||||||
|
timer = setTimeout(() => resolve(timeoutResult), time);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
basicType,
|
basicType,
|
||||||
|
wait,
|
||||||
|
timeout,
|
||||||
};
|
};
|
||||||
@@ -54,6 +54,7 @@ const libPrometheus = require("@clusterio/lib/prometheus");
|
|||||||
const libConfig = require("@clusterio/lib/config");
|
const libConfig = require("@clusterio/lib/config");
|
||||||
const libUsers = require("@clusterio/lib/users");
|
const libUsers = require("@clusterio/lib/users");
|
||||||
const libSharedCommands = require("@clusterio/lib/shared_commands");
|
const libSharedCommands = require("@clusterio/lib/shared_commands");
|
||||||
|
const libHelpers = require("@clusterio/lib/helpers");
|
||||||
|
|
||||||
const express = require("express");
|
const express = require("express");
|
||||||
const compression = require("compression");
|
const compression = require("compression");
|
||||||
@@ -147,8 +148,7 @@ async function getMetrics(req, res, next) {
|
|||||||
|
|
||||||
let requests = [];
|
let requests = [];
|
||||||
for (let slaveConnection of slaveConnections.values()) {
|
for (let slaveConnection of slaveConnections.values()) {
|
||||||
// XXX this needs a timeout
|
requests.push(libHelpers.timeout(libLink.messages.getMetrics.send(slaveConnection), 8e3, null));
|
||||||
requests.push(libLink.messages.getMetrics.send(slaveConnection));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for await (let result of await libPrometheus.defaultRegistry.collect()) {
|
for await (let result of await libPrometheus.defaultRegistry.collect()) {
|
||||||
@@ -157,6 +157,11 @@ async function getMetrics(req, res, next) {
|
|||||||
|
|
||||||
let resultMap = new Map();
|
let resultMap = new Map();
|
||||||
for (let response of await Promise.all(requests)) {
|
for (let response of await Promise.all(requests)) {
|
||||||
|
if (!response) {
|
||||||
|
// TODO: Log timeout occured?
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
for (let result of response.results) {
|
for (let result of response.results) {
|
||||||
if (!resultMap.has(result.metric.name)) {
|
if (!resultMap.has(result.metric.name)) {
|
||||||
resultMap.set(result.metric.name, result);
|
resultMap.set(result.metric.name, result);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
"use strict";
|
"use strict";
|
||||||
const assert = require("assert").strict;
|
const assert = require("assert").strict;
|
||||||
|
const hrtime = process.hrtime.bigint;
|
||||||
|
|
||||||
const libHelpers = require("@clusterio/lib/helpers");
|
const libHelpers = require("@clusterio/lib/helpers");
|
||||||
|
|
||||||
@@ -15,4 +16,30 @@ describe("lib/helpers", function() {
|
|||||||
assert.equal(libHelpers.basicType({ a: 1 }), "object");
|
assert.equal(libHelpers.basicType({ a: 1 }), "object");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("wait", function() {
|
||||||
|
it("should wait the approximate amount of time given", async function() {
|
||||||
|
let startNs = hrtime();
|
||||||
|
await libHelpers.wait(100);
|
||||||
|
let duration = Number(hrtime() - startNs) / 1e6;
|
||||||
|
|
||||||
|
assert(duration > 90 && duration < 110, `duration waited (${duration}ms) 10% of expected (100ms)`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("timeout", function() {
|
||||||
|
it("should return the result of an already resolved promise", async function() {
|
||||||
|
assert.equal(await libHelpers.timeout(Promise.resolve("value"), 10, "timeout"), "value");
|
||||||
|
});
|
||||||
|
it("should return the result of the promise if received before timeout", async function() {
|
||||||
|
assert.equal(await libHelpers.timeout(
|
||||||
|
(async () => { await libHelpers.wait(10); return "value"; })(), 20, "timeout"
|
||||||
|
), "value");
|
||||||
|
});
|
||||||
|
it("should return the timeoutResult if not received before timeout", async function() {
|
||||||
|
assert.equal(await libHelpers.timeout(
|
||||||
|
(async () => { await libHelpers.wait(20); return "value"; })(), 10, "timeout"
|
||||||
|
), "timeout");
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
Reference in new issue
Block a user