Require a token to scrape /metrics

The metrics endpoint fans out a gather to every connected host, so
anyone who could reach the controller could read cluster statistics and
put load on all hosts at will. Scraping now needs a user token holding
the new core.controller.metrics permission. The token is accepted via
Authorization: Bearer as well as X-Access-Token because Prometheus can
only send the former. Concurrent scrapes share one gather so hosts see
at most one metrics request at a time.

The three connection gauges in controller.ts now no-op before the
controller exists, like the other gauges there, so the endpoint can be
tested against the mock controller.

Closes #1009

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
bbassieandClaude Fable 5.1 committed 2026-09-12 22:28:43 +00:00
1 parent 2b981a5734
commit 87aa7b3b4c
6 files changed
+151 -10

No files matched your search

+25
View File
@@ -274,3 +274,28 @@ Remote updates are enabled by default but can be disabled on a per-machine basis
controller> config set controller.allow_plugin_install true/false controller> config set controller.allow_plugin_install true/false
Remote installs are disabled by default but can be enabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the above commands locally while the remote is offline. Remote installs are disabled by default but can be enabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the above commands locally while the remote is offline.
## Prometheus Metrics
The controller exports metrics for the whole cluster on the `/metrics` endpoint.
Fetching it requires a token for a user with the `core.controller.metrics` permission, sent either as an `Authorization: Bearer <token>` header or as an `X-Access-Token` header.
Requests without a valid token get a 401 response and requests for users lacking the permission get a 403 response.
Create a role with just this permission, a user to hold it, and generate a token for that user:
ctl> role create Prometheus --permissions core.controller.metrics
ctl> user create prometheus
ctl> user set-roles prometheus Prometheus
$ npx clusteriocontroller bootstrap generate-user-token prometheus
Then pass the token in the scrape job for the controller in `prometheus.yml`:
scrape_configs:
- job_name: clusterio
authorization:
credentials: <token>
static_configs:
- targets: ["localhost:8080"]
Concurrent scrapes share a single round of metrics gathering from the hosts, so multiple scrapers polling the controller do not multiply the load on the hosts.
+9
View File
@@ -101,6 +101,9 @@ void new lib.Gauge(
"clusterio_controller_websocket_active_connections", "clusterio_controller_websocket_active_connections",
"How many WebSocket connections are currently open to the controller", "How many WebSocket connections are currently open to the controller",
{ callback: function(gauge:lib.Gauge) { { callback: function(gauge:lib.Gauge) {
if (!controller) {
return;
}
gauge.set(controller.wsServer.activeConnectors.size); gauge.set(controller.wsServer.activeConnectors.size);
}} }}
); );
@@ -109,6 +112,9 @@ void new lib.Gauge(
"clusterio_controller_active_hosts", "clusterio_controller_active_hosts",
"How many hosts are currently connected to the controller", "How many hosts are currently connected to the controller",
{ callback: function(gauge:lib.Gauge) { { callback: function(gauge:lib.Gauge) {
if (!controller) {
return;
}
gauge.set(controller.wsServer.hostConnections.size); gauge.set(controller.wsServer.hostConnections.size);
}} }}
); );
@@ -118,6 +124,9 @@ void new lib.Gauge(
"How many clients are currently connected to this controller", "How many clients are currently connected to this controller",
{ {
labels: ["type"], callback: async function(gauge:lib.Gauge) { labels: ["type"], callback: async function(gauge:lib.Gauge) {
if (!controller) {
return;
}
gauge.labels("host").set(controller.wsServer.hostConnections.size); gauge.labels("host").set(controller.wsServer.hostConnections.size);
gauge.labels("control").set(controller.wsServer.controlConnections.size); gauge.labels("control").set(controller.wsServer.controlConnections.size);
}, },
+44 -9
View File
@@ -24,7 +24,8 @@ declare global {
mainBundle: string, mainBundle: string,
staticRoot: string | null, staticRoot: string | null,
devPlugins: Map<string, number>, devPlugins: Map<string, number>,
streams: Map<string, ProxyStream> streams: Map<string, ProxyStream>,
pendingMetrics?: Promise<string>,
} }
} }
} }
@@ -44,10 +45,8 @@ function mergeSamples(destinationResult: lib.CollectorResultSerialized, sourceRe
} }
} }
// Prometheus polling endpoint // Collect metrics from the controller, its plugins and all connected hosts
async function getMetrics(req: Request, res: Response, next: any) { async function gatherMetrics(controller: Controller) {
const controller: Controller = req.app.locals.controller;
let results: lib.CollectorResult[] = []; let results: lib.CollectorResult[] = [];
let pluginResults = await controller.hooks.metrics.collect(); let pluginResults = await controller.hooks.metrics.collect();
for (let metricIterator of pluginResults) { for (let metricIterator of pluginResults) {
@@ -99,7 +98,27 @@ async function getMetrics(req: Request, res: Response, next: any) {
results.push(lib.deserializeResult(result)); results.push(lib.deserializeResult(result));
} }
let text = await lib.exposition(results); return await lib.exposition(results);
}
// Prometheus polling endpoint
async function getMetrics(req: Request, res: Response) {
try {
res.locals.user.checkPermission("core.controller.metrics");
} catch (err: any) {
res.status(403).json({ request_errors: [err.message] });
return;
}
// Concurrent scrapes share one gather so hosts see at most one
// metrics request at a time regardless of how many clients poll.
const locals = req.app.locals;
if (!locals.pendingMetrics) {
locals.pendingMetrics = gatherMetrics(locals.controller).finally(() => {
locals.pendingMetrics = undefined;
});
}
const text = await locals.pendingMetrics;
res.set("Content-Type", lib.expositionContentType); res.set("Content-Type", lib.expositionContentType);
res.send(text); res.send(text);
} }
@@ -152,8 +171,21 @@ function getClusterName(req: Request, res: Response) {
res.json({ name: clusterName }); res.json({ name: clusterName });
} }
// Reads the access token from X-Access-Token or Authorization: Bearer
function getAccessToken(req: Request) {
const token = req.header("x-access-token");
if (token) {
return token;
}
const authorization = req.header("authorization");
if (authorization && /^Bearer /i.test(authorization)) {
return authorization.slice("Bearer ".length).trim();
}
return undefined;
}
function validateHostToken(req: Request, res: Response, next: any) { function validateHostToken(req: Request, res: Response, next: any) {
let token = req.header("x-access-token"); let token = getAccessToken(req);
if (!token) { if (!token) {
res.sendStatus(401); res.sendStatus(401);
return; return;
@@ -180,7 +212,7 @@ function validateHostToken(req: Request, res: Response, next: any) {
} }
function validateUserToken(req: Request, res: Response, next: any) { function validateUserToken(req: Request, res: Response, next: any) {
let token = req.header("x-access-token"); let token = getAccessToken(req);
if (!token) { if (!token) {
res.sendStatus(401); res.sendStatus(401);
return; return;
@@ -653,7 +685,10 @@ async function uploadMod(req: Request, res: Response) {
export function addRouteHandlers(app: Application) { export function addRouteHandlers(app: Application) {
app.get("/metrics", (req:Request, res:Response, next:any) => getMetrics(req, res, next).catch(next)); app.get("/metrics",
validateUserToken,
(req:Request, res:Response, next:any) => getMetrics(req, res).catch(next)
);
app.get("/api/plugins", getPlugins); app.get("/api/plugins", getPlugins);
app.get("/api/cluster-name", getClusterName); app.get("/api/cluster-name", getClusterName);
app.put("/api/upload-export", app.put("/api/upload-export",
+5
View File
@@ -144,6 +144,11 @@ export const corePermissions = [
title: "Modify controller config", title: "Modify controller config",
description: "Modify the controller config or entries of the controller config.", description: "Modify the controller config or entries of the controller config.",
}, },
{
name: "core.controller.metrics",
title: "Scrape metrics",
description: "Fetch the Prometheus metrics for the cluster from the controller's /metrics endpoint.",
},
{ {
name: "core.system.subscribe", name: "core.system.subscribe",
title: "Subscribe to system updates", title: "Subscribe to system updates",
+66 -1
View File
@@ -3,7 +3,7 @@ import events from "node:events";
import FormData from "form-data"; import FormData from "form-data";
import http from "node:http"; import http from "node:http";
import { wait } from "@clusterio/lib"; import { wait, expositionContentType } from "@clusterio/lib";
import * as routes from "@clusterio/controller/dist/node/src/routes.js"; import * as routes from "@clusterio/controller/dist/node/src/routes.js";
import * as mock from "../mock.js"; import * as mock from "../mock.js";
@@ -72,6 +72,71 @@ describe("controller/src/routes", function() {
assert.equal(await responses[1].text(), "test content"); assert.equal(await responses[1].text(), "test content");
}); });
}); });
describe("/metrics", function() {
let endpoint, testToken;
beforeEach(function() {
endpoint = `http://localhost:${port}/metrics`;
testToken = controller.users.signUserToken({ id: "test" });
});
it("should respond with 401 without a valid token", async function() {
let response;
response = await fetch(endpoint);
assert.equal(response.status, 401);
response = await fetch(endpoint, { headers: { "X-Access-Token": "invalid" } });
assert.equal(response.status, 401);
response = await fetch(endpoint, { headers: { "Authorization": "Bearer invalid" } });
assert.equal(response.status, 401);
response = await fetch(endpoint, { headers: { "Authorization": `Basic ${testToken}` } });
assert.equal(response.status, 401);
const user = controller.users.getOrCreateUser("test");
user.tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000);
user.saveRecord();
response = await fetch(endpoint, { headers: { "Authorization": `Bearer ${testToken}` } });
assert.equal(response.status, 401);
});
it("should respond with 403 when user has insufficient permission", async function() {
const user = controller.users.getOrCreateUser("player");
let response = await fetch(endpoint, {
headers: { "Authorization": `Bearer ${controller.users.signUserToken(user)}` },
});
assert.equal(response.status, 403);
});
it("should respond with metrics to an authorized user", async function() {
for (const headers of [
{ "X-Access-Token": testToken },
{ "Authorization": `Bearer ${testToken}` },
{ "Authorization": `bearer ${testToken}` },
]) {
let response = await fetch(endpoint, { headers });
assert.equal(response.status, 200);
const contentType = response.headers.get("content-type");
for (const part of expositionContentType.split("; ")) {
assert(contentType.includes(part), `expected ${part} in content type ${contentType}`);
}
assert(/^# HELP /m.test(await response.text()), "expected exposition text");
}
});
it("should share one gather between concurrent requests", async function() {
let calls = 0;
controller.hooks.metrics.attach("test", async () => {
calls += 1;
await wait(100);
});
const headers = { "X-Access-Token": testToken };
let responses = await Promise.all([
fetch(endpoint, { headers }),
fetch(endpoint, { headers }),
fetch(endpoint, { headers }),
]);
for (const response of responses) {
assert.equal(response.status, 200);
}
assert.equal(calls, 1);
let response = await fetch(endpoint, { headers });
assert.equal(response.status, 200);
assert.equal(calls, 2);
});
});
describe("stripStaticPrefix()", function() { describe("stripStaticPrefix()", function() {
it("should remove a leading static/", function() { it("should remove a leading static/", function() {
assert.equal(routes.stripStaticPrefix("static/main.abc.js"), "main.abc.js"); assert.equal(routes.stripStaticPrefix("static/main.abc.js"), "main.abc.js");
+2
View File
@@ -165,6 +165,7 @@ export class MockController {
["controller.public_url", "test"], ["controller.public_url", "test"],
["controller.auth_secret", "TestSecretDoNotUse"], ["controller.auth_secret", "TestSecretDoNotUse"],
["controller.proxy_stream_timeout", 1], ["controller.proxy_stream_timeout", 1],
["controller.metrics_timeout", 1],
]); ]);
this.config = { this.config = {
get: (name) => { get: (name) => {
@@ -199,6 +200,7 @@ export class MockController {
this.handles = new Map(); this.handles = new Map();
this.hooks = new ControllerHooks(new MockLogger()); this.hooks = new ControllerHooks(new MockLogger());
this.loadedPlugins = new Set(); this.loadedPlugins = new Set();
this.wsServer = { hostConnections: new Map() };
} }
get authSecret() { get authSecret() {