diff --git a/docs/managing-a-cluster.md b/docs/managing-a-cluster.md index aa3e8587..54dc1ce8 100644 --- a/docs/managing-a-cluster.md +++ b/docs/managing-a-cluster.md @@ -274,3 +274,29 @@ Remote updates are enabled by default but can be disabled on a per-machine basis controller> config set controller.allow_plugin_install true/false Remote installs are disabled by default but can be enabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the above commands locally while the remote is offline. + + +## Prometheus Metrics + +The controller exports metrics for the whole cluster on the `/metrics` endpoint. +Fetching it requires a token for a user with the `core.controller.metrics` permission, sent either as an `Authorization: Bearer ` header or as an `X-Access-Token` header. +Requests without a valid token get a 401 response and requests for users lacking the permission get a 403 response. + +Create a role with just this permission, a user to hold it, and generate a token for that user. +Give the user a name that is not a valid Factorio player name, such as one starting with `#`, so a player joining the cluster can never claim the account and its token: + + ctl> role create Prometheus --permissions core.controller.metrics + ctl> user create #prometheus + ctl> user set-roles #prometheus Prometheus + $ npx clusteriocontroller bootstrap generate-user-token '#prometheus' + +Then pass the token in the scrape job for the controller in `prometheus.yml`: + + scrape_configs: + - job_name: clusterio + authorization: + credentials: + static_configs: + - targets: ["localhost:8080"] + +Concurrent scrapes share a single round of metrics gathering from the hosts, so multiple scrapers polling the controller do not multiply the load on the hosts. diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index a61a0080..7e38e29f 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -101,6 +101,9 @@ void new lib.Gauge( "clusterio_controller_websocket_active_connections", "How many WebSocket connections are currently open to the controller", { callback: function(gauge:lib.Gauge) { + if (!controller) { + return; + } gauge.set(controller.wsServer.activeConnectors.size); }} ); @@ -109,6 +112,9 @@ void new lib.Gauge( "clusterio_controller_active_hosts", "How many hosts are currently connected to the controller", { callback: function(gauge:lib.Gauge) { + if (!controller) { + return; + } gauge.set(controller.wsServer.hostConnections.size); }} ); @@ -118,6 +124,9 @@ void new lib.Gauge( "How many clients are currently connected to this controller", { labels: ["type"], callback: async function(gauge:lib.Gauge) { + if (!controller) { + return; + } gauge.labels("host").set(controller.wsServer.hostConnections.size); gauge.labels("control").set(controller.wsServer.controlConnections.size); }, diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 993f16b9..94531b72 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -24,7 +24,8 @@ declare global { mainBundle: string, staticRoot: string | null, devPlugins: Map, - streams: Map + streams: Map, + pendingMetrics?: Promise, } } } @@ -44,10 +45,8 @@ function mergeSamples(destinationResult: lib.CollectorResultSerialized, sourceRe } } -// Prometheus polling endpoint -async function getMetrics(req: Request, res: Response, next: any) { - const controller: Controller = req.app.locals.controller; - +// Collect metrics from the controller, its plugins and all connected hosts +async function gatherMetrics(controller: Controller) { let results: lib.CollectorResult[] = []; let pluginResults = await controller.hooks.metrics.collect(); for (let metricIterator of pluginResults) { @@ -99,7 +98,27 @@ async function getMetrics(req: Request, res: Response, next: any) { results.push(lib.deserializeResult(result)); } - let text = await lib.exposition(results); + return await lib.exposition(results); +} + +// Prometheus polling endpoint +async function getMetrics(req: Request, res: Response) { + try { + res.locals.user.checkPermission("core.controller.metrics"); + } catch (err: any) { + res.status(403).json({ request_errors: [err.message] }); + return; + } + + // Concurrent scrapes share one gather so hosts see at most one + // metrics request at a time regardless of how many clients poll. + const locals = req.app.locals; + if (!locals.pendingMetrics) { + locals.pendingMetrics = gatherMetrics(locals.controller).finally(() => { + locals.pendingMetrics = undefined; + }); + } + const text = await locals.pendingMetrics; res.set("Content-Type", lib.expositionContentType); res.send(text); } @@ -152,8 +171,21 @@ function getClusterName(req: Request, res: Response) { res.json({ name: clusterName }); } +// Reads the access token from X-Access-Token or Authorization: Bearer +function getAccessToken(req: Request) { + const token = req.header("x-access-token"); + if (token) { + return token; + } + const authorization = req.header("authorization"); + if (authorization && /^Bearer /i.test(authorization)) { + return authorization.slice("Bearer ".length).trim(); + } + return undefined; +} + function validateHostToken(req: Request, res: Response, next: any) { - let token = req.header("x-access-token"); + let token = getAccessToken(req); if (!token) { res.sendStatus(401); return; @@ -180,7 +212,7 @@ function validateHostToken(req: Request, res: Response, next: any) { } function validateUserToken(req: Request, res: Response, next: any) { - let token = req.header("x-access-token"); + let token = getAccessToken(req); if (!token) { res.sendStatus(401); return; @@ -653,7 +685,10 @@ async function uploadMod(req: Request, res: Response) { export function addRouteHandlers(app: Application) { - app.get("/metrics", (req:Request, res:Response, next:any) => getMetrics(req, res, next).catch(next)); + app.get("/metrics", + validateUserToken, + (req:Request, res:Response, next:any) => getMetrics(req, res).catch(next) + ); app.get("/api/plugins", getPlugins); app.get("/api/cluster-name", getClusterName); app.put("/api/upload-export", diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index 2c9abed0..89b5ccf0 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -144,6 +144,11 @@ export const corePermissions = [ title: "Modify controller config", description: "Modify the controller config or entries of the controller config.", }, + { + name: "core.controller.metrics", + title: "Scrape metrics", + description: "Fetch the Prometheus metrics for the cluster from the controller's /metrics endpoint.", + }, { name: "core.system.subscribe", title: "Subscribe to system updates", diff --git a/test/controller/routes.js b/test/controller/routes.js index 7f122047..a15eccda 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -3,7 +3,7 @@ import events from "node:events"; import FormData from "form-data"; import http from "node:http"; -import { wait } from "@clusterio/lib"; +import { wait, expositionContentType } from "@clusterio/lib"; import * as routes from "@clusterio/controller/dist/node/src/routes.js"; import * as mock from "../mock.js"; @@ -72,6 +72,71 @@ describe("controller/src/routes", function() { assert.equal(await responses[1].text(), "test content"); }); }); + describe("/metrics", function() { + let endpoint, testToken; + beforeEach(function() { + endpoint = `http://localhost:${port}/metrics`; + testToken = controller.users.signUserToken({ id: "test" }); + }); + it("should respond with 401 without a valid token", async function() { + let response; + response = await fetch(endpoint); + assert.equal(response.status, 401); + response = await fetch(endpoint, { headers: { "X-Access-Token": "invalid" } }); + assert.equal(response.status, 401); + response = await fetch(endpoint, { headers: { "Authorization": "Bearer invalid" } }); + assert.equal(response.status, 401); + response = await fetch(endpoint, { headers: { "Authorization": `Basic ${testToken}` } }); + assert.equal(response.status, 401); + const user = controller.users.getOrCreateUser("test"); + user.tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000); + user.saveRecord(); + response = await fetch(endpoint, { headers: { "Authorization": `Bearer ${testToken}` } }); + assert.equal(response.status, 401); + }); + it("should respond with 403 when user has insufficient permission", async function() { + const user = controller.users.getOrCreateUser("player"); + let response = await fetch(endpoint, { + headers: { "Authorization": `Bearer ${controller.users.signUserToken(user)}` }, + }); + assert.equal(response.status, 403); + }); + it("should respond with metrics to an authorized user", async function() { + for (const headers of [ + { "X-Access-Token": testToken }, + { "Authorization": `Bearer ${testToken}` }, + { "Authorization": `bearer ${testToken}` }, + ]) { + let response = await fetch(endpoint, { headers }); + assert.equal(response.status, 200); + const contentType = response.headers.get("content-type"); + for (const part of expositionContentType.split("; ")) { + assert(contentType.includes(part), `expected ${part} in content type ${contentType}`); + } + assert(/^# HELP /m.test(await response.text()), "expected exposition text"); + } + }); + it("should share one gather between concurrent requests", async function() { + let calls = 0; + controller.hooks.metrics.attach("test", async () => { + calls += 1; + await wait(100); + }); + const headers = { "X-Access-Token": testToken }; + let responses = await Promise.all([ + fetch(endpoint, { headers }), + fetch(endpoint, { headers }), + fetch(endpoint, { headers }), + ]); + for (const response of responses) { + assert.equal(response.status, 200); + } + assert.equal(calls, 1); + let response = await fetch(endpoint, { headers }); + assert.equal(response.status, 200); + assert.equal(calls, 2); + }); + }); describe("stripStaticPrefix()", function() { it("should remove a leading static/", function() { assert.equal(routes.stripStaticPrefix("static/main.abc.js"), "main.abc.js"); diff --git a/test/mock.js b/test/mock.js index 0c52dfdc..de016ea0 100644 --- a/test/mock.js +++ b/test/mock.js @@ -165,6 +165,7 @@ export class MockController { ["controller.public_url", "test"], ["controller.auth_secret", "TestSecretDoNotUse"], ["controller.proxy_stream_timeout", 1], + ["controller.metrics_timeout", 1], ]); this.config = { get: (name) => { @@ -199,6 +200,7 @@ export class MockController { this.handles = new Map(); this.hooks = new ControllerHooks(new MockLogger()); this.loadedPlugins = new Set(); + this.wsServer = { hostConnections: new Map() }; } get authSecret() {