Merge pull request #1019 from bbassie/feature/metrics-auth

Require a token to scrape /metrics

docs/managing-a-cluster.md

@@ -274,3 +274,29 @@ Remote updates are enabled by default but can be disabled on a per-machine basis
     controller> config set controller.allow_plugin_install true/false
 
 Remote installs are disabled by default but can be enabled on a per-machine basis at the owner's discretion. For security reasons, this configuration setting cannot be changed remotely. To modify it, run the above commands locally while the remote is offline.
+
+
+## Prometheus Metrics
+
+The controller exports metrics for the whole cluster on the `/metrics` endpoint.
+Fetching it requires a token for a user with the `core.controller.metrics` permission, sent either as an `Authorization: Bearer <token>` header or as an `X-Access-Token` header.
+Requests without a valid token get a 401 response and requests for users lacking the permission get a 403 response.
+
+Create a role with just this permission, a user to hold it, and generate a token for that user.
+Give the user a name that is not a valid Factorio player name, such as one starting with `#`, so a player joining the cluster can never claim the account and its token:
+
+    ctl> role create Prometheus --permissions core.controller.metrics
+    ctl> user create #prometheus
+    ctl> user set-roles #prometheus Prometheus
+    $ npx clusteriocontroller bootstrap generate-user-token '#prometheus'
+
+Then pass the token in the scrape job for the controller in `prometheus.yml`:
+
+    scrape_configs:
+      - job_name: clusterio
+        authorization:
+          credentials: <token>
+        static_configs:
+          - targets: ["localhost:8080"]
+
+Concurrent scrapes share a single round of metrics gathering from the hosts, so multiple scrapers polling the controller do not multiply the load on the hosts.

packages/controller/controller.ts

@@ -101,6 +101,9 @@ void new lib.Gauge(
 	"clusterio_controller_websocket_active_connections",
 	"How many WebSocket connections are currently open to the controller",
 	{ callback: function(gauge:lib.Gauge) {
+		if (!controller) {
+			return;
+		}
 		gauge.set(controller.wsServer.activeConnectors.size);
 	}}
 );
@@ -109,6 +112,9 @@ void new lib.Gauge(
 	"clusterio_controller_active_hosts",
 	"How many hosts are currently connected to the controller",
 	{ callback: function(gauge:lib.Gauge) {
+		if (!controller) {
+			return;
+		}
 		gauge.set(controller.wsServer.hostConnections.size);
 	}}
 );
@@ -118,6 +124,9 @@ void new lib.Gauge(
 	"How many clients are currently connected to this controller",
 	{
 		labels: ["type"], callback: async function(gauge:lib.Gauge) {
+			if (!controller) {
+				return;
+			}
 			gauge.labels("host").set(controller.wsServer.hostConnections.size);
 			gauge.labels("control").set(controller.wsServer.controlConnections.size);
 		},

packages/controller/src/routes.ts

@@ -24,7 +24,8 @@ declare global {
 			mainBundle: string,
 			staticRoot: string | null,
 			devPlugins: Map<string, number>,
-			streams: Map<string, ProxyStream>
+			streams: Map<string, ProxyStream>,
+			pendingMetrics?: Promise<string>,
 		}
 	}
 }
@@ -44,10 +45,8 @@ function mergeSamples(destinationResult: lib.CollectorResultSerialized, sourceRe
 	}
 }
 
-// Prometheus polling endpoint
-async function getMetrics(req: Request, res: Response, next: any) {
-	const controller: Controller = req.app.locals.controller;
-
+// Collect metrics from the controller, its plugins and all connected hosts
+async function gatherMetrics(controller: Controller) {
 	let results: lib.CollectorResult[] = [];
 	let pluginResults = await controller.hooks.metrics.collect();
 	for (let metricIterator of pluginResults) {
@@ -99,7 +98,27 @@ async function getMetrics(req: Request, res: Response, next: any) {
 		results.push(lib.deserializeResult(result));
 	}
 
-	let text = await lib.exposition(results);
+	return await lib.exposition(results);
+}
+
+// Prometheus polling endpoint
+async function getMetrics(req: Request, res: Response) {
+	try {
+		res.locals.user.checkPermission("core.controller.metrics");
+	} catch (err: any) {
+		res.status(403).json({ request_errors: [err.message] });
+		return;
+	}
+
+	// Concurrent scrapes share one gather so hosts see at most one
+	// metrics request at a time regardless of how many clients poll.
+	const locals = req.app.locals;
+	if (!locals.pendingMetrics) {
+		locals.pendingMetrics = gatherMetrics(locals.controller).finally(() => {
+			locals.pendingMetrics = undefined;
+		});
+	}
+	const text = await locals.pendingMetrics;
 	res.set("Content-Type", lib.expositionContentType);
 	res.send(text);
 }
@@ -152,8 +171,21 @@ function getClusterName(req: Request, res: Response) {
 	res.json({ name: clusterName });
 }
 
+// Reads the access token from X-Access-Token or Authorization: Bearer
+function getAccessToken(req: Request) {
+	const token = req.header("x-access-token");
+	if (token) {
+		return token;
+	}
+	const authorization = req.header("authorization");
+	if (authorization && /^Bearer /i.test(authorization)) {
+		return authorization.slice("Bearer ".length).trim();
+	}
+	return undefined;
+}
+
 function validateHostToken(req: Request, res: Response, next: any) {
-	let token = req.header("x-access-token");
+	let token = getAccessToken(req);
 	if (!token) {
 		res.sendStatus(401);
 		return;
@@ -180,7 +212,7 @@ function validateHostToken(req: Request, res: Response, next: any) {
 }
 
 function validateUserToken(req: Request, res: Response, next: any) {
-	let token = req.header("x-access-token");
+	let token = getAccessToken(req);
 	if (!token) {
 		res.sendStatus(401);
 		return;
@@ -653,7 +685,10 @@ async function uploadMod(req: Request, res: Response) {
 
 
 export function addRouteHandlers(app: Application) {
-	app.get("/metrics", (req:Request, res:Response, next:any) => getMetrics(req, res, next).catch(next));
+	app.get("/metrics",
+		validateUserToken,
+		(req:Request, res:Response, next:any) => getMetrics(req, res).catch(next)
+	);
 	app.get("/api/plugins", getPlugins);
 	app.get("/api/cluster-name", getClusterName);
 	app.put("/api/upload-export",

packages/lib/src/permissions.ts

@@ -144,6 +144,11 @@ export const corePermissions = [
 		title: "Modify controller config",
 		description: "Modify the controller config or entries of the controller config.",
 	},
+	{
+		name: "core.controller.metrics",
+		title: "Scrape metrics",
+		description: "Fetch the Prometheus metrics for the cluster from the controller's /metrics endpoint.",
+	},
 	{
 		name: "core.system.subscribe",
 		title: "Subscribe to system updates",

test/controller/routes.js

@@ -3,7 +3,7 @@ import events from "node:events";
 import FormData from "form-data";
 import http from "node:http";
 
-import { wait } from "@clusterio/lib";
+import { wait, expositionContentType } from "@clusterio/lib";
 import * as routes from "@clusterio/controller/dist/node/src/routes.js";
 import * as mock from "../mock.js";
 
@@ -72,6 +72,71 @@ describe("controller/src/routes", function() {
 			assert.equal(await responses[1].text(), "test content");
 		});
 	});
+	describe("/metrics", function() {
+		let endpoint, testToken;
+		beforeEach(function() {
+			endpoint = `http://localhost:${port}/metrics`;
+			testToken = controller.users.signUserToken({ id: "test" });
+		});
+		it("should respond with 401 without a valid token", async function() {
+			let response;
+			response = await fetch(endpoint);
+			assert.equal(response.status, 401);
+			response = await fetch(endpoint, { headers: { "X-Access-Token": "invalid" } });
+			assert.equal(response.status, 401);
+			response = await fetch(endpoint, { headers: { "Authorization": "Bearer invalid" } });
+			assert.equal(response.status, 401);
+			response = await fetch(endpoint, { headers: { "Authorization": `Basic ${testToken}` } });
+			assert.equal(response.status, 401);
+			const user = controller.users.getOrCreateUser("test");
+			user.tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000);
+			user.saveRecord();
+			response = await fetch(endpoint, { headers: { "Authorization": `Bearer ${testToken}` } });
+			assert.equal(response.status, 401);
+		});
+		it("should respond with 403 when user has insufficient permission", async function() {
+			const user = controller.users.getOrCreateUser("player");
+			let response = await fetch(endpoint, {
+				headers: { "Authorization": `Bearer ${controller.users.signUserToken(user)}` },
+			});
+			assert.equal(response.status, 403);
+		});
+		it("should respond with metrics to an authorized user", async function() {
+			for (const headers of [
+				{ "X-Access-Token": testToken },
+				{ "Authorization": `Bearer ${testToken}` },
+				{ "Authorization": `bearer ${testToken}` },
+			]) {
+				let response = await fetch(endpoint, { headers });
+				assert.equal(response.status, 200);
+				const contentType = response.headers.get("content-type");
+				for (const part of expositionContentType.split("; ")) {
+					assert(contentType.includes(part), `expected ${part} in content type ${contentType}`);
+				}
+				assert(/^# HELP /m.test(await response.text()), "expected exposition text");
+			}
+		});
+		it("should share one gather between concurrent requests", async function() {
+			let calls = 0;
+			controller.hooks.metrics.attach("test", async () => {
+				calls += 1;
+				await wait(100);
+			});
+			const headers = { "X-Access-Token": testToken };
+			let responses = await Promise.all([
+				fetch(endpoint, { headers }),
+				fetch(endpoint, { headers }),
+				fetch(endpoint, { headers }),
+			]);
+			for (const response of responses) {
+				assert.equal(response.status, 200);
+			}
+			assert.equal(calls, 1);
+			let response = await fetch(endpoint, { headers });
+			assert.equal(response.status, 200);
+			assert.equal(calls, 2);
+		});
+	});
 	describe("stripStaticPrefix()", function() {
 		it("should remove a leading static/", function() {
 			assert.equal(routes.stripStaticPrefix("static/main.abc.js"), "main.abc.js");

test/mock.js

@@ -165,6 +165,7 @@ export class MockController {
 			["controller.public_url", "test"],
 			["controller.auth_secret", "TestSecretDoNotUse"],
 			["controller.proxy_stream_timeout", 1],
+			["controller.metrics_timeout", 1],
 		]);
 		this.config = {
 			get: (name) => {
@@ -199,6 +200,7 @@ export class MockController {
 		this.handles = new Map();
 		this.hooks = new ControllerHooks(new MockLogger());
 		this.loadedPlugins = new Set();
+		this.wsServer = { hostConnections: new Map() };
 	}
 
 	get authSecret() {