From d39b525b07f85535cc28e95290dd58ed555d65db Mon Sep 17 00:00:00 2001
From: Hornwitser <github@hornwitser.no>
Date: Thu, 9 Feb 2023 23:13:15 +0100
Subject: [PATCH] Add clusterio_master_log_bytes metric

Export a metric for total size of logs stored on the master server.
---
 packages/lib/file_ops.js      | 34 ++++++++++++++++++++++++++++++++++
 packages/master/src/Master.js | 19 +++++++++++++++++++
 test/lib/file_ops.js          | 15 +++++++++++++++
 3 files changed, 68 insertions(+)

diff --git a/packages/lib/file_ops.js b/packages/lib/file_ops.js
index be007912..e0b4639d 100644
--- a/packages/lib/file_ops.js
+++ b/packages/lib/file_ops.js
@@ -5,6 +5,7 @@
 const fs = require("fs-extra");
 const path = require("path");
 const crypto = require("crypto"); // needed for getTempFile
+
 /**
  * Returns the newest file in a directory
  *
@@ -32,6 +33,38 @@ async function getNewestFile(directory, filter = (name) => true) {
 	return newestFile;
 }
 
+/**
+ * Returns the total size of all files in a directory
+ *
+ * Sums up the file size of all files in the given directory if it exists.
+ * Error reading the size of files are ignored, and error reading the
+ * directory will result in 0 being returned.
+ *
+ * @param {string} directory - The directory to sum files in.
+ * @returns {Promise<number>} The size in bytes of all files in the directory.
+ */
+async function directorySize(directory) {
+	let dirEntries;
+	try {
+		dirEntries = await fs.readdir(directory, { withFileTypes: true });
+	} catch (err) {
+		if (err.code === "ENOENT") {
+			return 0;
+		}
+		throw err;
+	}
+	let statTasks = [];
+	for (let entry of dirEntries) {
+		if (entry.isFile()) {
+			statTasks.push(fs
+				.stat(path.join(directory, entry.name))
+				.then(stat => stat.size, _ => 0)
+			);
+		}
+	}
+	return (await Promise.all(statTasks)).reduce((a, v) => a + v, 0);
+}
+
 /**
  * Modifies name in case it already exisist at the given directory
  *
@@ -193,6 +226,7 @@ function cleanFilename(name) {
 
 module.exports = {
 	getNewestFile,
+	directorySize,
 	findUnusedName,
 	getTempFile,
 	safeOutputFile,
diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js
index 3e57f984..be8b5b2a 100644
--- a/packages/master/src/Master.js
+++ b/packages/master/src/Master.js
@@ -32,6 +32,11 @@ const endpointDurationSummary = new libPrometheus.Summary(
 	{ labels: ["route"] }
 );
 
+const logSizeGauge = new libPrometheus.Gauge(
+	"clusterio_master_log_bytes",
+	"Size of all log files currently stored on the master."
+);
+
 /**
  * Manages all master related operations
  * @alias module:master/src/Master
@@ -131,6 +136,19 @@ class Master {
 		}
 	}
 
+	/**
+	 * Get the total size of the logs stored
+	 *
+	 * @returns {Promise<number>} size in bytes of stored log files.
+	 */
+	async logSize() {
+		return (await Promise.all([
+			libFileOps.directorySize(path.join(this.logDirectory, "cluster")),
+			libFileOps.directorySize(path.join(this.logDirectory, "master")),
+			libFileOps.directorySize(path.join(this.logDirectory, "slave")),
+		])).reduce((a, v) => a + v, 0);
+	}
+
 	async _startInternal(args) {
 		this.logDirectory = args.logDirectory;
 		this.clusterLogIndex = await libLoggingUtils.LogIndex.load(path.join(this.logDirectory, "cluster"));
@@ -139,6 +157,7 @@ class Master {
 				err => logger.error(`Error building cluster log index:\n${err.stack}`)
 			);
 		}, 600e3);
+		logSizeGauge.callback = async () => { logSizeGauge.set(await this.logSize()); };
 
 		// Start webpack development server if enabled
 		if (args.dev || args.devPlugin) {
diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js
index da256995..e9184090 100644
--- a/test/lib/file_ops.js
+++ b/test/lib/file_ops.js
@@ -41,6 +41,21 @@ describe("lib/file_ops", function() {
 		});
 	});
 
+	describe("getNewestFile()", function() {
+		it("returns 0 if directory does not exist", async function() {
+			let size = await libFileOps.directorySize(path.join(baseDir, "invalid"));
+			assert.equal(size, 0);
+		});
+		it("returns 0 if directory is empty", async function() {
+			let size = await libFileOps.directorySize(path.join(baseDir, "test", "folder"));
+			assert.equal(size, 0);
+		});
+		it("returns size of files in directory", async function() {
+			let size = await libFileOps.directorySize(path.join(baseDir, "test"));
+			assert.equal(size, 21);
+		});
+	});
+
 	describe("findUnusedName()", function() {
 		it("should return named unchanged if it does not exist", async function() {
 			let cases = [