Merge pull request #1200 from bbassie/feature/log-retention

Delete log files older than a configurable number of days

docs/configuration.md

@@ -48,6 +48,16 @@ Must be greater than 0.
 Defaults to 60.
 
 
+### controller.log_retention_days
+
+Number of days to keep the controller and cluster log files in the log directory for.
+Log files are named by the UTC date they were written and a file is deleted once its date is more than this many days in the past.
+Old files are checked for on startup and every hour after that.
+Set to 0 to keep logs forever.
+
+Defaults to 30.
+
+
 ### controller.host_pairing_timeout
 
 Time in seconds a host can wait for approval before its request expires, after which the host requests again with a new code.
@@ -436,6 +446,15 @@ Must be greater than 0.
 Defaults to 60.
 
 
+### host.log_retention_days
+
+Number of days to keep the host log files in the log directory for.
+Works the same way as controller.log_retention_days.
+Set to 0 to keep logs forever.
+
+Defaults to 30.
+
+
 ### host.allow_remote_updates
 
 Whether the `host update` command in clusterioctl and the update button in the web interface are allowed to update the `@clusterio/host` package with npm.

packages/controller/controller.ts

@@ -492,6 +492,11 @@ async function startup() {
 		controller.stop();
 	});
 
+	lib.startLogRetention(
+		[path.join(args.logDirectory, "cluster"), path.join(args.logDirectory, "controller")],
+		() => controllerConfig.get("controller.log_retention_days"),
+	);
+
 	await controller.start(args);
 }
 

packages/host/host.ts

@@ -210,6 +210,11 @@ async function startHost() {
 	// From this point on the host should run
 	await hostConfigLock.acquire();
 
+	lib.startLogRetention(
+		[path.join(args.logDirectory, "host")],
+		() => hostConfig.get("host.log_retention_days"),
+	);
+
 	// If we get here the command was run
 	await fs.mkdir(hostConfig.get("host.instances_directory"), { recursive: true });
 	await fs.mkdir(hostConfig.get("host.mods_directory"), { recursive: true });

packages/lib/src/config/definitions.ts

@@ -37,6 +37,7 @@ export interface ControllerConfigFields {
 	"controller.default_mod_pack_id": number | null;
 	"controller.default_role_id": number | null;
 	"controller.autosave_interval": number;
+	"controller.log_retention_days": number;
 	"controller.host_pairing_timeout": number;
 	"controller.host_pairing_max_pending": number;
 	"controller.mod_portal_cache_duration_minutes": number;
@@ -100,6 +101,13 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			initialValue: 60,
 			validator: validators.greaterThanZero,
 		},
+		"controller.log_retention_days": {
+			title: "Log Retention Days",
+			description: "Days to keep controller and cluster log files for, 0 to keep them forever.",
+			type: "number",
+			initialValue: 30,
+			validator: validators.all(validators.integer, validators.greaterThanEqualZero),
+		},
 		"controller.host_pairing_timeout": {
 			title: "Host Pairing Timeout",
 			description: "Time in seconds a host can wait for approval before it has to request again, 0 for no limit.",
@@ -349,6 +357,7 @@ export interface HostConfigFields {
 	"host.factorio_username": string | null,
 	"host.factorio_token": string | null,
 	"host.max_reconnect_delay": number;
+	"host.log_retention_days": number;
 	"host.allow_remote_updates": boolean;
 	"host.allow_plugin_updates": boolean;
 	"host.allow_plugin_install": boolean;
@@ -458,6 +467,13 @@ export class HostConfig extends classes.Config<HostConfigFields> {
 			initialValue: 60,
 			validator: validators.greaterThanZero,
 		},
+		"host.log_retention_days": {
+			title: "Log Retention Days",
+			description: "Days to keep host log files for, 0 to keep them forever.",
+			type: "number",
+			initialValue: 30,
+			validator: validators.all(validators.integer, validators.greaterThanEqualZero),
+		},
 		"host.allow_remote_updates": {
 			description: "When true, allows a remote event to trigger a clusterio update via npm",
 			type: "boolean",

packages/lib/src/logging_utils.ts

@@ -358,8 +358,15 @@ export class LogIndex {
 	 */
 	async buildIndex() {
 		let today = new Date(Date.now() - 300e3).toISOString().slice(0, 10);
+		let files = await fs.readdir(this.logDirectory);
 
-		for (let file of await fs.readdir(this.logDirectory)) {
+		for (let file of this.index.keys()) {
+			if (!files.includes(file)) {
+				this.index.delete(file);
+			}
+		}
+
+		for (let file of files) {
 			let match = logFileGlob.exec(file);
 			if (!match || match[1] >= today || this.index.has(file)) {
 				continue;
@@ -412,6 +419,80 @@ export class LogIndex {
 	}
 }
 
+/**
+ * Delete dated log files older than the given number of days
+ *
+ * Only files named like name-YYYY-MM-DD.log are deleted, and only once
+ * every entry they can hold is older than retentionDays.
+ *
+ * @param logDirectory - Directory to delete log files from.
+ * @param retentionDays - Days to keep log files for, 0 keeps them forever.
+ * @param now - Current time in milliseconds since the epoch.
+ * @returns names of the deleted files.
+ */
+export async function deleteOldLogs(logDirectory: string, retentionDays: number, now = Date.now()) {
+	if (!(retentionDays > 0)) {
+		return [];
+	}
+
+	let files: string[];
+	try {
+		files = await fs.readdir(logDirectory);
+	} catch (err: any) {
+		if (err.code === "ENOENT") {
+			return [];
+		}
+		throw err;
+	}
+
+	const cutoff = new Date(now - retentionDays * 86400e3).toISOString().slice(0, 10);
+	const deleted: string[] = [];
+	for (const file of files) {
+		const match = logFileGlob.exec(file);
+		if (!match || match[1] >= cutoff) {
+			continue;
+		}
+		try {
+			await fs.unlink(path.join(logDirectory, file));
+		} catch (err: any) {
+			if (err.code !== "ENOENT") {
+				throw err;
+			}
+		}
+		deleted.push(file);
+	}
+	return deleted;
+}
+
+/**
+ * Delete old logs now and then every hour
+ *
+ * @param logDirectories - Directories to delete old log files from.
+ * @param getRetentionDays - Called before each run to get the retention.
+ * @returns the interval timer, which does not keep the process alive.
+ */
+export function startLogRetention(logDirectories: string[], getRetentionDays: () => number) {
+	async function run() {
+		const retentionDays = getRetentionDays();
+		for (const logDirectory of logDirectories) {
+			try {
+				const deleted = await deleteOldLogs(logDirectory, retentionDays);
+				if (deleted.length) {
+					logger.info(
+						`Deleted ${deleted.length} log files older than ${retentionDays} days from ${logDirectory}`
+					);
+				}
+			} catch (err: any) {
+				logger.error(`Failed to delete old logs from ${logDirectory}:\n${err.stack}`);
+			}
+		}
+	}
+	run();
+	const interval = setInterval(run, 3600e3);
+	interval.unref();
+	return interval;
+}
+
 /**
  * Filter object for querying logs
  */

test/lib/logging_utils.js

@@ -45,6 +45,19 @@ describe("lib/logging_utils.js", function() {
 				logIndex
 			);
 		});
+		it("should drop entries for files no longer present", async function() {
+			let tempDir = path.join("temp", "test", "log_index_prune");
+			await fs.rm(tempDir, { recursive: true, force: true });
+			await fs.cp(logDir, tempDir, { recursive: true });
+			let logIndex = await lib.LogIndex.load(tempDir);
+			await logIndex.buildIndex();
+			await fs.unlink(path.join(tempDir, "test-2021-12-10.log"));
+			await logIndex.buildIndex();
+			assert.deepEqual(
+				new Set(logIndex.index.keys()),
+				new Set(["test-2021-12-11.log", "test-2021-12-12.log"])
+			);
+		});
 		it("should handle loading broken index", async function() {
 			let tempDir = path.join("temp", "test", "log");
 			await fs.writeFile(path.join(tempDir, "index.json"), "Broken JSON");
@@ -53,6 +66,43 @@ describe("lib/logging_utils.js", function() {
 		});
 	});
 
+	describe("deleteOldLogs", function() {
+		let tempDir = path.join("temp", "test", "log_retention");
+		let files = [
+			"cluster-2026-10-01.log",
+			"cluster-2026-10-12.log",
+			"cluster-2026-10-13.log",
+			"cluster-2026-11-12.log",
+			"index.json",
+			"notes-2026-10-01.txt",
+			".0123abcd-audit.json",
+		];
+		let now = Date.parse("2026-11-12T08:00:00Z");
+		beforeEach(async function() {
+			await fs.rm(tempDir, { recursive: true, force: true });
+			await fs.mkdir(tempDir, { recursive: true });
+			for (let file of files) {
+				await fs.writeFile(path.join(tempDir, file), "");
+			}
+		});
+
+		it("should delete only dated log files past the retention", async function() {
+			let deleted = await lib.deleteOldLogs(tempDir, 30, now);
+			assert.deepEqual(deleted.sort(), ["cluster-2026-10-01.log", "cluster-2026-10-12.log"]);
+			assert.deepEqual(
+				(await fs.readdir(tempDir)).sort(),
+				files.filter(file => !deleted.includes(file)).sort(),
+			);
+		});
+		it("should keep everything with a retention of 0", async function() {
+			assert.deepEqual(await lib.deleteOldLogs(tempDir, 0, now), []);
+			assert.deepEqual((await fs.readdir(tempDir)).sort(), [...files].sort());
+		});
+		it("should ignore a missing directory", async function() {
+			assert.deepEqual(await lib.deleteOldLogs(path.join(tempDir, "missing"), 30, now), []);
+		});
+	});
+
 	let allLines = [];
 	let logIndex = null;
 	before(async function() {