diff --git a/docs/configuration.md b/docs/configuration.md index 803e51f8..2bc2e221 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -48,6 +48,16 @@ Must be greater than 0. Defaults to 60. +### controller.log_retention_days + +Number of days to keep the controller and cluster log files in the log directory for. +Log files are named by the UTC date they were written and a file is deleted once its date is more than this many days in the past. +Old files are checked for on startup and every hour after that. +Set to 0 to keep logs forever. + +Defaults to 30. + + ### controller.host_pairing_timeout Time in seconds a host can wait for approval before its request expires, after which the host requests again with a new code. @@ -436,6 +446,15 @@ Must be greater than 0. Defaults to 60. +### host.log_retention_days + +Number of days to keep the host log files in the log directory for. +Works the same way as controller.log_retention_days. +Set to 0 to keep logs forever. + +Defaults to 30. + + ### host.allow_remote_updates Whether the `host update` command in clusterioctl and the update button in the web interface are allowed to update the `@clusterio/host` package with npm. diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 29334ed4..cd1364a1 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -492,6 +492,11 @@ async function startup() { controller.stop(); }); + lib.startLogRetention( + [path.join(args.logDirectory, "cluster"), path.join(args.logDirectory, "controller")], + () => controllerConfig.get("controller.log_retention_days"), + ); + await controller.start(args); } diff --git a/packages/host/host.ts b/packages/host/host.ts index a5302b70..a250ab06 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -210,6 +210,11 @@ async function startHost() { // From this point on the host should run await hostConfigLock.acquire(); + lib.startLogRetention( + [path.join(args.logDirectory, "host")], + () => hostConfig.get("host.log_retention_days"), + ); + // If we get here the command was run await fs.mkdir(hostConfig.get("host.instances_directory"), { recursive: true }); await fs.mkdir(hostConfig.get("host.mods_directory"), { recursive: true }); diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 5fd9e64f..ecea3400 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -37,6 +37,7 @@ export interface ControllerConfigFields { "controller.default_mod_pack_id": number | null; "controller.default_role_id": number | null; "controller.autosave_interval": number; + "controller.log_retention_days": number; "controller.host_pairing_timeout": number; "controller.host_pairing_max_pending": number; "controller.mod_portal_cache_duration_minutes": number; @@ -100,6 +101,13 @@ export class ControllerConfig extends classes.Config { initialValue: 60, validator: validators.greaterThanZero, }, + "controller.log_retention_days": { + title: "Log Retention Days", + description: "Days to keep controller and cluster log files for, 0 to keep them forever.", + type: "number", + initialValue: 30, + validator: validators.all(validators.integer, validators.greaterThanEqualZero), + }, "controller.host_pairing_timeout": { title: "Host Pairing Timeout", description: "Time in seconds a host can wait for approval before it has to request again, 0 for no limit.", @@ -349,6 +357,7 @@ export interface HostConfigFields { "host.factorio_username": string | null, "host.factorio_token": string | null, "host.max_reconnect_delay": number; + "host.log_retention_days": number; "host.allow_remote_updates": boolean; "host.allow_plugin_updates": boolean; "host.allow_plugin_install": boolean; @@ -458,6 +467,13 @@ export class HostConfig extends classes.Config { initialValue: 60, validator: validators.greaterThanZero, }, + "host.log_retention_days": { + title: "Log Retention Days", + description: "Days to keep host log files for, 0 to keep them forever.", + type: "number", + initialValue: 30, + validator: validators.all(validators.integer, validators.greaterThanEqualZero), + }, "host.allow_remote_updates": { description: "When true, allows a remote event to trigger a clusterio update via npm", type: "boolean", diff --git a/packages/lib/src/logging_utils.ts b/packages/lib/src/logging_utils.ts index ab235182..ea062c96 100644 --- a/packages/lib/src/logging_utils.ts +++ b/packages/lib/src/logging_utils.ts @@ -358,8 +358,15 @@ export class LogIndex { */ async buildIndex() { let today = new Date(Date.now() - 300e3).toISOString().slice(0, 10); + let files = await fs.readdir(this.logDirectory); - for (let file of await fs.readdir(this.logDirectory)) { + for (let file of this.index.keys()) { + if (!files.includes(file)) { + this.index.delete(file); + } + } + + for (let file of files) { let match = logFileGlob.exec(file); if (!match || match[1] >= today || this.index.has(file)) { continue; @@ -412,6 +419,80 @@ export class LogIndex { } } +/** + * Delete dated log files older than the given number of days + * + * Only files named like name-YYYY-MM-DD.log are deleted, and only once + * every entry they can hold is older than retentionDays. + * + * @param logDirectory - Directory to delete log files from. + * @param retentionDays - Days to keep log files for, 0 keeps them forever. + * @param now - Current time in milliseconds since the epoch. + * @returns names of the deleted files. + */ +export async function deleteOldLogs(logDirectory: string, retentionDays: number, now = Date.now()) { + if (!(retentionDays > 0)) { + return []; + } + + let files: string[]; + try { + files = await fs.readdir(logDirectory); + } catch (err: any) { + if (err.code === "ENOENT") { + return []; + } + throw err; + } + + const cutoff = new Date(now - retentionDays * 86400e3).toISOString().slice(0, 10); + const deleted: string[] = []; + for (const file of files) { + const match = logFileGlob.exec(file); + if (!match || match[1] >= cutoff) { + continue; + } + try { + await fs.unlink(path.join(logDirectory, file)); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + } + deleted.push(file); + } + return deleted; +} + +/** + * Delete old logs now and then every hour + * + * @param logDirectories - Directories to delete old log files from. + * @param getRetentionDays - Called before each run to get the retention. + * @returns the interval timer, which does not keep the process alive. + */ +export function startLogRetention(logDirectories: string[], getRetentionDays: () => number) { + async function run() { + const retentionDays = getRetentionDays(); + for (const logDirectory of logDirectories) { + try { + const deleted = await deleteOldLogs(logDirectory, retentionDays); + if (deleted.length) { + logger.info( + `Deleted ${deleted.length} log files older than ${retentionDays} days from ${logDirectory}` + ); + } + } catch (err: any) { + logger.error(`Failed to delete old logs from ${logDirectory}:\n${err.stack}`); + } + } + } + run(); + const interval = setInterval(run, 3600e3); + interval.unref(); + return interval; +} + /** * Filter object for querying logs */ diff --git a/test/lib/logging_utils.js b/test/lib/logging_utils.js index 054e49e4..b8dab0b9 100644 --- a/test/lib/logging_utils.js +++ b/test/lib/logging_utils.js @@ -45,6 +45,19 @@ describe("lib/logging_utils.js", function() { logIndex ); }); + it("should drop entries for files no longer present", async function() { + let tempDir = path.join("temp", "test", "log_index_prune"); + await fs.rm(tempDir, { recursive: true, force: true }); + await fs.cp(logDir, tempDir, { recursive: true }); + let logIndex = await lib.LogIndex.load(tempDir); + await logIndex.buildIndex(); + await fs.unlink(path.join(tempDir, "test-2021-12-10.log")); + await logIndex.buildIndex(); + assert.deepEqual( + new Set(logIndex.index.keys()), + new Set(["test-2021-12-11.log", "test-2021-12-12.log"]) + ); + }); it("should handle loading broken index", async function() { let tempDir = path.join("temp", "test", "log"); await fs.writeFile(path.join(tempDir, "index.json"), "Broken JSON"); @@ -53,6 +66,43 @@ describe("lib/logging_utils.js", function() { }); }); + describe("deleteOldLogs", function() { + let tempDir = path.join("temp", "test", "log_retention"); + let files = [ + "cluster-2026-10-01.log", + "cluster-2026-10-12.log", + "cluster-2026-10-13.log", + "cluster-2026-11-12.log", + "index.json", + "notes-2026-10-01.txt", + ".0123abcd-audit.json", + ]; + let now = Date.parse("2026-11-12T08:00:00Z"); + beforeEach(async function() { + await fs.rm(tempDir, { recursive: true, force: true }); + await fs.mkdir(tempDir, { recursive: true }); + for (let file of files) { + await fs.writeFile(path.join(tempDir, file), ""); + } + }); + + it("should delete only dated log files past the retention", async function() { + let deleted = await lib.deleteOldLogs(tempDir, 30, now); + assert.deepEqual(deleted.sort(), ["cluster-2026-10-01.log", "cluster-2026-10-12.log"]); + assert.deepEqual( + (await fs.readdir(tempDir)).sort(), + files.filter(file => !deleted.includes(file)).sort(), + ); + }); + it("should keep everything with a retention of 0", async function() { + assert.deepEqual(await lib.deleteOldLogs(tempDir, 0, now), []); + assert.deepEqual((await fs.readdir(tempDir)).sort(), [...files].sort()); + }); + it("should ignore a missing directory", async function() { + assert.deepEqual(await lib.deleteOldLogs(path.join(tempDir, "missing"), 30, now), []); + }); + }); + let allLines = []; let logIndex = null; before(async function() {