Split logs by date

Move the default location of the log files into a configurable logs
folder and split the written logs up by UTC date to more easily allow
deleting of old logs to be performed.  The log rotation is implemented
by winston-daily-rotate-file.

The old log format will automatically be migrated to the new format and
location if the target folder for the new logs don't yet exists.
This commit is contained in:
Hornwitser committed 2021-12-19 01:42:34 +01:00
1 parent 200ccf8e06
commit fd1f3329bd
5 files changed
+125 -9

No files matched your search

+61
View File
@@ -1,4 +1,7 @@
"use strict";
const events = require("events");
const fs = require("fs-extra");
const path = require("path");
const winston = require("winston");
const Transport = require("winston-transport");
const { LEVEL, MESSAGE } = require("triple-beam");
@@ -6,6 +9,7 @@ const chalk = require("chalk");
const libLink = require("./link");
const libErrors = require("./errors");
const libStream = require("./stream");
/**
@@ -121,10 +125,67 @@ class LinkTransport extends Transport {
}
}
async function migrateLogs(log, directory, pattern) {
const dateRegExp = /^\d{4}-\d{2}-\d{2}/;
let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
let fileStream = fs.createReadStream(log);
fileStream.pipe(lineStream);
let currentDate = null;
let output = null;
async function switchOutput(newOutputPath) {
if (output) {
output.end();
await events.once(output, "close");
}
output = fs.createWriteStream(newOutputPath, { flags: "ax" });
try {
await events.once(output, "open");
// eslint-disable-next-line no-console
console.log(`writing ${newOutputPath}`);
} catch (err) {
if (err.code === "EEXIST") {
// eslint-disable-next-line no-console
console.warn(`Discarding logs for ${newDate} due to target logfile already existing`);
output = null;
} else {
throw err;
}
}
}
for await (let line of lineStream) {
let info = {};
try {
info = JSON.parse(line);
} catch (err) {
// Ignore
}
if (typeof info.timestamp === "string" && dateRegExp.test(info.timestamp)) {
let newDate = info.timestamp.slice(0, 10);
// Only update date forward in time.
if (newDate !== currentDate && (!currentDate || newDate > currentDate)) {
await switchOutput(path.join(directory, pattern.replace("%DATE%", newDate)));
currentDate = newDate;
}
}
if (output) {
if (!output.write(Buffer.concat([line, Buffer.from("\n")]))) {
await events.once(output, "drain");
}
}
}
}
module.exports = {
TerminalFormat,
LinkTransport,
migrateLogs,
// for testing only
_formatServerOutput: formatServerOutput,