Replace log query implementation

The interface provided by Winston is unusably slow for querying large
logs.  Reimplement the log querying with both forwards and backwards
line reading and a simple file based index to skip files which do not
contain any lines matching the filter.
This commit is contained in:
Hornwitser committed 2021-12-19 02:06:59 +01:00
1 parent fd1f3329bd
commit 601ebba919
10 files changed
+497 -54

No files matched your search

+1
View File
@@ -0,0 +1 @@
This file should be excluded by queries and the index
+9
View File
@@ -0,0 +1,9 @@
{"timestamp":"2021-12-10T10:00:00.000Z","level":"verbose","message":"Master start"}
{"timestamp":"2021-12-10T10:00:01.000Z","level":"verbose","message":"Loading"}
{"timestamp":"2021-12-10T10:00:03.000Z","level":"info","message":"Ready"}
{"timestamp":"2021-12-10T10:00:10.000Z","slave_id":1,"level":"verbose","message":"Slave connected"}
{"timestamp":"2021-12-10T10:00:11.000Z","slave_id":1,"level":"verbose","message":"Found instance 10"}
{"timestamp":"2021-12-10T10:00:13.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Starting up"}
{"timestamp":"2021-12-10T10:00:14.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Loading map"}
{"timestamp":"2021-12-10T10:00:15.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Game ready"}
{"timestamp":"2021-12-10T10:01:00.000Z","slave_id":1,"instance_id":10,"level":"error","message":"Unexpected crash"}
+3
View File
@@ -0,0 +1,3 @@
{"timestamp":"2021-12-11T10:00:00.000Z","level":"verbose","message":"Master start"}
{"timestamp":"2021-12-11T10:00:01.000Z","level":"verbose","message":"Loading"}
{"timestamp":"2021-12-11T10:00:03.000Z","level":"fatal","message":"An error occured"}
+1
View File
@@ -0,0 +1 @@
{"timestamp":"2021-12-12T10:00:00.000Z","leve
View File
Whitespace-only changes.
+111
View File
@@ -1,8 +1,13 @@
"use strict";
const assert = require("assert").strict;
const fs = require("fs-extra");
const path = require("path");
const util = require("util");
const { testLines } = require("./factorio/lines");
const libLoggingUtils = require("@clusterio/lib/logging_utils");
const libStream = require("@clusterio/lib/stream");
describe("lib/logging_utils.js", function() {
@@ -16,4 +21,110 @@ describe("lib/logging_utils.js", function() {
}
});
});
let logDir = path.join("test", "file", "log");
describe("class LogIndex", function() {
it("should create a new empty index if index.json does not exist", async function() {
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
assert.deepEqual(logIndex.index, new Map());
});
it("should build index of only past log files", async function() {
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
await logIndex.buildIndex();
assert.deepEqual(
new Set(logIndex.index.keys()),
new Set(["test-2021-12-10.log", "test-2021-12-11.log", "test-2021-12-12.log"])
);
});
it("should roundtrip index on save load cycle", async function() {
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
await logIndex.buildIndex();
let tempDir = path.join("temp", "test", "log");
logIndex.logDirectory = tempDir;
await logIndex.save();
let loadedIndex = await libLoggingUtils.LogIndex.load(tempDir);
assert.deepEqual(
loadedIndex,
logIndex
);
});
it("should handle loading broken index", async function() {
let tempDir = path.join("temp", "test", "log");
await fs.outputFile(path.join(tempDir, "index.json"), "Broken JSON");
let logIndex = await libLoggingUtils.LogIndex.load(tempDir);
assert.deepEqual(logIndex.index, new Map());
});
});
let allLines = [];
let logIndex = null;
before(async function() {
for (let file of (await fs.readdir(logDir)).sort()) {
if (file === "excluded.txt") {
continue;
}
let fileStream = fs.createReadStream(path.join(logDir, file));
let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
fileStream.pipe(lineStream);
for await (let line of lineStream) {
try {
allLines.push(JSON.parse(line));
} catch (err) {
allLines.push({ "level": "info", "message": line.toString() });
}
}
}
logIndex = await libLoggingUtils.LogIndex.load(logDir);
await logIndex.buildIndex();
});
describe("queryLog", function() {
for (let index of [() => undefined, () => logIndex]) {
describe(index() === undefined ? "without index" : "with index", function() {
it("returns nothing with an empty filter", async function() {
let log = await libLoggingUtils.queryLog(logDir, {}, index());
assert.deepEqual(log, []);
});
it("should return the whole log when querying all", async function() {
let log = await libLoggingUtils.queryLog(logDir, { all: true }, index());
assert.deepEqual(log, allLines);
});
it("should return the whole log in reverse when querying all desc", async function() {
let log = await libLoggingUtils.queryLog(logDir, { all: true, order: "desc" }, index());
assert.deepEqual(log, [...allLines].reverse());
});
it("should limit by limit", async function() {
let log = await libLoggingUtils.queryLog(logDir, { limit: 2, all: true }, index());
assert.deepEqual(log, allLines.slice(0, 2));
});
it("should filter by max_level", async function() {
let log = await libLoggingUtils.queryLog(logDir, { max_level: "info", all: true }, index());
assert.deepEqual(log, allLines.filter(
info => ["fatal", "error", "warn", "audit", "info"].includes(info.level)
));
log = await libLoggingUtils.queryLog(logDir, { max_level: "fatal", all: true }, index());
assert.deepEqual(log, allLines.filter(
info => ["fatal"].includes(info.level)
));
});
it("should filter by master", async function() {
let log = await libLoggingUtils.queryLog(logDir, { master: true }, index());
assert.deepEqual(log, allLines.filter(
info => info.slave_id === undefined && info.instance_id === undefined
));
});
it("should filter by slave_ids", async function() {
let log = await libLoggingUtils.queryLog(logDir, { slave_ids: [1, 2] }, index());
assert.deepEqual(log, allLines.filter(
info => [1, 2].includes(info.slave_id) && info.instance_id === undefined
));
});
it("should filter by instance_ids", async function() {
let log = await libLoggingUtils.queryLog(logDir, { instance_ids: [10, 11] }, index());
assert.deepEqual(log, allLines.filter(info => [10, 11].includes(info.instance_id)));
});
});
}
});
});