mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-19 23:33:13 +09:00
Replace log query implementation
The interface provided by Winston is unusably slow for querying large logs. Reimplement the log querying with both forwards and backwards line reading and a simple file based index to skip files which do not contain any lines matching the filter.
This commit is contained in:
1 parent
fd1f3329bd
commit
601ebba919
10 files changed
+497
-54
No files matched your search
@@ -0,0 +1 @@
|
||||
This file should be excluded by queries and the index
|
||||
@@ -0,0 +1,9 @@
|
||||
{"timestamp":"2021-12-10T10:00:00.000Z","level":"verbose","message":"Master start"}
|
||||
{"timestamp":"2021-12-10T10:00:01.000Z","level":"verbose","message":"Loading"}
|
||||
{"timestamp":"2021-12-10T10:00:03.000Z","level":"info","message":"Ready"}
|
||||
{"timestamp":"2021-12-10T10:00:10.000Z","slave_id":1,"level":"verbose","message":"Slave connected"}
|
||||
{"timestamp":"2021-12-10T10:00:11.000Z","slave_id":1,"level":"verbose","message":"Found instance 10"}
|
||||
{"timestamp":"2021-12-10T10:00:13.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Starting up"}
|
||||
{"timestamp":"2021-12-10T10:00:14.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Loading map"}
|
||||
{"timestamp":"2021-12-10T10:00:15.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Game ready"}
|
||||
{"timestamp":"2021-12-10T10:01:00.000Z","slave_id":1,"instance_id":10,"level":"error","message":"Unexpected crash"}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"timestamp":"2021-12-11T10:00:00.000Z","level":"verbose","message":"Master start"}
|
||||
{"timestamp":"2021-12-11T10:00:01.000Z","level":"verbose","message":"Loading"}
|
||||
{"timestamp":"2021-12-11T10:00:03.000Z","level":"fatal","message":"An error occured"}
|
||||
@@ -0,0 +1 @@
|
||||
{"timestamp":"2021-12-12T10:00:00.000Z","leve
|
||||
Whitespace-only changes.
@@ -1,8 +1,13 @@
|
||||
"use strict";
|
||||
const assert = require("assert").strict;
|
||||
const fs = require("fs-extra");
|
||||
const path = require("path");
|
||||
const util = require("util");
|
||||
|
||||
|
||||
const { testLines } = require("./factorio/lines");
|
||||
const libLoggingUtils = require("@clusterio/lib/logging_utils");
|
||||
const libStream = require("@clusterio/lib/stream");
|
||||
|
||||
|
||||
describe("lib/logging_utils.js", function() {
|
||||
@@ -16,4 +21,110 @@ describe("lib/logging_utils.js", function() {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let logDir = path.join("test", "file", "log");
|
||||
describe("class LogIndex", function() {
|
||||
it("should create a new empty index if index.json does not exist", async function() {
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
assert.deepEqual(logIndex.index, new Map());
|
||||
});
|
||||
it("should build index of only past log files", async function() {
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
await logIndex.buildIndex();
|
||||
assert.deepEqual(
|
||||
new Set(logIndex.index.keys()),
|
||||
new Set(["test-2021-12-10.log", "test-2021-12-11.log", "test-2021-12-12.log"])
|
||||
);
|
||||
});
|
||||
it("should roundtrip index on save load cycle", async function() {
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
await logIndex.buildIndex();
|
||||
let tempDir = path.join("temp", "test", "log");
|
||||
logIndex.logDirectory = tempDir;
|
||||
await logIndex.save();
|
||||
let loadedIndex = await libLoggingUtils.LogIndex.load(tempDir);
|
||||
assert.deepEqual(
|
||||
loadedIndex,
|
||||
logIndex
|
||||
);
|
||||
});
|
||||
it("should handle loading broken index", async function() {
|
||||
let tempDir = path.join("temp", "test", "log");
|
||||
await fs.outputFile(path.join(tempDir, "index.json"), "Broken JSON");
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(tempDir);
|
||||
assert.deepEqual(logIndex.index, new Map());
|
||||
});
|
||||
});
|
||||
|
||||
let allLines = [];
|
||||
let logIndex = null;
|
||||
before(async function() {
|
||||
for (let file of (await fs.readdir(logDir)).sort()) {
|
||||
if (file === "excluded.txt") {
|
||||
continue;
|
||||
}
|
||||
let fileStream = fs.createReadStream(path.join(logDir, file));
|
||||
let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
|
||||
fileStream.pipe(lineStream);
|
||||
for await (let line of lineStream) {
|
||||
try {
|
||||
allLines.push(JSON.parse(line));
|
||||
} catch (err) {
|
||||
allLines.push({ "level": "info", "message": line.toString() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
await logIndex.buildIndex();
|
||||
});
|
||||
|
||||
describe("queryLog", function() {
|
||||
for (let index of [() => undefined, () => logIndex]) {
|
||||
describe(index() === undefined ? "without index" : "with index", function() {
|
||||
it("returns nothing with an empty filter", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, {}, index());
|
||||
assert.deepEqual(log, []);
|
||||
});
|
||||
it("should return the whole log when querying all", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { all: true }, index());
|
||||
assert.deepEqual(log, allLines);
|
||||
});
|
||||
it("should return the whole log in reverse when querying all desc", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { all: true, order: "desc" }, index());
|
||||
assert.deepEqual(log, [...allLines].reverse());
|
||||
});
|
||||
it("should limit by limit", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { limit: 2, all: true }, index());
|
||||
assert.deepEqual(log, allLines.slice(0, 2));
|
||||
});
|
||||
it("should filter by max_level", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { max_level: "info", all: true }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => ["fatal", "error", "warn", "audit", "info"].includes(info.level)
|
||||
));
|
||||
log = await libLoggingUtils.queryLog(logDir, { max_level: "fatal", all: true }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => ["fatal"].includes(info.level)
|
||||
));
|
||||
});
|
||||
it("should filter by master", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { master: true }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => info.slave_id === undefined && info.instance_id === undefined
|
||||
));
|
||||
});
|
||||
it("should filter by slave_ids", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { slave_ids: [1, 2] }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => [1, 2].includes(info.slave_id) && info.instance_id === undefined
|
||||
));
|
||||
});
|
||||
it("should filter by instance_ids", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { instance_ids: [10, 11] }, index());
|
||||
assert.deepEqual(log, allLines.filter(info => [10, 11].includes(info.instance_id)));
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user