mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-16 07:24:53 +09:00
Replace log query implementation
The interface provided by Winston is unusably slow for querying large logs. Reimplement the log querying with both forwards and backwards line reading and a simple file based index to skip files which do not contain any lines matching the filter.
This commit is contained in:
1 parent
fd1f3329bd
commit
601ebba919
10 files changed
+497
-54
No files matched your search
+1
-2
@@ -1,7 +1,6 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
*debug.log*
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
/**
|
||||
* Utilities for logging on the Node.js side
|
||||
*
|
||||
* @author Hornwitser
|
||||
* @module lib/logging_utils
|
||||
*/
|
||||
"use strict";
|
||||
const events = require("events");
|
||||
const fs = require("fs-extra");
|
||||
@@ -9,6 +15,8 @@ const chalk = require("chalk");
|
||||
|
||||
const libLink = require("./link");
|
||||
const libErrors = require("./errors");
|
||||
const libFileOps = require("./file_ops");
|
||||
const { levels, logger } = require("./logging");
|
||||
const libStream = require("./stream");
|
||||
|
||||
|
||||
@@ -59,6 +67,7 @@ function formatServerOutput(parsed) {
|
||||
|
||||
/**
|
||||
* Formats winston log messages for a character terminal.
|
||||
* @static
|
||||
*/
|
||||
class TerminalFormat {
|
||||
constructor(options = {}) {
|
||||
@@ -99,6 +108,7 @@ class TerminalFormat {
|
||||
|
||||
/**
|
||||
* Sends logs over a lib/link connection.
|
||||
* @static
|
||||
*/
|
||||
class LinkTransport extends Transport {
|
||||
constructor(options) {
|
||||
@@ -125,6 +135,315 @@ class LinkTransport extends Transport {
|
||||
}
|
||||
}
|
||||
|
||||
const logFileGlob = /^[a-z]+-(\d{4}-\d{2}-\d{2})\.log$/;
|
||||
/**
|
||||
* Filter object for logs
|
||||
* @typedef {Object} module:lib/logging_utils~LogFilter
|
||||
* @property {string} [max_level] -
|
||||
* Maximum log level to include. Higher levels are more verbose.
|
||||
* @property {boolean} [all] -
|
||||
* Include log entries from master, all slaves and all instances.
|
||||
* @property {boolean} [master] -
|
||||
* Include log entries from the master server.
|
||||
* @property {Array<number>} [slave_ids] -
|
||||
* Include log entries for the given slaves and instances of those
|
||||
* slaves by id.
|
||||
* @property {Array<number>} [instance_ids] -
|
||||
* Include log entries for the given instances by id.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Keeps an index over a log directory to speed up queries to it
|
||||
* @static
|
||||
*/
|
||||
class LogIndex {
|
||||
constructor(logDirectory, serialized) {
|
||||
this.logDirectory = logDirectory;
|
||||
this.index = new Map();
|
||||
|
||||
if (serialized.version !== 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (let [file, serializedEntry] of Object.entries(serialized.files)) {
|
||||
this.index.set(file, {
|
||||
levels: new Set(serializedEntry.levels),
|
||||
master: serializedEntry.master,
|
||||
slave_ids: new Set(serializedEntry.slave_ids),
|
||||
instance_ids: new Set(serializedEntry.instance_ids),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
serialize() {
|
||||
let serialized = {
|
||||
version: 0,
|
||||
files: {},
|
||||
};
|
||||
|
||||
for (let [file, entry] of this.index) {
|
||||
serialized.files[file] = {
|
||||
levels: [...entry.levels],
|
||||
master: entry.master,
|
||||
slave_ids: [...entry.slave_ids],
|
||||
instance_ids: [...entry.instance_ids],
|
||||
};
|
||||
}
|
||||
|
||||
return serialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load or create an empty index for the given directory
|
||||
*
|
||||
* Loads index.json from the given log directy expecting it to contain
|
||||
* data about previously indexed files saved with {@link
|
||||
* module:lib/logging_utils.LogIndex#save}.
|
||||
*
|
||||
* @param {string} logDirectory - Path to directory to load from.
|
||||
* @returns {module:lib/logging_utils.LogIndex} loaded or created log index.
|
||||
*/
|
||||
static async load(logDirectory) {
|
||||
try {
|
||||
let content = await fs.readFile(path.join(logDirectory, "index.json"));
|
||||
return new LogIndex(logDirectory, JSON.parse(content));
|
||||
} catch (err) {
|
||||
if (err.code !== "ENOENT") {
|
||||
logger.warn(`Failed to load ${path.join(logDirectory, "index.json")}: ${err}`);
|
||||
}
|
||||
return new LogIndex(logDirectory, {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save index
|
||||
*
|
||||
* Saves corrently held index into index.json in the log directory it
|
||||
* was loaded/created from.
|
||||
*/
|
||||
async save() {
|
||||
await libFileOps.safeOutputFile(
|
||||
path.join(this.logDirectory, "index.json"),
|
||||
JSON.stringify(this.serialize(), null, 4)
|
||||
);
|
||||
}
|
||||
|
||||
async indexFile(file) {
|
||||
let filePath = path.join(this.logDirectory, file);
|
||||
let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
|
||||
let fileStream = fs.createReadStream(filePath);
|
||||
fileStream.pipe(lineStream);
|
||||
let entry = {
|
||||
levels: new Set(),
|
||||
master: false,
|
||||
slave_ids: new Set(),
|
||||
instance_ids: new Set(),
|
||||
};
|
||||
for await (let line of lineStream) {
|
||||
let info;
|
||||
try {
|
||||
info = JSON.parse(line);
|
||||
} catch (err) {
|
||||
entry.levels.add("info");
|
||||
entry.master = true;
|
||||
continue;
|
||||
}
|
||||
if (info.level) {
|
||||
entry.levels.add(info.level);
|
||||
}
|
||||
if (info.slave_id === undefined) {
|
||||
entry.master = true;
|
||||
} else if (info.instance_id === undefined) {
|
||||
entry.slave_ids.add(info.slave_id);
|
||||
}
|
||||
if (info.instance_id !== undefined) {
|
||||
entry.instance_ids.add(info.instance_id);
|
||||
}
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build index for files currently present in the log directory
|
||||
*/
|
||||
async buildIndex() {
|
||||
let today = new Date(Date.now() - 300e3).toISOString().slice(0, 10);
|
||||
|
||||
for (let file of await fs.readdir(this.logDirectory)) {
|
||||
let match = logFileGlob.exec(file);
|
||||
if (!match || match[1] >= today || this.index.has(file)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let entry = await this.indexFile(file);
|
||||
this.index.set(file, entry);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file includes entries for the given filter
|
||||
*
|
||||
* @param {string} file -
|
||||
* Name of file in the log directory to check for.
|
||||
* @param {module:lib/logging_utils~LogFilter} filter -
|
||||
* Filter to check index if file contains entries for.
|
||||
* @returns {boolean}
|
||||
* true if the file may contain entries included by the filter.
|
||||
*/
|
||||
filterIncludesFile(file, { max_level, all, master, instance_ids, slave_ids }) {
|
||||
let entry = this.index.get(file);
|
||||
if (!entry) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const hasOwnProperty = Object.prototype.hasOwnProperty;
|
||||
if (max_level && hasOwnProperty.call(levels, max_level)) {
|
||||
let fileLevels = [...entry.levels].map(
|
||||
fileLevel => (hasOwnProperty.call(levels, fileLevel) ? levels[fileLevel] : levels.verbose)
|
||||
);
|
||||
if (fileLevels.every(level => level > levels[max_level])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (all) {
|
||||
return true;
|
||||
}
|
||||
if (master && entry.master) {
|
||||
return true;
|
||||
}
|
||||
if (slave_ids && slave_ids.some(id => entry.slave_ids.has(id))) {
|
||||
return true;
|
||||
}
|
||||
if (instance_ids && instance_ids.some(id => entry.instance_ids.has(id))) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create log filter by level and source.
|
||||
*
|
||||
* @param {module:lib/logging_utils~LogFilter} filter -
|
||||
* Filter to filter log entries by.
|
||||
* @returns {function(object): boolean}
|
||||
* filter returning true for log entries that match it.
|
||||
* @static
|
||||
*/
|
||||
function logFilter({ all, master, slave_ids, instance_ids, max_level }) {
|
||||
return info => {
|
||||
// Note: reversed to filter out undefined levels
|
||||
if (max_level && !(levels[info.level] <= levels[max_level])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (all) {
|
||||
return true;
|
||||
}
|
||||
if (master && info.slave_id === undefined) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
slave_ids
|
||||
&& info.slave_id !== undefined
|
||||
&& info.instance_id === undefined
|
||||
&& slave_ids.includes(info.slave_id)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (instance_ids && info.instance_id !== undefined && instance_ids.includes(info.instance_id)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter object for querying logs
|
||||
* @typedef {Object} module:lib/logging_utils~QueryLogFilter
|
||||
* @property {number} [limit=100] - Maximum number of entries to return.
|
||||
* @property {string} [order="asc"] -
|
||||
* Return entries in ascending ("asc") date order or desceding ("desc")
|
||||
* date order.
|
||||
* @property {string} [max_level] -
|
||||
* Maximum log level to include. Higher levels are more verbose.
|
||||
* @property {boolean} [all] -
|
||||
* Include log entries from master, all slaves and all instances.
|
||||
* @property {boolean} [master] -
|
||||
* Include log entries from the master server.
|
||||
* @property {Array<number>} [slave_ids] -
|
||||
* Include log entries for the given slaves and instances of those
|
||||
* slaves by id.
|
||||
* @property {Array<number>} [instance_ids] -
|
||||
* Include log entries for the given instances by id.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Query log directory
|
||||
*
|
||||
* @param {string} logDirectory - path to directory with logs.
|
||||
* @param {module:lib/logging_utils~QueryLogFilter} filter -
|
||||
* Filter to limit logs by.
|
||||
* @param {module:lib/logging_utils.LogIndex=} index -
|
||||
* Index to speed up query with.
|
||||
* @returns {Array<Object>} log entries matching the filter
|
||||
*/
|
||||
async function queryLog(logDirectory, filter, index) {
|
||||
let files = (await fs.readdir(logDirectory)).filter(entry => logFileGlob.test(entry));
|
||||
|
||||
filter = {
|
||||
limit: 100,
|
||||
order: "asc",
|
||||
...filter,
|
||||
};
|
||||
|
||||
files.sort();
|
||||
if (filter.order === "desc") {
|
||||
files.reverse();
|
||||
}
|
||||
|
||||
const includeEntry = logFilter(filter);
|
||||
let log = [];
|
||||
for (let file of files) {
|
||||
if (index && !index.filterIncludesFile(file, filter)) {
|
||||
continue;
|
||||
}
|
||||
let filePath = path.join(logDirectory, file);
|
||||
let fileStream;
|
||||
let lineStream;
|
||||
if (filter.order === "asc") {
|
||||
lineStream = new libStream.LineSplitter({ readableObjectMode: true });
|
||||
fileStream = fs.createReadStream(filePath);
|
||||
} else {
|
||||
lineStream = new libStream.ReverseLineSplitter({ readableObjectMode: true });
|
||||
fileStream = await libStream.createReverseReadStream(filePath);
|
||||
}
|
||||
lineStream.on("data", line => {
|
||||
let info;
|
||||
try {
|
||||
info = JSON.parse(line);
|
||||
} catch (err) {
|
||||
info = { level: "info", message: line.toString("utf8") };
|
||||
}
|
||||
if (includeEntry(info)) {
|
||||
log.push(info);
|
||||
if (log.length >= filter.limit) {
|
||||
lineStream.destroy();
|
||||
fileStream.destroy();
|
||||
}
|
||||
}
|
||||
});
|
||||
fileStream.pipe(lineStream);
|
||||
await events.once(lineStream, "close");
|
||||
|
||||
if (log.length >= filter.limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return log;
|
||||
}
|
||||
|
||||
async function migrateLogs(log, directory, pattern) {
|
||||
const dateRegExp = /^\d{4}-\d{2}-\d{2}/;
|
||||
let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
|
||||
@@ -185,6 +504,9 @@ async function migrateLogs(log, directory, pattern) {
|
||||
module.exports = {
|
||||
TerminalFormat,
|
||||
LinkTransport,
|
||||
LogIndex,
|
||||
logFilter,
|
||||
queryLog,
|
||||
migrateLogs,
|
||||
|
||||
// for testing only
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
"use strict";
|
||||
const jwt = require("jsonwebtoken");
|
||||
const util = require("util");
|
||||
const winston = require("winston");
|
||||
|
||||
const libConfig = require("@clusterio/lib/config");
|
||||
const libErrors = require("@clusterio/lib/errors");
|
||||
const libLink = require("@clusterio/lib/link");
|
||||
const { levels, logger } = require("@clusterio/lib/logging");
|
||||
const { logger } = require("@clusterio/lib/logging");
|
||||
const libLoggingUtils = require("@clusterio/lib/logging_utils");
|
||||
const libPlugin = require("@clusterio/lib/plugin");
|
||||
const libPrometheus = require("@clusterio/lib/prometheus");
|
||||
@@ -402,29 +400,6 @@ class ControlConnection extends BaseConnection {
|
||||
this.updateLogSubscriptions();
|
||||
}
|
||||
|
||||
static logFilter({ all, master, slave_ids, instance_ids, max_level }) {
|
||||
return info => {
|
||||
// Note: reversed to filter out undefined levels
|
||||
if (max_level && !(levels[info.level] <= levels[max_level])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (all) {
|
||||
return true;
|
||||
}
|
||||
if (master && info.slave_id === undefined) {
|
||||
return true;
|
||||
}
|
||||
if (info.slave_id && slave_ids.includes(info.slave_id)) {
|
||||
return true;
|
||||
}
|
||||
if (info.instance_id && instance_ids.includes(info.instance_id)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
||||
|
||||
updateLogSubscriptions() {
|
||||
let { all, master, slave_ids, instance_ids } = this.logSubscriptions;
|
||||
if (all || master || slave_ids.length || instance_ids.length) {
|
||||
@@ -432,7 +407,7 @@ class ControlConnection extends BaseConnection {
|
||||
this.logTransport = new libLoggingUtils.LinkTransport({ link: this });
|
||||
this._master.clusterLogger.add(this.logTransport);
|
||||
}
|
||||
this.logTransport.filter = this.constructor.logFilter(this.logSubscriptions);
|
||||
this.logTransport.filter = libLoggingUtils.logFilter(this.logSubscriptions);
|
||||
|
||||
} else if (this.logTransport) {
|
||||
this._master.clusterLogger.remove(this.logTransport);
|
||||
@@ -441,34 +416,16 @@ class ControlConnection extends BaseConnection {
|
||||
}
|
||||
|
||||
async queryLogRequestHandler(message) {
|
||||
let transport = new winston.transports.File({ filename: "cluster.log" });
|
||||
let query = util.promisify((options, callback) => transport.query(options, callback));
|
||||
|
||||
// Available options and defaults inferred from reading the source
|
||||
// rows: number = limit || 10
|
||||
// limit: number // alias of rows.
|
||||
// start: number = 0
|
||||
// until: Date = now
|
||||
// from: Date = until - 24h
|
||||
// order: "asc"|"desc" = "desc"
|
||||
// fields: Array<string>
|
||||
// level: string
|
||||
// This interface is junk:
|
||||
// - The level option filters by exact match
|
||||
// - No way to add your own filter
|
||||
// - No way to stream results
|
||||
// - The log file is read starting from the beginning
|
||||
let setDuration = lastQueryLogTime.startTimer();
|
||||
let log = await query({
|
||||
order: message.data.order,
|
||||
limit: Infinity,
|
||||
from: new Date(0),
|
||||
});
|
||||
let { all, master, slave_ids, instance_ids } = message.data;
|
||||
|
||||
log = log.filter(this.constructor.logFilter(message.data));
|
||||
if (message.data.limit) {
|
||||
log = log.slice(0, message.data.limit);
|
||||
let log;
|
||||
if (!all && master && !slave_ids.length && !instance_ids.length) {
|
||||
log = await this._master.queryMasterLog(message.data);
|
||||
} else {
|
||||
log = await this._master.queryClusterLog(message.data);
|
||||
}
|
||||
|
||||
setDuration();
|
||||
return { log };
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ const libConfig = require("@clusterio/lib/config");
|
||||
const libErrors = require("@clusterio/lib/errors");
|
||||
const libFileOps = require("@clusterio/lib/file_ops");
|
||||
const libLink = require("@clusterio/lib/link");
|
||||
const libLoggingUtils = require("@clusterio/lib/logging_utils");
|
||||
const libPlugin = require("@clusterio/lib/plugin");
|
||||
const libPluginLoader = require("@clusterio/lib/plugin_loader");
|
||||
const libPrometheus = require("@clusterio/lib/prometheus");
|
||||
@@ -117,6 +118,14 @@ class Master {
|
||||
}
|
||||
|
||||
async _startInternal(args) {
|
||||
this.logDirectory = args.logDirectory;
|
||||
this.clusterLogIndex = await libLoggingUtils.LogIndex.load(path.join(this.logDirectory, "cluster"));
|
||||
this.clusterLogBuildInterval = setInterval(() => {
|
||||
this.clusterLogIndex.buildIndex().catch(
|
||||
err => logger.error(`Error building cluster log index:\n${err.stack}`)
|
||||
);
|
||||
}, 600e3);
|
||||
|
||||
// Start webpack development server if enabled
|
||||
if (args.dev || args.devPlugin) {
|
||||
logger.warn("Webpack development mode enabled");
|
||||
@@ -240,6 +249,10 @@ class Master {
|
||||
// This function should never throw.
|
||||
this._state = "stopping";
|
||||
logger.info("Stopping master");
|
||||
clearInterval(this.clusterLogBuildInterval);
|
||||
if (this.clusterLogIndex) {
|
||||
await this.clusterLogIndex.save();
|
||||
}
|
||||
|
||||
logger.info("Saving config");
|
||||
await libFileOps.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, 4));
|
||||
@@ -333,6 +346,33 @@ class Master {
|
||||
await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
|
||||
}
|
||||
|
||||
/**
|
||||
* Query master log
|
||||
*
|
||||
* @param {module:lib/logging_utils~QueryLogFilter} filter -
|
||||
* Filter to limit entries with. Note that only the master log can
|
||||
* be queried from this function.
|
||||
* @returns {Array<Object>} log entries matching the filter
|
||||
*/
|
||||
async queryMasterLog(filter) {
|
||||
return await libLoggingUtils.queryLog(
|
||||
path.join(this.logDirectory, "master"), filter,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query cluster log
|
||||
*
|
||||
* @param {module:lib/logging_utils~QueryLogFilter} filter -
|
||||
* Filter to limit entries with.
|
||||
* @returns {Array<Object>} log entries matching the filter
|
||||
*/
|
||||
async queryClusterLog(filter) {
|
||||
return await libLoggingUtils.queryLog(
|
||||
path.join(this.logDirectory, "cluster"), filter, this.clusterLogIndex,
|
||||
);
|
||||
}
|
||||
|
||||
static addAppRoutes(app, pluginInfos) {
|
||||
app.use((req, res, next) => {
|
||||
hitCounter.inc();
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
This file should be excluded by queries and the index
|
||||
@@ -0,0 +1,9 @@
|
||||
{"timestamp":"2021-12-10T10:00:00.000Z","level":"verbose","message":"Master start"}
|
||||
{"timestamp":"2021-12-10T10:00:01.000Z","level":"verbose","message":"Loading"}
|
||||
{"timestamp":"2021-12-10T10:00:03.000Z","level":"info","message":"Ready"}
|
||||
{"timestamp":"2021-12-10T10:00:10.000Z","slave_id":1,"level":"verbose","message":"Slave connected"}
|
||||
{"timestamp":"2021-12-10T10:00:11.000Z","slave_id":1,"level":"verbose","message":"Found instance 10"}
|
||||
{"timestamp":"2021-12-10T10:00:13.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Starting up"}
|
||||
{"timestamp":"2021-12-10T10:00:14.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Loading map"}
|
||||
{"timestamp":"2021-12-10T10:00:15.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Game ready"}
|
||||
{"timestamp":"2021-12-10T10:01:00.000Z","slave_id":1,"instance_id":10,"level":"error","message":"Unexpected crash"}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"timestamp":"2021-12-11T10:00:00.000Z","level":"verbose","message":"Master start"}
|
||||
{"timestamp":"2021-12-11T10:00:01.000Z","level":"verbose","message":"Loading"}
|
||||
{"timestamp":"2021-12-11T10:00:03.000Z","level":"fatal","message":"An error occured"}
|
||||
@@ -0,0 +1 @@
|
||||
{"timestamp":"2021-12-12T10:00:00.000Z","leve
|
||||
Whitespace-only changes.
@@ -1,8 +1,13 @@
|
||||
"use strict";
|
||||
const assert = require("assert").strict;
|
||||
const fs = require("fs-extra");
|
||||
const path = require("path");
|
||||
const util = require("util");
|
||||
|
||||
|
||||
const { testLines } = require("./factorio/lines");
|
||||
const libLoggingUtils = require("@clusterio/lib/logging_utils");
|
||||
const libStream = require("@clusterio/lib/stream");
|
||||
|
||||
|
||||
describe("lib/logging_utils.js", function() {
|
||||
@@ -16,4 +21,110 @@ describe("lib/logging_utils.js", function() {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let logDir = path.join("test", "file", "log");
|
||||
describe("class LogIndex", function() {
|
||||
it("should create a new empty index if index.json does not exist", async function() {
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
assert.deepEqual(logIndex.index, new Map());
|
||||
});
|
||||
it("should build index of only past log files", async function() {
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
await logIndex.buildIndex();
|
||||
assert.deepEqual(
|
||||
new Set(logIndex.index.keys()),
|
||||
new Set(["test-2021-12-10.log", "test-2021-12-11.log", "test-2021-12-12.log"])
|
||||
);
|
||||
});
|
||||
it("should roundtrip index on save load cycle", async function() {
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
await logIndex.buildIndex();
|
||||
let tempDir = path.join("temp", "test", "log");
|
||||
logIndex.logDirectory = tempDir;
|
||||
await logIndex.save();
|
||||
let loadedIndex = await libLoggingUtils.LogIndex.load(tempDir);
|
||||
assert.deepEqual(
|
||||
loadedIndex,
|
||||
logIndex
|
||||
);
|
||||
});
|
||||
it("should handle loading broken index", async function() {
|
||||
let tempDir = path.join("temp", "test", "log");
|
||||
await fs.outputFile(path.join(tempDir, "index.json"), "Broken JSON");
|
||||
let logIndex = await libLoggingUtils.LogIndex.load(tempDir);
|
||||
assert.deepEqual(logIndex.index, new Map());
|
||||
});
|
||||
});
|
||||
|
||||
let allLines = [];
|
||||
let logIndex = null;
|
||||
before(async function() {
|
||||
for (let file of (await fs.readdir(logDir)).sort()) {
|
||||
if (file === "excluded.txt") {
|
||||
continue;
|
||||
}
|
||||
let fileStream = fs.createReadStream(path.join(logDir, file));
|
||||
let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
|
||||
fileStream.pipe(lineStream);
|
||||
for await (let line of lineStream) {
|
||||
try {
|
||||
allLines.push(JSON.parse(line));
|
||||
} catch (err) {
|
||||
allLines.push({ "level": "info", "message": line.toString() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logIndex = await libLoggingUtils.LogIndex.load(logDir);
|
||||
await logIndex.buildIndex();
|
||||
});
|
||||
|
||||
describe("queryLog", function() {
|
||||
for (let index of [() => undefined, () => logIndex]) {
|
||||
describe(index() === undefined ? "without index" : "with index", function() {
|
||||
it("returns nothing with an empty filter", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, {}, index());
|
||||
assert.deepEqual(log, []);
|
||||
});
|
||||
it("should return the whole log when querying all", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { all: true }, index());
|
||||
assert.deepEqual(log, allLines);
|
||||
});
|
||||
it("should return the whole log in reverse when querying all desc", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { all: true, order: "desc" }, index());
|
||||
assert.deepEqual(log, [...allLines].reverse());
|
||||
});
|
||||
it("should limit by limit", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { limit: 2, all: true }, index());
|
||||
assert.deepEqual(log, allLines.slice(0, 2));
|
||||
});
|
||||
it("should filter by max_level", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { max_level: "info", all: true }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => ["fatal", "error", "warn", "audit", "info"].includes(info.level)
|
||||
));
|
||||
log = await libLoggingUtils.queryLog(logDir, { max_level: "fatal", all: true }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => ["fatal"].includes(info.level)
|
||||
));
|
||||
});
|
||||
it("should filter by master", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { master: true }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => info.slave_id === undefined && info.instance_id === undefined
|
||||
));
|
||||
});
|
||||
it("should filter by slave_ids", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { slave_ids: [1, 2] }, index());
|
||||
assert.deepEqual(log, allLines.filter(
|
||||
info => [1, 2].includes(info.slave_id) && info.instance_id === undefined
|
||||
));
|
||||
});
|
||||
it("should filter by instance_ids", async function() {
|
||||
let log = await libLoggingUtils.queryLog(logDir, { instance_ids: [10, 11] }, index());
|
||||
assert.deepEqual(log, allLines.filter(info => [10, 11].includes(info.instance_id)));
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user