Replace log query implementation

The interface provided by Winston is unusably slow for querying large
logs.  Reimplement the log querying with both forwards and backwards
line reading and a simple file based index to skip files which do not
contain any lines matching the filter.
This commit is contained in:
Hornwitser committed 2021-12-19 02:06:59 +01:00
1 parent fd1f3329bd
commit 601ebba919
10 files changed
+497 -54

No files matched your search

+9 -52
View File
@@ -1,12 +1,10 @@
"use strict";
const jwt = require("jsonwebtoken");
const util = require("util");
const winston = require("winston");
const libConfig = require("@clusterio/lib/config");
const libErrors = require("@clusterio/lib/errors");
const libLink = require("@clusterio/lib/link");
const { levels, logger } = require("@clusterio/lib/logging");
const { logger } = require("@clusterio/lib/logging");
const libLoggingUtils = require("@clusterio/lib/logging_utils");
const libPlugin = require("@clusterio/lib/plugin");
const libPrometheus = require("@clusterio/lib/prometheus");
@@ -402,29 +400,6 @@ class ControlConnection extends BaseConnection {
this.updateLogSubscriptions();
}
static logFilter({ all, master, slave_ids, instance_ids, max_level }) {
return info => {
// Note: reversed to filter out undefined levels
if (max_level && !(levels[info.level] <= levels[max_level])) {
return false;
}
if (all) {
return true;
}
if (master && info.slave_id === undefined) {
return true;
}
if (info.slave_id && slave_ids.includes(info.slave_id)) {
return true;
}
if (info.instance_id && instance_ids.includes(info.instance_id)) {
return true;
}
return false;
};
}
updateLogSubscriptions() {
let { all, master, slave_ids, instance_ids } = this.logSubscriptions;
if (all || master || slave_ids.length || instance_ids.length) {
@@ -432,7 +407,7 @@ class ControlConnection extends BaseConnection {
this.logTransport = new libLoggingUtils.LinkTransport({ link: this });
this._master.clusterLogger.add(this.logTransport);
}
this.logTransport.filter = this.constructor.logFilter(this.logSubscriptions);
this.logTransport.filter = libLoggingUtils.logFilter(this.logSubscriptions);
} else if (this.logTransport) {
this._master.clusterLogger.remove(this.logTransport);
@@ -441,34 +416,16 @@ class ControlConnection extends BaseConnection {
}
async queryLogRequestHandler(message) {
let transport = new winston.transports.File({ filename: "cluster.log" });
let query = util.promisify((options, callback) => transport.query(options, callback));
// Available options and defaults inferred from reading the source
// rows: number = limit || 10
// limit: number // alias of rows.
// start: number = 0
// until: Date = now
// from: Date = until - 24h
// order: "asc"|"desc" = "desc"
// fields: Array<string>
// level: string
// This interface is junk:
// - The level option filters by exact match
// - No way to add your own filter
// - No way to stream results
// - The log file is read starting from the beginning
let setDuration = lastQueryLogTime.startTimer();
let log = await query({
order: message.data.order,
limit: Infinity,
from: new Date(0),
});
let { all, master, slave_ids, instance_ids } = message.data;
log = log.filter(this.constructor.logFilter(message.data));
if (message.data.limit) {
log = log.slice(0, message.data.limit);
let log;
if (!all && master && !slave_ids.length && !instance_ids.length) {
log = await this._master.queryMasterLog(message.data);
} else {
log = await this._master.queryClusterLog(message.data);
}
setDuration();
return { log };
}
+40
View File
@@ -12,6 +12,7 @@ const libConfig = require("@clusterio/lib/config");
const libErrors = require("@clusterio/lib/errors");
const libFileOps = require("@clusterio/lib/file_ops");
const libLink = require("@clusterio/lib/link");
const libLoggingUtils = require("@clusterio/lib/logging_utils");
const libPlugin = require("@clusterio/lib/plugin");
const libPluginLoader = require("@clusterio/lib/plugin_loader");
const libPrometheus = require("@clusterio/lib/prometheus");
@@ -117,6 +118,14 @@ class Master {
}
async _startInternal(args) {
this.logDirectory = args.logDirectory;
this.clusterLogIndex = await libLoggingUtils.LogIndex.load(path.join(this.logDirectory, "cluster"));
this.clusterLogBuildInterval = setInterval(() => {
this.clusterLogIndex.buildIndex().catch(
err => logger.error(`Error building cluster log index:\n${err.stack}`)
);
}, 600e3);
// Start webpack development server if enabled
if (args.dev || args.devPlugin) {
logger.warn("Webpack development mode enabled");
@@ -240,6 +249,10 @@ class Master {
// This function should never throw.
this._state = "stopping";
logger.info("Stopping master");
clearInterval(this.clusterLogBuildInterval);
if (this.clusterLogIndex) {
await this.clusterLogIndex.save();
}
logger.info("Saving config");
await libFileOps.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, 4));
@@ -333,6 +346,33 @@ class Master {
await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
}
/**
* Query master log
*
* @param {module:lib/logging_utils~QueryLogFilter} filter -
* Filter to limit entries with. Note that only the master log can
* be queried from this function.
* @returns {Array<Object>} log entries matching the filter
*/
async queryMasterLog(filter) {
return await libLoggingUtils.queryLog(
path.join(this.logDirectory, "master"), filter,
);
}
/**
* Query cluster log
*
* @param {module:lib/logging_utils~QueryLogFilter} filter -
* Filter to limit entries with.
* @returns {Array<Object>} log entries matching the filter
*/
async queryClusterLog(filter) {
return await libLoggingUtils.queryLog(
path.join(this.logDirectory, "cluster"), filter, this.clusterLogIndex,
);
}
static addAppRoutes(app, pluginInfos) {
app.use((req, res, next) => {
hitCounter.inc();