mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-21 08:03:13 +09:00
Replace log query implementation
The interface provided by Winston is unusably slow for querying large logs. Reimplement the log querying with both forwards and backwards line reading and a simple file based index to skip files which do not contain any lines matching the filter.
This commit is contained in:
1 parent
fd1f3329bd
commit
601ebba919
10 files changed
+497
-54
No files matched your search
@@ -12,6 +12,7 @@ const libConfig = require("@clusterio/lib/config");
|
||||
const libErrors = require("@clusterio/lib/errors");
|
||||
const libFileOps = require("@clusterio/lib/file_ops");
|
||||
const libLink = require("@clusterio/lib/link");
|
||||
const libLoggingUtils = require("@clusterio/lib/logging_utils");
|
||||
const libPlugin = require("@clusterio/lib/plugin");
|
||||
const libPluginLoader = require("@clusterio/lib/plugin_loader");
|
||||
const libPrometheus = require("@clusterio/lib/prometheus");
|
||||
@@ -117,6 +118,14 @@ class Master {
|
||||
}
|
||||
|
||||
async _startInternal(args) {
|
||||
this.logDirectory = args.logDirectory;
|
||||
this.clusterLogIndex = await libLoggingUtils.LogIndex.load(path.join(this.logDirectory, "cluster"));
|
||||
this.clusterLogBuildInterval = setInterval(() => {
|
||||
this.clusterLogIndex.buildIndex().catch(
|
||||
err => logger.error(`Error building cluster log index:\n${err.stack}`)
|
||||
);
|
||||
}, 600e3);
|
||||
|
||||
// Start webpack development server if enabled
|
||||
if (args.dev || args.devPlugin) {
|
||||
logger.warn("Webpack development mode enabled");
|
||||
@@ -240,6 +249,10 @@ class Master {
|
||||
// This function should never throw.
|
||||
this._state = "stopping";
|
||||
logger.info("Stopping master");
|
||||
clearInterval(this.clusterLogBuildInterval);
|
||||
if (this.clusterLogIndex) {
|
||||
await this.clusterLogIndex.save();
|
||||
}
|
||||
|
||||
logger.info("Saving config");
|
||||
await libFileOps.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, 4));
|
||||
@@ -333,6 +346,33 @@ class Master {
|
||||
await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
|
||||
}
|
||||
|
||||
/**
|
||||
* Query master log
|
||||
*
|
||||
* @param {module:lib/logging_utils~QueryLogFilter} filter -
|
||||
* Filter to limit entries with. Note that only the master log can
|
||||
* be queried from this function.
|
||||
* @returns {Array<Object>} log entries matching the filter
|
||||
*/
|
||||
async queryMasterLog(filter) {
|
||||
return await libLoggingUtils.queryLog(
|
||||
path.join(this.logDirectory, "master"), filter,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query cluster log
|
||||
*
|
||||
* @param {module:lib/logging_utils~QueryLogFilter} filter -
|
||||
* Filter to limit entries with.
|
||||
* @returns {Array<Object>} log entries matching the filter
|
||||
*/
|
||||
async queryClusterLog(filter) {
|
||||
return await libLoggingUtils.queryLog(
|
||||
path.join(this.logDirectory, "cluster"), filter, this.clusterLogIndex,
|
||||
);
|
||||
}
|
||||
|
||||
static addAppRoutes(app, pluginInfos) {
|
||||
app.use((req, res, next) => {
|
||||
hitCounter.inc();
|
||||
|
||||
Reference in new issue
Block a user