Rewrite statistics export

Add automatic export of metrics from slaves to the master server.  And
rewrite the statistics export plugin to use this new system.
This commit is contained in:
Hornwitser committed 2019-12-20 21:26:29 +01:00
1 parent 8229d1955c
commit 26eefb0476
19 files changed
+334 -226

No files matched your search

@@ -1,9 +0,0 @@
/*
Clusterio plugin for gathering ingame statistics
*/
module.exports = {
name: "statisticsExporter",
version: "1.0.0",
description: "Gathers ingame stats such as production, consumption, kills and buildings placed",
masterPlugin: "masterPlugin.js"
}
-40
View File
@@ -1,40 +0,0 @@
const needle = require("needle");
module.exports = class remoteCommands {
constructor(mergedConfig, messageInterface, extras){
this.messageInterface = messageInterface;
this.config = mergedConfig;
const needleOptionsWithTokenAuthHeader = {
headers: {
'x-access-token': this.config.masterAuthToken
},
};
// Gather and submit statistics every 15 seconds
setInterval(async () => {
let stats = await this.getStats();
stats.forEach(force => {
// I have no idea why I need to do JSON.stringify before passing it to needle.post,
// but if you do forceData: force.data, it doesn't work...
let request = {
instanceID: this.config.unique,
force: force.forceName,
forceData: JSON.stringify(force.data),
};
needle.post(`${mergedConfig.masterURL}/api/logStats`,
request,
needleOptionsWithTokenAuthHeader, (err, resp) => {
if(!err && resp.body && resp.body.ok == false) console.log("Error calling /api/logStats: " + resp.body);
}
);
});
}, 15000);
}
async getStats() {
let string = await this.messageInterface('/sc remote.call("statisticsExporter", "export")');
return JSON.parse(string);
}
}
@@ -1,30 +0,0 @@
local statistics = {
"item_production_statistics",
"fluid_production_statistics",
"kill_count_statistics",
"entity_build_count_statistics",
}
local function export()
local stats = {}
for _, force in pairs(game.forces) do
local data = {}
for _, statName in pairs(statistics) do
data[statName] = {
input = force[statName].input_counts,
output = force[statName].output_counts,
}
end
table.insert(stats, {
forceName = force.name,
data = data,
})
end
rcon.print(game.table_to_json(stats))
end
return {
add_remote_interface = function()
remote.add_interface("statisticsExporter", { export = export })
end
}
@@ -1,67 +0,0 @@
const fs = require("fs-extra");
const path = require("path");
class masterPlugin {
constructor({config, socketio, express, Prometheus}){
const authenticate = require("lib/authenticate");
this.config = config;
this.io = socketio;
const app = express;
const prometheusStatisticsGauge = new Prometheus.Gauge({
name: 'clusterio_statistics_gauge',
help: 'Items produced/built/killed by a force',
labelNames: ["instanceID", "force", "statistic", "direction", "itemName"],
});
/** --- TODO: Replace this with a simpler Prometheus exporter. Requires changes to the mod as well.
POST endpoint to log production graph statistics. Should contain an instanceID (also reffered to as "unique")
and of course the timeSeries data organized by category.
@memberof clusterioMaster
@instance
@alias api/logStats
@param {object} JSON {instanceID: "string", force: "string" data: {"item_production_statistics": {"input":{"iron-plate":150}, "output":{}}, ...}}
@returns {string} failure
*/
app.post("/api/logStats", authenticate.middleware, function(req,res) {
let data = req.body.forceData;
if(typeof req.body.forceData == "string") {
data = JSON.parse(req.body.forceData);
}
// endpointHitCounter.labels(req.route.path).inc();
if(typeof req.body == "object"
&& req.body.instanceID
&& typeof req.body.instanceID == "string"
&& req.body.instanceID.length < 100
&& data
&& typeof data == "object"
&& !Array.isArray(data)){
try{
Object.keys(data).forEach(statisticName => {
Object.keys(data[statisticName]).forEach(direction => {
let statHash = data[statisticName][direction];
Object.keys(statHash).forEach(item => {
let value = Number(statHash[item]);
if(value) prometheusStatisticsGauge.labels(req.body.instanceID, req.body.force, statisticName, direction, item).set(value);
});
});
});
res.send({ok:true, msg:`Statistics successfully reported at ${Date.now()}`});
}catch(e){
res.send({ok:false, msg:"Statistics reporter crashed, see sharedPlugins/statisticsExporter/masterPlugin.js"});
console.log(e)
};
} else {
console.log(`/api/logStats: Invalid request. instanceID: ${req.body.instanceID}, dataIsArray: `
+ `${Array.isArray(data)}, ${typeof req.body.instanceID}, ${typeof data}`);
res.send({
ok:false,
msg:"Invalid request parameters"
});
}
});
}
}
module.exports = masterPlugin;
@@ -1,5 +0,0 @@
// These are libraries that does not have any units test, they are
// required here in order to track coverage for them.
require('../index');
require('../masterPlugin');