Merge pull request #271 from Hornwitser/remove-old-stats

Remove old statistics code
This commit is contained in:
Arvids Godjuks authored and GitHub committed 2019-12-25 12:02:18 +02:00
commit ac3aa5ffee
10 files changed
-349

No files matched your search

-5
View File
@@ -29,11 +29,6 @@
"public": true, "public": true,
"lan": true "lan": true
}, },
"__comment6": "Statistic gathering statistics, may affect server performance (master memory usage especially)",
"itemStats": {
"maxEntries":60,
"entriesPerSecond": 0.01666666667
},
"logItemTransfers": false, "logItemTransfers": false,
"__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.", "__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.",
"disableFairItemDistribution": false, "disableFairItemDistribution": false,
-4
View File
@@ -75,10 +75,6 @@ let downloadList = [
url: "https://code.jquery.com/ui/1.12.1/jquery-ui.js", url: "https://code.jquery.com/ui/1.12.1/jquery-ui.js",
file: "static/external/jquery-ui.js", file: "static/external/jquery-ui.js",
}, },
{
url: "https://cdnjs.cloudflare.com/ajax/libs/canvasjs/1.7.0/canvasjs.js",
file: "static/external/canvasjs.js",
},
{ {
url: "https://momentjs.com/downloads/moment.min.js", url: "https://momentjs.com/downloads/moment.min.js",
file: "static/external/moment.min.js", file: "static/external/moment.min.js",
-1
View File
@@ -30,7 +30,6 @@
}, },
"dependencies": { "dependencies": {
"as-table": "^1.0.31", "as-table": "^1.0.31",
"averaged-timeseries": "^1.4.0",
"base64url": "^3.0.0", "base64url": "^3.0.0",
"bcrypt": "=3.0.6", "bcrypt": "=3.0.6",
"bcrypt-promise": "^2.0.0", "bcrypt-promise": "^2.0.0",
-32
View File
@@ -1,5 +1,3 @@
const averagedTimeSeries = require("averaged-timeseries")
const doleNN = require("./dole_nn_base") const doleNN = require("./dole_nn_base")
const Prometheus = require('prom-client'); const Prometheus = require('prom-client');
@@ -58,7 +56,6 @@ class neuralDole {
res, res,
object, object,
config, config,
sentItemStatisticsBySlaveID,
prometheusImportGauge prometheusImportGauge
}){ }){
let magicData = doleNN.Dose( let magicData = doleNN.Dose(
@@ -89,20 +86,6 @@ class neuralDole {
if(config.logItemTransfers){ if(config.logItemTransfers){
console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName); console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
} }
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
});
}
sentItemStatistics.add({
key:object.name,
value:magicData[0],
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0); prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
res.send({ res.send({
@@ -118,7 +101,6 @@ function doleDivider({
itemCount, itemCount,
object, object,
items, items,
sentItemStatisticsBySlaveID,
config, config,
prometheusDoleFactorGauge, prometheusDoleFactorGauge,
prometheusImportGauge, prometheusImportGauge,
@@ -142,20 +124,6 @@ function doleDivider({
console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName); console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
} }
items.removeItem(object.name, object.count) items.removeItem(object.name, object.count)
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
});
}
sentItemStatistics.add({
key:object.name,
value:object.count,
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0); prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);
-33
View File
@@ -1,7 +1,3 @@
var recievedItemStatisticsBySlaveID = {};
var sentItemStatisticsBySlaveID = {};
const averagedTimeSeries = require("averaged-timeseries");
const express = require("express"); const express = require("express");
const path = require("path"); const path = require("path");
@@ -87,19 +83,6 @@ function addApiRoutes(
console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")"); console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
} }
// gather statistics // gather statistics
let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
if(recievedItemStatistics === undefined){
recievedItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
});
recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
}
recievedItemStatistics.add({
key:req.body.name,
value:req.body.count,
});
prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
// save items we get // save items we get
var count = Number(req.body.count); var count = Number(req.body.count);
@@ -160,27 +143,12 @@ function addApiRoutes(
// track statistics and do graphing things // track statistics and do graphing things
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0); prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
});
}
sentItemStatistics.add({
key:object.name,
value:numberToRemove,
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
} else if(config.useNeuralNetDoleDivider){ } else if(config.useNeuralNetDoleDivider){
// use fancy neural net to calculate a "fair" dole division rate. // use fancy neural net to calculate a "fair" dole division rate.
neuralDole.divider({ neuralDole.divider({
res, res,
object, object,
config, config,
sentItemStatisticsBySlaveID,
prometheusImportGauge prometheusImportGauge
}) })
} else { } else {
@@ -189,7 +157,6 @@ function addApiRoutes(
itemCount, itemCount,
object, object,
db, db,
sentItemStatisticsBySlaveID,
config, config,
prometheusDoleFactorGauge, prometheusDoleFactorGauge,
prometheusImportGauge, prometheusImportGauge,
@@ -21,7 +21,6 @@ describe("clusterioMod endpoints", function() {
app.use(bodyParser.json({ limit: '10mb', })); app.use(bodyParser.json({ limit: '10mb', }));
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true })); app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
const config = { const config = {
itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 },
disableFairItemDistribution: true, disableFairItemDistribution: true,
}; };
const items = new database.ItemDatabase(); const items = new database.ItemDatabase();
-1
View File
@@ -1,7 +1,6 @@
(function (){ (function (){
ensureSetting("Display offline slaves", "boolean", false); ensureSetting("Display offline slaves", "boolean", false);
ensureSetting("Enable production graphs", "boolean", true);
ensureSetting("Periodically update storage screen", "boolean", true); ensureSetting("Periodically update storage screen", "boolean", true);
}()) }())
-117
View File
@@ -1,24 +1,3 @@
// string, object, function(object)
function post(url, data, callback) {
console.log("POST " + url + JSON.stringify(data));
var xhr = new XMLHttpRequest();
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-type", "application/json");
xhr.onreadystatechange = function () {
if (xhr.readyState === 4 && xhr.status === 200) {
var json = JSON.parse(xhr.responseText);
callback(json);
}
};
xhr.send(JSON.stringify(data));
}
// function to sort arrays of objects after a keys value
function sortByKey(array, key) {
return array.sort(function(a, b) {
var x = Number(a[key]); var y = Number(b[key]);
return ((x < y) ? 1 : ((x > y) ? -1 : 0));
});
}
function getParameterByName(name, url) { function getParameterByName(name, url) {
if (!url) url = window.location.href; if (!url) url = window.location.href;
name = name.replace(/[\[\]]/g, "\\$&"); name = name.replace(/[\[\]]/g, "\\$&");
@@ -158,8 +137,6 @@ function populateSlaveInfo(){
HTML += "</div>"; // end of left container HTML += "</div>"; // end of left container
// chart
HTML += '<div id="' + slave.unique + '" class="productionGraph" style="width: calc(100% - 300px);"></div>';
// terminal // terminal
// HTML += '<div id="terminal"></div>'; // HTML += '<div id="terminal"></div>';
@@ -167,9 +144,6 @@ function populateSlaveInfo(){
document.querySelector("#body > #details").innerHTML = HTML; document.querySelector("#body > #details").innerHTML = HTML;
// make production graph
makeGraph(slave.unique, slave.unique)
// makeTerminal(); // makeTerminal();
$('#fetch-mod-data').on("click", downloadSaveModData); $('#fetch-mod-data').on("click", downloadSaveModData);
@@ -279,94 +253,3 @@ function maximizeTerminal(instant){
setTimeout(() => $('#terminal')[0].style.transition = 'none', (Number(instant) || 200)); setTimeout(() => $('#terminal')[0].style.transition = 'none', (Number(instant) || 200));
$('#terminal').draggable('enable'); $('#terminal').draggable('enable');
} }
// ID of slave, ID of canvasjs div without #
function makeGraph(instanceID, selector) {
let chartIgnoreList = [
"water",
"steam"
];
post("api/getStats", {instanceID: instanceID}, function(data){
//console.log("Building chart " + instanceID + " with this data:")
//console.log(data)
if(data.length > 0) {
// find keys
let itemNames = [];
for(let key in data[data.length - 1].data) {
itemNames[itemNames.length] = key
}
let chartData = [];
for(let o = 0; o < itemNames.length; o++) {
if(!chartIgnoreList.includes(itemNames[o])){
chartData[chartData.length] = generateLineChartArray(data, itemNames[o]);
}
}
//console.log(chartData)
drawChart(selector, chartData)
}
// callback(data);
})
}
function generateLineChartArray(data, nameKey) {
let chartData = [];
for(let i = 0; i < data.length; i++) {
// only show recent data
if(data[i].timestamp > Date.now() - (24*60*60*1000)){
let y = data[i].data[nameKey];
if(!data[i].data[nameKey]) {
y = 0;
} else if(y < 0) {
y = 0;
}
chartData[chartData.length] = {
x: new Date(data[i].timestamp),
y: Number(y)
}
}
}
chartData = sortByKey(chartData, "x");
let xyz = {};
xyz.name = nameKey;
xyz.type = "line";
if(nameKey === "copper-wire"||nameKey === "iron-plate"||nameKey === "copper-plate"||nameKey === "electronic-circuit"||nameKey === "steel-plate"||nameKey === "advanced-circuit"||nameKey === "crude-oil"||nameKey === "petroleum-gas"){
xyz.showInLegend = true;
}
xyz.dataPoints = chartData;
return xyz;
}
function drawChart(selector, chartData, title) {
// selector is ID of element, ex "chartContainer" or "-123199123"
console.log(chartData);
chartsByID[selector] = new CanvasJS.Chart(selector, {
title:{
text: title || "Production graph"
},
toolTip:{
content: "{name}: {y}"
},
zoomEnabled: true,
axisY:{
includeZero: true,
},
legend: {
cursor: "pointer",
itemclick: function (e) {
//console.log("legend click: " + e.dataPointIndex);
//console.log(e);
if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
e.dataSeries.visible = false;
} else {
e.dataSeries.visible = true;
}
e.chart.render();
}
},
data: chartData
});
chart = chartsByID[selector];
// console.log(chart)
chart.render();
return chart;
}
-154
View File
@@ -13,168 +13,14 @@ setTimeout(function() {
if(Date.now() - slaveData[key].time < 120000 || displayOffline) { if(Date.now() - slaveData[key].time < 120000 || displayOffline) {
let seenDate = moment(Number(slaveData[key].time)).format('DD.MM.YY, hh:mm:ss') let seenDate = moment(Number(slaveData[key].time)).format('DD.MM.YY, hh:mm:ss')
HTML += "<div class='slaveBox'>"; HTML += "<div class='slaveBox'>";
HTML += '<div id="' + slaveData[key].unique + '" class="productionGraph" style="width: calc(100% - 200px);"></div>';
HTML += "<a href='nodeDetails?instanceID="+slaveData[key].unique+"'><h2>" + slaveData[key].instanceName + "</h2></a><p>ID: " + slaveData[key].unique + "</p><p>Last seen: "+seenDate+"</p><p>Online players: "+slaveData[key].playerCount+"</p><p>IP: "+slaveData[key].publicIP +":"+ slaveData[key].serverPort+"</p>" HTML += "<a href='nodeDetails?instanceID="+slaveData[key].unique+"'><h2>" + slaveData[key].instanceName + "</h2></a><p>ID: " + slaveData[key].unique + "</p><p>Last seen: "+seenDate+"</p><p>Online players: "+slaveData[key].playerCount+"</p><p>IP: "+slaveData[key].publicIP +":"+ slaveData[key].serverPort+"</p>"
HTML += "<p>Host: "+slaveData[key].mac+"</p>"; HTML += "<p>Host: "+slaveData[key].mac+"</p>";
HTML += "<br><p>Graph tools:</p>";
HTML += "<button style='width:100px;' onclick='hideAllDatasets(\"" + slaveData[key].unique + "\")\'>Hide all</button>";
HTML += "<button style='width:100px;' onclick='showAllDatasets(\"" + slaveData[key].unique + "\")\'>Show all</button>";
HTML += "</div>"; HTML += "</div>";
} }
} }
document.querySelector("#slaves > #display").innerHTML = HTML document.querySelector("#slaves > #display").innerHTML = HTML
if(JSON.parse(localStorage.settings)["Enable production graphs"]) {
let graphs = document.querySelectorAll(".productionGraph");
for(let i = 0; i < graphs.length; i++) {
// execute functions to make graphs
// ID of slave, ID of canvasjs div without #
makeGraph(graphs[i].id, graphs[i].id)
}
}
} }
} }
xmlhttp.open("GET", "api/slaves", true); xmlhttp.open("GET", "api/slaves", true);
xmlhttp.send(); xmlhttp.send();
}, 0) }, 0)
let chartIgnoreList = [
"water",
"steam"
]
// ID of slave, ID of canvasjs div without #
function makeGraph(instanceID, selector) {
post("api/getTimelineStats", {instanceID: instanceID}, function(pointsInTime) {
//console.log("Building chart " + instanceID + " with this data:")
//Find all keys at all points in time.
let itemNames = {};
pointsInTime.forEach(function(point) {
for(let key in point.data) {
itemNames[key] = true;
}
});
itemNames = Object.keys(itemNames);
//Generate a chart list for each item.
const displayNames = itemNames.filter(function(name) {
return !chartIgnoreList.includes(name);
});
const chartData = displayNames.map(function(name) {
return generateLineChartArray(pointsInTime, name)
});
chartData.length && drawChart(selector, chartData)
})
}
// production info
// string, object, function(object)
function post(url, data, callback) {
console.log("POST " + url, data);
var xhr = new XMLHttpRequest();
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-type", "application/json");
xhr.onreadystatechange = function () {
if (xhr.readyState == 4 && xhr.status == 200) {
var json = JSON.parse(xhr.responseText);
callback(json);
}
}
xhr.send(JSON.stringify(data));
}
function generateLineChartArray(data, nameKey) {
const legendKeys = ["copper-cable", "iron-plate", "copper-plate", "electronic-circuit", "steel-plate", "advanced-circuit", "crude-oil", "petroleum-gas"];
let chartData = [];
for(let i = 0; i < data.length; i++) {
let y = data[i].data[nameKey]
if(!data[i].data[nameKey]) {
y = 0;
} else if(y < 0) {
y = 0;
}
chartData[chartData.length] = {
x: new Date(data[i].timestamp),
y: Number(y)
}
}
chartData = sortByKey(chartData, "x");
let xyz = {};
xyz.name = nameKey;
xyz.type = "line";
if(legendKeys.indexOf(nameKey) >= 0){
xyz.showInLegend = true;
}
xyz.dataPoints = chartData;
return xyz;
}
var chartsByID = {};
function drawChart(selector, chartData) {
// selector is ID of element, ex "chartContainer" or "-123199123"
// console.log(chartData)
chartsByID[selector] = new CanvasJS.Chart(selector, {
title:{
text: "Production graph"
},
toolTip:{
content: "{name}: {y}"
},
zoomEnabled: true,
axisY:{
includeZero: true,
},
legend: {
cursor: "pointer",
itemclick: function (e) {
//console.log("legend click: " + e.dataPointIndex);
//console.log(e);
if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
e.dataSeries.visible = false;
} else {
e.dataSeries.visible = true;
}
e.chart.render();
}
},
data: chartData
});
chart = chartsByID[selector];
// console.log(chart)
chart.render();
}
function hideAllDatasets(chartID) {
if(chartsByID[chartID]) {
let chart = chartsByID[chartID];
for(let i = 0; i < chart.options.data.length; i++) {
chart.options.data[i].visible = false;
}
chart.render();
} else {
return false;
}
}
function showAllDatasets(chartID) {
if(chartsByID[chartID]) {
let chart = chartsByID[chartID];
for(let i = 0; i < chart.options.data.length; i++) {
chart.options.data[i].visible = true;
}
chart.render();
} else {
return false;
}
}
// function to sort arrays of objects after a keys value
function sortByKey(array, key) {
return array.sort(function(a, b) {
var x = Number(a[key]); var y = Number(b[key]);
return ((x < y) ? 1 : ((x > y) ? -1 : 0));
});
}
-1
View File
@@ -2,7 +2,6 @@
<link rel="stylesheet" type="text/css" href="/style.css"> <link rel="stylesheet" type="text/css" href="/style.css">
<script src="/external/jquery.js"></script> <script src="/external/jquery.js"></script>
<script src="/external/jquery-ui.js"></script> <script src="/external/jquery-ui.js"></script>
<script src="/external/canvasjs.js"></script>
<script src="/external/moment.min.js"></script> <script src="/external/moment.min.js"></script>
<script src="/external/mousetrap.min.js"></script> <script src="/external/mousetrap.min.js"></script>
<script src="/main.js"></script> <script src="/main.js"></script>