From f700e98e8e0823542dad77a276573b4159b62eb2 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 21 Dec 2019 21:52:53 +0100 Subject: [PATCH] Remove old statistics code Remove left over code from the old statistics gathering and display which was replaced with Prometheus+Grafana. --- config.json.dist | 5 - lib/npmPostinstall.js | 4 - package.json | 1 - sharedPlugins/clusterioMod/dole.js | 32 ----- sharedPlugins/clusterioMod/routes.js | 33 ----- sharedPlugins/clusterioMod/test/routes.js | 1 - static/main.js | 1 - static/nodeDetails.js | 117 ---------------- static/slaveDisplay.js | 154 ---------------------- views/head.ejs | 1 - 10 files changed, 349 deletions(-) diff --git a/config.json.dist b/config.json.dist index 8b8bdb99..333a81b1 100644 --- a/config.json.dist +++ b/config.json.dist @@ -29,11 +29,6 @@ "public": true, "lan": true }, - "__comment6": "Statistic gathering statistics, may affect server performance (master memory usage especially)", - "itemStats": { - "maxEntries":60, - "entriesPerSecond": 0.01666666667 - }, "logItemTransfers": false, "__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.", "disableFairItemDistribution": false, diff --git a/lib/npmPostinstall.js b/lib/npmPostinstall.js index bdd539bf..73095cac 100644 --- a/lib/npmPostinstall.js +++ b/lib/npmPostinstall.js @@ -75,10 +75,6 @@ let downloadList = [ url: "https://code.jquery.com/ui/1.12.1/jquery-ui.js", file: "static/external/jquery-ui.js", }, - { - url: "https://cdnjs.cloudflare.com/ajax/libs/canvasjs/1.7.0/canvasjs.js", - file: "static/external/canvasjs.js", - }, { url: "https://momentjs.com/downloads/moment.min.js", file: "static/external/moment.min.js", diff --git a/package.json b/package.json index e69f172d..aed6e9f0 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ }, "dependencies": { "as-table": "^1.0.31", - "averaged-timeseries": "^1.4.0", "base64url": "^3.0.0", "bcrypt": "=3.0.6", "bcrypt-promise": "^2.0.0", diff --git a/sharedPlugins/clusterioMod/dole.js b/sharedPlugins/clusterioMod/dole.js index 5f1f197b..19f9189f 100644 --- a/sharedPlugins/clusterioMod/dole.js +++ b/sharedPlugins/clusterioMod/dole.js @@ -1,5 +1,3 @@ -const averagedTimeSeries = require("averaged-timeseries") - const doleNN = require("./dole_nn_base") const Prometheus = require('prom-client'); @@ -58,7 +56,6 @@ class neuralDole { res, object, config, - sentItemStatisticsBySlaveID, prometheusImportGauge }){ let magicData = doleNN.Dose( @@ -89,20 +86,6 @@ class neuralDole { if(config.logItemTransfers){ console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName); } - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }); - } - sentItemStatistics.add({ - key:object.name, - value:magicData[0], - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0); res.send({ @@ -118,7 +101,6 @@ function doleDivider({ itemCount, object, items, - sentItemStatisticsBySlaveID, config, prometheusDoleFactorGauge, prometheusImportGauge, @@ -142,20 +124,6 @@ function doleDivider({ console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName); } items.removeItem(object.name, object.count) - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }); - } - sentItemStatistics.add({ - key:object.name, - value:object.count, - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0); diff --git a/sharedPlugins/clusterioMod/routes.js b/sharedPlugins/clusterioMod/routes.js index db7ee7a4..bd0baece 100644 --- a/sharedPlugins/clusterioMod/routes.js +++ b/sharedPlugins/clusterioMod/routes.js @@ -1,7 +1,3 @@ -var recievedItemStatisticsBySlaveID = {}; -var sentItemStatisticsBySlaveID = {}; - -const averagedTimeSeries = require("averaged-timeseries"); const express = require("express"); const path = require("path"); @@ -87,19 +83,6 @@ function addApiRoutes( console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")"); } // gather statistics - let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID]; - if(recievedItemStatistics === undefined){ - recievedItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }); - recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics; - } - recievedItemStatistics.add({ - key:req.body.name, - value:req.body.count, - }); prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); // save items we get var count = Number(req.body.count); @@ -160,27 +143,12 @@ function addApiRoutes( // track statistics and do graphing things prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0); - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }); - } - sentItemStatistics.add({ - key:object.name, - value:numberToRemove, - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } else if(config.useNeuralNetDoleDivider){ // use fancy neural net to calculate a "fair" dole division rate. neuralDole.divider({ res, object, config, - sentItemStatisticsBySlaveID, prometheusImportGauge }) } else { @@ -189,7 +157,6 @@ function addApiRoutes( itemCount, object, db, - sentItemStatisticsBySlaveID, config, prometheusDoleFactorGauge, prometheusImportGauge, diff --git a/sharedPlugins/clusterioMod/test/routes.js b/sharedPlugins/clusterioMod/test/routes.js index 1e5a6e20..a2872284 100644 --- a/sharedPlugins/clusterioMod/test/routes.js +++ b/sharedPlugins/clusterioMod/test/routes.js @@ -21,7 +21,6 @@ describe("clusterioMod endpoints", function() { app.use(bodyParser.json({ limit: '10mb', })); app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true })); const config = { - itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 }, disableFairItemDistribution: true, }; const items = new database.ItemDatabase(); diff --git a/static/main.js b/static/main.js index 504b8d1a..65b7a1e8 100644 --- a/static/main.js +++ b/static/main.js @@ -1,7 +1,6 @@ (function (){ ensureSetting("Display offline slaves", "boolean", false); - ensureSetting("Enable production graphs", "boolean", true); ensureSetting("Periodically update storage screen", "boolean", true); }()) diff --git a/static/nodeDetails.js b/static/nodeDetails.js index 90fc42db..fc8ee509 100644 --- a/static/nodeDetails.js +++ b/static/nodeDetails.js @@ -1,24 +1,3 @@ -// string, object, function(object) -function post(url, data, callback) { - console.log("POST " + url + JSON.stringify(data)); - var xhr = new XMLHttpRequest(); - xhr.open("POST", url, true); - xhr.setRequestHeader("Content-type", "application/json"); - xhr.onreadystatechange = function () { - if (xhr.readyState === 4 && xhr.status === 200) { - var json = JSON.parse(xhr.responseText); - callback(json); - } - }; - xhr.send(JSON.stringify(data)); -} -// function to sort arrays of objects after a keys value -function sortByKey(array, key) { - return array.sort(function(a, b) { - var x = Number(a[key]); var y = Number(b[key]); - return ((x < y) ? 1 : ((x > y) ? -1 : 0)); - }); -} function getParameterByName(name, url) { if (!url) url = window.location.href; name = name.replace(/[\[\]]/g, "\\$&"); @@ -158,8 +137,6 @@ function populateSlaveInfo(){ HTML += ""; // end of left container - // chart - HTML += '
'; // terminal // HTML += '
'; @@ -167,9 +144,6 @@ function populateSlaveInfo(){ document.querySelector("#body > #details").innerHTML = HTML; - // make production graph - makeGraph(slave.unique, slave.unique) - // makeTerminal(); $('#fetch-mod-data').on("click", downloadSaveModData); @@ -279,94 +253,3 @@ function maximizeTerminal(instant){ setTimeout(() => $('#terminal')[0].style.transition = 'none', (Number(instant) || 200)); $('#terminal').draggable('enable'); } - -// ID of slave, ID of canvasjs div without # -function makeGraph(instanceID, selector) { - let chartIgnoreList = [ - "water", - "steam" - ]; - post("api/getStats", {instanceID: instanceID}, function(data){ - //console.log("Building chart " + instanceID + " with this data:") - //console.log(data) - if(data.length > 0) { - // find keys - let itemNames = []; - for(let key in data[data.length - 1].data) { - itemNames[itemNames.length] = key - } - let chartData = []; - for(let o = 0; o < itemNames.length; o++) { - if(!chartIgnoreList.includes(itemNames[o])){ - chartData[chartData.length] = generateLineChartArray(data, itemNames[o]); - } - } - //console.log(chartData) - drawChart(selector, chartData) - } - // callback(data); - }) -} - -function generateLineChartArray(data, nameKey) { - let chartData = []; - for(let i = 0; i < data.length; i++) { - // only show recent data - if(data[i].timestamp > Date.now() - (24*60*60*1000)){ - let y = data[i].data[nameKey]; - if(!data[i].data[nameKey]) { - y = 0; - } else if(y < 0) { - y = 0; - } - chartData[chartData.length] = { - x: new Date(data[i].timestamp), - y: Number(y) - } - } - } - chartData = sortByKey(chartData, "x"); - let xyz = {}; - xyz.name = nameKey; - xyz.type = "line"; - if(nameKey === "copper-wire"||nameKey === "iron-plate"||nameKey === "copper-plate"||nameKey === "electronic-circuit"||nameKey === "steel-plate"||nameKey === "advanced-circuit"||nameKey === "crude-oil"||nameKey === "petroleum-gas"){ - xyz.showInLegend = true; - } - xyz.dataPoints = chartData; - return xyz; -} - -function drawChart(selector, chartData, title) { - // selector is ID of element, ex "chartContainer" or "-123199123" - console.log(chartData); - chartsByID[selector] = new CanvasJS.Chart(selector, { - title:{ - text: title || "Production graph" - }, - toolTip:{ - content: "{name}: {y}" - }, - zoomEnabled: true, - axisY:{ - includeZero: true, - }, - legend: { - cursor: "pointer", - itemclick: function (e) { - //console.log("legend click: " + e.dataPointIndex); - //console.log(e); - if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) { - e.dataSeries.visible = false; - } else { - e.dataSeries.visible = true; - } - e.chart.render(); - } - }, - data: chartData - }); - chart = chartsByID[selector]; - // console.log(chart) - chart.render(); - return chart; -} diff --git a/static/slaveDisplay.js b/static/slaveDisplay.js index dbcc7e3e..6a0a5dc5 100644 --- a/static/slaveDisplay.js +++ b/static/slaveDisplay.js @@ -13,168 +13,14 @@ setTimeout(function() { if(Date.now() - slaveData[key].time < 120000 || displayOffline) { let seenDate = moment(Number(slaveData[key].time)).format('DD.MM.YY, hh:mm:ss') HTML += "
"; - HTML += '
'; HTML += "

" + slaveData[key].instanceName + "

ID: " + slaveData[key].unique + "

Last seen: "+seenDate+"

Online players: "+slaveData[key].playerCount+"

IP: "+slaveData[key].publicIP +":"+ slaveData[key].serverPort+"

" HTML += "

Host: "+slaveData[key].mac+"

"; - - HTML += "

Graph tools:

"; - HTML += ""; - HTML += ""; HTML += "
"; } } document.querySelector("#slaves > #display").innerHTML = HTML - - if(JSON.parse(localStorage.settings)["Enable production graphs"]) { - let graphs = document.querySelectorAll(".productionGraph"); - for(let i = 0; i < graphs.length; i++) { - // execute functions to make graphs - // ID of slave, ID of canvasjs div without # - makeGraph(graphs[i].id, graphs[i].id) - } - } } } xmlhttp.open("GET", "api/slaves", true); xmlhttp.send(); }, 0) - -let chartIgnoreList = [ - "water", - "steam" -] - -// ID of slave, ID of canvasjs div without # -function makeGraph(instanceID, selector) { - post("api/getTimelineStats", {instanceID: instanceID}, function(pointsInTime) { - //console.log("Building chart " + instanceID + " with this data:") - - //Find all keys at all points in time. - let itemNames = {}; - pointsInTime.forEach(function(point) { - for(let key in point.data) { - itemNames[key] = true; - } - }); - itemNames = Object.keys(itemNames); - - //Generate a chart list for each item. - const displayNames = itemNames.filter(function(name) { - return !chartIgnoreList.includes(name); - }); - const chartData = displayNames.map(function(name) { - return generateLineChartArray(pointsInTime, name) - }); - - chartData.length && drawChart(selector, chartData) - }) -} - -// production info -// string, object, function(object) -function post(url, data, callback) { - console.log("POST " + url, data); - var xhr = new XMLHttpRequest(); - xhr.open("POST", url, true); - xhr.setRequestHeader("Content-type", "application/json"); - xhr.onreadystatechange = function () { - if (xhr.readyState == 4 && xhr.status == 200) { - var json = JSON.parse(xhr.responseText); - callback(json); - } - } - xhr.send(JSON.stringify(data)); -} - -function generateLineChartArray(data, nameKey) { - const legendKeys = ["copper-cable", "iron-plate", "copper-plate", "electronic-circuit", "steel-plate", "advanced-circuit", "crude-oil", "petroleum-gas"]; - let chartData = []; - for(let i = 0; i < data.length; i++) { - let y = data[i].data[nameKey] - if(!data[i].data[nameKey]) { - y = 0; - } else if(y < 0) { - y = 0; - } - chartData[chartData.length] = { - x: new Date(data[i].timestamp), - y: Number(y) - } - } - chartData = sortByKey(chartData, "x"); - let xyz = {}; - xyz.name = nameKey; - xyz.type = "line"; - - if(legendKeys.indexOf(nameKey) >= 0){ - xyz.showInLegend = true; - } - xyz.dataPoints = chartData; - return xyz; -} - -var chartsByID = {}; -function drawChart(selector, chartData) { - // selector is ID of element, ex "chartContainer" or "-123199123" - // console.log(chartData) - chartsByID[selector] = new CanvasJS.Chart(selector, { - title:{ - text: "Production graph" - }, - toolTip:{ - content: "{name}: {y}" - }, - zoomEnabled: true, - axisY:{ - includeZero: true, - }, - legend: { - cursor: "pointer", - itemclick: function (e) { - //console.log("legend click: " + e.dataPointIndex); - //console.log(e); - if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) { - e.dataSeries.visible = false; - } else { - e.dataSeries.visible = true; - } - e.chart.render(); - } - }, - data: chartData - }); - chart = chartsByID[selector]; - // console.log(chart) - chart.render(); -} - -function hideAllDatasets(chartID) { - if(chartsByID[chartID]) { - let chart = chartsByID[chartID]; - for(let i = 0; i < chart.options.data.length; i++) { - chart.options.data[i].visible = false; - } - chart.render(); - } else { - return false; - } -} -function showAllDatasets(chartID) { - if(chartsByID[chartID]) { - let chart = chartsByID[chartID]; - for(let i = 0; i < chart.options.data.length; i++) { - chart.options.data[i].visible = true; - } - chart.render(); - } else { - return false; - } -} - -// function to sort arrays of objects after a keys value -function sortByKey(array, key) { - return array.sort(function(a, b) { - var x = Number(a[key]); var y = Number(b[key]); - return ((x < y) ? 1 : ((x > y) ? -1 : 0)); - }); -} \ No newline at end of file diff --git a/views/head.ejs b/views/head.ejs index 207b3f17..f2dec05e 100644 --- a/views/head.ejs +++ b/views/head.ejs @@ -2,7 +2,6 @@ -