Merge pull request #271 from Hornwitser/remove-old-stats

Remove old statistics code

config.json.dist

@@ -29,11 +29,6 @@
 		"public": true,
 		"lan": true
 	},
-	"__comment6": "Statistic gathering statistics, may affect server performance (master memory usage especially)",
-	"itemStats": {
-		"maxEntries":60,
-		"entriesPerSecond": 0.01666666667
-	},
 	"logItemTransfers": false,
 	"__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.",
 	"disableFairItemDistribution": false,

lib/npmPostinstall.js

@@ -75,10 +75,6 @@ let downloadList = [
 		url: "https://code.jquery.com/ui/1.12.1/jquery-ui.js",
 		file: "static/external/jquery-ui.js",
 	},
-	{
-		url: "https://cdnjs.cloudflare.com/ajax/libs/canvasjs/1.7.0/canvasjs.js",
-		file: "static/external/canvasjs.js",
-	},
 	{
 		url: "https://momentjs.com/downloads/moment.min.js",
 		file: "static/external/moment.min.js",

package.json

@@ -30,7 +30,6 @@
   },
   "dependencies": {
     "as-table": "^1.0.31",
-    "averaged-timeseries": "^1.4.0",
     "base64url": "^3.0.0",
     "bcrypt": "=3.0.6",
     "bcrypt-promise": "^2.0.0",

sharedPlugins/clusterioMod/dole.js

@@ -1,5 +1,3 @@
-const averagedTimeSeries = require("averaged-timeseries")
-
 const doleNN = require("./dole_nn_base")
 
 const Prometheus = require('prom-client');
@@ -58,7 +56,6 @@ class neuralDole {
         res,
         object,
         config,
-        sentItemStatisticsBySlaveID,
         prometheusImportGauge
     }){
         let magicData = doleNN.Dose(
@@ -89,20 +86,6 @@ class neuralDole {
 		if(config.logItemTransfers){
 			console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
 		}
-		let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-		if(sentItemStatistics === undefined){
-			sentItemStatistics = new averagedTimeSeries({
-				maxEntries: config.itemStats.maxEntries,
-				entriesPerSecond: config.itemStats.entriesPerSecond,
-				mergeMode: "add",
-			});
-		}
-		sentItemStatistics.add({
-			key:object.name,
-			value:magicData[0],
-		});
-		//console.log(sentItemStatistics.data)
-		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
 		prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
 
 		res.send({
@@ -118,7 +101,6 @@ function doleDivider({
 	itemCount,
     object,
 	items,
-    sentItemStatisticsBySlaveID,
     config,
     prometheusDoleFactorGauge,
     prometheusImportGauge,
@@ -142,20 +124,6 @@ function doleDivider({
 			console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
 		}
 		items.removeItem(object.name, object.count)
-		let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-		if(sentItemStatistics === undefined){
-			sentItemStatistics = new averagedTimeSeries({
-				maxEntries: config.itemStats.maxEntries,
-				entriesPerSecond: config.itemStats.entriesPerSecond,
-				mergeMode: "add",
-			});
-		}
-		sentItemStatistics.add({
-			key:object.name,
-			value:object.count,
-		});
-		//console.log(sentItemStatistics.data)
-		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
 
         prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
         prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);

sharedPlugins/clusterioMod/routes.js

@@ -1,7 +1,3 @@
-var recievedItemStatisticsBySlaveID = {};
-var sentItemStatisticsBySlaveID = {};
-
-const averagedTimeSeries = require("averaged-timeseries");
 const express = require("express");
 const path = require("path");
 
@@ -87,19 +83,6 @@ function addApiRoutes(
 				console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
 			}
 			// gather statistics
-			let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
-			if(recievedItemStatistics === undefined){
-				recievedItemStatistics = new averagedTimeSeries({
-					maxEntries: config.itemStats.maxEntries,
-					entriesPerSecond: config.itemStats.entriesPerSecond,
-					mergeMode: "add",
-				});
-				recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
-			}
-			recievedItemStatistics.add({
-				key:req.body.name,
-				value:req.body.count,
-			});
 			prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
 			// save items we get
 			var count = Number(req.body.count);
@@ -160,27 +143,12 @@ function addApiRoutes(
 
 			// track statistics and do graphing things
 			prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
-			let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-			if(sentItemStatistics === undefined){
-				sentItemStatistics = new averagedTimeSeries({
-					maxEntries: config.itemStats.maxEntries,
-					entriesPerSecond: config.itemStats.entriesPerSecond,
-					mergeMode: "add",
-				});
-			}
-			sentItemStatistics.add({
-				key:object.name,
-				value:numberToRemove,
-			});
-			//console.log(sentItemStatistics.data)
-			sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
 		} else if(config.useNeuralNetDoleDivider){
 			// use fancy neural net to calculate a "fair" dole division rate.
 			neuralDole.divider({
 				res,
 				object,
 				config,
-				sentItemStatisticsBySlaveID,
 				prometheusImportGauge
 			})
 		} else {
@@ -189,7 +157,6 @@ function addApiRoutes(
 				itemCount,
 				object,
 				db,
-				sentItemStatisticsBySlaveID,
 				config,
 				prometheusDoleFactorGauge,
 				prometheusImportGauge,

sharedPlugins/clusterioMod/test/routes.js

@@ -21,7 +21,6 @@ describe("clusterioMod endpoints", function() {
 	app.use(bodyParser.json({ limit: '10mb', }));
 	app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
 	const config = {
-		itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 },
 		disableFairItemDistribution: true,
 	};
 	const items = new database.ItemDatabase();

static/main.js

@@ -1,7 +1,6 @@
 
 (function (){
 	ensureSetting("Display offline slaves", "boolean", false);
-	ensureSetting("Enable production graphs", "boolean", true);
 	ensureSetting("Periodically update storage screen", "boolean", true);
 }())
 

static/nodeDetails.js

@@ -1,24 +1,3 @@
-// string, object, function(object)
-function post(url, data, callback) {
-	console.log("POST " + url + JSON.stringify(data));
-	var xhr = new XMLHttpRequest();
-	xhr.open("POST", url, true);
-	xhr.setRequestHeader("Content-type", "application/json");
-	xhr.onreadystatechange = function () {
-		if (xhr.readyState === 4 && xhr.status === 200) {
-			var json = JSON.parse(xhr.responseText);
-			callback(json);
-		}
-	};
-	xhr.send(JSON.stringify(data));
-}
-// function to sort arrays of objects after a keys value
-function sortByKey(array, key) {
-    return array.sort(function(a, b) {
-        var x = Number(a[key]); var y = Number(b[key]);
-        return ((x < y) ? 1 : ((x > y) ? -1 : 0));
-    });
-}
 function getParameterByName(name, url) {
     if (!url) url = window.location.href;
     name = name.replace(/[\[\]]/g, "\\$&");
@@ -158,8 +137,6 @@ function populateSlaveInfo(){
 		
 		HTML += "</div>"; // end of left container
 		
-		// chart
-		HTML += '<div id="' + slave.unique + '" class="productionGraph" style="width: calc(100% - 300px);"></div>';
 		// terminal
 		// HTML += '<div id="terminal"></div>';
 
@@ -167,9 +144,6 @@ function populateSlaveInfo(){
 		
 		document.querySelector("#body > #details").innerHTML = HTML;
 		
-		// make production graph
-		makeGraph(slave.unique, slave.unique)
-		
 		// makeTerminal();
 
         $('#fetch-mod-data').on("click", downloadSaveModData);
@@ -279,94 +253,3 @@ function maximizeTerminal(instant){
 	setTimeout(() => $('#terminal')[0].style.transition = 'none', (Number(instant) || 200));
 	$('#terminal').draggable('enable');
 }
-
-// ID of slave, ID of canvasjs div without #
-function makeGraph(instanceID, selector) {
-	let chartIgnoreList = [
-		"water",
-		"steam"
-	];
-	post("api/getStats", {instanceID: instanceID}, function(data){
-		//console.log("Building chart " + instanceID + " with this data:")
-		//console.log(data)
-		if(data.length > 0) {
-			// find keys
-			let itemNames = [];
-			for(let key in data[data.length - 1].data) {
-				itemNames[itemNames.length] = key
-			}
-			let chartData = [];
-			for(let o = 0; o < itemNames.length; o++) {
-				if(!chartIgnoreList.includes(itemNames[o])){
-					chartData[chartData.length] = generateLineChartArray(data, itemNames[o]);
-				}
-			}
-			//console.log(chartData)
-			drawChart(selector, chartData)
-		}
-		// callback(data);
-	})
-}
-
-function generateLineChartArray(data, nameKey) {
-	let chartData = [];
-	for(let i = 0; i < data.length; i++) {
-		// only show recent data
-		if(data[i].timestamp > Date.now() - (24*60*60*1000)){
-			let y = data[i].data[nameKey];
-			if(!data[i].data[nameKey]) {
-				y = 0;
-			} else if(y < 0) {
-				y = 0;
-			}
-			chartData[chartData.length] = {
-				x: new Date(data[i].timestamp),
-				y: Number(y)
-			}
-		}
-	}
-	chartData = sortByKey(chartData, "x");
-	let xyz = {};
-	xyz.name = nameKey;
-	xyz.type = "line";
-	if(nameKey === "copper-wire"||nameKey === "iron-plate"||nameKey === "copper-plate"||nameKey === "electronic-circuit"||nameKey === "steel-plate"||nameKey === "advanced-circuit"||nameKey === "crude-oil"||nameKey === "petroleum-gas"){
-		xyz.showInLegend = true;
-	}
-	xyz.dataPoints = chartData;
-	return xyz;
-}
-
-function drawChart(selector, chartData, title) {
-	// selector is ID of element, ex "chartContainer" or "-123199123"
-	console.log(chartData);
-	chartsByID[selector] = new CanvasJS.Chart(selector, {
-		title:{
-			text: title || "Production graph"
-		},
-		toolTip:{   
-			content: "{name}: {y}"	  
-		},
-		zoomEnabled: true,
-		axisY:{
-			includeZero: true,
-		},
-		legend: {
-			cursor: "pointer",
-			itemclick: function (e) {
-				//console.log("legend click: " + e.dataPointIndex);
-				//console.log(e);
-				if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
-					e.dataSeries.visible = false;
-				} else {
-					e.dataSeries.visible = true;
-				}	
-				e.chart.render();
-			}
-		},
-		data: chartData
-	});
-	chart = chartsByID[selector];
-	// console.log(chart)
-	chart.render();
-	return chart;
-}

static/slaveDisplay.js

@@ -13,168 +13,14 @@ setTimeout(function() {
 				if(Date.now() - slaveData[key].time < 120000 || displayOffline) {
 					let seenDate = moment(Number(slaveData[key].time)).format('DD.MM.YY, hh:mm:ss')
 					HTML += "<div class='slaveBox'>";
-					HTML += '<div id="' + slaveData[key].unique + '" class="productionGraph" style="width: calc(100% - 200px);"></div>';
 					HTML += "<a href='nodeDetails?instanceID="+slaveData[key].unique+"'><h2>" + slaveData[key].instanceName + "</h2></a><p>ID: " + slaveData[key].unique + "</p><p>Last seen: "+seenDate+"</p><p>Online players: "+slaveData[key].playerCount+"</p><p>IP: "+slaveData[key].publicIP +":"+ slaveData[key].serverPort+"</p>"
 					HTML += "<p>Host: "+slaveData[key].mac+"</p>";
-					
-					HTML += "<br><p>Graph tools:</p>";
-					HTML += "<button style='width:100px;' onclick='hideAllDatasets(\"" + slaveData[key].unique + "\")\'>Hide all</button>";
-					HTML += "<button style='width:100px;' onclick='showAllDatasets(\"" + slaveData[key].unique + "\")\'>Show all</button>";
 					HTML += "</div>";
 				}
 			}
 			document.querySelector("#slaves > #display").innerHTML = HTML
-			
-			if(JSON.parse(localStorage.settings)["Enable production graphs"]) {
-				let graphs = document.querySelectorAll(".productionGraph");
-				for(let i = 0; i < graphs.length; i++) {
-					// execute functions to make graphs
-					// ID of slave, ID of canvasjs div without #
-					makeGraph(graphs[i].id, graphs[i].id)
-				}
-			}
 		}
 	}
 	xmlhttp.open("GET", "api/slaves", true);
 	xmlhttp.send();
 }, 0)
-
-let chartIgnoreList = [
-	"water",
-	"steam"
-]
-
-// ID of slave, ID of canvasjs div without #
-function makeGraph(instanceID, selector) {
-	post("api/getTimelineStats", {instanceID: instanceID}, function(pointsInTime) {
-		//console.log("Building chart " + instanceID + " with this data:")
-		
-		//Find all keys at all points in time.
-		let itemNames = {};
-		pointsInTime.forEach(function(point) {
-			for(let key in point.data) {
-				itemNames[key] = true;
-			}
-		});
-		itemNames = Object.keys(itemNames);
-		
-		//Generate a chart list for each item.
-		const displayNames = itemNames.filter(function(name) { 
-			return !chartIgnoreList.includes(name);
-		});
-		const chartData = displayNames.map(function(name) {
-			return generateLineChartArray(pointsInTime, name)
-		});
-		
-		chartData.length && drawChart(selector, chartData)
-	})
-}
-
-// production info
-// string, object, function(object)
-function post(url, data, callback) {
-	console.log("POST " + url, data);
-	var xhr = new XMLHttpRequest();
-	xhr.open("POST", url, true);
-	xhr.setRequestHeader("Content-type", "application/json");
-	xhr.onreadystatechange = function () {
-		if (xhr.readyState == 4 && xhr.status == 200) {
-			var json = JSON.parse(xhr.responseText);
-			callback(json);
-		}
-	}
-	xhr.send(JSON.stringify(data));
-}
-
-function generateLineChartArray(data, nameKey) {
-	const legendKeys = ["copper-cable", "iron-plate", "copper-plate", "electronic-circuit", "steel-plate", "advanced-circuit", "crude-oil", "petroleum-gas"];
-	let chartData = [];
-	for(let i = 0; i < data.length; i++) {
-		let y = data[i].data[nameKey]
-		if(!data[i].data[nameKey]) {
-			y = 0;
-		} else if(y < 0) {
-			y = 0;
-		}
-		chartData[chartData.length] = {
-			x: new Date(data[i].timestamp),
-			y: Number(y)
-		}
-	}
-	chartData = sortByKey(chartData, "x");
-	let xyz = {};
-	xyz.name = nameKey;
-	xyz.type = "line";
-	
-	if(legendKeys.indexOf(nameKey) >= 0){
-		xyz.showInLegend = true;
-	}
-	xyz.dataPoints = chartData;
-	return xyz;
-}
-
-var chartsByID = {};
-function drawChart(selector, chartData) {
-	// selector is ID of element, ex "chartContainer" or "-123199123"
-	// console.log(chartData)
-	chartsByID[selector] = new CanvasJS.Chart(selector, {
-		title:{
-			text: "Production graph"
-		},
-		toolTip:{   
-			content: "{name}: {y}"	  
-		},
-		zoomEnabled: true,
-		axisY:{
-			includeZero: true,
-		},
-		legend: {
-			cursor: "pointer",
-			itemclick: function (e) {
-				//console.log("legend click: " + e.dataPointIndex);
-				//console.log(e);
-				if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
-					e.dataSeries.visible = false;
-				} else {
-					e.dataSeries.visible = true;
-				}	
-				e.chart.render();
-			}
-		},
-		data: chartData
-	});
-	chart = chartsByID[selector];
-	// console.log(chart)
-	chart.render();
-}
-
-function hideAllDatasets(chartID) {
-	if(chartsByID[chartID]) {
-		let chart = chartsByID[chartID];
-		for(let i = 0; i < chart.options.data.length; i++) {
-			chart.options.data[i].visible = false;
-		}
-		chart.render();
-	} else {
-		return false;
-	}
-}
-function showAllDatasets(chartID) {
-	if(chartsByID[chartID]) {
-		let chart = chartsByID[chartID];
-		for(let i = 0; i < chart.options.data.length; i++) {
-			chart.options.data[i].visible = true;
-		}
-		chart.render();
-	} else {
-		return false;
-	}
-}
-
-// function to sort arrays of objects after a keys value
-function sortByKey(array, key) {
-    return array.sort(function(a, b) {
-        var x = Number(a[key]); var y = Number(b[key]);
-        return ((x < y) ? 1 : ((x > y) ? -1 : 0));
-    });
-}

views/head.ejs

@@ -2,7 +2,6 @@
 <link rel="stylesheet" type="text/css" href="/style.css">
 <script src="/external/jquery.js"></script>
 <script src="/external/jquery-ui.js"></script>
-<script src="/external/canvasjs.js"></script>
 <script src="/external/moment.min.js"></script>
 <script src="/external/mousetrap.min.js"></script>
 <script src="/main.js"></script>