Data collection related to storage IO and graphing

config.json

@@ -24,5 +24,10 @@
 	"visibility": {
 		"public": true,
 		"lan": true
+	},
+	"__comment": "Statistic gathering statistics, may affect server performance (master memory usage especially)",
+	"itemStats": {
+		"maxEntries":60,
+		"entriesPerSecond": 0.01666666667
 	}
 }

master.js

@@ -13,6 +13,7 @@ const stringUtils = require("./lib/stringUtils");
 const mkdirp = require("mkdirp");
 mkdirp.sync("./database");
 mkdirp.sync(masterModFolder);
+const averagedTimeSeries = require("averaged-timeseries");
 const deepmerge = require("deepmerge");
 const path = require("path");
 const fs = require("fs");
@@ -77,23 +78,25 @@ db.flows = new LinvoDB("flows", {}, {});
 })()
 
 db.items.addItem = function(object) {
-	if(object.name == "addItem") {
+	if(object.name == "addItem" || object.name == "removeItem") {
 		console.error("Fuck you, that would screw everything up if you named your item that.");
-	}
-	if(this[object.name] && Number(this[object.name]) != NaN){
-		this[object.name] = Number(this[object.name]) + Number(object.count);
 	} else {
-		this[object.name] = object.count;
+		if(this[object.name] && Number(this[object.name]) != NaN){
+			this[object.name] = Number(this[object.name]) + Number(object.count);
+		} else {
+			this[object.name] = object.count;
+		}
 	}
 }
 db.items.removeItem = function(object) {
-	if(object.name == "addItem") {
+	if(object.name == "addItem" || object.name == "removeItem") {
 		console.error("Fuck you, that would screw everything up if you named your item that.");
-	}
-	if(this[object.name] && Number(this[object.name]) != NaN){
-		this[object.name] = Number(this[object.name]) - Number(object.count);
 	} else {
-		this[object.name] = 0;
+		if(this[object.name] && Number(this[object.name]) != NaN){
+			this[object.name] = Number(this[object.name]) - Number(object.count);
+		} else {
+			this[object.name] = 0;
+		}
 	}
 }
 
@@ -201,17 +204,46 @@ app.get("/api/slaves", function(req, res) {
 	}
 	res.send(copyOfSlaves);
 });
-
+var recievedItemStatisticsBySlaveID = {};
+var sentItemStatisticsBySlaveID = {};
+/*
+var recievedItemStatistics = new averagedTimeSeries({
+	maxEntries: config.itemStats.maxEntries,
+	entriesPerSecond: config.itemStats.entriesPerSecond,
+}, console.log);*/
 // endpoint to send items to
 app.post("/api/place", function(req, res) {
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	let x = req.body;
+	if(!x.instanceName) {
+		x.instanceName = "unknown";
+	}
+	if(!x.instanceID) {
+		x.instanceID = "unknown";
+	}
+	if(!x.unique) {
+		x.unique = "unknown";
+	}
 	if(	x.unique
-		&& x.instanceName // This is in no way a proper authentication or anything, its just to make sure everybody are registered as slaves before modifying the cluster
-		&& stringUtils.hashCode(slaves[x.unique].rconPassword) == x.unique){
+		&& x.instanceName
+		&& !isNaN(Number(x.count))// This is in no way a proper authentication or anything, its just to make sure everybody are registered as slaves before modifying the cluster (or not, to maintain backwards compat)
+		/*&& stringUtils.hashCode(slaves[x.unique].rconPassword) == x.unique*/){
 		
 		console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.unique+")");
+		// gather statistics
+		let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
+		if(recievedItemStatistics === undefined){
+			recievedItemStatistics = new averagedTimeSeries({
+				maxEntries: config.itemStats.maxEntries,
+				entriesPerSecond: config.itemStats.entriesPerSecond,
+			}, console.log);
+			recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
+		}
+		recievedItemStatistics.add({
+			key:req.body.name,
+			value:req.body.count,
+		});
 		// save items we get
 		db.items.addItem({
 			name:req.body.name,
@@ -223,7 +255,6 @@ app.post("/api/place", function(req, res) {
 		res.send("failure");
 	}
 });
-
 // endpoint to remove items from DB when client orders items
 _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 app.post("/api/remove", function(req, res) {
@@ -257,6 +288,17 @@ app.post("/api/remove", function(req, res) {
 			//console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
 			db.items.removeItem(object);
 			// res.send("successier");
+			let sentItemStatistics = sentItemStatisticsBySlaveID[x.instanceID];
+			if(sentItemStatistics === undefined){
+				sentItemStatistics = new averagedTimeSeries({
+					maxEntries: config.itemStats.maxEntries,
+					entriesPerSecond: config.itemStats.entriesPerSecond,
+				}, console.log);
+			}
+			sentItemStatistics.add({
+				key:object.name,
+				value:object.count,
+			});
 			res.send(object);
 		} else {
 			// if we didn't have enough, attempt giving out a smaller amount next time
@@ -328,8 +370,11 @@ app.post("/api/logStats", function(req,res) {
 });
 // {slaveID: string, fromTime: Date, toTime, Date}
 app.post("/api/getStats", function(req,res) {
+	if(typeof req.body == "string"){
+		req.body = JSON.stringify(req.body);
+	}
 	console.log(req.body);
-	if(typeof req.body == "object" && req.body.slaveID) {
+	if(typeof req.body == "object" && req.body.slaveID && req.body.statistic === undefined) {
 		// if not specified, get stats for last 24 hours
 		if(!req.body.fromTime) {
 			req.body.fromTime = Date.now() - 86400000 // 24 hours in MS
@@ -347,6 +392,26 @@ app.post("/api/getStats", function(req,res) {
 			// console.log(entries);
 			res.send(entries);
 		});
+	} else if(typeof req.body == "object" && req.body.slaveID && typeof req.body.slaveID == "string" && req.body.itemName){
+		if(req.body.statistic == "place"){
+			console.log("sending place data...");
+			// Gather data
+			console.log(recievedItemStatisticsBySlaveID)
+			let itemStats = recievedItemStatisticsBySlaveID[req.body.slaveID];
+			if(itemStats === undefined){
+				res.send({statusForDebugging:"no data available"});
+				return false;
+			}
+			let data = itemStats.get(config.itemStats.maxEntries, req.body.itemName);
+			
+			res.send({
+				maxEntries:config.itemStats.maxEntries,
+				entriesPerSecond: config.itemStats.entriesPerSecond,
+				data: data,
+			});
+		} else if(req.body.statistic == "remove"){
+			
+		}
 	}
 });
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "clusterio",
-  "version": "1.0.0",
+  "version": "1.3.0",
   "description": "Interserver communication for factorio",
   "repository": "https://github.com/Danielv123/factorioClusterio",
   "main": "master.js",
@@ -15,22 +15,23 @@
   "author": "Danielv123",
   "license": "ISC",
   "dependencies": {
-    "body-parser": "^1.15.2",
+    "averaged-timeseries": "^1.2.4",
+    "body-parser": "^1.18.2",
     "canvasjs": "^1.8.1",
     "child_process": "^1.0.2",
     "codecov": "^2.3.0",
-    "deepmerge": "^1.3.2",
-    "ejs": "^2.5.6",
-    "express": "^4.14.0",
+    "deepmerge": "^1.5.2",
+    "ejs": "^2.5.7",
+    "express": "^4.16.2",
     "express-fileupload": "0.0.6",
     "find-remove": "^1.0.0",
-    "follow-redirects": "^1.2.1",
+    "follow-redirects": "^1.2.5",
     "fs": "0.0.2",
     "fs-extra": "^1.0.0",
     "getmac": "^1.2.1",
     "hash-files": "^1.1.1",
     "ini": "^1.3.4",
-    "leveldown": "^1.8.0",
+    "leveldown": "^1.9.0",
     "linvodb": "0.0.0",
     "linvodb3": "^3.25.0",
     "luamin": "^1.0.4",
@@ -39,8 +40,8 @@
     "nedb": "^1.8.0",
     "needle": "^1.1.0",
     "portfinder": "^1.0.9",
-    "request": "^2.79.0",
-    "semver": "^5.3.0",
+    "request": "^2.83.0",
+    "semver": "^5.4.1",
     "simple-rcon": "^0.3.1",
     "sync-request": "^4.0.1",
     "underscore": "^1.8.3"
@@ -48,10 +49,10 @@
   "devDependencies": {
     "chai": "^3.5.0",
     "istanbul": "^0.4.5",
-    "mocha": "^3.4.1",
+    "mocha": "^3.5.3",
     "mocha-sinon": "^2.0.0",
     "mock-fs": "^4.3.0",
-    "nock": "^9.0.13",
-    "sinon": "^2.3.4"
+    "nock": "^9.0.22",
+    "sinon": "^2.4.1"
   }
 }

static/head.html

@@ -1,7 +1,7 @@
 <title>Clusterio</title>
 <link rel="stylesheet" type="text/css" href="style.css">
 <script src="https://code.jquery.com/jquery.js"></script>
-  <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
+<script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/canvasjs/1.7.0/canvasjs.js" type="text/javascript"></script>
 <script src="http://momentjs.com/downloads/moment.min.js" type="text/javascript"></script>
 <script src="main.js"></script>

static/index.html

@@ -13,7 +13,11 @@
 			<div id="contents">
 				<table><tbody><!-- populated by js --></tbody></table>
 			</div>
-			<canvas id="contentGraph" width="200" height="200"></canvas>
+		</div>
+		<div id="graphDisplay">
+			<div id="storageGraphContainer">
+				<div id="contentGraph" class="storageGraph" style="width: 100%;"></div>
+			</div>
 		</div>
 	</div>
 </body>

static/main.js

@@ -34,6 +34,58 @@ function djb2(str){
 	}
 	return hash;
 }
+// callback(err, json)
+function getJSON(url, callback) {
+	var xhr = new XMLHttpRequest();
+	xhr.open('GET', url, true);
+	xhr.responseType = 'json';
+	xhr.onload = function() {
+	  var status = xhr.status;
+	  if (status == 200) {
+		callback(null, xhr.response);
+	  } else {
+		callback(status);
+	  }
+	};
+	// triggers if connection is refused
+	xhr.onerror = function(e){
+		callback(e);
+	};
+	xhr.send();
+};
+// callback(err, json)
+function postJSON(url, data, callback) {
+	var xhr = new XMLHttpRequest();
+	xhr.open('POST', url, true);
+	xhr.responseType = 'json';
+	xhr.onload = function() {
+	  var status = xhr.status;
+	  if (status == 200) {
+		callback(null, xhr.response);
+	  } else {
+		callback(status);
+	  }
+	};
+	// triggers if connection is refused
+	xhr.onerror = function(e){
+		callback(e);
+	};
+	xhr.send(data);
+};
+// return Boolean
+function isJSON(string){
+	let stringIsJson = false;
+	let x
+	try {
+		x = JSON.parse(string);
+	} catch (e){
+		return false;
+	}
+	if(typeof x == "object" || typeof string == "object"){
+		stringIsJson = true;
+	}
+	return stringIsJson;
+}
 // hash a string to a color
 function hashColor(str) {
 	var hash = djb2(str);

static/nodeDetails.js

@@ -12,25 +12,6 @@ function post(url, data, callback) {
 	}
 	xhr.send(JSON.stringify(data));
 }
-// callback(err, json)
-function getJSON(url, callback) {
-	var xhr = new XMLHttpRequest();
-	xhr.open('GET', url, true);
-	xhr.responseType = 'json';
-	xhr.onload = function() {
-	  var status = xhr.status;
-	  if (status == 200) {
-		callback(null, xhr.response);
-	  } else {
-		callback(status);
-	  }
-	};
-	// triggers if connection is refused
-	xhr.onerror = function(e){
-		callback(e);
-	};
-	xhr.send();
-};
 // function to sort arrays of objects after a keys value
 function sortByKey(array, key) {
     return array.sort(function(a, b) {
@@ -71,6 +52,7 @@ setInterval(function(){
 	});
 }, 1000);
 
+populateSlaveInfo();
 function populateSlaveInfo(){
 	let slaveID = getParameterByName("slaveID");
 	if(!slaveID){
@@ -241,7 +223,6 @@ function maximizeTerminal(instant){
 	setTimeout(() => $('#terminal')[0].style.transition = 'none', (Number(instant) || 200));
 	$('#terminal').draggable('enable');
 }
-populateSlaveInfo();
 
 // ID of slave, ID of canvasjs div without #
 function makeGraph(slaveID, selector) {
@@ -301,7 +282,7 @@ function drawChart(selector, chartData) {
 	console.log(chartData)
 	chartsByID[selector] = new CanvasJS.Chart(selector, {
 		title:{
-			text: "Production graph"
+			text: title || "Production graph"
 		},
 		toolTip:{   
 			content: "{name}: {y}"	  
@@ -328,4 +309,5 @@ function drawChart(selector, chartData) {
 	chart = chartsByID[selector];
 	// console.log(chart)
 	chart.render();
-}
+	return chart;
+}

static/storage.js

@@ -32,6 +32,9 @@ function drawcontents(data) {
 		const name = row.querySelector('.name');
 		if(name.textContent !== item.name) {
 			name.textContent = item.name;
+			row.onclick = ()=>{
+				makeItemGraphs(item.name);
+			}
 		}
 		
 		const count = row.querySelector('.count');
@@ -73,4 +76,101 @@ function sortByKey(array, key) {
 
 document.querySelector("#search").addEventListener('input', function() { 
 	drawcontents();
-})
+});
+
+// Make graph of cluster IO items
+/*getJSON("/api/getStats", function(err, data){
+	
+}*/
+function getItemStatData(item, callback){
+	var xhr = new XMLHttpRequest();
+	var url = "/api/getStats";
+	xhr.open("POST", url, true);
+	xhr.setRequestHeader("Content-type", "application/json");
+	xhr.onreadystatechange = function () {
+		if (xhr.readyState === 4 && xhr.status === 200) {
+			var json = JSON.parse(xhr.responseText);
+			console.log(json.email + ", " + json.password);
+			callback(json);
+		}
+	};
+	var requestData = JSON.stringify({
+		"statistic": "place",
+		"itemName": "shit",
+	});
+	xhr.send(data);
+}
+////// drawChart("contentGraph", /api/getStats)
+
+
+function makeItemGraphs(itemName){
+	getJSON("/api/slaves", function(err, slaves){
+		if(err) throw err;
+		if(typeof slaves == "string"){
+			slaves = JSON.parse(slaves);
+		}
+		let slaveIDs = Object.keys(slaves);
+		let HTML = ""
+		// Draw in/out chart for unregistered slaves as well
+		slaveIDs.unshift("unknown");
+		slaveIDs.forEach(function(slaveID){
+			["place", "remove"].forEach(statistic => {
+				console.log("Making chart for "+slaveID+"'s "+statistic+" statistic")
+				HTML += '<div id="contentGraph'+slaveID+'" class="storageGraph" style="width: 100%;"></div>';
+				
+				setTimeout(function(){
+					let xmlhttp = new XMLHttpRequest();   // new HttpRequest instance 
+					xmlhttp.open("POST", "/api/getStats");
+					xmlhttp.setRequestHeader("Content-Type", "application/json");
+					xmlhttp.onreadystatechange = function(){
+						console.log(xmlhttp.status);
+						if(xmlhttp.readyState === XMLHttpRequest.DONE && xmlhttp.status === 200) {
+							// console.log(xmlhttp.responseText);
+							if(isJSON(xmlhttp.responseText) && JSON.parse(xmlhttp.responseText).statusForDebugging === undefined){
+								drawChart("contentGraph"+slaveID, [JSON.parse(xmlhttp.response).data]);
+								console.log(JSON.parse(xmlhttp.response).data)
+							}
+						}
+					}
+					xmlhttp.send(JSON.stringify({statistic:statistic, itemName:itemName, slaveID:slaveID}));
+				},1000);
+			});
+		});
+		document.querySelector("#graphDisplay > #storageGraphContainer").innerHTML = HTML;
+	});
+}
+let chartsByID = {};
+function drawChart(selector, chartData, title) {
+	// selector is ID of element, ex "chartContainer" or "-123199123"
+	console.log(chartData)
+	chartsByID[selector] = new CanvasJS.Chart(selector, {
+		title:{
+			text: title || "Production graph"
+		},
+		toolTip:{   
+			content: "{name}: {y}"	  
+		},
+		zoomEnabled: true,
+		axisY:{
+			includeZero: true,
+		},
+		legend: {
+			cursor: "pointer",
+			itemclick: function (e) {
+				//console.log("legend click: " + e.dataPointIndex);
+				//console.log(e);
+				if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
+					e.dataSeries.visible = false;
+				} else {
+					e.dataSeries.visible = true;
+				}	
+				e.chart.render();
+			}
+		},
+		data: chartData
+	});
+	chart = chartsByID[selector];
+	// console.log(chart)
+	chart.render();
+	return chart;
+}

static/style.css

@@ -19,10 +19,21 @@ button {
 	background-color: #262626;
 	color: 	#C8C8C8;
 }
+#storage {
+	float: left;
+}
 /* search field */
 #storage > input {
 	background-color:black;
 	width:300px;
+	box-sizing: border-box;
+}
+/* Graphs on the right of storage page */
+#body > #graphDisplay {
+	float:left;
+	width: calc(100% - 300px);
+	min-height:100px;
+	background-color: red;
 }
 /* Navbar on the right */
 #sidenav {