Merge branch 'dependencyRemoval'

README.md

@@ -16,12 +16,12 @@ Discord for development/support/play: https://discord.gg/5XuDkje
 
 * [Windows setup](#windows-setup)
 
+* [Optional plugins](#Plugins)
+
 * [Common problems](#Common-problems)
 
 * [Command cheatsheet](#cheatsheet)
 
-* [Optional plugins](#Plugins)
-
 ## Introduction
 
 Features:
@@ -36,7 +36,7 @@ Features:
 
 - Inventory combinator to display item levels in the cluster (and epoch time)
 
-- Reporting of graphs and UPS on master interface
+- Reporting of graphs and UPS on master interface (Also has extensive Prometheus reporting)
 
 Optional extras (see [Plugins](#Plugins))
 
@@ -205,6 +205,12 @@ Fancy game client that does the following steps automatically, but is really old
 
 3. Run factorio and connect to slave as a normal MP game. You will find the port number to connect to at http://[masterAddress]:8080
 
+## Plugins
+Here are the known Clusterio plugins in the wild:
+1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
+2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
+3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.
+
 ## Common problems
 
 ### Cannot find module: `/../../config`
@@ -227,6 +233,22 @@ if that does not suffice do
 
 If it still doesn't work, come over to our support discord linked at the top of this readme.
 
+### Portforwarding doesn't work on the master server when running under WSL
+
+If you follow the ubuntu guide on WSL (Windows Subsystem for Linux, Bash on Ubuntu on Windows specifically), you will find that the website works on localhost and on your local ip, but not on the global ip. This is also true when you correctly port-forwarded the correct ports. Even when routing this server through nginx in WSL, the issue persists. Then, on a hunch, I tried to run nginx from windows itself and found that this DID work. It came to me that the only usage difference between the 2 versions of nginx is that I got a Windows Firewall popup. 
+
+TLDR: the tested fix is: 
+
+- open your windows firewall and go to advanced settings
+
+- click on inbound rules and click on new rule...
+
+- select port and click next >
+
+- select TCP and select specific local ports and type in the ports that you want to open (comma separated) and click next > 3 times
+
+- give the rule a name (like 'web server' or something), give it a description (optionally) and click finish
+
 ### Other fixes for other potential problems:
 
 Sometimes the install fails. Try `node ./lib/npmPostinstall.js` to complete it.
@@ -259,9 +281,3 @@ npm install
 ```
 node client.js manage shared mods add clusterio
 ```
-
-## Plugins
-Here are the known Clusterio plugins in the wild:
-1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
-2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
-3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.

master.js

@@ -10,6 +10,10 @@ combinator signals.
 node master.js
 */
 
+// Attempt updating
+// const updater = require("./updater.js");
+// updater.update().then(console.log);
+
 (async ()=>{
 
 // Set the process title, shows up as the title of the CMD window on windows
@@ -35,7 +39,6 @@ const averagedTimeSeries = require("averaged-timeseries");
 const deepmerge = require("deepmerge");
 const path = require("path");
 const fs = require("fs-extra");
-const nedb = require("nedb");
 const jwt = require('jsonwebtoken');
 const bcrypt = require('bcryptjs');
 const crypto = require('crypto');
@@ -145,11 +148,6 @@ const prometheusWsUsageCounter = new Prometheus.Counter({
 	help: 'Websocket traffic',
 	labelNames: ["connectionType", "instanceID"],
 });
-const prometheusProductionGauge = new Prometheus.Gauge({
-	name: prometheusPrefix+'production_gauge',
-	help: 'Items produced by instance',
-	labelNames: ["instanceID", "itemName"],
-});
 const prometheusExportGauge = new Prometheus.Gauge({
 	name: prometheusPrefix+'export_gauge',
 	help: 'Items exported by instance',
@@ -199,7 +197,7 @@ app.get('/metrics', (req, res) => {
 	res.set('Content-Type', Prometheus.register.contentType);
 	
 	/// gather some static metrics
-	reisterMoreMetrics();
+	registerMoreMetrics();
 	res.end(Prometheus.register.metrics());
 });
 
@@ -230,24 +228,7 @@ function registerMoreMetrics(){
 }
 
 // set up database
-var Datastore = require('nedb');
-db = {};
-var LinvoDB = require("linvodb3");
-
-// TODO: This breaks. databaseDirectory is either "database" or "./database". `LinvoDB.dbPath = "./database/linvodb/";` is the old working version.
-LinvoDB.dbPath = path.resolve(config.databaseDirectory, "linvodb");
-// LinvoDB.dbPath = "./database/linvodb/";
-
-// database for items in system
-// db.items = new Datastore({ filename: 'database/items.db', autoload: true });
-
-// in memory database for combinator signals
-db.signals = new Datastore({ filename: path.resolve(config.databaseDirectory, '/signals.db'), autoload: true, inMemoryOnly: true});
-db.signals.ensureIndex({ fieldName: 'time', expireAfterSeconds: 3600 }, function (err) {});
-
-// production chart database
-db.flows = new LinvoDB("flows", {}, {});
-
+const db = {};
 
 (function(){
 	try{
@@ -618,43 +599,6 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	}
 });
 
-// circuit stuff
-/**
-POST endpoint to send and store circuit frames on the master.
-Gives no response
-
-@memberof clusterioMaster
-@instance
-@alias api/setSignal
-@param {object} circuitFrame
-@param {number} circuitFrame.time
-*/
-app.post("/api/setSignal", authenticate.middleware, function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	if(typeof req.body == "object" && req.body.time){
-		db.signals.insert(req.body);
-		// console.log("signal set");
-	}
-});
-/**
-POST endpoint to read database of circuit signals sent to master
-
-@memberof clusterioMaster
-@instance
-@alias api/readSignal
-@returns {object} circuitFrame
-*/
-app.post("/api/readSignal", function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	// request.body should be an object
-	// {since:UNIXTIMESTAMP,}
-	// we should send an array of all signals since then
-	db.signals.find({time:{$gte: req.body.since}}, function (err, docs) {
-		// $gte means greater than or equal to, meaning we only get entries newer than the timestamp
-		res.send(docs);
-		// console.log(docs);
-	});
-});
 /**
 GET endpoint to read the masters current inventory of items.
 
@@ -693,171 +637,6 @@ app.get("/api/inventoryAsObject", function(req, res) {
 	res.send(JSON.stringify(db.items));
 });
 
-// post flowstats here for production graphs
-// {timestamp: Date, instanceID: string, data: {"item":number}}
-/**
-POST endpoint to log production graph statistics. Should contain a timestamp
-gathered from Date.now(), a instanceID (also reffered to as "unique") and of
-course the timeSeries data.
-
-@memberof clusterioMaster
-@instance
-@alias api/logStats
-@param {object} JSON {timestamp: Date.now(), instanceID: "string", data: {"item": number}}
-@returns {string} failure
-*/
-app.post("/api/logStats", authenticate.middleware, function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	if(typeof req.body == "object" && req.body.instanceID && req.body.timestamp && req.body.data) {
-		if(Number(req.body.timestamp) != NaN){
-			req.body.timestamp = Number(req.body.timestamp);
-			db.flows.insert({
-				instanceID: req.body.instanceID,
-				timestamp: req.body.timestamp,
-				data: req.body.data,
-			});
-			try{
-			Object.keys(req.body.data).forEach(itemName => {
-				prometheusProductionGauge.labels(req.body.instanceID, itemName).inc(Number(req.body.data[itemName]) || 0);
-			});
-			}catch(e){console.log(e)};
-			console.log("inserted: " + req.body.instanceID + " | " + req.body.timestamp);
-		} else {
-			console.log("error invalid timestamp " + req.body.timestamp);
-			res.send("failure");
-		}
-	} else {
-		res.send("failure");
-	}
-});
-// {instanceID: string, fromTime: Date, toTime, Date}
-/**
-POST endpoint to get timeSeries statistics stored on the master. Can give production
-graphs and other IO statistics.
-
-@memberof clusterioMaster
-@instance
-@alias api/getStats
-@param {object} JSON
-@returns {object} - with statistics
-*/
-app.post("/api/getStats", function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	if(typeof req.body == "string"){
-		req.body = JSON.parse(req.body);
-	}
-	// console.log(req.body);
-	if(typeof req.body == "object" && req.body.instanceID && req.body.statistic === undefined) {
-		// if not specified, get stats for last 24 hours
-		if(!req.body.fromTime) {
-			req.body.fromTime = Date.now() - 86400000 // 24 hours in MS
-		}
-		if(!req.body.toTime) {
-			req.body.toTime = Date.now();
-		}
-		console.log("Looking... " + JSON.stringify(req.body));
-		db.flows.find({
-			instanceID: req.body.instanceID,
-		}, function(err, docs) {
-			let entries = docs.filter(function (el) {
-				return el.timestamp <= req.body.toTime && el.timestamp >= req.body.fromTime;
-			});
-			// console.log(entries);
-			res.send(entries);
-		});
-	} else if(typeof req.body == "object" && req.body.instanceID && typeof req.body.instanceID == "string" && req.body.itemName){
-		if(req.body.statistic == "place"){
-			console.log(`sending place data for instanceID ${req.body.instanceID} ${req.body.itemName}`);
-			// Gather data
-			//console.log(recievedItemStatisticsBySlaveID)
-			let itemStats = recievedItemStatisticsBySlaveID[req.body.instanceID];
-			if(itemStats === undefined){
-				res.send({statusForDebugging:"no data available"});
-				return false;
-			}
-			let data = itemStats.get(config.itemStats.maxEntries, req.body.itemName);
-			
-			res.send({
-				maxEntries:config.itemStats.maxEntries,
-				entriesPerSecond: config.itemStats.entriesPerSecond,
-				data: data,
-			});
-		} else if(req.body.statistic == "remove"){
-			console.log(`sending remove data for instanceID ${req.body.instanceID} ${req.body.itemName}`);
-			// Gather data
-			// console.log(sentItemStatisticsBySlaveID)
-			let itemStats = sentItemStatisticsBySlaveID[req.body.instanceID];
-			console.log(itemStats)
-			if(typeof itemStats == "object"){
-				let data = itemStats.get(config.itemStats.maxEntries, req.body.itemName);
-				//console.log(itemStats.get(config.itemStats.maxEntries, req.body.itemName));
-				res.send({
-					maxEntries:config.itemStats.maxEntries,
-					entriesPerSecond: config.itemStats.entriesPerSecond,
-					data: data,
-				});
-			} else {
-				res.send({statusForDebugging:"no data available"});
-			}
-		}
-	}
-});
-
-/**
-POST endpoint. Modified version of api/getStats which feeds the web interface production graphs
-at /nodes. It is like getStats but it does not report items which were not
-produced at the moment of recording. (This is to save space, the 0-items were
-making up about 92% of the response body weight.)
-
-@memberof clusterioMaster
-@instance
-@alias api/getTimelineStats
-@param {object} JSON {instanceID:1941029, fromTime: ???, toTime: ???}
-@returns {object[]} timeseries where each entry is a set point in time.
-*/
-app.post("/api/getTimelineStats", function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	console.log(req.body);
-	if(typeof req.body == "object" && req.body.instanceID) {
-		// if not specified, get stats for last 24 hours
-		if(!req.body.fromTime) {
-			req.body.fromTime = Date.now() - 86400000 // 24 hours in MS
-		}
-		if(!req.body.toTime) {
-			req.body.toTime = Date.now();
-		}
-		console.log("Looking... " + JSON.stringify(req.body));
-		db.flows.find({
-			instanceID: req.body.instanceID,
-		}, function(err, docs) {
-			if(err) { 
-				console.error(err);
-				return;
-			}
-			
-			//Filter out all the entries outside the time range.
-			docs = docs.filter(function (el) {
-				return el.timestamp <= req.body.toTime && el.timestamp >= req.body.fromTime;
-			});
-			
-			//Filter out all the elements that weren't produced.
-			docs.forEach(el => {
-				for(let key in el.data) {
-					if(el.data[key] === '0') {
-						// Set value to undefined instead of using the delete keyword.
-						// This is because the delete keyword is super duper slow.
-						// https://stackoverflow.com/questions/208105/how-do-i-remove-a-property-from-a-javascript-object
-						
-						//delete el.data[key];
-						el.data[key] = undefined;
-					}
-				}
-			});
-			
-			res.send(docs);
-		});
-	}
-});
 /**
 POST endpoint for running commands on slaves.
 Requires x-access-token header to be set. Find you api token in secret-api-token.txt on the master (after running it once)
@@ -1152,6 +931,7 @@ async function getPlugins(){
 						path: pluginConfig.pluginPath,
 						socketio: io,
 						express: app,
+						Prometheus,
 					}),
 					pluginConfig,
 				});

package.json

@@ -44,15 +44,12 @@
     "html5-validator": "^1.0.7",
     "ini": "^1.3.5",
     "jsonwebtoken": "^8.2.1",
-    "leveldown": "^1.9.0",
-    "linvodb3": "^3.25.1",
     "luamin": "^1.0.4",
     "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
     "mkdirp-promise": "^5.0.1",
     "moment": "^2.22.2",
     "ncp": "^2.0.0",
-    "nedb": "^1.8.0",
     "needle": "^2.2.1",
     "node-factorio-api": "^0.3.1-alpha",
     "node-forge": "^0.7.1",

sharedPlugins/clusterioMod/masterPlugin.js

@@ -3,8 +3,6 @@ const path = require("path");
 const Express = require("express");
 const ejs = require("ejs");
 
-const pmSockets = [];
-
 class masterPlugin {
 	constructor({config, pluginConfig, pluginPath, socketio, express}){
 		this.config = config;

sharedPlugins/statisticsExporter/Lua/exportStatistics.lua

@@ -0,0 +1,19 @@
+local statisticsData = "["
+for _, force in pairs(game.forces) do
+	statisticsData = statisticsData .. '{"forceName":"'..force.name..'","data":{'
+	local statistics = {"item_production_statistics", "fluid_production_statistics", "kill_count_statistics", "entity_build_count_statistics"}
+	for key, stat in pairs(statistics) do
+		statisticsData = statisticsData..'"'..stat..'":{"input":{'
+		for name, value in pairs(force[stat].input_counts) do
+			statisticsData = statisticsData..'"'..name..'":'..value..','
+		end
+		statisticsData = statisticsData..'},"output":{'
+		for name, value in pairs(force[stat].output_counts) do
+			statisticsData = statisticsData..'"'..name..'":'..value..','
+		end
+		statisticsData = statisticsData .. '}},'
+	end
+	statisticsData = statisticsData..'}},'
+end
+statisticsData = statisticsData..']'
+rcon.print(statisticsData)

sharedPlugins/statisticsExporter/config.js

@@ -0,0 +1,10 @@
+/*
+	Clusterio plugin for gathering ingame statistics
+*/
+module.exports = {
+	name: "statisticsExporter",
+	version: "1.0.0",
+	binary: "nodePackage",
+	description: "Gathers ingame stats such as production, consumption, kills and buildings placed",
+	masterPlugin: "masterPlugin.js"
+}

sharedPlugins/statisticsExporter/index.js

@@ -0,0 +1,40 @@
+const needle = require("needle");
+const fs = require("fs-extra");
+const path = require("path");
+
+const clusterTools = require("_app/clusterTools")();
+const pluginConfig = require("./config");
+
+module.exports = class remoteCommands {
+	constructor(mergedConfig, messageInterface, extras){
+		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		
+		const needleOptionsWithTokenAuthHeader = {
+			headers: {
+				'x-access-token': this.config.masterAuthToken
+			},
+		};
+		
+		// Gather and submit statistics every 15 seconds
+		setInterval(async () => {
+			let stats = await this.getStats();
+			stats.forEach(force => {
+				needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, {
+					instanceID: this.config.unique,
+					force: force.forceName,
+					data: force.data,
+				}, needleOptionsWithTokenAuthHeader, (err, resp) => {
+					if(!err && resp.body && resp.body.ok == false) console.log(resp.body);
+				});
+			});
+		}, 15000);
+	}
+	async getStats(){
+		let command = fs.readFileSync(path.join(__dirname, "Lua/exportStatistics.lua"), "utf8")//clusterTools.getLua(path.join(__dirname, "Lua/exportStatistics.lua"), false);
+		let string = await this.messageInterface(`/silent-command ${command}`);
+		var data;
+		eval(`data = ${string}`);
+		return data;
+	}
+}

sharedPlugins/statisticsExporter/masterPlugin.js

@@ -0,0 +1,58 @@
+const fs = require("fs-extra");
+const path = require("path");
+
+class masterPlugin {
+	constructor({config, socketio, express, Prometheus}){
+		const authenticate = require("./../../lib/authenticate")(config);
+		
+		this.config = config;
+		this.io = socketio;
+		const app = express;
+		
+		const prometheusStatisticsGauge = new Prometheus.Gauge({
+			name: 'clusterio_statistics_gauge',
+			help: 'Items produced/built/killed by a force',
+			labelNames: ["instanceID", "force", "statistic", "direction", "itemName"],
+		});
+		
+		/** --- TODO: Replace this with a simpler Prometheus exporter. Requires changes to the mod as well.
+		POST endpoint to log production graph statistics. Should contain an instanceID (also reffered to as "unique")
+		and of course the timeSeries data organized by category.
+
+		@memberof clusterioMaster
+		@instance
+		@alias api/logStats
+		@param {object} JSON {instanceID: "string", force: "string" data: {"item_production_statistics": {"input":{"iron-plate":150}, "output":{}}, ...}}
+		@returns {string} failure
+		*/
+		app.post("/api/logStats", authenticate.middleware, function(req,res) {
+			// endpointHitCounter.labels(req.route.path).inc();
+			if(typeof req.body == "object"
+			&& req.body.instanceID
+			&& typeof req.body.instanceID == "string"
+			&& req.body.instanceID.length < 100
+			&& req.body.data
+			&& typeof req.body.data == "object"
+			&& !Array.isArray(req.body.data)){
+				try{
+					Object.keys(req.body.data).forEach(statisticName => {
+						Object.keys(req.body.data[statisticName]).forEach(direction => {
+							let statHash = req.body.data[statisticName][direction];
+							Object.keys(statHash).forEach(item => {
+								let value = Number(statHash[item]);
+								if(value) prometheusStatisticsGauge.labels(req.body.instanceID, req.body.force, statisticName, direction, item).set(value);
+							});
+						});
+					});
+					res.send({ok:true, msg:`Statistics successfully reported at ${Date.now()}`});
+				}catch(e){
+					res.send({ok:false, msg:"Statistics reporter crashed, see sharedPlugins/statisticsExporter/masterPlugin.js"});
+					console.log(e)
+				};
+			} else {
+				res.send("failure");
+			}
+		});
+	}
+}
+module.exports = masterPlugin;