Merge pull request #2 from Danielv123/master

get latest

.gitignore

@@ -39,6 +39,7 @@ mod_cache.json
 static/pictures/spritesheet.js
 static/pictures/spritesheet.png
 static/external
+saves
 
 # weird pattern to ignore sharedPlugins except not ignore some core modules
 !sharedPlugins/
@@ -48,3 +49,6 @@ sharedPlugins/*
 !sharedPlugins/remoteMap/
 !sharedPlugins/remoteCommands/
 !sharedPlugins/researchSync/
+!sharedPlugins/globalChat/
+
+\.cache-require-paths\.json

README.md

@@ -1,12 +1,12 @@
 # factorioClusterio
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2FDanielv123%2FfactorioClusterio.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FDanielv123%2FfactorioClusterio?ref=badge_shield)
 
-| Branch | Coverage | Build |
-|--------|----------| ----- |
-| Master | [![codecov](https://codecov.io/gh/Danielv123/factorioClusterio/branch/master/graph/badge.svg)](https://codecov.io/gh/Danielv123/factorioClusterio) | [![Build Status](https://travis-ci.org/Danielv123/factorioClusterio.svg?branch=master)](https://travis-ci.org/Danielv123/factorioClusterio) |
-| Dev    | [![codecov](https://codecov.io/gh/Danielv123/factorioClusterio/branch/dev/graph/badge.svg)](https://codecov.io/gh/Danielv123/factorioClusterio) | [![Build Status](https://travis-ci.org/Danielv123/factorioClusterio.svg?branch=dev)](https://travis-ci.org/Danielv123/factorioClusterio) |
+Discord for development/support/play: https://discord.gg/5XuDkje
 
-If you want to connect to a clusterio cluster, please reffer to the [client](https://github.com/Danielv123/factorioClusterioClient)
+### Ways to support me/the project:
+
+* Contribute with code/documentation. Always nice to move the project forward
+
+* Support me monetarily on [patreon](https://www.patreon.com/danielv123) or paypal: danielv@live.no
 
 ### Table of contents
 
@@ -16,6 +16,8 @@ If you want to connect to a clusterio cluster, please reffer to the [client](htt
 
 * [Windows setup](#windows-setup)
 
+* [Common problems](#Common-problems)
+
 * [Command cheatsheet](#cheatsheet)
 
 ## Introduction
@@ -70,40 +72,56 @@ NodeJS does not support EOL ubuntu releases. Make sure you are on the most recen
 
 Master and all slaves:
 
-    sudo curl -sL https://deb.nodesource.com/setup_9.x | sudo -E bash - \
-    sudo apt install -y nodejs python-dev git wget curl tar build-essential \
-    sudo npm install pm2 -g \
-    git clone https://github.com/Danielv123/factorioClusterio.git \
-    cd factorioClusterio \
-    npm install \
-    curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 \
+    sudo curl -sL https://deb.nodesource.com/setup_9.x | sudo -E bash -
+    sudo apt install -y nodejs python-dev git wget curl tar build-essential
+    sudo npm install pm2 -g
+    git clone https://github.com/Danielv123/factorioClusterio.git
+    cd factorioClusterio
+    curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64
     tar -xf factorio.tar.gz
+    npm install
+    cp config.json.dist config.json
+    node ./lib/npmPostinstall.json
+	
 
 downloads and installs nodejs, pm2, git and clusterio. To specify a version, change "latest" in the link to a version number like 0.14.21.
 
+Now you need to edit the `config.json` file. If you skip this step nothing will work.
+Pretty much all the blank fields should be filled in, except on the master where a few can be omitted.
+
+* You get the `masterAuthToken` from `secret-api-token.txt` in the master install dir after running the master twice.
+
+* You get your factorio matchmaking token from factorio.com
+
+* The `masterAuthSecret` should never be touched unless you want to invalidate everyones authentication tokens
+
 **Master**
 
     pm2 start master.js --name master
+	
+OR
+
+    node master.js
     
 **Server Host**
     
-To download the mod for all its non vanilla features and items, (optional)
-
-    node client.js download
+To download the mod for all its non vanilla features and items, (optional, but very recommended)
 
+    node client.js manage shared mods add clusterio
+	
 To create a new instance (its own save, set of mods and config files)
 
     node client.js start [instancename]
 
 To launch an instance with pm2
 
-    pm2 start --name slave client.js -- start [instancename]
+    pm2 start --name [instancename] client.js -- start [instancename]
 
 use `nano config.json` to change settings.
 
 **Ubuntu with Docker**
 
-Clusterio has limited support for using docker.
+Clusterio has *very* limited support for using docker.
 
     sudo docker build -t clusterio --no-cache --force-rm factorioClusterio
 	
@@ -123,13 +141,9 @@ Server host (Slave) = client.js + factorio server
 
 Game Client = The people connecting to the server
 
-All Server Hosts (Slaves) AND Game Clients need to be running the clusterio mod located at [github](https://github.com/Danielv123/factorioClusterioMod) Install it by dropping it into either the sharedMods folder or instances/[instanceName]/instanceMods folder.
-
-There are no requirements for other mods, they can be ran in any configuration allowed by the base game.
-
 **Requirements**
 
-download and install nodeJS 6+ from http://nodejs.org
+download and install nodeJS 8 or 9 from http://nodejs.org
 
 download and install git from https://git-scm.com/
 
@@ -139,29 +153,40 @@ reboot when you are done, then proceed to the next steps. *reboots matter*
 
 1. download and run https://puu.sh/toFHl/01eebbb333.bat
 
-2. You do not *need* to follow the given instructions, but you should
+2. Copy config.json.dist to config.json
 
-3. type `node master.js`
+3. Follow the instructions in the bat file
+3. Some of the instructions are outdated. If you get stuck somewhere, look at the Ubuntu section.
+
+4. Run `node client.js manage shared mods add clusterio`
+
+5. type `node master.js` to start the server
 
 **Server Host**
 
 1. download and run https://puu.sh/toFHl/01eebbb333.bat
 
-2. Follow the instructions given
+2. Copy config.json.dist to config.json
 
-3. Type `node client.js start [instancename]` to create a new instance.
+3. Follow the instructions given. 
+
+3.5 Some of the instructions are outdated. If you get stuck somewhere, look at the Ubuntu section.
+
+4. Type `node client.js start [instancename]` to create a new instance.
 
 To connect to a master server running on a remote machine, open config.json with your favourite text editor (notepad++). You can also set it up to use the official server browser.
 
-Change masterIP to something like 31.152.123.14 (provided by master server owner)
+Change `masterIP `to something like `31.152.123.14` (provided by master server owner)
 
-Change masterPort to something like 8080 (provided by master server owner)
+Change `masterPort` to something like `8080` (provided by master server owner)
 
-Repeat step 3 for more servers on one machine. You should be able to find its port by looking at the slave section on master:8080 (the web interface)
+Change `masterAuthToken` to the value found in `secret-api-token.txt` on the master server
+
+Repeat step 4 for more servers on one machine. You should be able to find its port by looking at the slave section on master:8080 (the web interface)
 
 **GameClient**
 
-Fancy game client that does the following steps automatically: [clusterioClient](https://github.com/Danielv123/factorioClusterioClient)
+Fancy game client that does the following steps automatically, but is really old so be warned: [clusterioClient](https://github.com/Danielv123/factorioClusterioClient)
 
 1. Download the same version of the mod as the slave is running from [the mod portal](https://mods.factorio.com/mods/Danielv123/clusterio) or [github](https://github.com/Danielv123/factorioClusterioMod
 
@@ -169,6 +194,21 @@ Fancy game client that does the following steps automatically: [clusterioClient]
 
 3. Run factorio and connect to slave as a normal MP game. You will find the port number to connect to at http://[masterAddress]:8080
 
+## Common problems
+
+### Cannot find module: `/../../config`
+
+Copy your config.json.dist to config.json and configure it.
+
+### EACCESS [...] LISTEN 443
+
+Some systems don't let non root processes listen to ports below 1000. Either run with `sudo` or change config.json to use higher port numbers.
+
+### Other fixes for other potential problems:
+
+Sometimes the install fails. Try `node ./lib/npmPostinstall.js` to complete it.
+
+
 ## Cheatsheet
 
 **To create a new instance/start it**
@@ -194,5 +234,5 @@ npm install
 
 3. Download the latest version of the factorioClusterioMod from its github repo
 ```
-node client.js download
+node client.js manage shared mods add clusterio
 ```

client.js

@@ -6,7 +6,7 @@ const path = require('path');
 const syncRequest = require('sync-request');
 const request = require("request");
 const ncp = require('ncp').ncp;
-const Rcon = require('simple-rcon');
+const Rcon = require('rcon-client').Rcon;
 const hashFiles = require('hash-files');
 const _ = require('underscore');
 const deepmerge = require("deepmerge");
@@ -21,12 +21,15 @@ const stringUtils = require("./lib/stringUtils.js");
 const modManager = require("./lib/manager/modManager.js");
 const configManager = require("./lib/manager/configManager.js");
 
+// argument parsing
+const args = require('minimist')(process.argv.slice(2));
+
 // require config.json
-var config = require('./config');
+var config = require(args.config || './config');
 var global = {};
 
-if (!fs.existsSync("./instances/")) {
-	fs.mkdirSync("instances");
+if (!fs.existsSync(config.instanceDirectory)) {
+	fs.mkdirSync(config.instanceDirectory);
 }
 if (!fs.existsSync("./sharedPlugins/")) {
 	fs.mkdirSync("sharedPlugins");
@@ -35,23 +38,44 @@ if (!fs.existsSync("./sharedMods/")) {
 	fs.mkdirSync("sharedMods");
 }
 const instance = process.argv[3];
-const instancedirectory = './instances/' + instance;
+const instancedirectory = config.instanceDirectory + '/' + instance;
 const command = process.argv[2];
 
 // Set the process title, shows up as the title of the CMD window on windows
 // and as the process name in ps/top on linux.
 process.title = "clusterioClient "+instance;
 
+// make sure we have the master access token (can't write to master without it since clusterio 2.0)
+if(!config.masterAuthToken || typeof config.masterAuthToken !== "string"){
+	console.log("ERROR invalid config!");
+	console.log("Master server now needs an access token for write operations. As clusterio slaves depends \
+	upon this, please add your token to config.js in the field named masterAuthToken. \
+	You can retrieve your auth token from the master in secret-api-token.txt after running it once.");
+}
+const needleOptionsWithTokenAuthHeader = {
+	headers: {
+		'x-access-token': config.masterAuthToken
+	},
+};
+
 var instanceInfo = {};
+var commandBuffer=[];
+// messageInterface Management
+setInterval(function(){
+	let command=commandBuffer.shift();
+	if(command){
+		messageInterfaceInternal(command[0], command[1], command[2], command[3]);
+	}
+},config.msBetweenCommands || 50);
 
 // function to handle sending commands into the game
-function messageInterface(command, callback) {
+async function messageInterfaceInternal(command, callback, resolve, reject) {
 	// try to save us if you send a buffer instead of string
 	if(typeof command == "object") {
 		command = command.toString('utf8');
 	}
 	
-	if(process.platform == "linux" && typeof command == "string" && serverprocess) {
+	if(false && process.platform == "linux" && typeof command == "string" && serverprocess) {
 		/*
 			to send to stdin, use:
 			serverprocess.stdin.write("/c command;\n")
@@ -60,17 +84,28 @@ function messageInterface(command, callback) {
 		if(typeof callback == "function"){
 			callback();
 		}
-	} else if(typeof command == "string" && client && client.exec && typeof client.exec == "function") {
+		resolve();
+	} else if(typeof command == "string" && client && client.send && typeof client.send == "function") {
 		try {
-			client.exec(command+"\n", callback);
+			let str = await client.send(command+"\n");
+			if(typeof callback == "function") callback(str)
+			resolve(str)
 		} catch (err) {
-			console.log(err);
-		}
-		if(typeof callback == "function"){
-			callback();
+			console.log("RCON failed, server might be paused or shutting down");
+			// console.log(err);
+			// reject(err);
+			if(typeof callback == "function"){
+				callback();
+			}
 		}
 	}
 }
+function messageInterface(command, callback) {
+	return new Promise((resolve,reject) => {
+		commandBuffer.push([command,callback, resolve, reject]);
+	});
+}
+
 
 // handle commandline parameters
 if (!command || command == "help" || command == "--help") {
@@ -84,7 +119,7 @@ if (!command || command == "help" || command == "--help") {
 	console.error("node client.js manage");
 	process.exit(1);
 } else if (command == "list") {
-	let instanceNames = fileOps.getDirectoriesSync("./instances/");
+	let instanceNames = fileOps.getDirectoriesSync(config.instanceDirectory);
 	instanceNames.unshift("Name:");
 	let longestInstanceName = 0;
 	// determine longest instance name
@@ -107,7 +142,7 @@ if (!command || command == "help" || command == "--help") {
 		if(instance.includes("Name:")){
 			factorioPort = "Port:"
 		} else {
-			factorioPort = require("./instances/"+instance+"/config").factorioPort;
+			factorioPort = require(config.instanceDirectory+"/"+instance+"/config").factorioPort;
 		}
 		factorioPorts.push(factorioPort);
 	});
@@ -126,7 +161,7 @@ if (!command || command == "help" || command == "--help") {
 	});
 	
 	displayLines.forEach(line => console.log(line));
-	process.exit(1);
+	process.exit(0);
 } else if (command == "manage"){
 	// console.log("Launching mod manager");
 	//const fullUsage = 'node client.js manage [instance, "shared"] ["mods", "config"] ...';
@@ -143,20 +178,26 @@ if (!command || command == "help" || command == "--help") {
 	const action = process.argv[5] || "";
 	if(instance){
 		if(tool == "mods"){
-			// allow managing mods
-			if(action == "list"){
-				modManager.listMods(instance);
-			} else if(action == "search"){
-				modManager.findMods(process.argv[6]);
-			} else if(action == "add" || action == "download"){
-				modManager.addMod(process.argv[6], instance);
-			} else if(action == "remove" || action == "rm" || action == "delete"){
-				modManager.removeMod(process.argv[6], instance);
-			} else if(action == "update"){
-				modManager.updateAllMods();
-			} else {
-				usage(instance, tool);
-			}
+			(async function(){try{
+				// allow managing mods
+				if(action == "list"){
+					console.log(await modManager.listMods(instance));
+				} else if(action == "search"){
+					console.log(await modManager.findMods(process.argv[6]));
+				} else if(action == "add" || action == "download"){
+					await modManager.addMod(process.argv[6], instance);
+				} else if(action == "remove" || action == "rm" || action == "delete"){
+					await modManager.removeMod(process.argv[6], instance);
+				} else if(action == "update"){
+					await modManager.updateAllMods();
+				} else {
+					usage(instance, tool);
+				}
+				process.exit(0);
+			}catch(e){
+				console.log("Got error from modManager:")
+				console.log(e);
+			}})();
 		} else if(tool == "config"){
 			// allow managing the config
 			if(action == "list" || action == "show" || action == "display"){
@@ -181,18 +222,18 @@ if (!command || command == "help" || command == "--help") {
 		console.log('Usage:');
 		usage(instance);
 	}
-	// process.exit(1);
+	// process.exit(0);
 } else if (command == "delete") {
 	if (!process.argv[3]) {
 		console.error("Usage: node client.js delete [instance]");
 		process.exit(1);
-	} else if (typeof process.argv[3] == "string" && fs.existsSync("./instances/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") {
-		fileOps.deleteFolderRecursiveSync("./instances/" + process.argv[3]);
+	} else if (typeof process.argv[3] == "string" && fs.existsSync(config.instanceDirectory+"/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") {
+		fileOps.deleteFolderRecursiveSync(config.instanceDirectory+"/" + process.argv[3]);
 		console.log("Deleted instance " + process.argv[3]);
-		process.exit(1);
+		process.exit(0);
 	} else {
 		console.error("Instance not found: " + process.argv[3]);
-		process.exit(1);
+		process.exit(0);
 	}
 } else if (command == "download") {
 	console.log("Downloading mods...");
@@ -204,9 +245,12 @@ if (!command || command == "help" || command == "--help") {
 		console.log(url);
 		let file = fs.createWriteStream("sharedMods/"+name);
 		https.get(url, function(response) {
+			response.on('end', function () {
+				console.log("Downloaded "+name);
+				process.exit(0);                
+			});
 			response.pipe(file);
-			console.log("Downloaded "+name);
-		});
+		}).end();
 	}
 } else if (command == "start" && instance === undefined) {
 	console.log("ERROR: No instanceName provided!");
@@ -224,17 +268,17 @@ if (!command || command == "help" || command == "--help") {
 	fs.mkdirSync(instancedirectory + "/mods/");
 	fs.mkdirSync(instancedirectory + "/instanceMods/");
 	// fs.symlinkSync('../../../sharedMods', instancedirectory + "/mods", 'junction') // This is broken because it can only take a file as first argument, not a folder
-	fs.writeFileSync(instancedirectory + "/config.ini", "[path]\r\n\
-read-data=__PATH__executable__/../../data\r\n\
-write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
-	");
+	fs.writeFileSync(instancedirectory + `/config.ini`, `[path]\r\n
+read-data=${ path.resolve(config.factorioDirectory, "data") }\r\n
+write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
+	`);
 	
 	// this line is probably not needed anymore but Im not gonna remove it
-	fs.copySync('sharedMods', instancedirectory + "/mods");
+	fs.copySync('sharedMods', path.join(instancedirectory, "mods"));
 	let instconf = {
-		"factorioPort": process.env.FACTORIOPORT || Math.floor(Math.random() * 65535),
-		"clientPort": process.env.RCONPORT || Math.floor(Math.random() * 65535),
-		"clientPassword": Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8),
+		"factorioPort": args.port || process.env.FACTORIOPORT || Math.floor(Math.random() * 65535),
+		"clientPort": args["rcon-port"] || process.env.RCONPORT || Math.floor(Math.random() * 65535),
+		"clientPassword": args["rcon-password"] || Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8),
 	}
 	console.log("Clusterio | Created instance with settings:")
 	console.log(instconf);
@@ -266,15 +310,16 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 	fs.writeFileSync(instancedirectory + "/server-settings.json", JSON.stringify(serversettings, null, 4));
 	let createSave = child_process.spawnSync(
 		'./' + config.factorioDirectory + '/bin/x64/factorio', [
-			'-c', instancedirectory + '/config.ini',
-			'--create', instancedirectory + '/saves/save.zip',
+			'-c', path.resolve(instancedirectory, 'config.ini'),
+			'--create', path.resolve(instancedirectory, 'saves', 'save.zip'),
 		]
 	);
 	console.log("Instance created!");
+	process.exit(0);
 } else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)) {
 	// Exit if no instance specified (it should be, just a safeguard);
-	if(instancedirectory != "./instances/undefined"){
-		var instanceconfig = require(instancedirectory + '/config');
+	if(instancedirectory != config.instanceDirectory+"/undefined"){
+		var instanceconfig = require(path.resolve(instancedirectory,'config'));
 		instanceconfig.unique = stringUtils.hashCode(instanceconfig.clientPassword);
 		if(process.env.FACTORIOPORT){
 			instanceconfig.factorioPort = process.env.FACTORIOPORT;
@@ -286,17 +331,17 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 		process.exit(1);
 	}
 	console.log("Deleting .tmp.zip files");
-	let savefiles = fs.readdirSync(instancedirectory + "/saves/");
+	let savefiles = fs.readdirSync(path.join(instancedirectory,"saves"));
 	for(i = 0; i < savefiles.length; i++){
 		if(savefiles[i].substr(savefiles[i].length - 8, 8) == ".tmp.zip") {
-			fs.unlinkSync(instancedirectory + "/saves/" + savefiles[i]);
+			fs.unlinkSync(path.resolve(instancedirectory, "saves", savefiles[i]));
 		}
 	}
 	console.log("Deleting logs");
 	// clean old log file to avoid crash
 	// file exists, delete so we don't get in trouble
 	try {
-		fs.unlinkSync(instancedirectory+'/factorio-current.log');
+		fs.unlinkSync(path.join(instancedirectory,'factorio-current.log'));
 	} catch (err){
 		if(err){
 			console.log(err);
@@ -306,21 +351,19 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 	}
 	
 	// move mods from ./sharedMods to the instances mod directory
-	try {
-		fs.mkdirSync(instancedirectory + "/instanceMods/");
-	} catch(e){}
-	try{rmdirSync(instancedirectory + "/mods");}catch(e){}
+	try{fs.mkdirSync(path.join(instancedirectory, "instanceMods"));}catch(e){}
+	try{rmdirSync(path.join(instancedirectory, "mods"));}catch(e){}
 	try {
 		// mods directory that will be emptied (deleted) when closing the server to facilitate seperation of instanceMods and sharedMods
-		fs.mkdirSync(instancedirectory + "/mods/");
+		fs.mkdirSync(path.join(instancedirectory, "mods"));
 	} catch(e){}
 	console.log("Clusterio | Moving shared mods from sharedMods/ to instance/mods...");
-	fs.copySync('sharedMods', instancedirectory + "/mods");
+	fs.copySync('sharedMods', path.join(instancedirectory, "mods"));
 	console.log("Clusterio | Moving instance specific mods from instance/instanceMods to instance/mods...");
-	fs.copySync(instancedirectory + "/instanceMods", instancedirectory + "/mods");
+	fs.copySync(path.join(instancedirectory, "instanceMods"), path.join(instancedirectory, "mods"));
 
 	process.on('SIGINT', function () {
-		console.log("Caught interrupt signal");
+		console.log("Caught interrupt signal, sending /quit");
 		messageInterface("/quit");
 	});
 
@@ -328,8 +371,9 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 	//var serverprocess = child_process.exec(commandline);
 	fileOps.getNewestFile(instancedirectory + "/saves/", fs.readdirSync(instancedirectory + "/saves/"),function(err, latestSave) {
 		if(err) {
-			console.log("Your savefile seems to be missing. This might because you created an instance without having factorio\
-			installed and configured properly. Try installing factorio and adding your savefile to instances/[instancename]/saves/");
+			console.error("ERROR!");
+			console.error("Your savefile seems to be missing. This might because you created an instance without having factorio\
+ installed and configured properly. Try installing factorio and adding your savefile to instances/[instancename]/saves/\n");
 			throw err;
 		}
 		// implicit global
@@ -337,10 +381,10 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 			'./' + config.factorioDirectory + '/bin/x64/factorio', [
 				'-c', instancedirectory + '/config.ini',
 				'--start-server', latestSave.file,
-				'--rcon-port', Number(process.env.RCONPORT) || instanceconfig.clientPort,
-				'--rcon-password', instanceconfig.clientPassword,
+				'--rcon-port', args["rcon-port"] || Number(process.env.RCONPORT) || instanceconfig.clientPort,
+				'--rcon-password', args["rcon-password"] || instanceconfig.clientPassword,
 				'--server-settings', instancedirectory + '/server-settings.json',
-				'--port', Number(process.env.FACTORIOPORT) || instanceconfig.factorioPort
+				'--port', args.port || Number(process.env.FACTORIOPORT) || instanceconfig.factorioPort
 			], {
 				'stdio': ['pipe', 'pipe', 'pipe']
 			}
@@ -358,24 +402,17 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 				client.close();
 				client.connect();
 			}
-			if(process.platform == "linux"){
-				// we have to log on linux because linux only shows stdout of the first process launched
-				console.log('Factorio: ' + data);
-			}
+			// we have to do this to make logs visible on linux and in powershell. Causes log duplication for people with CMD.
+			console.log('Fact: ' + data.toString("utf8").replace("\n", ""));
 		});
 		serverprocess.stderr.on('data', (chunk) => {
 			console.log('ERR: ' + chunk);
 		});
 
 		// connect to the server with rcon
-		if(process.platform != "linux"){
+		if(true || process.platform != "linux"){
 			// IP, port, password
-			client = new Rcon({
-				host: 'localhost',
-				port: Number(process.env.RCONPORT) || instanceconfig.clientPort,
-				password: instanceconfig.clientPassword,
-				timeout: 0
-			});
+			client = new Rcon();
 			
 			// check the logfile to see if the RCON interface is running as there is no way to continue without it
 			// we read the log every 2 seconds and stop looping when we start connecting to factorio
@@ -383,28 +420,35 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 				fs.readFile(instancedirectory+"/factorio-current.log", function (err, data) {
 					// if (err) console.log(err);
 					if(data && data.indexOf('Starting RCON interface') > 0){
-						client.connect();
+						client.connect({
+							host: 'localhost',
+							port: args["rcon-port"] || Number(process.env.RCONPORT) || instanceconfig.clientPort,
+							password: args["rcon-password"] || instanceconfig.clientPassword,
+							timeout: 5000
+						});
 					} else {
 						setTimeout(function(){
 							checkRcon();
-						},2000);
+						},5000);
 					}
 				});
 			}
 			setTimeout(checkRcon, 5000);
 		
-			client.on('authenticated', function () {
+			client.onDidAuthenticate(() => {
 				console.log('Clusterio | RCON Authenticated!');
 				instanceManagement(); // start using rcons
-			}).on('connected', function () {
+			});
+			client.onDidConnect(() => {
 				console.log('Clusterio | RCON Connected!');
 				// getID();
-			}).on('disconnected', function () {
+			});
+			client.onDidDisconnect(() => {
 				console.log('Clusterio | RCON Disconnected!');
 				process.exit(0); // exit because RCON disconnecting is undefined behaviour and we rather just wanna restart now
 			});
 		} else if(process.platform == "linux"){
-			// TODO: Check if this works fine on linux or if it has to be delayed until the server starts properly
+			// don't open an RCON connection and just use stdio instead, does not work on windows.
 			instanceManagement();
 		}
 
@@ -412,6 +456,9 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\
 		confirmedOrders = [];
 		lastSignalCheck = Date.now();
 	});
+} else {
+	console.error("Invalid arguments, quitting.");
+	process.exit(1);
 }
 
 // ensure instancemanagement only ever runs once
@@ -447,15 +494,20 @@ function instanceManagement() {
 		if(pluginConfig.binary == "nodePackage"){
 			// require index.js.main() of plugin and execute it as a class
 			let pluginClass = require("./sharedPlugins/" + pluginDirectories[I] + "/index.js");
-			plugins[I] = new pluginClass(combinedConfig, function(data){
+			plugins[I] = new pluginClass(combinedConfig, async function(data, callback){
 				if(data.toString('utf8')[0] != "/") {
 					log("Stdout: " + data.toString('utf8'));
+					return true;
 				} else {
-					messageInterface(data.toString('utf8'));
+					return messageInterface(data.toString('utf8'), callback);
 				}
 			}, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat.
 				socket, // socket.io connection to master (and ES6 destructuring, yay)
 			});
+			if(plugins[I].factorioOutput && typeof plugins[I].factorioOutput === "function"){
+				// when factorio logs a line, send it to the plugin. This includes things like autosaves, chat, errors etc
+				serverprocess.stdout.on("data", data => plugins[I].factorioOutput(data.toString()));
+			}
 			if(pluginConfig.scriptOutputFileSubscription && typeof pluginConfig.scriptOutputFileSubscription == "string"){
 				if(global.subscribedFiles[pluginConfig.scriptOutputFileSubscription]) {
 					// please choose a unique file to subscribe to. If you need plugins to share this interface, set up a direct communication
@@ -487,7 +539,9 @@ function instanceManagement() {
 								}
 								for(let i = 0; i < stuff.length; i++) {
 									if(stuff[i] && !stuff[i].includes('\u0000\u0000')) {
-										plugins[I].scriptOutput(stuff[i]);
+										try{
+											plugins[I].scriptOutput(stuff[i]);
+										}catch(e){console.log(e)}
 									}
 								}
 							},
@@ -497,7 +551,7 @@ function instanceManagement() {
 				}
 			}
 			console.log("Clusterio | Loaded plugin " + pluginDirectories[i]);
-		} else if(pluginConfig.binary != "nodePackage"){
+		} else if(pluginConfig.binary != "nodePackage" && pluginConfig.binary){
 			// handle as fragile executable plugin
 			let args = pluginConfig.args || [];
 			plugins[I]=child_process.spawn(pluginConfig.binary, args, {
@@ -558,6 +612,8 @@ function instanceManagement() {
 			plugins[i].on('close', (code) => {
 				log(`child process exited with code ${code}`);
 			});
+		} else {
+			// This plugin doesn't have a client portion
 		}
 	}
 	
@@ -566,26 +622,7 @@ function instanceManagement() {
 		setInterval(getID, 10000);
 		getID();
 		function getID() {
-			messageInterface("/silent-command game.write_file('tempfile.txt', 'connected_players ' .. #game.connected_players .. '\\n', true, 0)", function(err) {setTimeout(function(){
-				// get array of lines in file
-				if(fs.existsSync(instancedirectory + "/script-output/tempfile.txt")) {
-					var data = fs.readFileSync(instancedirectory + "/script-output/tempfile.txt", "utf8").split("\n");
-					// delete when we are done
-					fs.unlink(instancedirectory + "/script-output/tempfile.txt", function(){});
-				}
-				// if we actually got anything from the file, proceed to categorize it
-				if (data && data[0]) {
-					while (data[0]) {
-						let q = data[0].split(" ");
-						// delete array element
-						data.splice(0,1);
-						if(q[0] == "connected_players" && Number(q[1]) != NaN) {
-							instanceInfo.playerCount = q[1];
-						}
-					}
-				} else {
-					instanceInfo.playerCount = 0;
-				}
+			messageInterface("/silent-command rcon.print(#game.connected_players)", function(playerCount) {
 				var payload = {
 					time: Date.now(),
 					rconPort: instanceconfig.clientPort,
@@ -594,37 +631,35 @@ function instanceManagement() {
 					unique: instanceconfig.unique,
 					publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config
 					mods:modHashes,
-					playerCount: instanceInfo.playerCount || 0,
 					instanceName: instance,
+					playerCount:playerCount.replace(/(\r\n\t|\n|\r\t)/gm, ""),
 				}
 				
 				function callback(err, mac) {
 					if (err) {
+						mac = "unknown";
 						console.log("##### getMac crashed, but we don't really give a shit because we are probably closing down #####");
-					} else {
-						payload.mac = mac
-						console.log("Registered our precense with master "+config.masterIP+" at " + payload.time);
-						needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, function (err, response, body) {
-							if (err && err.code != "ECONNRESET"){
-								console.error("We got problems, something went wrong when contacting master");
-								console.error(err);
-							} else if (response && response.body) {
-								// In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care.
-								console.log(response.body);
-							}
-						});
 					}
+					payload.mac = mac;
+					console.log("Registered our presence with master "+config.masterIP+" at " + payload.time);
+					needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) {
+						if (err && err.code != "ECONNRESET"){
+							console.error("We got problems, something went wrong when contacting master");
+							console.error(err);
+						} else if (response && response.body) {
+							// In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care.
+							console.log(response.body);
+						}
+					});
 				}
 				if(global.mac){
 					callback(undefined, global.mac);
 				} else {
 					getMac(callback);
 				}
-			},1000)});
+			});
 		}
 	});
-	
-	
 	// Mod uploading and management -----------------------------------------------
 	// get mod names and hashes
 	// string: instance, function: callback
@@ -637,17 +672,21 @@ function instanceManagement() {
 				modName: modHashes[i].modName,
 				hash: modHashes[i].hash,
 			}
-			needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, function (err, response, body) {
+			needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) {
 				if(err) console.log(new Error("Unable to contact master server! Please check your config.json."));
 				if(response && body && body == "found") {
-					console.log("master has mod");
+					console.log("master has mod "+modHashes[i].modName);
 				} else if (response && body && typeof body == "string") {
 					let mod = response.body;
 					if(config.uploadModsToMaster){
 						console.log("Sending mod: " + mod);
 						// Send mods master says it wants
 						// response.body is a string which is a modName.zip
-						var req = request.post("http://"+config.masterIP + ":" + config.masterPort + '/api/uploadMod', function (err, resp, body) {
+						var req = request.post({url: "http://"+config.masterIP + ":" + config.masterPort + '/api/uploadMod',
+							headers: {
+								"x-access-token": config.masterAuthToken,
+							},
+						}, function (err, resp, body) {
 							if (err) {
 								console.log(new Error("Unable to contact master server! Please check your config.json."));
 							} else {
@@ -655,7 +694,7 @@ function instanceManagement() {
 							}
 						});
 						var form = req.form();
-						form.append('file', fs.createReadStream("./instances/"+instance+"/mods/"+mod));
+						form.append('file', fs.createReadStream(config.instanceDirectory+"/"+instance+"/mods/"+mod));
 					} else {
 						console.log("Not sending mod: " + mod + " to master because config.uploadModsToMaster is not enabled")
 					}
@@ -705,7 +744,7 @@ function instanceManagement() {
 						}
 					}
 					console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]);
-					needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: instanceconfig.unique,data: payload}, function (err, response, body) {
+					needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: instanceconfig.unique,data: payload}, needleOptionsWithTokenAuthHeader, function (err, response, body) {
 						// we did it, keep going
 					});
 				}
@@ -737,7 +776,7 @@ function instanceManagement() {
 					count: g[1],
 					instanceName: instance, // name of instance
 					instanceID: instanceconfig.unique, // a hash computed from the randomly generated rcon password
-				}, function (err, resp, body) {
+				}, needleOptionsWithTokenAuthHeader, function (err, resp, body) {
 					if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]);
 					if(config.logItemTransfers){
 						if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`);
@@ -794,12 +833,10 @@ function instanceManagement() {
 			// request our items, one item at a time
 			for (let i = 0; i < Object.keys(preparedPackage).length; i++) {
 				// console.log(preparedPackage[Object.keys(preparedPackage)[i]]);
-				needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], function (err, response, body) {
+				needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) {
 					if (response && response.body && typeof response.body == "object") {
 						// buffer confirmed orders
-						confirmedOrders[confirmedOrders.length] = {
-							[response.body.name]: response.body.count,
-						}
+						confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count}
 						if(config.logItemTransfers){
 							console.log(`Imported ${response.body.count} ${response.body.name} from master`);
 						}
@@ -808,11 +845,21 @@ function instanceManagement() {
 			}
 			// if we got some confirmed orders
 			// console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders));
-			sadas = JSON.stringify(confirmedOrders);
-			confirmedOrders = [];
-			// send our RCON command with whatever we got
-			messageInterface("/silent-command remote.call('clusterio', 'importMany', '" + sadas + "')");
-			sadas = null;
+			//if (!(confirmedOrders.length>0)){return;}
+			let cmd="local t={";
+			for(let i=0;i<confirmedOrders.length;i++)
+			{
+			    cmd+='["'+confirmedOrders[i].name+'"]='+confirmedOrders[i].count+',';
+			    if(cmd.length>320) // Factorio max packet size is 508
+			    {
+			        messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
+			        cmd="local t={";
+			    }
+			}
+			if (!(cmd==="local t={")){
+				messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
+			}
+			confirmedOrders=[];
 		}
 	}, 1000);
 	// COMBINATOR SIGNALS ---------------------------------------------------------
@@ -826,22 +873,37 @@ function instanceManagement() {
 				if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver
 				try {
 					var inventory = JSON.parse(response.body);
+					var inventoryFrame = {};
+					for (let i = 0; i < inventory.length; i++) {
+						inventoryFrame[inventory[i].name] = Number(inventory[i].count);
+						if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){
+							inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice
+						}
+					}
+					inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000);
+					// console.log("RCONing inventory! " + JSON.stringify(inventoryFrame));
+					let first = true;
+					let cmd="local s={";
+					for (let key in inventoryFrame)
+					{
+						cmd+='["'+key+'"]='+inventoryFrame[key]+",";
+						if(first && cmd.length>300 || !first && cmd.length > 320) // Factorio max packet size is 508
+						{
+					       		messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
+					       		cmd="local s={";
+					       		first = false;
+						}
+					}
+					if (!(cmd==="local s={")){
+						messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
+					}
+					messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')");
 				} catch (e){
 					console.log(e);
 				}
-				var inventoryFrame = {};
-				for (let i = 0; i < inventory.length; i++) {
-					inventoryFrame[inventory[i].name] = Number(inventory[i].count);
-					if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){
-						inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice
-					}
-				}
-				inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000);
-				// console.log("RCONing inventory! " + JSON.stringify(inventoryFrame));
-				messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')");
 			}
 		});
-	}, 1000);
+	}, 1550);
 	// Make sure world has its worldID
 	setTimeout(function(){
 		messageInterface("/silent-command remote.call('clusterio','setWorldID',"+instanceconfig.unique+")")
@@ -917,7 +979,7 @@ function hashMods(instanceName, callback) {
 	}
 	let hashedMods = [];
 	/*let mods = fs.readdirSync("./sharedMods/")*/
-	let instanceMods = fs.readdirSync("./instances/"+instanceName+"/mods/");
+	let instanceMods = fs.readdirSync(config.instanceDirectory+"/"+instanceName+"/mods/");
 	if(instanceMods.length == 0){
 		callback({}); // there are no mods installed, return an empty object
 	}
@@ -927,7 +989,7 @@ function hashMods(instanceName, callback) {
 		}
 	}
 	for(let i=0; i<instanceMods.length; i++){
-		let path = "./instances/"+instanceName+"/mods/"+instanceMods[i];
+		let path = config.instanceDirectory+"/"+instanceName+"/mods/"+instanceMods[i];
 		let name = instanceMods[i];
 		let options = {
 			files:path,

config.json.dist

@@ -1,27 +1,38 @@
 {
 	"__comment": "Where factorio folder. Should be in the same folder as this config file.",
 	"factorioDirectory": "factorio",
+	"instanceDirectory": "instances",
+	"databaseDirectory": "database",
 	
 	"__comment1": "IP players should connect to to join your servers, should be portforwarded and stuff",
 	"publicIP": "localhost",
 	
 	"__comment2": "IP the master web server runs at, can be ignored on the master system",
 	"masterIP": "localhost",
+	
+	"__commentMasterPort": "HTTP port for the master to listen to, set to 0 or an empty string to not listen for HTTP requests",
 	"masterPort": 8080,
+	
+	"__commentsslPort": "HTTPS port for the master to listen to. Set to 0 or an empty string to not use HTTPS.",
+	"sslPort": 443,
+	"sslCert": "database/certificates/cert.crt",
+	"sslPrivKey": "database/certificates/cert.key",
+	
+	"__commentMasterAuthToken": "Authentication token to connect to the master server. Find it in secret-api-token.txt on the master.",
 	"masterAuthToken": "",
 	
-	"__commentAuthSecret": "This is a long random string generated by keyboard mashing. Please mash you own string for security.",
-	"masterAuthSecret": "anfeåpvunuennuusnvzunsrevunåzeufnewnufasiojfåqwoiuajknwkjfdabewfkjadfnewåfundjkcvnewmfqewrmn123",
+	"__commentAuthSecret": "This is a long random string used for generating authentication tokens. Do not share. Regenerates when secret-api-token.txt is deleted.",
+	"masterAuthSecret": "",
 	
-	"__comment3": "Your factorio.com username for factorio MP services, also username will be admin",
+	"__comment3": "Your factorio.com username for factorio MP services, also username will be admin on new instances",
 	"username": "",
 	"__comment4": "Token from your factorio.com profile page. Required for MP and mod manager services.",
 	"token": "",
-	"factorio_version": "0.16.41",
 	"__comment7": "Used for communication with the mod portal, not needed if you manage mods by hand",
+	"factorio_version": "0.16.41",
 	
 	"__comment5": "General user settings",
-	"verify_user_identity": false,
+	"verify_user_identity": true,
 	"auto_pause": false,
 	"allow_commands": "admins-only",
 	"game_password": "",
@@ -37,5 +48,8 @@
 	},
 	"logItemTransfers": false,
 	"uploadModsToMaster": true,
-	"allowRemoteCommandExecution": false
+	"msBetweenCommands": 10,
+	"allowRemoteCommandExecution": true,
+	"enableCrossServerShout": true,
+	"mirrorAllChat": false
 }

lib/authenticate.js

@@ -0,0 +1,17 @@
+const jwt = require('jsonwebtoken');
+
+// const config = require("./../config");
+
+module.exports = function(config){
+	return {
+		middleware: function (req, res, next) {
+			var token = req.headers['x-access-token'];
+			if(!token) return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path});
+			
+			jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
+				if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path});
+				next();
+			});
+		},
+	}
+};

lib/getFactorioLocale.js

@@ -6,6 +6,8 @@ Module for grabbing factorio locale from a local factorio install
 const fs = require("fs");
 const ini = require("ini");
 
+const cache = {};
+
 /**
 Gets raw factorio locale from the .ini file.
 Not available outside of module through exports.
@@ -32,14 +34,20 @@ Gets factorios locale as an object, does not respect mods.
 */
 module.exports.asObject = function getLocaleAsObject(factorioDirectory, languageCode, callback){
 	if(typeof factorioDirectory != "string" || typeof languageCode != "string" || typeof callback != "function"){
-		throw "Error: wrong parameters provided";
+		throw new Error("Error: wrong parameters provided");
+	}
+	
+	if(!cache[languageCode]){
+		getLocale(factorioDirectory, languageCode, (err, rawLocale) => {
+			if(err){
+				callback(err);
+			} else {
+				let processedLocale = ini.parse(rawLocale);
+				cache[languageCode] = processedLocale;
+				callback(undefined, processedLocale);
+			}
+		});
+	} else {
+		callback(undefined, cache[languageCode]);
 	}
-	getLocale(factorioDirectory, languageCode, (err, rawLocale) => {
-		if(err){
-			callback(err);
-		} else {
-			let processedLocale = ini.parse(rawLocale)
-			callback(undefined, processedLocale);
-		}
-	});
 }

lib/getInstances.js

@@ -1,14 +1,17 @@
 const fileOps = require("_app/fileOps");
+// const config = require("./../config");
 
-module.exports = {
-	sync: function(){
-		let instances;
-		try {
-			instances = fileOps.getDirectoriesSync("./instances/");
-		} catch (e) {
-			// there are no instances created yet, probably missing the folder
-			instances = [];
-		}
-		return instances;
+module.exports = function(config) {
+	return {
+		sync: function(){
+			let instances;
+			try {
+				instances = fileOps.getDirectoriesSync(config.instanceDirectory);
+			} catch (e) {
+				// there are no instances created yet, probably missing the folder
+				instances = [];
+			}
+			return instances;
+		},
 	}
 }

lib/manager/modManager.js

@@ -11,11 +11,11 @@ const fs = require("fs");
 const https = require('follow-redirects').https;
 const factorioAPI = require("node-factorio-api");
 const sLog = require("single-line-log").stdout;
+const query = require('cli-interact').getYesNo;
 
 const fileOps = require("_app/fileOps");
-const config = require("./../../config");
+// const config = require("./../../config");
 
-const instances = require("./../getInstances.js").sync();
 
 function ellipse(str, max){
    return str.length > (max - 3) ? str.substring(0,max-3) + '...' : str;
@@ -23,209 +23,249 @@ function ellipse(str, max){
 // https://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
 function formatBytes(a,b){if(0==a)return"0 Bytes";var c=1024,d=b||2,e=["Bytes","KB","MB","GB","TB","PB","EB","ZB","YB"],f=Math.floor(Math.log(a)/Math.log(c));return parseFloat((a/Math.pow(c,f)).toFixed(d))+" "+e[f]};
 
-/**
-Lists mods for an instance as a table. Warning, handles printing internally.
-@param {string} instance - Name of the instance or "shared"
 
-@example
-var modManager = require("modManager");
-modManager.listMods("testInstance");
-// console.logs table of mods installed on instance
-modManager.listMods("shared");
-// console.logs mods located in /sharedMods as a nice table
-*/
-module.exports.listMods = function(instance){
-	if(instance && instances.includes(instance)){
-		factorioAPI.init(false, 'instances/'+instance+'/instanceMods', 'instances/'+instance+'/saves', config.factorio_version);
-		factorioAPI.authenticate({
-			username: config.username,
-			token: config.token,
-			require_ownership: false
-		}).then(token => {
-			console.log("* Currently installed:");
-			factorioAPI.readModZips().then(list => {
-				console.log(asTable(list.map(mod => {
-					return {
-						name:(mod.title || mod.name),
-						version:mod.version,
-						date:mod.date,
-						factorio_ver:mod.factorio_version,
-						author:mod.author,
-					}
-				})));
-			});
-			// Had issues earlier due to bug in node-factorio-api, see https://github.com/Danacus/node-factorio-api/issues/2
-			// bug is resolved, thanks to @Danacus and @brotherpot
-			factorioAPI.getModsFromSaves().then(list => {
-				console.log("* Mods used in save:");
-				console.log(asTable(list[0].mods))
-			});
-		}).catch(reason => {
-			console.log("Failed to authenticate to factorio.com! Please check your connection and the username+token used for authentication in config.json");
-		});
-	} else {
-		// list shared mods in /sharedMods/
-		factorioAPI.init(false, 'sharedMods/', '', config.factorio_version);
-		factorioAPI.authenticate({
-			username: config.username,
-			token: config.token,
-			require_ownership: false
-		}).then(() => {
-			console.log("\r\n")
-			console.log("* Mods shared by all instances:")
-			factorioAPI.getInstalledMods().then(mods => {
-				console.log(asTable(mods));
-			});
-		});
-	}
+module.exports = function(config){
+	if(!config) config = require("./../../config");
+	const instances = require("./../getInstances.js")(config).sync();
 	
-	// get JSON data about GH releases
-	/*let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
-	let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-	let name = JSON.parse(res.getBody())[0].assets[0].name;
-	console.log(JSON.parse(res.getBody()))
-	*/
-}
-/**
-Searches mods.factorio.com for mods matching a search term and prints
-the results to console as a neat looking table
+	return {
+		/**
+		Lists mods for an instance as a table. Warning, handles printing internally.
+		@param {string} instance - Name of the instance or "shared"
 
-@param {string} searchTerm What to search for
-*/
-module.exports.findMods = function(searchTerm){
-	factorioAPI.init(false);
-	factorioAPI.authenticate({
-		username: config.username,
-		token: config.token,
-		require_ownership: false
-	}).then(token => {
-		factorioAPI.searchMods(searchTerm).then(mods => {
-			console.log(asTable(mods.reverse().map(mod => {
-				return {
-					title:ellipse(mod.title, 50),
-					name:ellipse(mod.name, 30),
-					downloads:mod.downloads_count,
-					updated: moment(mod.updated_at).format("DD.MM.YY"),
-					latest_version: mod.latest_release.version,
-					owner:mod.owner,
-					//license: mod.license_name,
-				}
-			})))
-		})
-	});
-}
-
-module.exports.addMod = function(term, instance){
-	if(instance == "shared"){ // modDirectory has to end in /
-		var modDirectory = "sharedMods/";
-	} else {
-		var modDirectory = "instances/"+instance+"/instanceMods/";
-	}
-	if(term && term.length > 0){
-		if(term.includes("mods.factorio.com/mods/")){ // https://mods.factorio.com/mods/Danielv123/ChooChoo
-			// treat it as a link download
-			factorioAPI.init(false, modDirectory, "", config.factorio_version);
-			factorioAPI.authenticate({
-				username: config.username,
-				token: config.token,
-				require_ownership: false
-			}).then(() => {
-				factorioAPI.searchMods(term.split("/")[term.split("/").length-1]).then(body => {
-					console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
-					factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
-						console.log("Mod downloaded");
+		@example
+		var modManager = require("modManager");
+		modManager.listMods("testInstance");
+		// console.logs table of mods installed on instance
+		modManager.listMods("shared");
+		// console.logs mods located in /sharedMods as a nice table
+		*/
+		listMods: function(instance){
+			return new Promise((resolve, reject) => {
+				if(instance && instances.includes(instance)){
+					factorioAPI.init(false, config.instanceDirectory+'/'+instance+'/instanceMods', config.instanceDirectory+'/'+instance+'/saves', config.factorio_version);
+					factorioAPI.authenticate({
+						username: config.username,
+						token: config.token,
+						require_ownership: false
+					}).then(token => {
+						console.log("* Currently installed:");
+						factorioAPI.readModZips().then(list => {
+							console.log(asTable(list.map(mod => {
+								return {
+									name:(mod.title || mod.name),
+									version:mod.version,
+									date:mod.date,
+									factorio_ver:mod.factorio_version,
+									author:mod.author,
+								}
+							})));
+						});
+						// Had issues earlier due to bug in node-factorio-api, see https://github.com/Danacus/node-factorio-api/issues/2
+						// bug is resolved, thanks to @Danacus and @brotherpot
+						factorioAPI.getModsFromSaves().then(list => {
+							console.log("* Mods used in save:");
+							console.log(asTable(list[0].mods))
+						});
+					}).catch(reason => {
+						console.log("Failed to authenticate to factorio.com! Please check your connection and the username+token used for authentication in config.json");
+						resolve();
 					});
+				} else {
+					// list shared mods in /sharedMods/
+					factorioAPI.init(false, 'sharedMods/', '', config.factorio_version);
+					factorioAPI.authenticate({
+						username: config.username,
+						token: config.token,
+						require_ownership: false
+					}).then(() => {
+						console.log("\r\n")
+						console.log("* Mods shared by all instances:")
+						factorioAPI.getInstalledMods().then(mods => {
+							console.log(asTable(mods));
+							resolve();
+						});
+					}).catch(e => {
+						reject(new Error(e));
+					});
+				}
+				
+				// get JSON data about GH releases
+				/*let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+				let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+				let name = JSON.parse(res.getBody())[0].assets[0].name;
+				console.log(JSON.parse(res.getBody()))
+				*/
+			});
+		},
+		/**
+		Searches mods.factorio.com for mods matching a search term and prints
+		the results to console as a neat looking table
+
+		@param {string} searchTerm What to search for
+		*/
+		findMods: function(searchTerm){
+			return new Promise((resolve, reject) => {
+				factorioAPI.init(false);
+				factorioAPI.authenticate({
+					username: config.username,
+					token: config.token,
+					require_ownership: false
+				}).then(token => {
+					factorioAPI.searchMods(searchTerm).then(mods => {
+						let result = asTable(mods.reverse().map(mod => {
+							return {
+								title:ellipse(mod.title, 50),
+								name:ellipse(mod.name, 30),
+								downloads:mod.downloads_count,
+								updated: moment(mod.updated_at).format("DD.MM.YY"),
+								latest_version: mod.latest_release.version,
+								owner:mod.owner,
+								//license: mod.license_name,
+							}
+						}));
+						resolve(result);
+						// console.log(result);
+					});
+				}).catch(e => {
+					if(e.contains("Insufficient information")) console.log("We are missing some parameters required for logging in! Make sure your factorio username and token are specified in config.json.");
+					reject(new Error(e));
 				});
 			});
-		} else if(term.includes("github.com")){ // https://github.com/Danielv123/factorioClusterioMod
-			// treat it as a github releases download
-			let apiURL = "https://api.github.com/repos/" + term.split("/")[3] + "/" + term.split("/")[4] + "/releases"; // https://api.github.com/repos/Danielv123/factorioClusterioMod/releases
-			
-			let res = syncRequest('GET', apiURL, {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
-			let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+		},
+		addMod: function(term, instance){
+			return new Promise((resolve, reject) => {
+				if(instance == "shared"){ // modDirectory has to end in /
+					var modDirectory = "sharedMods/";
+				} else {
+				var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
+				}
+				if(term && term.length > 0){
+					if(term.includes("mods.factorio.com/mods/")){ // https://mods.factorio.com/mods/Danielv123/ChooChoo
+						// treat it as a link download
+						factorioAPI.init(false, modDirectory, "", config.factorio_version);
+						factorioAPI.authenticate({
+							username: config.username,
+							token: config.token,
+							require_ownership: false
+						}).then(() => {
+							factorioAPI.searchMods(term.split("/")[term.split("/").length-1]).then(body => {
+								console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
+								factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
+									console.log("Mod downloaded");
+									resolve();
+								});
+							});
+						});
+					} else if(term.includes("github.com")){ // https://github.com/Danielv123/factorioClusterioMod
+						// treat it as a github releases download
+						let apiURL = "https://api.github.com/repos/" + term.split("/")[3] + "/" + term.split("/")[4] + "/releases"; // https://api.github.com/repos/Danielv123/factorioClusterioMod/releases
+						
+						let res = syncRequest('GET', apiURL, {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+						let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+						let name = JSON.parse(res.getBody())[0].assets[0].name;
+						if(url) {
+							console.log("Downloading " + url);
+							let file = fs.createWriteStream(modDirectory + name);
+							let bytesDownloaded = 0;
+							let downloadStartTime = Date.now();
+							https.get(url, function(response) {
+								let bytesToDownload = response.headers['content-length'];
+								response.pipe(file);
+								response.on("data", chunk => {
+									bytesDownloaded += chunk.length;
+									let downloadTimeElapsed = Date.now() - downloadStartTime;
+									let bytesPerSecond = bytesDownloaded / (downloadTimeElapsed/1000);
+									let timeToComplete = Math.floor((bytesToDownload - bytesDownloaded)/bytesPerSecond);
+									sLog("Downloading "+name+" "+(bytesDownloaded/bytesToDownload*100).toPrecision(3)+"% ("+formatBytes(bytesDownloaded)+" / "+formatBytes(bytesToDownload)+")\r\n"+
+									"about "+timeToComplete+" seconds remaining ("+formatBytes(bytesPerSecond)+"/s)");
+								}).on("end", x => {
+									sLog.clear();
+									console.log(" Downloaded in "+moment.utc(moment.duration(Date.now()-downloadStartTime).as('milliseconds')).format('mm:ss')); // because sLog.clear() doesn't seem to do the job of ending the line properly
+									resolve();
+								});
+							});
+						}
+					} else {
+						// treat it as a search and download the top result
+						factorioAPI.init(false, modDirectory, "", config.factorio_version);
+						factorioAPI.authenticate({
+							username: config.username,
+							token: config.token,
+							require_ownership: false
+						}).then(() => {
+							factorioAPI.searchMods(term).then(body => {
+								console.log(body)
+								console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
+								factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
+									console.log("Mod downloaded");
+									resolve();
+								});
+							});
+						});
+					}
+				} else {
+					reject(new Error("Invalid search term"));
+				}
+			});
+		},
+		removeMod: function removeMod(term, instance, force = false){
+			return new Promise((resolve, reject) => {
+				if(instance == "shared"){ // modDirectory has to end in /
+					var modDirectory = "sharedMods/";
+				} else {
+				var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
+				}
+				if(term && typeof term == "string" && term.length > 0){
+					let mods = fs.readdirSync(modDirectory);
+					let modsMatchingFilter = mods.filter(mod => mod.toLowerCase().includes(term.toLowerCase()));
+					if(modsMatchingFilter.length < 1){
+						console.log("No mods found matching search");
+					} else if(modsMatchingFilter.length == 1){
+						console.log("Deleting "+modsMatchingFilter[0]);
+						fs.unlinkSync(modDirectory+modsMatchingFilter[0]);
+						console.log("Deleted.");
+					} else if(modsMatchingFilter.length > 1){
+						console.log("Found multiple mods");
+						modsMatchingFilter.forEach(mod => console.log(mod));
+						if(query("Delete all mods matching filter?")){
+							let modsToDelete = modsMatchingFilter.length;
+							for(let i = 0; i < modsToDelete; i++){
+								fs.unlinkSync(modDirectory+modsMatchingFilter[i]);
+								console.log("Deleted "+modsMatchingFilter[i]);
+							}
+						} else console.log("Did not delete mods.")
+					}
+					resolve();
+				} else {
+					reject(new Error("Invalid search term"));
+				}
+			});
+		},
+
+		// 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases'
+		/**
+		Download mod from github repo by looking for the first binary download in
+		the latest release, saving it to /sharedMods
+
+		@param {string} url a github repo  URL, for example https://api.github.com/repos/Danielv123/factorioClusterioMod/releases. Must link to /releases
+		*/
+		downloadFromGitURL: function(url){
+			console.log("Downloading mod from "+url);
+			// get JSON data about releases
+			let res = syncRequest('GET', url, {"headers":{"User-Agent":"Please don't require setting a user agent"}});
+			let downloadUrl = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
 			let name = JSON.parse(res.getBody())[0].assets[0].name;
-			if(url) {
-				console.log("Downloading " + url);
-				let file = fs.createWriteStream(modDirectory + name);
-				let bytesDownloaded = 0;
-				let downloadStartTime = Date.now();
-				https.get(url, function(response) {
-					let bytesToDownload = response.headers['content-length'];
+			console.log(JSON.parse(res.getBody()))
+			if(downloadUrl) {
+				console.log(downloadUrl);
+				let file = fs.createWriteStream("sharedMods/"+name);
+				https.get(downloadUrl, function(response) {
 					response.pipe(file);
-					response.on("data", chunk => {
-						bytesDownloaded += chunk.length;
-						let downloadTimeElapsed = Date.now() - downloadStartTime;
-						let bytesPerSecond = bytesDownloaded / (downloadTimeElapsed/1000);
-						let timeToComplete = Math.floor((bytesToDownload - bytesDownloaded)/bytesPerSecond);
-						sLog("Downloading "+name+" "+(bytesDownloaded/bytesToDownload*100).toPrecision(3)+"% ("+formatBytes(bytesDownloaded)+" / "+formatBytes(bytesToDownload)+")\r\n"+
-						"about "+timeToComplete+" seconds remaining ("+formatBytes(bytesPerSecond)+"/s)");
-					}).on("end", x => {
-						sLog.clear();
-						console.log(" Downloaded in "+moment.utc(moment.duration(Date.now()-downloadStartTime).as('milliseconds')).format('mm:ss')); // because sLog.clear() doesn't seem to do the job of ending the line properly
-					});
+					console.log("Downloaded "+name);
 				});
 			}
-		} else {
-			// treat it as a search and download the top result
-			factorioAPI.init(false, modDirectory, "", config.factorio_version);
-			factorioAPI.authenticate({
-				username: config.username,
-				token: config.token,
-				require_ownership: false
-			}).then(() => {
-				factorioAPI.searchMods(term).then(body => {
-					console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
-					factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
-						console.log("Mod downloaded");
-					});
-				});
-			});
-		}
-	}
-}
-module.exports.removeMod = function removeMod(term, instance, force = false){
-	if(instance == "shared"){ // modDirectory has to end in /
-		var modDirectory = "sharedMods/";
-	} else {
-		var modDirectory = "instances/"+instance+"/instanceMods/";
-	}
-	if(term && typeof term == "string" && term.length > 0){
-		let mods = fs.readdirSync(modDirectory);
-		let modsMatchingFilter = mods.filter(mod => mod.toLowerCase().includes(term.toLowerCase()));
-		if(modsMatchingFilter.length < 1){
-			console.log("No mods found matching search");
-		} else if(modsMatchingFilter.length == 1){
-			console.log("Deleting "+modsMatchingFilter[0]);
-			fs.unlinkSync(modDirectory+modsMatchingFilter[0]);
-			console.log("Deleted.");
-		} else if(modsMatchingFilter.length > 1){
-			console.log("Found multiple mods, not doing anything");
-			modsMatchingFilter.forEach(mod => console.log(mod));
-		}
-	}
-}
-
-// 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases'
-/**
-Download mod from github repo by looking for the first binary download in
-the latest release, saving it to /sharedMods
-
-@param {string} url a github repo  URL, for example https://api.github.com/repos/Danielv123/factorioClusterioMod/releases. Must link to /releases
-*/
-module.exports.downloadFromGitURL = function(url){
-	console.log("Downloading mod from "+url);
-	// get JSON data about releases
-	let res = syncRequest('GET', url, {"headers":{"User-Agent":"Please don't require setting a user agent"}});
-	let downloadUrl = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-	let name = JSON.parse(res.getBody())[0].assets[0].name;
-	console.log(JSON.parse(res.getBody()))
-	if(downloadUrl) {
-		console.log(downloadUrl);
-		let file = fs.createWriteStream("sharedMods/"+name);
-		https.get(downloadUrl, function(response) {
-			response.pipe(file);
-			console.log("Downloaded "+name);
-		});
+		},
 	}
 }
 /*

lib/npmPostinstall.js

@@ -5,7 +5,7 @@ var fs = require('fs');
 
 var modManager = require("./manager/modManager.js");
 
-modManager.addMod("clusterio", "shared");
+// modManager.addMod("clusterio", "shared");
 
 function makeDirectory(path){
 	return new Promise((resolve, reject) => {

master.js

@@ -20,18 +20,15 @@ process.on('unhandledRejection', r => console.log(r));
 // configgy stuff
 debug = false;
 
-// constants
-const masterModFolder = "./database/masterMods/";
-var config = require('./config');
+// reduces loadtime from 1700 ms to 1200 ms on my i3 + SSD
+// tested by logging Date.now() here and when the webserver started listening
+require('cache-require-paths');
 
-// homebrew modules
-const getFactorioLocale = require("./lib/getFactorioLocale");
-const stringUtils = require("./lib/stringUtils");
+// argument parsing
+const args = require('minimist')(process.argv.slice(2));
 
 // Library for create folder recursively if it does not exist
 const mkdirp = require("mkdirp");
-mkdirp.sync("./database");
-mkdirp.sync(masterModFolder);
 const averagedTimeSeries = require("averaged-timeseries");
 const deepmerge = require("deepmerge");
 const path = require("path");
@@ -39,8 +36,44 @@ const fs = require("fs");
 const nedb = require("nedb");
 const jwt = require('jsonwebtoken');
 const bcrypt = require('bcryptjs');
+const crypto = require('crypto');
+const base64url = require('base64url');
+
+// constants
+const config = require(args.config || './config');
+config.databaseDirectory = args.databaseDirectory || config.databaseDirectory || "./database";
+const masterModFolder = path.join(config.databaseDirectory, "/masterMods/");
+mkdirp.sync(config.databaseDirectory);
+mkdirp.sync(masterModFolder);
+
+// homebrew modules
+const getFactorioLocale = require("./lib/getFactorioLocale");
+const stringUtils = require("./lib/stringUtils");
+const fileOps = require("_app/fileOps");
+
+// homemade express middleware for token auth
+const authenticate = require("./lib/authenticate")(config);
+
+/** Sync */
+function randomStringAsBase64Url(size) {
+  return base64url(crypto.randomBytes(size));
+}
+if (!fs.existsSync("secret-api-token.txt")) {
+	config.masterAuthSecret = randomStringAsBase64Url(256);
+    fs.writeFileSync("config.json",JSON.stringify(config, null, 4));
+	fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
+		expiresIn: 86400*365 // expires in 1 year
+	}));
+	console.log("Generated new master authentication private key!");
+	process.exit(0);
+}
+// write an auth token to file
+fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
+	expiresIn: 86400*365 // expires in 1 year
+}));
 
 const express = require("express");
+const compression = require('compression');
 const ejs = require("ejs");
 // Required for express post requests
 const bodyParser = require("body-parser");
@@ -48,18 +81,28 @@ const fileUpload = require('express-fileupload');
 var app = express();
 
 app.use(bodyParser.json());
-app.use(bodyParser.urlencoded({ extended: true }));
+app.use(bodyParser.urlencoded({
+	parameterLimit: 100000,
+	limit: '10mb',
+	extended: true
+}));
 app.use(fileUpload());
+app.use(compression());
 
 // dynamic HTML generations with EJS
 app.set('views', path.join(__dirname, 'static'));
 app.set('view engine', 'html');
 app.engine('html', ejs.renderFile);
 
-var routes = require("./routes.js");
-routes(app);
-
+// give ejs access to some interesting information
+app.use(function(req, res, next){
+	res.locals.res = res;
+	res.locals.req = req;
+	res.locals.masterPlugins = masterPlugins;
+	next();
+});
 
+require("./routes.js")(app);
 // Set folder to serve static content from (the website)
 app.use(express.static('static'));
 // mod downloads
@@ -88,18 +131,43 @@ const prometheusConnectedInstancesCounter = new Prometheus.Gauge({
 const prometheusWsUsageCounter = new Prometheus.Counter({
 	name: prometheusPrefix+'websocket_usage_counter',
 	help: 'Websocket traffic',
-	labelNames: ["connectionType"],
+	labelNames: ["connectionType", "instanceID"],
 });
 const prometheusProductionGauge = new Prometheus.Gauge({
 	name: prometheusPrefix+'production_gauge',
 	help: 'Items produced by instance',
 	labelNames: ["instanceID", "itemName"],
 });
+const prometheusExportGauge = new Prometheus.Gauge({
+	name: prometheusPrefix+'export_gauge',
+	help: 'Items exported by instance',
+	labelNames: ["instanceID", "itemName"],
+});
+const prometheusImportGauge = new Prometheus.Gauge({
+	name: prometheusPrefix+'import_gauge',
+	help: 'Items imported by instance',
+	labelNames: ["instanceID", "itemName"],
+});
+const prometheusDoleFactorGauge = new Prometheus.Gauge({
+	name: prometheusPrefix+'dole_factor_gauge',
+	help: 'The current dole division factor for this item',
+	labelNames: ["itemName"],
+});
 const prometheusPlayerCountGauge = new Prometheus.Gauge({
 	name: prometheusPrefix+'player_count_gauge',
 	help: 'Amount of players connected to this cluster',
 	labelNames: ["instanceID", "instanceName"],
 });
+const prometheusUPSGauge = new Prometheus.Gauge({
+	name: prometheusPrefix+'UPS_gauge',
+	help: 'UPS of the current server',
+	labelNames: ["instanceID", "instanceName"],
+});
+const prometheusTickGauge = new Prometheus.Gauge({
+	name: prometheusPrefix+'tick_gauge',
+	help: 'Servers current gametick',
+	labelNames: ["instanceID", "instanceName"],
+});
 const prometheusMasterInventoryGauge = new Prometheus.Gauge({
 	name: prometheusPrefix+'master_inventory_gauge',
 	help: 'Amount of items stored on master',
@@ -124,10 +192,17 @@ app.get('/metrics', (req, res) => {
 	res.set('Content-Type', Prometheus.register.contentType);
 	
 	/// gather some static metrics
-	// playercount
-	for(let instanceID in slaves){try{
-		prometheusPlayerCountGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].playerCount) || 0);
-	}catch(e){}}
+	for(let instanceID in slaves){
+		// playercount
+		try{
+			prometheusPlayerCountGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].playerCount) || 0);
+		}catch(e){}
+		// UPS
+		try{
+			prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.UPS) || 60);
+			if(slaves[instanceID].meta.tick && typeof slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.tick) || 0);
+		}catch(e){}
+	}
 	// inventory
 	for(let key in db.items){
 		if(typeof db.items[key] == "number" || typeof db.items[key] == "string"){
@@ -143,12 +218,16 @@ app.get('/metrics', (req, res) => {
 var Datastore = require('nedb');
 db = {};
 var LinvoDB = require("linvodb3");
-LinvoDB.dbPath = "./database/linvodb/";
+
+// TODO: This breaks. databaseDirectory is either "database" or "./database". `LinvoDB.dbPath = "./database/linvodb/";` is the old working version.
+LinvoDB.dbPath = path.resolve(config.databaseDirectory, "linvodb");
+// LinvoDB.dbPath = "./database/linvodb/";
+
 // database for items in system
 // db.items = new Datastore({ filename: 'database/items.db', autoload: true });
 
 // in memory database for combinator signals
-db.signals = new Datastore({ filename: 'database/signals.db', autoload: true, inMemoryOnly: true});
+db.signals = new Datastore({ filename: path.resolve(config.databaseDirectory, '/signals.db'), autoload: true, inMemoryOnly: true});
 db.signals.ensureIndex({ fieldName: 'time', expireAfterSeconds: 3600 }, function (err) {});
 
 // production chart database
@@ -157,16 +236,16 @@ db.flows = new LinvoDB("flows", {}, {});
 
 (function(){
 	try{
-		let x = fs.statSync("database/slaves.json");
-		console.log("loading slaves from database/slaves.json");
-		slaves = JSON.parse(fs.readFileSync("database/slaves.json"));
+		let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json"));
+		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
+		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json")));
 	} catch (e){
 		slaves = {};
 	}
 	try{
-		x = fs.statSync("database/items.json");
-		console.log("loading items from database/items.json");
-		db.items = JSON.parse(fs.readFileSync("database/items.json"));
+		x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
+		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
+		db.items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")));
 	} catch (e){
 		db.items = {};
 	}
@@ -200,23 +279,35 @@ db.items.removeItem = function(object) {
 }
 
 // store slaves and inventory in a .json full of JSON data
-process.on('SIGINT', function () {
+process.on('SIGINT', shutdown); // ctrl + c
+process.on('SIGHUP', shutdown); // terminal closed
+async function shutdown() {
 	console.log('Ctrl-C...');
+	let exitStartTime = Date.now();
 	// set insane limit to slave length, if its longer than this we are probably being ddosed or something
 	if(slaves && Object.keys(slaves).length < 2000000){
 		console.log("saving to slaves.json");
-		fs.writeFileSync("database/slaves.json", JSON.stringify(slaves));
+		fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
 	} else if(slaves) {
 		console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
 	}
 	if(db.items && Object.keys(db.items).length < 50000){
 		console.log("saving to items.json");
-		fs.writeFileSync("database/items.json", JSON.stringify(db.items));
+		fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
 	} else if(slaves) {
 		console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
 	}
-	process.exit(2);
-});
+	for(let i in masterPlugins){
+		let plugin = masterPlugins[i];
+		if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
+			let startTime = Date.now();
+			await plugin.main.onExit();
+			console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
+		}
+	}
+	console.log("Exited in "+(Date.now()-exitStartTime)+"ms");
+	process.exit(0);
+}
 
 /**
 POST World ID management. Slaves post here to tell the server they exist
@@ -224,7 +315,7 @@ POST World ID management. Slaves post here to tell the server they exist
 @instance
 @alias /api/getID
 */
-app.post("/api/getID", function(req,res) {
+app.post("/api/getID", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
 	// request.body should be an object
@@ -253,7 +344,7 @@ POST Allows you to add metadata related to slaves for other tools, like owner da
 @instance
 @alias /api/editSlaveMeta
 */
-app.post("/api/editSlaveMeta", function(req,res) {
+app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
 	// request.body should be an object
@@ -283,6 +374,8 @@ POST Get metadata from a single slave. For whenever you find the data returned b
 @alias /api/getSlaveMeta
 */
 app.post("/api/getSlaveMeta", function (req, res) {
+	endpointHitCounter.labels(req.route.path).inc();
+	let reqStartTime = Date.now();
 	console.log("body", req.body);
     if(req.body && req.body.instanceID && req.body.password){
     	console.log("returning meta for ", req.body.instanceID);
@@ -291,22 +384,7 @@ app.post("/api/getSlaveMeta", function (req, res) {
     	res.status(400);
     	res.send('{"INVALID REQUEST":1}');
 	}
-});
-// TODO: Remove deprecated function, also update researchSync to not use it
-/**
-POST DEPRECATED Get metadata from all slaves. This is already covered by /api/slaves, please use that one instead.
-@memberof clusterioMaster
-@instance
-@alias /api/getSlavesMeta
-*/
-app.post("/api/getSlavesMeta", function (req, res) {
-    if(req.body.password){
-    	let metas = {};
-        res.send(JSON.stringify(Object.keys(slaves).map(key => metas[key] = slaves[key].meta)));
-    } else {
-        res.status(400);
-        res.send('{"INVALID REQUEST":1}');
-    }
+	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 // mod management
 // should handle uploading and checking if mods are uploaded
@@ -341,15 +419,12 @@ POST endpoint for uploading mods to the master server. Required for automatic mo
 @instance
 @alias /api/uploadMod
 */
-app.post("/api/uploadMod", function(req,res) {
+app.post("/api/uploadMod", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
-	if (!req.files) {
-        res.send('No files were uploaded.');
-        return;
-    } else {
+	if (req.files && req.files.file) {
 		console.log(req.files.file);
-		req.files.file.mv('./database/masterMods/'+req.files.file.name, function(err) {
+		req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err) {
 			if (err) {
 				res.status(500).send(err);
 			} else {
@@ -358,7 +433,10 @@ app.post("/api/uploadMod", function(req,res) {
 			}
 			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 		});
-	}
+	} else {
+		res.send('No files were uploaded.');
+		return;
+    }
 });
 /**
 GET endpoint for getting information about all our slaves
@@ -366,17 +444,25 @@ GET endpoint for getting information about all our slaves
 @instance
 @alias /api/slaves
 */
+let slaveCache = {
+	timestamp: Date.now(),
+};
 app.get("/api/slaves", function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-	let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
-	// filter out the rcon password because thats kindof not a safe thing to share
-	for(key in copyOfSlaves){
-		copyOfSlaves[key].rconPassword = "hidden";
+	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){
+		let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
+		// filter out the rcon password because thats kindof not a safe thing to share
+		for(key in copyOfSlaves){
+			copyOfSlaves[key].rconPassword = "hidden";
+		}
+		slaveCache.cache = copyOfSlaves;
+		slaveCache.timestamp = Date.now();
 	}
-	res.send(copyOfSlaves);
+	
+	res.send(slaveCache.cache);
 	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 var recievedItemStatisticsBySlaveID = {};
@@ -398,7 +484,7 @@ POST endpoint for storing items in master's inventory.
 @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name]
 @returns {string} status either "success" or "failure"
 */
-app.post("/api/place", function(req, res) {
+app.post("/api/place", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
 	res.header("Access-Control-Allow-Origin", "*");
@@ -441,11 +527,13 @@ app.post("/api/place", function(req, res) {
 			key:req.body.name,
 			value:req.body.count,
 		});
+		prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
 		// save items we get
 		db.items.addItem({
 			name:req.body.name,
 			count:req.body.count
 		});
+		
 		// Attempt confirming
 		res.send("success");
 	} else {
@@ -465,7 +553,7 @@ POST endpoint to remove items from DB when client orders items.
 @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
 */
 _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
-app.post("/api/remove", function(req, res) {
+app.post("/api/remove", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
 	const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
@@ -517,6 +605,9 @@ app.post("/api/remove", function(req, res) {
 				//console.log(sentItemStatistics.data)
 				sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
 			}
+			
+			prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+			prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
 			res.send({count: object.count, name: object.name});
 			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 		} else {
@@ -540,7 +631,7 @@ Gives no response
 @param {object} circuitFrame
 @param {number} circuitFrame.time
 */
-app.post("/api/setSignal", function(req,res) {
+app.post("/api/setSignal", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
 	if(typeof req.body == "object" && req.body.time){
@@ -625,7 +716,7 @@ course the timeSeries data.
 @param {object} JSON {timestamp: Date.now(), instanceID: "string", data: {"item": number}}
 @returns {string} failure
 */
-app.post("/api/logStats", function(req,res) {
+app.post("/api/logStats", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let reqStartTime = Date.now();
 	if(typeof req.body == "object" && req.body.instanceID && req.body.timestamp && req.body.data) {
@@ -707,7 +798,7 @@ app.post("/api/getStats", function(req,res) {
 		} else if(req.body.statistic == "remove"){
 			console.log(`sending remove data for instanceID ${req.body.instanceID} ${req.body.itemName}`);
 			// Gather data
-			console.log(sentItemStatisticsBySlaveID)
+			// console.log(sentItemStatisticsBySlaveID)
 			let itemStats = sentItemStatisticsBySlaveID[req.body.instanceID];
 			console.log(itemStats)
 			if(typeof itemStats == "object"){
@@ -790,14 +881,12 @@ Requires x-access-token header to be set. Find you api token in secret-api-token
 @memberof clusterioMaster
 @instance
 @alias api/runCommand
-@param {object} JSON {instanceID:19412312, command:"/c game.print('hello')"}
+@param {object} JSON {instanceID:19412312, broadcast:false, command:"/c game.print('hello')"}
 @returns {object} Status {auth: bool, message: "Informative error", data:{}}
 */
-// write an auth token to file
-fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
-	expiresIn: 86400*365 // expires in 1 year
-}));
 app.post("/api/runCommand", (req,res) => {
+	endpointHitCounter.labels(req.route.path).inc();
+	let reqStartTime = Date.now();
 	var token = req.headers['x-access-token'];
 	if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' });
 	
@@ -805,19 +894,38 @@ app.post("/api/runCommand", (req,res) => {
 		if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' });
 		
 		// validate request.body
-		let body = req.body;
-		if(body.instanceID
-		&& Object.keys(wsSlaves).includes(body.instanceID)
-		&& body.command
-		&& typeof body.command == "string"
-		&& body.command[0] == "/"){
-			// execute command
-			wsSlaves[body.instanceID].runCommand(body.command, data => {
-				res.status(200).send({auth: true, message: "success", data});
-			});
-		} else {
-			res.status(400).send({auth: true, message: "Error: invalid request.body"});
-		}
+        let body = req.body;
+
+        if (
+            body.broadcast
+            && body.command
+            && typeof body.command == "string"
+            && body.command[0] == "/")
+        {
+        	let instanceResponses = [];
+            for (let instanceID in wsSlaves) {
+                // skip loop if the property is from prototype
+                if (!wsSlaves.hasOwnProperty(instanceID)) continue;
+                wsSlaves[instanceID].runCommand(body.command);
+            }
+            res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"});
+			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
+        } else if (
+            body.instanceID
+            && wsSlaves[body.instanceID]
+            && body.command
+            && typeof body.command == "string"
+            && body.command[0] == "/")
+        {
+            // execute command
+            wsSlaves[body.instanceID].runCommand(body.command, data => {
+                res.status(200).send({auth: true, message: "success", data});
+				httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
+            });
+        } else {
+            res.status(400).send({auth: true, message: "Error: invalid request.body"});
+			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
+        }
 	});
 });
 /**
@@ -836,10 +944,26 @@ app.get("/api/getFactorioLocale", function(req,res){
 		httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 	});
 });
-var server = require("http").Server(app);
-server.listen(config.masterPort || 8080, function () {
-	console.log("Listening on port %s...", server.address().port);
-});
+if(args.sslPort || config.sslPort){
+	try{
+		var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt'));
+		var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key'));
+
+		httpsServer = require("https").createServer({
+			key: privateKey,
+			cert: certificate
+		}, app).listen(args.sslPort || config.sslPort);
+	} catch(e){
+		console.error(e)
+	}
+}
+if(args.masterPort || config.masterPort){
+	server = require("http").Server(app);
+	server.listen(args.masterPort || config.masterPort || 8080, function () {
+		console.log("Listening on port %s...", server.address().port);
+	});
+}
+
 /* Websockets for remoteMap */
 var io = require("socket.io")(server);
 var slaveMappers = {};
@@ -850,14 +974,17 @@ class slaveMapper {
 		this.lastBeat = Date.now();
 		
 		this.socket.on("heartbeat", () => {
+			prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc();
 			// we aren't ready to die yet apparently
 			this.lastBeat = Date.now();
 		});
 		this.socket.on("sendChunk", function(data){
+			prometheusWsUsageCounter.labels('sendChunk', this.instanceID).inc();
 			mapRequesters[data.requesterID].socket.emit("displayChunk", data);
 		});
 		// slaveMapper sent us an entity update, process
 		this.socket.on("sendEntity", entity => {
+			prometheusWsUsageCounter.labels('sendEntity', this.instanceID).inc();
 			Object.keys(mapRequesters).forEach(requesterName => {
 				let requester = mapRequesters[requesterName];
 				
@@ -878,12 +1005,12 @@ class mapRequester {
 		this.lastBeat = Date.now();
 		
 		this.socket.on("heartbeat", () => {
-			prometheusWsUsageCounter.labels('heartbeat').inc();
+			prometheusWsUsageCounter.labels('heartbeat', "other").inc();
 			// we aren't ready to die yet apparently
 			this.lastBeat = Date.now();
 		});
 		this.socket.on("requestChunk", loc => {
-			prometheusWsUsageCounter.labels('requestChunk').inc();
+			prometheusWsUsageCounter.labels('requestChunk', "other").inc();
 			loc.requesterID = this.requesterID;
 			let instanceID = loc.instanceID || this.instanceID;
 			if(slaveMappers[instanceID] && typeof loc.x == "number" && typeof loc.y == "number"){
@@ -891,7 +1018,7 @@ class mapRequester {
 			}
 		});
 		this.socket.on("requestEntity", req => {
-			prometheusWsUsageCounter.labels('requestEntity').inc();
+			prometheusWsUsageCounter.labels('requestEntity', "other").inc();
 			req.requesterID = this.requesterID;
 			let instanceID = req.instanceID || this.instanceID;
 			if(slaveMappers[instanceID] && typeof req.x == "number" && typeof req.y == "number"){
@@ -899,7 +1026,7 @@ class mapRequester {
 			}
 		});
 		this.socket.on("placeEntity", req => {
-			prometheusWsUsageCounter.labels('placeEntity').inc();
+			prometheusWsUsageCounter.labels('placeEntity', "other").inc();
 			req.requesterID = this.requesterID;
 			let instanceID = req.instanceID || this.instanceID;
 			if(slaveMappers[instanceID]){
@@ -917,11 +1044,11 @@ class wsSlave {
 		this.lastBeat = Date.now();
 		
 		this.socket.on("heartbeat", () => {
-			prometheusWsUsageCounter.labels('heartbeat').inc();
+			prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc();
 			this.lastBeat = Date.now();
 		});
 		this.socket.on("combinatorSignal", circuitFrameWithMeta => {
-			prometheusWsUsageCounter.labels('combinatorSignal').inc();
+			prometheusWsUsageCounter.labels('combinatorSignal', this.instanceID).inc();
 			if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object"){
 				Object.keys(wsSlaves).forEach(instanceID => {
 					wsSlaves[instanceID].socket.emit("processCombinatorSignal", circuitFrameWithMeta);
@@ -931,16 +1058,39 @@ class wsSlave {
 		// handle command return values
 		this.commandsWaitingForReturn = {};
 		this.socket.on("runCommandReturnValue", resp => {
+			prometheusWsUsageCounter.labels('runCommandReturnValue', this.instanceID).inc();
 			if(resp.commandID && resp.body && this.commandsWaitingForReturn[resp.commandID] && this.commandsWaitingForReturn[resp.commandID].callback && typeof this.commandsWaitingForReturn[resp.commandID].callback == "function"){
 				this.commandsWaitingForReturn[resp.commandID].callback(resp.body);
 				delete this.commandsWaitingForReturn[resp.commandID];
 			}
 		});
+		
+		this.socket.on("gameChat", data => {
+			prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc();
+			if(typeof data === "object"){
+				data.timestamp = Date.now();
+				if(!global.wsChatRecievers) global.wsChatRecievers = [];
+				global.wsChatRecievers.forEach(socket => {
+					socket.emit("gameChat", data);
+				});
+			}
+		});
+		
+		this.socket.on("alert", alert => {
+			prometheusWsUsageCounter.labels('alert', this.instanceID).inc();
+			if(typeof alert === "object"){
+				alert.timestamp = Date.now();
+				if(!global.wsAlertRecievers) global.wsAlertRecievers = [];
+				global.wsAlertRecievers.forEach(socket => {
+					socket.emit("alert", alert);
+				});
+			}
+		});
 	}
 	runCommand(command, callback){
 		let commandID = Math.random().toString();
 		this.socket.emit("runCommand", {command, commandID});
-		this.commandsWaitingForReturn[commandID] = {callback, timestamp: Date.now()};
+		if(commandID) this.commandsWaitingForReturn[commandID] = {callback, timestamp: Date.now()};
 	}
 }
 io.on('connection', function (socket) {
@@ -963,13 +1113,13 @@ io.on('connection', function (socket) {
 	
 	/* initial processing for remoteMap */
 	socket.on('registerSlaveMapper', function (data) {
-		prometheusWsUsageCounter.labels('registerSlaveMapper').inc();
+		prometheusWsUsageCounter.labels('registerSlaveMapper', "other").inc();
 		slaveMappers[data.instanceID] = new slaveMapper(data.instanceID, socket);
 		console.log("remoteMap | SOCKET registered map provider for "+data.instanceID);
 	});
 	socket.on('registerMapRequester', function(data){
 		// data {instanceID:""}
-		prometheusWsUsageCounter.labels('registerMapRequester').inc();
+		prometheusWsUsageCounter.labels('registerMapRequester', "other").inc();
 		let requesterID = Math.random().toString();
 		mapRequesters[requesterID] = new mapRequester(requesterID, socket, data.instanceID);
 		socket.emit("mapRequesterReady", true);
@@ -978,13 +1128,85 @@ io.on('connection', function (socket) {
 	
 	/* Websockets for send and recieve combinators */
 	socket.on("registerSlave", function(data) {
-		prometheusWsUsageCounter.labels('registerSlave').inc();
+		prometheusWsUsageCounter.labels('registerSlave', "other").inc();
 		if(data && data.instanceID){
 			wsSlaves[data.instanceID] = new wsSlave(data.instanceID, socket);
 			console.log("SOCKET | Created new wsSlave: "+ data.instanceID);
 		}
 	});
-	
+	socket.on("registerChatReciever", function(data){
+		prometheusWsUsageCounter.labels("registerChatReciever", "other").inc();
+		if(!global.wsChatRecievers) global.wsChatRecievers = [];
+		global.wsChatRecievers.push(socket);
+	});
+	socket.on("registerAlertReciever", function(data){
+		prometheusWsUsageCounter.labels("registerAlertReciever", "other").inc();
+		if(!global.wsAlertRecievers) global.wsAlertRecievers = [];
+		global.wsAlertRecievers.push(socket);
+	});
 });
 
+// handle plugins on the master
+pluginManagement();
+masterPlugins = [];
+async function pluginManagement(){
+	let startPluginLoad = Date.now();
+	masterPlugins = await getPlugins("sharedPlugins");
+	console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms");
+}
+
+function getPlugins(pluginDirectory){
+	return new Promise((resolve, reject) => {
+		let filesLeft = 0;
+		let plugins = [];
+		
+		function checkLoadComplete(plugin){
+			if(!filesLeft){
+				resolve(plugins);
+			}
+		}
+		fs.readdir(pluginDirectory, function(err, files){
+			if(err){
+				reject(err);
+			} else {
+				files.forEach(file => {
+					let pluginStartedLoading = Date.now(); // just for logging
+					filesLeft++;
+					let pluginPath = path.join(pluginDirectory, file);
+					fs.stat(pluginPath, function(err, stats){
+						if(err){
+							reject(err);
+							checkLoadComplete();
+						} else if(stats.isDirectory()){
+							let pluginConfig = require(path.resolve(pluginPath, "config"));
+							--filesLeft;
+							if(pluginConfig.masterPlugin){
+								let masterPlugin = require(path.resolve(pluginPath, pluginConfig.masterPlugin));
+								plugins.push({
+									main:new masterPlugin({
+										config,
+										pluginConfig,
+										path: pluginPath,
+										socketio: io,
+										express: app,
+									}),
+									pluginConfig,
+								});
+								
+								console.log("Loaded plugin "+pluginConfig.name+" in "+(Date.now() - pluginStartedLoading)+"ms");
+								checkLoadComplete();
+							} else {
+								checkLoadComplete();
+							}
+						} else {
+							checkLoadComplete();
+						}
+					});
+				});
+			}
+		});
+	});
+}
+
+
 module.exports = app;

package.json

@@ -21,15 +21,21 @@
     "app-root-path": "^2.0.1",
     "as-table": "^1.0.31",
     "averaged-timeseries": "^1.4.0",
+    "base64url": "^3.0.0",
+    "bcrypt": "^2.0.1",
+    "bcrypt-promise": "^2.0.0",
     "bcryptjs": "^2.4.3",
     "body-parser": "^1.18.2",
+    "cache-require-paths": "^0.3.0",
+    "cli-interact": "^0.1.9",
     "codecov": "^3.0.1",
+    "compression": "^1.7.2",
     "deepmerge": "^1.5.2",
     "ejs": "^2.5.7",
     "express": "^4.16.2",
     "express-fileupload": "0.0.6",
-    "follow-redirects": "^1.4.1",
-    "fs-extra": "^1.0.0",
+    "follow-redirects": "^1.5.0",
+    "fs-extra": "^6.0.1",
     "getmac": "^1.4.0",
     "hash-files": "^1.1.1",
     "html5-validator": "^1.0.7",
@@ -38,19 +44,21 @@
     "leveldown": "^1.9.0",
     "linvodb3": "^3.25.1",
     "luamin": "^1.0.4",
+    "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
     "mkdirp-promise": "^5.0.1",
     "moment": "^2.20.1",
     "ncp": "^2.0.0",
     "nedb": "^1.8.0",
-    "needle": "^1.1.0",
+    "needle": "^2.2.1",
     "node-factorio-api": "^0.3.1-alpha",
     "node-forge": "^0.7.1",
     "prom-client": "^10.2.3",
+    "rcon-client": "^3.0.2",
     "request": "^2.83.0",
     "right-pad": "^1.0.1",
     "rmdir-sync": "^1.0.1",
-    "simple-rcon": "^0.3.1",
+    "sanitizer": "^0.1.3",
     "single-line-log": "^1.1.2",
     "socket.io": "^2.0.4",
     "socket.io-client": "^2.0.4",

sharedPlugins/UPSdisplay/index.js

@@ -3,12 +3,20 @@ const needle = require("needle");
 
 
 class UPSdisplay {
-	constructor(slaveConfig, msgInterface){
+	constructor(slaveConfig, msgInterface, extras){
 		this.config = slaveConfig;
 		this.messageInterface = msgInterface;
+		this.socket = extras.socket;
+		
 		this.historicalTicks = [];
 		setInterval(() => {
-			this.messageInterface("/silent-command game.write_file('UPSdisplay.txt', game.tick, true, 0)");
+			this.messageInterface("/silent-command rcon.print(game.tick)", data => {
+				this.tick = Number(data);
+				this.historicalTicks[this.historicalTicks.length] = {tick:Number(data), timestamp:Date.now()};
+				if(this.historicalTicks.length > 30){
+					this.historicalTicks.shift(); // delete last element in array (position 0);
+				}
+			});
 		},1000);
 		setInterval(() => {
 			if(this.historicalTicks && this.historicalTicks[0]){
@@ -17,24 +25,15 @@ class UPSdisplay {
 				let UPS = Math.round(ticksInPeriod / (timePeriod/1000));
 				// console.log("UPS: " + UPS);
 				try{
-					needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS}}, function(err, resp) {
+					needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS, tick:this.tick}}, {headers: {'x-access-token': this.config.masterAuthToken}}, function(err, resp) {
 						// success?
 					});
 				} catch (err){
 					console.log(err);
 				}
 			}
-		}, 10000);
+		}, 5000);
 		this.messageInterface("/silent-command game.print('UPSdisplay enabled')");
 	}
-	scriptOutput(data){
-		if (data !== null && !isNaN(data)){
-			// console.log("Tick: " + data);
-			this.historicalTicks[this.historicalTicks.length] = {tick:Number(data), timestamp:Date.now()};
-			if(this.historicalTicks.length > 30){
-				this.historicalTicks.shift(); // delete last element in array (position 0);
-			}
-		}
-	}
 }
 module.exports = UPSdisplay;

sharedPlugins/globalChat/config.js

@@ -0,0 +1,10 @@
+/*
+	Clusterio plugin to allow for chat between instances.
+*/
+module.exports = {
+	// Name of package. For display somewhere I guess.
+	name: "globalChat",
+	version: "1.0.0",
+	binary: "nodePackage",
+	description: "Clusterio plugin to allow for chat between instances.",
+}

sharedPlugins/globalChat/index.js

@@ -0,0 +1,75 @@
+const needle = require("needle");
+
+module.exports = class remoteCommands {
+	constructor(mergedConfig, messageInterface, extras){
+		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		this.socket = extras.socket;
+		this.instances = {};
+		
+		this.socket.on("hello", () => this.socket.emit("registerChatReciever"));
+	
+		this.socket.on("gameChat", async data => {
+			if(data.instanceID && data.instanceID != this.config.unique.toString()){
+				if(data.data.includes("[CHAT]")
+				// check if we are allowed to cross chat at all, default to true
+				&& (mergedConfig.enableCrossServerShout == undefined || mergedConfig.enableCrossServerShout)
+				// check if we are supposed to relay this message
+				&& (mergedConfig.mirrorAllChat || data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){
+					let words = data.data.trim().split(" ");
+					words.shift();
+					words.shift();
+					words.shift();
+					let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout", "");
+					
+					let instance = await this.getInstanceName(data.instanceID);
+					
+					this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')");
+				}
+			}
+		});
+	}
+	async factorioOutput(data){
+		try{
+		this.socket.emit("gameChat", {
+			instanceID: this.config.unique.toString(),
+			data,
+		});
+		if(data.includes("[CHAT]") && data.includes("!info")){
+			let infoMessage = [
+				"=== Server info ===",
+				"Name: "+await this.getInstanceName(this.config.unique)+" ("+this.config.unique+")",
+			];
+			infoMessage.forEach((line, i) => {
+				setTimeout(()=>this.messageInterface("/silent-command game.print('"+line+"')"),i*200);
+			});
+		}
+		}catch(e){console.log(e)}
+	}
+	getInstanceName(instanceID){
+		return new Promise((resolve, reject) => {
+			let instance = this.instances[instanceID];
+			if(!instance){
+				needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => {
+					if(err || response.statusCode != 200) {
+						console.log("Unable to get JSON master/api/slaves, master might be unaccessible");
+					} else if (response && response.body) {	
+						if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();}
+							try {
+								for (let index in response.body)
+									this.instances[index] = response.body[index].instanceName;
+							} catch (e){
+								console.log(e);
+								return null;
+							}
+						instance = this.instances[instanceID] 							
+						if (!instance) instance = instanceID;  //somehow the master doesn't know the instance	
+						resolve(instance);
+					}
+				});
+			} else {
+				resolve(instance);
+			}
+		});
+	}
+}

sharedPlugins/inventoryImports/interfacingFunctions.js

@@ -76,7 +76,7 @@ function handleInventory(json, config, callback){
 						}
 						//console.log("Requesting: "+JSON.stringify(request));
 						// request the difference from master
-						needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', request, {json:true}, function (err, response, body) {
+						needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', request, {json:true,headers: {'x-access-token': config.masterAuthToken}}, function (err, response, body) {
 							if (!err && response && response.body && (typeof response.body == "object" || isJson(response.body))) {
 								// confirmed orders, already removed from master inventory
 								//response.body.name: response.body.count
@@ -103,7 +103,7 @@ function handleInventory(json, config, callback){
 				for(let i = 0; i < stackArray.length; i++){
 					if(stackArray[i].count > 0){
 						// console.log("Returning overflow: " + JSON.stringify(stackArray[i]));
-						needle.post(config.masterIP + ":" + config.masterPort + '/api/place', stackArray[i], function (err, resp, body) {
+						needle.post(config.masterIP + ":" + config.masterPort + '/api/place', stackArray[i], {headers: {'x-access-token': config.masterAuthToken}}, function (err, resp, body) {
 							// console.log(body);
 						});
 					}

sharedPlugins/remoteCommands/index.js

@@ -22,4 +22,4 @@ module.exports = class remoteCommands {
 	scriptOutput(data){
         // we don't return anything yet
     }
-}
+}

sharedPlugins/remoteMap/chunkStore.js

@@ -2,11 +2,12 @@ var mkdirp = require("mkdirp-promise");
 var mkdirpSync = require("mkdirp").sync;
 var fs = require("fs");
 var events = require("events");
+var path = require("path");
 
 var objectOps = require("./../../lib/objectOps");
 
 var LinvoDB = require("linvodb3");
-LinvoDB.dbPath = "./database/linvodb/";
+// LinvoDB.dbPath = "./database/linvodb/";
 
 
 let allowLogging = true;
@@ -20,6 +21,7 @@ module.exports = class chunkStore {
 		this.chunkSize = chunkSize;
 		this.path = (path[path.length] == "/") ? path : path+"/";
 		mkdirpSync(path+name);
+		LinvoDB.dbPath = this.path;
 		
 		this.db = new LinvoDB(name, {}, {});
 		

sharedPlugins/researchSync/index.js

@@ -83,7 +83,7 @@ class ResearchSync {
                 instanceID: this.config.unique,
                 password: this.config.clientPassword,
                 meta: {research: this.research}
-            }, function (err, resp) {
+            }, {headers: {'x-access-token': this.config.masterAuthToken}}, function (err, resp) {
                 // success?
             });
         });

static/head.html

@@ -1,9 +1,9 @@
 <title>Clusterio</title>
-<link rel="stylesheet" type="text/css" href="style.css">
-<script src="external/jquery.js"></script>
-<script src="external/jquery-ui.js"></script>
-<script src="external/canvasjs.js"></script>
-<script src="external/moment.min.js"></script>
-<script src="external/mousetrap.min.js"></script>
-<script src="main.js"></script>
-<link href="external/css/Roboto.css" rel="stylesheet">
+<link rel="stylesheet" type="text/css" href="/style.css">
+<script src="/external/jquery.js"></script>
+<script src="/external/jquery-ui.js"></script>
+<script src="/external/canvasjs.js"></script>
+<script src="/external/moment.min.js"></script>
+<script src="/external/mousetrap.min.js"></script>
+<script src="/main.js"></script>
+<link href="/external/css/Roboto.css" rel="stylesheet">

static/main.js

@@ -49,6 +49,24 @@ debugTools = {
 	}
 }
 
+let cachedInstanceNames = {};
+async function getInstanceName(instanceID){
+	let instance = cachedInstanceNames[instanceID];
+	if(!instance){
+		let slaves = await getJSON("/api/slaves");
+		try {
+			for(id in slaves){
+				cachedInstanceNames[id] = slaves[id].instanceName;
+			}
+		} catch(e){
+			console.log("Error while looking for instance name!");
+			console.log(e);
+		}
+		return cachedInstanceNames[instanceID] || instanceID;
+	} else {
+		return cachedInstanceNames[instanceID];
+	}
+}
 var g = {}
 contents = {
 	"iron-plate":100,
@@ -63,44 +81,66 @@ function djb2(str){
 	}
 	return hash;
 }
+
+// get queryString parameters
+function getParameterByName(name, url) {
+    if (!url) url = window.location.href;
+    name = name.replace(/[\[\]]/g, "\\$&");
+    var regex = new RegExp("[?&]" + name + "(=([^&#]*)|&|#|$)"),
+        results = regex.exec(url);
+    if (!results) return null;
+    if (!results[2]) return '';
+    return decodeURIComponent(results[2].replace(/\+/g, " "));
+}
+
 // callback(err, json)
 function getJSON(url, callback) {
-	var xhr = new XMLHttpRequest();
-	xhr.open('GET', url, true);
-	xhr.responseType = 'json';
-	xhr.onload = function() {
-	  var status = xhr.status;
-	  if (status == 200) {
-		callback(null, xhr.response);
-	  } else {
-		callback(status);
-	  }
-	};
-	// triggers if connection is refused
-	xhr.onerror = function(e){
-		callback(e);
-	};
-	xhr.send();
+	return new Promise((resolve, reject) => {
+		var xhr = new XMLHttpRequest();
+		xhr.open('GET', url, true);
+		xhr.responseType = 'json';
+		xhr.onload = function() {
+			var status = xhr.status;
+			if (status == 200) {
+				if(callback) callback(null, xhr.response);
+				resolve(xhr.response);
+			} else {
+				if(callback) callback(status);
+				reject(status);
+			}
+		};
+		// triggers if connection is refused
+		xhr.onerror = function(e){
+			if(callback) callback(e);
+			reject(e);
+		};
+		xhr.send();
+	});
 };
 // callback(err, json)
 function postJSON(url, data, callback) {
-	var xhr = new XMLHttpRequest();
-	xhr.open('POST', url, true);
-	xhr.setRequestHeader('Content-Type', 'application/json; charset=UTF-8');
-	xhr.responseType = 'json';
-	xhr.onload = function() {
-	  var status = xhr.status;
-	  if (status == 200) {
-		callback(null, xhr.response);
-	  } else {
-		callback(status);
-	  }
-	};
-	// triggers if connection is refused
-	xhr.onerror = function(e){
-		callback(e);
-	};
-	xhr.send(JSON.stringify(data));
+	return new Promise((resolve, reject) => {
+		var xhr = new XMLHttpRequest();
+		xhr.open('POST', url, true);
+		xhr.setRequestHeader('Content-Type', 'application/json; charset=UTF-8');
+		xhr.responseType = 'json';
+		xhr.onload = function() {
+			var status = xhr.status;
+			if (status == 200) {
+				if(callback) callback(null, xhr.response);
+				resolve(xhr.response);
+			} else {
+				if(callback) callback(status);
+				reject(status);
+			}
+		};
+		// triggers if connection is refused
+		xhr.onerror = function(e){
+			if(callback) callback(e);
+			reject(e);
+		};
+		xhr.send(JSON.stringify(data));
+	});
 };
 // return Boolean
 function isJSON(string){

static/sidebar.html

@@ -2,16 +2,35 @@
 	
 </div>
 <div id="sidenav">
-	<div id="logo">
+	<div id="logo" style="height:50px;">
+		<img src="/pictures/clusterio_icon.png" style="width:50px;height:50px; float:left;"></img>
 		<h1>Clusterio</h1>
 	</div>
 	<a href="/"><div id="storage-menu" class="button-black menuItem">
 		<p>Storage</p>
 	</div></a>
-	<a href="nodes"><div id="slaves-menu" class="button-black menuItem">
+	<a href="/nodes"><div id="slaves-menu" class="button-black menuItem">
 		<p>Slaves</p>
 	</div></a>
-	<a href="settings"><div id="settings-menu" class="button-black menuItem">
+	<%# Very ugly ejs code to hide the Players button if the playerManager plugin isn`t installed %>
+	<%
+		sidebarPluginButtonsHtml = "";
+		masterPlugins.forEach(plug => {
+			if(plug.main
+			&& plug.main.ui
+			&& typeof plug.main.ui.sidebar == "object"
+			&& Array.isArray(plug.main.ui.sidebar)){
+				let sidebarEntries = plug.main.ui.sidebar;
+				sidebarEntries.forEach(button => {
+					if(button.getHtml && typeof button.getHtml == "function"){
+						sidebarPluginButtonsHtml += button.getHtml();
+					}
+				});
+			}
+		});	
+	%>
+	<%- sidebarPluginButtonsHtml %>
+	<a href="/settings"><div id="settings-menu" class="button-black menuItem">
 		<p>Settings</p>
 	</div></a>
 </div>

static/storage.js

@@ -16,6 +16,9 @@ function drawcontents(data) {
 	
 	//update existing rows or create new ones
 	data.forEach(function(item, i) {
+		// format count to have .s in it for large number readability (1,000,000)
+		item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+		
 		let row = rows[i];
 		if(!row) {
 			row = document.createElement('tr');

static/style.css

@@ -19,6 +19,10 @@ button {
 	background-color: #262626;
 	color: 	#C8C8C8;
 }
+#logo > h1 {
+	padding-top: 6px;
+	transform: translateX(5%);
+}
 #storage {
 	float: left;
 }
@@ -188,7 +192,7 @@ a:hover {
 
 .button-black {
 	width:100%;
-	height:40px;
+	height:50px;
 	background-color:black;
 	color:white;
 	transition: all 300ms;