Fixed remoteMap stuff

Neural Dole v2

.gitignore

@@ -22,6 +22,7 @@ coverage
 
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
+dist/
 
 node_modules/*
 !node_modules/_app
@@ -50,5 +51,20 @@ sharedPlugins/*
 !sharedPlugins/remoteCommands/
 !sharedPlugins/researchSync/
 !sharedPlugins/globalChat/
+!sharedPlugins/clusterioMod/
+!sharedPlugins/serverManager/
+sharedPlugins/serverManager/database/*
+sharedPlugins/serverManager/config.json
+sharedPlugins/serverManager/secret-api-token.txt
+!sharedPlugins/scenario_loader/
+sharedPlugins/scenario_loader/scenarios
+!sharedPlugins/statisticsExporter/
+
+# File used for marking plugins as disabled
+sharedPlugins/*/DISABLED
 
 \.cache-require-paths\.json
+
+\.vs/
+
+bin/

.snyk

@@ -0,0 +1,45 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+version: v1.13.3
+ignore: {}
+# patches apply the minimum changes required to fix a vulnerability
+patch:
+  'npm:lodash:20180130':
+    - html5-validator > caporal > cli-table2 > lodash:
+        patched: '2018-07-03T06:07:55.532Z'
+  'npm:hawk:20160119':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > hawk:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:http-signature:20150122':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > http-signature:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:mime:20170907':
+    - node-factorio-api > documentation > highlight.js > gear-lib > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > form-data > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:minimatch:20160620':
+    - node-factorio-api > documentation > highlight.js > gear > liftoff > findup-sync > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > gear > liftoff > findup-sync > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jslint > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jshint > cli > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jshint > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:request:20160119':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:tunnel-agent:20170305':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > tunnel-agent:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:uglify-js:20151024':
+    - node-factorio-api > documentation > highlight.js > gear-lib > handlebars > uglify-js:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > uglify-js:
+        patched: '2019-02-25T06:48:21.845Z'

.travis.yml

@@ -1,6 +1,6 @@
 language: node_js
 node_js:
-  - "node"
+  - "9"
 before_install:
   - sudo apt-get -qq update
   - sudo apt-get install -y curl git tar

.vs/ProjectSettings.json

@@ -0,0 +1,3 @@
+{
+  "CurrentProjectSetting": null
+}

.vs/VSWorkspaceState.json

@@ -0,0 +1,8 @@
+{
+  "ExpandedNodes": [
+    "",
+    "\\sharedPlugins"
+  ],
+  "SelectedNode": "\\.gitignore",
+  "PreviewInSolutionExplorer": false
+}

Dockerfile

@@ -1,15 +1,17 @@
-FROM node:latest
+FROM node:9.11.2
-RUN apt-get update && apt install git -y
+RUN apt-get update && apt install git curl tar -y
-
 RUN mkdir factorioClusterio
-RUN git clone -b dev https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xvzf factorio.tar.gz
+
+RUN git clone -b master https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install
+RUN cd factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz
 
 WORKDIR factorioClusterio
-RUN mkdir instances sharedMods sharedPlugins
+RUN mkdir instances sharedMods
+RUN cp config.json.dist config.json
 
 RUN node client.js download
 
-LABEL maintainer "danielv@live.no"
+LABEL maintainer "Sir3lit@gmail.com"
 
 EXPOSE 8080 34167
 VOLUME /factorioClusterio/instances

LICENSE.md

@@ -1,4 +1,4 @@
-MIT License
+This product uses the MIT License
 
 Copyright (c) 2016 Daniel Vestøl
 
@@ -19,3 +19,8 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+The icons for angels modded items located in /static/pictures/ are licensed
+by https://mods.factorio.com/user/Arch666Angel under CC BY-NC-ND 4.0
+https://creativecommons.org/licenses/by-nc-nd/4.0/
+

README.md

@@ -1,12 +1,18 @@
+<img src="./logo.svg" width="100%" align="right">
+
+<br/>
+<br/>
+<br/>
+
 # factorioClusterio
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2FDanielv123%2FfactorioClusterio.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FDanielv123%2FfactorioClusterio?ref=badge_shield)
 
-| Branch | Coverage | Build |
+Discord for development/support/play: https://discord.gg/5XuDkje
-|--------|----------| ----- |
-| Master | [![codecov](https://codecov.io/gh/Danielv123/factorioClusterio/branch/master/graph/badge.svg)](https://codecov.io/gh/Danielv123/factorioClusterio) | [![Build Status](https://travis-ci.org/Danielv123/factorioClusterio.svg?branch=master)](https://travis-ci.org/Danielv123/factorioClusterio) |
-| Dev    | [![codecov](https://codecov.io/gh/Danielv123/factorioClusterio/branch/dev/graph/badge.svg)](https://codecov.io/gh/Danielv123/factorioClusterio) | [![Build Status](https://travis-ci.org/Danielv123/factorioClusterio.svg?branch=dev)](https://travis-ci.org/Danielv123/factorioClusterio) |
 
-If you want to connect to a clusterio cluster, please reffer to the [client](https://github.com/Danielv123/factorioClusterioClient)
+### Ways to support me/the project:
+
+* Contribute with code/documentation. Always nice to move the project forward
+
+* Support me monetarily on [patreon](https://www.patreon.com/danielv123) or paypal: danielv@live.no
 
 ### Table of contents
 
@@ -16,6 +22,10 @@ If you want to connect to a clusterio cluster, please reffer to the [client](htt
 
 * [Windows setup](#windows-setup)
 
+* [Optional plugins](#Plugins)
+
+* [Common problems](#Common-problems)
+
 * [Command cheatsheet](#cheatsheet)
 
 ## Introduction
@@ -32,7 +42,16 @@ Features:
 
 - Inventory combinator to display item levels in the cluster (and epoch time)
 
-- Reporting of graphs and UPS on master interface
+- Reporting of graphs and UPS on master interface (Also has extensive Prometheus reporting)
+
+Optional extras (see [Plugins](#Plugins))
+
+- Have your inventory synchronize across servers
+
+- Teleport trans from the border of one world to te next
+
+- Show in-game chat in discord
+
 
 Connection diagram:
 
@@ -59,10 +78,9 @@ Clusterio exploits one of the games logging features, game.write_file and RCON t
 
 7. client.js on server2: We were allowed to import x of item y, run command /c remote.call("clusterio", "importMany", "{'copper-plate':120}")
 
-This process works the same for both items and liquids, independant on what mods are used. Yes, modded items are fully supported.
+This process works the same for both items and liquids, independent on what mods are used. Yes, modded items are fully supported.
 
-Clusterio also handles a few other neat things, such as giving you access to epoch time, transmitting combinator signals between worlds (and any other application who wants to) and 
+Clusterio can also do a few other neat things, such as giving you access to epoch time, syncing player inventories between servers, keeping track of playtime (playerManager plugin), teleporting trains between servers (trainTeleports) and exporting tons of factorio related statistics to Prometheus for graphing in grafana.
-creating graphs on the master web interface.
 
 ## Ubuntu setup
 
@@ -78,24 +96,35 @@ Master and all slaves:
     curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64
     tar -xf factorio.tar.gz
     npm install
+    cp config.json.dist config.json
     node ./lib/npmPostinstall.js
 
+
 downloads and installs nodejs, pm2, git and clusterio. To specify a version, change "latest" in the link to a version number like 0.14.21.
 
-*Copy config.json.dist to config.json* - Otherwise it will crash, as you'd expect.
+Now you need to edit the `config.json` file. If you skip this step nothing will work.
+Pretty much all the blank fields should be filled in, except on the master where a few can be omitted.
+
+* You get the `masterAuthToken` from `secret-api-token.txt` in the master install dir after running the master twice.
+
+* You get your factorio matchmaking token from factorio.com
+
+* The `masterAuthSecret` should never be touched unless you want to invalidate everyones authentication tokens
 
 **Master**
 
     pm2 start master.js --name master
 
+OR
+
+    node master.js
+
 **Server Host**
 
 To download the mod for all its non vanilla features and items, (optional, but very recommended)
 
     node client.js manage shared mods add clusterio
 
-Add the master API key, found in the masters install directory (secret-api-token.txt) to your config.json masterAuthToken field
-
 To create a new instance (its own save, set of mods and config files)
 
     node client.js start [instancename]
@@ -108,7 +137,7 @@ use `nano config.json` to change settings.
 
 **Ubuntu with Docker**
 
-Clusterio has limited support for using docker.
+Clusterio has *very* limited support for using docker.
 
     sudo docker build -t clusterio --no-cache --force-rm factorioClusterio
 
@@ -128,10 +157,6 @@ Server host (Slave) = client.js + factorio server
 
 Game Client = The people connecting to the server
 
-All Server Hosts (Slaves) AND Game Clients need to be running the clusterio mod located at [github](https://github.com/Danielv123/factorioClusterioMod) Install it by dropping it into either the sharedMods folder or instances/[instanceName]/instanceMods folder.
-
-There are no requirements for other mods, they can be ran in any configuration allowed by the base game.
-
 **Requirements**
 
 download and install nodeJS 8 or 9 from http://nodejs.org
@@ -147,6 +172,7 @@ reboot when you are done, then proceed to the next steps. *reboots matter*
 2. Copy config.json.dist to config.json
 
 3. Follow the instructions in the bat file
+3. Some of the instructions are outdated. If you get stuck somewhere, look at the Ubuntu section.
 
 4. Run `node client.js manage shared mods add clusterio`
 
@@ -158,17 +184,21 @@ reboot when you are done, then proceed to the next steps. *reboots matter*
 
 2. Copy config.json.dist to config.json
 
-3. Follow the instructions given
+3. Follow the instructions given.
+
+3.5 Some of the instructions are outdated. If you get stuck somewhere, look at the Ubuntu section.
 
 4. Type `node client.js start [instancename]` to create a new instance.
 
 To connect to a master server running on a remote machine, open config.json with your favourite text editor (notepad++). You can also set it up to use the official server browser.
 
-Change masterIP to something like 31.152.123.14 (provided by master server owner)
+Change `masterIP `to something like `31.152.123.14` (provided by master server owner)
-e war
-Change masterPort to something like 8080 (provided by master server owner)
 
-Repeat step 3 for more servers on one machine. You should be able to find its port by looking at the slave section on master:8080 (the web interface)
+Change `masterPort` to something like `8080` (provided by master server owner)
+
+Change `masterAuthToken` to the value found in `secret-api-token.txt` on the master server
+
+Repeat step 4 for more servers on one machine. You should be able to find its port by looking at the slave section on master:8080 (the web interface)
 
 **GameClient**
 
@@ -180,6 +210,55 @@ Fancy game client that does the following steps automatically, but is really old
 
 3. Run factorio and connect to slave as a normal MP game. You will find the port number to connect to at http://[masterAddress]:8080
 
+## Plugins
+Here are the known Clusterio plugins in the wild:
+1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
+2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
+3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.
+
+## Common problems
+
+### Cannot find module: `/../../config`
+
+Copy your config.json.dist to config.json and configure it.
+
+### EACCESS [...] LISTEN 443
+
+Some systems don't let non root processes listen to ports below 1000. Either run with `sudo` or change config.json to use higher port numbers.
+
+### InitializationError: First argument must be an abstract-leveldown compliant store
+
+do:
+
+    npm install linvodb3@3.25.1
+
+if that does not suffice do
+
+    npm install levelup@1.3.9
+
+If it still doesn't work, come over to our support discord linked at the top of this readme.
+
+### Portforwarding doesn't work on the master server when running under WSL
+
+If you follow the ubuntu guide on WSL (Windows Subsystem for Linux, Bash on Ubuntu on Windows specifically), you will find that the website works on localhost and on your local ip, but not on the global ip. This is also true when you correctly port-forwarded the correct ports. Even when routing this server through nginx in WSL, the issue persists. Then, on a hunch, I tried to run nginx from windows itself and found that this DID work. It came to me that the only usage difference between the 2 versions of nginx is that I got a Windows Firewall popup.
+
+TLDR: the tested fix is:
+
+- open your windows firewall and go to advanced settings
+
+- click on inbound rules and click on new rule...
+
+- select port and click next >
+
+- select TCP and select specific local ports and type in the ports that you want to open (comma separated) and click next > 3 times
+
+- give the rule a name (like 'web server' or something), give it a description (optionally) and click finish
+
+### Other fixes for other potential problems:
+
+Sometimes the install fails. Try `node ./lib/npmPostinstall.js` to complete it.
+
+
 ## Cheatsheet
 
 **To create a new instance/start it**

client.js

@@ -18,11 +18,13 @@ const ioClient = require("socket.io-client");
 const objectOps = require("./lib/objectOps.js");
 const fileOps = require("_app/fileOps");
 const stringUtils = require("./lib/stringUtils.js");
-const modManager = require("./lib/manager/modManager.js");
 const configManager = require("./lib/manager/configManager.js");
 
+// argument parsing
+const args = require('minimist')(process.argv.slice(2));
+
 // require config.json
-var config = require('./config');
+var config = require(args.config || './config');
 var global = {};
 
 if (!fs.existsSync(config.instanceDirectory)) {
@@ -42,11 +44,14 @@ const command = process.argv[2];
 // and as the process name in ps/top on linux.
 process.title = "clusterioClient "+instance;
 
+// add better stack traces on promise rejection
+process.on('unhandledRejection', r => console.log(r));
+
 // make sure we have the master access token (can't write to master without it since clusterio 2.0)
 if(!config.masterAuthToken || typeof config.masterAuthToken !== "string"){
-	console.log("ERROR invalid config!");
+	console.error("ERROR invalid config!");
-	console.log("Master server now needs an access token for write operations. As clusterio slaves depends \
+	console.error("Master server now needs an access token for write operations. As clusterio slaves depends \
-	upon this, please add your token to config.js in the field named masterAuthToken. \
+	upon this, please add your token to config.json in the field named masterAuthToken. \
 	You can retrieve your auth token from the master in secret-api-token.txt after running it once.");
 }
 const needleOptionsWithTokenAuthHeader = {
@@ -94,6 +99,7 @@ async function messageInterfaceInternal(command, callback, resolve, reject) {
 			if(typeof callback == "function"){
 				callback();
 			}
+			reject(err)
 		}
 	}
 }
@@ -112,6 +118,8 @@ if (!command || command == "help" || command == "--help") {
 	console.error("node client.js delete [instance]");
 	console.error("To download the latest version of the Clusterio lua mod, do");
 	console.error("node client.js manage shared mods download clusterio");
+	console.error("To download a clusterio plugin, do");
+	console.error("node client.js manage shared plugins install https://github.com/Danielv123/playerManager");
 	console.error("For more management options, do");
 	console.error("node client.js manage");
 	process.exit(1);
@@ -134,12 +142,12 @@ if (!command || command == "help" || command == "--help") {
 	// create port colloumn
 	let factorioPorts = [];
 	instanceNames.forEach(function(instance){
-		let factorioPort
+		let factorioPort;
 		
 		if(instance.includes("Name:")){
 			factorioPort = "Port:"
 		} else {
-			factorioPort = require(config.instanceDirectory+"/"+instance+"/config").factorioPort;
+			factorioPort = require(path.resolve(config.instanceDirectory, instance, 'config')).factorioPort;
 		}
 		factorioPorts.push(factorioPort);
 	});
@@ -167,8 +175,10 @@ if (!command || command == "help" || command == "--help") {
 			console.log('node client.js manage '+instance+' '+tool+' ["list", "search", "add", "remove", "update"]');
 		} else if(tool && tool == "config") {
 			console.log('node client.js manage '+instance+' '+tool+' ["list", "edit"]');
+		} else if(tool && tool == "plugins") {
+			console.log(`node client.js manage ${instance} ${tool} ["list", "add", "remove"]`);
 		} else {
-			console.log('node client.js manage '+(instance || '[instance, "shared"]') +' '+ (tool || '["mods", "config"]') + ' ...');
+			console.log('node client.js manage '+(instance || '[instance, "shared"]') +' '+ (tool || '["mods", "config", "plugins"]') + ' ...');
 		}
 	}
 	const tool = process.argv[4] || "";
@@ -176,6 +186,9 @@ if (!command || command == "help" || command == "--help") {
 	if(instance){
 		if(tool == "mods"){
 			(async function(){try{
+				// do require down here to reduce master load time
+				const modManager = require("./lib/manager/modManager.js")(config);
+				
 				// allow managing mods
 				if(action == "list"){
 					console.log(await modManager.listMods(instance));
@@ -212,6 +225,28 @@ if (!command || command == "help" || command == "--help") {
 			} else {
 				usage(instance, tool);
 			}
+		} else if(tool == "plugins"){
+			(async function(){try{
+				const pluginManager = require("./lib/manager/pluginManager.js")(config);
+				if(action == "list"){
+					await pluginManager.listPlugins();
+				} else if(action == "add" || action == "install" || action == "download"){
+					let status = await pluginManager.addPlugin(process.argv[6]);
+				} else if(action == "remove" || action == "uninstall" || action == "delete"){
+					let status = await pluginManager.removePlugin(process.argv[6]);
+					if(status && status.msg) console.log(status.msg);
+				} else if(action == "enable"){
+					let status = await pluginManager.enablePlugin(process.argv[6]);
+					if(status && status.msg) console.log(status.msg);
+				} else if(action == "disable"){
+					let status = await pluginManager.disablePlugin(process.argv[6]);
+					if(status && status.msg) console.log(status.msg);
+				}
+				process.exit(0);
+			}catch(e){
+				console.error(e);
+				process.exit(1);
+			}})();
 		} else {
 			usage(instance);
 		}
@@ -225,7 +260,7 @@ if (!command || command == "help" || command == "--help") {
 		console.error("Usage: node client.js delete [instance]");
 		process.exit(1);
 	} else if (typeof process.argv[3] == "string" && fs.existsSync(config.instanceDirectory+"/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") {
-		fileOps.deleteFolderRecursiveSync(config.instanceDirectory+"/" + process.argv[3]);
+		fileOps.deleteFolderRecursiveSync(path.resolve(config.instanceDirectory, process.argv[3])); // TODO: Check if this can cause i-craft users to format their server by using wrong paths
 		console.log("Deleted instance " + process.argv[3]);
 		process.exit(0);
 	} else {
@@ -242,14 +277,16 @@ if (!command || command == "help" || command == "--help") {
 		console.log(url);
 		let file = fs.createWriteStream("sharedMods/"+name);
 		https.get(url, function(response) {
+			response.on('end', function () {
+				console.log("Downloaded "+name);
+				process.exit(0);
+			});
 			response.pipe(file);
-			console.log("Downloaded "+name);
+		}).end();
-			process.exit(0);
-		});
 	}
 } else if (command == "start" && instance === undefined) {
-	console.log("ERROR: No instanceName provided!");
+	console.error("ERROR: No instanceName provided!");
-	console.log("Usage: node client.js start [instanceName]");
+	console.error("Usage: node client.js start [instanceName]");
 	process.exit(0);
 } else if (command == "start" && typeof instance == "string" && instance != "/" && !fs.existsSync(instancedirectory)) {
 	// if instance does not exist, create it
@@ -262,6 +299,11 @@ if (!command || command == "help" || command == "--help") {
 	fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", "");
 	fs.mkdirSync(instancedirectory + "/mods/");
 	fs.mkdirSync(instancedirectory + "/instanceMods/");
+    fs.mkdirSync(instancedirectory + "/scenarios/");
+    ncp("./lib/scenarios", path.resolve(instancedirectory, "scenarios"), err => {
+        if (err) console.error(err)
+    });
+
 	// fs.symlinkSync('../../../sharedMods', instancedirectory + "/mods", 'junction') // This is broken because it can only take a file as first argument, not a folder
 	fs.writeFileSync(instancedirectory + `/config.ini`, `[path]\r\n
 read-data=${ path.resolve(config.factorioDirectory, "data") }\r\n
@@ -269,11 +311,12 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 	`);
 
 	// this line is probably not needed anymore but Im not gonna remove it
-	fs.copySync('sharedMods', instancedirectory + "/mods");
+	fs.copySync('sharedMods', path.join(instancedirectory, "mods"));
 	let instconf = {
-		"factorioPort": process.env.FACTORIOPORT || Math.floor(Math.random() * 65535),
+		"factorioPort": args.port || process.env.FACTORIOPORT || Math.floor(Math.random() * 65535),
-		"clientPort": process.env.RCONPORT || Math.floor(Math.random() * 65535),
+		"clientPort": args["rcon-port"] || process.env.RCONPORT || Math.floor(Math.random() * 65535),
-		"clientPassword": Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8),
+		"__comment_clientPassword": "This is the rcon password. Its also used for making an instanceID. Make sure its unique and not blank.",
+		"clientPassword": args["rcon-password"] || Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8),
 	}
 	console.log("Clusterio | Created instance with settings:")
 	console.log(instconf);
@@ -301,17 +344,56 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 		"autosave_slots": 5,
 		"afk_autokick_interval": 0,
 		"auto_pause": config.auto_pause,
-	}
+	};
 	fs.writeFileSync(instancedirectory + "/server-settings.json", JSON.stringify(serversettings, null, 4));
-	let createSave = child_process.spawnSync(
+    console.log("Server settings: "+JSON.stringify(serversettings, null, 4));
-		'./' + config.factorioDirectory + '/bin/x64/factorio', [
+    console.log("Creating save .....");
-			'-c', instancedirectory + '/config.ini',
+    let factorio = child_process.spawn(
-			'--create', instancedirectory + '/saves/save.zip',
+        './' + config.factorioDirectory + '/bin/x64/factorio', [
-		]
+            '-c', instancedirectory + '/config.ini',
-	);
+            // '--create', instancedirectory + '/saves/save.zip',
-	console.log("Instance created!");
+            '--start-server-load-scenario', 'Hotpatch',
-	process.exit(0);
+            '--server-settings', instancedirectory + '/server-settings.json',
-} else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)) {
+            '--rcon-port', Number(process.env.RCONPORT) || instconf.clientPort,
+            '--rcon-password', instconf.clientPassword,
+        ], {
+            'stdio': ['pipe', 'pipe', 'pipe']
+        }
+    );
+    factorio.stdout.on("data", data => {
+        data = data.toString("utf8").replace(/(\r\n\t|\n|\r\t)/gm,"");
+        console.log(data);
+        if(data.includes("Starting RCON interface")){
+            let client = new Rcon();
+            client.connect({
+                host: 'localhost',
+                port: Number(process.env.RCONPORT) || instconf.clientPort,
+                password: instconf.clientPassword,
+                timeout: 5000
+            });
+            client.onDidAuthenticate(() => {
+                console.log('Clusterio | RCON Authenticated!');
+            });
+            client.onDidConnect(() => {
+                console.log('Clusterio | RCON Connected, starting save');
+                client.send("/c game.server_save('hotpachSave')");
+            });
+        }
+        if(data.includes("Saving finished")){
+            console.log("Map saved as hotpachSave.zip, exiting...");
+            console.log("Instance created!")
+            process.exit(0);
+        }
+		if(data.includes("Downloading from auth server failed")){
+			console.error("Instance creation failed, unable to establish auth server connection.");
+			console.error("Deleting broken instance...");
+			
+			fileOps.deleteFolderRecursiveSync(path.resolve(config.instanceDirectory, process.argv[3]));
+			
+			process.exit(0);
+		}
+    });
+} else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)){(async () => {
 	// Exit if no instance specified (it should be, just a safeguard);
 	if(instancedirectory != config.instanceDirectory+"/undefined"){
 		var instanceconfig = require(path.resolve(instancedirectory,'config'));
@@ -326,47 +408,51 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 		process.exit(1);
 	}
 	console.log("Deleting .tmp.zip files");
-	let savefiles = fs.readdirSync(instancedirectory + "/saves/");
+	let savefiles = fs.readdirSync(path.join(instancedirectory,"saves"));
 	for(i = 0; i < savefiles.length; i++){
 		if(savefiles[i].substr(savefiles[i].length - 8, 8) == ".tmp.zip") {
-			fs.unlinkSync(instancedirectory + "/saves/" + savefiles[i]);
+			fs.unlinkSync(path.resolve(instancedirectory, "saves", savefiles[i]));
 		}
 	}
-	console.log("Deleting logs");
+	console.log("Clusterio | Rotating old logs...");
 	// clean old log file to avoid crash
-	// file exists, delete so we don't get in trouble
+	try{
-	try {
+		let logPath = path.join(instancedirectory,'factorio-current.log');
-		fs.unlinkSync(instancedirectory+'/factorio-current.log');
+		let stat = await fs.stat(logPath);
-	} catch (err){
+		console.log(stat)
-		if(err){
+		console.log(stat.isFile())
-			console.log(err);
+		if(stat.isFile()){
-		} else {
+			let logFilename = `factorio-${Math.floor(Date.parse(stat.mtime)/1000)}.log`;
-			console.log("Clusterio | Deleting old logs...");
+			await fs.rename(logPath, path.join(instancedirectory, logFilename));
+			console.log(`Log rotated as ${logFilename}`);
 		}
-	}
+	}catch(e){}
+	// Math.floor(Date.now()/1000)
+	// (new Date).toGMTString()
 	
 	// move mods from ./sharedMods to the instances mod directory
-	try{fs.mkdirSync(instancedirectory + "/instanceMods/");}catch(e){}
+	try{fs.mkdirSync(path.join(instancedirectory, "instanceMods"));}catch(e){}
-	try{rmdirSync(instancedirectory + "/mods");}catch(e){}
+	try{rmdirSync(path.join(instancedirectory, "mods"));}catch(e){}
 	try {
 		// mods directory that will be emptied (deleted) when closing the server to facilitate seperation of instanceMods and sharedMods
-		fs.mkdirSync(instancedirectory + "/mods/");
+		fs.mkdirSync(path.join(instancedirectory, "mods"));
 	} catch(e){}
 	console.log("Clusterio | Moving shared mods from sharedMods/ to instance/mods...");
-	fs.copySync('sharedMods', instancedirectory + "/mods");
+	fs.copySync('sharedMods', path.join(instancedirectory, "mods"));
 	console.log("Clusterio | Moving instance specific mods from instance/instanceMods to instance/mods...");
-	fs.copySync(instancedirectory + "/instanceMods", instancedirectory + "/mods");
+	fs.copySync(path.join(instancedirectory, "instanceMods"), path.join(instancedirectory, "mods"));
 	
-	process.on('SIGINT', function () {
+	// Set paths for factorio so it reads and writes from the correct place even if the instance is imported from somewhere else
-		console.log("Caught interrupt signal, sending /quit");
+	fs.writeFileSync(instancedirectory + `/config.ini`, `[path]\r\n
-		messageInterface("/quit");
+read-data=${ path.resolve(config.factorioDirectory, "data") }\r\n
-	});
+write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
+	`);
 	
 	// Spawn factorio server
 	//var serverprocess = child_process.exec(commandline);
 	fileOps.getNewestFile(instancedirectory + "/saves/", fs.readdirSync(instancedirectory + "/saves/"),function(err, latestSave) {
 		if(err) {
-			console.error("ERROR!")
+			console.error("ERROR!");
 			console.error("Your savefile seems to be missing. This might because you created an instance without having factorio\
  installed and configured properly. Try installing factorio and adding your savefile to instances/[instancename]/saves/\n");
 			throw err;
@@ -376,29 +462,29 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 			'./' + config.factorioDirectory + '/bin/x64/factorio', [
 				'-c', instancedirectory + '/config.ini',
 				'--start-server', latestSave.file,
-				'--rcon-port', Number(process.env.RCONPORT) || instanceconfig.clientPort,
+				'--rcon-port', args["rcon-port"] || Number(process.env.RCONPORT) || instanceconfig.clientPort,
-				'--rcon-password', instanceconfig.clientPassword,
+				'--rcon-password', args["rcon-password"] || instanceconfig.clientPassword,
 				'--server-settings', instancedirectory + '/server-settings.json',
-				'--port', Number(process.env.FACTORIOPORT) || instanceconfig.factorioPort
+				'--port', args.port || Number(process.env.FACTORIOPORT) || instanceconfig.factorioPort
 			], {
 				'stdio': ['pipe', 'pipe', 'pipe']
 			}
 		);
 
-		serverprocess.on('close', (code) => {
+		serverprocess.on('close', code => {
 			console.log(`child process exited with code ${code}`);
 			process.exit();
 		});
-		serverprocess.stdout.on("data", (data) => {
+		serverprocess.stdout.on("data", data => {
 			// log("Stdout: " + data);
 			if(data.toString('utf8').includes("Couldn't parse RCON data: Maximum payload size exceeded")){
 				console.error("ERROR: RCON CONNECTION BROKE DUE TO TOO LARGE PACKET!");
 				console.error("Attempting reconnect...");
-				client.close();
+				client.disconnect();
 				client.connect();
 			}
 			// we have to do this to make logs visible on linux and in powershell. Causes log duplication for people with CMD.
-			console.log('Fact: ' + data);
+			console.log('Fact: ' + data.toString("utf8").replace("\n", ""));
 		});
 		serverprocess.stderr.on('data', (chunk) => {
 			console.log('ERR: ' + chunk);
@@ -413,12 +499,12 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 			// we read the log every 2 seconds and stop looping when we start connecting to factorio
 			function checkRcon() {
 				fs.readFile(instancedirectory+"/factorio-current.log", function (err, data) {
-					// if (err) console.log(err);
+					// if (err) console.error(err);
 					if(data && data.indexOf('Starting RCON interface') > 0){
 						client.connect({
 							host: 'localhost',
-							port: Number(process.env.RCONPORT) || instanceconfig.clientPort,
+							port: args["rcon-port"] || Number(process.env.RCONPORT) || instanceconfig.clientPort,
-							password: instanceconfig.clientPassword,
+							password: args["rcon-password"] || instanceconfig.clientPassword,
 							timeout: 5000
 						});
 					} else {
@@ -432,7 +518,7 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 		
 			client.onDidAuthenticate(() => {
 				console.log('Clusterio | RCON Authenticated!');
-				instanceManagement(); // start using rcons
+				instanceManagement(instanceconfig); // start using rcons
 			});
 			client.onDidConnect(() => {
 				console.log('Clusterio | RCON Connected!');
@@ -440,71 +526,89 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 			});
 			client.onDidDisconnect(() => {
 				console.log('Clusterio | RCON Disconnected!');
-				process.exit(0); // exit because RCON disconnecting is undefined behaviour and we rather just wanna restart now
+				// process.exit(0); // exit because RCON disconnecting is undefined behaviour and we rather just wanna restart now
+			});
+			process.on('SIGINT', function () {
+				console.log("Caught interrupt signal, disconnecting RCON");
+				client.disconnect().then(()=>{
+					console.log("Rcon disconnected, Sending ^C");
+					// We don't actually do this on windows, because ctrl+c in windows CMD sends it to all subprocesses as well. Doing it twice will abort factorios save.
+					if(process.platform == "linux"){
+						serverprocess.kill("SIGINT");
+					}
+				});
 			});
 		} else if(process.platform == "linux"){
 			// don't open an RCON connection and just use stdio instead, does not work on windows.
-			instanceManagement();
+			instanceManagement(instanceconfig);
+			process.on('SIGINT', function () {
+				console.log("Caught interrupt signal, sending ^C");
+				serverprocess.kill("SIGINT");
+			});
 		}
 
 		// set some globals
 		confirmedOrders = [];
 		lastSignalCheck = Date.now();
 	});
+})()} else {
+	console.error("Invalid arguments, quitting.");
+	process.exit(1);
 }
 
 // ensure instancemanagement only ever runs once
 _.once(instanceManagement);
-function instanceManagement() {
+async function instanceManagement(instanceconfig) {
-	/* Open websocket connection to master */
+    console.log("Started instanceManagement();");
+
+    /* Open websocket connection to master */
 	var socket = ioClient("http://"+config.masterIP+":"+config.masterPort);
 	socket.on("hello", data => {
 		console.log("SOCKET | registering slave!");
 		socket.emit("registerSlave", {
 			instanceID: instanceconfig.unique,
 		});
-		setInterval(B=> socket.emit("heartbeat"), 10000);
 	});
+	setInterval(B=> socket.emit("heartbeat"), 10000);
 
-	console.log("Started instanceManagement();");
 	// load plugins and execute onLoad event
-	let pluginDirectories = fileOps.getDirectoriesSync("./sharedPlugins/");
+	const pluginManager = require("./lib/manager/pluginManager.js")(config);
+	let pluginsToLoad = await pluginManager.getPlugins();
 	let plugins = [];
-	for(let i=0; i<pluginDirectories.length; i++) {
+	
-		let I = i
+	for(let i = 0; i < pluginsToLoad.length; i++){
+		let pluginLoadStarted = Date.now();
 		let log = function(message) {
-			console.log("Clusterio | "+ pluginDirectories[I] + " | " + message);
+			console.log("Clusterio | "+ pluginsToLoad[i].name + " | " + message);
 		}
-		// these are our two config files. We need to send these in case plugin
+		let combinedConfig = deepmerge(instanceconfig,config,{clone:true});
-		// wants to contact master or know something.
+		let pluginConfig = pluginsToLoad[i];
-		let combinedConfig = deepmerge(instanceconfig,config,{clone:true})
-		let pluginConfig = require("./sharedPlugins/" + pluginDirectories[i] + "/config.js");
 		
 		if(!global.subscribedFiles) {
 			global.subscribedFiles = {};
 		}
-		if(pluginConfig.binary == "nodePackage"){
+		if(pluginConfig.binary == "nodePackage" && pluginConfig.enabled){
-			// require index.js.main() of plugin and execute it as a class
+			// require plugin class and execute it
-			let pluginClass = require("./sharedPlugins/" + pluginDirectories[I] + "/index.js");
+			let pluginClass = require(path.resolve(pluginConfig.pluginPath, "index.js"));
-			plugins[I] = new pluginClass(combinedConfig, async function(data, callback){
+			plugins[i] = new pluginClass(combinedConfig, async function(data, callback){
-				if(data.toString('utf8')[0] != "/") {
+				if(data && data.toString('utf8')[0] != "/") {
-					log("Stdout: " + data.toString('utf8'));
+					log(data.toString('utf8'));
 					return true;
-				} else {
+				} else if (data && data.toString('utf8')[0] == "/"){
 					return messageInterface(data.toString('utf8'), callback);
 				}
 			}, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat.
 				socket, // socket.io connection to master (and ES6 destructuring, yay)
 			});
-			if(plugins[I].factorioOutput && typeof plugins[I].factorioOutput === "function"){
+			if(plugins[i].factorioOutput && typeof plugins[i].factorioOutput === "function"){
 				// when factorio logs a line, send it to the plugin. This includes things like autosaves, chat, errors etc
-				serverprocess.stdout.on("data", data => plugins[I].factorioOutput(data.toString()));
+				serverprocess.stdout.on("data", data => plugins[i].factorioOutput(data.toString()));
 			}
 			if(pluginConfig.scriptOutputFileSubscription && typeof pluginConfig.scriptOutputFileSubscription == "string"){
 				if(global.subscribedFiles[pluginConfig.scriptOutputFileSubscription]) {
 					// please choose a unique file to subscribe to. If you need plugins to share this interface, set up a direct communication
 					// between those plugins instead.
-					throw "FATAL ERROR IN " + pluginDirectories[i] + " FILE ALREADY SUBSCRIBED " + pluginConfig.scriptOutputFileSubscription;
+					throw "FATAL ERROR IN " + pluginConfig.name + " FILE ALREADY SUBSCRIBED " + pluginConfig.scriptOutputFileSubscription;
 				}
 				
 				if (!fs.existsSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription)) {
@@ -513,10 +617,13 @@ function instanceManagement() {
 				}
 				global.subscribedFiles[pluginConfig.scriptOutputFileSubscription] = true;
 				console.log("Clusterio | Registered file subscription on "+instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription);
+				
 				fs.watch(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, fileChangeHandler);
 				// run once in case a plugin wrote out information before the plugin loaded fully
-				fileChangeHandler(false, pluginConfig.scriptOutputFileSubscription);
+				// delay, so the socket got enough time to connect
-				
+				setTimeout(()=> {
+                    fileChangeHandler(false, pluginConfig.scriptOutputFileSubscription);
+                }, 500);
 				// send file contents to plugin for processing
 				function fileChangeHandler(eventType, filename) {
 					if(filename != null){
@@ -529,11 +636,13 @@ function instanceManagement() {
 								if (stuff[0]) {
 									fs.writeFileSync(instancedirectory + "/script-output/" + filename, "");
 								}
-								for(let i = 0; i < stuff.length; i++) {
+								for(let o = 0; o < stuff.length; o++) {
-									if(stuff[i] && !stuff[i].includes('\u0000\u0000')) {
+									if(stuff[o] && !stuff[o].includes('\u0000\u0000')) {
 										try{
-											plugins[I].scriptOutput(stuff[i]);
+											plugins[i].scriptOutput(stuff[o]);
-										}catch(e){console.log(e)}
+										}catch(e){
+											throw new Error(`Plugin ${pluginConfig.name} has a fileOutputSubscription but plugin.scriptOutput is not a function`)
+										}
 									}
 								}
 							},
@@ -542,70 +651,9 @@ function instanceManagement() {
 					}
 				}
 			}
-			console.log("Clusterio | Loaded plugin " + pluginDirectories[i]);
+			console.log(`Clusterio | Loaded plugin ${pluginsToLoad[i].name} in ${Date.now() - pluginLoadStarted}ms`);
-		} else if(pluginConfig.binary != "nodePackage" && pluginConfig.binary){
-			// handle as fragile executable plugin
-			let args = pluginConfig.args || [];
-			plugins[I]=child_process.spawn(pluginConfig.binary, args, {
-				cwd: "./sharedPlugins/"+pluginDirectories[i],
-				stdio: ['pipe', 'pipe', 'pipe'],
-			});
-			
-			/*
-				to send to stdin, use:
-				spawn.stdin.write("text\n");
-			*/
-			// If plugin has subscribed to a file, send any text appearing in that file to stdin
-			if(pluginConfig.scriptOutputFileSubscription && typeof pluginConfig.scriptOutputFileSubscription == "string") {
-				if(global.subscribedFiles[pluginConfig.scriptOutputFileSubscription]) {
-					// please choose a unique file to subscribe to. If you need plugins to share this interface, set up a direct communication
-					// between those plugins instead.
-					throw "FATAL ERROR IN " + pluginDirectories[i] + " FILE ALREADY SUBSCRIBED " + pluginConfig.scriptOutputFileSubscription;
-				}
-				
-				if (!fs.existsSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription)) {
-					// Do something
-					fs.writeFileSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, "");
-				}
-				global.subscribedFiles[pluginConfig.scriptOutputFileSubscription] = true;
-				fs.watch(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, function (eventType, filename) {
-					// get array of lines in file
-					// TODO async
-					let stuff = fs.readFileSync(instancedirectory + "/script-output/" + filename, "utf8").split("\n");
-					// if you found anything, reset the file
-					if (stuff[0]) {
-						fs.writeFileSync(instancedirectory + "/script-output/" + filename, "");
-					}
-					for (let i = 0; i < stuff.length; i++) {
-						if (stuff[i]) {
-							plugins[I].stdin.write(stuff[i]);
-						}
-					}
-				});
-			}
-			// these are our two config files. We need to send these in case plugin
-			// wants to contact master or know something.
-			// send through script-output file, maybe more compat?
-			fs.writeFileSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, JSON.stringify(combinedConfig)+"\r\n");
-			// send directly through stdin
-			// plugins[i].stdin.write(JSON.stringify(combinedConfig)+"\n");
-			
-			console.log("Clusterio | Loaded plugin " + pluginDirectories[i]);
-			plugins[i].stdout.on("data", (data) => {
-				if(data.toString('utf8')[0] != "/") {
-					log("Stdout: " + data.toString('utf8'))
-				} else {
-					messageInterface(data.toString('utf8'));
-				}
-			});
-			plugins[i].stderr.on("data", (data) => {
-				log("STDERR: " + data);
-			});
-			plugins[i].on('close', (code) => {
-				log(`child process exited with code ${code}`);
-			});
 		} else {
-			// This plugin doesn't have a client portion
+			// this plugin doesn't have a client portion. Maybe it runs on the master only?
 		}
 	}
 
@@ -624,7 +672,11 @@ function instanceManagement() {
 					publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config
 					mods:modHashes,
 					instanceName: instance,
-					playerCount:playerCount.replace(/(\r\n\t|\n|\r\t)/gm, ""),
+				}
+				if(playerCount){
+					payload.playerCount = playerCount.replace(/(\r\n\t|\n|\r\t)/gm, "");
+				} else {
+					payload.playerCount = 0;
 				}
 				
 				function callback(err, mac) {
@@ -632,15 +684,19 @@ function instanceManagement() {
 						mac = "unknown";
 						console.log("##### getMac crashed, but we don't really give a shit because we are probably closing down #####");
 					}
+					global.mac = mac;
 					payload.mac = mac;
-					console.log("Registered our precense with master "+config.masterIP+" at " + payload.time);
+					// console.log("Registered our presence with master "+config.masterIP+" at " + payload.time);
 					needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) {
 						if (err && err.code != "ECONNRESET"){
-							console.error("We got problems, something went wrong when contacting master");
+                            console.error("We got problems, something went wrong when contacting master"+config.masterIP+" at " + payload.time);
 							console.error(err);
 						} else if (response && response.body) {
 							// In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care.
-							console.log(response.body);
+							if(response.body !== "ok") {
+                                console.log("Got no \"ok\" while registering our precense with master "+config.masterIP+" at " + payload.time);
+                                console.log(response.body);
+                            }
 						}
 					});
 				}
@@ -665,7 +721,7 @@ function instanceManagement() {
 				hash: modHashes[i].hash,
 			}
 			needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) {
-				if(err) console.log(new Error("Unable to contact master server! Please check your config.json."));
+				if(err) console.error("Unable to contact master server /api/checkMod! Please check your config.json.");
 				if(response && body && body == "found") {
 					console.log("master has mod "+modHashes[i].modName);
 				} else if (response && body && typeof body == "string") {
@@ -680,7 +736,7 @@ function instanceManagement() {
 							},
 						}, function (err, resp, body) {
 							if (err) {
-								console.log(new Error("Unable to contact master server! Please check your config.json."));
+								console.error(new Error("Unable to contact master server /api/uploadMod! Please check your config.json."));
 							} else {
 								console.log('URL: ' + body);
 							}
@@ -694,262 +750,6 @@ function instanceManagement() {
 			});
 		}
 	}
-	
-	// flow/production statistics ------------------------------------------------------------
-	oldFlowStats = false
-	setInterval(function(){
-		fs.readFile(instancedirectory + "/script-output/flows.txt", {encoding: "utf8"}, function(err, data) {
-			if(!err && data) {
-				let timestamp = Date.now();
-				data = data.split("\n");
-				let flowStats = [];
-				for(let i = 0; i < data.length; i++) {
-					// try catch to remove any invalid json
-					try{
-						flowStats[flowStats.length] = JSON.parse(data[i]);
-					} catch (e) {
-						// console.log(" invalid json: " + i);
-						// some lines of JSON are invalid but don't worry, we just filter em out
-					}
-				}
-				// fluids
-				let flowStat1 = flowStats[flowStats.length-1].flows.player.input_counts
-				// items
-				let flowStat2 = flowStats[flowStats.length-2].flows.player.input_counts
-				// merge fluid and item flows
-				let totalFlows = {};
-				for(let key in flowStat1) totalFlows[key] = flowStat1[key];
-				for(let key in flowStat2) totalFlows[key] = flowStat2[key];
-				if(oldFlowStats && totalFlows && oldTimestamp) {
-					let payload = objectOps.deepclone(totalFlows);
-					// change from total reported to per time unit
-					for(let key in oldFlowStats) {
-						// get production per minute
-						payload[key] = Math.floor((payload[key] - oldFlowStats[key])/(timestamp - oldTimestamp)*60000);
-						if(payload[key] < 0) {
-							payload[key] = 0;
-						}
-					}
-					for(let key in payload) {
-						if(payload[key] == '0') {
-							delete payload[key];
-						}
-					}
-					console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]);
-					needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: instanceconfig.unique,data: payload}, needleOptionsWithTokenAuthHeader, function (err, response, body) {
-						// we did it, keep going
-					});
-				}
-				oldTimestamp = timestamp;
-				oldFlowStats = totalFlows;
-				fs.writeFileSync(instancedirectory + "/script-output/flows.txt", "");
-			}
-		});
-		// we don't need to update stats quickly as that could be expensive
-	}, 60000*5);
-	
-	// provide items --------------------------------------------------------------
-	// trigger when something happens to output.txt
-	fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) {
-		// get array of lines in file
-		let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n");
-		// if you found anything, reset the file
-		if (items[0]) {
-			fs.writeFileSync(instancedirectory + "/script-output/output.txt", "");
-		}
-		for (let i = 0; i < items.length; i++) {
-			if (items[i]) {
-				let g = items[i].split(" ");
-				g[0] = g[0].replace("\u0000", "");
-				// console.log("exporting " + JSON.stringify(g));
-				// send our entity and count to the master for him to keep track of
-				needle.post(config.masterIP + ":" + config.masterPort + '/api/place', {
-					name: g[0],
-					count: g[1],
-					instanceName: instance, // name of instance
-					instanceID: instanceconfig.unique, // a hash computed from the randomly generated rcon password
-				}, needleOptionsWithTokenAuthHeader, function (err, resp, body) {
-					if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]);
-					if(config.logItemTransfers){
-						if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`);
-					}
-				});
-			}
-		}
-	});
-	// request items --------------------------------------------------------------
-	setInterval(function () {
-		// get array of lines in file
-		let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n");
-		// if we actually got anything from the file, proceed and reset file
-		if (items[0]) {
-			fs.writeFileSync(instancedirectory + "/script-output/orders.txt", "");
-			// prepare a package of all our requested items in a more tranfer friendly format
-			var preparedPackage = {};
-			for (let i = 0; i < items.length; i++) {
-				(function (i) {
-					if (items[i]) {
-						items[i] = items[i].split(" ");
-						items[i][0] = items[i][0].replace("\u0000", "");
-						items[i][0] = items[i][0].replace(",", "");
-						if (preparedPackage[items[i][0]]) {
-							// if we have buffered some already, sum the new items
-							if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") {
-								preparedPackage[items[i][0]] = {
-									"name": items[i][0],
-									"count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]),
-									"instanceName":instance,
-									"instanceID":instanceconfig.unique,
-								};
-							// else just add em in without summing
-							} else if (typeof Number(items[i][1]) == "number") {
-								preparedPackage[items[i][0]] = {
-									"name": items[i][0],
-									"count": Number(items[i][1]),
-									"instanceName":instance,
-									"instanceID":instanceconfig.unique,
-								};
-							}
-						// this condition will NEVER be triggered but we know how that goes
-						} else if (typeof Number(items[i][1]) == "number") {
-							preparedPackage[items[i][0]] = {
-								"name": items[i][0],
-								"count": Number(items[i][1]),
-								"instanceName":instance,
-								"instanceID":instanceconfig.unique,
-							};
-						}
-					}
-				})(i);
-			}
-			// request our items, one item at a time
-			for (let i = 0; i < Object.keys(preparedPackage).length; i++) {
-				// console.log(preparedPackage[Object.keys(preparedPackage)[i]]);
-				needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) {
-					if (response && response.body && typeof response.body == "object") {
-						// buffer confirmed orders
-						confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count}
-						if(config.logItemTransfers){
-							console.log(`Imported ${response.body.count} ${response.body.name} from master`);
-						}
-					}
-				});
-			}
-			// if we got some confirmed orders
-			// console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders));
-			//if (!(confirmedOrders.length>0)){return;}
-			let cmd="local t={";
-			for(let i=0;i<confirmedOrders.length;i++)
-			{
-			    cmd+='["'+confirmedOrders[i].name+'"]='+confirmedOrders[i].count+',';
-			    if(cmd.length>320) // Factorio max packet size is 508
-			    {
-			        messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
-			        cmd="local t={";
-			    }
-			}
-			if (!(cmd==="local t={")){
-				messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
-			}
-			confirmedOrders=[];
-		}
-	}, 1000);
-	// COMBINATOR SIGNALS ---------------------------------------------------------
-	// get inventory from Master and RCON it to our slave
-	setInterval(function () {
-		needle.get(config.masterIP + ":" + config.masterPort + '/api/inventory', function (err, response, body) {
-			if(err){
-				console.log("Unable to get JSON master/api/inventory, master might be unaccessible");
-			} else if (response && response.body) {
-				// Take the inventory we (hopefully) got and turn it into the format LUA accepts
-				if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver
-				try {
-					var inventory = JSON.parse(response.body);
-					var inventoryFrame = {};
-					for (let i = 0; i < inventory.length; i++) {
-						inventoryFrame[inventory[i].name] = Number(inventory[i].count);
-						if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){
-							inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice
-						}
-					}
-					inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000);
-					// console.log("RCONing inventory! " + JSON.stringify(inventoryFrame));
-					let first = true;
-					let cmd="local s={";
-					for (let key in inventoryFrame)
-					{
-						cmd+='["'+key+'"]='+inventoryFrame[key]+",";
-						if(first && cmd.length>300 || !first && cmd.length > 320) // Factorio max packet size is 508
-						{
-					       		messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
-					       		cmd="local s={";
-					       		first = false;
-						}
-					}
-					if (!(cmd==="local s={")){
-						messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
-					}
-					messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')");
-				} catch (e){
-					console.log(e);
-				}
-			}
-		});
-	}, 1550);
-	// Make sure world has its worldID
-	setTimeout(function(){
-		messageInterface("/silent-command remote.call('clusterio','setWorldID',"+instanceconfig.unique+")")
-	}, 20000);
-	/* REMOTE SIGNALLING
-	 * send any signals the slave has been told to send
-	 * Fetch combinator signals from the server
-	*/
-	socket.on("processCombinatorSignal", circuitFrameWithMeta => {
-		if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object" && circuitFrameWithMeta.frame && Array.isArray(circuitFrameWithMeta.frame)){
-			messageInterface("/silent-command remote.call('clusterio', 'receiveFrame', '"+JSON.stringify(circuitFrameWithMeta.frame)+"')");
-		}
-	});
-	// get outbound frames from file and send to master
-	// get array of lines in file, each line should correspond to a JSON encoded frame
-	let signals = fs.readFileSync(instancedirectory + "/script-output/txbuffer.txt", "utf8").split("\n");
-	// if we actually got anything from the file, proceed and reset file
-	let readingTxBufferSoon = false;
-	let txBufferClearCounter = 0;
-	fs.watch(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (eventType, filename) => {
-		if(!readingTxBufferSoon){ // use a 100ms delay to avoid messing with rapid sequential writes from factorio (I think that might be a problem maybe?)
-			readingTxBufferSoon = true;
-			setTimeout(()=>{
-				txBufferClearCounter++;
-				fs.readFile(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (err, signals) => {
-					signals = signals.split("\n");
-					if (signals[0]) {
-						//if(txBufferClearCounter > 500){
-							fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", "");
-						//	txBufferClearCounter = 0;
-						//}
-						
-						// loop through all our frames
-						for (let i = 0; i < signals.length; i++) {
-							if (signals[i] && objectOps.isJSON(signals[i])) {
-								// signals[i] is a JSON array called a "frame" of signals. We timestamp it for storage on master
-								// then we unpack and RCON in this.frame to the game later.
-								let framepart = JSON.parse(signals[i]);
-								let doneframe = {
-									time: Date.now(),
-									frame: framepart, // thats our array of objects(single signals);
-								}
-								// send to master using socket.io, opened at the top of instanceManagement()
-								socket.emit("combinatorSignal", doneframe);
-							} else {
-								// console.log("Invalid jsony: "+typeof signals[i])
-							}
-						}
-					}
-				});
-				readingTxBufferSoon = false;
-			},100);
-		}
-	});
 } // END OF INSTANCE START ---------------------------------------------------------------------
 
 // string, function

cluster_rcon.js

@@ -0,0 +1,33 @@
+const needle = require("needle");
+const config = require("./config.json");
+
+const masterIP = `${config.masterIP}:${config.masterPort}`;
+const token = config.masterAuthToken
+let commandArray = process.argv;
+commandArray.shift();
+commandArray.shift();
+const needleOptionsWithTokenAuthHeader = {
+	headers: {
+		'x-access-token': token,
+	},
+};
+
+let command = commandArray.join(" ");
+let killBiters = `/silent-command local surface=game.surfaces[1] for key, entity in pairs(surface.find_entities_filtered({force="enemy"})) do entity.destroy() end`;
+
+(async ()=>{
+	needle.post(masterIP+"/api/runCommand", {
+		broadcast:true,
+		command,
+	}, needleOptionsWithTokenAuthHeader, (err, resp) => {
+		console.log(resp.body)
+	});
+	// setInterval(()=>{
+		// needle.post(masterIP+"/api/runCommand", {
+		// broadcast:true,
+		// command: killBiters,
+	// }, needleOptionsWithTokenAuthHeader, (err, resp) => {
+		// console.log(resp.body)
+	// });
+	// }, 1000*60*5);
+})();

config.json.dist

@@ -1,7 +1,8 @@
 {
 	"__comment": "Where factorio folder. Should be in the same folder as this config file.",
-	"factorioDirectory": "factorio",
+	"factorioDirectory": "./factorio",
-	"instanceDirectory": "instances",
+	"instanceDirectory": "./instances",
+	"databaseDirectory": "./database",
 	
 	"__comment1": "IP players should connect to to join your servers, should be portforwarded and stuff",
 	"publicIP": "localhost",
@@ -9,10 +10,10 @@
 	"__comment2": "IP the master web server runs at, can be ignored on the master system",
 	"masterIP": "localhost",
 	
-	"__commentMasterPort": "HTTP port for the master to listen to, set to 0 or an empty string to not listen for HTTP requests",
+	"__commentMasterPort": "HTTP port for the master to listen to",
 	"masterPort": 8080,
 	
-	"__commentsslPort": "HTTPS port for the master to listen to. Set to 0 or an empty string to not use HTTPS.",
+	"__commentsslPort": "HTTPS port for the master to listen to. Set to 0 or an empty string to not listen for HTTPS.",
 	"sslPort": 443,
 	"sslCert": "database/certificates/cert.crt",
 	"sslPrivKey": "database/certificates/cert.key",
@@ -46,7 +47,17 @@
 		"entriesPerSecond": 0.01666666667
 	},
 	"logItemTransfers": false,
+	"__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.",
+	"disableFairItemDistribution": false,
+	"useNeuralNetDoleDivider": false,
 	"uploadModsToMaster": true,
 	"msBetweenCommands": 10,
-	"allowRemoteCommandExecution": true
+	"allowRemoteCommandExecution": true,
+	"enableCrossServerShout": true,
+	"mirrorAllChat": false,
+	"__comment7": "Autosave interval for items in the master storage in milliseconds. Defaults to one minute when set to 0 or not set.",
+	"autosaveInterval": 600000,
+	"disablePrometheusPushgateway": false,
+	"__comment8": "This setting only affects the master. It makes every import for things that arent 'electricity' return a maximum of 0. Effectively disables teleport chests.",
+	"disableImportsOfEverythingExceptElectricity": false
 }

lib/authenticate.js

@@ -1,15 +1,28 @@
 const jwt = require('jsonwebtoken');
 
-const config = require("./../config");
+// const config = require("./../config");
 
-var middleware = function (req, res, next) {
+module.exports = function(config){
-	var token = req.headers['x-access-token'];
+	return {
-	if(!token) return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path});
+		middleware: function (req, res, next) {
+			var token = req.headers['x-access-token'];
+			if(!token) return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path});
 			
-	jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
+			jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
-		if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path});
+				if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path});
-		next();
+				next();
-	});
+			});
-}
+		},
-
+		check: async function(token){
-module.exports = {middleware};
+			return new Promise((resolve, reject) => {
+				jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
+					if(err){
+						resolve({ok:false, msg:err})
+					} else {
+						resolve({ok:true, msg:"Successfully authenticated"});
+					}
+				});
+			});
+		}
+	}
+};

lib/dole_nn_base.js

@@ -0,0 +1,118 @@
+//Sigmoids...
+
+function sigmoid0(x)
+{
+	return x/(1+Math.abs(x));
+}
+function sigmoid1(x)
+{
+	if (x>=0.0) {
+		return 1.0
+	} else {
+		return 0.0
+	}
+}
+function sigmoid2(x)
+{
+	if (x>=0.0){
+		return x
+	} else {
+		return 0.0
+	}
+}
+function sigmoid3(x)
+{
+	return x;
+}
+
+//DOSE - Doses items for item X slave Y
+//Numreq - number of items that are requested for item X for slave Y
+//Instore - number of items for item X in master gauge (currently)
+//store_last_tick - Number of items for item X in master gauge at time of last tick
+//dole - Current dole for item X
+//carry - carry for item X slave Y
+//prev_req - Numreq for previous request for item X slave Y
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+
+//Returns: Array of 3 elements
+//0.Number of items to give in that dose
+//1.New dole for item X
+//2.New carry for item X slave Y
+function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj)
+{
+	numreq=Number(numreq);
+	instore=Number(instore);
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	
+	var b=Math.trunc(prev_req/instore_adj*numreq);
+	var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
+	var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b];//5..9
+	var inputs=inp1.concat(inp2);
+	var internal=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0];
+	var outputs=[0,0,0];
+	//NN: Please don't touch :), code is generated
+	
+outputs[0]=inputs[2]*3.101320028;
+outputs[0]=outputs[0]+inputs[7]*0.5471593142;
+outputs[0]=outputs[0]+inputs[3]*2.691268921;
+outputs[0]=outputs[0]+outputs[1]*-1.64255929;
+outputs[0]=outputs[0]+inputs[6]*0.7746122479;
+outputs[0]=sigmoid0(outputs[0]);
+outputs[1]=inputs[2]*-0.9946503043;
+outputs[1]=outputs[1]+inputs[1]*0.2128598392;
+outputs[1]=sigmoid0(outputs[1]);
+outputs[2]=sigmoid2(outputs[2]);
+
+	outputs[0]=Number(outputs[0]);
+	if (outputs[0]<0) outputs[0]=0;
+	outputs[0]=outputs[0]*numreq;
+	var res1=Math.round(outputs[0]);
+	if (res1>numreq) res1=numreq;
+	if (res1>instore) res1=instore;
+	if (res1<0) res1=0;
+	carry=sigmoid0(outputs[2]);
+	return [res1,sigmoid0(outputs[1]),carry];
+}
+
+//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
+//Instore - number of items for item X in master gauge (currently)
+//dole - Current dole for item X
+//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+
+
+//Returns: Array of 2 elements
+//0.New dole for item X
+//1.Guessed percentage average of items supplied in last 10 ticks
+function Tick(instore,dole,store_last_tick,numreq_total_adj)
+{
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj];
+	var internal=[0,0,0,0,0];
+	var outputs=[0,0];
+	//NN: Please don't touch :), code is generated
+	
+internal[2]=sigmoid0(internal[2]);
+outputs[0]=internal[2]*0.4049594998;
+outputs[0]=outputs[0]+1.0*-5.026565075;
+outputs[0]=outputs[0]+inputs[0]*0.9100194573;
+outputs[0]=outputs[0]+inputs[1]*-1.063795686;
+outputs[0]=sigmoid1(outputs[0]);
+outputs[1]=outputs[0]*1.049500823;
+outputs[1]=outputs[1]*inputs[0]*3.437163115;
+outputs[1]=sigmoid0(outputs[1]);
+
+
+
+	avg=outputs[1];
+	if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
+	if (avg>1) avg=1;
+	if (avg<0) avg=0;
+	
+	return [sigmoid0(outputs[0]),avg];
+}
+
+module.exports = {Dose,Tick}

lib/getInstances.js

@@ -1,15 +1,17 @@
 const fileOps = require("_app/fileOps");
-const config = require("./../config");
+// const config = require("./../config");
 
-module.exports = {
+module.exports = function(config) {
-	sync: function(){
+	return {
-		let instances;
+		sync: function(){
-		try {
+			let instances;
-			instances = fileOps.getDirectoriesSync(config.instanceDirectory);
+			try {
-		} catch (e) {
+				instances = fileOps.getDirectoriesSync(config.instanceDirectory);
-			// there are no instances created yet, probably missing the folder
+			} catch (e) {
-			instances = [];
+				// there are no instances created yet, probably missing the folder
-		}
+				instances = [];
-		return instances;
+			}
+			return instances;
+		},
 	}
 }

lib/manager/configManager.js

@@ -15,6 +15,9 @@ console.log()s each entry in the config
 @param {string} instance Either undefined, "shared" or the instance to show the config for.
 */
 module.exports.displayConfig = function(instance){
+	console.log(asTable(this.getConfig(instance)));
+}
+module.exports.getConfig = async function(instance){
 	if(!instance || instance == "shared"){
 		let configArray = [];
 		Object.keys(config).forEach(key=>{
@@ -24,7 +27,7 @@ module.exports.displayConfig = function(instance){
 				} else configArray.push({entry: key, value: config[key]});
 			}
 		});
-		console.log(asTable(configArray));
+		return configArray;
 	}
 }
 module.exports.editConfig = function(instance, entry, newValue){
@@ -33,16 +36,34 @@ module.exports.editConfig = function(instance, entry, newValue){
 			if(objectOps.isJSON(newValue)){
 				config[entry] = JSON.parse(newValue);
 				console.log("Changed config entry "+entry+" from "+JSON.stringify(config[entry])+" to "+JSON.stringify(newValue));
-				console.log("new config file:");
+				// console.log("new config file:");
-				console.log(JSON.stringify(config, null, "	"))
+				// console.log(JSON.stringify(config, null, "	"))
 				fs.writeFileSync("config.json", JSON.stringify(config, null, 4));
+				return {
+					ok:true,
+					msg:`Changed config entry ${entry} from Object to ${JSON.stringify(newValue)}`,
+				}
 			} else {
 				config[entry] = newValue;
 				console.log("Changed config entry "+entry+" from "+config[entry]+" to '"+newValue+"'");
-				console.log("new config file:");
+				// console.log("new config file:");
-				console.log(JSON.stringify(config, null, "	"))
+				// console.log(JSON.stringify(config, null, "	"))
 				fs.writeFileSync("config.json", JSON.stringify(config, null, 4));
+				return {
+					ok:true,
+					msg:`Changed config entry ${entry} from ***** to '${newValue}'`,
+				};
 			}
+		} else {
+			return {
+				ok:false,
+				msg:`No such config parameter`,
+			}
+		}
+	} else {
+		return {
+			ok:false,
+			msg:`Editing config for single instances is not yet supported`,
 		}
 	}
 };

lib/manager/instanceManager.js

@@ -0,0 +1,2 @@
+const fs = require("fs-extra");
+const needle = require("needle");

lib/manager/modManager.js

@@ -7,16 +7,15 @@ const rightpad = require("right-pad");
 const asTable = require("as-table").configure({delimiter: ' | '});
 const moment = require("moment");
 const syncRequest = require("sync-request");
-const fs = require("fs");
+const fs = require("fs-extra");
 const https = require('follow-redirects').https;
 const factorioAPI = require("node-factorio-api");
 const sLog = require("single-line-log").stdout;
 const query = require('cli-interact').getYesNo;
 
 const fileOps = require("_app/fileOps");
-const config = require("./../../config");
+// const config = require("./../../config");
 
-const instances = require("./../getInstances.js").sync();
 
 function ellipse(str, max){
    return str.length > (max - 3) ? str.substring(0,max-3) + '...' : str;
@@ -24,242 +23,248 @@ function ellipse(str, max){
 // https://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
 function formatBytes(a,b){if(0==a)return"0 Bytes";var c=1024,d=b||2,e=["Bytes","KB","MB","GB","TB","PB","EB","ZB","YB"],f=Math.floor(Math.log(a)/Math.log(c));return parseFloat((a/Math.pow(c,f)).toFixed(d))+" "+e[f]};
 
-/**
-Lists mods for an instance as a table. Warning, handles printing internally.
-@param {string} instance - Name of the instance or "shared"
 
-@example
+module.exports = function(config){
-var modManager = require("modManager");
+	if(!config) config = require("./../../config");
-modManager.listMods("testInstance");
+	const instances = require("./../getInstances.js")(config).sync();
-// console.logs table of mods installed on instance
+	return {
-modManager.listMods("shared");
+		/**
-// console.logs mods located in /sharedMods as a nice table
+		Lists mods for an instance as a table. Warning, handles printing internally.
-*/
+		@param {string} instance - Name of the instance or "shared"
-module.exports.listMods = function(instance){
-	return new Promise((resolve, reject) => {
-		if(instance && instances.includes(instance)){
-			factorioAPI.init(false, config.instanceDirectory+'/'+instance+'/instanceMods', config.instanceDirectory+'/'+instance+'/saves', config.factorio_version);
-			factorioAPI.authenticate({
-				username: config.username,
-				token: config.token,
-				require_ownership: false
-			}).then(token => {
-				console.log("* Currently installed:");
-				factorioAPI.readModZips().then(list => {
-					console.log(asTable(list.map(mod => {
-						return {
-							name:(mod.title || mod.name),
-							version:mod.version,
-							date:mod.date,
-							factorio_ver:mod.factorio_version,
-							author:mod.author,
-						}
-					})));
-				});
-				// Had issues earlier due to bug in node-factorio-api, see https://github.com/Danacus/node-factorio-api/issues/2
-				// bug is resolved, thanks to @Danacus and @brotherpot
-				factorioAPI.getModsFromSaves().then(list => {
-					console.log("* Mods used in save:");
-					console.log(asTable(list[0].mods))
-				});
-			}).catch(reason => {
-				console.log("Failed to authenticate to factorio.com! Please check your connection and the username+token used for authentication in config.json");
-				resolve();
-			});
-		} else {
-			// list shared mods in /sharedMods/
-			factorioAPI.init(false, 'sharedMods/', '', config.factorio_version);
-			factorioAPI.authenticate({
-				username: config.username,
-				token: config.token,
-				require_ownership: false
-			}).then(() => {
-				console.log("\r\n")
-				console.log("* Mods shared by all instances:")
-				factorioAPI.getInstalledMods().then(mods => {
-					console.log(asTable(mods));
-					resolve();
-				});
-			}).catch(e => {
-				reject(new Error(e));
-			});
-		}
 
-		// get JSON data about GH releases
+		@example
-		/*let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+		var modManager = require("modManager")(config);
-		let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+		modManager.listMods("testInstance");
-		let name = JSON.parse(res.getBody())[0].assets[0].name;
+		// console.logs table of mods installed on instance
-		console.log(JSON.parse(res.getBody()))
+		modManager.listMods("shared");
+		// console.logs mods located in /sharedMods as a nice table
 		*/
-	});
+		listMods: function(instance){
-}
+			return new Promise((resolve, reject) => {
-/**
+				if(instance && instances.includes(instance)){
-Searches mods.factorio.com for mods matching a search term and prints
+					factorioAPI.init(false, config.instanceDirectory+'/'+instance+'/instanceMods', config.instanceDirectory+'/'+instance+'/saves', config.factorio_version);
-the results to console as a neat looking table
+					factorioAPI.authenticate({
-
+						username: config.username,
-@param {string} searchTerm What to search for
+						token: config.token,
-*/
+						require_ownership: false
-module.exports.findMods = function(searchTerm){
+					}).then(token => {
-	return new Promise((resolve, reject) => {
+						console.log("* Currently installed:");
-		factorioAPI.init(false);
+						factorioAPI.readModZips().then(list => {
-		factorioAPI.authenticate({
+							console.log(asTable(list.map(mod => {
-			username: config.username,
+								return {
-			token: config.token,
+									name:(mod.title || mod.name),
-			require_ownership: false
+									version:mod.version,
-		}).then(token => {
+									date:mod.date,
-			factorioAPI.searchMods(searchTerm).then(mods => {
+									factorio_ver:mod.factorio_version,
-				let result = asTable(mods.reverse().map(mod => {
+									author:mod.author,
-					return {
+								}
-						title:ellipse(mod.title, 50),
+							})));
-						name:ellipse(mod.name, 30),
-						downloads:mod.downloads_count,
-						updated: moment(mod.updated_at).format("DD.MM.YY"),
-						latest_version: mod.latest_release.version,
-						owner:mod.owner,
-						//license: mod.license_name,
-					}
-				}));
-				resolve(result);
-				// console.log(result);
-			});
-		}).catch(e => {
-			if(e.contains("Insufficient information")) console.log("We are missing some parameters required for logging in! Make sure your factorio username and token are specified in config.json.");
-			reject(new Error(e));
-		});
-	});
-}
-
-module.exports.addMod = function(term, instance){
-	return new Promise((resolve, reject) => {
-		if(instance == "shared"){ // modDirectory has to end in /
-			var modDirectory = "sharedMods/";
-		} else {
-		var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
-		}
-		if(term && term.length > 0){
-			if(term.includes("mods.factorio.com/mods/")){ // https://mods.factorio.com/mods/Danielv123/ChooChoo
-				// treat it as a link download
-				factorioAPI.init(false, modDirectory, "", config.factorio_version);
-				factorioAPI.authenticate({
-					username: config.username,
-					token: config.token,
-					require_ownership: false
-				}).then(() => {
-					factorioAPI.searchMods(term.split("/")[term.split("/").length-1]).then(body => {
-						console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
-						factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
-							console.log("Mod downloaded");
-							resolve();
 						});
+						// Had issues earlier due to bug in node-factorio-api, see https://github.com/Danacus/node-factorio-api/issues/2
+						// bug is resolved, thanks to @Danacus and @brotherpot
+						factorioAPI.getModsFromSaves().then(list => {
+							console.log("* Mods used in save:");
+							console.log(asTable(list[0].mods))
+						});
+					}).catch(reason => {
+						console.log("Failed to authenticate to factorio.com! Please check your connection and the username+token used for authentication in config.json");
+						resolve();
 					});
-				});
+				} else {
-			} else if(term.includes("github.com")){ // https://github.com/Danielv123/factorioClusterioMod
+					// list shared mods in /sharedMods/
-				// treat it as a github releases download
+					factorioAPI.init(false, 'sharedMods/', '', config.factorio_version);
-				let apiURL = "https://api.github.com/repos/" + term.split("/")[3] + "/" + term.split("/")[4] + "/releases"; // https://api.github.com/repos/Danielv123/factorioClusterioMod/releases
+					factorioAPI.authenticate({
-				
+						username: config.username,
-				let res = syncRequest('GET', apiURL, {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+						token: config.token,
-				let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+						require_ownership: false
-				let name = JSON.parse(res.getBody())[0].assets[0].name;
+					}).then(() => {
-				if(url) {
+						console.log("\r\n")
-					console.log("Downloading " + url);
+						console.log("* Mods shared by all instances:")
-					let file = fs.createWriteStream(modDirectory + name);
+						factorioAPI.getInstalledMods().then(mods => {
-					let bytesDownloaded = 0;
+							console.log(asTable(mods));
-					let downloadStartTime = Date.now();
-					https.get(url, function(response) {
-						let bytesToDownload = response.headers['content-length'];
-						response.pipe(file);
-						response.on("data", chunk => {
-							bytesDownloaded += chunk.length;
-							let downloadTimeElapsed = Date.now() - downloadStartTime;
-							let bytesPerSecond = bytesDownloaded / (downloadTimeElapsed/1000);
-							let timeToComplete = Math.floor((bytesToDownload - bytesDownloaded)/bytesPerSecond);
-							sLog("Downloading "+name+" "+(bytesDownloaded/bytesToDownload*100).toPrecision(3)+"% ("+formatBytes(bytesDownloaded)+" / "+formatBytes(bytesToDownload)+")\r\n"+
-							"about "+timeToComplete+" seconds remaining ("+formatBytes(bytesPerSecond)+"/s)");
-						}).on("end", x => {
-							sLog.clear();
-							console.log(" Downloaded in "+moment.utc(moment.duration(Date.now()-downloadStartTime).as('milliseconds')).format('mm:ss')); // because sLog.clear() doesn't seem to do the job of ending the line properly
 							resolve();
 						});
+					}).catch(e => {
+						reject(new Error(e));
 					});
 				}
-			} else {
+				
-				// treat it as a search and download the top result
+				// get JSON data about GH releases
-				factorioAPI.init(false, modDirectory, "", config.factorio_version);
+				/*let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+				let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+				let name = JSON.parse(res.getBody())[0].assets[0].name;
+				console.log(JSON.parse(res.getBody()))
+				*/
+			});
+		},
+		/**
+		Searches mods.factorio.com for mods matching a search term and prints
+		the results to console as a neat looking table
+
+		@param {string} searchTerm What to search for
+		*/
+		findMods: function(searchTerm){
+			return new Promise((resolve, reject) => {
+				factorioAPI.init(false);
 				factorioAPI.authenticate({
 					username: config.username,
 					token: config.token,
 					require_ownership: false
-				}).then(() => {
+				}).then(token => {
-					factorioAPI.searchMods(term).then(body => {
+					factorioAPI.searchMods(searchTerm).then(mods => {
-						console.log(body)
+						let result = asTable(mods.reverse().map(mod => {
-						console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
+							return {
-						factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
+								title:ellipse(mod.title, 50),
-							console.log("Mod downloaded");
+								name:ellipse(mod.name, 30),
-							resolve();
+								downloads:mod.downloads_count,
-						});
+								updated: moment(mod.updated_at).format("DD.MM.YY"),
+								latest_version: mod.latest_release.version,
+								owner:mod.owner,
+								//license: mod.license_name,
+							}
+						}));
+						resolve(result);
+						// console.log(result);
 					});
+				}).catch(e => {
+					if(e.contains("Insufficient information")) console.log("We are missing some parameters required for logging in! Make sure your factorio username and token are specified in config.json.");
+					reject(new Error(e));
+				});
+			});
+		},
+		addMod: function(term, instance){
+			return new Promise((resolve, reject) => {
+				if(instance == "shared"){ // modDirectory has to end in /
+					var modDirectory = "sharedMods/";
+				} else {
+				var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
+				}
+				if(term && term.length > 0){
+					if(term.includes("mods.factorio.com/mods/")){ // https://mods.factorio.com/mods/Danielv123/ChooChoo
+						// treat it as a link download
+						factorioAPI.init(false, modDirectory, "", config.factorio_version);
+						factorioAPI.authenticate({
+							username: config.username,
+							token: config.token,
+							require_ownership: false
+						}).then(() => {
+							factorioAPI.searchMods(term.split("/")[term.split("/").length-1]).then(body => {
+								console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
+								factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
+									console.log("Mod downloaded");
+									resolve();
+								});
+							});
+						});
+					} else if(term.includes("github.com")){ // https://github.com/Danielv123/factorioClusterioMod
+						// treat it as a github releases download
+						let apiURL = "https://api.github.com/repos/" + term.split("/")[3] + "/" + term.split("/")[4] + "/releases"; // https://api.github.com/repos/Danielv123/factorioClusterioMod/releases
+						
+						let res = syncRequest('GET', apiURL, {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+						let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+						let name = JSON.parse(res.getBody())[0].assets[0].name;
+						if(url) {
+							console.log("Downloading " + url);
+							let file = fs.createWriteStream(modDirectory + name);
+							let bytesDownloaded = 0;
+							let downloadStartTime = Date.now();
+							https.get(url, function(response) {
+								let bytesToDownload = response.headers['content-length'];
+								response.pipe(file);
+								response.on("data", chunk => {
+									bytesDownloaded += chunk.length;
+									let downloadTimeElapsed = Date.now() - downloadStartTime;
+									let bytesPerSecond = bytesDownloaded / (downloadTimeElapsed/1000);
+									let timeToComplete = Math.floor((bytesToDownload - bytesDownloaded)/bytesPerSecond);
+									sLog("Downloading "+name+" "+(bytesDownloaded/bytesToDownload*100).toPrecision(3)+"% ("+formatBytes(bytesDownloaded)+" / "+formatBytes(bytesToDownload)+")\r\n"+
+									"about "+timeToComplete+" seconds remaining ("+formatBytes(bytesPerSecond)+"/s)");
+								}).on("end", x => {
+									sLog.clear();
+									console.log(" Downloaded in "+moment.utc(moment.duration(Date.now()-downloadStartTime).as('milliseconds')).format('mm:ss')); // because sLog.clear() doesn't seem to do the job of ending the line properly
+									resolve();
+								});
+							});
+						}
+					} else {
+						// treat it as a search and download the top result
+						factorioAPI.init(false, modDirectory, "", config.factorio_version);
+						factorioAPI.authenticate({
+							username: config.username,
+							token: config.token,
+							require_ownership: false
+						}).then(() => {
+							factorioAPI.searchMods(term).then(body => {
+								console.log(body)
+								console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
+								factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
+									console.log("Mod downloaded");
+									resolve();
+								});
+							});
+						});
+					}
+				} else {
+					reject(new Error("Invalid search term"));
+				}
+			});
+		},
+		removeMod: function removeMod(term, instance, force = false){
+			return new Promise((resolve, reject) => {
+				if(instance == "shared"){ // modDirectory has to end in /
+					var modDirectory = "sharedMods/";
+				} else {
+				var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
+				}
+				if(term && typeof term == "string" && term.length > 0){
+					let mods = fs.readdirSync(modDirectory);
+					let modsMatchingFilter = mods.filter(mod => mod.toLowerCase().includes(term.toLowerCase()));
+					if(modsMatchingFilter.length < 1){
+						console.log("No mods found matching search");
+					} else if(modsMatchingFilter.length == 1){
+						console.log("Deleting "+modsMatchingFilter[0]);
+						fs.unlinkSync(modDirectory+modsMatchingFilter[0]);
+						console.log("Deleted.");
+					} else if(modsMatchingFilter.length > 1){
+						console.log("Found multiple mods");
+						modsMatchingFilter.forEach(mod => console.log(mod));
+						if(query("Delete all mods matching filter?")){
+							let modsToDelete = modsMatchingFilter.length;
+							for(let i = 0; i < modsToDelete; i++){
+								fs.unlinkSync(modDirectory+modsMatchingFilter[i]);
+								console.log("Deleted "+modsMatchingFilter[i]);
+							}
+						} else console.log("Did not delete mods.")
+					}
+					resolve();
+				} else {
+					reject(new Error("Invalid search term"));
+				}
+			});
+		},
+
+		// 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases'
+		/**
+		Download mod from github repo by looking for the first binary download in
+		the latest release, saving it to /sharedMods
+
+		@param {string} url a github repo  URL, for example https://api.github.com/repos/Danielv123/factorioClusterioMod/releases. Must link to /releases
+		*/
+		downloadFromGitURL: function(url){
+			console.log("Downloading mod from "+url);
+			// get JSON data about releases
+			let res = syncRequest('GET', url, {"headers":{"User-Agent":"Please don't require setting a user agent"}});
+			let downloadUrl = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
+			let name = JSON.parse(res.getBody())[0].assets[0].name;
+			console.log(JSON.parse(res.getBody()))
+			if(downloadUrl) {
+				console.log(downloadUrl);
+				let file = fs.createWriteStream("sharedMods/"+name);
+				https.get(downloadUrl, function(response) {
+					response.pipe(file);
+					console.log("Downloaded "+name);
 				});
 			}
-		} else {
+		},
-			reject(new Error("Invalid search term"));
-		}
-	});
-}
-module.exports.removeMod = function removeMod(term, instance, force = false){
-	return new Promise((resolve, reject) => {
-		if(instance == "shared"){ // modDirectory has to end in /
-			var modDirectory = "sharedMods/";
-		} else {
-		var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
-		}
-		if(term && typeof term == "string" && term.length > 0){
-			let mods = fs.readdirSync(modDirectory);
-			let modsMatchingFilter = mods.filter(mod => mod.toLowerCase().includes(term.toLowerCase()));
-			if(modsMatchingFilter.length < 1){
-				console.log("No mods found matching search");
-			} else if(modsMatchingFilter.length == 1){
-				console.log("Deleting "+modsMatchingFilter[0]);
-				fs.unlinkSync(modDirectory+modsMatchingFilter[0]);
-				console.log("Deleted.");
-			} else if(modsMatchingFilter.length > 1){
-				console.log("Found multiple mods");
-				modsMatchingFilter.forEach(mod => console.log(mod));
-				if(query("Delete all mods matching filter?")){
-					let modsToDelete = modsMatchingFilter.length;
-					for(let i = 0; i < modsToDelete; i++){
-						fs.unlinkSync(modDirectory+modsMatchingFilter[i]);
-						console.log("Deleted "+modsMatchingFilter[i]);
-					}
-				} else console.log("Did not delete mods.")
-			}
-			resolve();
-		} else {
-			reject(new Error("Invalid search term"));
-		}
-	});
-}
-
-// 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases'
-/**
-Download mod from github repo by looking for the first binary download in
-the latest release, saving it to /sharedMods
-
-@param {string} url a github repo  URL, for example https://api.github.com/repos/Danielv123/factorioClusterioMod/releases. Must link to /releases
-*/
-module.exports.downloadFromGitURL = function(url){
-	console.log("Downloading mod from "+url);
-	// get JSON data about releases
-	let res = syncRequest('GET', url, {"headers":{"User-Agent":"Please don't require setting a user agent"}});
-	let downloadUrl = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-	let name = JSON.parse(res.getBody())[0].assets[0].name;
-	console.log(JSON.parse(res.getBody()))
-	if(downloadUrl) {
-		console.log(downloadUrl);
-		let file = fs.createWriteStream("sharedMods/"+name);
-		https.get(downloadUrl, function(response) {
-			response.pipe(file);
-			console.log("Downloaded "+name);
-		});
 	}
 }
 /*

lib/manager/pluginManager.js

@@ -0,0 +1,116 @@
+const fs = require("fs-extra");
+const path = require("path");
+const git = require("simple-git")("./sharedPlugins");
+
+
+module.exports = function(config){
+	if(!config) config = require("./../../config");
+	
+	return {
+		listPlugins: async function(){
+			let plugins = await this.getPlugins();
+			plugins.forEach(plug => {
+				console.log(`${plug.name}`.padEnd(22).substr(0,22)+` - ${plug.description}`.substr(0,80-22));
+			});
+		},
+		getPlugins: async function(){
+			let directories = await fs.readdir("./sharedPlugins");
+			let returnArray = [];
+			for(let i in directories){
+				let dir = directories[i];
+				let pluginContents = await fs.readdir(path.join("./sharedPlugins", dir));
+				// console.log(pluginContents)
+				if(pluginContents.includes("config.js") || pluginContents.includes("config.json")){
+					try{
+						var pluginConfig = require(path.join("./../../sharedPlugins", dir, "config"));
+						pluginConfig.pluginPath = path.join("./sharedPlugins", dir);
+					}catch(e){
+						console.log(e)
+					}	
+					
+					// check if plugin is enabled
+					try{
+						await fs.stat(path.join(pluginConfig.pluginPath, "DISABLED"))
+						pluginConfig.enabled = false;
+					}catch(e){
+						pluginConfig.enabled = true;
+					}
+					returnArray.push(pluginConfig);
+				}
+			}
+			return returnArray;
+		},
+		addPlugin: function(path){
+			return new Promise((resolve, reject) => {
+				if(path.includes("git")){ // we are decently sure this is a git repo
+					git.clone(path, data => resolve(data))
+				}
+			});
+		},
+		removePlugin: async function(name){
+			let plugins = await this.getPlugins();
+			for(let i in plugins){
+				let plug = plugins[i];
+				if(plug.name.includes(name)){
+					// console.log(`Removing plugin ${plug.name}`);
+					await fs.remove(plug.pluginPath);
+					return {
+						ok:true,
+						msg:`Deleted plugin ${plug.name}`
+					}
+				}
+			}
+			return {
+				ok:false,
+				msg:`Plugin matching "${name}" not found`,
+			}
+		},
+		enablePlugin: async function(name, instance){
+			let plugins = await this.getPlugins();
+			let plugin = plugins[findInArray("name", name, plugins)[0]];
+			if(plugin){
+				let files = await fs.readdir(plugin.pluginPath);
+				if(files.includes("DISABLED")){
+					await fs.unlink(path.join(plugin.pluginPath, "DISABLED"));
+					return {
+						ok:true,
+						msg:`Globally enabled plugin ${name}`,
+					}
+				} else {
+					return {
+						ok:true,
+						msg:`Plugin ${name} already enabled`,
+					}
+				}
+			} else {
+				return {
+					ok:false,
+					msg:`Found no excact match for plugin ${name}`,
+				}
+			}
+		},
+		disablePlugin: async function(name, instance){
+			let plugins = await this.getPlugins();
+			let plugin = plugins[findInArray("name", name, plugins)[0]];
+			if(plugin){
+				await fs.writeFile(path.join(plugin.pluginPath, "DISABLED"), "");
+				return {
+					ok:true,
+					msg:`Plugin ${name} is now disabled on this host`,
+				}
+			} else {
+				return {
+					ok:false,
+					msg:`Found no excact match for plugin ${name}`,
+				}
+			}
+		}
+	}
+}
+function findInArray(key, value, array){
+	let indexes = [];
+	for(let i in array){
+		if(array[i][key] && array[i][key] === value) indexes.push(i);
+	}
+	return indexes;
+}

lib/npmPostinstall.js

@@ -61,17 +61,17 @@ var directories = [
 	"instances",
 ];
 let downloadList = [
-	{
+	// {
-		url: "http://fbpviewer.trakos.pl/images/spritesheet.json",
+		// url: "http://fbpviewer.trakos.pl/images/spritesheet.json",
-		file: "static/pictures/spritesheet.js",
+		// file: "static/pictures/spritesheet.js",
-		callback: function(file){
+		// callback: function(file){
-			fs.writeFileSync("static/pictures/spritesheet.js", "export default\n"+fs.readFileSync("static/pictures/spritesheet.js"));
+			// fs.writeFileSync("static/pictures/spritesheet.js", "export default\n"+fs.readFileSync("static/pictures/spritesheet.js"));
-		},
+		// },
-	},
+	// },
-	{
+	// {
-		url: "http://fbpviewer.trakos.pl/images/spritesheet.png",
+		// url: "http://fbpviewer.trakos.pl/images/spritesheet.png",
-		file: "static/pictures/spritesheet.png",
+		// file: "static/pictures/spritesheet.png",
-	},
+	// },
 	{
 		url: "https://code.jquery.com/jquery.js",
 		file: "static/external/jquery.js",

lib/scenarios/Hotpatch/control.lua

@@ -0,0 +1,153 @@
+--[[
+
+Copyright 2018 Chrisgbk
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+]]
+-- MIT License, https://opensource.org/licenses/MIT
+
+-- Hotpatchable control.lua for scenarios
+-- Supports multiple simultaneously loaded softmods
+-- This is a WIP
+-- Version 1.1 alpha
+-- probably some performance improvements to be made
+
+_ENV.debug_settings = {
+    level = 'info',
+    log_to_console_only = false,
+    log_to_RCON = false,
+    log_on_tick = false,
+}
+
+local hotpatch_tools = require 'hotpatch.mod-tools'
+require 'hotpatch.remote-interface'
+require 'hotpatch.commands'
+require 'hotpatch.gui'
+
+-- mod code goes here
+
+-- This is the Factorio Freeplay scenario
+-- This code, and the freeplay locale files in the zip file, is the property of Wube
+-- As this is part of the base mod, the developers have given permission for modders to use/adapt the assets therein, for modding Factorio
+-- Any other use would require licensing/permission from Wube
+-- If you don't want to run this, either comment this out before creating your scenario, OR uninstall freeplay at runtime
+hotpatch_tools.static_mod('freeplay', '1.0.0', [===[
+local util = require("util")
+local silo_script = require("silo-script")
+
+local created_items = function()
+  return
+  {
+    ["iron-plate"] = 8,
+    ["wood"] = 1,
+    ["pistol"] = 1,
+    ["firearm-magazine"] = 10,
+    ["burner-mining-drill"] = 1,
+    ["stone-furnace"] = 1
+  }
+end
+
+local respawn_items = function()
+  return
+  {
+    ["pistol"] = 1,
+    ["firearm-magazine"] = 10
+  }
+end
+
+for k,v in pairs(silo_script.get_events()) do
+  script.on_event(k, v)
+end
+
+script.on_event(defines.events.on_player_created, function(event)
+  local player = game.players[event.player_index]
+  util.insert_safe(player, global.created_items)
+
+  local r = global.chart_distance or 200
+  player.force.chart(player.surface, {{player.position.x - r, player.position.y - r}, {player.position.x + r, player.position.y + r}})
+
+  if not global.skip_intro then
+    if game.is_multiplayer() then
+      player.print({"msg-intro"})
+    else
+      game.show_message_dialog{text = {"msg-intro"}}
+    end
+  end
+
+  silo_script.on_event(event)
+end)
+
+script.on_event(defines.events.on_player_respawned, function(event)
+  local player = game.players[event.player_index]
+  util.insert_safe(player, global.respawn_items)
+  silo_script.on_event(event)
+end)
+
+script.on_configuration_changed(function(event)
+  global.created_items = global.created_items or created_items()
+  global.respawn_items = global.respawn_items or respawn_items()
+  silo_script.on_configuration_changed(event)
+end)
+
+script.on_load(function()
+  silo_script.on_load()
+end)
+
+script.on_init(function()
+  global.created_items = created_items()
+  global.respawn_items = respawn_items()
+  silo_script.on_init()
+end)
+
+silo_script.add_remote_interface()
+silo_script.add_commands()
+
+remote.add_interface("freeplay",
+{
+  get_created_items = function()
+    return global.created_items
+  end,
+  set_created_items = function(map)
+    global.created_items = map
+  end,
+  get_respawn_items = function()
+    return global.respawn_items
+  end,
+  set_respawn_items = function(map)
+    global.respawn_items = map
+  end,
+  set_skip_intro = function(bool)
+    global.skip_intro = bool
+  end,
+  set_chart_distance = function(value)
+    global.chart_distance = tonumber(value)
+  end
+})
+]===])
+
+-- Multi-file test
+--[[
+local files = {}
+
+files['testfolder.anothertest.test'] = [===[
+    script.on_event(defines.events.on_player_changed_position, function(e) game.print('position changed') end)
+]===]
+
+files['testfolder.test'] = [===[
+    require 'anothertest.test'
+]===]
+
+files['test'] = [===[
+    require 'testfolder.test'
+]===]
+
+hotpatch_tools.static_mod('require-test', '1.0.0', [===[
+    require 'test'
+]===], files)
+--]]
+
+-- end of mod code

lib/scenarios/Hotpatch/hotpatch/commands.lua

@@ -0,0 +1,135 @@
+local hotpatch_tools = require 'hotpatch.mod-tools'
+hotpatch_tools.static_mod('hotpatch-commands', '1.0.1', [===[
+--[[
+
+Copyright 2018 Chrisgbk
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+]]
+-- MIT License, https://opensource.org/licenses/MIT
+
+local hotpatch_tools = require 'hotpatch.mod-tools'
+
+--load private API
+-- mod installation/uninstallation support functions
+-- These take a mod NAME as a first argument
+local install_mod = hotpatch_tools.install_mod
+local find_installed_mod = hotpatch_tools.find_installed_mod
+local install_mod_file = hotpatch_tools.install_mod_file
+local uninstall_mod = hotpatch_tools.uninstall_mod
+
+-- mod interaction functions
+-- These take a LOADED INDEX as a first argument, except load_mod, which takes an INSTALLED INDEX
+local load_mod = hotpatch_tools.load_mod
+local find_loaded_mod = hotpatch_tools.find_loaded_mod
+local run_mod = hotpatch_tools.run_mod
+local reset_mod = hotpatch_tools.reset_mod
+local reset_mod_events = hotpatch_tools.reset_mod_events
+local register_mod_events = hotpatch_tools.register_mod_events
+local unload_mod = hotpatch_tools.unload_mod
+
+-- internal callbacks when a mod registers events
+local register_event = hotpatch_tools.register_event
+local register_nth_tick = hotpatch_tools.register_nth_tick
+local register_on_tick = hotpatch_tools.register_on_tick
+
+-- mod bootstrap functions
+-- These take a LOADED INDEX as a first argument
+local mod_on_init = hotpatch_tools.mod_on_init
+local mod_on_load = hotpatch_tools.mod_on_load
+local mod_on_configuration_changed = hotpatch_tools.mod_on_configuration_changed
+
+local static_mods = hotpatch_tools.static_mods
+local console = hotpatch_tools.console
+local debug_log = hotpatch_tools.debug_log
+local loaded_mods = hotpatch_tools.loaded_mods
+local installed_mods = hotpatch_tools.installed_mods
+
+local help_commands = {
+    help = '/hotpatch help\nYou got this far, you probably know all you need to know',
+    list = '/hotpatch list [type]\nLists \'installed\', \'loaded\' or \'running\' mods',
+}
+
+local sub_commands
+sub_commands = {
+    help = function(player_index, param)
+        local caller = (player_index and game.players[player_index]) or console
+        if (not param) or (param == '') then
+            caller.print('Usage: /hotpatch [command] [parameters]')
+            caller.print('Available commands:')
+            for k, _ in pairs(sub_commands) do caller.print(k) end
+            caller.print('Use /hotpatch help [command] for help on a particular command')
+        else
+            local text = help_commands[param]
+            if text then
+                caller.print(text)
+            end
+        end
+    end,
+    list = function(player_index, param)
+        local caller = (player_index and game.players[player_index]) or console
+        if not param or (param == '') then
+            caller.print('Installed mods:')
+            for k, v in pairs(installed_mods) do
+                caller.print(table.concat{'[', k, '] ', v.name, ' ', v.version})
+            end
+            caller.print('Loaded/Running mods:')
+            for k, v in pairs(loaded_mods) do
+                caller.print(table.concat{'[', k, '] ', v.name, ' ', v.version, ' ', (v.loaded and 'loaded') or 'not loaded', ' ', (v.running and 'running') or 'not running'})
+            end
+        else
+
+        end
+    end,
+}
+
+local admin_commands = {
+    reinstall = function(player_index, param)
+        local caller = (player_index and game.players[player_index]) or console
+        local mod
+        local installed_index
+        local loaded_index
+        for i = 1, #static_mods do
+            mod = static_mods[i]
+            if mod.name:match('^hotpatch%-.*$') then
+                loaded_index = find_loaded_mod(mod.name)
+                if loaded_index then
+                    unload_mod(loaded_index)
+                end
+                install_mod(mod.name, mod.version, mod.code, mod.files)
+                installed_index = find_installed_mod(mod.name)
+                load_mod(installed_index)
+                loaded_index = find_loaded_mod(mod.name)
+                run_mod(loaded_index)
+                mod_on_init(loaded_index)
+            end
+        end
+    end,
+}
+
+_ENV.commands.add_command('hotpatch', 'Commands for hotpatch. Run /hotpatch help for details.', function(e)
+    local caller = (e.player_index and game.players[e.player_index]) or console
+    if (not e.parameter) or (e.parameter == '') then
+        sub_commands['help'](e.player_index)
+        return
+    end
+    local sub_command, rest = e.parameter:match('(.-)%s(.*)')
+    if not sub_command then
+        sub_command = e.parameter
+    end
+
+    local f = sub_commands[sub_command]
+    if f then f(e.player_index, rest) end
+    if caller.admin then
+        f = admin_commands[sub_command]
+        if f then f(e.player_index, rest) end
+    end
+end)
+
+--]===])
+
+return true

lib/scenarios/Hotpatch/hotpatch/gui.lua

@@ -0,0 +1,377 @@
+local hotpatch_tools = require 'hotpatch.mod-tools'
+hotpatch_tools.static_mod('hotpatch-gui', '1.0.4', [===[
+--[[
+
+Copyright 2018 Chrisgbk
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+]]
+-- MIT License, https://opensource.org/licenses/MIT
+
+local hotpatch_tools = require 'hotpatch.mod-tools'
+local mod_gui = require 'mod-gui'
+
+--load private API
+-- mod installation/uninstallation support functions
+-- These take a mod NAME as a first argument
+local install_mod = hotpatch_tools.install_mod
+local find_installed_mod = hotpatch_tools.find_installed_mod
+local install_mod_file = hotpatch_tools.install_mod_file
+local uninstall_mod = hotpatch_tools.uninstall_mod
+
+-- mod interaction functions
+-- These take a LOADED INDEX as a first argument, except load_mod, which takes an INSTALLED INDEX
+local load_mod = hotpatch_tools.load_mod
+local find_loaded_mod = hotpatch_tools.find_loaded_mod
+local run_mod = hotpatch_tools.run_mod
+local reset_mod = hotpatch_tools.reset_mod
+local reset_mod_events = hotpatch_tools.reset_mod_events
+local register_mod_events = hotpatch_tools.register_mod_events
+local unload_mod = hotpatch_tools.unload_mod
+
+-- internal callbacks when a mod registers events
+local register_event = hotpatch_tools.register_event
+local register_nth_tick = hotpatch_tools.register_nth_tick
+local register_on_tick = hotpatch_tools.register_on_tick
+
+-- mod bootstrap functions
+-- These take a LOADED INDEX as a first argument
+local mod_on_init = hotpatch_tools.mod_on_init
+local mod_on_load = hotpatch_tools.mod_on_load
+local mod_on_configuration_changed = hotpatch_tools.mod_on_configuration_changed
+
+local static_mods = hotpatch_tools.static_mods
+local console = hotpatch_tools.console
+local debug_log = hotpatch_tools.debug_log
+local loaded_mods = hotpatch_tools.loaded_mods
+local installed_mods = hotpatch_tools.installed_mods
+
+script.on_event(defines.events.on_player_joined_game, function(e)
+    local player = game.players[e.player_index]
+    local top = mod_gui.get_button_flow(player)
+    local left = mod_gui.get_frame_flow(player)
+    local center = player.gui.center
+
+    local button = top['hotpatch-button']
+    if button then
+        button.destroy()
+    end
+    local menu = left['hotpatch-menu']
+    if menu then
+        menu.destroy()
+    end
+
+    local IDE = center['hotpatch-IDE']
+    if IDE then
+        IDE.destroy()
+    end
+
+    local main = center['hotpatch-main']
+    if main then
+        main.destroy()
+    end
+
+    button = top.add{type = 'sprite-button', name = 'hotpatch-button', sprite='utility/heat_exchange_indication', tooltip = 'Hotpatch', style = mod_gui.button_style}
+end)
+
+local on_gui_click_handlers
+local on_gui_selection_state_changed_handlers
+
+on_gui_click_handlers = {
+    ['hotpatch-button'] = function(e)
+        --on_gui_click_handlers['hotpatch-menu.IDE'](e)
+        --do return end
+        local player = game.players[e.player_index]
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local menu = left['hotpatch-menu']
+        if not menu then
+            menu = left.add{type = 'frame', name = 'hotpatch-menu', direction = 'vertical'}
+            menu.add{type = 'button', name = 'hotpatch-menu.IDE', caption = 'Mod IDE', tooltip = 'Open debugging GUI'}
+            menu.add{type = 'button', name = 'hotpatch-menu.console', caption = 'Mod Console', tooltip = 'Open debugging console'}
+            menu.visible = true
+            return
+        end
+        menu.visible = not menu.visible
+    end,
+    ['hotpatch-menu'] = function(e)
+        local player = game.players[e.player_index]
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local menu = left['hotpatch-menu']
+        menu.visible = not menu.visible
+
+        on_gui_click_handlers[e.element.name](e)
+    end,
+    ['hotpatch-menu.IDE'] = function(e)
+        local player = game.players[e.player_index]
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local IDE = center['hotpatch-IDE']
+        if not IDE then
+            IDE = center.add{type = 'frame', name = 'hotpatch-IDE', direction = 'vertical', caption = 'Hotpatch IDE', style = mod_gui.frame_style}
+            local top_flow = IDE.add{type = 'flow', name = 'hotpatch-IDE-top', direction = 'horizontal'}
+            top_flow.add{type = 'label', name = 'hotpatch-IDE-mod-label', caption = 'Mod: '}
+            local IDE_dropdown = top_flow.add{type = 'drop-down', name = 'hotpatch-IDE-mod-selector'}
+            top_flow.add{type = 'label', name = 'hotpatch-IDE-mod-version', caption = 'Version: No mod selected'}
+            local IDE_table = IDE.add{type = 'table', name = 'hotpatch-IDE-table', column_count = 2}
+            local files = IDE_table.add{type = 'scroll-pane', name = 'hotpatch-IDE-files', direction='vertical'}
+            files.style.height = 600
+            files.vertical_scroll_policy = 'always'
+            files = files.add{type = 'table', name = 'hotpatch-IDE-files-table', column_count = 1}
+            local code = IDE_table.add{type = 'text-box', name = 'hotpatch-IDE-code'}
+            code.word_wrap = true
+            files.style.width = 400
+            files.style.cell_padding = 0
+            files.style.top_padding = 0
+            files.style.bottom_padding = 0
+            code.style.width = 600
+            code.style.height = 600
+            IDE.visible = false
+        end
+
+        local IDE_dropdown = IDE['hotpatch-IDE-top']['hotpatch-IDE-mod-selector']
+        IDE_dropdown.clear_items()
+        for _, v in ipairs(installed_mods) do
+            IDE_dropdown.add_item(v.name)
+        end
+
+        IDE.visible = not IDE.visible
+        if IDE.visible then
+            player.opened = IDE
+        else
+            player.opened = nil
+        end
+    end,
+    ['hotpatch-menu.console'] = function(e)
+        local player = game.players[e.player_index]
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local hotpatch_console = center['hotpatch-console']
+        if not hotpatch_console then
+            hotpatch_console = center.add{type = 'frame', name = 'hotpatch-console', direction = 'vertical', caption = 'Hotpatch Console'}
+            local top_flow = hotpatch_console.add{type = 'flow', name = 'hotpatch-console-top', direction = 'horizontal'}
+            top_flow.add{type = 'label', name = 'hotpatch-console-mod-label', caption = 'Mod: '}
+            local console_dropdown = top_flow.add{type = 'drop-down', name = 'hotpatch-console-mod-selector'}
+            top_flow.add{type = 'label', name = 'hotpatch-console-mod-version', caption = 'Version: No mod selected'}
+            local bottom_flow = hotpatch_console.add{type = 'flow', name = 'hotpatch-console-bottom', direction = 'vertical'}
+            local output = bottom_flow.add{type = 'text-box', name = 'hotpatch-console-output'}
+            local bottom_input_flow = bottom_flow.add{type = 'flow', name = 'hotpatch-console-bottom-input', direction = 'horizontal'}
+            local input = bottom_input_flow.add{type = 'textfield', name = 'hotpatch-console-input'}
+            bottom_input_flow.add{type = 'button', name = 'hotpatch-console-run', caption = 'Run', tooltip = 'Run Lua Code'}
+            output.style.width = 800
+            output.style.height = 600
+            input.style.width = 600
+			hotpatch_console.visible = false
+        end
+
+        local console_dropdown = hotpatch_console['hotpatch-console-top']['hotpatch-console-mod-selector']
+        console_dropdown.clear_items()
+        for _, v in ipairs(loaded_mods) do
+            console_dropdown.add_item(v.name)
+        end
+
+        hotpatch_console.visible = not hotpatch_console.visible
+        if hotpatch_console.visible then
+            player.opened = hotpatch_console
+        else
+            player.opened = nil
+        end
+    end,
+    ['hotpatch-console-run'] = function(e)
+        local player = game.players[e.player_index]
+
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local hotpatch_console = center['hotpatch-console']
+        local output = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-output']
+        local input = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-bottom-input']['hotpatch-console-input']
+        local console_dropdown = hotpatch_console['hotpatch-console-top']['hotpatch-console-mod-selector']
+
+        if not player.admin then
+            output.text = table.concat{output.text, '\n', 'Only admins!'}
+            return
+        end
+
+        if not (console_dropdown.selected_index > 0) then
+            output.text = table.concat{output.text, '\n', 'Select a mod!'}
+            return
+        end
+
+        local mod_name = console_dropdown.items[console_dropdown.selected_index]
+        local loaded_index = find_loaded_mod(mod_name)
+        local env = loaded_mods[loaded_index].env
+        local old_print = env.print
+        local old_log = env.log
+
+
+
+
+
+        env.print = function(...)
+            --local count = select('#', ...)
+            local t = table.pack(...)
+            for k, v in pairs(t) do
+                t[k] = tostring(v)
+            end
+            local text = table.concat(t, '\t')
+            output.text = table.concat{output.text, '\n', text}
+        end
+
+        env.log = function(text)
+            output.text = table.concat{output.text, '\n', text}
+        end
+
+        local success
+        local code, err = load(input.text, input.text, 't', env)
+        if code then
+            success, err = pcall(code)
+            if not success then
+                output.text = table.concat{output.text, '\n', err}
+            end
+        else
+            output.text = table.concat{output.text, '\n', err}
+        end
+
+        env.print = old_print
+        env.log = old_log
+    end,
+    ['hotpatch-IDE-file'] = function(e)
+        local player = game.players[e.player_index]
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local element = e.element
+        local previous
+        for _, v in pairs(center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-files']['hotpatch-IDE-files-table'].children) do
+            if table.compare(v.style.font_color or {}, {r=1.0, g=1.0, b=0.0, a=1.0}) then
+                previous = v
+                break
+            end
+        end
+        if previous then
+            previous.style.font_color = {r=1.0, g=1.0, b=1.0, a=1.0}
+        end
+        element.style.font_color = {r=1.0, g=1.0, b=0.0, a=1.0}
+        local file = element.name:match('.-%.(.*)')
+        local selected = center['hotpatch-IDE']['hotpatch-IDE-top']['hotpatch-IDE-mod-selector']
+        local mod_name = selected.items[selected.selected_index]
+        local code = center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-code']
+        local index = find_installed_mod(mod_name)
+        if index then
+            local mod = installed_mods[index]
+            if file == 'control' then
+                code.text = mod.code
+            else
+                code.text = mod.files[file]
+            end
+        end
+    end,
+}
+
+script.on_event(defines.events.on_gui_click, function(e)
+    local element = e.element
+    if element.valid then
+        local name = element.name:match('([^%.]*)%.?.-')
+        local handler = on_gui_click_handlers[name]
+        if handler then handler(e) end
+    end
+end)
+
+on_gui_selection_state_changed_handlers = {
+    ['hotpatch-console-mod-selector'] = function(e)
+        local player = game.players[e.player_index]
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local element = e.element
+        local name = element.items[element.selected_index]
+        if name then
+            local index = find_loaded_mod(name)
+            if index then
+                local mod = loaded_mods[index]
+                local version_label = center['hotpatch-console']['hotpatch-console-top']['hotpatch-console-mod-version']
+                version_label.caption = 'Version: ' .. mod.version
+                local hotpatch_console = center['hotpatch-console']
+                local output = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-output']
+                output.text = ''
+            end
+        end
+    end,
+    ['hotpatch-IDE-mod-selector'] = function(e)
+        local player = game.players[e.player_index]
+        local top = mod_gui.get_button_flow(player)
+        local left = mod_gui.get_frame_flow(player)
+        local center = player.gui.center
+
+        local element = e.element
+
+        local name = element.items[element.selected_index]
+        if name then
+            local index = find_installed_mod(name)
+            if index then
+                local mod = installed_mods[index]
+                local version_label = center['hotpatch-IDE']['hotpatch-IDE-top']['hotpatch-IDE-mod-version']
+                version_label.caption = 'Version: ' .. mod.version
+                local list = center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-files']['hotpatch-IDE-files-table']
+                list.clear()
+                local file = list.add{type = 'label', style = 'hoverable_bold_label', caption = 'control', name = 'hotpatch-IDE-file.control'}
+                file.style.bottom_padding = 0
+                file.style.top_padding = 0
+                for k, _ in pairs(mod.files) do
+                    file = list.add{type = 'label', style = 'hoverable_bold_label', caption = k, name = 'hotpatch-IDE-file.' .. k}
+                    file.style.bottom_padding = 0
+                    file.style.top_padding = 0
+                end
+                local code = center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-code']
+                code.text = ''
+            end
+        end
+    end,
+}
+
+script.on_event(defines.events.on_gui_selection_state_changed, function(e)
+    local element = e.element
+    local handler = on_gui_selection_state_changed_handlers[element.name]
+    if handler then handler(e) end
+end)
+
+script.on_event(defines.events.on_gui_closed, function(e)
+    local player = game.players[e.player_index]
+    local element = e.element
+    if e.gui_type == defines.gui_type.custom then
+        if element and element.valid then
+            if element.name == 'hotpatch-IDE' then
+                if element.visible then
+                    player.play_sound{path = "utility/gui_click"}
+                end
+                element.visible = false
+            elseif element.name == 'hotpatch-console' then
+                if element.visible then
+                    player.play_sound{path = "utility/gui_click"}
+                end
+                element.visible = false
+            end
+        end
+    end
+end)
+
+--]===])
+
+return true

lib/scenarios/Hotpatch/hotpatch/remote-interface.lua

@@ -0,0 +1,322 @@
+local hotpatch_tools = require 'hotpatch.mod-tools'
+hotpatch_tools.static_mod('hotpatch-remote-interface', '1.0.2', [===[
+--[[
+
+Copyright 2018 Chrisgbk
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+]]
+-- MIT License, https://opensource.org/licenses/MIT
+
+local hotpatch_tools = require 'hotpatch.mod-tools'
+
+--load private API
+-- mod installation/uninstallation support functions
+-- These take a mod NAME as a first argument
+local install_mod = hotpatch_tools.install_mod
+local find_installed_mod = hotpatch_tools.find_installed_mod
+local install_mod_file = hotpatch_tools.install_mod_file
+local uninstall_mod = hotpatch_tools.uninstall_mod
+
+-- mod interaction functions
+-- These take a LOADED INDEX as a first argument, except load_mod, which takes an INSTALLED INDEX
+local load_mod = hotpatch_tools.load_mod
+local find_loaded_mod = hotpatch_tools.find_loaded_mod
+local run_mod = hotpatch_tools.run_mod
+local reset_mod = hotpatch_tools.reset_mod
+local reset_mod_events = hotpatch_tools.reset_mod_events
+local register_mod_events = hotpatch_tools.register_mod_events
+local unload_mod = hotpatch_tools.unload_mod
+
+-- internal callbacks when a mod registers events
+local register_event = hotpatch_tools.register_event
+local register_nth_tick = hotpatch_tools.register_nth_tick
+local register_on_tick = hotpatch_tools.register_on_tick
+
+-- mod bootstrap functions
+-- These take a LOADED INDEX as a first argument
+local mod_on_init = hotpatch_tools.mod_on_init
+local mod_on_load = hotpatch_tools.mod_on_load
+local mod_on_configuration_changed = hotpatch_tools.mod_on_configuration_changed
+
+local static_mods = hotpatch_tools.static_mods
+local console = hotpatch_tools.console
+local debug_log = hotpatch_tools.debug_log
+local loaded_mods = hotpatch_tools.loaded_mods
+local installed_mods = hotpatch_tools.installed_mods
+
+local remote_interface = {}
+
+--IMPORTANT: WHEN COPY PASTING CODE TO CONSOLE THE CODE MUST HAVE SINGLE LINE COMMENTS REMOVED
+-- FACTORIO CONSOLE STRIPS LINEFEEDS WHICH MAKES ALL THE CODE BECOME COMMENTED OUT
+--TODO: gate these behind admin permissions; if (game.player and game.player.admin) should work for console commands?
+
+debug_log({'hotpatch-info.remote-installing'})
+
+remote_interface['install'] = function(mod_name, mod_version, mod_code, mod_files, only_install)
+    -- this installs a new mod and runs on_init, then registers events
+    -- Note that mods may expect that certain events haven't been called yet when their on_init is ran
+    -- This may prevent them from functioning properly, without manually calling the events they expect
+    -- examples: on_player_created
+    local caller = game.player or console
+    if caller.admin then
+		if mod_files then
+			for k,_ in pairs(mod_files) do
+				if k == 'control' then
+					caller.print('ERROR: control.lua loaded twice for mod ' .. mod_name)
+					return
+				end
+			end
+		end
+        local installed_index = find_installed_mod(mod_name)
+        local loaded_index = find_loaded_mod(mod_name)
+        if installed_index then
+            local old_version = installed_mods[installed_index].version
+            if old_version then
+                debug_log({'hotpatch-warning.already-exists', mod_name, old_version})
+                debug_log({'hotpatch-warning.reinstalling', mod_name, old_version})
+            end
+            if loaded_index then
+                unload_mod(loaded_index)
+            end
+        end
+
+        install_mod(mod_name, mod_version, mod_code, mod_files)
+        if not only_install then
+            installed_index = find_installed_mod(mod_name)
+            if installed_index then
+                if not load_mod(installed_index) then
+                    caller.print('compilation failed for mod ' .. mod_name)
+                    return
+                end
+                loaded_index = find_loaded_mod(mod_name)
+                if loaded_index then
+                    if not run_mod(loaded_index) then
+                        caller.print('execution failed for mod ' .. mod_name)
+						unload_mod(loaded_index)
+                        return
+                    end
+                    if not mod_on_init(loaded_index) then
+                        caller.print('on_init failed for mod ' .. mod_name)
+						unload_mod(loaded_index)
+                        return
+                    end
+                    -- TODO: notify all mods
+                    -- TODO: determine vanilla behaviour and replicate it
+                    if not mod_on_configuration_changed(loaded_index, {mod_changes = {mod_name={new_version=mod_version}}}) then
+                        caller.print('on_configuration_changed failed for mod ' .. mod_name)
+						unload_mod(loaded_index)
+                        return
+                    end
+                else
+                end
+            else
+            end
+        end
+    else
+        caller.print('You must be an admin to run this command.')
+    end
+end
+
+remote_interface['run'] = function(mod_name)
+    -- this runs on_init, then registers events
+    -- Note that mods may expect that certain events haven't been called yet when their on_init is ran
+    -- This may prevent them from functioning properly, without manually calling the events they expect
+    -- examples: on_player_created
+    local caller = game.player or console
+    if caller.admin then
+        local installed_index = find_installed_mod(mod_name)
+        local loaded_index = find_loaded_mod(mod_name)
+		local mod
+        local version
+        if installed_index and not loaded_index then
+            if not load_mod(installed_index) then
+                caller.print('compilation failed for mod ' .. mod_name)
+                return
+            end
+            loaded_index = find_loaded_mod(mod_name)
+        else
+            if loaded_index then
+                mod = loaded_mods[loaded_index]
+                version = mod.version
+                debug_log({'hotpatch-warning.already-loaded', mod_name, version})
+                debug_log({'hotpatch-warning.reinitializing', mod_name, version})
+            end
+        end
+
+        if loaded_index then
+            if not run_mod(loaded_index) then
+                caller.print('execution failed for mod ' .. mod_name)
+				unload_mod(loaded_index)
+                return
+            end
+            if not mod_on_init(loaded_index) then
+                caller.print('on_init failed for mod ' .. mod_name)
+				unload_mod(loaded_index)
+                return
+            end
+            -- TODO: notify all mods
+            -- TODO: determine vanilla behaviour and replicate it
+            if not mod_on_configuration_changed(loaded_index, {mod_changes = {mod_name={new_version=version}}}) then
+                caller.print('on_configuration_changed failed for mod ' .. mod_name)
+				unload_mod(loaded_index)
+                return
+            end
+        end
+    else
+        caller.print('You must be an admin to run this command.')
+    end
+end
+
+remote_interface['update'] = function(mod_name, mod_version, mod_code, mod_files)
+    -- this updates an existing mod
+    -- the current mods events are de-registered, the new code is installed, on_load is triggered, and then events are registered
+    -- finally, the mod is informed of the update, so it can run migrations from the previous version
+    -- TODO: validation
+    local caller = game.player or console
+    if caller.admin then
+		if mod_files then
+			for k,_ in pairs(mod_files) do
+				if k == 'control' then
+					caller.print('ERROR: control.lua loaded twice for mod ' .. mod_name)
+					return
+				end
+			end
+		end
+        local installed_index = find_installed_mod(mod_name)
+		if installed_index then
+			;
+		end
+        local loaded_index = find_loaded_mod(mod_name)
+        local old_version
+        if loaded_index then
+            old_version = loaded_mods[loaded_index].version
+            unload_mod(loaded_index)
+        end
+        install_mod(mod_name, mod_version, mod_code, mod_files)
+        installed_index = find_installed_mod(mod_name)
+        if not load_mod(installed_index) then
+            caller.print('compilation failed for mod ' .. mod_name)
+            return
+        end
+        loaded_index = find_loaded_mod(mod_name)
+        if not run_mod(loaded_index) then
+            caller.print('execution failed for mod ' .. mod_name)
+			unload_mod(loaded_index)
+            return
+        end
+        if old_version then
+            if not mod_on_load(loaded_index) then
+                caller.print('on_load failed for mod ' .. mod_name)
+				unload_mod(loaded_index)
+                return
+            end
+            -- The mod must do any migrations here
+            -- TODO: notify all mods
+            if not mod_on_configuration_changed(loaded_index, {mod_changes = {mod_name={old_version=old_version, new_version=mod_version}}}) then
+                caller.print('on_configuration_changed failed for mod ' .. mod_name)
+				unload_mod(loaded_index)
+                return
+            end
+        else
+            -- first time install
+            if not mod_on_init(loaded_index) then
+                caller.print('on_init failed for mod ' .. mod_name)
+				unload_mod(loaded_index)
+                return
+            end
+            -- TODO: notify all mods
+            if not mod_on_configuration_changed(loaded_index, {mod_changes = {mod_name={new_version=mod_version}}}) then
+                caller.print('on_configuration_changed failed for mod ' .. mod_name)
+				unload_mod(loaded_index)
+                return
+            end
+        end
+    else
+        caller.print('You must be an admin to run this command.')
+    end
+end
+
+remote_interface['install_mod_file'] = function(mod_name, mod_file, mod_file_code)
+    local caller = game.player or console
+    if caller.admin then
+        install_mod_file(mod_name, mod_file, mod_file_code)
+    else
+        caller.print('You must be an admin to run this command.')
+    end
+end
+
+remote_interface['exec'] = function(mod_name, exec_code)
+    local caller = game.player or console
+    if caller.admin then
+		local loaded_index = find_loaded_mod(mod_name)
+		local env = loaded_mods[loaded_index].env
+		
+		local success
+		local code, err = load(exec_code, exec_code, 't', env)
+		if code then
+			success, err = pcall(code)
+			if not success then
+				debug_log(err)
+			end
+		else
+			debug_log(err)
+		end
+	else
+		caller.print('You must be an admin to run this command.')
+    end
+end
+
+--TODO: most of this function
+remote_interface['clean'] = function(mod_name)
+    -- Removes ALL mod data and reinitializes
+    -- doesn't remove things a mod may add like surfaces, etc
+    local caller = game.player or console
+    if caller.admin then
+        --mod_reset(mod_name)
+        --run_mod(mod_name)
+        --mod_on_init(mod_name)
+		local loaded_index = find_loaded_mod(mod_name)
+		local mod = loaded_mods[loaded_index]
+        local version = mod.version
+        mod_on_configuration_changed(loaded_index, {mod_changes = {mod_name={new_version=version}}})
+    else
+        caller.print('You must be an admin to run this command.')
+    end
+end
+
+remote_interface['uninstall'] = function(mod_name)
+    -- Uninstalls a mod
+    local caller = game.player or console
+    if caller.admin then
+        uninstall_mod(mod_name)
+    else
+        caller.print('You must be an admin to run this command.')
+    end
+end
+
+remote.add_interface('hotpatch', remote_interface)
+
+debug_log({'hotpatch-info.complete', {'hotpatch-info.remote-installing'}})
+
+--]===])
+
+--[[
+
+remote.call('hotpatch, 'install', 'test', '1.0.0', [===[
+script.on_event(defines.events.on_player_changed_position, function(e)
+    game.print('changed position')
+end
+]===]
+
+local updated_code = ....
+remote.call('hotpatch', 'update', 'test', '1.0.1', updated_code)
+
+
+]]
+
+return true

lib/scenarios/Hotpatch/locale/af/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Hierdie is Factorio vryspel. Jou taak is om 'n vuurpyl in die buitenste ruimte in te stuur. Doen dit deur die bou van 'n Vuurpyl Silo en van daar af 'n vuurpyl te lanseer met a satelliet aan boord. Jy sal baie gevorderde tegnologiëe moet navors om die Vuurpyl Silo te ontsluit. Begin klein, werk jou pad op met outomatisering en moenie vergeet om jouself te beskerm teen die inboorlinge nie.
+
+

lib/scenarios/Hotpatch/locale/ar/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=.نييلصألا ناكسلا نم كسفن ةيامح ىسنت الو ةياهنلا ىلإ لصتل كتقيرطب لمعأو ،رفصلا نم أدبتس .خيراوصلا عماوص حتف لجأ نم ةمدقتم ايجولونكت نع ثحبلل جاتحت فوس .ةيعانصلا رامقألاب خيراوصلا قالطإو خيراوصلا عماوص ءانب قيرط نع كلذب مايقلا عيطتست .ءاضفلل اخوراص قلطت نا يه كتمهم .Factorio ـل رحلا بعللا وه اذه
+
+

lib/scenarios/Hotpatch/locale/be/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Гэта рэжым свабоднай гульні Factorio. Вашая мэта - запусціць ракету ў космас. Дзеля гэтага вам трэба пабудаваць ракетную шахту і саму ракету з спадарожнікам. Вы мусіце даследваць перадавыя тэхналогіі, каб навучыцца будаваць ракетную шахту. Пачынайце з малога, працуйце над аўтаматызацыяй вытворчасці, даследуйце новыя тэхналогіі і не забывайцеся на абарону ад "тутэйшых"!
+
+

lib/scenarios/Hotpatch/locale/bg/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Това е режимът на свободна игра. Задачата ви е да изстреляте ракета в космоса. За това ще е нужно да построите ракетна шахта и изстреляте ракетата със спътник. Ще трябва да проучите развити технологии, които ще отключат ракетната шахта. Започнете с малко и постепенно автоматизирайте процеса, като не забравяте да се защитавате от местните форми на живот.
+
+

lib/scenarios/Hotpatch/locale/ca/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Aquest és el mode lliure del Factorio. La vostra tasca és construir un coet i enviar-lo a l'espai. Ho haureu de fer construint un centre de coets espacials i llençant un coet amb un satèl·lit. Necessitareu investigar tecnologies avançades per poder-ho fer. Aneu pas a pas, avanceu cap a l'automatització i no oblideu protegir-vos dels natius.
+
+

lib/scenarios/Hotpatch/locale/cs/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Toto je volná hra Factorio. Tvým úkolem je vypustit raketu do vesmíru. Postav raketové silo a vypusť raketu se satelitem. Pro odemknutí raketového sila budeš muset vyzkoumat pokročilé technologie. Začni v malém, postupuj s automatizací a nezapomeň se chránit před domorodci.
+
+

lib/scenarios/Hotpatch/locale/da/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Det her er Factorio fri leg. Din opgave er at affyre en raket for at udforske rummet. For at gøre det skal du bygge en raket silo og affyre en raket med en satellit i. Du bliver nød til at udforske avancerede teknologier for at at kunne bygge en raket silo. Start småt, og arbejd din vej op med hjælp af automatisering, glem ikke at beskytte dig fra de indfødte.
+
+

lib/scenarios/Hotpatch/locale/de/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Dies ist das freie Spiel von Factorio. Deine Aufgabe ist es, eine Rakete in den Weltraum zu starten. Dazu musst Du ein Raketensilo bauen und eine Rakete mit einem Satelliten starten. Du wirst erweiterte Technologien erforschen müssen, um das Raketensilo freizuschalten. Fange klein an, arbeite Dich durch Automatisierung hoch und vergiss nicht, Dich vor den Einheimischen zu schützen.
+
+

lib/scenarios/Hotpatch/locale/el/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Αυτό είναι το ελεύθερο παιχνίδι του Factorio. Ο στόχος σου είναι να εκτοξεύσεις έναν πύραυλο στο διάστημα. Αυτό το κάνεις κατασκευάζοντας ένα σιλό πυραύλου και εκτοξεύοντας έναν πυραύλο με δορυφόρο. Θα πρέπει να ερευνήσεις προηγμένες τεχνολογίες για να ξεκλειδώσεις το σιλό πυραύλου. Ξεκινάς περιορισμένος, δούλεψε με αυτοματισμούς για την άνοδο σου και μην ξεχνάς να προστατέψεις τον εαυτό σου από τους ντόπιους.
+
+

lib/scenarios/Hotpatch/locale/en/freeplay.cfg

@@ -0,0 +1,2 @@
+msg-intro=This is the Factorio freeplay. Your task is to launch a rocket into space. Do this by constructing a Rocket Silo and launching a rocket with a satellite. You will need to research advanced technologies in order to unlock the Rocket Silo. Start small, work your way up with automation and don't forget to protect yourself from the natives.
+

lib/scenarios/Hotpatch/locale/en/hotpatch.cfg

@@ -0,0 +1,81 @@
+[hotpatch]
+log=@__1__:__2__] __3__
+log-mod=@__1__:__2__][__4__] __3__
+severe=SEVERE ERROR: __1__
+error=ERROR: __1__
+warning=WARNING: __1__
+info=INFO: __1__
+verbose=INFO: __1__
+trace=TRACE: __1__
+
+[hotpatch-info]
+logging-enabled=Logging enabled
+metatable-installed=_ENV metatable installed
+complete=__1__ Complete!
+on-init=initializing...
+on-load=loading...
+on-configuration-changed=mod configuration changed...
+installing-included-mods=installing included mods...
+loading-installed-mods=loading installed mods...
+loading-libs=loading Factorio.data.core.lualib...
+loading-library=loading library: __1__
+uninstalling=Uninstalling mod...
+installing=Installing version __1__...
+installing-file=Installing file __1__...
+script-shim=setting up mod script shim...
+setting-env=setting up mod _ENV...
+loading=loading...
+unloading=unloading...
+running=running...
+must-be-admin=You must be an admin to run this command
+remote-installing=installing remote interface
+
+[hotpatch-trace]
+nil-var-access=_ENV nil variable access: __1__
+nil-var-assignment=_ENV variable assignment: __1__
+event-registering=registering events...
+on-tick-event-registered=registered on_tick event...
+on-event-registered=registered event __1__...
+on-nth-tick-event-registered=registered nth_tick event __1__...
+event-running=running event __1__...
+nth-tick-event-running=running nth_event __1__...
+mod-on-init=running on_init...
+mod-on-load=running on_load...
+mod-on-configuration-changed=running on_configuration_changed...
+adding-event=adding event __1__
+adding-nth-tick-event=adding nth_tick event __1__
+caching-event=caching event: __1__...
+caching-nth-tick-event=caching nth_tick event: __1__...
+event-processing=processing event __1__...
+nth-tick-event-processing=processing nth_tick event __1__...
+nth-tick-handler-added=added nth_tick handler: __1__
+nth-tick-handler-removed=removed nth_tick handler: __1__
+on-tick-handler-added=added on_tick handler
+on-tick-handler-removed=removed on_tick handler
+on-event-handler-added=added event handler: __1__
+on-event-handler-removed=removed event handler: __1__
+cached-load-require=loading cached require'd file: __1__...
+load-require=loading require'd file: __1__...
+load-core-lib=loading from Factorio.data.core.lualib: __1__...
+
+[hotpatch-warning]
+contains-comments=mod code contains comments!
+contains-comments-no-lf=mod contains comments and no linefeed!
+contains-comments-console=comments from console will comment out the entire code!
+reset-events-not-running=tried to reset events for mod that isn't running!
+remote-interface-exists=remote interface __1__ already exists, removing...
+command-exists=command __1__ already exists, removing...
+already-exists=mod already exists: __1__ __2__
+reinstalling=reinstalling mod in-place: __1__ __2__
+already-loaded=mod already loaded: __1__ __2__
+reinitializing=reinitializing: __1__ __2__
+
+[hotpatch-error]
+invalid-API-access=Invalid API access: __1__
+not-installed=mod not installed, cannot install file for mod that does not exist
+compilation-failed=compilation failed for mod
+execution-failed=execution failed for mod
+path-not-found=path '__1__' not found
+on-init-failed=on_init failed
+on-load-failed=on_load failed
+on-configuration-changed-failed=on_configuration_changed failed

lib/scenarios/Hotpatch/locale/eo/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Tio estas la Factorio Libera Ludado. Via tasko estas ĵeti raketon en spacon. Fari tiun per konstruanta Raketo Silo kaj ĵeti raketon kun satelito. Vi bezonos esplori altnivelaj teknologioj por malŝlosi la Raketo Silo. Start malgranda, labori vian vojon supren per aŭtomatigo kaj ne forgesu protekti vin mem de la indiĝenoj.
+
+

lib/scenarios/Hotpatch/locale/es-ES/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Este es el modo de juego libre de Factorio. Tu tarea es la de lanzar un cohete al espacio. Logra esto construyendo un Silo de Cohetes y lanzando un cohete con un satélite. Necesitarás investigar tecnologías avanzadas para poder desbloquear el Silo de Cohetes. Comienza poco a poco, trabaja en tu sistema de automatización y no te olvides de protegerte de los nativos.
+
+

lib/scenarios/Hotpatch/locale/et/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=See on Factorio vabamäng. Sinu eesmärgiks on saata rakett avakosmosesse. Seda saad teha ehitades Raketi Silotorni ja lennutades Raketti koos satelliidiga. Selleks pead õppima selgeks keerukad tehnoloogiaharud, et Raketi Silotorni lahti lukustada. Alusta väikselt ning liigu edasi automatiseerimise abil ja ära unusta ennast kaitsta kohalike eluvormide eest.
+
+

lib/scenarios/Hotpatch/locale/fi/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Tämä on Factorion vapaapelimuoto. Tavoitteesi on lähettää satelliitti ulkoavaruuteen rakentamalla rakettisiilo ja laukaisemalla raketti. Rakettisiilon voit rakentaa vasta, kun olet kehittänyt siihen tarvittavat huipputeknologiat. Aloita pienestä ja automatisoi kaikki tarvittava, äläkä unohda suojella itseäsi alkuasukkailta.
+
+

lib/scenarios/Hotpatch/locale/fr/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Il s'agit du mode de jeu libre de Factorio. Votre tâche consiste à lancer une fusée dans l'espace. Ceci en construisant un silo à fusée, et en lançant un satellite avec la fusée. Vous aurez besoin de rechercher des technologies de pointe afin de débloquer le silo. Commencez petit, évoluez en automatisant et n'oubliez pas de vous protéger contre les autochtones.
+
+

lib/scenarios/Hotpatch/locale/he/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=.םיימוקמה םירוציה ינפמ םכמצע לע ןגהל וחכשת לאו ,יטמוטוא ןוכימ תועצמאב םכלש הישעתה תא וביחרה ,ןטקב וליחתה .ןקתמה תיינב תא רשפאל ידכ תומדקתמ תויגולונכט רוקחל םכילע היהי .ןייוול ליכמה ליט ותועצמאב רגשלו םיליט רוגיש ןקתמ תונבל םכילע .ללחל ליט רגשל איה םכתמישמ .ויִרֹוטְקַפ לש ישפוחה שיחרתה והז
+
+

lib/scenarios/Hotpatch/locale/hu/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Ez a Factorio szabad játék módja. A feladatod az, hogy kilőj egy rakétát az űr felderítésére. Ehhez egy Rakétasilót kell építened, melyből aztán egy műholddal felszerelt rakétát kell kilőnöd. A Rakétasiló kifejlesztéséhez különféle technológiákat kell fejlesztened. Kezdj kicsiben és küzdd fel magad az automatizálásban. Ne felejts el védekezni a honos fajok ellen.
+
+

lib/scenarios/Hotpatch/locale/it/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Questa è la modalità libera di Factorio. Il tuo obiettivo è quello di lanciare il razzo nello spazio. Completalo costruendo un Silo per Razzi e lanciando il razzo con il satellite. Avrai bisogno di ricercare tecnologie avanzate per riuscire a sbloccare il Silo per Razzi. Inizia in piccolo, espandi con l'automazione e non dimenticare di proteggerti dai nativi.
+
+

lib/scenarios/Hotpatch/locale/ja/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=これは、Factorioのフリープレイです。ロケットサイロを建造し、衛星を載せたロケットを宇宙へ打ち上げるのがあなたの任務です。ロケットサイロをアンロックするためには高度な技術研究をする必要があります。小規模に始め、自動化しつつ進み、原住生物から身を守ることも忘れないでください。
+
+

lib/scenarios/Hotpatch/locale/ko/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Factorio 자유플레이입니다. 당신의 목표는 우주로 로켓을 발사하는 것입니다. 로켓 격납고를 건설하고 인공위성을 탑재한 로켓을 발사하여 이 목표를 달성하세요. 로켓 격납고의 잠금을 해제하기 위해 고급 기술들을 연구할 필요가 있습니다. 작은 것부터 시작해 자동화로 차근차근 발전하고, 토착 생물로부터 자신을 보호하는 것도 잊지 마세요.
+
+

lib/scenarios/Hotpatch/locale/lt/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Tai Factorio laisvas žaidimas. Jūsų užduotis - paleisti raketą į kosmosą. Tam reikia pastatyti Raketos šachtą ir paleisti raketą su palydovu. Reikės atrasti pažangias technologijas kad atrakintumėte Raketos šachtą. Pradėkite palengva, tobulinkite automatiką ir neužmirškite apsisaugoti nuo vietinių.
+
+

lib/scenarios/Hotpatch/locale/lv/freeplay.cfg

@@ -0,0 +1,2 @@
+msg-intro=Šis ir Factorio brīvais režīms. Tavs uzdevums ir palaist raķeti kosmosā. Lai to izdarītu, tev vajadzēs uzbūvēt palaišanas platformu un palaist raķeti ar satelītu. Tev būs jāizpēta sarežģītas tehnoloģijas, lai atbloķētu palaišanas platformu. Sāc no mazumiņa, strādā un neaizmirsti pasargāt sevi no šīs planētas iemītniekiem...
+

lib/scenarios/Hotpatch/locale/nl/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Dit is Factorio vrij spel. Je taak is om een raket te lanceren in de ruimte. Bouw hiervoor een raket-silo en lanceer een raket met een satelliet. Je zult geavanceerde technologieën moeten onderzoeken om de raket-silo vrij te spelen. Begin op kleine schaal, breidt uit en automatiseer je fabriek en vergeet niet jezelf te beschermen tegen de inheemse bewoners.
+
+

lib/scenarios/Hotpatch/locale/no/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Dette er Factorio frispilling. Din oppgave er å sende en rakett til verdensrommet. Bygg en rakettsilo og send opp en rakett med en satellitt. Du blir nødt til å utforske avanserte teknologier for å få tilgang til rakettsiloen. Begynn i det små, jobb deg oppover med hjelp av automatisering, men ikke glem å beskytte deg mot de innfødte.
+
+

lib/scenarios/Hotpatch/locale/pl/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=To jest swobodna rozgrywka Factorio. Twoim zadaniem jest wysłanie rakiety w kosmos. Skonstruuj silos rakietowy i wystrzel z niego rakietę z satelitą. Żeby odblokować silos rakietowy będziesz musiał odkryć zaawansowane technologie. Rozpocznij od podstaw, rozwijaj się wykorzystując automatyzację i nie zapomnij o obronie przed rodzimymi mieszkańcami planety.
+
+

lib/scenarios/Hotpatch/locale/pt-BR/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Este é o modo livre do Factorio. Sua missão é lançar um foguete no espaço. Para fazer isso construa um centro de lançamento de foguetes e lance o foguete com um satélite. Você vai precisar pesquisar tecnologias avançadas para conseguir desbloquear um Centro de lançamento. Comece do começo, continue na automação e não se esqueça de se proteger dos nativos.
+
+

lib/scenarios/Hotpatch/locale/pt-PT/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Este é o jogo livre Factorio. A tua tarefa é lançar um foguetão para explorar o espaço. Faça isto, construindo um Silo de Foguetão e lançar um foguetão com um satélite. Tu vais precisar de pesquisar tecnologias avançadas para poderes desbloquear o Silo de Foguetão. Começa simples, trabalha e evolui com a automação e não te esqueças de proteger-te dos nativos.
+
+

lib/scenarios/Hotpatch/locale/ro/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Acesta este modul Factorio freeplay. Sarcina ta este de a lansa o rachetă în spațiu. Fă acest lucru prin construirea unui siloz de rachete și lansarea unei rachete cu satelit. Va trebui să cercetezi tehnologiile avansate pentru a debloca silozul de rachete. Începe încet, avansează cu automatizarea și nu uita să te protejezi de inamici.
+
+

lib/scenarios/Hotpatch/locale/ru/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Это режим свободной игры Factorio. Ваша задача запустить спутник в космос. Для его запуска потребуется создать ракету и ракетную шахту. Начните с малого, работайте над автоматизацией производства, исследуйте новые технологии и не забывайте о защите от местных форм жизни.
+
+

lib/scenarios/Hotpatch/locale/sk/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Toto je voľná hra Factorio. Tvojou úlohou je vypustiť raketu do vesmíru. Postav raketové silo a vypusti raketu so satelitom. Pre odomknutie raketového sila budeš musieť vyskúmať pokročilé technológie. Začni v malom, postupuj s automatizáciou a nezabnudni sa chrániť pred domácimi.
+
+

lib/scenarios/Hotpatch/locale/sl/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=To je Factorio freeplay. Vaša naloga je poslati raketo v vesolje. To storite tako, da zgradite silos za raketo in jo skupaj s satelitom izstrelite v vesolje. Raziskati bo potrebno napredne tehnologije, da bi lahko odklenili Raketni silos. Začnete z majhnim, nato nadgradite vse s pomočjo avtomatizacije in ne pozabite, zaščitite se pred domorodci.
+
+

lib/scenarios/Hotpatch/locale/sr/freeplay.cfg

@@ -0,0 +1,2 @@
+msg-intro=Ово је Factorio freeplay. Твој задатак је да лансираш ракету са сателитом у свемир. Уради ово грађом ракетног силоса. Треба да истражиш пуно напредних технологија да би откључао ракетни силос. Почни мало, а онда напредуј са аутоматизацијом и не заборави да се заштитиш од ванземаљаца.
+

lib/scenarios/Hotpatch/locale/sv-SE/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Det här är Factorio frispel. Ditt uppdrag är att avfyra en raket ut i rymden. Gör det genom att tillverka en raket silo och avfyra en raket med en satellit. Du kommer att behöva forska fram avancerade teknologier för att kunna låsa upp raket silon. Börja smått, jobba dig uppåt med hjälp av automatisering och glöm inte att försvara dig själv mot infödingarna.
+
+

lib/scenarios/Hotpatch/locale/th/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=นี้คือโหมดอิสระของ Factorio ภารกิจของคุณคือส่งจรวดสู่อวกาศ ซึ่งทำได้โดยการสร้างไซโลจรวดและส่งจรวดขึ้นไปพร้อมกับดาวเทียม คุณต้องวิจัยค้นคว้าเทคโนโลยีขั้นสูงเพื่อปลดล็อคไซโลจรวด เริ่มจากเล็กๆไปเรื่อยๆด้วยระบบอัตโนมัติและอย่าลืมป้องกันตัวเองจากคนพื้นเมืองด้วย
+
+

lib/scenarios/Hotpatch/locale/tr/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Bu Factorio'nun serbest oynanışı. Görevin roketleri uzaya yollamak. Bunu Roket Silosu yaparak ve uyduyla roket yollayarak yapacaksın. Roket Silo'sunu açmak için gelişmiş teknolojileri sırayla açmalısın. Yavaş başla, otomatikleş ve yerlilerden kendini korumayı untuma.
+
+

lib/scenarios/Hotpatch/locale/uk/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Це один з типів гри в Factorio - "Вільна гра". Ваше завдання - запустити ракету у космічний простір. Зробіть це, побудувавши ракетну шахту та запустивши ракету зі супутником. Вам потрібно дослідити передові технології, щоб мати змогу побудувати ракетну шахту. Починайте з малого, розвивайтеся за допомогою автоматизації, але не забувайте і про оборону!
+
+

lib/scenarios/Hotpatch/locale/vi/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=Đầy là chế độ freeplay của Factorio. Nhiệm cụ của bạn là phóng một chiếc tên lửa lên vũ trụ. Hãy xây dựng một bệ phóng tên lửa để phóng một chiếc tên lửa kèm theo vệ tinh. Bạn sẽ cần phải nghiên cứu nhiều công nghệ tiên tiến để có thể xây dựng bệ phóng tên lửa. Bắt đầu từ quy mô nhỏ, phát triển dần dần bằng tự động hóa, và đừng quên bảo vệ bản thân mình khỏi những sinh vật sinh sống ở đây.
+
+

lib/scenarios/Hotpatch/locale/zh-CN/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=欢迎来到异星工厂标准模式，你的目标是要把火箭发射到太空。为此你需要研发一系列高等科技，解锁并修建火箭发射井，直到把携带有卫星的火箭发射出去。从小作坊生产起步，逐渐建立起属于自己的自动化大工厂吧！同时，别忘了防范异星虫子的骚扰 ...
+
+

lib/scenarios/Hotpatch/locale/zh-TW/freeplay.cfg

@@ -0,0 +1,3 @@
+msg-intro=這是異星工廠的自由模式。你的任務是發射火箭，探索外太空。請建造一個火箭發射井，並發射攜帶著衛星的火箭。你會需要研究大量的進階科技以解鎖火箭發射井。從頭開始，靠著自動化科技的協助向目標邁進時，也別忘記從外星住民手的中保護你自己啊！
+
+

logo.svg

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg width="100%" height="100%" viewBox="0 0 154 24" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:1.41421;"><path d="M105.009,19.309l-3.12,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68l0,6.2l-0.96,0.96l-7.596,-0.003l0,4.816l-0.96,0.953l0,0.007Zm6.876,0l-3.12,0l0,-3.944l0.96,-0.96l4.8,0l0,2.212l-2.64,2.692Zm-63.405,0l-9.917,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l2.718,0l0.002,-8.867l0.96,-0.96l3.117,0l1.68,1.68l0,10.293l-2.64,2.64Zm-15.652,0l-9.918,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l8.478,0.009l0,2.137l-2.64,2.64Zm-15.712,0l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.942l0,0.018l-7.596,0l0,5.04l8.556,0l0,2.146l-2.64,2.64Zm47.185,0l-9.997,0l-1.68,-1.68l0,-2.142l0.96,-0.96l7.597,-0.002l0,-0.993l-8.557,0l0,-6.196l1.68,-1.68l0,0l0.96,-0.96c2.029,0 7.282,0 9.997,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.96l-7.597,0.002l0,0.957l8.557,0l0,6.227l-0.96,0.96l0,0l-1.68,1.68Zm12.444,0l-3.12,0l-1.68,-1.68l0,-6.216l0.96,-0.96l6.48,0l0,6.216l-2.64,2.64Zm19.325,0l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-1.086,0.96l-7.47,0l0,0.959l5.841,0l0,2.13l-0.96,0.96l-4.881,0l0,0.991l8.556,0l0,2.146l-2.64,2.64Zm20.065,-7.896l0.96,-0.96l6.481,0l0,5.953l-2.64,2.903l-3.121,0l-1.68,-1.943l0,-5.953Zm21.411,7.896l-2.872,0l0,-1.178c-1.382,-0.123 -2.749,-0.431 -4.028,-0.977l-0.056,-0.024l-1.307,0.804l-2.03,-1.25l1.277,-0.786c-0.81,-0.666 -1.487,-1.522 -1.761,-2.534l-1.749,0l0,-1.768l0.96,-0.96l1.141,0c0.332,-0.616 0.809,-1.156 1.35,-1.61l-1.198,-0.78l0.94,-0.93l2.03,-1.25l1.227,0.755c0.949,-0.424 2.033,-0.741 3.204,-0.922l0,-0.248l0.013,0l0.947,-0.96l2.872,0l0,1.116c1.505,0.156 2.889,0.535 4.059,1.079l1.332,-0.82l2.03,1.25l-1.12,1.059c0.701,0.671 1.181,1.437 1.377,2.261l1.976,0l0,1.768l-0.96,0.962l-0.002,-0.002l-1.327,0c-0.228,0.624 -0.547,1.236 -1.053,1.681l1.109,0.679l-0.968,0.978l-0.015,-0.004l-2.007,1.236l-1.411,-0.868c-0.958,0.432 -1.978,0.728 -3.02,0.908l0,0.375l-0.96,0.93l0,0.03Zm-15.17,-7.656l-6.001,0l0,5.624l1.55,1.792l2.901,0l1.55,-1.792l0,-5.624Zm-44.191,0l-6,0l0,5.877l1.54,1.539l2.921,0l1.539,-1.539l0,-5.877Zm35.14,3.952l-4.32,0l0,3.464l2.778,0l1.542,-1.59l0,-1.874Zm-96.308,-9.709l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-6.48l8.556,0l0,-1.807l-1.539,-1.54Zm8.771,0l-2.778,0l-1.54,1.54l0,10.094l1.54,1.539l9.719,0l1.539,-1.539l0,-1.799l-8.478,-0.008l-0.002,-9.827Zm15.653,0l-2.779,0l-1.539,1.54l0,10.094l1.539,1.539l9.719,0l1.539,-1.539l0,-10.094l-1.539,-1.54l-2.778,0l-0.003,9.827l-4.157,0l-0.002,-9.827Zm54.53,0l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-2.431l5.841,0l0,-1.65l-5.841,0l0,-2.399l8.556,0l0,-1.807l-1.539,-1.54Zm-31.77,0l-9.797,0l-0.82,0.82l-0.72,0.72l0,5.856l8.557,0l0,2.432l-8.557,0.003l0,1.803l1.54,1.539l9.797,0l1.54,-1.539l0,-5.888l-8.557,0l0,-2.396l8.557,-0.003l0,-1.807l-1.54,-1.54Zm47.585,0l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l2.781,0l0,-5.777l8.556,0.004l0,-5.86l-1.539,-1.54Zm28.324,7.453c0.327,-0.363 0.421,-0.95 0.212,-1.53c-0.459,-1.279 -2.339,-2.021 -4.278,-2.122c-0.058,0 -0.058,0 -0.115,0c-2.792,0.201 -3.386,1.31 -3.433,1.781c-0.151,1.505 1.994,2.74 4.45,2.74c1.269,0 2.7,-0.357 3.164,-0.869Zm-31.101,-1.707l0,-2.398l-4.24,0.002l0,2.396l4.24,0Zm-3.04,-2.157l0,0.957l2.8,0l0,-0.958l-2.8,0.001Zm-24.048,0.012l-13.471,0l0,-2.161l2.64,-2.64l10.111,0l1.68,1.68c0,1.034 0,2.161 0,2.161l-0.96,0.96Zm40.606,-0.048l-6.392,0l0,-2.113l2.64,-2.64l4.801,-0.005l0,3.798l-1.049,0.96Zm-42.386,-3.553l-9.912,0l-1.539,1.54l0,1.821l12.991,0l0,-1.821l-1.54,-1.54Zm37.774,0l-1.54,1.54l0,1.773l6.001,0l0,-3.318c-1.487,0.002 -2.974,0.003 -4.461,0.005Z" style="fill:#141414;"/><path d="M73.625,19.309l3.12,0l-3.12,0Zm47.201,-0.24l-2.901,0l-1.55,-1.792l0,-5.624l6.001,0l0,5.624l-1.55,1.792Zm-44.18,0l-2.921,0l-1.54,-1.539l0,-5.877l6,0l0,5.877l-1.539,1.539Zm60.66,0l-2.392,0l0,-1.157l-0.288,-0.026c-1.392,-0.131 -2.761,-0.465 -4.055,-1.026l-1.288,0.792l-1.572,-0.968l1.175,-0.724l0.005,-0.076l-0.25,-0.205c-0.598,-0.5 -1.12,-1.098 -1.438,-1.808c-0.159,-0.355 -0.249,-0.743 -0.25,-0.747l-1.693,0l0,-1.288l1.676,0l0.056,-0.226c0.27,-1.047 1.011,-1.878 1.84,-2.574l-0.005,-0.072l-1.116,-0.688l1.572,-0.968l1.208,0.743l0.228,-0.1c1.335,-0.574 2.76,-0.86 4.195,-0.988l0,-1.072l2.392,0l0,1.092l0.284,0.03c1.394,0.155 2.76,0.518 4.031,1.105l1.316,-0.81l1.572,0.968l-1.259,0.776l0.236,0.276c0.559,0.497 1.039,1.085 1.324,1.77c0.148,0.356 0.225,0.734 0.226,0.738l1.924,0l0,1.288l-1.941,0l-0.059,0.219c-0.367,1.3 -1.768,2.53 -1.768,2.53l1.317,0.811l-1.572,0.968l-1.394,-0.858lLine truncated

master.js

@@ -10,6 +10,12 @@ combinator signals.
 node master.js
 */
 
+// Attempt updating
+// const updater = require("./updater.js");
+// updater.update().then(console.log);
+
+(async ()=>{
+
 // Set the process title, shows up as the title of the CMD window on windows
 // and as the process name in ps/top on linux.
 process.title = "clusterioMaster";
@@ -24,9 +30,29 @@ debug = false;
 // tested by logging Date.now() here and when the webserver started listening
 require('cache-require-paths');
 
+// argument parsing
+const args = require('minimist')(process.argv.slice(2));
+
+// Library for create folder recursively if it does not exist
+const mkdirp = require("mkdirp");
+const averagedTimeSeries = require("averaged-timeseries");
+const deepmerge = require("deepmerge");
+const path = require("path");
+const fs = require("fs-extra");
+const jwt = require('jsonwebtoken');
+const bcrypt = require('bcryptjs');
+const crypto = require('crypto');
+const base64url = require('base64url');
+const moment = require("moment");
+const request = require("request")
+
 // constants
-const masterModFolder = "./database/masterMods/";
+console.log(`Requiring config from ${args.config || './config'}`)
-const config = require('./config');
+const config = require(args.config || './config');
+config.databaseDirectory = args.databaseDirectory || config.databaseDirectory || "./database";
+const masterModFolder = path.join(config.databaseDirectory, "/masterMods/");
+mkdirp.sync(config.databaseDirectory);
+mkdirp.sync(masterModFolder);
 
 // homebrew modules
 const getFactorioLocale = require("./lib/getFactorioLocale");
@@ -34,24 +60,7 @@ const stringUtils = require("./lib/stringUtils");
 const fileOps = require("_app/fileOps");
 
 // homemade express middleware for token auth
-const authenticate = require("./lib/authenticate");
+const authenticate = require("./lib/authenticate")(config);
-
-// Library for create folder recursively if it does not exist
-const mkdirp = require("mkdirp");
-mkdirp.sync("./database");
-mkdirp.sync(masterModFolder);
-const averagedTimeSeries = require("averaged-timeseries");
-const deepmerge = require("deepmerge");
-const path = require("path");
-const fs = require("fs");
-const nedb = require("nedb");
-const https = require("https");
-
-const jwt = require('jsonwebtoken');
-const bcrypt = require('bcryptjs');
-
-const crypto = require('crypto');
-const base64url = require('base64url');
 
 /** Sync */
 function randomStringAsBase64Url(size) {
@@ -73,12 +82,14 @@ fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAu
 
 const express = require("express");
 const compression = require('compression');
+const cookieParser = require('cookie-parser');
 const ejs = require("ejs");
 // Required for express post requests
 const bodyParser = require("body-parser");
 const fileUpload = require('express-fileupload');
 var app = express();
 
+app.use(cookieParser());
 app.use(bodyParser.json());
 app.use(bodyParser.urlencoded({
 	parameterLimit: 100000,
@@ -98,10 +109,13 @@ app.use(function(req, res, next){
 	res.locals.res = res;
 	res.locals.req = req;
 	res.locals.masterPlugins = masterPlugins;
+	res.locals.slaves = slaves;
+	res.locals.moment = moment;
 	next();
 });
 
 require("./routes.js")(app);
+require("./routes/api/getPictures.js")(app);
 // Set folder to serve static content from (the website)
 app.use(express.static('static'));
 // mod downloads
@@ -110,14 +124,19 @@ app.use(express.static(masterModFolder));
 // set up logging software
 const prometheusPrefix = "clusterio_";
 const Prometheus = require('prom-client');
+const expressPrometheus = require('express-prometheus-request-metrics');
 Prometheus.collectDefaultMetrics({ timeout: 10000 }); // collects RAM usage etc every 10 s
-const httpRequestDurationMilliseconds = new Prometheus.Histogram({
+if(!config.disablePrometheusPushgateway){
-  name: prometheusPrefix+'http_request_duration_ms',
+	const pushgateway = new Prometheus.Pushgateway('http://hme.danielv.no:9091');
-  help: 'Duration of HTTP requests in ms',
+	setInterval(() => {
-  labelNames: ['route'],
+		registerMoreMetrics();
-  // buckets for response time from 0.1ms to 500ms
+		pushgateway.push({ jobName: 'clusterio', groupings: {instance: config.publicIP + ":" + config.masterPort, owner: config.username}}, function(err, resp, body) {})
-  buckets: [0.10, 5, 15, 50, 100, 200, 300, 400, 500]
+	}, 15000)
-});
+}
+
+// collect express request durations ms
+app.use(expressPrometheus(Prometheus));
+
 const endpointHitCounter = new Prometheus.Gauge({
 	name: prometheusPrefix+'endpoint_hit_gauge',
 	help: "How many requests a particular endpoint has gotten",
@@ -132,11 +151,6 @@ const prometheusWsUsageCounter = new Prometheus.Counter({
 	help: 'Websocket traffic',
 	labelNames: ["connectionType", "instanceID"],
 });
-const prometheusProductionGauge = new Prometheus.Gauge({
-	name: prometheusPrefix+'production_gauge',
-	help: 'Items produced by instance',
-	labelNames: ["instanceID", "itemName"],
-});
 const prometheusExportGauge = new Prometheus.Gauge({
 	name: prometheusPrefix+'export_gauge',
 	help: 'Items exported by instance',
@@ -173,12 +187,8 @@ const prometheusMasterInventoryGauge = new Prometheus.Gauge({
 	labelNames: ["itemName"],
 });
 setInterval(()=>{
-	let numberOfActiveSlaves = 0;
+    fs.writeFileSync("database/items.json", JSON.stringify(db.items));
-	for(let instance in slaves){
+},config.autosaveInterval || 60000);
-		if(Date.now() - Number(slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
-	}
-	prometheusConnectedInstancesCounter.set(numberOfActiveSlaves);
-},10000);
 /**
 GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format.
 @memberof clusterioMaster
@@ -187,10 +197,14 @@ GET Prometheus metrics endpoint. Returns performance and usage metrics in a prom
 */
 app.get('/metrics', (req, res) => {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	res.set('Content-Type', Prometheus.register.contentType);
 	
 	/// gather some static metrics
+	registerMoreMetrics();
+	res.end(Prometheus.register.metrics());
+});
+
+function registerMoreMetrics(){
 	for(let instanceID in slaves){
 		// playercount
 		try{
@@ -208,39 +222,29 @@ app.get('/metrics', (req, res) => {
 			prometheusMasterInventoryGauge.labels(key).set(Number(db.items[key]) || 0);
 		}
 	}
-	
+	// Slave count
-	res.end(Prometheus.register.metrics());
+	let numberOfActiveSlaves = 0;
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
+	for(let instance in slaves){
-});
+		if(Date.now() - Number(slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
+	}
+	prometheusConnectedInstancesCounter.set(numberOfActiveSlaves);
+}
 
 // set up database
-var Datastore = require('nedb');
+const db = {};
-db = {};
-var LinvoDB = require("linvodb3");
-LinvoDB.dbPath = "./database/linvodb/";
-// database for items in system
-// db.items = new Datastore({ filename: 'database/items.db', autoload: true });
-
-// in memory database for combinator signals
-db.signals = new Datastore({ filename: 'database/signals.db', autoload: true, inMemoryOnly: true});
-db.signals.ensureIndex({ fieldName: 'time', expireAfterSeconds: 3600 }, function (err) {});
-
-// production chart database
-db.flows = new LinvoDB("flows", {}, {});
-
 
 (function(){
 	try{
-		let x = fs.statSync("database/slaves.json");
+		let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json"));
-		console.log("loading slaves from database/slaves.json");
+		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
-		slaves = JSON.parse(fs.readFileSync("database/slaves.json"));
+		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json")));
 	} catch (e){
 		slaves = {};
 	}
 	try{
-		x = fs.statSync("database/items.json");
+		x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
-		console.log("loading items from database/items.json");
+		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
-		db.items = JSON.parse(fs.readFileSync("database/items.json"));
+		db.items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")));
 	} catch (e){
 		db.items = {};
 	}
@@ -254,7 +258,7 @@ db.items.addItem = function(object) {
 		if(this[object.name] && Number(this[object.name]) != NaN){
 			this[object.name] = Number(this[object.name]) + Number(object.count);
 		} else {
-			this[object.name] = object.count;
+			this[object.name] = Number(object.count);
 		}
 		return true;
 	}
@@ -274,33 +278,41 @@ db.items.removeItem = function(object) {
 }
 
 // store slaves and inventory in a .json full of JSON data
-process.on('SIGINT', async function () {
+process.on('SIGINT', shutdown); // ctrl + c
+process.on('SIGHUP', shutdown); // terminal closed
+async function shutdown() {
 	console.log('Ctrl-C...');
 	let exitStartTime = Date.now();
-	// set insane limit to slave length, if its longer than this we are probably being ddosed or something
+	try {
-	if(slaves && Object.keys(slaves).length < 2000000){
+		// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-		console.log("saving to slaves.json");
+		if(slaves && Object.keys(slaves).length < 2000000){
-		fs.writeFileSync("database/slaves.json", JSON.stringify(slaves));
+			console.log("saving to slaves.json");
-	} else if(slaves) {
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
-		console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
+		} else if(slaves) {
-	}
+			console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
-	if(db.items && Object.keys(db.items).length < 50000){
-		console.log("saving to items.json");
-		fs.writeFileSync("database/items.json", JSON.stringify(db.items));
-	} else if(slaves) {
-		console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
-	}
-	for(let i in masterPlugins){
-		let plugin = masterPlugins[i];
-		if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
-			let startTime = Date.now();
-			await plugin.main.onExit();
-			console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
 		}
+		if(db.items && Object.keys(db.items).length < 50000){
+			console.log("saving to items.json");
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
+		} else if(slaves) {
+			console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
+		}
+		for(let i in masterPlugins){
+			let plugin = masterPlugins[i];
+			if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
+				let startTime = Date.now();
+				await plugin.main.onExit();
+				console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
+			}
+		}
+		console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
+		process.exit(0);
+	} catch(e) {
+		console.log(e)
+		console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms")
+		process.exit(1)
 	}
-	console.log("Exited in "+(Date.now()-exitStartTime)+"ms");
+}
-	process.exit(2);
-});
 
 /**
 POST World ID management. Slaves post here to tell the server they exist
@@ -310,7 +322,6 @@ POST World ID management. Slaves post here to tell the server they exist
 */
 app.post("/api/getID", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	// request.body should be an object
 	// {rconPort, rconPassword, serverPort, mac, time}
 	// time us a unix timestamp we can use to check for how long the server has been unresponsive
@@ -327,9 +338,8 @@ app.post("/api/getID", authenticate.middleware, function(req,res) {
 				slaves[req.body.unique][k] = req.body[k];
 			}
 		}
-		console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName);
+		// console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName);
 	}
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 /**
 POST Allows you to add metadata related to slaves for other tools, like owner data, descriptions or statistics.
@@ -339,7 +349,6 @@ POST Allows you to add metadata related to slaves for other tools, like owner da
 */
 app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	// request.body should be an object
 	// {instanceID, pass, meta:{x,y,z}}
 	
@@ -352,12 +361,11 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 			slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
 			let metaPortion = JSON.stringify(req.body.meta);
 			if(metaPortion.length > 50) metaPortion = metaPortion.substring(0,20) + "...";
-			console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion);
+			// console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion);
 		} else {
 			res.send("ERROR: Invalid instanceID or password")
 		}
 	}
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 
 /**
@@ -368,7 +376,6 @@ POST Get metadata from a single slave. For whenever you find the data returned b
 */
 app.post("/api/getSlaveMeta", function (req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	console.log("body", req.body);
     if(req.body && req.body.instanceID && req.body.password){
     	console.log("returning meta for ", req.body.instanceID);
@@ -377,7 +384,6 @@ app.post("/api/getSlaveMeta", function (req, res) {
     	res.status(400);
     	res.send('{"INVALID REQUEST":1}');
 	}
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 // mod management
 // should handle uploading and checking if mods are uploaded
@@ -389,7 +395,6 @@ POST Check if a mod has been uploaded to the master before. Only checks against
 */
 app.post("/api/checkMod", function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	let files = fs.readdirSync(masterModFolder);
 	let found = false;
 	files.forEach(file => {
@@ -404,7 +409,6 @@ app.post("/api/checkMod", function(req,res) {
 		res.send("found");
 	}
 	res.end();
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 /**
 POST endpoint for uploading mods to the master server. Required for automatic mod downloads with factorioClusterioClient (electron app, see seperate repo)
@@ -414,17 +418,15 @@ POST endpoint for uploading mods to the master server. Required for automatic mo
 */
 app.post("/api/uploadMod", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	if (req.files && req.files.file) {
-		console.log(req.files.file);
+		// console.log(req.files.file);
-		req.files.file.mv('./database/masterMods/'+req.files.file.name, function(err) {
+		req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err) {
 			if (err) {
 				res.status(500).send(err);
 			} else {
 				res.send('File uploaded!');
 				console.log("Uploaded mod: " + req.files.file.name);
 			}
-			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 		});
 	} else {
 		res.send('No files were uploaded.');
@@ -442,7 +444,6 @@ let slaveCache = {
 };
 app.get("/api/slaves", function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){
@@ -456,7 +457,6 @@ app.get("/api/slaves", function(req, res) {
 	}
 	
 	res.send(slaveCache.cache);
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 var recievedItemStatisticsBySlaveID = {};
 var sentItemStatisticsBySlaveID = {};
@@ -479,7 +479,6 @@ POST endpoint for storing items in master's inventory.
 */
 app.post("/api/place", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	let x;
@@ -499,10 +498,12 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 			}
 		});
 	}
-	if(	x.instanceID
+	if(x.instanceID
-		&& x.instanceName
+	&& x.instanceName
-		&& !isNaN(Number(x.count))// This is in no way a proper authentication or anything, its just to make sure everybody are registered as slaves before modifying the cluster (or not, to maintain backwards compat)
+	&& !isNaN(Number(x.count))// This is in no way a proper authentication or anything, its just to make sure everybody are registered as slaves before modifying the cluster (or not, to maintain backwards compat)
-		/*&& stringUtils.hashCode(slaves[x.unique].rconPassword) == x.unique*/){
+	/*&& stringUtils.hashCode(slaves[x.unique].rconPassword) == x.unique*/
+	&& x.name
+	&& typeof x.name == "string"){
 		if(config.logItemTransfers){
 			console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
 		}
@@ -532,8 +533,14 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	} else {
 		res.send("failure");
 	}
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
+const routes_api_remove = require("./routes/api/remove.js")
+var neuralDole = null
+
+if(config.useNeuralNetDoleDivider)//Only initialize neural network when it's enabled, otherwise it might override gauge
+	neuralDole=new routes_api_remove.neuralDole({
+		items: db.items, gaugePrefix: prometheusPrefix
+	})
 /**
 POST endpoint to remove items from DB when client orders items.
 
@@ -545,12 +552,8 @@ POST endpoint to remove items from DB when client orders items.
 @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name]
 @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
 */
-_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 app.post("/api/remove", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
-	const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
-	const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// save items we get
@@ -564,96 +567,61 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	if(slaves[object.instanceID]) object.instanceName = slaves[object.instanceID].instanceName;
 	let item = db.items[object.name]
 		// console.dir(doc);
-	if (!item) {
+	if((item === undefined)
+	||((
+		config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
+		&& object.name != "electricity")
+	){
 		if(config.logItemTransfers){
 			console.log('failure could not find ' + object.name);
 		}
 		res.send({name:object.name, count:0});
-	} else {
+	} else if(config.disableFairItemDistribution){
-		const originalCount = Number(object.count) || 0;
+		// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
-		object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+		let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item));
-		object.count = Math.round(object.count);
+		db.items.removeItem({count: numberToRemove, name: object.name});
-		if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
+		res.send({count: numberToRemove, name: object.name});
 		
-		// Update existing items if item name already exists
+		// track statistics and do graphing things
-		if(Number(item) > Number(object.count)) {
+		prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
-			//If successful, increase dole
+		let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-			_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+		if(sentItemStatistics === undefined){
-			if(config.logItemTransfers){
+			sentItemStatistics = new averagedTimeSeries({
-				console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
+				maxEntries: config.itemStats.maxEntries,
-			}
+				entriesPerSecond: config.itemStats.entriesPerSecond,
-			if(db.items.removeItem({count: object.count, name: object.name})){
+				mergeMode: "add",
-				let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+			}, console.log);
-				if(sentItemStatistics === undefined){
-					sentItemStatistics = new averagedTimeSeries({
-						maxEntries: config.itemStats.maxEntries,
-						entriesPerSecond: config.itemStats.entriesPerSecond,
-						mergeMode: "add",
-					}, console.log);
-				}
-				sentItemStatistics.add({
-					key:object.name,
-					value:object.count,
-				});
-				//console.log(sentItemStatistics.data)
-				sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-			}
-			
-			prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
-			prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
-			res.send({count: object.count, name: object.name});
-			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
-		} else {
-			// if we didn't have enough, attempt giving out a smaller amount next time
-			_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
-			res.send({name:object.name, count:0});
-			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
-			//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
 		}
+		sentItemStatistics.add({
+			key:object.name,
+			value:numberToRemove,
+		});
+		//console.log(sentItemStatistics.data)
+		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+	} else if(config.useNeuralNetDoleDivider){
+		// use fancy neural net to calculate a "fair" dole division rate.
+		neuralDole.divider({
+			res,
+			object,
+			config,
+			sentItemStatisticsBySlaveID,
+			prometheusImportGauge
+		})
+	} else {
+		// Use dole division. Makes it really slow to drain out the last little bit.
+		routes_api_remove.doleDivider({
+			item,
+			object,
+			db,
+			sentItemStatisticsBySlaveID,
+			config,
+			prometheusDoleFactorGauge,
+			prometheusImportGauge,
+			req,res,
+		})
 	}
 });
 
-// circuit stuff
-/**
-POST endpoint to send and store circuit frames on the master.
-Gives no response
-
-@memberof clusterioMaster
-@instance
-@alias api/setSignal
-@param {object} circuitFrame
-@param {number} circuitFrame.time
-*/
-app.post("/api/setSignal", authenticate.middleware, function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
-	if(typeof req.body == "object" && req.body.time){
-		db.signals.insert(req.body);
-		httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
-		// console.log("signal set");
-	}
-});
-/**
-POST endpoint to read database of circuit signals sent to master
-
-@memberof clusterioMaster
-@instance
-@alias api/readSignal
-@returns {object} circuitFrame
-*/
-app.post("/api/readSignal", function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
-	// request.body should be an object
-	// {since:UNIXTIMESTAMP,}
-	// we should send an array of all signals since then
-	db.signals.find({time:{$gte: req.body.since}}, function (err, docs) {
-		// $gte means greater than or equal to, meaning we only get entries newer than the timestamp
-		res.send(docs);
-		httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
-		// console.log(docs);
-	});
-});
 /**
 GET endpoint to read the masters current inventory of items.
 
@@ -665,7 +633,6 @@ GET endpoint to read the masters current inventory of items.
 // endpoint for getting an inventory of what we got
 app.get("/api/inventory", function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// Check it and send it
@@ -676,7 +643,6 @@ app.get("/api/inventory", function(req, res) {
 		}
 	}
 	res.send(JSON.stringify(inventory));
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 /**
 GET endpoint to read the masters inventory as an object with key:value pairs
@@ -688,185 +654,39 @@ GET endpoint to read the masters inventory as an object with key:value pairs
 */
 app.get("/api/inventoryAsObject", function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// Check it and send it
 	res.send(JSON.stringify(db.items));
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
-});
-
-// post flowstats here for production graphs
-// {timestamp: Date, instanceID: string, data: {"item":number}}
-/**
-POST endpoint to log production graph statistics. Should contain a timestamp
-gathered from Date.now(), a instanceID (also reffered to as "unique") and of
-course the timeSeries data.
-
-@memberof clusterioMaster
-@instance
-@alias api/logStats
-@param {object} JSON {timestamp: Date.now(), instanceID: "string", data: {"item": number}}
-@returns {string} failure
-*/
-app.post("/api/logStats", authenticate.middleware, function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
-	if(typeof req.body == "object" && req.body.instanceID && req.body.timestamp && req.body.data) {
-		if(Number(req.body.timestamp) != NaN){
-			req.body.timestamp = Number(req.body.timestamp);
-			db.flows.insert({
-				instanceID: req.body.instanceID,
-				timestamp: req.body.timestamp,
-				data: req.body.data,
-			});
-			try{
-			Object.keys(req.body.data).forEach(itemName => {
-				prometheusProductionGauge.labels(req.body.instanceID, itemName).inc(Number(req.body.data[itemName]) || 0);
-			});
-			}catch(e){console.log(e)};
-			console.log("inserted: " + req.body.instanceID + " | " + req.body.timestamp);
-		} else {
-			console.log("error invalid timestamp " + req.body.timestamp);
-			res.send("failure");
-		}
-	} else {
-		res.send("failure");
-	}
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
-});
-// {instanceID: string, fromTime: Date, toTime, Date}
-/**
-POST endpoint to get timeSeries statistics stored on the master. Can give production
-graphs and other IO statistics.
-
-@memberof clusterioMaster
-@instance
-@alias api/getStats
-@param {object} JSON
-@returns {object} - with statistics
-*/
-app.post("/api/getStats", function(req,res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
-	if(typeof req.body == "string"){
-		req.body = JSON.parse(req.body);
-	}
-	// console.log(req.body);
-	if(typeof req.body == "object" && req.body.instanceID && req.body.statistic === undefined) {
-		// if not specified, get stats for last 24 hours
-		if(!req.body.fromTime) {
-			req.body.fromTime = Date.now() - 86400000 // 24 hours in MS
-		}
-		if(!req.body.toTime) {
-			req.body.toTime = Date.now();
-		}
-		console.log("Looking... " + JSON.stringify(req.body));
-		db.flows.find({
-			instanceID: req.body.instanceID,
-		}, function(err, docs) {
-			let entries = docs.filter(function (el) {
-				return el.timestamp <= req.body.toTime && el.timestamp >= req.body.fromTime;
-			});
-			// console.log(entries);
-			res.send(entries);
-		});
-	} else if(typeof req.body == "object" && req.body.instanceID && typeof req.body.instanceID == "string" && req.body.itemName){
-		if(req.body.statistic == "place"){
-			console.log(`sending place data for instanceID ${req.body.instanceID} ${req.body.itemName}`);
-			// Gather data
-			//console.log(recievedItemStatisticsBySlaveID)
-			let itemStats = recievedItemStatisticsBySlaveID[req.body.instanceID];
-			if(itemStats === undefined){
-				res.send({statusForDebugging:"no data available"});
-				return false;
-			}
-			let data = itemStats.get(config.itemStats.maxEntries, req.body.itemName);
-			
-			res.send({
-				maxEntries:config.itemStats.maxEntries,
-				entriesPerSecond: config.itemStats.entriesPerSecond,
-				data: data,
-			});
-		} else if(req.body.statistic == "remove"){
-			console.log(`sending remove data for instanceID ${req.body.instanceID} ${req.body.itemName}`);
-			// Gather data
-			// console.log(sentItemStatisticsBySlaveID)
-			let itemStats = sentItemStatisticsBySlaveID[req.body.instanceID];
-			console.log(itemStats)
-			if(typeof itemStats == "object"){
-				let data = itemStats.get(config.itemStats.maxEntries, req.body.itemName);
-				//console.log(itemStats.get(config.itemStats.maxEntries, req.body.itemName));
-				res.send({
-					maxEntries:config.itemStats.maxEntries,
-					entriesPerSecond: config.itemStats.entriesPerSecond,
-					data: data,
-				});
-			} else {
-				res.send({statusForDebugging:"no data available"});
-			}
-		}
-	}
-	httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 });
 
 /**
-POST endpoint. Modified version of api/getStats which feeds the web interface production graphs
+ GET endpoint to read the masters inventory as an object with key:value pairs
-at /nodes. It is like getStats but it does not report items which were not
-produced at the moment of recording. (This is to save space, the 0-items were
-making up about 92% of the response body weight.)
 
-@memberof clusterioMaster
+ @memberof clusterioMaster
-@instance
+ @instance
-@alias api/getTimelineStats
+ @alias api/modData
-@param {object} JSON {instanceID:1941029, fromTime: ???, toTime: ???}
+ @returns {object} JSON
-@returns {object[]} timeseries where each entry is a set point in time.
+ */
-*/
+
-app.post("/api/getTimelineStats", function(req,res) {
+// wrap a request in an promise
+    async function downloadPage(url) {
+        return new Promise((resolve, reject) => {
+            request(url, (error, response, body) => {
+                resolve(body);
+            });
+        });
+    }
+
+app.get("/api/modmeta", async function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
+    res.header("Access-Control-Allow-Origin", "*");
-	console.log(req.body);
+    res.setHeader('Content-Type', 'application/json');
-	if(typeof req.body == "object" && req.body.instanceID) {
+    let modData = await downloadPage("https://mods.factorio.com/api/mods/" + req.query.modname);
-		// if not specified, get stats for last 24 hours
+	res.send(modData);
-		if(!req.body.fromTime) {
-			req.body.fromTime = Date.now() - 86400000 // 24 hours in MS
-		}
-		if(!req.body.toTime) {
-			req.body.toTime = Date.now();
-		}
-		console.log("Looking... " + JSON.stringify(req.body));
-		db.flows.find({
-			instanceID: req.body.instanceID,
-		}, function(err, docs) {
-			if(err) { 
-				console.error(err);
-				return;
-			}
-			
-			//Filter out all the entries outside the time range.
-			docs = docs.filter(function (el) {
-				return el.timestamp <= req.body.toTime && el.timestamp >= req.body.fromTime;
-			});
-			
-			//Filter out all the elements that weren't produced.
-			docs.forEach(el => {
-				for(let key in el.data) {
-					if(el.data[key] === '0') {
-						// Set value to undefined instead of using the delete keyword.
-						// This is because the delete keyword is super duper slow.
-						// https://stackoverflow.com/questions/208105/how-do-i-remove-a-property-from-a-javascript-object
-						
-						//delete el.data[key];
-						el.data[key] = undefined;
-					}
-				}
-			});
-			
-			res.send(docs);
-			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
-		});
-	}
 });
+
+
 /**
 POST endpoint for running commands on slaves.
 Requires x-access-token header to be set. Find you api token in secret-api-token.txt on the master (after running it once)
@@ -879,7 +699,6 @@ Requires x-access-token header to be set. Find you api token in secret-api-token
 */
 app.post("/api/runCommand", (req,res) => {
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	var token = req.headers['x-access-token'];
 	if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' });
 	
@@ -902,7 +721,6 @@ app.post("/api/runCommand", (req,res) => {
                 wsSlaves[instanceID].runCommand(body.command);
             }
             res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"});
-			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
         } else if (
             body.instanceID
             && wsSlaves[body.instanceID]
@@ -913,11 +731,9 @@ app.post("/api/runCommand", (req,res) => {
             // execute command
             wsSlaves[body.instanceID].runCommand(body.command, data => {
                 res.status(200).send({auth: true, message: "success", data});
-				httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
             });
         } else {
             res.status(400).send({auth: true, message: "Error: invalid request.body"});
-			httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
         }
 	});
 });
@@ -931,30 +747,35 @@ GET endpoint. Returns factorio's base locale as a JSON object.
 */
 app.get("/api/getFactorioLocale", function(req,res){
 	endpointHitCounter.labels(req.route.path).inc();
-	let reqStartTime = Date.now();
 	getFactorioLocale.asObject(config.factorioDirectory, "en", (err, factorioLocale) => {
 		res.send(factorioLocale);
-		httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime);
 	});
 });
-if(config.sslPort){
+if(args.sslPort || config.sslPort){
-	var certificate = fs.readFileSync( 'database/certificates/cert.crt' );
+	try{
-	var privateKey = fs.readFileSync( 'database/certificates/cert.key' );
+		var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt'));
+		var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key'));
 
-	httpsServer = https.createServer({
+		httpsServer = require("https").createServer({
-		key: privateKey,
+			key: privateKey,
-		cert: certificate
+			cert: certificate
-	}, app).listen(config.sslPort);
+		}, app).listen(args.sslPort || config.sslPort);
+	} catch(e){
+		console.error(e)
+	}
 }
-if(config.masterPort){
+if(args.masterPort || config.masterPort){
 	server = require("http").Server(app);
-	server.listen(config.masterPort || 8080, function () {
+	server.listen(args.masterPort || config.masterPort || 8080, function () {
 		console.log("Listening on port %s...", server.address().port);
 	});
+	module.exports = server;
 }
 
 /* Websockets for remoteMap */
 var io = require("socket.io")(server);
+const ioMetrics = require("socket.io-prometheus");
+ioMetrics(io);
 var slaveMappers = {};
 class slaveMapper {
 	constructor(instanceID, socket) {
@@ -1098,7 +919,7 @@ io.on('connection', function (socket) {
 	if(terminatedConnections > 0) console.log("SOCKET | There are currently "+currentConnections+" websocket connections, deleting "+terminatedConnections+" on timeout");
 	
 	// tell our friend that we are listening
-	socket.emit('hello', { hello: 'world' });
+	setTimeout(()=>socket.emit('hello', { hello: 'world' }), 5000);
 	
 	/* initial processing for remoteMap */
 	socket.on('registerSlaveMapper', function (data) {
@@ -1140,62 +961,43 @@ pluginManagement();
 masterPlugins = [];
 async function pluginManagement(){
 	let startPluginLoad = Date.now();
-	masterPlugins = await getPlugins("sharedPlugins");
+	masterPlugins = await getPlugins();
 	console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms");
+	return;
 }
 
-function getPlugins(pluginDirectory){
+async function getPlugins(){
-	return new Promise((resolve, reject) => {
+	const pluginManager = require("./lib/manager/pluginManager.js")(config);
-		let filesLeft = 0;
+	let plugins = [];
-		let plugins = [];
+	let pluginsToLoad = await pluginManager.getPlugins();
+	for(let i = 0; i < pluginsToLoad.length; i++){
+		let pluginStartedLoading = Date.now(); // just for logging
+		let stats = await fs.stat(pluginsToLoad[i].pluginPath);
+		if(stats.isDirectory()){
+			let pluginConfig = pluginsToLoad[i];
+			if(pluginConfig.masterPlugin && pluginConfig.enabled){
+				let masterPlugin = require(path.resolve(pluginConfig.pluginPath, pluginConfig.masterPlugin));
+				plugins.push({
+					main:new masterPlugin({
+						config,
+						pluginConfig,
+						path: pluginConfig.pluginPath,
+						socketio: io,
+						express: app,
+						Prometheus,
+					}),
+					pluginConfig,
+				});
 				
-		function checkLoadComplete(plugin){
+				console.log("Loaded plugin "+pluginConfig.name+" in "+(Date.now() - pluginStartedLoading)+"ms");
-			if(!filesLeft){
-				resolve(plugins);
 			}
 		}
-		fs.readdir(pluginDirectory, function(err, files){
+	}
-			if(err){
+	for(let i in plugins){
-				reject(err);
+		let plugin = plugins[i];
-			} else {
+		if(plugin.main.onLoadFinish && typeof plugin.main.onLoadFinish == "function") await plugin.main.onLoadFinish({plugins});
-				files.forEach(file => {
+	}
-					let pluginStartedLoading = Date.now(); // just for logging
+	return plugins;
-					filesLeft++;
-					let pluginPath = path.join(pluginDirectory, file);
-					fs.stat(pluginPath, function(err, stats){
-						if(err){
-							reject(err);
-							checkLoadComplete();
-						} else if(stats.isDirectory()){
-							let pluginConfig = require(path.resolve(pluginPath, "config"));
-							--filesLeft;
-							if(pluginConfig.masterPlugin){
-								let masterPlugin = require(path.resolve(pluginPath, pluginConfig.masterPlugin));
-								plugins.push({
-									main:new masterPlugin({
-										config,
-										pluginConfig,
-										path: pluginPath,
-										socketio: io,
-										express: app,
-									}),
-									pluginConfig,
-								});
-								
-								console.log("Loaded plugin "+pluginConfig.name+" in "+(Date.now() - pluginStartedLoading)+"ms");
-								checkLoadComplete();
-							} else {
-								checkLoadComplete();
-							}
-						} else {
-							checkLoadComplete();
-						}
-					});
-				});
-			}
-		});
-	});
 }
 
-
+})();
-module.exports = app;

master.spec.js

@@ -76,17 +76,6 @@ describe('Master server endpoint testing', function() {
 				});
 			});
 		});
-		it("sends some HTML when accessing /remoteMap",function(done){
-			this.timeout(6000);
-			request(app).get("/remoteMap").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					// there should be 1 error regarding complaining about me using ES6 modules before they are fully supported
-					assert(result.messages.length === 2, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
 	});
 	let persistentMaster = request(app);
 	describe("#POST /api/place", function(){

node_modules/_app/clusterTools/checkHotpatchInstallation.js

@@ -0,0 +1,9 @@
+module.exports = async function checkHotpatchInstallation(messageInterface){
+	let yn = await messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end");
+	yn = yn.replace(/(\r\n\t|\n|\r\t)/gm, "");
+	if(yn == "true"){
+		return true;
+	} else if(yn == "false"){
+		return false;
+	}
+}

node_modules/_app/clusterTools/getCommand.js

@@ -0,0 +1,16 @@
+const fs = require("fs-extra");
+
+const getLua = require("./getLua.js");
+
+let commandCache;
+module.exports = async function getCommand(file, COMPRESS = true){
+	commandCache = commandCache || {};
+	if(!commandCache[file]){
+		commandCache[file] = await getLua(file);
+		return commandCache[file];
+	} else if(typeof commandCache[file] == "string"){
+		return commandCache[file];
+	} else {
+		throw new Error("Command not found");
+	}
+}

node_modules/_app/clusterTools/getInstanceName.js

@@ -0,0 +1,40 @@
+const needle = require("needle");
+
+module.exports = function(config){
+	const instances = [];
+	
+	if(config){
+		return function getInstanceName(instanceID){
+			return new Promise((resolve, reject) => {
+				let instance = instances[instanceID];
+				if(!instance){
+					needle.get(config.masterIP+":"+config.masterPort+ '/api/slaves', (err, response) => {
+						if(err || response.statusCode != 200) {
+							console.log("Unable to get JSON master/api/slaves, master might be unaccessible");
+						} else if (response && response.body) {	
+							if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); reject(new Error())}
+								try {
+									for (let index in response.body)
+										instances[index] = response.body[index].instanceName;
+								} catch (e){
+									console.log(e);
+									return null;
+								}
+							instance = instances[instanceID] 							
+							if (!instance) instance = instanceID;  //somehow the master doesn't know the instance	
+							resolve(instance);
+						}
+					});
+				} else {
+					resolve(instance);
+				}
+			});
+		}
+	} else {
+		return async function thisAintGonnaWork(){
+			let err = new Error("getInstanceName from _app/clusterTools needs to be called with {config} to work");
+			console.log(err);
+			throw err;
+		}
+	}
+}

node_modules/_app/clusterTools/getLua.js

@@ -0,0 +1,40 @@
+const fs = require("fs-extra");
+
+module.exports = function getLua(filePath, COMPRESS_LUA = true){
+	return new Promise((resolve, reject) => {
+		fs.readFile(filePath, "utf8", (err, contents) => {
+			if(err){
+				reject(err);
+			} else {
+				// split content into lines
+				contents = contents.split(/\r?\n/);
+
+				// join those lines after making them safe again
+				contents = contents.reduce((acc, val) => {
+					val = val.replace(/\\/g ,'\\\\');
+					// remove leading and trailing spaces
+					val = val.trim();
+					// escape single quotes
+					val = val.replace(/'/g ,'\\\'');
+
+					// remove single line comments
+					let singleLineCommentPosition = val.indexOf("--");
+					let multiLineCommentPosition = val.indexOf("--[[");
+
+					if(multiLineCommentPosition === -1 && singleLineCommentPosition !== -1) {
+						val = val.substr(0, singleLineCommentPosition);
+					}
+
+					return acc + val + '\\n';
+				}, ""); // need the "" or it will not process the first row, potentially leaving a single line comment in that disables the whole code
+
+				// console.log(contents);
+
+				// this takes about 46 ms to minify train_stop_tracking.lua in my tests on an i3 6100u
+				if(COMPRESS_LUA) contents = require("luamin").minify(contents);
+				
+				resolve(contents);
+			}
+		});
+	});
+}

node_modules/_app/clusterTools/index.js

@@ -0,0 +1,41 @@
+/**
+
+Functions frequently needed from different places in the cluster. Ex:
+  * Loading Lua scripts
+  * Checking if hotpatch is installed
+  * Getting an instance's displayname from their ID
+
+*/
+
+module.exports = ({config} = {}) => ({
+	/**
+		async function to get Lua from a file, remove comments, minify and return.
+		
+		arg1 is the file path
+		arg2 is whether or not to compress the Lua. Makes debugging harder, but reduces bandwidth when many players are online. Recommended for polling commands. Defaults to true.
+		Promise resolves with a string of Lua.
+	*/
+	getLua: require("./getLua.js"),
+	/**
+		async function to check whether hotpatch is installed in a server.
+		
+		First argument is the servers async messageInterface for rcon return values
+		Promise resolves with boolean.
+	*/
+	checkHotpatchInstallation: require("./checkHotpatchInstallation.js"),
+	/**
+		async function to get an instances displayname from their instanceID
+		
+		arg1 is the instanceID
+		Promise resolves with a string, either the displayName or instanceID if not found.
+	*/
+	getInstanceName: require("./getInstanceName.js")(config),
+	/**
+		async function same as getLua, but using an in memory cache to lower FS operations.
+		
+		arg1 is the file path
+		arg2 is whether or not to compress, default true
+		Promise resolves with a string of Lua.
+	*/
+	getCommand: require("./getCommand.js"),
+})

package.json

@@ -10,7 +10,10 @@
     "cover": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{{,node_modules/_app/**/},!(node_modules)/**/}*.spec.js\" -R spec || echo . && codecov -t f7b38085-36eb-4d72-84e6-64721910171e",
     "docs": "documentation build index.js --config documentation.yml -f html -o docs",
     "livedocs": "documentation serve --watch *.js lib/** --config documentation.yml",
-    "install": "node lib/npmPostinstall.js"
+    "install": "node lib/npmPostinstall.js",
+    "bundle-dependencies": "bundle-dependencies",
+    "snyk-protect": "snyk protect",
+    "prepublish": "npm run snyk-protect"
   },
   "keywords": [
     "factorio"
@@ -30,10 +33,12 @@
     "cli-interact": "^0.1.9",
     "codecov": "^3.0.1",
     "compression": "^1.7.2",
+    "cookie-parser": "^1.4.3",
     "deepmerge": "^1.5.2",
     "ejs": "^2.5.7",
     "express": "^4.16.2",
     "express-fileupload": "0.0.6",
+    "express-prometheus-request-metrics": "github:burnash/express-prometheus-request-metrics",
     "follow-redirects": "^1.5.0",
     "fs-extra": "^6.0.1",
     "getmac": "^1.4.0",
@@ -41,15 +46,13 @@
     "html5-validator": "^1.0.7",
     "ini": "^1.3.5",
     "jsonwebtoken": "^8.2.1",
-    "leveldown": "^1.9.0",
-    "linvodb3": "^3.25.1",
     "luamin": "^1.0.4",
+    "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
     "mkdirp-promise": "^5.0.1",
-    "moment": "^2.20.1",
+    "moment": "^2.22.2",
     "ncp": "^2.0.0",
-    "nedb": "^1.8.0",
+    "needle": "^2.2.4",
-    "needle": "^1.1.0",
     "node-factorio-api": "^0.3.1-alpha",
     "node-forge": "^0.7.1",
     "prom-client": "^10.2.3",
@@ -58,11 +61,14 @@
     "right-pad": "^1.0.1",
     "rmdir-sync": "^1.0.1",
     "sanitizer": "^0.1.3",
+    "simple-git": "^1.96.0",
     "single-line-log": "^1.1.2",
     "socket.io": "^2.0.4",
     "socket.io-client": "^2.0.4",
+    "socket.io-prometheus": "^0.2.1",
     "sync-request": "^4.0.1",
-    "underscore": "^1.8.3"
+    "underscore": "^1.8.3",
+    "snyk": "^1.134.2"
   },
   "devDependencies": {
     "documentation": "^5.4.0",
@@ -73,5 +79,6 @@
     "nock": "^9.1.6",
     "sinon": "^2.4.1",
     "supertest": "^3.0.0"
-  }
+  },
+  "snyk": true
 }

routes.js

@@ -11,7 +11,4 @@ module.exports = function(app) {
 	app.get("/nodeDetails", function(req, res) {
 		res.render("nodeDetails");
 	});
-	app.get("/remoteMap", function(req, res) {
-		res.render("remoteMap");
-	});
 };

routes/api/getPictures.js

@@ -0,0 +1,16 @@
+const fs = require("fs-extra")
+const path = require("path")
+
+module.exports = function(app){
+	app.get("/api/getPictures", async (req, res) => {
+		let pictures = (await fs.readdir(path.join(__dirname, `../../static/pictures`)))
+		.filter(path => path.includes(".png"))
+		.map(name => ({
+			name: name.replace(".png",""),
+			path: `/pictures/${name}`
+		}))
+		
+		
+		res.send(pictures)
+	})
+}

routes/api/remove.js

@@ -0,0 +1,180 @@
+const averagedTimeSeries = require("averaged-timeseries")
+
+const doleNN = require("./../../lib/dole_nn_base.js")
+
+const Prometheus = require('prom-client');
+
+class neuralDole {
+    
+    getRequestStats(itemname,samples)
+    {
+        var sum=0;
+        samples=Math.min(samples,(this.stats[itemname] || []).length-1);
+        if (samples<1) return 0.1;
+        for (var i = 1; i <= samples; i++) {
+            sum+=this.stats[itemname][i].req;
+        }
+        if (sum==0) return 0.1;
+        return sum/samples;
+    }
+    
+    constructor({
+        items, gaugePrefix
+    }){
+        // Set some storage variables for the dole divider
+        this.prometheusNNDoleGauge = new Prometheus.Gauge({
+            name: gaugePrefix+'nn_dole_gauge',
+            help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
+            labelNames: ["itemName"],
+        });
+        this.items = items
+        this.itemsLastTick = JSON.parse(JSON.stringify(items))
+        this.dole = {}
+        this.carry = {}
+        this.lastRequest = {}
+        this.stats=[]
+
+        setInterval(()=>{
+            for(let name in this.items){
+                // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties.
+                if(typeof this.items[name] !== "number") continue
+
+                let count = this.items[name]
+                let magicData = doleNN.Tick(
+                    count,
+                    this.dole[name],
+                    this.itemsLastTick[name],
+                    this.getRequestStats(name,5)
+                )
+                this.stats[name]=this.stats[name] || []
+                this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
+                if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
+                
+                this.dole[name] = magicData[0]
+                // DONE handle magicData[1] for graphing for our users
+                this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
+            }
+            this.itemsLastTick = JSON.parse(JSON.stringify(this.items))
+        }, 1000)
+    }
+    divider({
+        res,
+        object,
+        config,
+        sentItemStatisticsBySlaveID,
+        prometheusImportGauge
+    }){
+        let magicData = doleNN.Dose(
+            object.count, // numReq
+            this.items[object.name],
+            this.itemsLastTick[object.name] || 0,
+            this.dole[object.name],
+            this.carry[object.name+" "+object.instanceID] || 0,
+            this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
+            this.getRequestStats(object.name,5)
+        )
+        if ((this.stats[object.name] || []).length>0)
+        {
+            this.stats[object.name][0].req+=Number(object.count);
+            this.stats[object.name][0].given+=Number(magicData[0]);
+        }
+        this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
+        //0.Number of items to give in that dose
+        //1.New dole for item X
+        //2.New carry for item X slave Y
+        this.dole[object.name] = magicData[1]
+        this.carry[object.name+" "+object.instanceID] = magicData[2]
+        
+        // Remove item from DB and send it
+        if(this.items.removeItem({count: magicData[0], name: object.name})){
+            if(config.logItemTransfers){
+                console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName);
+            }
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:magicData[0],
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+            prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
+
+            res.send({
+                name: object.name,
+                count: magicData[0],
+            })
+        } else {
+            res.send({
+                name: object.name,
+                count: 0,
+            })
+        }        
+    }
+}
+
+_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
+function doleDivider({
+    item,
+    object,
+    db,
+    sentItemStatisticsBySlaveID,
+    config,
+    prometheusDoleFactorGauge,
+    prometheusImportGauge,
+    req,res,
+}){
+    const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
+    const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
+    
+    const originalCount = Number(object.count) || 0;
+    object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+    object.count = Math.round(object.count);
+    if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
+    
+    // Update existing items if item name already exists
+    if(Number(item) > Number(object.count)) {
+        //If successful, increase dole
+        _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+        if(config.logItemTransfers){
+            console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
+        }
+        if(db.items.removeItem({count: object.count, name: object.name})){
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:object.count,
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+        }
+        
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
+        res.send({count: object.count, name: object.name});
+    } else {
+        // if we didn't have enough, attempt giving out a smaller amount next time
+        _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        res.send({name:object.name, count:0});
+        //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
+    }
+}
+
+module.exports = {
+    doleDivider,
+    neuralDole
+}

scenarios/remoteMap/control.lua

@@ -2,6 +2,8 @@
 -- 
 -- Copyright (c) 2017 Florian Jung
 -- 
+-- Modified by Daniel Vestøl
+--
 -- Permission is hereby granted, free of charge, to any person obtaining a
 -- copy of this factorio lua stub and associated
 -- documentation files (the "Software"), to deal in the Software without
@@ -22,12 +24,13 @@
 -- DEALINGS IN THE SOFTWARE.
 
 function write_file(data)
-	stringsToWriteNextTick = stringsToWriteNextTick .. data .. '\n'
+	global.tileQueue = global.tileQueue .. data .. '\n'
+	-- stringsToWriteNextTick = stringsToWriteNextTick .. data .. '\n'
 	-- game.write_file("remoteMap.txt", data, true)
 end
 
 function dump_cached_writes()
-	game.write_file("remoteMap.txt", stringsToWriteNextTick, true, 0)
+	-- game.write_file("remoteMap.txt", stringsToWriteNextTick, true, 0)
 end
 
 function complain(text)
@@ -89,7 +92,7 @@ function on_chunk_generated(event)
 	end
 
 	-- writeout_resources(surface, area)
-	writeout_objects(surface, area)
+	-- writeout_objects(surface, area)
 	-- writeout_tiles(surface, area)
 end
 function on_some_entity_created(event, entp)
@@ -104,7 +107,7 @@ function on_some_entity_created(event, entp)
 		complain("wtf, on_some_entity_created has nil entity")
 		return
 	end
-	local entityData = ent.name..","..ent.position.x..","..ent.position.y
+	local entityData = "name="..ent.name..",x="..ent.position.x..",y="..ent.position.y
 	if ent.supports_direction then
 		entityData = entityData .. ",rot="..ent.direction
 	end
@@ -126,7 +129,7 @@ function on_some_entity_deleted(event)
 	--local area = {left_top={x=math.floor(ent.position.x), y=math.floor(ent.position.y)}, right_bottom={x=math.floor(ent.position.x)+1, y=math.floor(ent.position.y)+1}}
 
 	--table.insert(todo_next_tick, function () writeout_objects(surface, area ) end)
-	write_file("deleted,"..ent.position.x..","..ent.position.y)
+	write_file("name=deleted,x="..ent.position.x..",y="..ent.position.y)
 	--complain("on_some_entity_deleted: "..ent.name.." at "..ent.position.x..","..ent.position.y)
 end
 
@@ -143,3 +146,22 @@ script.on_event(defines.events.on_player_mined_entity, on_some_entity_deleted) -
 script.on_event(defines.events.on_robot_mined_entity, on_some_entity_deleted) --entity
 script.on_event(defines.events.on_resource_depleted, on_some_entity_deleted) --entity
 
+script.on_init(function()
+	global.tileQueue = ""
+end)
+
+remote.remove_interface("remoteMap")
+remote.add_interface("remoteMap", {
+	exportTiles = function()
+		rcon.print(global.tileQueue)
+		global.tileQueue = ""
+	end,
+	exportChunk = function(xc, yc, ref)
+		local data = ""
+		local entities = game.surfaces[1].find_entities({{xc*32,yc*32},{xc*32+32,yc*32+32}})
+		for _, ent in pairs(entities) do
+			data = data .. "|name:"..ent.name..",direction:"..ent.direction..",x:"..ent.position.x..",y:"..ent.position.y
+		end
+		rcon.print(data)
+	end
+})

sharedPlugins/clusterioMod/config.js

@@ -0,0 +1,11 @@
+/*
+	Clusterio plugin to allow for chat between instances.
+*/
+module.exports = {
+	// Name of package. For display somewhere I guess.
+	name: "clusterioMod",
+	version: "1.0.0",
+	binary: "nodePackage",
+	description: "Server side handling of functionality in the clusterio lua mod",
+	masterPlugin: "masterPlugin.js",
+}

sharedPlugins/clusterioMod/index.js

@@ -0,0 +1,384 @@
+const needle = require("needle");
+const fs = require("fs-extra");
+
+const objectOps = require("./../../lib/objectOps.js");
+const fileOps = require("_app/fileOps");
+const stringUtils = require("./../../lib/stringUtils.js");
+
+const pluginConfig = require("./config");
+const COMPRESS_LUA = false;
+
+module.exports = class remoteCommands {
+	constructor(mergedConfig, messageInterface, extras){
+		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		this.socket = extras.socket;
+		this.socket.on("hello", () => {
+			
+		});
+		const that = this;
+		const instance = process.argv[3];
+		const instancedirectory = this.config.instanceDirectory + '/' + instance;
+		
+		const needleOptionsWithTokenAuthHeader = {
+			headers: {
+				'x-access-token': this.config.masterAuthToken
+			},
+		};
+		
+		(async ()=>{
+			let hotpatchInstallStatus = await this.checkHotpatchInstallation();
+			this.messageInterface("Hotpach installation status: "+hotpatchInstallStatus);
+			if(hotpatchInstallStatus){
+				// let mainCode = await this.getSafeLua("sharedPlugins/playerManager/lua/playerTracking.lua");
+				// if(mainCode) var returnValue = await messageInterface("/silent-command remote.call('hotpatch', 'update', '"+pluginConfig.name+"', '"+pluginConfig.version+"', '"+mainCode+"')");
+			}
+		})().catch(e => console.log(e));
+		
+		// flow/production statistics ------------------------------------------------------------
+		var oldFlowStats = false;
+		var oldTimestamp;
+		var oldFlowStats;
+		setInterval(function(){
+			fs.readFile(instancedirectory + "/script-output/flows.txt", {encoding: "utf8"}, function(err, data) {
+				if(!err && data) {
+					let timestamp = Date.now();
+					data = data.split("\n");
+					let flowStats = [];
+					for(let i = 0; i < data.length; i++) {
+						// try catch to remove any invalid json
+						try{
+							flowStats[flowStats.length] = JSON.parse(data[i]);
+						} catch (e) {
+							// console.log(" invalid json: " + i);
+							// some lines of JSON are invalid but don't worry, we just filter em out
+						}
+					}
+					// fluids
+					let flowStat1 = flowStats[flowStats.length-1].flows.player.input_counts
+					// items
+					let flowStat2 = flowStats[flowStats.length-2].flows.player.input_counts
+					// merge fluid and item flows
+					let totalFlows = {};
+					for(let key in flowStat1) totalFlows[key] = flowStat1[key];
+					for(let key in flowStat2) totalFlows[key] = flowStat2[key];
+					if(oldFlowStats && totalFlows && oldTimestamp) {
+						let payload = objectOps.deepclone(totalFlows);
+						// change from total reported to per time unit
+						for(let key in oldFlowStats) {
+							// get production per minute
+							payload[key] = Math.floor((payload[key] - oldFlowStats[key])/(timestamp - oldTimestamp)*60000);
+							if(payload[key] < 0) {
+								payload[key] = 0;
+							}
+						}
+						for(let key in payload) {
+							if(payload[key] == '0') {
+								delete payload[key];
+							}
+						}
+						console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]);
+						needle.post(that.config.masterIP + ":" + that.config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: that.config.unique, data: payload}, needleOptionsWithTokenAuthHeader, function (err, response, body) {
+							// we did it, keep going
+						});
+					}
+					oldTimestamp = timestamp;
+					oldFlowStats = totalFlows;
+					fs.writeFileSync(instancedirectory + "/script-output/flows.txt", "");
+				}
+			});
+			// we don't need to update stats quickly as that could be expensive
+		}, 60000*5);
+		
+		// provide items --------------------------------------------------------------
+		// trigger when something happens to output.txt
+		fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) {
+			// get array of lines in file
+			let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n");
+			// if you found anything, reset the file
+			if (items[0]) {
+				fs.writeFileSync(instancedirectory + "/script-output/output.txt", "");
+			}
+			for (let i = 0; i < items.length; i++) {
+				if (items[i]) {
+					let g = items[i].split(" ");
+					g[0] = g[0].replace("\u0000", "");
+					// console.log("exporting " + JSON.stringify(g));
+					// send our entity and count to the master for him to keep track of
+					needle.post(that.config.masterIP + ":" + that.config.masterPort + '/api/place', {
+						name: g[0],
+						count: g[1],
+						instanceName: instance, // name of instance
+						instanceID: that.config.unique, // a hash computed from the randomly generated rcon password
+					}, needleOptionsWithTokenAuthHeader, function (err, resp, body) {
+						if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]);
+						if(that.config.logItemTransfers){
+							if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`);
+						}
+					});
+				}
+			}
+		});
+		
+		// request items --------------------------------------------------------------
+		setInterval(function () {
+			// get array of lines in file
+			let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n");
+			// if we actually got anything from the file, proceed and reset file
+			if (items[0]) {
+				fs.writeFileSync(instancedirectory + "/script-output/orders.txt", "");
+				// prepare a package of all our requested items in a more transfer friendly format
+				var preparedPackage = {};
+				for (let i = 0; i < items.length; i++) {
+					(function (i) {
+						if (items[i]) {
+							items[i] = items[i].split(" ");
+							items[i][0] = items[i][0].replace("\u0000", "");
+							items[i][0] = items[i][0].replace(",", "");
+							if (preparedPackage[items[i][0]]) {
+								// if we have buffered some already, sum the new items
+								if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") {
+									preparedPackage[items[i][0]] = {
+										"name": items[i][0],
+										"count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]),
+										"instanceName":instance,
+										"instanceID":that.config.unique,
+									};
+								// else just add em in without summing
+								} else if (typeof Number(items[i][1]) == "number") {
+									preparedPackage[items[i][0]] = {
+										"name": items[i][0],
+										"count": Number(items[i][1]),
+										"instanceName":instance,
+										"instanceID":that.config.unique,
+									};
+								}
+							// this condition will NEVER be triggered but we know how that goes
+							} else if (typeof Number(items[i][1]) == "number") {
+								preparedPackage[items[i][0]] = {
+									"name": items[i][0],
+									"count": Number(items[i][1]),
+									"instanceName":instance,
+									"instanceID":that.config.unique,
+								};
+							}
+						}
+					})(i);
+				}
+				// request our items, one item at a time
+				for (let i = 0; i < Object.keys(preparedPackage).length; i++) {
+					// console.log(preparedPackage[Object.keys(preparedPackage)[i]]);
+					needle.post(that.config.masterIP + ":" + that.config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) {
+						if (response && response.body && typeof response.body == "object") {
+							// buffer confirmed orders
+							confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count}
+							if(that.config.logItemTransfers){
+								console.log(`Imported ${response.body.count} ${response.body.name} from master`);
+							}
+						}
+					});
+				}
+				// if we got some confirmed orders
+				// console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders));
+				//if (!(confirmedOrders.length>0)){return;}
+				let cmd="local t={";
+				for(let i=0;i<confirmedOrders.length;i++)
+				{
+					cmd+='["'+confirmedOrders[i].name+'"]='+confirmedOrders[i].count+',';
+				}
+				if (!(cmd==="local t={")){
+					that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
+				}
+				confirmedOrders=[];
+			}
+		}, 1000);
+		// COMBINATOR SIGNALS ---------------------------------------------------------
+		// get inventory from Master and RCON it to our slave
+		setInterval(() => {
+			needle.get(that.config.masterIP + ":" + that.config.masterPort + '/api/inventory', function (err, response, body) {
+				if(err){
+					console.error("Unable to get JSON master/api/inventory, master might be unaccessible");
+				} else if (response && response.body) {
+					// Take the inventory we (hopefully) got and turn it into the format LUA accepts
+					if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver
+					try {
+						var inventory = JSON.parse(response.body);
+						var inventoryFrame = {};
+						for (let i = 0; i < inventory.length; i++) {
+							inventoryFrame[inventory[i].name] = Number(inventory[i].count);
+							if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){
+								inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice
+							}
+						}
+						inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000);
+						// console.log("RCONing inventory! " + JSON.stringify(inventoryFrame));
+						let first = true;
+						let cmd="local s={";
+						for (let key in inventoryFrame)
+						{
+							cmd+='["'+key+'"]='+inventoryFrame[key]+",";
+						}
+						if (!(cmd==="local s={")){
+							that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
+						}
+						that.messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')");
+					} catch (e){
+						console.log(e);
+					}
+				}
+			});
+		}, 1550);
+		// Make sure world has its worldID
+		setTimeout(function(){
+			messageInterface("/silent-command remote.call('clusterio','setWorldID',"+that.config.unique+")")
+		}, 20000);
+		/* REMOTE SIGNALLING
+		 * send any signals the slave has been told to send
+		 * Fetch combinator signals from the server
+		*/
+		this.socket.on("processCombinatorSignal", circuitFrameWithMeta => {
+			if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object" && circuitFrameWithMeta.frame && Array.isArray(circuitFrameWithMeta.frame)){
+				messageInterface("/silent-command remote.call('clusterio', 'receiveFrame', '"+JSON.stringify(circuitFrameWithMeta.frame)+"')");
+			}
+		});
+		// get outbound frames from file and send to master
+		// get array of lines in file, each line should correspond to a JSON encoded frame
+		let signals = fs.readFileSync(instancedirectory + "/script-output/txbuffer.txt", "utf8").split("\n");
+		// if we actually got anything from the file, proceed and reset file
+		let readingTxBufferSoon = false;
+		let txBufferClearCounter = 0;
+		fs.watch(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (eventType, filename) => {
+			if(!readingTxBufferSoon){ // use a 100ms delay to avoid messing with rapid sequential writes from factorio (I think that might be a problem maybe?)
+				readingTxBufferSoon = true;
+				setTimeout(()=>{
+					txBufferClearCounter++;
+					fs.readFile(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (err, signals) => {
+						signals = signals.split("\n");
+						if (signals[0]) {
+							//if(txBufferClearCounter > 500){
+								fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", "");
+							//	txBufferClearCounter = 0;
+							//}
+							
+							// loop through all our frames
+							for (let i = 0; i < signals.length; i++) {
+								if (signals[i] && objectOps.isJSON(signals[i])) {
+									// signals[i] is a JSON array called a "frame" of signals. We timestamp it for storage on master
+									// then we unpack and RCON in this.frame to the game later.
+									let framepart = JSON.parse(signals[i]);
+									let doneframe = {
+										time: Date.now(),
+										frame: framepart, // thats our array of objects(single signals);
+									}
+									// send to master using socket.io, opened at the top of instanceManagement()
+									this.socket.emit("combinatorSignal", doneframe);
+								} else {
+									// console.log("Invalid jsony: "+typeof signals[i])
+								}
+							}
+						}
+					});
+					readingTxBufferSoon = false;
+				},100);
+			}
+		});
+	}
+	async getCommand(file){
+		this.commandCache = this.commandCache || {};
+		if(!this.commandCache[file]){
+			try{
+				let command = (await fs.readFile(file)).toString();
+				this.commandCache[file] = command.replace(/(\r\n\t|\n|\r\t)/gm, " "); // remove newlines
+				return this.commandCache[file];
+			} catch(e){
+				console.log("Unable to get command from file!");
+				console.log(e)
+			}
+		} else if(typeof this.commandCache[file] == "string"){
+			return this.commandCache[file];
+		} else {
+			throw new Error("Command not found");
+		}
+	}
+	async factorioOutput(data){
+		
+	}
+	getInstanceName(instanceID){
+		return new Promise((resolve, reject) => {
+			let instance = this.instances[instanceID];
+			if(!instance){
+				needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => {
+					if(err || response.statusCode != 200) {
+						console.log("Unable to get JSON master/api/slaves, master might be unaccessible");
+					} else if (response && response.body) {	
+						if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();}
+							try {
+								for (let index in response.body)
+									this.instances[index] = response.body[index].instanceName;
+							} catch (e){
+								console.log(e);
+								return null;
+							}
+						instance = this.instances[instanceID] 							
+						if (!instance) instance = instanceID;  //somehow the master doesn't know the instance	
+						resolve(instance);
+					}
+				});
+			} else {
+				resolve(instance);
+			}
+		});
+	}
+	async getSafeLua(filePath){
+		return new Promise((resolve, reject) => {
+			fs.readFile(filePath, "utf8", (err, contents) => {
+				if(err){
+					reject(err);
+				} else {
+                    // split content into lines
+					contents = contents.split(/\r?\n/);
+
+					// join those lines after making them safe again
+					contents = contents.reduce((acc, val) => {
+                        val = val.replace(/\\/g ,'\\\\');
+                        // remove leading and trailing spaces
+					    val = val.trim();
+                        // escape single quotes
+					    val = val.replace(/'/g ,'\\\'');
+
+					    // remove single line comments
+                        let singleLineCommentPosition = val.indexOf("--");
+                        let multiLineCommentPosition = val.indexOf("--[[");
+
+						if(multiLineCommentPosition === -1 && singleLineCommentPosition !== -1) {
+							val = val.substr(0, singleLineCommentPosition);
+						}
+
+                        return acc + val + '\\n';
+					}, ""); // need the "" or it will not process the first row, potentially leaving a single line comment in that disables the whole code
+
+					// console.log(contents);
+
+					// this takes about 46 ms to minify train_stop_tracking.lua in my tests on an i3
+					if(COMPRESS_LUA) contents = require("luamin").minify(contents);
+					
+					resolve(contents);
+				}
+			});
+		});
+	}
+	async checkHotpatchInstallation(){
+		let yn = await this.messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end");
+		yn = yn.replace(/(\r\n\t|\n|\r\t)/gm, "");
+		if(yn == "true"){
+			return true;
+		} else if(yn == "false"){
+			return false;
+		}
+	}
+}
+async function sleep(s){
+	return new Promise((resolve, reject) => {
+		setTimeout(resolve, s*1000);
+	});
+}

sharedPlugins/clusterioMod/masterPlugin.js

@@ -0,0 +1,83 @@
+const fs = require("fs-extra");
+const path = require("path");
+const Express = require("express");
+const ejs = require("ejs");
+
+class masterPlugin {
+	constructor({config, pluginConfig, pluginPath, socketio, express}){
+		this.config = config;
+		this.pluginConfig = pluginConfig;
+		this.pluginPath = pluginPath;
+		this.io = socketio;
+		this.app = express;
+		
+		// load databases
+		this.items = getDatabaseSync("database/items.json");
+		
+		this.clients = {};
+		this.slaves = {};
+		
+		// initialize web API
+		// require("./js/api-endpoints.js")(this);
+		
+		// expose UI elements embedded in the master
+		// this.ui = require("./js/ui.js").ui;
+		
+		this.io.on("connection", socket => {
+			
+		});
+	}
+	async broadcastCommand(command){
+		let returnValues = [];
+		for(let instanceID in this.slaves){
+			let slave = this.slaves[instanceID];
+			slave.emit("runCommand", {
+				// commandID:Math.random(),
+				command,
+			});
+		};
+		return returnValues;
+	}
+	findInArray(key, value, array){
+		let indexes = [];
+		for(let i in array){
+			if(array[i][key] && array[i][key] === value) indexes.push(i);
+		}
+		return indexes;
+	}
+	async onExit(){
+		// await saveDatabase("database/items.json", this.items);
+		return;
+	}
+}
+module.exports = masterPlugin;
+
+function arrayRemoveDuplicates(array){
+	let newArray = [];
+	array.forEach(value => {
+		if(!newArray.includes(value)) newArray.push(value);
+	});
+	return newArray;
+}
+function getDatabaseSync(path){
+	let db;
+	try {
+		db = JSON.parse(fs.readFileSync(path, "utf8"));
+	} catch(e){
+		db = {};
+	}
+	return db;
+}
+async function saveDatabase(path, database){
+	if(!path){
+		throw new Error("No path provided!");
+	} else if(!database){
+		throw new Error("No database provided!");
+	} else {
+		try {
+			await fs.writeFile(path, JSON.stringify(database, null, 4));
+		} catch(e){
+			throw new Error("Unable to write to database! "+path);
+		}
+	}
+}

sharedPlugins/globalChat/index.js

@@ -11,7 +11,13 @@ module.exports = class remoteCommands {
 	
 		this.socket.on("gameChat", async data => {
 			if(data.instanceID && data.instanceID != this.config.unique.toString()){
-				if(data.data.includes("[CHAT]") && (data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){
+				if(data.data.includes("[CHAT]")
+				// if this isn't used as an info command or something (reduce spam)
+				&& !data.data.includes("!info")
+				// check if we are allowed to cross chat at all, default to true
+				&& (mergedConfig.enableCrossServerShout == undefined || mergedConfig.enableCrossServerShout)
+				// check if we are supposed to relay this message
+				&& (mergedConfig.mirrorAllChat || data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){
 					let words = data.data.trim().split(" ");
 					words.shift();
 					words.shift();

sharedPlugins/inventoryImports/index.js

@@ -12,7 +12,7 @@ class inventoryImports {
 		this.messageInterface = messageInterface;
 		messageInterface("/silent-command game.print('InventoryImports enabled')");
 		setInterval(function(){
-			messageInterface("Polling inventories\n")
+			// messageInterface("Polling inventories\n")
 			messageInterface(functions.pollInventories(pluginConfig.scriptOutputFileSubscription));
 		}, 10000);
 	}

sharedPlugins/inventoryImports/interfacingFunctions.js

@@ -141,7 +141,7 @@ function pollInventories(outputFile){
 	//console.log("/silent-command "+'local a=game.players["Danielv123"].get_inventory(defines.inventory.player_main).get_contents()local b=game.players["Danielv123"].get_quickbar().get_contents()local c={}for d=1,game.players["Danielv123"].force.character_logistic_slot_count do c[d]=game.players["Danielv123"].character.get_request_slot(d)end;local e="{inventory:{"for f,g in pairs(a)do e=e.."['+"'"+'"..f.."'+"'"+']:"..g..","end;for f,g in pairs(b)do e=e.."['+"'"+'"..f.."'+"'"+']:"..g..","end;e=e.."},requestSlots:{"for f,g in pairs(c)do e=e.."['+"'"+'"..g["name"].."'+"'"+']:"..g["count"]..", "end;e=e.."}}"game.write_file("'+pluginConfig.scriptOutputFileSubscription+'",e)')
 	// console.log("/silent-command "+'local a="{"for b,c in pairs(game.players)do local d=game.players[b].get_inventory(defines.inventory.player_main).get_contents()local e=game.players[b].get_quickbar().get_contents()local f={}for g=1,game.players[b].force.character_logistic_slot_count do f[g]=game.players[b].character.get_request_slot(g)end;a=a..b..":{inventory:{"for h,i in pairs(d)do a=a.."['+"'"+'"..h.."'+"'"+']:"..i..","end;for h,i in pairs(e)do a=a.."['+"'"+'"..h.."'+"'"+']:"..i..","end;a=a.."},requestSlots:{"for h,i in pairs(f)do a=a.."['+"'"+'"..i["name"].."'+"'"+']:"..i["count"]..", "end;a=a.."}},"end;a=a.."}"game.write_file("'+pluginConfig.scriptOutputFileSubscription+'",a)')
 	//return "/silent-command "+'local a="{"for b,c in pairs(game.players)do if c.connected then local d=game.players[b].get_inventory(defines.inventory.player_main).get_contents()local e=game.players[b].get_quickbar().get_contents()local f={}for g=1,game.players[b].force.character_logistic_slot_count do f[g]=game.players[b].character.get_request_slot(g)end;a=a..b..":{inventory:{"for h,i in pairs(d)do a=a.."['+"'"+'"..h.."'+"'"+']:"..i..","end;for h,i in pairs(e)do a=a.."['+"'"+'"..h.."'+"'"+']:"..i..","end;a=a.."},requestSlots:{"for h,i in pairs(f)do a=a.."['+"'"+'"..i["name"].."'+"'"+']:"..i["count"]..", "end;a=a.."}},"end end;a=a.."}"game.write_file("'+outputFile+'",a)'
-	return '/silent-command '+'local a='+"'"+'{"players":{'+"'"+'local b=false;for c,d in pairs(game.players)do if d.connected and d.character then if b then a=a..'+"'"+','+"'"+'else b=true end;local e=game.players[c].get_inventory(defines.inventory.player_main).get_contents()local f=game.players[c].get_quickbar().get_contents()local g={}for h=1,game.players[c].force.character_logistic_slot_count do g[h]=game.players[c].character.get_request_slot(h)end;a=a..'+"'"+'"'+"'"+'..c..'+"'"+'":{"inventory":{'+"'"+'local i=false;for j,k in pairs(e)do if i then a=a..'+"'"+','+"'"+'else i=true end;a=a..'+"'"+'"'+"'"+'..j..'+"'"+'":'+"'"+'..k end;for j,k in pairs(f)do if i then a=a..'+"'"+','+"'"+'else i=true end;a=a..'+"'"+'"'+"'"+'..j..'+"'"+'":'+"'"+'..k end;a=a..'+"'"+'},"requestSlots":{'+"'"+'i=false;for j,k in pairs(g)do if i then a=a..'+"'"+','+"'"+'else i=true end;a=a..'+"'"+'"'+"'"+'..k['+"'"+'name'+"'"+']..'+"'"+'":'+"'"+'..k['+"'"+'count'+"'"+']end;a=a..'+"'"+'}}'+"'"+'end end;a=a.."}}"game.write_file("'+outputFile+'",a,true,0)'
+	return '/silent-command '+'local a='+"'"+'{"players":{'+"'"+'local b=false;for c,d in pairs(game.players)do if d.connected and d.character then if b then a=a..'+"'"+','+"'"+'else b=true end;local e=game.players[c].get_inventory(defines.inventory.player_main).get_contents()local f=game.players[c].get_quickbar().get_contents()local g={}local z={}for j,k in pairs(e)do if z[j] ~= nil then z[j] = z[j] + k else z[j] = k end end for j,k in pairs(f)do if z[j] ~= nil then z[j] = z[j] + k else z[j] = k end end for h=1,game.players[c].force.character_logistic_slot_count do g[h]=game.players[c].character.get_request_slot(h)end;a=a..'+"'"+'"'+"'"+'..c..'+"'"+'":{"inventory":{'+"'"+'local i=false;for j,k in pairs(z)do if i then a=a..'+"'"+','+"'"+'else i=true end;a=a..'+"'"+'"'+"'"+'..j..'+"'"+'":'+"'"+'..k end;a=a..'+"'"+'},"requestSlots":{'+"'"+'i=false;for j,k in pairs(g)do if i then a=a..'+"'"+','+"'"+'else i=true end;a=a..'+"'"+'"'+"'"+'..k['+"'"+'name'+"'"+']..'+"'"+'":'+"'"+'..k['+"'"+'count'+"'"+']end;a=a..'+"'"+'}}'+"'"+'end end;a=a.."}}"game.write_file("'+outputFile+'",a,true,0)'
 	/*
 	needle.post(config.masterIP+':'+config.masterPort+'/api/editSlaveMeta', {instanceID: config.unique, password: config.clientPassword, meta: {UPS:UPS}}, function(err, resp) {
 		// success?

sharedPlugins/remoteMap/chunkStore.js

@@ -1,98 +0,0 @@
-var mkdirp = require("mkdirp-promise");
-var mkdirpSync = require("mkdirp").sync;
-var fs = require("fs");
-var events = require("events");
-
-var objectOps = require("./../../lib/objectOps");
-
-var LinvoDB = require("linvodb3");
-LinvoDB.dbPath = "./database/linvodb/";
-
-
-let allowLogging = true;
-let log = string => (allowLogging)? console.log(string):""
-
-module.exports = class chunkStore {
-	constructor(name, chunkSize = 32, path = "./chunkStore/"){
-		if(!name || (typeof name != "string" && typeof name != "number")) throw new Error("chunkstore requires a name string as first param");
-		name = name.toString();
-		this.name = name;
-		this.chunkSize = chunkSize;
-		this.path = (path[path.length] == "/") ? path : path+"/";
-		mkdirpSync(path+name);
-		
-		this.db = new LinvoDB(name, {}, {});
-		
-		this.eventEmitter = new events.EventEmitter();
-	}
-	setEntity(x/**{number}*/, y/**{number}*/, newEntity/**{object}*/){
-		return new Promise((resolve, reject) => {
-			x = Math.floor(x);
-			y = Math.floor(y);
-			this.db.find({x, y}, (err, docs) => {
-				if(typeof newEntity == "object"){
-					if(err) reject(err);
-					let entityDoc = {x, y, entity: newEntity};
-					if(docs && docs[0] && docs[0]._id){
-						entityDoc._id = docs[0]._id;
-					}
-					this.db.save(entityDoc, (err,docs) => {
-						if(err) reject(err);
-						resolve(docs);
-						this.eventEmitter.emit("change", docs)
-					});
-				} else {
-					docs.forEach(doc => {
-						doc.remove(()=>{
-							// removed :)
-						});
-					});
-					resolve();
-					this.eventEmitter.emit("change", {x,y});
-				}
-			});
-		});
-	}
-	getEntity(x, y){
-		return new Promise((resolve, reject) => {
-			this.db.find({x:Math.floor(x), y:Math.floor(y)}, (err, docs) => {
-				if(err) reject(err);
-				resolve(docs);
-			});
-		});
-	}
-	onEntityChange(eventhandler/**function*/){
-		if(typeof eventhandler == "function") {
-			this.eventEmitter.on("change", eventhandler);
-		}
-	}
-	getChunk(x, y){
-		return new Promise((resolve, reject) => {
-			
-			/*
-			// make new chunk in case we can't find an existing one
-			let chunk = {
-				dataObject: {},
-				position:{x,y}, // fancy ES6 feature that generates to {x:x, y:y}
-				chunkSize:this.chunkSize
-			};
-			fs.readFile(this.path+this.name+"/"+x+"/"+y, (err, chunkFile) => {
-				
-				if(err){
-					log(err)
-					resolve(chunk);
-				} else {
-					if(objectOps.isJSON(chunkFile)) {
-						log("returned from file")
-						let chunk = JSON.parse(chunkFile);
-						resolve(chunk);
-					} else {
-						log("file failed")
-						resolve(chunk);
-					}
-				}
-			}); // add x and y values as well and we are done?
-			*/
-		});
-	}
-}

sharedPlugins/remoteMap/chunkStore.spec.js

@@ -1,64 +0,0 @@
-var assert = require("assert");
-
-var chunkStore = require("./chunkStore.js");
-
-describe("remoteMap/chunkStore.js", () => {
-	// define some values for our test
-	const inserterX = 14.5; // rounds to 15
-	const inserterY = -17.5; // rounds to -17, +64 would be 47
-	const inserterName = "inserter";
-	
-	let store // make our store elevated
-	it("is a constructor", ()=>{
-		assert.equal(typeof chunkStore, "function");
-		// create a new store and elevate it
-		store = new chunkStore("910239"/* instanceID, use defaults for the rest */, undefined, "./database/unitTests/chunkStore/");
-		assert.equal(store.name, "910239");
-	});
-	it("allows you to store the locations of entities", done => {
-		store.setEntity(inserterX, inserterY, {name:inserterName, otherData:"can be arbitrary"}).then(() => {
-			done();
-			// nothing to do here really
-		});
-	});
-	it("returns entities at positions you query for", done => {
-		setTimeout(()=>{
-			store.getEntity(inserterX, inserterY).then(entities => {
-				entity = entities[0]
-				// console.log(entity)
-				assert.equal(entity.x, 14);
-				assert.equal(entity.y, -18);
-				assert.equal(entity.entity.name, "inserter");
-				assert.equal(entity.entity.otherData, "can be arbitrary");
-				done();
-			});
-		},300);
-	});
-	it("deletes entities when not provided with an object as 3rd arg", done => {
-		store.setEntity(inserterX, inserterY).then(entity => {
-			assert(entity === undefined);
-			setTimeout(()=>{
-				store.getEntity(inserterX, inserterY).then(entities => {
-					// console.log(entities)
-					assert.equal(entities.length, 0, "Why are we getting results, it should be empty?");
-					done();
-				});
-			},300);
-		});
-	});
-	describe("validates input by throwing when it is invalid", ()=>{
-		it("throws if called without a string as the first parameter", ()=>{
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore(undefined, undefined, "asdasda");
-			});
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore({}, undefined, "asdasdas");
-			});
-		});
-		it("converts numbers to strings automatically if they are provided  as the first parameter", ()=>{
-			assert.ok(()=>{
-				let nonThrowingChunkStore = new chunkStore(42, undefined, "./database/unitTests/chunkStore/");
-			});
-		});
-	});
-});

sharedPlugins/remoteMap/config.js

@@ -13,4 +13,5 @@ module.exports = {
 	description: modInfo.description,
 	// We'll send everything in this file to your stdin. Beware.
 	scriptOutputFileSubscription: "remoteMap.txt",
+	masterPlugin: "masterPlugin.js",
 }

sharedPlugins/remoteMap/index.js

@@ -1,130 +1,71 @@
 const needle = require("needle");
-const ioClient = require("socket.io-client");
 
-const chunkStore = require("./chunkStore.js");
+const debug = true
-
+module.exports = class remoteMap {
-const moduleConfig = require("./config"); // not to be confused with clusterio config. This config is private to this plugin.
+	constructor(mergedConfig, messageInterface, extras){
-
-class remoteMap {
-	constructor(slaveConfig, messageInterface){
-		this.config = slaveConfig;
 		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		this.socket = extras.socket;
+		this.instances = {};
 		
-		// initialize chunk database
+        this.socket.on("hello", () => this.socket.emit("registerMapServer", {
-		this.chunkMap = new chunkStore(this.config.unique, 64, "./database/chunkStore/");
+			instanceID: this.config.unique, // instanceID
-		this.chunkMap.onEntityChange(entity => {
+		}));
-			// this.messageInterface("Entity changed: ")
+
-			// this.messageInterface(entity);
+        this.socket.on("createEntity", req => {
-			// check valid coordinates (dunno what could go wrong, but can never be too sure)
+			// req = {name:"", position:{x,y}, ?direction:4}
-			if(entity && entity.x !== undefined && !isNaN(Number(entity.x)) && entity.y !== undefined && !isNaN(Number(entity.y))){
+			// game.player.surface.create_entity{name="small-ship-wreck", position={game.player.position.x-2, game.player.position.y+1}, direction=..., force=...}
-				this.socket.emit("sendEntity", entity);
+			this.messageInterface("placing "+JSON.stringify(req));
+			if(req && req.name && typeof req.name == "string" && req.position && !isNaN(Number(req.position.x)) && !isNaN(Number(req.position.y))){
+				// delete any entity in that spot first, just in case to avoid overlaps
+				this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
+				// build the entity
+				let command = '/silent-command game.surfaces[1].create_entity{name="'+req.name+'", position={'+Number(req.position.x)+', '+Number(req.position.y)+'}, force="player"'
+				if(req.direction) command += ', direction='+req.direction;
+				command += '}';
+				this.messageInterface(command); // execute command
 			}
-		});
+        })
+        this.socket.on("deleteEntity", req => {
+			// req: {position: {x, y}}
+			this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
+        })
+		this.socket.on("getChunk", async (data, callback) => {
+			let chunkData = await this.messageInterface(`/silent-command remote.call("remoteMap", "exportChunk", ${data.x}, ${data.y})`)
+			console.log(`Sending chunk data: ${chunkData}`)
+			callback(chunkData);
+		})
 		
-		// set up websocket communication and handle requests from web interface users (which are going through master)
+		// detect new entities created/deleted on the server and update the master
-		// socket should be a global
+		setInterval(async ()=>{
-		this.socket = ioClient("http://"+this.config.masterIP+":"+this.config.masterPort);
+			let data = await this.messageInterface(`/silent-command remote.call("remoteMap", "exportTiles")`)
-		this.socket.on("hello", data => {
+			if(data.length > 1) this.socket.emit("updateEntities", data)
-			this.socket.emit("registerSlaveMapper", {instanceID: this.config.unique});
+		}, 2000)
-			setInterval(()=>{
+	}
-				this.socket.emit("heartbeat"); // send our heartbeat to prevent being assumed dead
+	getInstanceName(instanceID){
-			},10000);
+		return new Promise((resolve, reject) => {
-		});
+			let instance = this.instances[instanceID];
-		this.socket.on("getChunk", req => {
+			if(!instance){
-			this.messageInterface("getChunk is not supported!");
+				needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => {
-			/*this.chunkMap.getChunk(req.x, req.y).then(chunk => {
+					if(err || response.statusCode != 200) {
-				chunk.requesterID = req.requesterID;
+						console.log("Unable to get JSON master/api/slaves, master might be unaccessible");
-				this.socket.emit("sendChunk", chunk);
+					} else if (response && response.body) {	
-			});*/
+						if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();}
-		});
+							try {
-		this.socket.on("getEntity", req => {
+								for (let index in response.body)
-			// check for valid coordinates
+									this.instances[index] = response.body[index].instanceName;
-			if(req.x !== undefined && !isNaN(Number(req.x)) && req.y !== undefined && !isNaN(Number(req.y))){
+							} catch (e){
-				this.chunkMap.getEntity(req.x, req.y).then(entities => {
+								console.log(e);
-					if(entities && entities.length > 0){
+								return null;
-						entities.forEach(entity => {
+							}
-							this.socket.emit("sendEntity", entity);
+						instance = this.instances[instanceID] 							
-						});
+						if (!instance) instance = instanceID;  //somehow the master doesn't know the instance	
+						resolve(instance);
 					}
 				});
 			} else {
-				this.messageInterface("socket.getEntity triggered called with invalid parameters");
+				resolve(instance);
 			}
 		});
-		// both placing and deleting entities via the web interface
-		this.socket.on("placeEntity", req => {
-			// game.player.surface.create_entity{name="small-ship-wreck", position={game.player.position.x-2, game.player.position.y+1}, direction=..., force=...}
-			this.messageInterface("placing "+JSON.stringify(req));
-			if(req.name && typeof req.name == "string" && req.position && !isNaN(Number(req.position.x)) && !isNaN(Number(req.position.y))){
-				if(req.name == "deleted"){ // {{'+req.position.x+','+req.position.y+'},{'+req.position.x+','+req.position.y+'}}
-					// delete entity from game world using LUA command
-					this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+1)+','+(req.position.y+1)+'}}) for i, entity in pairs(toDelete) do entity.die() end');
-					// this position is now empty, delete whatever was there from DB
-					this.deleteEntity(req.position.x, req.position.y);
-				} else {
-					let command = '/silent-command game.surfaces[1].create_entity{name="'+req.name+'", position={'+Number(req.position.x)+', '+Number(req.position.y)+'}, force="neutral"'
-					if(req.direction) command += ', direction='+req.direction;
-					command += '}';
-					this.messageInterface(command); // execute command
-					
-					// add entity to the chunkStore database because create_entity does not trigger any events
-					let name = req.name;
-					let entity = {name};
-					if(typeof req.direction == "string"){
-						entity.rot = req.direction;
-					}
-					this.addEntity(req.position.x, req.position.y, entity);
-				}
-			}
-		});
-		
-		this.messageInterface("/silent-command game.print('"+moduleConfig.name+" version "+moduleConfig.version+" enabled')");
-	}
-	addEntity(x,y,entity){ // add entity to database
-		this.chunkMap.setEntity(Number(x), Number(y), entity).then(data => {
-			this.messageInterface("Added "+entity.name+" to data "+Number(x)+", "+Number(y));
-		}).catch((err)=>this.messageInterface(err));
-	}
-	deleteEntity(x,y){ // remove entity from database
-		// this position is now empty, delete whatever was there from DB
-		this.chunkMap.setEntity(x, y, "delete this entity").then(() => {
-			this.messageInterface("Deleted entity at "+x+", "+y);
-		}).catch((err)=>this.messageInterface(err));
-	}
-	scriptOutput(data){
-		if(data){
-			this.messageInterface(data);
-			// express-transport-belt,-35.5,-14.5,[...]
-			data = data.split(",");
-			if(data && data[0] == undefined){
-				this.messageInterface("empty: ");
-				this.messageInterface(data);
-				return
-			}
-			let name = data[0];
-			let xPos = data[1];
-			let yPos = data[2];
-			if(xPos && yPos && name){
-				if(name == "deleted"){
-					// this position is now empty, delete whatever was there from DB
-					this.deleteEntity(xPos, yPos);
-				} else {
-					let entity = {name};
-					for(let i = 3; i < data.length; i++){
-						if(data[i].includes("=")){
-							let kv = data[i].split("=");
-							entity[kv[0]] = kv[1];
-						}
-					}
-					// save this entity in the database
-					this.addEntity(xPos, yPos, entity);
-					/* this.chunkMap.setEntity(xPos, yPos, entity).then(data => {
-						this.messageInterface("Added "+name+" to data "+xPos+", "+yPos);
-					}).catch((err)=>this.messageInterface(err)); */
-				}
-			}
-			
-		}
 	}
 }
-module.exports = remoteMap;

sharedPlugins/remoteMap/info.json

@@ -2,7 +2,7 @@
 {
 "name": "remoteMap",
 "title": "Clusterio remote map",
-"version": "2.2.0",
+"version": "3.0.0",
 "date": "18-11-2017",
 "author": "Danielv123",
 "dependencies": ["base >= 0.15"],

sharedPlugins/remoteMap/masterPlugin.js

@@ -0,0 +1,189 @@
+const fs = require("fs-extra");
+const path = require("path");
+const sanitizer = require('sanitizer');
+
+const editorSockets = [];
+
+const blacklistedEntities = [
+	"fish",
+	"player",
+]
+
+class masterPlugin {
+	constructor({ config, pluginConfig, pluginPath, socketio, express }) {
+		this.config = config;
+		this.pluginConfig = pluginConfig;
+		this.pluginPath = pluginPath;
+		this.io = socketio;
+		this.app = express;
+		
+		this.clients = {};
+		this.slaves = {};
+		
+		// initialize token auth module
+		this.authenticate = require("./../../lib/authenticate.js")(config);
+
+		// expose UI elements embedded in the master
+		//this.ui = require("./js/ui.js").ui;
+
+		// handle websocket connections
+		this.io.on("connection", socket => {
+			let instanceID = "unknown";
+			socket.on("registerMapServer", data => {
+				console.log("Registered mapServer socket")
+				if (data.instanceID && !isNaN(Number(data.instanceID))) {
+					instanceID = data.instanceID;
+					this.slaves[instanceID] = socket;
+				}
+				socket.on("updateEntities", data => {
+					console.log(data)
+					if(data){
+						// transform from weirdly deliminated string to entity object
+						let entities = data
+						.split("\n")
+						.filter(str => str.length > 0)
+						.map(ent => ent.replace(/[-]/g, "_"))											// replace - with _
+						.map(str => str
+							.split(",")
+							.filter(x => x.length > 0)
+							.map(prop => ({
+								k: prop.split("=")[0],
+								v: prop.split("=")[1]})
+							)
+						)
+						.map(arr => {
+							let obj = {}
+							arr.forEach(property => obj[property.k] = property.v)
+							return obj
+						})
+						console.log(entities)
+						editorSockets.forEach(sock => sock.emit("updateEntity", {entities}))
+					}
+				})
+			});
+			socket.on("registerMapEditor", () => {
+				console.log("Registered mapEditor socket")
+				editorSockets.push(socket);
+				
+				setTimeout(()=>socket.emit("createEntity", {entity:{
+					name:"fast_inserter",
+					position: {x:30,y:30},
+					direction: 2,
+				}}),10000)
+				
+				socket.on("createEntity", data => {
+					console.log(`Creating entity: ${JSON.stringify(data)}`)
+					this.slaves[Object.keys(this.slaves)[0]].emit("createEntity", data.entity)
+				})
+				socket.on("deleteEntity", data => {
+					console.log(`Deleting entity: ${JSON.stringify(data)}`)
+					this.slaves[Object.keys(this.slaves)[0]].emit("deleteEntity", data.entity)
+				})
+				socket.on("getChunk", (data, callback) => {
+					console.log(`Getting chunk x:${data.x} y:${data.y} from slave`)
+					this.slaves[Object.keys(this.slaves)[0]].emit("getChunk", data, resp => {
+						// parse response
+						// |name:transport-belt,direction:2,x:99,y:1231|name:assembly-machine,direction:6,x:939,y:88
+						console.log(resp)
+						callback(resp
+							.split("|")																	// split into entities
+							.map(str => str.replace("\n",""))											// remove newlines that happen at the end of the last entity (maybe more as well???)
+							.filter(str => str.length > 0)												// remove blank entities
+							.filter(str => 
+							// !str.includes("tree") &&													// don't send entities that can't be rendered anyways
+							!str.includes("dead") &&
+							// !str.includes("rock") &&
+							!str.includes("crude-oil") &&
+							!str.includes("remnants") &&
+							!str.includes("cliff") &&
+							!blacklistedEntities.includes(str) &&
+							!str.includes("ore"))
+							.map(ent => ent.split(",")													// split entity into properties
+								.map(prop => [prop.split(":")[0], prop.split(":")[1]])					// split properties into KV pairs
+							)
+							.map(entity => {															// turn entities into objects
+								let entityObj = {}
+								entity.forEach(property => entityObj[property[0]] = property[1])
+								return entityObj
+							})
+							.map(entity => {
+								entity.name = entity.name.replace(/[-]/g, "_")							// replace - with _ in entity names as thats what factorio.blueprint-editor preffers
+								return entity
+							})
+							.filter(entity => Object.keys(entity).length > 1)
+							.filter(entity => !blacklistedEntities.includes(entity.name))
+							/*.map(entity => {
+								if(entity.name.includes("tree")) entity.name = "tree"
+								return entity
+							})*/
+						)
+					})
+				})
+				
+				socket.on("disconnect", function () {
+					let i = editorSockets.indexOf(socket);
+					console.log("editorSocket " + (i + 1) + " disconnected, " + (editorSockets.length - 1) + " left");
+					editorSockets.splice(i, 1);
+				});
+			});
+			socket.on("gameChat", async data => {
+				let chatLine = data.data.replace(/(\r\n\t|\n|\r\t)/gm, "").replace("\r", "");
+				//if (typeof chatLine == "string") this.handleChatLine(chatLine, instanceID);
+			});
+		});
+
+	}
+	async onExit() {
+		return;
+	}
+	async onLoadFinish({ plugins }) {
+		this.masterPlugins = plugins;
+	}
+	parseData(data, sharedData = {}) {
+		let parsedData = [];
+		data = data.split("|");
+		data.forEach(player => {
+			if (player) {
+				let playerData = {};
+				player = player.split(`~`);
+				player.forEach(kv => {
+					kv = kv.split(":");
+					if (kv[1] === undefined || kv[0] === undefined) {
+						console.log(new Error(`Something is wrong! Key:${kv[0]} Value:${kv[1]}`));
+					} else {
+						playerData[kv[0]] = kv[1].trim();
+					}
+				});
+				for (let k in sharedData) {
+					playerData[k] = sharedData[k];
+				}
+				parsedData.push(playerData);
+			}
+		});
+		return parsedData;
+	}
+}
+module.exports = masterPlugin;
+
+function getDatabaseSync(path) {
+	let db;
+	try {
+		db = JSON.parse(fs.readFileSync(path, "utf8"));
+	} catch (e) {
+		db = {};
+	}
+	return db;
+}
+async function saveDatabase(path, database) {
+	if (!path) {
+		throw new Error("No path provided!");
+	} else if (!database) {
+		throw new Error("No database provided!");
+	} else {
+		try {
+			await fs.writeFile(path, JSON.stringify(database, null, 4));
+		} catch (e) {
+			throw new Error("Unable to write to database! " + path);
+		}
+	}
+}

sharedPlugins/remoteMap/readme.md

@@ -0,0 +1,15 @@
+# Remote Map
+
+A clusterio plugin for remotely exploring and editing the factorio world
+
+### Setup
+
+Make a database for storing entities
+
+    docker run --name some-mariadb -e MYSQL_ROOT_PASSWORD=clusterio-remote-map --restart=unless-stopped -d mariadb:latest
+
+### Dev setup
+
+Make a database container and expose the ports
+
+    docker run -p 3306:3306 --name some-mariadb -e MYSQL_ROOT_PASSWORD=clusterio-remote-map -d mariadb:latest

sharedPlugins/scenario_loader/config.js

@@ -0,0 +1,10 @@
+/*
+	Clusterio plugin for loading softmod scenarios using Hotpatch
+	To load a scenario, simply drop it inside sharedPlugins/scenario_loader/scenarios/
+*/
+module.exports = {
+	name: "scenario_loader",
+	version: "1.0.0",
+	binary: "nodePackage",
+	description: "Simple way to load scenarios using Hotpatch"
+}

sharedPlugins/scenario_loader/index.js

@@ -0,0 +1,116 @@
+const fs = require("fs-extra");
+const path = require("path");
+const clusterTools = require("_app/clusterTools")();
+
+module.exports = class remoteCommands {
+	constructor(mergedConfig, messageInterface, extras){
+		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		
+		(async () => {
+			let hotpatchInstallStatus = await this.checkHotpatchInstallation();
+			messageInterface("Hotpach installation status: "+hotpatchInstallStatus);
+			if(hotpatchInstallStatus){
+				await this.loadScenariosFromFolder("scenarios") // located in project root folder (with sharedMods, sharedPlugins etc)
+			}
+		})();
+	}
+	async loadScenariosFromFolder(scenarioDir){
+		await fs.ensureDir(scenarioDir)
+		let scenarios = await fs.readdir(scenarioDir);
+		for(let i = 0; i < scenarios.length; i++){
+			this.messageInterface(`Loading scenario ${scenarios[i]}`)
+			let startTime = Date.now();
+			let scenPath = path.join(scenarioDir, scenarios[i]);
+			let stat = await fs.stat(scenPath);
+			if(stat.isDirectory){
+				async function loadFiles(dir, rootdir){
+					let returnFiles = [];
+					let entries = await fs.readdir(path.join(rootdir, dir));
+					for(let o = 0; o < entries.length; o++){
+						let stat = await fs.stat(path.join(rootdir, dir, entries[o]));
+						if(stat.isDirectory()){
+							// recurse
+							let subFolder = await loadFiles(path.join(dir, entries[o]), rootdir);
+							returnFiles = returnFiles.concat(subFolder);
+						} else if(entries[o].split(".")[entries[o].split(".").length-1].toLowerCase() == "lua"){
+							// if it is a Lua file, read its contents
+							// let data = await fs.readFile(path.join(rootdir, dir, entries[o]));
+							let data = await clusterTools.getLua(path.join(rootdir, dir, entries[o]), false);
+							returnFiles.push({
+								name: path.join(dir, entries[o].split(".")[0]),
+								data,
+							});
+						}
+					}
+					return returnFiles;
+				}
+				let fileMap = await loadFiles("", scenPath);
+				let files = {};
+				fileMap.forEach(map => {
+					files[map.name] = map.data;
+					console.log(map.name);
+				});
+				
+				let fileImportString = `\{`;
+				for(let k in files){
+					let name = k.replace(/\\/g, '/');
+					name = name.replace(/ /g, '');
+					// make sure *not* to include control.lua as it is provided as a seperate argument
+					if(name != "control") fileImportString += `["${name}"] = '${files[k]}', `;
+				}
+				fileImportString += `\}`;
+				if(files.control) var returnValue = await this.messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
+				if(returnValue) this.messageInterface(returnValue);
+				this.messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
+			}
+		}
+	}
+	async checkHotpatchInstallation(){
+		let yn = await this.messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end");
+		yn = yn.replace(/(\r\n\t|\n|\r\t)/gm, "");
+		if(yn == "true"){
+			return true;
+		} else if(yn == "false"){
+			return false;
+		}
+	}
+}
+
+// this function is bad in terms of scope since it relies on a global for return values, should be refactored.
+var filesCollection = [];
+const directoriesToSkip = ['bower_components', 'node_modules', 'www', 'platforms', '.git'];
+
+function readDirectorySynchronously(directory) {
+    var currentDirectorypath = /*path.join(__dirname + */directory//);
+
+    var currentDirectory = fs.readdirSync(currentDirectorypath, 'utf8');
+
+    currentDirectory.forEach(file => {
+        var fileShouldBeSkipped = directoriesToSkip.indexOf(file) > -1;
+        var pathOfCurrentItem = path.join(/*__dirname + */directory + '/' + file);
+        if (!fileShouldBeSkipped && fs.statSync(pathOfCurrentItem).isFile()) {
+            filesCollection.push(pathOfCurrentItem);
+        }
+        else if (!fileShouldBeSkipped) {
+            var directorypath = path.join(directory, file);
+            readDirectorySynchronously(directorypath);
+        }
+    });
+}
+class AsyncArray extends Array {
+	constructor(arr) {
+		super(arr)
+		this.data = arr; // In place of Array subclassing
+	}
+	async filterAsync(predicate) {
+		// Take a copy of the array, it might mutate by the time we've finished
+		const data = Array.from(this.data);
+		// Transform all the elements into an array of promises using the predicate
+		// as the promise
+		const result = await Promise.all(data.map((element, index) => predicate(element, index, data)));
+		return data.filter((element, index) => {
+			return result[index];
+		});
+	}
+}

sharedPlugins/serverManager/config.js

@@ -0,0 +1,13 @@
+/*
+	Clusterio plugin for remote server management
+*/
+module.exports = {
+	name: "serverManager",
+	version: "1.0.0",
+	binary: "nodePackage",
+	description: "Server management tools for remote config edits and web based server management",
+	masterPlugin: "masterPlugin.js",
+	dependencies: [
+		"playerManager",
+	]
+}

sharedPlugins/serverManager/index.js

@@ -0,0 +1,43 @@
+const needle = require("needle");
+const fs = require("fs-extra");
+
+const pluginConfig = require("./config");
+const COMPRESS_LUA = false;
+
+module.exports = class remoteCommands {
+	constructor(mergedConfig, messageInterface, extras){
+		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		this.socket = extras.socket;
+		const pluginManager = require("./../../lib/manager/pluginManager.js")(this.config);
+		const configManager = require("./../../lib/manager/configManager.js");
+		// const modManager = require("./../../lib/manager/modManager.js")(this.config);
+		this.socket.on("hello", () => {
+			this.socket.emit("registerServerManager");
+		});
+		this.socket.on("serverManagerGetStatus", async data => {
+			
+		});
+		this.socket.on("serverManagerGetMods", async (data, callback) => {
+			// TODO: Implement this function separately from listMods (which console.logs it)
+			// let mods = modManager.getMods()
+			// callback(mods);
+		});
+		this.socket.on("serverManagerGetConfig", async (data, callback) => {
+			let config = await configManager.getConfig(data.instance);
+			callback(config);
+		});
+		this.socket.on("serverManagerEditConfig", async (data, callback) => {
+			callback(configManager.editConfig(data.instance, data.entry, data.value));
+		});
+		this.socket.on("serverManagerGetPlugins", async (data, callback) => {
+			callback(await pluginManager.getPlugins());
+		});
+		this.socket.on("serverManagerEnablePlugin", async (data, callback) => {
+			callback(await pluginManager.enablePlugin(data.name, data.instanceID /* Currently ignored */));
+		});
+		this.socket.on("serverManagerDisablePlugin", async (data, callback) => {
+			callback(await pluginManager.disablePlugin(data.name, data.instanceID /* Currently ignored */));
+		});
+	}
+}