Changed stuff in the dockerfile, no idea if its gonna work

.gitignore

@@ -63,3 +63,7 @@ sharedPlugins/scenario_loader/scenarios
 sharedPlugins/*/DISABLED
 
 \.cache-require-paths\.json
+
+\.vs/
+
+bin/

Dockerfile

@@ -2,20 +2,22 @@ FROM node:9.11.2
 RUN apt-get update && apt install git curl tar -y
 RUN mkdir factorioClusterio
 
-RUN git clone -b master https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install
+#RUN git clone -b master https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install
-RUN cd factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz
+COPY . /factorioClusterio
+RUN cd /factorioClusterio && npm install
+RUN cd /factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
 
-WORKDIR factorioClusterio
+WORKDIR /factorioClusterio
 RUN mkdir instances sharedMods
 RUN cp config.json.dist config.json
 
 RUN node client.js download
 
-LABEL maintainer "Sir3lit@gmail.com"
+LABEL maintainer "danielv@live.no"
 
 EXPOSE 8080 34167
 VOLUME /factorioClusterio/instances
 VOLUME /factorioClusterio/sharedMods
 VOLUME /factorioClusterio/sharedPlugins
 
-CMD RCONPORT="$RCONPORT" FACTORIOPORT="$FACTORIOPORT" MODE="$MODE" node $MODE\.js start $INSTANCE
+CMD RCONPORT="$RCONPORT" FACTORIOPORT="34167" MODE="$MODE" node $MODE\.js start $INSTANCE

lib/getDatabase.ts

@@ -0,0 +1,15 @@
+import { getItemDatabase } from "./getItemDatabase";
+import { getSlaveDatabase } from "./getSlaveDatabase";
+
+export type db = {
+    items: {[name:string]: any},
+    slaves: Slaves,
+}
+
+export function getDatabase(config: ClusterioConfig) {
+    let db: db = {
+        items: getItemDatabase(config),
+        slaves: getSlaveDatabase(config),
+    };
+    return db
+}

lib/getFactorioLocale.d.ts

@@ -0,0 +1,6 @@
+declare module "getFactorioLocale" {
+    interface callback {
+        (error: Error, factorioLocale?: any): void
+    }
+    export function asObject(factorioDirectory: string, languageCode: string, callback: callback): void
+}

lib/getItemDatabase.ts

@@ -0,0 +1,40 @@
+import * as fs from "fs"
+import * as path from "path"
+
+export function getItemDatabase(config: ClusterioConfig): {[name:string]: number | Function} {
+	var items: {[name: string]: number | Function} = {}
+	try{
+		let x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
+		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
+		items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")).toString());
+	} catch (e){
+		items = {};
+	}
+	items.addItem = function addItem(object: item): boolean {
+		if(object.name == "addItem" || object.name == "removeItem") {
+			console.error("Fuck you, that would screw everything up if you named your item that.");
+			return false;
+		} else {
+			if(this[object.name] && Number(this[object.name]) != NaN){
+				this[object.name] = Number(this[object.name]) + Number(object.count);
+			} else {
+				this[object.name] = object.count;
+			}
+			return true;
+		}
+	},
+	items.removeItem = function removeItem(object: item): boolean {
+		if(object.name == "addItem" || object.name == "removeItem") {
+			console.error("Fuck you, that would screw everything up if you named your item that.");
+			return false;
+		} else {
+			if(this[object.name] && Number(this[object.name]) != NaN){
+				this[object.name] = Number(this[object.name]) - Number(object.count);
+			} else {
+				this[object.name] = 0;
+			}
+			return true;
+		}
+	}
+	return items
+}

lib/getSlaveDatabase.ts

@@ -0,0 +1,14 @@
+import * as fs from "fs"
+import * as path from "path"
+
+export function getSlaveDatabase(config: ClusterioConfig): Slaves {
+    var slaves: Slaves = {};
+    try{
+		let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json"));
+		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
+		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json"), "utf-8"));
+	} catch (e){
+		slaves = {};
+    }
+    return slaves
+}

master.d.ts

@@ -0,0 +1,16 @@
+type MasterPluginInstance = {
+    config: object,
+    pluginConfig: object,
+    path: string,
+    socketio: any,
+    express: any,
+    Prometheus: any,
+    onExit?: Function,
+}
+type pluginConfig = {
+    name: string,
+}
+interface MasterPlugin {
+    main: MasterPluginInstance,
+    pluginConfig: pluginConfig,
+}

master.spec.js

@@ -76,17 +76,6 @@ describe('Master server endpoint testing', function() {
 				});
 			});
 		});
-		it("sends some HTML when accessing /remoteMap",function(done){
-			this.timeout(6000);
-			request(app).get("/remoteMap").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					// there should be 1 error regarding complaining about me using ES6 modules before they are fully supported
-					assert(result.messages.length === 2, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
 	});
 	let persistentMaster = request(app);
 	describe("#POST /api/place", function(){

master.ts

@@ -1,3 +1,6 @@
+import { getDatabase } from "./lib/getDatabase";
+import { db } from "./lib/getDatabase";
+
 /**
 Clusterio master server. Facilitates communication between slaves through
 a webserver, storing data related to slaves like production graphs and
@@ -13,7 +16,9 @@ node master.js
 // Attempt updating
 // const updater = require("./updater.js");
 // updater.update().then(console.log);
-
+namespace Master {
+// homebrew types
+//import {MasterPlugin} from "master"
 (async ()=>{
 
 // Set the process title, shows up as the title of the CMD window on windows
@@ -24,7 +29,7 @@ process.title = "clusterioMaster";
 process.on('unhandledRejection', r => console.log(r));
 
 // configgy stuff
-debug = false;
+const debug = false;
 
 // reduces loadtime from 1700 ms to 1200 ms on my i3 + SSD
 // tested by logging Date.now() here and when the webserver started listening
@@ -63,7 +68,7 @@ const fileOps = require("_app/fileOps");
 const authenticate = require("./lib/authenticate")(config);
 
 /** Sync */
-function randomStringAsBase64Url(size) {
+function randomStringAsBase64Url(size: Number) {
   return base64url(crypto.randomBytes(size));
 }
 if (!fs.existsSync("secret-api-token.txt")) {
@@ -105,11 +110,11 @@ app.set('view engine', 'html');
 app.engine('html', ejs.renderFile);
 
 // give ejs access to some interesting information
-app.use(function(req, res, next){
+app.use(function(req: Request, res: any, next: any){
 	res.locals.res = res;
 	res.locals.req = req;
 	res.locals.masterPlugins = masterPlugins;
-	res.locals.slaves = slaves;
+	res.locals.slaves = db.slaves;
 	res.locals.moment = moment;
 	next();
 });
@@ -205,15 +210,15 @@ app.get('/metrics', (req, res) => {
 });
 
 function registerMoreMetrics(){
-	for(let instanceID in slaves){
+	for(let instanceID in db.slaves){
 		// playercount
 		try{
-			prometheusPlayerCountGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].playerCount) || 0);
+			prometheusPlayerCountGauge.labels(instanceID, db.slaves[instanceID].instanceName).set(Number(db.slaves[instanceID].playerCount) || 0);
 		}catch(e){}
 		// UPS
 		try{
-			prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.UPS) || 60);
+			prometheusUPSGauge.labels(instanceID, db.slaves[instanceID].instanceName).set(Number(db.slaves[instanceID].meta.UPS) || 60);
-			if(slaves[instanceID].meta.tick && typeof slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.tick) || 0);
+			if(db.slaves[instanceID].meta.tick && typeof db.slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, db.slaves[instanceID].instanceName).set(Number(db.slaves[instanceID].meta.tick) || 0);
 		}catch(e){}
 	}
 	// inventory
@@ -224,58 +229,15 @@ function registerMoreMetrics(){
 	}
 	// Slave count
 	let numberOfActiveSlaves = 0;
-	for(let instance in slaves){
+	for(let instance in db.slaves){
-		if(Date.now() - Number(slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
+		if(Date.now() - Number(db.slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
 	}
 	prometheusConnectedInstancesCounter.set(numberOfActiveSlaves);
 }
 
 // set up database
-const db = {};
+const db: db = getDatabase(config);
 
-(function(){
-	try{
-		let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json"));
-		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
-		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json")));
-	} catch (e){
-		slaves = {};
-	}
-	try{
-		x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
-		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
-		db.items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")));
-	} catch (e){
-		db.items = {};
-	}
-})()
-
-db.items.addItem = function(object) {
-	if(object.name == "addItem" || object.name == "removeItem") {
-		console.error("Fuck you, that would screw everything up if you named your item that.");
-		return false;
-	} else {
-		if(this[object.name] && Number(this[object.name]) != NaN){
-			this[object.name] = Number(this[object.name]) + Number(object.count);
-		} else {
-			this[object.name] = object.count;
-		}
-		return true;
-	}
-}
-db.items.removeItem = function(object) {
-	if(object.name == "addItem" || object.name == "removeItem") {
-		console.error("Fuck you, that would screw everything up if you named your item that.");
-		return false;
-	} else {
-		if(this[object.name] && Number(this[object.name]) != NaN){
-			this[object.name] = Number(this[object.name]) - Number(object.count);
-		} else {
-			this[object.name] = 0;
-		}
-		return true;
-	}
-}
 
 // store slaves and inventory in a .json full of JSON data
 process.on('SIGINT', shutdown); // ctrl + c
@@ -284,17 +246,17 @@ async function shutdown() {
 	console.log('Ctrl-C...');
 	let exitStartTime = Date.now();
 	// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-	if(slaves && Object.keys(slaves).length < 2000000){
+	if(db.slaves && Object.keys(db.slaves).length < 2000000){
 		console.log("saving to slaves.json");
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
+		fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(db.slaves));
-	} else if(slaves) {
+	} else if(db.slaves) {
-		console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
+		console.log("Slave database too large, not saving ("+Object.keys(db.slaves).length+")");
 	}
 	if(db.items && Object.keys(db.items).length < 50000){
 		console.log("saving to items.json");
 		fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
-	} else if(slaves) {
+	} else if(db.slaves) {
-		console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
+		console.log("Item database too large, not saving ("+Object.keys(db.slaves).length+")");
 	}
 	for(let i in masterPlugins){
 		let plugin = masterPlugins[i];
@@ -314,7 +276,7 @@ POST World ID management. Slaves post here to tell the server they exist
 @instance
 @alias /api/getID
 */
-app.post("/api/getID", authenticate.middleware, function(req,res) {
+app.post("/api/getID", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	// request.body should be an object
 	// {rconPort, rconPassword, serverPort, mac, time}
@@ -324,12 +286,12 @@ app.post("/api/getID", authenticate.middleware, function(req,res) {
 		if(debug){
 			console.log(req.body)
 		}
-		if(!slaves[req.body.unique]){
+		if(!db.slaves[req.body.unique]){
-			slaves[req.body.unique] = {};
+			db.slaves[req.body.unique] = {};
 		}
-		for(k in req.body){
+		for(let k in req.body){
 			if(k != "meta" && req.body.hasOwnProperty(k)){
-				slaves[req.body.unique][k] = req.body[k];
+				db.slaves[req.body.unique][k] = req.body[k];
 			}
 		}
 		// console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName);
@@ -348,11 +310,11 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 	
 	if(req.body && req.body.instanceID && req.body.password && req.body.meta){
 		// check for editing permissions
-		if(slaves[req.body.instanceID] && slaves[req.body.instanceID].rconPassword == req.body.password){
+		if(db.slaves[req.body.instanceID] && db.slaves[req.body.instanceID].rconPassword == req.body.password){
-			if(!slaves[req.body.instanceID].meta){
+			if(!db.slaves[req.body.instanceID].meta){
-				slaves[req.body.instanceID].meta = {};
+				db.slaves[req.body.instanceID].meta = {};
 			}
-			slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
+			db.slaves[req.body.instanceID].meta = deepmerge(db.slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
 			let metaPortion = JSON.stringify(req.body.meta);
 			if(metaPortion.length > 50) metaPortion = metaPortion.substring(0,20) + "...";
 			// console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion);
@@ -373,7 +335,7 @@ app.post("/api/getSlaveMeta", function (req, res) {
 	console.log("body", req.body);
     if(req.body && req.body.instanceID && req.body.password){
     	console.log("returning meta for ", req.body.instanceID);
-    	res.send(JSON.stringify(slaves[req.body.instanceID].meta))
+    	res.send(JSON.stringify(db.slaves[req.body.instanceID].meta))
 	} else {
     	res.status(400);
     	res.send('{"INVALID REQUEST":1}');
@@ -389,9 +351,9 @@ POST Check if a mod has been uploaded to the master before. Only checks against
 */
 app.post("/api/checkMod", function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let files = fs.readdirSync(masterModFolder);
+	let files: string[] = fs.readdirSync(masterModFolder);
 	let found = false;
-	files.forEach(file => {
+	files.forEach((file) => {
 		if(file == req.body.modName) {
 			found = true;
 		}
@@ -414,7 +376,7 @@ app.post("/api/uploadMod", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	if (req.files && req.files.file) {
 		// console.log(req.files.file);
-		req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err) {
+		req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err: Error) {
 			if (err) {
 				res.status(500).send(err);
 			} else {
@@ -433,7 +395,7 @@ GET endpoint for getting information about all our slaves
 @instance
 @alias /api/slaves
 */
-let slaveCache = {
+let slaveCache: {timestamp:number, cache?:Slaves} = {
 	timestamp: Date.now(),
 };
 app.get("/api/slaves", function(req, res) {
@@ -441,9 +403,9 @@ app.get("/api/slaves", function(req, res) {
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){
-		let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
+		let copyOfSlaves: Slaves = JSON.parse(JSON.stringify(db.slaves));
 		// filter out the rcon password because thats kindof not a safe thing to share
-		for(key in copyOfSlaves){
+		for(let key in copyOfSlaves){
 			copyOfSlaves[key].rconPassword = "hidden";
 		}
 		slaveCache.cache = copyOfSlaves;
@@ -452,8 +414,8 @@ app.get("/api/slaves", function(req, res) {
 	
 	res.send(slaveCache.cache);
 });
-var recievedItemStatisticsBySlaveID = {};
+var recievedItemStatisticsBySlaveID: any = {};
-var sentItemStatisticsBySlaveID = {};
+var sentItemStatisticsBySlaveID: any = {};
 /*
 var recievedItemStatistics = new averagedTimeSeries({
 	maxEntries: config.itemStats.maxEntries,
@@ -475,7 +437,7 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-	let x;
+	let x: item;
 	try {
 		x = JSON.parse(req.body);
 	} catch (e) {
@@ -486,8 +448,8 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	}
 	if(!x.instanceID) {
 		x.instanceID = "unknown";
-		Object.keys(slaves).forEach(instanceID => {
+		Object.keys(db.slaves).forEach(instanceID => {
-			if(slaves[instanceID].instanceName == req.body.instanceName) {
+			if(db.slaves[instanceID].instanceName == req.body.instanceName) {
 				x.instanceID = instanceID;
 			}
 		});
@@ -539,7 +501,7 @@ POST endpoint to remove items from DB when client orders items.
 @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name]
 @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
 */
-_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
+var _doleDivisionFactor: {[name:string]: number} = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 app.post("/api/remove", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
@@ -554,7 +516,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	if(!object.instanceName) {
 		object.instanceName = "unknown";
 	}
-	if(slaves[object.instanceID]) object.instanceName = slaves[object.instanceID].instanceName;
+	if(db.slaves[object.instanceID]) object.instanceName = db.slaves[object.instanceID].instanceName;
 	let item = db.items[object.name]
 		// console.dir(doc);
 	if(!item
@@ -672,18 +634,18 @@ app.get("/api/inventoryAsObject", function(req, res) {
 
  @memberof clusterioMaster
  @instance
- @alias api/modData
+ @alias api/modmeta
  @returns {object} JSON
  */
 
 // wrap a request in an promise
-    async function downloadPage(url) {
+async function downloadPage(url: string): Promise<object> {
-        return new Promise((resolve, reject) => {
+	return new Promise((resolve, reject) => {
-            request(url, (error, response, body) => {
+		request(url, (_error: any, _response: any, body: object) => {
-                resolve(body);
+			resolve(body);
-            });
+		});
-        });
+	});
-    }
+}
 
 app.get("/api/modmeta", async function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
@@ -709,7 +671,7 @@ app.post("/api/runCommand", (req,res) => {
 	var token = req.headers['x-access-token'];
 	if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' });
 	
-	jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
+	jwt.verify(token, config.masterAuthSecret, function(err: Error) {
 		if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' });
 		
 		// validate request.body
@@ -736,7 +698,7 @@ app.post("/api/runCommand", (req,res) => {
             && body.command[0] == "/")
         {
             // execute command
-            wsSlaves[body.instanceID].runCommand(body.command, data => {
+            wsSlaves[body.instanceID].runCommand(body.command, (data: string) => {
                 res.status(200).send({auth: true, message: "success", data});
             });
         } else {
@@ -754,7 +716,7 @@ GET endpoint. Returns factorio's base locale as a JSON object.
 */
 app.get("/api/getFactorioLocale", function(req,res){
 	endpointHitCounter.labels(req.route.path).inc();
-	getFactorioLocale.asObject(config.factorioDirectory, "en", (err, factorioLocale) => {
+	getFactorioLocale.asObject(config.factorioDirectory, "en", (_err, factorioLocale) => {
 		res.send(factorioLocale);
 	});
 });
@@ -763,7 +725,7 @@ if(args.sslPort || config.sslPort){
 		var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt'));
 		var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key'));
 
-		httpsServer = require("https").createServer({
+		var httpsServer = require("https").createServer({
 			key: privateKey,
 			cert: certificate
 		}, app).listen(args.sslPort || config.sslPort);
@@ -772,10 +734,11 @@ if(args.sslPort || config.sslPort){
 	}
 }
 if(args.masterPort || config.masterPort){
-	server = require("http").Server(app);
+	var server = require("http").Server(app);
 	server.listen(args.masterPort || config.masterPort || 8080, function () {
 		console.log("Listening on port %s...", server.address().port);
 	});
+	module.exports = server;
 }
 
 /* Websockets for remoteMap */
@@ -784,6 +747,9 @@ const ioMetrics = require("socket.io-prometheus");
 ioMetrics(io);
 var slaveMappers = {};
 class slaveMapper {
+	instanceID: string
+	socket: any
+	lastBeat: number
 	constructor(instanceID, socket) {
 		this.instanceID = instanceID;
 		this.socket = socket;
@@ -814,6 +780,10 @@ class slaveMapper {
 }
 var mapRequesters = {};
 class mapRequester {
+	requesterID: string
+	socket: any
+	instanceID: string
+	lastBeat: number
 	constructor(requesterID, socket, instanceID){
 		this.requesterID = requesterID;
 		this.socket = socket;
@@ -853,7 +823,13 @@ class mapRequester {
 }
 /* Websockets for send and recieve combinators */
 var wsSlaves = {};
+var wsChatRecievers = [];
+var wsAlertRecievers = [];
 class wsSlave {
+	instanceID: string
+	socket: any
+	lastBeat: number
+	commandsWaitingForReturn: object
 	constructor(instanceID, socket){
 		this.instanceID = instanceID;
 		this.socket = socket;
@@ -885,8 +861,8 @@ class wsSlave {
 			prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc();
 			if(typeof data === "object"){
 				data.timestamp = Date.now();
-				if(!global.wsChatRecievers) global.wsChatRecievers = [];
+				if(!wsChatRecievers) wsChatRecievers = [];
-				global.wsChatRecievers.forEach(socket => {
+				wsChatRecievers.forEach(socket => {
 					socket.emit("gameChat", data);
 				});
 			}
@@ -896,8 +872,8 @@ class wsSlave {
 			prometheusWsUsageCounter.labels('alert', this.instanceID).inc();
 			if(typeof alert === "object"){
 				alert.timestamp = Date.now();
-				if(!global.wsAlertRecievers) global.wsAlertRecievers = [];
+				if(!wsAlertRecievers) wsAlertRecievers = [];
-				global.wsAlertRecievers.forEach(socket => {
+				wsAlertRecievers.forEach(socket => {
 					socket.emit("alert", alert);
 				});
 			}
@@ -952,27 +928,26 @@ io.on('connection', function (socket) {
 	});
 	socket.on("registerChatReciever", function(data){
 		prometheusWsUsageCounter.labels("registerChatReciever", "other").inc();
-		if(!global.wsChatRecievers) global.wsChatRecievers = [];
+		if(!wsChatRecievers) wsChatRecievers = [];
-		global.wsChatRecievers.push(socket);
+		wsChatRecievers.push(socket);
 	});
 	socket.on("registerAlertReciever", function(data){
 		prometheusWsUsageCounter.labels("registerAlertReciever", "other").inc();
-		if(!global.wsAlertRecievers) global.wsAlertRecievers = [];
+		if(!wsAlertRecievers) wsAlertRecievers = [];
-		global.wsAlertRecievers.push(socket);
+		wsAlertRecievers.push(socket);
 	});
 });
 
 // handle plugins on the master
 pluginManagement();
-masterPlugins = [];
+var masterPlugins: MasterPlugin[] = [];
 async function pluginManagement(){
 	let startPluginLoad = Date.now();
 	masterPlugins = await getPlugins();
 	console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms");
 	return;
 }
-
+async function getPlugins(): Promise<MasterPlugin[]>{
-async function getPlugins(){
 	const pluginManager = require("./lib/manager/pluginManager.js")(config);
 	let plugins = [];
 	let pluginsToLoad = await pluginManager.getPlugins();
@@ -1006,5 +981,5 @@ async function getPlugins(){
 	return plugins;
 }
 
-module.exports = app;
 })();
+}

package.json

@@ -5,6 +5,10 @@
   "repository": "https://github.com/Danielv123/factorioClusterio",
   "main": "master.js",
   "scripts": {
+    "build": "tsc",
+    "watch-ts": "tsc -w",
+    "watch-node": "nodemon dist/master.js",
+    "watch": "concurrently -k -p \"[{name}]\" -n \"TypeScript, Node\" -c \"yello.bold, cyan.bold\" \"npm run watch-ts\" \"npm run watch-node\"",
     "test": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{{,node_modules/_app/**/},!(node_modules)/**/}*.spec.js\" -R spec || echo .",
     "start": "echo Run node client.js start [instance] to start slaves && node master.js",
     "cover": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{{,node_modules/_app/**/},!(node_modules)/**/}*.spec.js\" -R spec || echo . && codecov -t f7b38085-36eb-4d72-84e6-64721910171e",
@@ -19,6 +23,17 @@
   "author": "Danielv123",
   "license": "ISC",
   "dependencies": {
+    "@types/bcrypt": "^3.0.0",
+    "@types/body-parser": "^1.17.0",
+    "@types/deepmerge": "^2.2.0",
+    "@types/ejs": "^2.6.3",
+    "@types/express": "^4.16.1",
+    "@types/fs-extra": "^5.0.5",
+    "@types/mkdirp": "^0.5.2",
+    "@types/moment": "^2.13.0",
+    "@types/needle": "^2.0.4",
+    "@types/request": "^2.48.1",
+    "@types/socket.io": "^2.1.2",
     "app-root-path": "^2.0.1",
     "as-table": "^1.0.31",
     "averaged-timeseries": "^1.4.0",
@@ -65,6 +80,7 @@
     "socket.io-client": "^2.0.4",
     "socket.io-prometheus": "^0.2.1",
     "sync-request": "^4.0.1",
+    "typescript-require": "^0.2.10",
     "underscore": "^1.8.3"
   },
   "devDependencies": {

routes.js

@@ -11,7 +11,4 @@ module.exports = function(app) {
 	app.get("/nodeDetails", function(req, res) {
 		res.render("nodeDetails");
 	});
-	app.get("/remoteMap", function(req, res) {
-		res.render("remoteMap");
-	});
 };

scenarios/remoteMap/control.lua

@@ -2,6 +2,8 @@
 -- 
 -- Copyright (c) 2017 Florian Jung
 -- 
+-- Modified by Daniel Vestøl
+--
 -- Permission is hereby granted, free of charge, to any person obtaining a
 -- copy of this factorio lua stub and associated
 -- documentation files (the "Software"), to deal in the Software without
@@ -22,12 +24,13 @@
 -- DEALINGS IN THE SOFTWARE.
 
 function write_file(data)
-	stringsToWriteNextTick = stringsToWriteNextTick .. data .. '\n'
+	global.tileQueue = global.tileQueue .. data .. '\n'
+	-- stringsToWriteNextTick = stringsToWriteNextTick .. data .. '\n'
 	-- game.write_file("remoteMap.txt", data, true)
 end
 
 function dump_cached_writes()
-	game.write_file("remoteMap.txt", stringsToWriteNextTick, true, 0)
+	-- game.write_file("remoteMap.txt", stringsToWriteNextTick, true, 0)
 end
 
 function complain(text)
@@ -89,7 +92,7 @@ function on_chunk_generated(event)
 	end
 
 	-- writeout_resources(surface, area)
-	writeout_objects(surface, area)
+	-- writeout_objects(surface, area)
 	-- writeout_tiles(surface, area)
 end
 function on_some_entity_created(event, entp)
@@ -104,7 +107,7 @@ function on_some_entity_created(event, entp)
 		complain("wtf, on_some_entity_created has nil entity")
 		return
 	end
-	local entityData = ent.name..","..ent.position.x..","..ent.position.y
+	local entityData = "name="..ent.name..",x="..ent.position.x..",y="..ent.position.y
 	if ent.supports_direction then
 		entityData = entityData .. ",rot="..ent.direction
 	end
@@ -126,7 +129,7 @@ function on_some_entity_deleted(event)
 	--local area = {left_top={x=math.floor(ent.position.x), y=math.floor(ent.position.y)}, right_bottom={x=math.floor(ent.position.x)+1, y=math.floor(ent.position.y)+1}}
 
 	--table.insert(todo_next_tick, function () writeout_objects(surface, area ) end)
-	write_file("deleted,"..ent.position.x..","..ent.position.y)
+	write_file("name=deleted,x="..ent.position.x..",y="..ent.position.y)
 	--complain("on_some_entity_deleted: "..ent.name.." at "..ent.position.x..","..ent.position.y)
 end
 
@@ -143,3 +146,22 @@ script.on_event(defines.events.on_player_mined_entity, on_some_entity_deleted) -
 script.on_event(defines.events.on_robot_mined_entity, on_some_entity_deleted) --entity
 script.on_event(defines.events.on_resource_depleted, on_some_entity_deleted) --entity
 
+script.on_init(function()
+	global.tileQueue = ""
+end)
+
+remote.remove_interface("remoteMap")
+remote.add_interface("remoteMap", {
+	exportTiles = function()
+		rcon.print(global.tileQueue)
+		global.tileQueue = ""
+	end,
+	exportChunk = function(xc, yc, ref)
+		local data = ""
+		local entities = game.surfaces[1].find_entities({{xc*32,yc*32},{xc*32+32,yc*32+32}})
+		for _, ent in pairs(entities) do
+			data = data .. "|name:"..ent.name..",direction:"..ent.direction..",x:"..ent.position.x..",y:"..ent.position.y
+		end
+		rcon.print(data)
+	end
+})

sharedPlugins/remoteMap/chunkStore.js

@@ -1,100 +0,0 @@
-var mkdirp = require("mkdirp-promise");
-var mkdirpSync = require("mkdirp").sync;
-var fs = require("fs");
-var events = require("events");
-var path = require("path");
-
-var objectOps = require("./../../lib/objectOps");
-
-var LinvoDB = require("linvodb3");
-// LinvoDB.dbPath = "./database/linvodb/";
-
-
-let allowLogging = true;
-let log = string => (allowLogging)? console.log(string):""
-
-module.exports = class chunkStore {
-	constructor(name, chunkSize = 32, path = "./chunkStore/"){
-		if(!name || (typeof name != "string" && typeof name != "number")) throw new Error("chunkstore requires a name string as first param");
-		name = name.toString();
-		this.name = name;
-		this.chunkSize = chunkSize;
-		this.path = (path[path.length] == "/") ? path : path+"/";
-		mkdirpSync(path+name);
-		LinvoDB.dbPath = this.path;
-		
-		this.db = new LinvoDB(name, {}, {});
-		
-		this.eventEmitter = new events.EventEmitter();
-	}
-	setEntity(x/**{number}*/, y/**{number}*/, newEntity/**{object}*/){
-		return new Promise((resolve, reject) => {
-			x = Math.floor(x);
-			y = Math.floor(y);
-			this.db.find({x, y}, (err, docs) => {
-				if(typeof newEntity == "object"){
-					if(err) reject(err);
-					let entityDoc = {x, y, entity: newEntity};
-					if(docs && docs[0] && docs[0]._id){
-						entityDoc._id = docs[0]._id;
-					}
-					this.db.save(entityDoc, (err,docs) => {
-						if(err) reject(err);
-						resolve(docs);
-						this.eventEmitter.emit("change", docs)
-					});
-				} else {
-					docs.forEach(doc => {
-						doc.remove(()=>{
-							// removed :)
-						});
-					});
-					resolve();
-					this.eventEmitter.emit("change", {x,y});
-				}
-			});
-		});
-	}
-	getEntity(x, y){
-		return new Promise((resolve, reject) => {
-			this.db.find({x:Math.floor(x), y:Math.floor(y)}, (err, docs) => {
-				if(err) reject(err);
-				resolve(docs);
-			});
-		});
-	}
-	onEntityChange(eventhandler/**function*/){
-		if(typeof eventhandler == "function") {
-			this.eventEmitter.on("change", eventhandler);
-		}
-	}
-	getChunk(x, y){
-		return new Promise((resolve, reject) => {
-			
-			/*
-			// make new chunk in case we can't find an existing one
-			let chunk = {
-				dataObject: {},
-				position:{x,y}, // fancy ES6 feature that generates to {x:x, y:y}
-				chunkSize:this.chunkSize
-			};
-			fs.readFile(this.path+this.name+"/"+x+"/"+y, (err, chunkFile) => {
-				
-				if(err){
-					log(err)
-					resolve(chunk);
-				} else {
-					if(objectOps.isJSON(chunkFile)) {
-						log("returned from file")
-						let chunk = JSON.parse(chunkFile);
-						resolve(chunk);
-					} else {
-						log("file failed")
-						resolve(chunk);
-					}
-				}
-			}); // add x and y values as well and we are done?
-			*/
-		});
-	}
-}

sharedPlugins/remoteMap/chunkStore.spec.js

@@ -1,64 +0,0 @@
-var assert = require("assert");
-
-var chunkStore = require("./chunkStore.js");
-
-describe("remoteMap/chunkStore.js", () => {
-	// define some values for our test
-	const inserterX = 14.5; // rounds to 15
-	const inserterY = -17.5; // rounds to -17, +64 would be 47
-	const inserterName = "inserter";
-	
-	let store // make our store elevated
-	it("is a constructor", ()=>{
-		assert.equal(typeof chunkStore, "function");
-		// create a new store and elevate it
-		store = new chunkStore("910239"/* instanceID, use defaults for the rest */, undefined, "./database/unitTests/chunkStore/");
-		assert.equal(store.name, "910239");
-	});
-	it("allows you to store the locations of entities", done => {
-		store.setEntity(inserterX, inserterY, {name:inserterName, otherData:"can be arbitrary"}).then(() => {
-			done();
-			// nothing to do here really
-		});
-	});
-	it("returns entities at positions you query for", done => {
-		setTimeout(()=>{
-			store.getEntity(inserterX, inserterY).then(entities => {
-				entity = entities[0]
-				// console.log(entity)
-				assert.equal(entity.x, 14);
-				assert.equal(entity.y, -18);
-				assert.equal(entity.entity.name, "inserter");
-				assert.equal(entity.entity.otherData, "can be arbitrary");
-				done();
-			});
-		},300);
-	});
-	it("deletes entities when not provided with an object as 3rd arg", done => {
-		store.setEntity(inserterX, inserterY).then(entity => {
-			assert(entity === undefined);
-			setTimeout(()=>{
-				store.getEntity(inserterX, inserterY).then(entities => {
-					// console.log(entities)
-					assert.equal(entities.length, 0, "Why are we getting results, it should be empty?");
-					done();
-				});
-			},300);
-		});
-	});
-	describe("validates input by throwing when it is invalid", ()=>{
-		it("throws if called without a string as the first parameter", ()=>{
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore(undefined, undefined, "asdasda");
-			});
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore({}, undefined, "asdasdas");
-			});
-		});
-		it("converts numbers to strings automatically if they are provided  as the first parameter", ()=>{
-			assert.ok(()=>{
-				let nonThrowingChunkStore = new chunkStore(42, undefined, "./database/unitTests/chunkStore/");
-			});
-		});
-	});
-});

sharedPlugins/remoteMap/config.js

@@ -13,4 +13,5 @@ module.exports = {
 	description: modInfo.description,
 	// We'll send everything in this file to your stdin. Beware.
 	scriptOutputFileSubscription: "remoteMap.txt",
+	masterPlugin: "masterPlugin.js",
 }

sharedPlugins/remoteMap/index.js

@@ -1,130 +1,67 @@
 const needle = require("needle");
-const ioClient = require("socket.io-client");
 
-const chunkStore = require("./chunkStore.js");
+const debug = true
-
+module.exports = class remoteMap {
-const moduleConfig = require("./config"); // not to be confused with clusterio config. This config is private to this plugin.
+	constructor(mergedConfig, messageInterface, extras){
-
-class remoteMap {
-	constructor(slaveConfig, messageInterface){
-		this.config = slaveConfig;
 		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		this.socket = extras.socket;
+		this.instances = {};
 		
-		// initialize chunk database
+        this.socket.on("hello", () => this.socket.emit("registerMapServer"));
-		this.chunkMap = new chunkStore(this.config.unique, 64, "./database/chunkStore/");
+
-		this.chunkMap.onEntityChange(entity => {
+        this.socket.on("createEntity", req => {
-			// this.messageInterface("Entity changed: ")
+			// req = {name:"", position:{x,y}, ?direction:4}
-			// this.messageInterface(entity);
+			// game.player.surface.create_entity{name="small-ship-wreck", position={game.player.position.x-2, game.player.position.y+1}, direction=..., force=...}
-			// check valid coordinates (dunno what could go wrong, but can never be too sure)
+			this.messageInterface("placing "+JSON.stringify(req));
-			if(entity && entity.x !== undefined && !isNaN(Number(entity.x)) && entity.y !== undefined && !isNaN(Number(entity.y))){
+			if(req && req.name && typeof req.name == "string" && req.position && !isNaN(Number(req.position.x)) && !isNaN(Number(req.position.y))){
-				this.socket.emit("sendEntity", entity);
+				// delete any entity in that spot first, just in case to avoid overlaps
+				this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
+				// build the entity
+				let command = '/silent-command game.surfaces[1].create_entity{name="'+req.name+'", position={'+Number(req.position.x)+', '+Number(req.position.y)+'}, force="player"'
+				if(req.direction) command += ', direction='+req.direction;
+				command += '}';
+				this.messageInterface(command); // execute command
 			}
-		});
+        })
+        this.socket.on("deleteEntity", req => {
+			// req: {position: {x, y}}
+			this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
+        })
+		this.socket.on("getChunk", async (data, callback) => {
+			callback(await this.messageInterface(`/silent-command remote.call("remoteMap", "exportChunk", ${data.x}, ${data.y})`));
+		})
 		
-		// set up websocket communication and handle requests from web interface users (which are going through master)
+		// detect new entities created/deleted on the server and update the master
-		// socket should be a global
+		setInterval(async ()=>{
-		this.socket = ioClient("http://"+this.config.masterIP+":"+this.config.masterPort);
+			let data = await this.messageInterface(`/silent-command remote.call("remoteMap", "exportTiles")`)
-		this.socket.on("hello", data => {
+			if(data.length > 1) this.socket.emit("updateEntities", data)
-			this.socket.emit("registerSlaveMapper", {instanceID: this.config.unique});
+		}, 2000)
-			setInterval(()=>{
+	}
-				this.socket.emit("heartbeat"); // send our heartbeat to prevent being assumed dead
+	getInstanceName(instanceID){
-			},10000);
+		return new Promise((resolve, reject) => {
-		});
+			let instance = this.instances[instanceID];
-		this.socket.on("getChunk", req => {
+			if(!instance){
-			this.messageInterface("getChunk is not supported!");
+				needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => {
-			/*this.chunkMap.getChunk(req.x, req.y).then(chunk => {
+					if(err || response.statusCode != 200) {
-				chunk.requesterID = req.requesterID;
+						console.log("Unable to get JSON master/api/slaves, master might be unaccessible");
-				this.socket.emit("sendChunk", chunk);
+					} else if (response && response.body) {	
-			});*/
+						if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();}
-		});
+							try {
-		this.socket.on("getEntity", req => {
+								for (let index in response.body)
-			// check for valid coordinates
+									this.instances[index] = response.body[index].instanceName;
-			if(req.x !== undefined && !isNaN(Number(req.x)) && req.y !== undefined && !isNaN(Number(req.y))){
+							} catch (e){
-				this.chunkMap.getEntity(req.x, req.y).then(entities => {
+								console.log(e);
-					if(entities && entities.length > 0){
+								return null;
-						entities.forEach(entity => {
+							}
-							this.socket.emit("sendEntity", entity);
+						instance = this.instances[instanceID] 							
-						});
+						if (!instance) instance = instanceID;  //somehow the master doesn't know the instance	
+						resolve(instance);
 					}
 				});
 			} else {
-				this.messageInterface("socket.getEntity triggered called with invalid parameters");
+				resolve(instance);
 			}
 		});
-		// both placing and deleting entities via the web interface
-		this.socket.on("placeEntity", req => {
-			// game.player.surface.create_entity{name="small-ship-wreck", position={game.player.position.x-2, game.player.position.y+1}, direction=..., force=...}
-			this.messageInterface("placing "+JSON.stringify(req));
-			if(req.name && typeof req.name == "string" && req.position && !isNaN(Number(req.position.x)) && !isNaN(Number(req.position.y))){
-				if(req.name == "deleted"){ // {{'+req.position.x+','+req.position.y+'},{'+req.position.x+','+req.position.y+'}}
-					// delete entity from game world using LUA command
-					this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+1)+','+(req.position.y+1)+'}}) for i, entity in pairs(toDelete) do entity.die() end');
-					// this position is now empty, delete whatever was there from DB
-					this.deleteEntity(req.position.x, req.position.y);
-				} else {
-					let command = '/silent-command game.surfaces[1].create_entity{name="'+req.name+'", position={'+Number(req.position.x)+', '+Number(req.position.y)+'}, force="neutral"'
-					if(req.direction) command += ', direction='+req.direction;
-					command += '}';
-					this.messageInterface(command); // execute command
-					
-					// add entity to the chunkStore database because create_entity does not trigger any events
-					let name = req.name;
-					let entity = {name};
-					if(typeof req.direction == "string"){
-						entity.rot = req.direction;
-					}
-					this.addEntity(req.position.x, req.position.y, entity);
-				}
-			}
-		});
-		
-		this.messageInterface("/silent-command game.print('"+moduleConfig.name+" version "+moduleConfig.version+" enabled')");
 	}
-	addEntity(x,y,entity){ // add entity to database
+}
-		this.chunkMap.setEntity(Number(x), Number(y), entity).then(data => {
-			this.messageInterface("Added "+entity.name+" to data "+Number(x)+", "+Number(y));
-		}).catch((err)=>this.messageInterface(err));
-	}
-	deleteEntity(x,y){ // remove entity from database
-		// this position is now empty, delete whatever was there from DB
-		this.chunkMap.setEntity(x, y, "delete this entity").then(() => {
-			this.messageInterface("Deleted entity at "+x+", "+y);
-		}).catch((err)=>this.messageInterface(err));
-	}
-	scriptOutput(data){
-		if(data){
-			this.messageInterface(data);
-			// express-transport-belt,-35.5,-14.5,[...]
-			data = data.split(",");
-			if(data && data[0] == undefined){
-				this.messageInterface("empty: ");
-				this.messageInterface(data);
-				return
-			}
-			let name = data[0];
-			let xPos = data[1];
-			let yPos = data[2];
-			if(xPos && yPos && name){
-				if(name == "deleted"){
-					// this position is now empty, delete whatever was there from DB
-					this.deleteEntity(xPos, yPos);
-				} else {
-					let entity = {name};
-					for(let i = 3; i < data.length; i++){
-						if(data[i].includes("=")){
-							let kv = data[i].split("=");
-							entity[kv[0]] = kv[1];
-						}
-					}
-					// save this entity in the database
-					this.addEntity(xPos, yPos, entity);
-					/* this.chunkMap.setEntity(xPos, yPos, entity).then(data => {
-						this.messageInterface("Added "+name+" to data "+xPos+", "+yPos);
-					}).catch((err)=>this.messageInterface(err)); */
-				}
-			}
-			
-		}
-	}
-}
-module.exports = remoteMap;

sharedPlugins/remoteMap/info.json

@@ -2,7 +2,7 @@
 {
 "name": "remoteMap",
 "title": "Clusterio remote map",
-"version": "2.2.0",
+"version": "3.0.0",
 "date": "18-11-2017",
 "author": "Danielv123",
 "dependencies": ["base >= 0.15"],

sharedPlugins/remoteMap/masterPlugin.js

@@ -0,0 +1,175 @@
+const fs = require("fs-extra");
+const path = require("path");
+const sanitizer = require('sanitizer');
+
+const editorSockets = [];
+
+class masterPlugin {
+	constructor({ config, pluginConfig, pluginPath, socketio, express }) {
+		this.config = config;
+		this.pluginConfig = pluginConfig;
+		this.pluginPath = pluginPath;
+		this.io = socketio;
+		this.app = express;
+		
+		this.clients = {};
+		this.slaves = {};
+		
+		// initialize token auth module
+		this.authenticate = require("./../../lib/authenticate.js")(config);
+
+		// expose UI elements embedded in the master
+		//this.ui = require("./js/ui.js").ui;
+
+		// handle websocket connections
+		this.io.on("connection", socket => {
+			let instanceID = "unknown";
+			socket.on("registerSlave", data => {
+				if (data.instanceID && !isNaN(Number(data.instanceID))) {
+					instanceID = data.instanceID;
+					this.slaves[instanceID] = socket;
+				}
+				socket.on("updateEntities", data => {
+					console.log(data)
+					if(data){
+						// transform from weirdly deliminated string to entity object
+						let entities = data
+						.split("\n")
+						.filter(str => str.length > 0)
+						.map(ent => ent.replace(/[-]/g, "_"))											// replace - with _
+						.map(str => str
+							.split(",")
+							.filter(x => x.length > 0)
+							.map(prop => ({
+								k: prop.split("=")[0],
+								v: prop.split("=")[1]})
+							)
+						)
+						.map(arr => {
+							let obj = {}
+							arr.forEach(property => obj[property.k] = property.v)
+							return obj
+						})
+						console.log(entities)
+						editorSockets.forEach(sock => sock.emit("updateEntity", {entities}))
+					}
+				})
+			});
+			socket.on("registerMapEditor", () => {
+				console.log("Registered mapEditor socket")
+				editorSockets.push(socket);
+				
+				setTimeout(()=>socket.emit("createEntity", {entity:{
+					name:"fast_inserter",
+					position: {x:30,y:30},
+					direction: 2,
+				}}),10000)
+				
+				socket.on("createEntity", data => {
+					this.slaves[Object.keys(this.slaves)[0]].emit("createEntity", data.entity)
+				})
+				socket.on("deleteEntity", data => {
+					this.slaves[Object.keys(this.slaves)[0]].emit("deleteEntity", data.entity)
+				})
+				socket.on("getChunk", (data, callback) => {
+					this.slaves[Object.keys(this.slaves)[0]].emit("getChunk", data, resp => {
+						// parse response
+						// |name:transport-belt,direction:2,x:99,y:1231|name:assembly-machine,direction:6,x:939,y:88
+						console.log(resp)
+						callback(resp
+							.split("|")																	// split into entities
+							.map(str => str.replace("\n",""))											// remove newlines that happen at the end of the last entity (maybe more as well???)
+							.filter(str => str.length > 0)												// remove blank entities
+							.filter(str => 
+							// !str.includes("tree") &&													// don't send entities that can't be rendered anyways
+							!str.includes("dead") &&
+							// !str.includes("rock") &&
+							!str.includes("crude-oil") &&
+							!str.includes("remnants") &&
+							!str.includes("cliff") &&
+							!str.includes("player") &&
+							!str.includes("ore"))
+							.map(ent => ent.split(",")													// split entity into properties
+								.map(prop => [prop.split(":")[0], prop.split(":")[1]])					// split properties into KV pairs
+							)
+							.map(entity => {															// turn entities into objects
+								let entityObj = {}
+								entity.forEach(property => entityObj[property[0]] = property[1])
+								return entityObj
+							})
+							.map(entity => {
+								entity.name = entity.name.replace(/[-]/g, "_")							// replace - with _ in entity names as thats what factorio.blueprint-editor preffers
+								return entity
+							})
+							.filter(entity => Object.keys(entity).length > 1)
+						)
+					})
+				})
+				
+				socket.on("disconnect", function () {
+					let i = editorSockets.indexOf(socket);
+					console.log("editorSocket " + (i + 1) + " disconnected, " + (editorSockets.length - 1) + " left");
+					editorSockets.splice(i, 1);
+				});
+			});
+			socket.on("gameChat", async data => {
+				let chatLine = data.data.replace(/(\r\n\t|\n|\r\t)/gm, "").replace("\r", "");
+				//if (typeof chatLine == "string") this.handleChatLine(chatLine, instanceID);
+			});
+		});
+
+	}
+	async onExit() {
+		return;
+	}
+	async onLoadFinish({ plugins }) {
+		this.masterPlugins = plugins;
+	}
+	parseData(data, sharedData = {}) {
+		let parsedData = [];
+		data = data.split("|");
+		data.forEach(player => {
+			if (player) {
+				let playerData = {};
+				player = player.split(`~`);
+				player.forEach(kv => {
+					kv = kv.split(":");
+					if (kv[1] === undefined || kv[0] === undefined) {
+						console.log(new Error(`Something is wrong! Key:${kv[0]} Value:${kv[1]}`));
+					} else {
+						playerData[kv[0]] = kv[1].trim();
+					}
+				});
+				for (let k in sharedData) {
+					playerData[k] = sharedData[k];
+				}
+				parsedData.push(playerData);
+			}
+		});
+		return parsedData;
+	}
+}
+module.exports = masterPlugin;
+
+function getDatabaseSync(path) {
+	let db;
+	try {
+		db = JSON.parse(fs.readFileSync(path, "utf8"));
+	} catch (e) {
+		db = {};
+	}
+	return db;
+}
+async function saveDatabase(path, database) {
+	if (!path) {
+		throw new Error("No path provided!");
+	} else if (!database) {
+		throw new Error("No database provided!");
+	} else {
+		try {
+			await fs.writeFile(path, JSON.stringify(database, null, 4));
+		} catch (e) {
+			throw new Error("Unable to write to database! " + path);
+		}
+	}
+}

sharedPlugins/remoteMap/readme.md

@@ -0,0 +1,15 @@
+# Remote Map
+
+A clusterio plugin for remotely exploring and editing the factorio world
+
+### Setup
+
+Make a database for storing entities
+
+    docker run --name some-mariadb -e MYSQL_ROOT_PASSWORD=clusterio-remote-map --restart=unless-stopped -d mariadb:latest
+
+### Dev setup
+
+Make a database container and expose the ports
+
+    docker run -p 3306:3306 --name some-mariadb -e MYSQL_ROOT_PASSWORD=clusterio-remote-map -d mariadb:latest

sharedPlugins/scenario_loader/index.js

@@ -11,59 +11,69 @@ module.exports = class remoteCommands {
 			let hotpatchInstallStatus = await this.checkHotpatchInstallation();
 			messageInterface("Hotpach installation status: "+hotpatchInstallStatus);
 			if(hotpatchInstallStatus){
-				let scenarioDir = "scenarios";
+				await this.loadScenariosFromFolder("scenarios") // located in project root folder (with sharedMods, sharedPlugins etc)
-				await fs.ensureDir(scenarioDir)
+				let plugins = new AsyncArray((await fs.readdir(path.join(__dirname, "../"))))
-				let scenarios = await fs.readdir(scenarioDir);
+				plugins.filterAsync(async plugin => {
-				for(let i = 0; i < scenarios.length; i++){
+					// check if plugin is a folder and if plugin has a scenarios folder
-					messageInterface(`Loading scenario ${scenarios[i]}`)
+					
-					let startTime = Date.now();
+				})
-					let scenPath = path.join(scenarioDir, scenarios[i]);
+				for(let g = 0; g < plugins.length; g++){
-					let stat = await fs.stat(scenPath);
+					
-					if(stat.isDirectory){
-						async function loadFiles(dir, rootdir){
-							let returnFiles = [];
-							let entries = await fs.readdir(path.join(rootdir, dir));
-							for(let o = 0; o < entries.length; o++){
-								let stat = await fs.stat(path.join(rootdir, dir, entries[o]));
-								if(stat.isDirectory()){
-									// recurse
-									let subFolder = await loadFiles(path.join(dir, entries[o]), rootdir);
-									returnFiles = returnFiles.concat(subFolder);
-								} else if(entries[o].split(".")[entries[o].split(".").length-1].toLowerCase() == "lua"){
-									// if it is a Lua file, read its contents
-									// let data = await fs.readFile(path.join(rootdir, dir, entries[o]));
-									let data = await clusterTools.getLua(path.join(rootdir, dir, entries[o]), false);
-									returnFiles.push({
-										name: path.join(dir, entries[o].split(".")[0]),
-										data,
-									});
-								}
-							}
-							return returnFiles;
-						}
-						let fileMap = await loadFiles("", scenPath);
-						let files = {};
-						fileMap.forEach(map => {
-							files[map.name] = map.data;
-							console.log(map.name);
-						});
-						
-						let fileImportString = `\{`;
-						for(let k in files){
-							let name = k.replace(/\\/g, '/');
-							name = name.replace(/ /g, '');
-							// make sure *not* to include control.lua as it is provided as a seperate argument
-							if(name != "control") fileImportString += `["${name}"] = '${files[k]}', `;
-						}
-						fileImportString += `\}`;
-						if(files.control) var returnValue = await messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
-						if(returnValue) messageInterface(returnValue);
-						messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
-					}
 				}
 			}
 		})();
 	}
+	async loadScenariosFromFolder(scenarioDir){
+		await fs.ensureDir(scenarioDir)
+		let scenarios = await fs.readdir(scenarioDir);
+		for(let i = 0; i < scenarios.length; i++){
+			messageInterface(`Loading scenario ${scenarios[i]}`)
+			let startTime = Date.now();
+			let scenPath = path.join(scenarioDir, scenarios[i]);
+			let stat = await fs.stat(scenPath);
+			if(stat.isDirectory){
+				async function loadFiles(dir, rootdir){
+					let returnFiles = [];
+					let entries = await fs.readdir(path.join(rootdir, dir));
+					for(let o = 0; o < entries.length; o++){
+						let stat = await fs.stat(path.join(rootdir, dir, entries[o]));
+						if(stat.isDirectory()){
+							// recurse
+							let subFolder = await loadFiles(path.join(dir, entries[o]), rootdir);
+							returnFiles = returnFiles.concat(subFolder);
+						} else if(entries[o].split(".")[entries[o].split(".").length-1].toLowerCase() == "lua"){
+							// if it is a Lua file, read its contents
+							// let data = await fs.readFile(path.join(rootdir, dir, entries[o]));
+							let data = await clusterTools.getLua(path.join(rootdir, dir, entries[o]), false);
+							returnFiles.push({
+								name: path.join(dir, entries[o].split(".")[0]),
+								data,
+							});
+						}
+					}
+					return returnFiles;
+				}
+				let fileMap = await loadFiles("", scenPath);
+				let files = {};
+				fileMap.forEach(map => {
+					files[map.name] = map.data;
+					console.log(map.name);
+				});
+				
+				let fileImportString = `\{`;
+				for(let k in files){
+					let name = k.replace(/\\/g, '/');
+					name = name.replace(/ /g, '');
+					// make sure *not* to include control.lua as it is provided as a seperate argument
+					if(name != "control") fileImportString += `["${name}"] = '${files[k]}', `;
+				}
+				fileImportString += `\}`;
+				if(files.control) var returnValue = await messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
+				if(returnValue) messageInterface(returnValue);
+				messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
+			}
+		}
+	}
 	async checkHotpatchInstallation(){
 		let yn = await this.messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end");
 		yn = yn.replace(/(\r\n\t|\n|\r\t)/gm, "");
@@ -96,3 +106,22 @@ function readDirectorySynchronously(directory) {
         }
     });
 }
+class AsyncArray extends Array {
+	constructor(arr) {
+		// this.data = arr; // In place of Array subclassing
+	}
+
+	filterAsync(predicate) {
+		// Take a copy of the array, it might mutate by the time we've finished
+		const data = Array.from(this.data);
+		// Transform all the elements into an array of promises using the predicate
+		// as the promise
+		return Promise.all(data.map((element, index) => predicate(element, index, data)))
+		// Use the result of the promises to call the underlying sync filter function
+		.then(result => {
+			return data.filter((element, index) => {
+				return result[index];
+			});
+		});
+	}
+}

tsconfig.json

@@ -0,0 +1,23 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "esModuleInterop": false,
+        "target": "es6",
+        "noImplicitAny": false,
+        "moduleResolution": "node",
+        "sourceMap": true,
+        "outDir": "dist",
+        "baseUrl": ".",
+        "paths": {
+            "*": [
+                "node_modules/",
+                "src/types/*",
+                "master.d.ts",
+                "types/*"
+            ]
+        }
+    },
+    "include": [
+        "**/*"
+    ]
+}

types/ClusterioConfig.d.ts

@@ -0,0 +1,39 @@
+type ClusterioConfig = {
+    factorioDirectory: string,
+    instanceDirectory: string,
+    databaseDirectory: string,
+    publicIP: string,
+    masterIP: string,
+    masterPort: number,
+    sslPort: number,
+    sslCert: string,
+    sslPrivKey: string,
+    masterAuthToken: string,
+    masterAuthSecret: string,
+    username: string,
+    token: string,
+    factorio_version: string,
+    verify_user_identity: boolean,
+    auto_pause: boolean,
+    allow_commands: "admins-only" | "true" | "false",
+    game_password: string,
+    description: string,
+    visibility: {
+        public: boolean,
+        lan: boolean
+    },
+    itemStats: {
+        maxEntries: number,
+        entriesPerSecond: number
+    },
+    logItemTransfers: boolean,
+    disableFairItemDistribution: boolean,
+    uploadModsToMaster: boolean,
+    msBetweenCommands: 10,
+    allowRemoteCommandExecution: boolean,
+    enableCrossServerShout: boolean,
+    mirrorAllChat: boolean,
+    autosaveInterval: 600000,
+    disablePrometheusPushgateway: boolean,
+    disableImportsOfEverythingExceptElectricity: boolean
+}

types/Mod.d.ts

@@ -0,0 +1,4 @@
+type Mod = {
+    modName: string,
+    hash: string,
+}

types/Request.d.ts

@@ -0,0 +1,9 @@
+interface Request {
+       route?: any,
+       body?: requestBody,
+    }
+    interface requestBody extends Body {
+        unique: string,
+        
+    }
+ 

types/Slave.d.ts

@@ -0,0 +1,15 @@
+interface Slave {
+    time?: string,
+    rconPort?: string,
+    rconPassword?: string,
+    serverPort?: string,
+    unique?: string, // this one isn't really optional, but I got tired of fighting with types and rewriting logic in master.ts
+    publicIP?: string,
+    mods?: Mod[],
+    instanceName?: string,
+    playerCount?: string | number,
+    mac?: string,
+    meta?: {
+        [name:string]:any,
+    },
+}

types/Slaves.d.ts

@@ -0,0 +1,3 @@
+type Slaves = {
+    [name: string]: Slave,
+}

types/item.d.ts

@@ -0,0 +1,6 @@
+type item = {
+    name: string,
+    count: number,
+    instanceName?: string,
+    instanceID?: string,
+}

types/master.d.ts

@@ -0,0 +1,16 @@
+type MasterPluginInstance = {
+    config: object,
+    pluginConfig: object,
+    path: string,
+    socketio: any,
+    express: any,
+    Prometheus: any,
+    onExit?: Function,
+}
+type pluginConfig = {
+    name: string,
+}
+interface MasterPlugin {
+    main: MasterPluginInstance,
+    pluginConfig: pluginConfig,
+}

types/slaveMapper.d.ts

@@ -0,0 +1,3 @@
+interface slaveMapper {
+    instanceID: string,
+}