Fixed remoteMap stuff

.gitignore

@@ -64,3 +64,7 @@ sharedPlugins/scenario_loader/scenarios
 sharedPlugins/*/DISABLED
 
 \.cache-require-paths\.json
+
+\.vs/
+
+bin/

client.js

@@ -640,7 +640,9 @@ async function instanceManagement(instanceconfig) {
 									if(stuff[o] && !stuff[o].includes('\u0000\u0000')) {
 										try{
 											plugins[i].scriptOutput(stuff[o]);
-										}catch(e){console.error(e)}
+										}catch(e){
+											throw new Error(`Plugin ${pluginConfig.name} has a fileOutputSubscription but plugin.scriptOutput is not a function`)
+										}
 									}
 								}
 							},

master.js

@@ -769,6 +769,7 @@ if(args.masterPort || config.masterPort){
 	server.listen(args.masterPort || config.masterPort || 8080, function () {
 		console.log("Listening on port %s...", server.address().port);
 	});
+	module.exports = server;
 }
 
 /* Websockets for remoteMap */
@@ -999,5 +1000,4 @@ async function getPlugins(){
 	return plugins;
 }
 
-module.exports = app;
 })();

master.spec.js

@@ -76,17 +76,6 @@ describe('Master server endpoint testing', function() {
 				});
 			});
 		});
-		it("sends some HTML when accessing /remoteMap",function(done){
-			this.timeout(6000);
-			request(app).get("/remoteMap").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					// there should be 1 error regarding complaining about me using ES6 modules before they are fully supported
-					assert(result.messages.length === 2, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
 	});
 	let persistentMaster = request(app);
 	describe("#POST /api/place", function(){

routes.js

@@ -11,7 +11,4 @@ module.exports = function(app) {
 	app.get("/nodeDetails", function(req, res) {
 		res.render("nodeDetails");
 	});
-	app.get("/remoteMap", function(req, res) {
-		res.render("remoteMap");
-	});
 };

scenarios/remoteMap/control.lua

@@ -2,6 +2,8 @@
 -- 
 -- Copyright (c) 2017 Florian Jung
 -- 
+-- Modified by Daniel Vestøl
+--
 -- Permission is hereby granted, free of charge, to any person obtaining a
 -- copy of this factorio lua stub and associated
 -- documentation files (the "Software"), to deal in the Software without
@@ -22,12 +24,13 @@
 -- DEALINGS IN THE SOFTWARE.
 
 function write_file(data)
-	stringsToWriteNextTick = stringsToWriteNextTick .. data .. '\n'
+	global.tileQueue = global.tileQueue .. data .. '\n'
+	-- stringsToWriteNextTick = stringsToWriteNextTick .. data .. '\n'
 	-- game.write_file("remoteMap.txt", data, true)
 end
 
 function dump_cached_writes()
-	game.write_file("remoteMap.txt", stringsToWriteNextTick, true, 0)
+	-- game.write_file("remoteMap.txt", stringsToWriteNextTick, true, 0)
 end
 
 function complain(text)
@@ -89,7 +92,7 @@ function on_chunk_generated(event)
 	end
 
 	-- writeout_resources(surface, area)
-	writeout_objects(surface, area)
+	-- writeout_objects(surface, area)
 	-- writeout_tiles(surface, area)
 end
 function on_some_entity_created(event, entp)
@@ -104,7 +107,7 @@ function on_some_entity_created(event, entp)
 		complain("wtf, on_some_entity_created has nil entity")
 		return
 	end
-	local entityData = ent.name..","..ent.position.x..","..ent.position.y
+	local entityData = "name="..ent.name..",x="..ent.position.x..",y="..ent.position.y
 	if ent.supports_direction then
 		entityData = entityData .. ",rot="..ent.direction
 	end
@@ -126,7 +129,7 @@ function on_some_entity_deleted(event)
 	--local area = {left_top={x=math.floor(ent.position.x), y=math.floor(ent.position.y)}, right_bottom={x=math.floor(ent.position.x)+1, y=math.floor(ent.position.y)+1}}
 
 	--table.insert(todo_next_tick, function () writeout_objects(surface, area ) end)
-	write_file("deleted,"..ent.position.x..","..ent.position.y)
+	write_file("name=deleted,x="..ent.position.x..",y="..ent.position.y)
 	--complain("on_some_entity_deleted: "..ent.name.." at "..ent.position.x..","..ent.position.y)
 end
 
@@ -143,3 +146,22 @@ script.on_event(defines.events.on_player_mined_entity, on_some_entity_deleted) -
 script.on_event(defines.events.on_robot_mined_entity, on_some_entity_deleted) --entity
 script.on_event(defines.events.on_resource_depleted, on_some_entity_deleted) --entity
 
+script.on_init(function()
+	global.tileQueue = ""
+end)
+
+remote.remove_interface("remoteMap")
+remote.add_interface("remoteMap", {
+	exportTiles = function()
+		rcon.print(global.tileQueue)
+		global.tileQueue = ""
+	end,
+	exportChunk = function(xc, yc, ref)
+		local data = ""
+		local entities = game.surfaces[1].find_entities({{xc*32,yc*32},{xc*32+32,yc*32+32}})
+		for _, ent in pairs(entities) do
+			data = data .. "|name:"..ent.name..",direction:"..ent.direction..",x:"..ent.position.x..",y:"..ent.position.y
+		end
+		rcon.print(data)
+	end
+})

sharedPlugins/remoteMap/chunkStore.js

@@ -1,100 +0,0 @@
-var mkdirp = require("mkdirp-promise");
-var mkdirpSync = require("mkdirp").sync;
-var fs = require("fs");
-var events = require("events");
-var path = require("path");
-
-var objectOps = require("./../../lib/objectOps");
-
-var LinvoDB = require("linvodb3");
-// LinvoDB.dbPath = "./database/linvodb/";
-
-
-let allowLogging = true;
-let log = string => (allowLogging)? console.log(string):""
-
-module.exports = class chunkStore {
-	constructor(name, chunkSize = 32, path = "./chunkStore/"){
-		if(!name || (typeof name != "string" && typeof name != "number")) throw new Error("chunkstore requires a name string as first param");
-		name = name.toString();
-		this.name = name;
-		this.chunkSize = chunkSize;
-		this.path = (path[path.length] == "/") ? path : path+"/";
-		mkdirpSync(path+name);
-		LinvoDB.dbPath = this.path;
-		
-		this.db = new LinvoDB(name, {}, {});
-		
-		this.eventEmitter = new events.EventEmitter();
-	}
-	setEntity(x/**{number}*/, y/**{number}*/, newEntity/**{object}*/){
-		return new Promise((resolve, reject) => {
-			x = Math.floor(x);
-			y = Math.floor(y);
-			this.db.find({x, y}, (err, docs) => {
-				if(typeof newEntity == "object"){
-					if(err) reject(err);
-					let entityDoc = {x, y, entity: newEntity};
-					if(docs && docs[0] && docs[0]._id){
-						entityDoc._id = docs[0]._id;
-					}
-					this.db.save(entityDoc, (err,docs) => {
-						if(err) reject(err);
-						resolve(docs);
-						this.eventEmitter.emit("change", docs)
-					});
-				} else {
-					docs.forEach(doc => {
-						doc.remove(()=>{
-							// removed :)
-						});
-					});
-					resolve();
-					this.eventEmitter.emit("change", {x,y});
-				}
-			});
-		});
-	}
-	getEntity(x, y){
-		return new Promise((resolve, reject) => {
-			this.db.find({x:Math.floor(x), y:Math.floor(y)}, (err, docs) => {
-				if(err) reject(err);
-				resolve(docs);
-			});
-		});
-	}
-	onEntityChange(eventhandler/**function*/){
-		if(typeof eventhandler == "function") {
-			this.eventEmitter.on("change", eventhandler);
-		}
-	}
-	getChunk(x, y){
-		return new Promise((resolve, reject) => {
-			
-			/*
-			// make new chunk in case we can't find an existing one
-			let chunk = {
-				dataObject: {},
-				position:{x,y}, // fancy ES6 feature that generates to {x:x, y:y}
-				chunkSize:this.chunkSize
-			};
-			fs.readFile(this.path+this.name+"/"+x+"/"+y, (err, chunkFile) => {
-				
-				if(err){
-					log(err)
-					resolve(chunk);
-				} else {
-					if(objectOps.isJSON(chunkFile)) {
-						log("returned from file")
-						let chunk = JSON.parse(chunkFile);
-						resolve(chunk);
-					} else {
-						log("file failed")
-						resolve(chunk);
-					}
-				}
-			}); // add x and y values as well and we are done?
-			*/
-		});
-	}
-}

sharedPlugins/remoteMap/chunkStore.spec.js

@@ -1,64 +0,0 @@
-var assert = require("assert");
-
-var chunkStore = require("./chunkStore.js");
-
-describe("remoteMap/chunkStore.js", () => {
-	// define some values for our test
-	const inserterX = 14.5; // rounds to 15
-	const inserterY = -17.5; // rounds to -17, +64 would be 47
-	const inserterName = "inserter";
-	
-	let store // make our store elevated
-	it("is a constructor", ()=>{
-		assert.equal(typeof chunkStore, "function");
-		// create a new store and elevate it
-		store = new chunkStore("910239"/* instanceID, use defaults for the rest */, undefined, "./database/unitTests/chunkStore/");
-		assert.equal(store.name, "910239");
-	});
-	it("allows you to store the locations of entities", done => {
-		store.setEntity(inserterX, inserterY, {name:inserterName, otherData:"can be arbitrary"}).then(() => {
-			done();
-			// nothing to do here really
-		});
-	});
-	it("returns entities at positions you query for", done => {
-		setTimeout(()=>{
-			store.getEntity(inserterX, inserterY).then(entities => {
-				entity = entities[0]
-				// console.log(entity)
-				assert.equal(entity.x, 14);
-				assert.equal(entity.y, -18);
-				assert.equal(entity.entity.name, "inserter");
-				assert.equal(entity.entity.otherData, "can be arbitrary");
-				done();
-			});
-		},300);
-	});
-	it("deletes entities when not provided with an object as 3rd arg", done => {
-		store.setEntity(inserterX, inserterY).then(entity => {
-			assert(entity === undefined);
-			setTimeout(()=>{
-				store.getEntity(inserterX, inserterY).then(entities => {
-					// console.log(entities)
-					assert.equal(entities.length, 0, "Why are we getting results, it should be empty?");
-					done();
-				});
-			},300);
-		});
-	});
-	describe("validates input by throwing when it is invalid", ()=>{
-		it("throws if called without a string as the first parameter", ()=>{
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore(undefined, undefined, "asdasda");
-			});
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore({}, undefined, "asdasdas");
-			});
-		});
-		it("converts numbers to strings automatically if they are provided  as the first parameter", ()=>{
-			assert.ok(()=>{
-				let nonThrowingChunkStore = new chunkStore(42, undefined, "./database/unitTests/chunkStore/");
-			});
-		});
-	});
-});

sharedPlugins/remoteMap/config.js

@@ -13,4 +13,5 @@ module.exports = {
 	description: modInfo.description,
 	// We'll send everything in this file to your stdin. Beware.
 	scriptOutputFileSubscription: "remoteMap.txt",
+	masterPlugin: "masterPlugin.js",
 }

sharedPlugins/remoteMap/index.js

@@ -1,130 +1,71 @@
 const needle = require("needle");
-const ioClient = require("socket.io-client");
 
-const chunkStore = require("./chunkStore.js");
+const debug = true
-
+module.exports = class remoteMap {
-const moduleConfig = require("./config"); // not to be confused with clusterio config. This config is private to this plugin.
+	constructor(mergedConfig, messageInterface, extras){
-
-class remoteMap {
-	constructor(slaveConfig, messageInterface){
-		this.config = slaveConfig;
 		this.messageInterface = messageInterface;
+		this.config = mergedConfig;
+		this.socket = extras.socket;
+		this.instances = {};
 		
-		// initialize chunk database
+        this.socket.on("hello", () => this.socket.emit("registerMapServer", {
-		this.chunkMap = new chunkStore(this.config.unique, 64, "./database/chunkStore/");
+			instanceID: this.config.unique, // instanceID
-		this.chunkMap.onEntityChange(entity => {
+		}));
-			// this.messageInterface("Entity changed: ")
+
-			// this.messageInterface(entity);
+        this.socket.on("createEntity", req => {
-			// check valid coordinates (dunno what could go wrong, but can never be too sure)
+			// req = {name:"", position:{x,y}, ?direction:4}
-			if(entity && entity.x !== undefined && !isNaN(Number(entity.x)) && entity.y !== undefined && !isNaN(Number(entity.y))){
+			// game.player.surface.create_entity{name="small-ship-wreck", position={game.player.position.x-2, game.player.position.y+1}, direction=..., force=...}
-				this.socket.emit("sendEntity", entity);
+			this.messageInterface("placing "+JSON.stringify(req));
+			if(req && req.name && typeof req.name == "string" && req.position && !isNaN(Number(req.position.x)) && !isNaN(Number(req.position.y))){
+				// delete any entity in that spot first, just in case to avoid overlaps
+				this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
+				// build the entity
+				let command = '/silent-command game.surfaces[1].create_entity{name="'+req.name+'", position={'+Number(req.position.x)+', '+Number(req.position.y)+'}, force="player"'
+				if(req.direction) command += ', direction='+req.direction;
+				command += '}';
+				this.messageInterface(command); // execute command
 			}
-		});
+        })
+        this.socket.on("deleteEntity", req => {
+			// req: {position: {x, y}}
+			this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
+        })
+		this.socket.on("getChunk", async (data, callback) => {
+			let chunkData = await this.messageInterface(`/silent-command remote.call("remoteMap", "exportChunk", ${data.x}, ${data.y})`)
+			console.log(`Sending chunk data: ${chunkData}`)
+			callback(chunkData);
+		})
 		
-		// set up websocket communication and handle requests from web interface users (which are going through master)
+		// detect new entities created/deleted on the server and update the master
-		// socket should be a global
+		setInterval(async ()=>{
-		this.socket = ioClient("http://"+this.config.masterIP+":"+this.config.masterPort);
+			let data = await this.messageInterface(`/silent-command remote.call("remoteMap", "exportTiles")`)
-		this.socket.on("hello", data => {
+			if(data.length > 1) this.socket.emit("updateEntities", data)
-			this.socket.emit("registerSlaveMapper", {instanceID: this.config.unique});
+		}, 2000)
-			setInterval(()=>{
+	}
-				this.socket.emit("heartbeat"); // send our heartbeat to prevent being assumed dead
+	getInstanceName(instanceID){
-			},10000);
+		return new Promise((resolve, reject) => {
-		});
+			let instance = this.instances[instanceID];
-		this.socket.on("getChunk", req => {
+			if(!instance){
-			this.messageInterface("getChunk is not supported!");
+				needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => {
-			/*this.chunkMap.getChunk(req.x, req.y).then(chunk => {
+					if(err || response.statusCode != 200) {
-				chunk.requesterID = req.requesterID;
+						console.log("Unable to get JSON master/api/slaves, master might be unaccessible");
-				this.socket.emit("sendChunk", chunk);
+					} else if (response && response.body) {	
-			});*/
+						if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();}
-		});
+							try {
-		this.socket.on("getEntity", req => {
+								for (let index in response.body)
-			// check for valid coordinates
+									this.instances[index] = response.body[index].instanceName;
-			if(req.x !== undefined && !isNaN(Number(req.x)) && req.y !== undefined && !isNaN(Number(req.y))){
+							} catch (e){
-				this.chunkMap.getEntity(req.x, req.y).then(entities => {
+								console.log(e);
-					if(entities && entities.length > 0){
+								return null;
-						entities.forEach(entity => {
+							}
-							this.socket.emit("sendEntity", entity);
+						instance = this.instances[instanceID] 							
-						});
+						if (!instance) instance = instanceID;  //somehow the master doesn't know the instance	
+						resolve(instance);
 					}
 				});
 			} else {
-				this.messageInterface("socket.getEntity triggered called with invalid parameters");
+				resolve(instance);
 			}
 		});
-		// both placing and deleting entities via the web interface
-		this.socket.on("placeEntity", req => {
-			// game.player.surface.create_entity{name="small-ship-wreck", position={game.player.position.x-2, game.player.position.y+1}, direction=..., force=...}
-			this.messageInterface("placing "+JSON.stringify(req));
-			if(req.name && typeof req.name == "string" && req.position && !isNaN(Number(req.position.x)) && !isNaN(Number(req.position.y))){
-				if(req.name == "deleted"){ // {{'+req.position.x+','+req.position.y+'},{'+req.position.x+','+req.position.y+'}}
-					// delete entity from game world using LUA command
-					this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+1)+','+(req.position.y+1)+'}}) for i, entity in pairs(toDelete) do entity.die() end');
-					// this position is now empty, delete whatever was there from DB
-					this.deleteEntity(req.position.x, req.position.y);
-				} else {
-					let command = '/silent-command game.surfaces[1].create_entity{name="'+req.name+'", position={'+Number(req.position.x)+', '+Number(req.position.y)+'}, force="neutral"'
-					if(req.direction) command += ', direction='+req.direction;
-					command += '}';
-					this.messageInterface(command); // execute command
-					
-					// add entity to the chunkStore database because create_entity does not trigger any events
-					let name = req.name;
-					let entity = {name};
-					if(typeof req.direction == "string"){
-						entity.rot = req.direction;
-					}
-					this.addEntity(req.position.x, req.position.y, entity);
-				}
-			}
-		});
-		
-		this.messageInterface("/silent-command game.print('"+moduleConfig.name+" version "+moduleConfig.version+" enabled')");
 	}
-	addEntity(x,y,entity){ // add entity to database
+}
-		this.chunkMap.setEntity(Number(x), Number(y), entity).then(data => {
-			this.messageInterface("Added "+entity.name+" to data "+Number(x)+", "+Number(y));
-		}).catch((err)=>this.messageInterface(err));
-	}
-	deleteEntity(x,y){ // remove entity from database
-		// this position is now empty, delete whatever was there from DB
-		this.chunkMap.setEntity(x, y, "delete this entity").then(() => {
-			this.messageInterface("Deleted entity at "+x+", "+y);
-		}).catch((err)=>this.messageInterface(err));
-	}
-	scriptOutput(data){
-		if(data){
-			this.messageInterface(data);
-			// express-transport-belt,-35.5,-14.5,[...]
-			data = data.split(",");
-			if(data && data[0] == undefined){
-				this.messageInterface("empty: ");
-				this.messageInterface(data);
-				return
-			}
-			let name = data[0];
-			let xPos = data[1];
-			let yPos = data[2];
-			if(xPos && yPos && name){
-				if(name == "deleted"){
-					// this position is now empty, delete whatever was there from DB
-					this.deleteEntity(xPos, yPos);
-				} else {
-					let entity = {name};
-					for(let i = 3; i < data.length; i++){
-						if(data[i].includes("=")){
-							let kv = data[i].split("=");
-							entity[kv[0]] = kv[1];
-						}
-					}
-					// save this entity in the database
-					this.addEntity(xPos, yPos, entity);
-					/* this.chunkMap.setEntity(xPos, yPos, entity).then(data => {
-						this.messageInterface("Added "+name+" to data "+xPos+", "+yPos);
-					}).catch((err)=>this.messageInterface(err)); */
-				}
-			}
-			
-		}
-	}
-}
-module.exports = remoteMap;

sharedPlugins/remoteMap/info.json

@@ -2,7 +2,7 @@
 {
 "name": "remoteMap",
 "title": "Clusterio remote map",
-"version": "2.2.0",
+"version": "3.0.0",
 "date": "18-11-2017",
 "author": "Danielv123",
 "dependencies": ["base >= 0.15"],

sharedPlugins/remoteMap/masterPlugin.js

@@ -0,0 +1,189 @@
+const fs = require("fs-extra");
+const path = require("path");
+const sanitizer = require('sanitizer');
+
+const editorSockets = [];
+
+const blacklistedEntities = [
+	"fish",
+	"player",
+]
+
+class masterPlugin {
+	constructor({ config, pluginConfig, pluginPath, socketio, express }) {
+		this.config = config;
+		this.pluginConfig = pluginConfig;
+		this.pluginPath = pluginPath;
+		this.io = socketio;
+		this.app = express;
+		
+		this.clients = {};
+		this.slaves = {};
+		
+		// initialize token auth module
+		this.authenticate = require("./../../lib/authenticate.js")(config);
+
+		// expose UI elements embedded in the master
+		//this.ui = require("./js/ui.js").ui;
+
+		// handle websocket connections
+		this.io.on("connection", socket => {
+			let instanceID = "unknown";
+			socket.on("registerMapServer", data => {
+				console.log("Registered mapServer socket")
+				if (data.instanceID && !isNaN(Number(data.instanceID))) {
+					instanceID = data.instanceID;
+					this.slaves[instanceID] = socket;
+				}
+				socket.on("updateEntities", data => {
+					console.log(data)
+					if(data){
+						// transform from weirdly deliminated string to entity object
+						let entities = data
+						.split("\n")
+						.filter(str => str.length > 0)
+						.map(ent => ent.replace(/[-]/g, "_"))											// replace - with _
+						.map(str => str
+							.split(",")
+							.filter(x => x.length > 0)
+							.map(prop => ({
+								k: prop.split("=")[0],
+								v: prop.split("=")[1]})
+							)
+						)
+						.map(arr => {
+							let obj = {}
+							arr.forEach(property => obj[property.k] = property.v)
+							return obj
+						})
+						console.log(entities)
+						editorSockets.forEach(sock => sock.emit("updateEntity", {entities}))
+					}
+				})
+			});
+			socket.on("registerMapEditor", () => {
+				console.log("Registered mapEditor socket")
+				editorSockets.push(socket);
+				
+				setTimeout(()=>socket.emit("createEntity", {entity:{
+					name:"fast_inserter",
+					position: {x:30,y:30},
+					direction: 2,
+				}}),10000)
+				
+				socket.on("createEntity", data => {
+					console.log(`Creating entity: ${JSON.stringify(data)}`)
+					this.slaves[Object.keys(this.slaves)[0]].emit("createEntity", data.entity)
+				})
+				socket.on("deleteEntity", data => {
+					console.log(`Deleting entity: ${JSON.stringify(data)}`)
+					this.slaves[Object.keys(this.slaves)[0]].emit("deleteEntity", data.entity)
+				})
+				socket.on("getChunk", (data, callback) => {
+					console.log(`Getting chunk x:${data.x} y:${data.y} from slave`)
+					this.slaves[Object.keys(this.slaves)[0]].emit("getChunk", data, resp => {
+						// parse response
+						// |name:transport-belt,direction:2,x:99,y:1231|name:assembly-machine,direction:6,x:939,y:88
+						console.log(resp)
+						callback(resp
+							.split("|")																	// split into entities
+							.map(str => str.replace("\n",""))											// remove newlines that happen at the end of the last entity (maybe more as well???)
+							.filter(str => str.length > 0)												// remove blank entities
+							.filter(str => 
+							// !str.includes("tree") &&													// don't send entities that can't be rendered anyways
+							!str.includes("dead") &&
+							// !str.includes("rock") &&
+							!str.includes("crude-oil") &&
+							!str.includes("remnants") &&
+							!str.includes("cliff") &&
+							!blacklistedEntities.includes(str) &&
+							!str.includes("ore"))
+							.map(ent => ent.split(",")													// split entity into properties
+								.map(prop => [prop.split(":")[0], prop.split(":")[1]])					// split properties into KV pairs
+							)
+							.map(entity => {															// turn entities into objects
+								let entityObj = {}
+								entity.forEach(property => entityObj[property[0]] = property[1])
+								return entityObj
+							})
+							.map(entity => {
+								entity.name = entity.name.replace(/[-]/g, "_")							// replace - with _ in entity names as thats what factorio.blueprint-editor preffers
+								return entity
+							})
+							.filter(entity => Object.keys(entity).length > 1)
+							.filter(entity => !blacklistedEntities.includes(entity.name))
+							/*.map(entity => {
+								if(entity.name.includes("tree")) entity.name = "tree"
+								return entity
+							})*/
+						)
+					})
+				})
+				
+				socket.on("disconnect", function () {
+					let i = editorSockets.indexOf(socket);
+					console.log("editorSocket " + (i + 1) + " disconnected, " + (editorSockets.length - 1) + " left");
+					editorSockets.splice(i, 1);
+				});
+			});
+			socket.on("gameChat", async data => {
+				let chatLine = data.data.replace(/(\r\n\t|\n|\r\t)/gm, "").replace("\r", "");
+				//if (typeof chatLine == "string") this.handleChatLine(chatLine, instanceID);
+			});
+		});
+
+	}
+	async onExit() {
+		return;
+	}
+	async onLoadFinish({ plugins }) {
+		this.masterPlugins = plugins;
+	}
+	parseData(data, sharedData = {}) {
+		let parsedData = [];
+		data = data.split("|");
+		data.forEach(player => {
+			if (player) {
+				let playerData = {};
+				player = player.split(`~`);
+				player.forEach(kv => {
+					kv = kv.split(":");
+					if (kv[1] === undefined || kv[0] === undefined) {
+						console.log(new Error(`Something is wrong! Key:${kv[0]} Value:${kv[1]}`));
+					} else {
+						playerData[kv[0]] = kv[1].trim();
+					}
+				});
+				for (let k in sharedData) {
+					playerData[k] = sharedData[k];
+				}
+				parsedData.push(playerData);
+			}
+		});
+		return parsedData;
+	}
+}
+module.exports = masterPlugin;
+
+function getDatabaseSync(path) {
+	let db;
+	try {
+		db = JSON.parse(fs.readFileSync(path, "utf8"));
+	} catch (e) {
+		db = {};
+	}
+	return db;
+}
+async function saveDatabase(path, database) {
+	if (!path) {
+		throw new Error("No path provided!");
+	} else if (!database) {
+		throw new Error("No database provided!");
+	} else {
+		try {
+			await fs.writeFile(path, JSON.stringify(database, null, 4));
+		} catch (e) {
+			throw new Error("Unable to write to database! " + path);
+		}
+	}
+}

sharedPlugins/remoteMap/readme.md

@@ -0,0 +1,15 @@
+# Remote Map
+
+A clusterio plugin for remotely exploring and editing the factorio world
+
+### Setup
+
+Make a database for storing entities
+
+    docker run --name some-mariadb -e MYSQL_ROOT_PASSWORD=clusterio-remote-map --restart=unless-stopped -d mariadb:latest
+
+### Dev setup
+
+Make a database container and expose the ports
+
+    docker run -p 3306:3306 --name some-mariadb -e MYSQL_ROOT_PASSWORD=clusterio-remote-map -d mariadb:latest

sharedPlugins/scenario_loader/index.js

@@ -42,25 +42,8 @@ module.exports = class remoteCommands {
 								data,
 							});
 						}
-						let fileMap = await loadFiles("", scenPath);
-						let files = {};
-						fileMap.forEach(map => {
-							files[map.name] = map.data;
-							console.log(map.name);
-						});
-						
-						let fileImportString = `\{`;
-						for(let k in files){
-							let name = k.replace(/\\/g, '/');
-							name = name.replace(/ /g, '');
-							// make sure *not* to include control.lua as it is provided as a seperate argument
-							if(name != "control") fileImportString += `["${name}"] = '${files[k]}', `;
-						}
-						fileImportString += `\}`;
-						if(files.control) var returnValue = await messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
-						if(returnValue) messageInterface(returnValue);
-						messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
 					}
+					return returnFiles;
 				}
 				let fileMap = await loadFiles("", scenPath);
 				let files = {};
@@ -120,4 +103,14 @@ class AsyncArray extends Array {
 		super(arr)
 		this.data = arr; // In place of Array subclassing
 	}
+	async filterAsync(predicate) {
+		// Take a copy of the array, it might mutate by the time we've finished
+		const data = Array.from(this.data);
+		// Transform all the elements into an array of promises using the predicate
+		// as the promise
+		const result = await Promise.all(data.map((element, index) => predicate(element, index, data)));
+		return data.filter((element, index) => {
+			return result[index];
+		});
+	}
 }