Refactor database and item retreival

Change the ugly global slaves object to be a member of the db object and
implement the items database as a separate integrety verifying class in
lib/database.

lib/database.js

@@ -0,0 +1,231 @@
+/**
+ * Implementation of master databases
+ *
+ * @module
+ * @author Hornwitser
+ */
+
+"use strict";
+const fs = require("fs-extra");
+
+
+/**
+ * Converts a Map with only string keys to a JavaScript object.
+ *
+ * @param {Map} map - the Map to convert to an object.
+ * @return {object} - Object with the mapping's key-values as properties.
+ * @throws {Error} if there are non-string keys in map.
+ */
+function mapToObject(map) {
+	let obj = {};
+	for (let [key, value] of map) {
+		if (typeof key !== 'string') {
+			throw new Error(
+				`Expected all keys to be string but got ${typeof key}`
+			);
+		}
+
+		obj[key] = value;
+	}
+	return obj;
+}
+
+/**
+ * Load JSON file as a Map
+ *
+ * Loads the JSON file specified by path an converts it into a JavaScript Map.
+ * Returns an empty Map if the file does not exist.
+ *
+ * @param {string} filePath - The path to the JSON file to load.
+ * @throws {Error} If JSON file did not contain an object.
+ * @throws {Error} If an error occured reading the file.
+ */
+async function loadJsonAsMap(filePath) {
+	try {
+		let content = await fs.readFile(filePath);
+		let parsed = JSON.parse(content);
+
+		if (parsed instanceof Array) {
+			throw new Error(`Expected object but got array`);
+		}
+
+		if (parsed === null) {
+			throw new Error(`Expected object but got null`);
+		}
+
+		if (typeof parsed !== "object") {
+			throw new Error(`Expected object but got ${typeof parsed}`);
+		}
+
+		return new Map(Object.entries(parsed));
+
+	} catch (err) {
+		if (err.code == "ENOENT") {
+			// File does not exist, return empty map
+			return new Map();
+
+		} else {
+			throw err;
+		}
+	}
+}
+
+/**
+ * Save a Map as a JSON file
+ *
+ * Save the content of a key-value Map with only string keys as a JSON file.
+ * If the directory the file is to be saved into doesn't exist it will be
+ * created.
+ *
+ * @param {string} filePath - The path to the JSON file that will be saved.
+ * @param {Map} map - Map with only string keys to save.
+ * @throws {Error} if there are non-string keys in map.
+ * @throws {Error} if an error occured writing to the file.
+ */
+async function saveMapAsJson(filePath, map) {
+	let obj = mapToObject(map);
+	await fs.outputFile(filePath, JSON.stringify(obj, null, 4));
+}
+
+
+function checkName(name) {
+	if (typeof name !== "string") {
+		throw new Error("name must be a string");
+	}
+}
+
+function checkCount(count) {
+	if (typeof count !== "number" || isNaN(count)) {
+		throw new Error("count must be a number");
+	}
+}
+
+
+/**
+ * Item database
+ *
+ * Stores counts of items for the master server.  Items that have not been
+ * stored in the database are treated as having a stored count of zero.  When
+ * serialized the database discards entries with a zero counts, and when
+ * deserialized the content is verified.
+ */
+class ItemDatabase {
+
+	/**
+	 * Create a new item database
+	 *
+	 * @param {object|undefined} serialized - An object from a previous call to
+	 *     {@link module:lib/database~ItemDatabase#serialize} to restore the
+	 *     database from.  An empty database will be created if this parameter
+	 *     is left undefined.
+	 */
+	constructor(serialized) {
+		this._items = new Map();
+
+		// Verify the content of the serialized database
+		if (serialized !== undefined) {
+			for (let [name, count] of Object.entries(serialized)) {
+				checkName(name);
+				checkCount(count);
+
+				this._items.set(name, count);
+			}
+		}
+	}
+
+	/**
+	 * Serialize item database
+	 *
+	 * Serialize the item database into a plain JavaScript object that can be
+	 * turned inta a string with JSON.stringify().
+	 *
+	 * @returns {object} Serialized representation of the database
+	 */
+	serialize() {
+		let obj = {};
+		for (let [name, count] of this._items) {
+			if (count !== 0) {
+				obj[name] = count;
+			}
+		}
+		return obj;
+	}
+
+	/**
+	 * Approximate size of the items database.
+	 *
+	 * Does not account for entries being zero in the item database.
+	 */
+	get size() {
+		return this._items.size;
+	}
+
+
+	/**
+	 * Get the count of an item stored in the database
+	 *
+	 * Returns the count of a given item that has been stored in to the
+	 * database.  If the item has not been previously stored into the database
+	 * then 0 is returned.
+	 *
+	 * @param {string} name - The name of the item to get the count of.
+	 * @returns {number} The count of the item stored.
+	 */
+	getItemCount(name) {
+		checkName(name);
+
+		if (!this._items.has(name)) {
+			return 0
+		}
+
+		return this._items.get(name);
+	}
+
+	/**
+	 * Adds count of an item to the database
+	 *
+	 * Add count copies of the item specified by name to the item database.
+	 *
+	 * @param {string} name - The name of the item to add.
+	 * @param {number} count - The count of item to remove.
+	 */
+	addItem(name, count) {
+		checkName(name);
+		checkCount(count);
+
+		if (this._items.has(name)) {
+			count += this._items.get(name);
+		}
+
+		this._items.set(name, count);
+	};
+
+	/**
+	 * Removes count of an item from the database.
+	 *
+	 * Remove count copies of the item specified by name from the item database
+	 * If count is greater than the currently stored count the stored item
+	 * count will become negative.
+	 *
+	 * @param {string} name - The name of the item to remove.
+	 * @param {number} count - The count of items to remove.
+	 */
+	removeItem(name, count) {
+		checkName(name);
+		checkCount(count);
+
+		count = -count;
+		if (this._items.has(name)) {
+			count += this._items.get(name);
+		}
+
+		this._items.set(name, count);
+	};
+}
+
+module.exports = {
+	mapToObject,
+	loadJsonAsMap,
+	saveMapAsJson,
+	ItemDatabase,
+}

lib/database.spec.js

@@ -0,0 +1,212 @@
+const assert = require("assert").strict;
+const fs = require("fs-extra");
+
+const database = require("lib/database");
+
+describe("lib/database", function() {
+	describe("mapToObject()", function() {
+		it("should throw on non-string key", function() {
+			assert.throws(
+				() => database.mapToObject(new Map([[1, 1]])),
+				new Error("Expected all keys to be string but got number")
+			);
+			assert.throws(
+				() => database.mapToObject(new Map([[Symbol(), 1]])),
+				new Error("Expected all keys to be string but got symbol")
+			);
+		});
+
+		it("should throw convert a map to an object", function() {
+			assert.deepEqual(
+				database.mapToObject(new Map([['a', 1], ['b', true]])),
+				{'a': 1, 'b': true}
+			);
+		});
+	});
+
+	describe("loadJsonAsMap()", function() {
+		let badTypes = ['null', 'array', 'number', 'string', 'boolean'];
+
+		for (let type of badTypes) {
+			it(`should reject on ${type} JSON`, async function() {
+				await assert.rejects(
+					database.loadJsonAsMap(`test/json/${type}.json`),
+					new Error(`Expected object but got ${type}`)
+				);
+			});
+		}
+
+		it("should work on empty object JSON", async function() {
+			assert.deepEqual(
+				await database.loadJsonAsMap('test/json/object.json'),
+				new Map()
+			);
+		});
+
+		it("should work on object JSON", async function() {
+			assert.deepEqual(
+				await database.loadJsonAsMap('test/json/load_map.json'),
+				new Map([['a', 1], ['b', true]])
+			);
+		});
+
+		it("should give an empty Map for non-existant file", async function() {
+			assert.deepEqual(
+				await database.loadJsonAsMap('test/json/does-not-exist'),
+				new Map()
+			);
+		});
+	});
+
+	describe("saveMapAsJson()", function() {
+		it("should save a mapping as JSON", async function() {
+			let testFile = 'test/json/save_map.json';
+			async function deleteTestFile() {
+				try {
+					await fs.unlink(testFile);
+				} catch (err) {
+					/* istanbul ignore if */
+					if (err.code !== 'ENOENT') {
+						throw err;
+					}
+				}
+			}
+
+			await deleteTestFile();
+			await database.saveMapAsJson(
+				testFile, new Map([['c', {}], ['d', "foo"]])
+			);
+
+			assert.equal(
+				await fs.readFile(testFile, {encoding: 'utf-8'}),
+				'{\n    "c": {},\n    "d": "foo"\n}'
+			);
+			await deleteTestFile();
+		});
+	});
+
+	describe("class ItemDatabase", function() {
+		describe("constructor()", function() {
+			it("should create an empty database with no args", function() {
+				let items = new database.ItemDatabase();
+				assert.deepEqual(items._items, new Map());
+			});
+
+			it("should restore the passed serialized database", function() {
+				let items = new database.ItemDatabase({'a': 1, 'b': 2});
+				assert.deepEqual(items._items, new Map([['a', 1], ['b', 2]]));
+			});
+
+			it("should throw on invalid serialized database", function() {
+				assert.throws(
+					() => new database.ItemDatabase({'a': NaN}),
+					new Error("count must be a number")
+				);
+
+				assert.throws(
+					() => new database.ItemDatabase({'a': 'a'}),
+					new Error("count must be a number")
+				);
+			});
+		});
+
+		describe(".serialize()", function() {
+			it("should return a serialized database", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				assert.deepEqual(items.serialize(), {'a': 10});
+			});
+
+			it("should remove zero count entries", function() {
+				let items = new database.ItemDatabase({'a': 0});
+				assert.deepEqual(items.serialize(), {});
+			});
+		});
+
+		describe(".size", function() {
+			it("should give an approximate size of the database", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				assert.equal(items.size, 1);
+			});
+		});
+
+		describe(".getItemCount()", function() {
+			it("should return the count of the given item", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				assert.equal(items.getItemCount('a'), 10);
+			});
+
+			it("should return zero if item does not exist", function() {
+				let items = new database.ItemDatabase();
+				assert.equal(items.getItemCount('b'), 0);
+			});
+
+			it("should throw on invalid name", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.getItemCount(2),
+					new Error("name must be a string")
+				);
+			});
+		});
+
+		describe(".addItem()", function() {
+			it("should add a new item", function() {
+				let items = new database.ItemDatabase();
+				items.addItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', 10]]));
+			})
+
+			it("should add an existing item", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				items.addItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', 20]]));
+			})
+
+			it("should throw on invalid name", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.addItem(2, 10),
+					new Error("name must be a string")
+				);
+			});
+
+			it("should throw on invalid count", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.addItem('a', NaN),
+					new Error("count must be a number")
+				);
+			});
+		});
+
+		describe(".removeItem()", function() {
+			it("should remove an existing item", function() {
+				let items = new database.ItemDatabase({'a': 20});
+				items.removeItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', 10]]));
+			})
+
+			it("should turn a non-existing item negative", function() {
+				let items = new database.ItemDatabase();
+				items.removeItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', -10]]));
+			})
+
+			it("should throw on invalid name", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.removeItem(2, 10),
+					new Error("name must be a string")
+				);
+			});
+
+			it("should throw on invalid count", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.removeItem('a', 'b'),
+					new Error("count must be a number")
+				);
+			});
+		});
+	});
+});

master.js

@@ -34,6 +34,7 @@ const config = require(args.config || './config');
 // homebrew modules
 const generateSSLcert = require("lib/generateSSLcert");
 const getFactorioLocale = require("lib/getFactorioLocale");
+const database = require("lib/database");
 
 // homemade express middleware for token auth
 const authenticate = require("lib/authenticate");
@@ -67,7 +68,7 @@ app.use(function(req, res, next){
 	res.locals.res = res;
 	res.locals.req = req;
 	res.locals.masterPlugins = masterPlugins;
-	res.locals.slaves = slaves;
+	res.locals.slaves = db.slaves;
 	res.locals.moment = moment;
 	next();
 });
@@ -146,27 +147,25 @@ app.get('/metrics', (req, res) => {
 });
 
 function registerMoreMetrics(){
-	for(let instanceID in slaves){
+	for (let [instanceID, slave] of db.slaves) {
 		// playercount
 		try{
-			prometheusPlayerCountGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].playerCount) || 0);
+			prometheusPlayerCountGauge.labels(instanceID, slave.instanceName).set(Number(slave.playerCount) || 0);
 		}catch(e){}
 		// UPS
 		try{
-			prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.UPS) || 60);
-			if(slaves[instanceID].meta.tick && typeof slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.tick) || 0);
+			prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.UPS) || 60);
+			if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0);
 		}catch(e){}
 	}
 	// inventory
-	for(let key in db.items){
-		if(typeof db.items[key] == "number" || typeof db.items[key] == "string"){
-			prometheusMasterInventoryGauge.labels(key).set(Number(db.items[key]) || 0);
-		}
+	for (let [key, count] of db.items._items) {
+		prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
 	}
 	// Slave count
 	let numberOfActiveSlaves = 0;
-	for(let instance in slaves){
-		if(Date.now() - Number(slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
+	for(let slave of db.slaves.values()){
+		if(Date.now() - Number(slave.time) < 1000 * 30) numberOfActiveSlaves++;
 	}
 	prometheusConnectedInstancesCounter.set(numberOfActiveSlaves);
 }
@@ -174,49 +173,28 @@ function registerMoreMetrics(){
 // set up database
 const db = {};
 
-(function(){
-	try{
-		let x = fs.statSync(path.resolve(config.databaseDirectory, "slaves.json"));
-		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
-		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json")));
-	} catch (e){
-		slaves = {};
-	}
-	try{
-		x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
-		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
-		db.items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")));
-	} catch (e){
-		db.items = {};
-	}
-})();
+async function loadDatabase(databaseDirectory) {
+	let slavesPath = path.resolve(databaseDirectory, "slaves.json");
+	console.log(`Loading ${slavesPath}`);
+	db.slaves = await database.loadJsonAsMap(slavesPath);
+
+	let itemsPath = path.resolve(databaseDirectory, "items.json");
+	console.log(`Loading ${itemsPath}`);
+	try {
+		let content = await fs.readFile(itemsPath);
+		db.items = new database.ItemDatabase(JSON.parse(content));
+
+	} catch (err) {
+		if (err.code === "ENOENT") {
+			console.log("Creating new item database");
+			db.items = new database.ItemDatabase();
 
-db.items.addItem = function(object) {
-	if(object.name == "addItem" || object.name == "removeItem") {
-		console.error("Fuck you, that would screw everything up if you named your item that.");
-		return false;
-	} else {
-		if(this[object.name] && Number(this[object.name]) != NaN){
-			this[object.name] = Number(this[object.name]) + Number(object.count);
 		} else {
-			this[object.name] = Number(object.count);
+			throw err;
 		}
-		return true;
 	}
-};
-db.items.removeItem = function(object) {
-	if(object.name == "addItem" || object.name == "removeItem") {
-		console.error("Fuck you, that would screw everything up if you named your item that.");
-		return false;
-	} else {
-		if(this[object.name] && Number(this[object.name]) != NaN){
-			this[object.name] = Number(this[object.name]) - Number(object.count);
-		} else {
-			this[object.name] = 0;
-		}
-		return true;
-	}
-};
+}
+
 
 // store slaves and inventory in a .json full of JSON data
 async function shutdown() {
@@ -224,17 +202,21 @@ async function shutdown() {
 	let exitStartTime = Date.now();
 	try {
 		// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-		if(slaves && Object.keys(slaves).length < 2000000){
-			console.log("saving to slaves.json");
-			fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
-		} else if(slaves) {
-			console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
+		if (db.slaves && db.slaves.size < 2000000) {
+			let file = path.resolve(config.databaseDirectory, "slaves.json");
+			console.log(`writing ${file}`);
+			await database.saveMapAsJson(file, db.slaves);
+		} else if (db.slaves) {
+			console.error(`Slave database too large, not saving (${db.slaves.size}`);
 		}
-		if(db.items && Object.keys(db.items).length < 50000){
-			console.log("saving to items.json");
-			fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
-		} else if(slaves) {
-			console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
+
+		if (db.items && db.items.size < 50000) {
+			let file = path.resolve(config.databaseDirectory, "items.json");
+			console.log(`writing ${file}`);
+			let content = JSON.stringify(db.items.serialize());
+			await fs.outputFile(file, content);
+		} else if (db.items) {
+			console.error(`Item database too large, not saving (${db.items.size})`);
 		}
 		for(let i in masterPlugins){
 			let plugin = masterPlugins[i];
@@ -266,15 +248,16 @@ app.post("/api/getID", authenticate.middleware, function(req,res) {
 	// time us a unix timestamp we can use to check for how long the server has been unresponsive
 	// we should save that somewhere and give appropriate response
 	if(req.body){
-		if(!slaves[req.body.unique]){
-			slaves[req.body.unique] = {};
+		if (!db.slaves.has(req.body.unique)) {
+			db.slaves.set(req.body.unique, {});
 		}
-		for(k in req.body){
+		let slave = db.slaves.get(req.body.unique);
+		for(let k in req.body){
 			if(k != "meta" && req.body.hasOwnProperty(k)){
-				slaves[req.body.unique][k] = req.body[k];
+				slave[k] = req.body[k];
 			}
 		}
-		// console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName);
+		// console.log("Slave registered: " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP + " with name " + slave.instanceName);
 	}
 });
 /**
@@ -289,16 +272,17 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 	// {instanceID, pass, meta:{x,y,z}}
 
 	if(req.body && req.body.instanceID && req.body.meta){
-		if(slaves[req.body.instanceID]){
-			if(!slaves[req.body.instanceID].meta) {
-				slaves[req.body.instanceID].meta = {};
+		if (db.slaves.has(req.body.instanceID)) {
+			let slave = db.slaves.get(req);
+			if(!slave.meta) {
+				slave.meta = {};
 			}
-			slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
+			slave.meta = deepmerge(slave.meta, req.body.meta, {clone:true});
 			let metaPortion = JSON.stringify(req.body.meta);
 			if(metaPortion.length > 50) {
 				metaPortion = metaPortion.substring(0,20) + "...";
 			}
-			// console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion);
+			// console.log("Updating slave "+slave.instanceName+": " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP, metaPortion);
 			res.sendStatus(200);
 		} else {
 			res.send("ERROR: Invalid instanceID or password")
@@ -316,12 +300,13 @@ app.post("/api/getSlaveMeta", function (req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	console.log("body", req.body);
 	if(req.body && req.body.instanceID){
-		if (slaves[req.body.instanceID]) {
+		if (db.slaves.has(req.body.instanceID)) {
+			let slave = db.slaves.get(req.body.instanceID);
 			console.log("returning meta for ", req.body.instanceID);
-			if(!slaves[req.body.instanceID].meta) {
-				slaves[req.body.instanceID].meta = {};
+			if(!slave.meta) {
+				slave.meta = {};
 			}
-			res.send(JSON.stringify(slaves[req.body.instanceID].meta))
+			res.send(JSON.stringify(slave.meta))
 		} else {
 			res.status(404);
 			res.send('{"status": 404, "info": "Slave not registered"}')
@@ -387,7 +372,7 @@ let slaveCache = {
 };
 function getSlaves() {
 	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000) {
-		let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
+		let copyOfSlaves = JSON.parse(JSON.stringify(database.mapToObject(db.slaves)));
 		slaveCache.cache = copyOfSlaves;
 		slaveCache.timestamp = Date.now();
 	}
@@ -460,11 +445,12 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	}
 	if(!x.instanceID) {
 		x.instanceID = "unknown";
-		Object.keys(slaves).forEach(instanceID => {
-			if(slaves[instanceID].instanceName == req.body.instanceName) {
+		for (let [instanceID, slave] of db.slaves) {
+			if(slave.instanceName == req.body.instanceName) {
 				x.instanceID = instanceID;
+				break;
 			}
-		});
+		}
 	}
 	if(x.instanceID
 	&& x.instanceName
@@ -490,10 +476,7 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 		});
 		prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
 		// save items we get
-		db.items.addItem({
-			name:req.body.name,
-			count:req.body.count
-		});
+		db.items.addItem(req.body.name, req.body.count);
 
 		// Attempt confirming
 		res.send("success");
@@ -502,12 +485,8 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	}
 });
 const routes_api_remove = require("./routes/api/remove.js");
-var neuralDole = null;
+let neuralDole;
 
-if(config.useNeuralNetDoleDivider)//Only initialize neural network when it's enabled, otherwise it might override gauge
-	neuralDole=new routes_api_remove.neuralDole({
-		items: db.items, gaugePrefix: prometheusPrefix
-	});
 /**
 POST endpoint to remove items from DB when client orders items.
 
@@ -531,22 +510,22 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	if(!object.instanceName) {
 		object.instanceName = "unknown";
 	}
-	if(slaves[object.instanceID]) object.instanceName = slaves[object.instanceID].instanceName;
-	let item = db.items[object.name];
-		// console.dir(doc);
-	if((item === undefined)
-	||((
-		config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
-		&& object.name != "electricity")
-	){
+	if (db.slaves.has(object.instanceID)) {
+		object.instanceName = db.slaves.get(object.instanceID).instanceName;
+	}
+	let itemCount = db.items.getItemCount(object.name);
+	if (
+		(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
+		&& object.name != "electricity"
+	) {
 		if(config.logItemTransfers){
 			console.log('failure could not find ' + object.name);
 		}
 		res.send({name:object.name, count:0});
 	} else if(config.disableFairItemDistribution){
 		// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
-		let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item));
-		db.items.removeItem({count: numberToRemove, name: object.name});
+		let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
+		db.items.removeItem(object.name, numberToRemove);
 		res.send({count: numberToRemove, name: object.name});
 
 		// track statistics and do graphing things
@@ -577,7 +556,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	} else {
 		// Use dole division. Makes it really slow to drain out the last little bit.
 		routes_api_remove.doleDivider({
-			item,
+			itemCount,
 			object,
 			db,
 			sentItemStatisticsBySlaveID,
@@ -604,10 +583,8 @@ app.get("/api/inventory", function(req, res) {
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// Check it and send it
 	var inventory = [];
-	for(let key in db.items){
-		if(typeof db.items[key] == "number" || typeof db.items[key] == "string"){
-			inventory.push({name:key, count:db.items[key]});
-		}
+	for (let [name, count] of db.items._items) {
+		inventory.push({ name, count });
 	}
 	res.send(JSON.stringify(inventory));
 });
@@ -624,7 +601,7 @@ app.get("/api/inventoryAsObject", function(req, res) {
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// Check it and send it
-	res.send(JSON.stringify(db.items));
+	res.send(JSON.stringify(db.items.serialise()));
 });
 
 /**
@@ -906,8 +883,11 @@ async function startServer() {
 	fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
 		expiresIn: 86400*365 // expires in 1 year
 	}));
-	setInterval(()=>{
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
+	setInterval(async function() {
+		let file = path.resolve(config.databaseDirectory, "items.json");
+		console.log(`autosaving ${file}`);
+		let content = JSON.stringify(db.items.serialize());
+		await fs.outputFile(file, content);
 	},config.autosaveInterval || 60000);
 	process.on('SIGINT', shutdown); // ctrl + c
 	process.on('SIGHUP', shutdown); // terminal closed
@@ -921,8 +901,16 @@ async function startServer() {
 	// mod downloads
 	app.use(express.static(masterModFolder));
 
+	await loadDatabase(config.databaseDirectory);
 	authenticate.setAuthSecret(config.masterAuthSecret);
 
+	// Only initialize neural network when it's enabled, otherwise it might override gauge
+	if(config.useNeuralNetDoleDivider) {
+		neuralDole=new routes_api_remove.neuralDole({
+			db, gaugePrefix: prometheusPrefix
+		});
+	}
+
 	// Make sure we're actually going to listen on a port
 	let httpPort = args.masterPort || config.masterPort;
 	let httpsPort = args.sslPort || config.sslPort;
@@ -973,6 +961,9 @@ async function startServer() {
 
 module.exports = {
 	app,
+
+	// For testing only
+	_db: db,
 }
 
 if (module === require.main) {

master.spec.js

@@ -7,9 +7,12 @@ const jwt = require("jsonwebtoken");
 const master = require('./master.js');
 const config = require('./config.json');
 const authenticate = require('lib/authenticate');
+const database = require('lib/database');
 
 
 authenticate.setAuthSecret(config.masterAuthSecret);
+master._db.slaves = new Map();
+master._db.items = new database.ItemDatabase();
 
 const token =
 	jwt.sign( { id: "api" }, config.masterAuthSecret, { expiresIn: 600 });
@@ -25,7 +28,7 @@ async function getValidate(path) {
 	assert(filtered.length === 0, "there are HTML errors on the page, please fix: "+JSON.stringify(validation.messages, null, 4));
 }
 
-describe('Master server endpoint testing', function() {
+describe('Master testing', function() {
 	describe('#GET /api/getFactorioLocale', function() {
 		it('should get the basegame factorio locale', async function() {
 			let res = await get('/api/getFactorioLocale');
@@ -86,19 +89,31 @@ describe('Master server endpoint testing', function() {
 				assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
 			});
 		});
-		it("returns an empty itemStack if you try to request addItem or removeItem", function() {
-			function remove() {
-				return persistentMaster.post("/api/remove").send({
-					name:"addItem",
-					count:10
+		it("works correctly for items named addItem or removeItem", async function() {
+			function add(item) {
+				return persistentMaster.post("/api/place").send({
+					name: item,
+					count: 10
 				}).set(
 					"X-Access-Token", token
 				).expect(200).then(function(res) {
-					assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0");
+					assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
+				})
+			}
+
+			function remove(item) {
+				return persistentMaster.post("/api/remove").send({
+					name: item,
+					count: 10
+				}).set(
+					"X-Access-Token", token
+				).expect(200).then(function(res) {
+					assert.deepEqual(res.body, {"name":item,"count":10});
 				});
 			}
 
-			return Promise.all([remove(), remove()]);
+			await Promise.all([add("addItem"), add("removeItem")]);
+			await Promise.all([remove("addItem"), remove("removeItem")]);
 		});
 		it("returns an empty itemStack if you don't have any of the item you request", function() {
 			return persistentMaster.post("/api/remove").send({

routes/api/remove.js

@@ -19,7 +19,7 @@ class neuralDole {
     }
     
     constructor({
-        items, gaugePrefix
+		db, gaugePrefix
     }){
         // Set some storage variables for the dole divider
         this.prometheusNNDoleGauge = new Prometheus.Gauge({
@@ -27,8 +27,8 @@ class neuralDole {
             help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
             labelNames: ["itemName"],
         });
-        this.items = items
-        this.itemsLastTick = JSON.parse(JSON.stringify(items))
+		this.db = db
+		this.itemsLastTick = new Map(db.items._items)
         this.dole = {}
         this.carry = {}
         this.lastRequest = {}
@@ -36,15 +36,11 @@ class neuralDole {
 		this.debt= {}
 
         setInterval(()=>{
-            for(let name in this.items){
-                // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties.
-                if(typeof this.items[name] !== "number") continue
-
-                let count = this.items[name]
+			for (let [name, count] of this.db.items._items) {
                 let magicData = doleNN.Tick(
                     count,
                     this.dole[name],
-                    this.itemsLastTick[name],
+					this.itemsLastTick.get(name),
                     this.getRequestStats(name,5)
                 )
                 this.stats[name]=this.stats[name] || []
@@ -55,7 +51,7 @@ class neuralDole {
                 // DONE handle magicData[1] for graphing for our users
                 this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
             }
-            this.itemsLastTick = JSON.parse(JSON.stringify(this.items))
+			this.itemsLastTick = new Map(this.db.items._items)
         }, 1000)
     }
     divider({
@@ -67,8 +63,8 @@ class neuralDole {
     }){
         let magicData = doleNN.Dose(
             object.count, // numReq
-            this.items[object.name],
-            this.itemsLastTick[object.name] || 0,
+			this.db.items.getItemCount(object.name),
+			this.itemsLastTick.get(object.name) || 0,
             this.dole[object.name],
             this.carry[object.name+" "+object.instanceID] || 0,
             this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
@@ -87,46 +83,40 @@ class neuralDole {
         this.dole[object.name] = magicData[1]
         this.carry[object.name+" "+object.instanceID] = magicData[2]
 		this.debt[object.name+" "+object.instanceID] = magicData[3]
-        
-        // Remove item from DB and send it
-        if(this.items.removeItem({count: magicData[0], name: object.name})){
-            if(config.logItemTransfers){
-                console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName);
-            }
-            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-            if(sentItemStatistics === undefined){
-                sentItemStatistics = new averagedTimeSeries({
-                    maxEntries: config.itemStats.maxEntries,
-                    entriesPerSecond: config.itemStats.entriesPerSecond,
-                    mergeMode: "add",
-                }, console.log);
-            }
-            sentItemStatistics.add({
-                key:object.name,
-                value:magicData[0],
-            });
-            //console.log(sentItemStatistics.data)
-            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-            prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
 
-            res.send({
-                name: object.name,
-                count: magicData[0],
-            })
-        } else {
-            res.send({
-                name: object.name,
-                count: 0,
-            })
-        }        
+		// Remove item from DB and send it
+		this.db.items.removeItem(object.name, magicData[0]);
+		if(config.logItemTransfers){
+			console.log("removed: " + object.name + " " + magicData[0] + " . " + this.db.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
+		}
+		let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+		if(sentItemStatistics === undefined){
+			sentItemStatistics = new averagedTimeSeries({
+				maxEntries: config.itemStats.maxEntries,
+				entriesPerSecond: config.itemStats.entriesPerSecond,
+				mergeMode: "add",
+			}, console.log);
+		}
+		sentItemStatistics.add({
+			key:object.name,
+			value:magicData[0],
+		});
+		//console.log(sentItemStatistics.data)
+		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+		prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
+
+		res.send({
+			name: object.name,
+			count: magicData[0],
+		})
     }
 }
 
 _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 function doleDivider({
-    item,
+	itemCount,
     object,
-    db,
+	db,
     sentItemStatisticsBySlaveID,
     config,
     prometheusDoleFactorGauge,
@@ -139,34 +129,35 @@ function doleDivider({
     const originalCount = Number(object.count) || 0;
     object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
     object.count = Math.round(object.count);
-    if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
-    
-    // Update existing items if item name already exists
-    if(Number(item) > Number(object.count)) {
+	if (itemCount > 40) {
+		console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`);
+	}
+
+	// Update existing items if item name already exists
+	if (itemCount > object.count) {
         //If successful, increase dole
         _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
-        if(config.logItemTransfers){
-            console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
-        }
-        if(db.items.removeItem({count: object.count, name: object.name})){
-            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-            if(sentItemStatistics === undefined){
-                sentItemStatistics = new averagedTimeSeries({
-                    maxEntries: config.itemStats.maxEntries,
-                    entriesPerSecond: config.itemStats.entriesPerSecond,
-                    mergeMode: "add",
-                }, console.log);
-            }
-            sentItemStatistics.add({
-                key:object.name,
-                value:object.count,
-            });
-            //console.log(sentItemStatistics.data)
-            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-        }
-        
+		if (config.logItemTransfers) {
+			console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
+		}
+		db.items.removeItem(object.name, object.count)
+		let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+		if(sentItemStatistics === undefined){
+			sentItemStatistics = new averagedTimeSeries({
+				maxEntries: config.itemStats.maxEntries,
+				entriesPerSecond: config.itemStats.entriesPerSecond,
+				mergeMode: "add",
+			}, console.log);
+		}
+		sentItemStatistics.add({
+			key:object.name,
+			value:object.count,
+		});
+		//console.log(sentItemStatistics.data)
+		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+
         prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
-        prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
+        prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);
         res.send({count: object.count, name: object.name});
     } else {
         // if we didn't have enough, attempt giving out a smaller amount next time

sharedPlugins/serverManager/static/index.html

@@ -13,10 +13,10 @@
 				html = "";
 				if(req.query
 				&& req.query.instanceID
-				&& slaves[req.query.instanceID]
-				&& slaves[req.query.instanceID].mods
+				&& locals.slaves.get(req.query.instanceID)
+				&& locals.slaves.get(req.query.instanceID).mods
 				&& serverManager.serverSockets[req.query.instanceID]){
-					let slave = slaves[req.query.instanceID];
+					let slave = locals.slaves.get(req.query.instanceID);
 					html += `
 						<div id="mods">
 							${slave.mods.map(mod => mod.name).reduce((acc, modname) => acc + "<li>" + modname + "</li>")}

static/nodes.html

@@ -15,8 +15,7 @@
 			<div id="display">
 				<%
                     let html = ""
-					for(let id in slaves){
-						let slave = slaves[id];
+					for(let [id, slave] of locals.slaves){
 						html += `
 						<div id="${id}" class="slaveBox card mb-3">
                             <div class="card-body">

test/json/array.json

@@ -0,0 +1 @@
+[]

test/json/boolean.json

@@ -0,0 +1 @@
+false

test/json/load_map.json

@@ -0,0 +1 @@
+{"a":1,"b":true}

test/json/null.json

@@ -0,0 +1 @@
+null

test/json/number.json

@@ -0,0 +1 @@
+0

test/json/object.json

@@ -0,0 +1 @@
+{}

test/json/string.json

@@ -0,0 +1 @@
+""