Merge pull request #222 from Hornwitser/config-part-1

Config Rewrite Part 1

.travis.yml

@@ -4,7 +4,6 @@ node_js:
 before_install:
   - curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64
   - tar -xf factorio.tar.gz
-  - cp -n config.json.test config.json
   - npm install -g codecov
 script:
   - npm run-script ci-cover

CHANGELOG.md

@@ -8,3 +8,5 @@ Version 2.0.0
 Breaking Changes
 ----------------
 - Node.js versions below 10 are no longer supported.
+- lib/authenticate no longer requires config to be passed to it.  Breaks
+  playerManager.

client.js

@@ -18,6 +18,7 @@ const objectOps = require("lib/objectOps.js");
 const fileOps = require("lib/fileOps");
 const stringUtils = require("lib/stringUtils.js");
 const configManager = require("lib/manager/configManager.js");
+const pluginManager = require("lib/manager/pluginManager");
 const hashFile = require('lib/hash').hashFile;
 
 console.warn(`
@@ -235,7 +236,6 @@ if (!command || command == "help" || command == "--help") {
 			}
 		} else if(tool == "plugins"){
 			(async function(){try{
-				const pluginManager = require("lib/manager/pluginManager.js")(config);
 				if(action == "list"){
 					await pluginManager.listPlugins();
 				} else if(action == "add" || action == "install" || action == "download"){
@@ -587,7 +587,6 @@ async function instanceManagement(instanceconfig) {
 	setInterval(B=> socket.emit("heartbeat"), 10000);
 
 	// load plugins and execute onLoad event
-	const pluginManager = require("lib/manager/pluginManager.js")(config);
 	let pluginsToLoad = await pluginManager.getPlugins();
 	let plugins = [];
 	

config.json.test

@@ -1,46 +0,0 @@
-{
-	"factorioDirectory": "./factorio",
-	"instanceDirectory": "./instances",
-	"databaseDirectory": "./database",
-
-	"publicIP": "localhost",
-
-	"masterIP": "localhost",
-
-	"masterPort": 8080,
-
-	"sslPort": 443,
-	"sslCert": "database/certificates/cert.crt",
-	"sslPrivKey": "database/certificates/cert.key",
-	"masterAuthToken": "",
-	"masterAuthSecret": "TestSecretDoNotUse",
-
-	"username": "",
-	"token": "",
-	"factorio_version": "0.16.41",
-
-	"verify_user_identity": true,
-	"auto_pause": false,
-	"allow_commands": "admins-only",
-	"game_password": "",
-	"description": "This server runs Clusterio",
-	"visibility": {
-		"public": true,
-		"lan": true
-	},
-
-	"itemStats": {
-		"maxEntries":60,
-		"entriesPerSecond": 0.01666666667
-	},
-	"logItemTransfers": false,
-	"disableFairItemDistribution": true,
-	"useNeuralNetDoleDivider": false,
-	"uploadModsToMaster": true,
-	"msBetweenCommands": 10,
-	"allowRemoteCommandExecution": true,
-	"enableCrossServerShout": true,
-	"mirrorAllChat": false,
-	"autosaveInterval": 600000,
-	"disableImportsOfEverythingExceptElectricity": false
-}

lib/authenticate.js

@@ -1,28 +1,48 @@
 const jwt = require('jsonwebtoken');
 
-// const config = require("./../config");
+let authSecret = null;
+function setAuthSecret(secret) {
+	authSecret = secret;
+}
 
-module.exports = function(config){
-	return {
-		middleware: function (req, res, next) {
-			var token = req.headers['x-access-token'];
-			if(!token) return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path});
-			
-			jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
-				if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path});
-				next();
-			});
-		},
-		check: async function(token){
-			return new Promise((resolve, reject) => {
-				jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
-					if(err){
-						resolve({ok:false, msg:err})
-					} else {
-						resolve({ok:true, msg:"Successfully authenticated"});
-					}
-				});
-			});
-		}
+function middleware(req, res, next) {
+	// This should not happen
+	if (!authSecret) {
+		return res.status(500).send({ auth: false, message: 'Secret not set', endpoint: req.route.path });
 	}
+
+	var token = req.headers['x-access-token'];
+	if (!token) {
+		return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path });
+	}
+
+	jwt.verify(token, authSecret, function(err, decoded) {
+		if (err) {
+			return res.status(401).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path });
+		}
+		next();
+	});
+}
+
+async function check(token) {
+	return new Promise((resolve, reject) => {
+		// This should not happen
+		if (!authSecret) {
+			resolve({ ok: false, msg: 'Secret not set' });
+		}
+
+		jwt.verify(token, authSecret, function(err, decoded) {
+			if (err) {
+				resolve({ok:false, msg:err})
+			} else {
+				resolve({ok:true, msg:"Successfully authenticated"});
+			}
+		});
+	});
+}
+
+module.exports = {
+	setAuthSecret,
+	middleware,
+	check,
 };

lib/database.js

@@ -0,0 +1,231 @@
+/**
+ * Implementation of master databases
+ *
+ * @module
+ * @author Hornwitser
+ */
+
+"use strict";
+const fs = require("fs-extra");
+
+
+/**
+ * Converts a Map with only string keys to a JavaScript object.
+ *
+ * @param {Map} map - the Map to convert to an object.
+ * @return {object} - Object with the mapping's key-values as properties.
+ * @throws {Error} if there are non-string keys in map.
+ */
+function mapToObject(map) {
+	let obj = {};
+	for (let [key, value] of map) {
+		if (typeof key !== 'string') {
+			throw new Error(
+				`Expected all keys to be string but got ${typeof key}`
+			);
+		}
+
+		obj[key] = value;
+	}
+	return obj;
+}
+
+/**
+ * Load JSON file as a Map
+ *
+ * Loads the JSON file specified by path an converts it into a JavaScript Map.
+ * Returns an empty Map if the file does not exist.
+ *
+ * @param {string} filePath - The path to the JSON file to load.
+ * @throws {Error} If JSON file did not contain an object.
+ * @throws {Error} If an error occured reading the file.
+ */
+async function loadJsonAsMap(filePath) {
+	try {
+		let content = await fs.readFile(filePath);
+		let parsed = JSON.parse(content);
+
+		if (parsed instanceof Array) {
+			throw new Error(`Expected object but got array`);
+		}
+
+		if (parsed === null) {
+			throw new Error(`Expected object but got null`);
+		}
+
+		if (typeof parsed !== "object") {
+			throw new Error(`Expected object but got ${typeof parsed}`);
+		}
+
+		return new Map(Object.entries(parsed));
+
+	} catch (err) {
+		if (err.code == "ENOENT") {
+			// File does not exist, return empty map
+			return new Map();
+
+		} else {
+			throw err;
+		}
+	}
+}
+
+/**
+ * Save a Map as a JSON file
+ *
+ * Save the content of a key-value Map with only string keys as a JSON file.
+ * If the directory the file is to be saved into doesn't exist it will be
+ * created.
+ *
+ * @param {string} filePath - The path to the JSON file that will be saved.
+ * @param {Map} map - Map with only string keys to save.
+ * @throws {Error} if there are non-string keys in map.
+ * @throws {Error} if an error occured writing to the file.
+ */
+async function saveMapAsJson(filePath, map) {
+	let obj = mapToObject(map);
+	await fs.outputFile(filePath, JSON.stringify(obj, null, 4));
+}
+
+
+function checkName(name) {
+	if (typeof name !== "string") {
+		throw new Error("name must be a string");
+	}
+}
+
+function checkCount(count) {
+	if (typeof count !== "number" || isNaN(count)) {
+		throw new Error("count must be a number");
+	}
+}
+
+
+/**
+ * Item database
+ *
+ * Stores counts of items for the master server.  Items that have not been
+ * stored in the database are treated as having a stored count of zero.  When
+ * serialized the database discards entries with a zero counts, and when
+ * deserialized the content is verified.
+ */
+class ItemDatabase {
+
+	/**
+	 * Create a new item database
+	 *
+	 * @param {object|undefined} serialized - An object from a previous call to
+	 *     {@link module:lib/database~ItemDatabase#serialize} to restore the
+	 *     database from.  An empty database will be created if this parameter
+	 *     is left undefined.
+	 */
+	constructor(serialized) {
+		this._items = new Map();
+
+		// Verify the content of the serialized database
+		if (serialized !== undefined) {
+			for (let [name, count] of Object.entries(serialized)) {
+				checkName(name);
+				checkCount(count);
+
+				this._items.set(name, count);
+			}
+		}
+	}
+
+	/**
+	 * Serialize item database
+	 *
+	 * Serialize the item database into a plain JavaScript object that can be
+	 * turned inta a string with JSON.stringify().
+	 *
+	 * @returns {object} Serialized representation of the database
+	 */
+	serialize() {
+		let obj = {};
+		for (let [name, count] of this._items) {
+			if (count !== 0) {
+				obj[name] = count;
+			}
+		}
+		return obj;
+	}
+
+	/**
+	 * Approximate size of the items database.
+	 *
+	 * Does not account for entries being zero in the item database.
+	 */
+	get size() {
+		return this._items.size;
+	}
+
+
+	/**
+	 * Get the count of an item stored in the database
+	 *
+	 * Returns the count of a given item that has been stored in to the
+	 * database.  If the item has not been previously stored into the database
+	 * then 0 is returned.
+	 *
+	 * @param {string} name - The name of the item to get the count of.
+	 * @returns {number} The count of the item stored.
+	 */
+	getItemCount(name) {
+		checkName(name);
+
+		if (!this._items.has(name)) {
+			return 0
+		}
+
+		return this._items.get(name);
+	}
+
+	/**
+	 * Adds count of an item to the database
+	 *
+	 * Add count copies of the item specified by name to the item database.
+	 *
+	 * @param {string} name - The name of the item to add.
+	 * @param {number} count - The count of item to remove.
+	 */
+	addItem(name, count) {
+		checkName(name);
+		checkCount(count);
+
+		if (this._items.has(name)) {
+			count += this._items.get(name);
+		}
+
+		this._items.set(name, count);
+	};
+
+	/**
+	 * Removes count of an item from the database.
+	 *
+	 * Remove count copies of the item specified by name from the item database
+	 * If count is greater than the currently stored count the stored item
+	 * count will become negative.
+	 *
+	 * @param {string} name - The name of the item to remove.
+	 * @param {number} count - The count of items to remove.
+	 */
+	removeItem(name, count) {
+		checkName(name);
+		checkCount(count);
+
+		count = -count;
+		if (this._items.has(name)) {
+			count += this._items.get(name);
+		}
+
+		this._items.set(name, count);
+	};
+}
+
+module.exports = {
+	mapToObject,
+	loadJsonAsMap,
+	saveMapAsJson,
+	ItemDatabase,
+}

lib/database.spec.js

@@ -0,0 +1,212 @@
+const assert = require("assert").strict;
+const fs = require("fs-extra");
+
+const database = require("lib/database");
+
+describe("lib/database", function() {
+	describe("mapToObject()", function() {
+		it("should throw on non-string key", function() {
+			assert.throws(
+				() => database.mapToObject(new Map([[1, 1]])),
+				new Error("Expected all keys to be string but got number")
+			);
+			assert.throws(
+				() => database.mapToObject(new Map([[Symbol(), 1]])),
+				new Error("Expected all keys to be string but got symbol")
+			);
+		});
+
+		it("should throw convert a map to an object", function() {
+			assert.deepEqual(
+				database.mapToObject(new Map([['a', 1], ['b', true]])),
+				{'a': 1, 'b': true}
+			);
+		});
+	});
+
+	describe("loadJsonAsMap()", function() {
+		let badTypes = ['null', 'array', 'number', 'string', 'boolean'];
+
+		for (let type of badTypes) {
+			it(`should reject on ${type} JSON`, async function() {
+				await assert.rejects(
+					database.loadJsonAsMap(`test/json/${type}.json`),
+					new Error(`Expected object but got ${type}`)
+				);
+			});
+		}
+
+		it("should work on empty object JSON", async function() {
+			assert.deepEqual(
+				await database.loadJsonAsMap('test/json/object.json'),
+				new Map()
+			);
+		});
+
+		it("should work on object JSON", async function() {
+			assert.deepEqual(
+				await database.loadJsonAsMap('test/json/load_map.json'),
+				new Map([['a', 1], ['b', true]])
+			);
+		});
+
+		it("should give an empty Map for non-existant file", async function() {
+			assert.deepEqual(
+				await database.loadJsonAsMap('test/json/does-not-exist'),
+				new Map()
+			);
+		});
+	});
+
+	describe("saveMapAsJson()", function() {
+		it("should save a mapping as JSON", async function() {
+			let testFile = 'test/json/save_map.json';
+			async function deleteTestFile() {
+				try {
+					await fs.unlink(testFile);
+				} catch (err) {
+					/* istanbul ignore if */
+					if (err.code !== 'ENOENT') {
+						throw err;
+					}
+				}
+			}
+
+			await deleteTestFile();
+			await database.saveMapAsJson(
+				testFile, new Map([['c', {}], ['d', "foo"]])
+			);
+
+			assert.equal(
+				await fs.readFile(testFile, {encoding: 'utf-8'}),
+				'{\n    "c": {},\n    "d": "foo"\n}'
+			);
+			await deleteTestFile();
+		});
+	});
+
+	describe("class ItemDatabase", function() {
+		describe("constructor()", function() {
+			it("should create an empty database with no args", function() {
+				let items = new database.ItemDatabase();
+				assert.deepEqual(items._items, new Map());
+			});
+
+			it("should restore the passed serialized database", function() {
+				let items = new database.ItemDatabase({'a': 1, 'b': 2});
+				assert.deepEqual(items._items, new Map([['a', 1], ['b', 2]]));
+			});
+
+			it("should throw on invalid serialized database", function() {
+				assert.throws(
+					() => new database.ItemDatabase({'a': NaN}),
+					new Error("count must be a number")
+				);
+
+				assert.throws(
+					() => new database.ItemDatabase({'a': 'a'}),
+					new Error("count must be a number")
+				);
+			});
+		});
+
+		describe(".serialize()", function() {
+			it("should return a serialized database", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				assert.deepEqual(items.serialize(), {'a': 10});
+			});
+
+			it("should remove zero count entries", function() {
+				let items = new database.ItemDatabase({'a': 0});
+				assert.deepEqual(items.serialize(), {});
+			});
+		});
+
+		describe(".size", function() {
+			it("should give an approximate size of the database", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				assert.equal(items.size, 1);
+			});
+		});
+
+		describe(".getItemCount()", function() {
+			it("should return the count of the given item", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				assert.equal(items.getItemCount('a'), 10);
+			});
+
+			it("should return zero if item does not exist", function() {
+				let items = new database.ItemDatabase();
+				assert.equal(items.getItemCount('b'), 0);
+			});
+
+			it("should throw on invalid name", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.getItemCount(2),
+					new Error("name must be a string")
+				);
+			});
+		});
+
+		describe(".addItem()", function() {
+			it("should add a new item", function() {
+				let items = new database.ItemDatabase();
+				items.addItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', 10]]));
+			})
+
+			it("should add an existing item", function() {
+				let items = new database.ItemDatabase({'a': 10});
+				items.addItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', 20]]));
+			})
+
+			it("should throw on invalid name", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.addItem(2, 10),
+					new Error("name must be a string")
+				);
+			});
+
+			it("should throw on invalid count", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.addItem('a', NaN),
+					new Error("count must be a number")
+				);
+			});
+		});
+
+		describe(".removeItem()", function() {
+			it("should remove an existing item", function() {
+				let items = new database.ItemDatabase({'a': 20});
+				items.removeItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', 10]]));
+			})
+
+			it("should turn a non-existing item negative", function() {
+				let items = new database.ItemDatabase();
+				items.removeItem('a', 10);
+				assert.deepEqual(items._items, new Map([['a', -10]]));
+			})
+
+			it("should throw on invalid name", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.removeItem(2, 10),
+					new Error("name must be a string")
+				);
+			});
+
+			it("should throw on invalid count", function() {
+				let items = new database.ItemDatabase();
+				assert.throws(
+					() => items.removeItem('a', 'b'),
+					new Error("count must be a number")
+				);
+			});
+		});
+	});
+});

lib/manager/pluginManager.js

@@ -3,114 +3,123 @@ const path = require("path");
 const git = require("simple-git")("./sharedPlugins");
 
 
-module.exports = function(config){
-	if(!config) config = require("./../../config");
-	
-	return {
-		listPlugins: async function(){
-			let plugins = await this.getPlugins();
-			plugins.forEach(plug => {
-				console.log(`${plug.name}`.padEnd(22).substr(0,22)+` - ${plug.description}`.substr(0,80-22));
-			});
-		},
-		getPlugins: async function(){
-			let directories = await fs.readdir("./sharedPlugins");
-			let returnArray = [];
-			for(let i in directories){
-				let dir = directories[i];
-				let pluginContents = await fs.readdir(path.join("./sharedPlugins", dir));
-				// console.log(pluginContents)
-				if(pluginContents.includes("config.js") || pluginContents.includes("config.json")){
-					try{
-						var pluginConfig = require(path.join("./../../sharedPlugins", dir, "config"));
-						pluginConfig.pluginPath = path.join("./sharedPlugins", dir);
-					}catch(e){
-						console.log(e)
-					}	
-					
-					// check if plugin is enabled
-					try{
-						await fs.stat(path.join(pluginConfig.pluginPath, "DISABLED"))
-						pluginConfig.enabled = false;
-					}catch(e){
-						pluginConfig.enabled = true;
-					}
-					returnArray.push(pluginConfig);
-				}
+async function listPlugins() {
+	let plugins = await this.getPlugins();
+	plugins.forEach(plug => {
+		console.log(`${plug.name}`.padEnd(22).substr(0,22)+` - ${plug.description}`.substr(0,80-22));
+	});
+}
+
+async function getPlugins() {
+	let directories = await fs.readdir("./sharedPlugins");
+	let returnArray = [];
+	for(let i in directories){
+		let dir = directories[i];
+		let pluginContents = await fs.readdir(path.join("./sharedPlugins", dir));
+		// console.log(pluginContents)
+		if(pluginContents.includes("config.js") || pluginContents.includes("config.json")){
+			try{
+				var pluginConfig = require(path.join("./../../sharedPlugins", dir, "config"));
+				pluginConfig.pluginPath = path.join("./sharedPlugins", dir);
+			}catch(e){
+				console.log(e)
 			}
-			return returnArray;
-		},
-		addPlugin: function(path){
-			return new Promise((resolve, reject) => {
-				if(path.includes("git")){ // we are decently sure this is a git repo
-					git.clone(path, data => resolve(data))
-				}
-			});
-		},
-		removePlugin: async function(name){
-			let plugins = await this.getPlugins();
-			for(let i in plugins){
-				let plug = plugins[i];
-				if(plug.name.includes(name)){
-					// console.log(`Removing plugin ${plug.name}`);
-					await fs.remove(plug.pluginPath);
-					return {
-						ok:true,
-						msg:`Deleted plugin ${plug.name}`
-					}
-				}
+
+			// check if plugin is enabled
+			try{
+				await fs.stat(path.join(pluginConfig.pluginPath, "DISABLED"))
+				pluginConfig.enabled = false;
+			}catch(e){
+				pluginConfig.enabled = true;
 			}
+			returnArray.push(pluginConfig);
+		}
+	}
+	return returnArray;
+}
+
+function addPlugin(path) {
+	return new Promise((resolve, reject) => {
+		if(path.includes("git")){ // we are decently sure this is a git repo
+			git.clone(path, data => resolve(data))
+		}
+	});
+}
+
+async function removePlugin(name) {
+	let plugins = await this.getPlugins();
+	for(let i in plugins){
+		let plug = plugins[i];
+		if(plug.name.includes(name)){
+			// console.log(`Removing plugin ${plug.name}`);
+			await fs.remove(plug.pluginPath);
 			return {
-				ok:false,
-				msg:`Plugin matching "${name}" not found`,
-			}
-		},
-		enablePlugin: async function(name, instance){
-			let plugins = await this.getPlugins();
-			let plugin = plugins[findInArray("name", name, plugins)[0]];
-			if(plugin){
-				let files = await fs.readdir(plugin.pluginPath);
-				if(files.includes("DISABLED")){
-					await fs.unlink(path.join(plugin.pluginPath, "DISABLED"));
-					return {
-						ok:true,
-						msg:`Globally enabled plugin ${name}`,
-					}
-				} else {
-					return {
-						ok:true,
-						msg:`Plugin ${name} already enabled`,
-					}
-				}
-			} else {
-				return {
-					ok:false,
-					msg:`Found no excact match for plugin ${name}`,
-				}
-			}
-		},
-		disablePlugin: async function(name, instance){
-			let plugins = await this.getPlugins();
-			let plugin = plugins[findInArray("name", name, plugins)[0]];
-			if(plugin){
-				await fs.writeFile(path.join(plugin.pluginPath, "DISABLED"), "");
-				return {
-					ok:true,
-					msg:`Plugin ${name} is now disabled on this host`,
-				}
-			} else {
-				return {
-					ok:false,
-					msg:`Found no excact match for plugin ${name}`,
-				}
+				ok:true,
+				msg:`Deleted plugin ${plug.name}`
 			}
 		}
 	}
+	return {
+		ok:false,
+		msg:`Plugin matching "${name}" not found`,
+	}
 }
-function findInArray(key, value, array){
+
+async function enablePlugin(name, instance) {
+	let plugins = await this.getPlugins();
+	let plugin = plugins[findInArray("name", name, plugins)[0]];
+	if(plugin){
+		let files = await fs.readdir(plugin.pluginPath);
+		if(files.includes("DISABLED")){
+			await fs.unlink(path.join(plugin.pluginPath, "DISABLED"));
+			return {
+				ok:true,
+				msg:`Globally enabled plugin ${name}`,
+			}
+		} else {
+			return {
+				ok:true,
+				msg:`Plugin ${name} already enabled`,
+			}
+		}
+	} else {
+		return {
+			ok:false,
+			msg:`Found no excact match for plugin ${name}`,
+		}
+	}
+}
+
+async function disablePlugin(name, instance) {
+	let plugins = await this.getPlugins();
+	let plugin = plugins[findInArray("name", name, plugins)[0]];
+	if(plugin){
+		await fs.writeFile(path.join(plugin.pluginPath, "DISABLED"), "");
+		return {
+			ok:true,
+			msg:`Plugin ${name} is now disabled on this host`,
+		}
+	} else {
+		return {
+			ok:false,
+			msg:`Found no excact match for plugin ${name}`,
+		}
+	}
+}
+
+function findInArray(key, value, array) {
 	let indexes = [];
 	for(let i in array){
 		if(array[i][key] && array[i][key] === value) indexes.push(i);
 	}
 	return indexes;
 }
+
+module.exports = {
+	listPlugins,
+	getPlugins,
+	addPlugin,
+	removePlugin,
+	enablePlugin,
+	disablePlugin,
+};

master.js

@@ -28,15 +28,16 @@ const moment = require("moment");
 const request = require("request");
 
 // constants
-console.log(`Requiring config from ${args.config || './config'}`);
-const config = require(args.config || './config');
+let config = {};
 
 // homebrew modules
 const generateSSLcert = require("lib/generateSSLcert");
 const getFactorioLocale = require("lib/getFactorioLocale");
+const pluginManager = require("lib/manager/pluginManager.js");
+const database = require("lib/database");
 
 // homemade express middleware for token auth
-const authenticate = require("lib/authenticate")(config);
+const authenticate = require("lib/authenticate");
 
 const express = require("express");
 const compression = require('compression');
@@ -67,7 +68,7 @@ app.use(function(req, res, next){
 	res.locals.res = res;
 	res.locals.req = req;
 	res.locals.masterPlugins = masterPlugins;
-	res.locals.slaves = slaves;
+	res.locals.slaves = db.slaves;
 	res.locals.moment = moment;
 	next();
 });
@@ -146,27 +147,25 @@ app.get('/metrics', (req, res) => {
 });
 
 function registerMoreMetrics(){
-	for(let instanceID in slaves){
+	for (let [instanceID, slave] of db.slaves) {
 		// playercount
 		try{
-			prometheusPlayerCountGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].playerCount) || 0);
+			prometheusPlayerCountGauge.labels(instanceID, slave.instanceName).set(Number(slave.playerCount) || 0);
 		}catch(e){}
 		// UPS
 		try{
-			prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.UPS) || 60);
-			if(slaves[instanceID].meta.tick && typeof slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.tick) || 0);
+			prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.UPS) || 60);
+			if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0);
 		}catch(e){}
 	}
 	// inventory
-	for(let key in db.items){
-		if(typeof db.items[key] == "number" || typeof db.items[key] == "string"){
-			prometheusMasterInventoryGauge.labels(key).set(Number(db.items[key]) || 0);
-		}
+	for (let [key, count] of db.items._items) {
+		prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
 	}
 	// Slave count
 	let numberOfActiveSlaves = 0;
-	for(let instance in slaves){
-		if(Date.now() - Number(slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
+	for(let slave of db.slaves.values()){
+		if(Date.now() - Number(slave.time) < 1000 * 30) numberOfActiveSlaves++;
 	}
 	prometheusConnectedInstancesCounter.set(numberOfActiveSlaves);
 }
@@ -174,49 +173,28 @@ function registerMoreMetrics(){
 // set up database
 const db = {};
 
-(function(){
-	try{
-		let x = fs.statSync(path.resolve(config.databaseDirectory, "slaves.json"));
-		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
-		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json")));
-	} catch (e){
-		slaves = {};
-	}
-	try{
-		x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
-		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
-		db.items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")));
-	} catch (e){
-		db.items = {};
-	}
-})();
+async function loadDatabase(databaseDirectory) {
+	let slavesPath = path.resolve(databaseDirectory, "slaves.json");
+	console.log(`Loading ${slavesPath}`);
+	db.slaves = await database.loadJsonAsMap(slavesPath);
+
+	let itemsPath = path.resolve(databaseDirectory, "items.json");
+	console.log(`Loading ${itemsPath}`);
+	try {
+		let content = await fs.readFile(itemsPath);
+		db.items = new database.ItemDatabase(JSON.parse(content));
+
+	} catch (err) {
+		if (err.code === "ENOENT") {
+			console.log("Creating new item database");
+			db.items = new database.ItemDatabase();
 
-db.items.addItem = function(object) {
-	if(object.name == "addItem" || object.name == "removeItem") {
-		console.error("Fuck you, that would screw everything up if you named your item that.");
-		return false;
-	} else {
-		if(this[object.name] && Number(this[object.name]) != NaN){
-			this[object.name] = Number(this[object.name]) + Number(object.count);
 		} else {
-			this[object.name] = Number(object.count);
+			throw err;
 		}
-		return true;
 	}
-};
-db.items.removeItem = function(object) {
-	if(object.name == "addItem" || object.name == "removeItem") {
-		console.error("Fuck you, that would screw everything up if you named your item that.");
-		return false;
-	} else {
-		if(this[object.name] && Number(this[object.name]) != NaN){
-			this[object.name] = Number(this[object.name]) - Number(object.count);
-		} else {
-			this[object.name] = 0;
-		}
-		return true;
-	}
-};
+}
+
 
 // store slaves and inventory in a .json full of JSON data
 async function shutdown() {
@@ -224,17 +202,21 @@ async function shutdown() {
 	let exitStartTime = Date.now();
 	try {
 		// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-		if(slaves && Object.keys(slaves).length < 2000000){
-			console.log("saving to slaves.json");
-			fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
-		} else if(slaves) {
-			console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
+		if (db.slaves && db.slaves.size < 2000000) {
+			let file = path.resolve(config.databaseDirectory, "slaves.json");
+			console.log(`writing ${file}`);
+			await database.saveMapAsJson(file, db.slaves);
+		} else if (db.slaves) {
+			console.error(`Slave database too large, not saving (${db.slaves.size}`);
 		}
-		if(db.items && Object.keys(db.items).length < 50000){
-			console.log("saving to items.json");
-			fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
-		} else if(slaves) {
-			console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
+
+		if (db.items && db.items.size < 50000) {
+			let file = path.resolve(config.databaseDirectory, "items.json");
+			console.log(`writing ${file}`);
+			let content = JSON.stringify(db.items.serialize());
+			await fs.outputFile(file, content);
+		} else if (db.items) {
+			console.error(`Item database too large, not saving (${db.items.size})`);
 		}
 		for(let i in masterPlugins){
 			let plugin = masterPlugins[i];
@@ -266,15 +248,16 @@ app.post("/api/getID", authenticate.middleware, function(req,res) {
 	// time us a unix timestamp we can use to check for how long the server has been unresponsive
 	// we should save that somewhere and give appropriate response
 	if(req.body){
-		if(!slaves[req.body.unique]){
-			slaves[req.body.unique] = {};
+		if (!db.slaves.has(req.body.unique)) {
+			db.slaves.set(req.body.unique, {});
 		}
-		for(k in req.body){
+		let slave = db.slaves.get(req.body.unique);
+		for(let k in req.body){
 			if(k != "meta" && req.body.hasOwnProperty(k)){
-				slaves[req.body.unique][k] = req.body[k];
+				slave[k] = req.body[k];
 			}
 		}
-		// console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName);
+		// console.log("Slave registered: " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP + " with name " + slave.instanceName);
 	}
 });
 /**
@@ -289,16 +272,17 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 	// {instanceID, pass, meta:{x,y,z}}
 
 	if(req.body && req.body.instanceID && req.body.meta){
-		if(slaves[req.body.instanceID]){
-			if(!slaves[req.body.instanceID].meta) {
-				slaves[req.body.instanceID].meta = {};
+		if (db.slaves.has(req.body.instanceID)) {
+			let slave = db.slaves.get(req);
+			if(!slave.meta) {
+				slave.meta = {};
 			}
-			slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
+			slave.meta = deepmerge(slave.meta, req.body.meta, {clone:true});
 			let metaPortion = JSON.stringify(req.body.meta);
 			if(metaPortion.length > 50) {
 				metaPortion = metaPortion.substring(0,20) + "...";
 			}
-			// console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion);
+			// console.log("Updating slave "+slave.instanceName+": " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP, metaPortion);
 			res.sendStatus(200);
 		} else {
 			res.send("ERROR: Invalid instanceID or password")
@@ -316,12 +300,13 @@ app.post("/api/getSlaveMeta", function (req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	console.log("body", req.body);
 	if(req.body && req.body.instanceID){
-		if (slaves[req.body.instanceID]) {
+		if (db.slaves.has(req.body.instanceID)) {
+			let slave = db.slaves.get(req.body.instanceID);
 			console.log("returning meta for ", req.body.instanceID);
-			if(!slaves[req.body.instanceID].meta) {
-				slaves[req.body.instanceID].meta = {};
+			if(!slave.meta) {
+				slave.meta = {};
 			}
-			res.send(JSON.stringify(slaves[req.body.instanceID].meta))
+			res.send(JSON.stringify(slave.meta))
 		} else {
 			res.status(404);
 			res.send('{"status": 404, "info": "Slave not registered"}')
@@ -387,7 +372,7 @@ let slaveCache = {
 };
 function getSlaves() {
 	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000) {
-		let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
+		let copyOfSlaves = JSON.parse(JSON.stringify(database.mapToObject(db.slaves)));
 		slaveCache.cache = copyOfSlaves;
 		slaveCache.timestamp = Date.now();
 	}
@@ -460,11 +445,12 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	}
 	if(!x.instanceID) {
 		x.instanceID = "unknown";
-		Object.keys(slaves).forEach(instanceID => {
-			if(slaves[instanceID].instanceName == req.body.instanceName) {
+		for (let [instanceID, slave] of db.slaves) {
+			if(slave.instanceName == req.body.instanceName) {
 				x.instanceID = instanceID;
+				break;
 			}
-		});
+		}
 	}
 	if(x.instanceID
 	&& x.instanceName
@@ -490,10 +476,7 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 		});
 		prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
 		// save items we get
-		db.items.addItem({
-			name:req.body.name,
-			count:req.body.count
-		});
+		db.items.addItem(req.body.name, req.body.count);
 
 		// Attempt confirming
 		res.send("success");
@@ -502,12 +485,8 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	}
 });
 const routes_api_remove = require("./routes/api/remove.js");
-var neuralDole = null;
+let neuralDole;
 
-if(config.useNeuralNetDoleDivider)//Only initialize neural network when it's enabled, otherwise it might override gauge
-	neuralDole=new routes_api_remove.neuralDole({
-		items: db.items, gaugePrefix: prometheusPrefix
-	});
 /**
 POST endpoint to remove items from DB when client orders items.
 
@@ -531,22 +510,22 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	if(!object.instanceName) {
 		object.instanceName = "unknown";
 	}
-	if(slaves[object.instanceID]) object.instanceName = slaves[object.instanceID].instanceName;
-	let item = db.items[object.name];
-		// console.dir(doc);
-	if((item === undefined)
-	||((
-		config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
-		&& object.name != "electricity")
-	){
+	if (db.slaves.has(object.instanceID)) {
+		object.instanceName = db.slaves.get(object.instanceID).instanceName;
+	}
+	let itemCount = db.items.getItemCount(object.name);
+	if (
+		(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
+		&& object.name != "electricity"
+	) {
 		if(config.logItemTransfers){
 			console.log('failure could not find ' + object.name);
 		}
 		res.send({name:object.name, count:0});
 	} else if(config.disableFairItemDistribution){
 		// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
-		let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item));
-		db.items.removeItem({count: numberToRemove, name: object.name});
+		let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
+		db.items.removeItem(object.name, numberToRemove);
 		res.send({count: numberToRemove, name: object.name});
 
 		// track statistics and do graphing things
@@ -577,7 +556,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	} else {
 		// Use dole division. Makes it really slow to drain out the last little bit.
 		routes_api_remove.doleDivider({
-			item,
+			itemCount,
 			object,
 			db,
 			sentItemStatisticsBySlaveID,
@@ -604,10 +583,8 @@ app.get("/api/inventory", function(req, res) {
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// Check it and send it
 	var inventory = [];
-	for(let key in db.items){
-		if(typeof db.items[key] == "number" || typeof db.items[key] == "string"){
-			inventory.push({name:key, count:db.items[key]});
-		}
+	for (let [name, count] of db.items._items) {
+		inventory.push({ name, count });
 	}
 	res.send(JSON.stringify(inventory));
 });
@@ -624,7 +601,7 @@ app.get("/api/inventoryAsObject", function(req, res) {
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// Check it and send it
-	res.send(JSON.stringify(db.items));
+	res.send(JSON.stringify(db.items.serialise()));
 });
 
 /**
@@ -824,7 +801,6 @@ async function pluginManagement(){
 }
 
 async function getPlugins(){
-	const pluginManager = require("lib/manager/pluginManager.js")(config);
 	let plugins = [];
 	let pluginsToLoad = await pluginManager.getPlugins();
 	for(let i = 0; i < pluginsToLoad.length; i++){
@@ -888,6 +864,9 @@ async function startServer() {
 	// add better stack traces on promise rejection
 	process.on('unhandledRejection', r => console.log(r));
 
+	console.log(`Requiring config from ${args.config || './config'}`);
+	config = require(args.config || './config');
+
 	/** Sync */
 	function randomStringAsBase64Url(size) {
 	  return base64url(crypto.randomBytes(size));
@@ -906,8 +885,11 @@ async function startServer() {
 	fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
 		expiresIn: 86400*365 // expires in 1 year
 	}));
-	setInterval(()=>{
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
+	setInterval(async function() {
+		let file = path.resolve(config.databaseDirectory, "items.json");
+		console.log(`autosaving ${file}`);
+		let content = JSON.stringify(db.items.serialize());
+		await fs.outputFile(file, content);
 	},config.autosaveInterval || 60000);
 	process.on('SIGINT', shutdown); // ctrl + c
 	process.on('SIGHUP', shutdown); // terminal closed
@@ -921,6 +903,16 @@ async function startServer() {
 	// mod downloads
 	app.use(express.static(masterModFolder));
 
+	await loadDatabase(config.databaseDirectory);
+	authenticate.setAuthSecret(config.masterAuthSecret);
+
+	// Only initialize neural network when it's enabled, otherwise it might override gauge
+	if(config.useNeuralNetDoleDivider) {
+		neuralDole=new routes_api_remove.neuralDole({
+			db, gaugePrefix: prometheusPrefix
+		});
+	}
+
 	// Make sure we're actually going to listen on a port
 	let httpPort = args.masterPort || config.masterPort;
 	let httpsPort = args.sslPort || config.sslPort;
@@ -974,7 +966,13 @@ async function startServer() {
 	}
 }
 
-module.exports = app;
+module.exports = {
+	app,
+
+	// For testing only
+	_db: db,
+	_config: config,
+}
 
 if (module === require.main) {
 	console.warn(`

master.spec.js

@@ -1,17 +1,29 @@
-const app = require('./master.js');
-const config = require('./config.json');
 const assert = require('assert').strict;
 const request = require('supertest');
 const validateHTML = require('html5-validator');
 const parallel = require('mocha.parallel');
 const jwt = require("jsonwebtoken");
 
+const master = require('./master.js');
+const authenticate = require('lib/authenticate');
+const database = require('lib/database');
 
-const token =
-	jwt.sign( { id: "api" }, config.masterAuthSecret, { expiresIn: 600 });
+
+master._db.slaves = new Map();
+master._db.items = new database.ItemDatabase();
+
+// Fields required in the config
+master._config.factorioDirectory = "factorio";
+master._config.itemStats = { maxEntries:60, entriesPerSecond: 1 / 60 };
+master._config.disableFairItemDistribution = true;
+
+const testSecret = "TestSecretDoNotUse";
+authenticate.setAuthSecret(testSecret);
+
+const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
 
 async function get(path) {
-	return await request(app).get(path).expect(200);
+	return await request(master.app).get(path).expect(200);
 }
 
 async function getValidate(path) {
@@ -21,7 +33,7 @@ async function getValidate(path) {
 	assert(filtered.length === 0, "there are HTML errors on the page, please fix: "+JSON.stringify(validation.messages, null, 4));
 }
 
-describe('Master server endpoint testing', function() {
+describe('Master testing', function() {
 	describe('#GET /api/getFactorioLocale', function() {
 		it('should get the basegame factorio locale', async function() {
 			let res = await get('/api/getFactorioLocale');
@@ -42,7 +54,7 @@ describe('Master server endpoint testing', function() {
 			it(`sends some HTML when accessing ${path}`, () => getValidate(path));
 		}
 	});
-	let persistentMaster = request(app);
+	let persistentMaster = request(master.app);
 	describe("#POST /api/place", function() {
 		it("adds an itemStack to the masters inventory", function() {
 			return persistentMaster.post("/api/place").send({
@@ -82,19 +94,31 @@ describe('Master server endpoint testing', function() {
 				assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
 			});
 		});
-		it("returns an empty itemStack if you try to request addItem or removeItem", function() {
-			function remove() {
-				return persistentMaster.post("/api/remove").send({
-					name:"addItem",
-					count:10
+		it("works correctly for items named addItem or removeItem", async function() {
+			function add(item) {
+				return persistentMaster.post("/api/place").send({
+					name: item,
+					count: 10
 				}).set(
 					"X-Access-Token", token
 				).expect(200).then(function(res) {
-					assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0");
+					assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
+				})
+			}
+
+			function remove(item) {
+				return persistentMaster.post("/api/remove").send({
+					name: item,
+					count: 10
+				}).set(
+					"X-Access-Token", token
+				).expect(200).then(function(res) {
+					assert.deepEqual(res.body, {"name":item,"count":10});
 				});
 			}
 
-			return Promise.all([remove(), remove()]);
+			await Promise.all([add("addItem"), add("removeItem")]);
+			await Promise.all([remove("addItem"), remove("removeItem")]);
 		});
 		it("returns an empty itemStack if you don't have any of the item you request", function() {
 			return persistentMaster.post("/api/remove").send({

routes/api/remove.js

@@ -19,7 +19,7 @@ class neuralDole {
     }
     
     constructor({
-        items, gaugePrefix
+		db, gaugePrefix
     }){
         // Set some storage variables for the dole divider
         this.prometheusNNDoleGauge = new Prometheus.Gauge({
@@ -27,8 +27,8 @@ class neuralDole {
             help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
             labelNames: ["itemName"],
         });
-        this.items = items
-        this.itemsLastTick = JSON.parse(JSON.stringify(items))
+		this.db = db
+		this.itemsLastTick = new Map(db.items._items)
         this.dole = {}
         this.carry = {}
         this.lastRequest = {}
@@ -36,15 +36,11 @@ class neuralDole {
 		this.debt= {}
 
         setInterval(()=>{
-            for(let name in this.items){
-                // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties.
-                if(typeof this.items[name] !== "number") continue
-
-                let count = this.items[name]
+			for (let [name, count] of this.db.items._items) {
                 let magicData = doleNN.Tick(
                     count,
                     this.dole[name],
-                    this.itemsLastTick[name],
+					this.itemsLastTick.get(name),
                     this.getRequestStats(name,5)
                 )
                 this.stats[name]=this.stats[name] || []
@@ -55,7 +51,7 @@ class neuralDole {
                 // DONE handle magicData[1] for graphing for our users
                 this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
             }
-            this.itemsLastTick = JSON.parse(JSON.stringify(this.items))
+			this.itemsLastTick = new Map(this.db.items._items)
         }, 1000)
     }
     divider({
@@ -67,8 +63,8 @@ class neuralDole {
     }){
         let magicData = doleNN.Dose(
             object.count, // numReq
-            this.items[object.name],
-            this.itemsLastTick[object.name] || 0,
+			this.db.items.getItemCount(object.name),
+			this.itemsLastTick.get(object.name) || 0,
             this.dole[object.name],
             this.carry[object.name+" "+object.instanceID] || 0,
             this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
@@ -87,46 +83,40 @@ class neuralDole {
         this.dole[object.name] = magicData[1]
         this.carry[object.name+" "+object.instanceID] = magicData[2]
 		this.debt[object.name+" "+object.instanceID] = magicData[3]
-        
-        // Remove item from DB and send it
-        if(this.items.removeItem({count: magicData[0], name: object.name})){
-            if(config.logItemTransfers){
-                console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName);
-            }
-            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-            if(sentItemStatistics === undefined){
-                sentItemStatistics = new averagedTimeSeries({
-                    maxEntries: config.itemStats.maxEntries,
-                    entriesPerSecond: config.itemStats.entriesPerSecond,
-                    mergeMode: "add",
-                }, console.log);
-            }
-            sentItemStatistics.add({
-                key:object.name,
-                value:magicData[0],
-            });
-            //console.log(sentItemStatistics.data)
-            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-            prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
 
-            res.send({
-                name: object.name,
-                count: magicData[0],
-            })
-        } else {
-            res.send({
-                name: object.name,
-                count: 0,
-            })
-        }        
+		// Remove item from DB and send it
+		this.db.items.removeItem(object.name, magicData[0]);
+		if(config.logItemTransfers){
+			console.log("removed: " + object.name + " " + magicData[0] + " . " + this.db.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
+		}
+		let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+		if(sentItemStatistics === undefined){
+			sentItemStatistics = new averagedTimeSeries({
+				maxEntries: config.itemStats.maxEntries,
+				entriesPerSecond: config.itemStats.entriesPerSecond,
+				mergeMode: "add",
+			}, console.log);
+		}
+		sentItemStatistics.add({
+			key:object.name,
+			value:magicData[0],
+		});
+		//console.log(sentItemStatistics.data)
+		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+		prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
+
+		res.send({
+			name: object.name,
+			count: magicData[0],
+		})
     }
 }
 
 _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 function doleDivider({
-    item,
+	itemCount,
     object,
-    db,
+	db,
     sentItemStatisticsBySlaveID,
     config,
     prometheusDoleFactorGauge,
@@ -139,34 +129,35 @@ function doleDivider({
     const originalCount = Number(object.count) || 0;
     object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
     object.count = Math.round(object.count);
-    if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
-    
-    // Update existing items if item name already exists
-    if(Number(item) > Number(object.count)) {
+	if (itemCount > 40) {
+		console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`);
+	}
+
+	// Update existing items if item name already exists
+	if (itemCount > object.count) {
         //If successful, increase dole
         _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
-        if(config.logItemTransfers){
-            console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
-        }
-        if(db.items.removeItem({count: object.count, name: object.name})){
-            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-            if(sentItemStatistics === undefined){
-                sentItemStatistics = new averagedTimeSeries({
-                    maxEntries: config.itemStats.maxEntries,
-                    entriesPerSecond: config.itemStats.entriesPerSecond,
-                    mergeMode: "add",
-                }, console.log);
-            }
-            sentItemStatistics.add({
-                key:object.name,
-                value:object.count,
-            });
-            //console.log(sentItemStatistics.data)
-            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-        }
-        
+		if (config.logItemTransfers) {
+			console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
+		}
+		db.items.removeItem(object.name, object.count)
+		let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+		if(sentItemStatistics === undefined){
+			sentItemStatistics = new averagedTimeSeries({
+				maxEntries: config.itemStats.maxEntries,
+				entriesPerSecond: config.itemStats.entriesPerSecond,
+				mergeMode: "add",
+			}, console.log);
+		}
+		sentItemStatistics.add({
+			key:object.name,
+			value:object.count,
+		});
+		//console.log(sentItemStatistics.data)
+		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+
         prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
-        prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
+        prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);
         res.send({count: object.count, name: object.name});
     } else {
         // if we didn't have enough, attempt giving out a smaller amount next time

sharedPlugins/serverManager/index.js

@@ -9,7 +9,7 @@ module.exports = class remoteCommands {
 		this.messageInterface = messageInterface;
 		this.config = mergedConfig;
 		this.socket = extras.socket;
-		const pluginManager = require("lib/manager/pluginManager.js")(this.config);
+		const pluginManager = require("lib/manager/pluginManager.js");
 		const configManager = require("lib/manager/configManager.js");
 		// const modManager = require("lib/manager/modManager.js")(this.config);
 		this.socket.on("hello", () => {

sharedPlugins/serverManager/static/index.html

@@ -13,10 +13,10 @@
 				html = "";
 				if(req.query
 				&& req.query.instanceID
-				&& slaves[req.query.instanceID]
-				&& slaves[req.query.instanceID].mods
+				&& locals.slaves.get(req.query.instanceID)
+				&& locals.slaves.get(req.query.instanceID).mods
 				&& serverManager.serverSockets[req.query.instanceID]){
-					let slave = slaves[req.query.instanceID];
+					let slave = locals.slaves.get(req.query.instanceID);
 					html += `
 						<div id="mods">
 							${slave.mods.map(mod => mod.name).reduce((acc, modname) => acc + "<li>" + modname + "</li>")}

sharedPlugins/statisticsExporter/masterPlugin.js

@@ -3,7 +3,7 @@ const path = require("path");
 
 class masterPlugin {
 	constructor({config, socketio, express, Prometheus}){
-		const authenticate = require("lib/authenticate")(config);
+		const authenticate = require("lib/authenticate");
 		
 		this.config = config;
 		this.io = socketio;

static/nodes.html

@@ -15,8 +15,7 @@
 			<div id="display">
 				<%
                     let html = ""
-					for(let id in slaves){
-						let slave = slaves[id];
+					for(let [id, slave] of locals.slaves){
 						html += `
 						<div id="${id}" class="slaveBox card mb-3">
                             <div class="card-body">

test/json/array.json

@@ -0,0 +1 @@
+[]

test/json/boolean.json

@@ -0,0 +1 @@
+false

test/json/load_map.json

@@ -0,0 +1 @@
+{"a":1,"b":true}

test/json/null.json

@@ -0,0 +1 @@
+null

test/json/number.json

@@ -0,0 +1 @@
+0

test/json/object.json

@@ -0,0 +1 @@
+{}

test/json/string.json

@@ -0,0 +1 @@
+""