Merge pull request #204 from Hornwitser/refactor-master

Refactor master to fix unit tests

.travis.yml

@@ -6,5 +6,6 @@ before_install:
   - sudo apt-get install -y curl git tar
   - curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64
   - tar -xvf factorio.tar.gz
+  - cp -n config.json.test config.json
 script:
   - npm run cover

config.json.test

@@ -0,0 +1,47 @@
+{
+	"factorioDirectory": "./factorio",
+	"instanceDirectory": "./instances",
+	"databaseDirectory": "./database",
+
+	"publicIP": "localhost",
+
+	"masterIP": "localhost",
+
+	"masterPort": 8080,
+
+	"sslPort": 443,
+	"sslCert": "database/certificates/cert.crt",
+	"sslPrivKey": "database/certificates/cert.key",
+	"masterAuthToken": "",
+	"masterAuthSecret": "TestSecretDoNotUse",
+
+	"username": "",
+	"token": "",
+	"factorio_version": "0.16.41",
+
+	"verify_user_identity": true,
+	"auto_pause": false,
+	"allow_commands": "admins-only",
+	"game_password": "",
+	"description": "This server runs Clusterio",
+	"visibility": {
+		"public": true,
+		"lan": true
+	},
+
+	"itemStats": {
+		"maxEntries":60,
+		"entriesPerSecond": 0.01666666667
+	},
+	"logItemTransfers": false,
+	"disableFairItemDistribution": true,
+	"useNeuralNetDoleDivider": false,
+	"uploadModsToMaster": true,
+	"msBetweenCommands": 10,
+	"allowRemoteCommandExecution": true,
+	"enableCrossServerShout": true,
+	"mirrorAllChat": false,
+	"autosaveInterval": 600000,
+	"disablePrometheusPushgateway": false,
+	"disableImportsOfEverythingExceptElectricity": false
+}

master.js

@@ -14,15 +14,6 @@ node master.js
 // const updater = require("./updater.js");
 // updater.update().then(console.log);
 
-(async ()=>{
-
-// Set the process title, shows up as the title of the CMD window on windows
-// and as the process name in ps/top on linux.
-process.title = "clusterioMaster";
-
-// add better stack traces on promise rejection
-process.on('unhandledRejection', r => console.log(r));
-
 // configgy stuff
 debug = false;
 
@@ -55,24 +46,6 @@ const getFactorioLocale = require("lib/getFactorioLocale");
 // homemade express middleware for token auth
 const authenticate = require("lib/authenticate")(config);
 
-/** Sync */
-function randomStringAsBase64Url(size) {
-  return base64url(crypto.randomBytes(size));
-}
-if (!fs.existsSync("secret-api-token.txt")) {
-	config.masterAuthSecret = randomStringAsBase64Url(256);
-	fs.writeFileSync("config.json",JSON.stringify(config, null, 4));
-	fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
-		expiresIn: 86400*365 // expires in 1 year
-	}));
-	console.log("Generated new master authentication private key!");
-	process.exit(0);
-}
-// write an auth token to file
-fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
-	expiresIn: 86400*365 // expires in 1 year
-}));
-
 const express = require("express");
 const compression = require('compression');
 const cookieParser = require('cookie-parser');
@@ -174,9 +147,6 @@ const prometheusMasterInventoryGauge = new Prometheus.Gauge({
 	help: 'Amount of items stored on master',
 	labelNames: ["itemName"],
 });
-setInterval(()=>{
-	fs.writeFileSync("database/items.json", JSON.stringify(db.items));
-},config.autosaveInterval || 60000);
 /**
 GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format.
 @memberof clusterioMaster
@@ -266,8 +236,6 @@ db.items.removeItem = function(object) {
 };
 
 // store slaves and inventory in a .json full of JSON data
-process.on('SIGINT', shutdown); // ctrl + c
-process.on('SIGHUP', shutdown); // terminal closed
 async function shutdown() {
 	console.log('Ctrl-C...');
 	let exitStartTime = Date.now();
@@ -764,28 +732,9 @@ app.get("/api/getFactorioLocale", function(req,res){
 		res.send(factorioLocale);
 	});
 });
-if(args.sslPort || config.sslPort){
-	try{
-		var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt'));
-		var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key'));
-
-		httpsServer = require("https").createServer({
-			key: privateKey,
-			cert: certificate
-		}, app).listen(args.sslPort || config.sslPort);
-	} catch(e){
-		console.error(e)
-	}
-}
-if(args.masterPort || config.masterPort){
-	server = require("http").Server(app);
-	server.listen(args.masterPort || config.masterPort || 8080, function () {
-		console.log("Listening on port %s...", server.address().port);
-	});
-}
 
 /* Websockets for remoteMap */
-var io = require("socket.io")();
+var io = require("socket.io")({});
 const ioMetrics = require("socket.io-prometheus");
 ioMetrics(io);
 
@@ -975,8 +924,14 @@ io.on('connection', function (socket) {
 });
 
 // handle plugins on the master
-async function getPlugins(){
+var masterPlugins = [];
+async function pluginManagement(){
 	let startPluginLoad = Date.now();
+	masterPlugins = await getPlugins();
+	console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms");
+}
+
+async function getPlugins(){
 	const pluginManager = require("lib/manager/pluginManager.js")(config);
 	let plugins = [];
 	let pluginsToLoad = await pluginManager.getPlugins();
@@ -1007,15 +962,74 @@ async function getPlugins(){
 		let plugin = plugins[i];
 		if(plugin.main.onLoadFinish && typeof plugin.main.onLoadFinish == "function") await plugin.main.onLoadFinish({plugins});
 	}
-	console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms");
 	return plugins;
 }
 
+async function startServer() {
+	// Set the process title, shows up as the title of the CMD window on windows
+	// and as the process name in ps/top on linux.
+	process.title = "clusterioMaster";
 
-masterPlugins = await getPlugins();
+	// add better stack traces on promise rejection
+	process.on('unhandledRejection', r => console.log(r));
 
+	/** Sync */
+	function randomStringAsBase64Url(size) {
+	  return base64url(crypto.randomBytes(size));
+	}
 
-io.attach(server);
+	if (!fs.existsSync("secret-api-token.txt")) {
+		config.masterAuthSecret = randomStringAsBase64Url(256);
+		fs.writeFileSync("config.json",JSON.stringify(config, null, 4));
+		fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
+			expiresIn: 86400*365 // expires in 1 year
+		}));
+		console.log("Generated new master authentication private key!");
+		process.exit(0);
+	}
+	// write an auth token to file
+	fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
+		expiresIn: 86400*365 // expires in 1 year
+	}));
+	setInterval(()=>{
+		fs.writeFileSync("database/items.json", JSON.stringify(db.items));
+	},config.autosaveInterval || 60000);
+	process.on('SIGINT', shutdown); // ctrl + c
+	process.on('SIGHUP', shutdown); // terminal closed
+
+	// Make sure we're actually going to listen on a port
+	let httpPort = args.masterPort || config.masterPort;
+	let httpsPort = args.sslPort || config.sslPort;
+	if (!httpPort && !httpsPort) {
+		console.error("Error: at least one of httpPort and sslPort must be configured");
+		process.exit(1);
+	}
+
+	// Load plugins
+	await pluginManagement();
+
+	// Only start listening for connections after all plugins have loaded
+	if (httpPort) {
+		let httpServer = require("http").Server(app);
+		httpServer.listen(httpPort);
+		console.log("Listening for HTTP on port %s...", httpServer.address().port);
+	}
+
+	if (httpsPort) {
+		var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt'));
+		var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key'));
+
+		let httpsServer = require("https").createServer({
+			key: privateKey,
+			cert: certificate
+		}, app)
+		httpsServer.listen(httpsPort);
+		console.log("Listening for HTTPS on port %s...", httpsServer.address().port);
+	}
+}
 
 module.exports = app;
-})();
+
+if (module === require.main) {
+	startServer();
+}

master.spec.js

@@ -1,183 +1,122 @@
-var app = require('./master.js'),
-  assert = require('assert'),
-  request = require('supertest');
+const app = require('./master.js');
+const config = require('./config.json');
+const assert = require('assert').strict;
+const request = require('supertest');
 const validateHTML = require('html5-validator');
-var parallel = require('mocha.parallel');
+const parallel = require('mocha.parallel');
+const jwt = require("jsonwebtoken");
 
 
+const token =
+	jwt.sign( { id: "api" }, config.masterAuthSecret, { expiresIn: 600 });
+
+async function get(path) {
+	return await request(app).get(path).expect(200);
+}
+
+async function getValidate(path) {
+	let res = await get(path);
+	let validation = await validateHTML(res.text);
+	let filtered = validation.messages.filter(msg => msg.type !== "info");
+	assert(filtered.length === 0, "there are HTML errors on the page, please fix: "+JSON.stringify(validation.messages, null, 4));
+}
+
 describe('Master server endpoint testing', function() {
-	describe('#GET /api/getFactorioLocale', function() { 
-		it('should get the basegame factorio locale', function(done) { 
-			request(app).get('/api/getFactorioLocale').end(function(err, res) {
-				assert.equal(res.statusCode, 200);
-				
-				let object = res.body;
-				
-				// test that it is actual factorio locale, copied from getFactorioLocale.spec.js
-				assert.equal(typeof object, "object");
-				assert.equal(object["entity-name"]["fish"], "Fish");
-				assert.equal(object["entity-name"]["small-lamp"], "Lamp");
-				Object.keys(object).forEach(key => {
-					// first level of the nested object is always an object
-					assert.equal(typeof object[key], "object");
-					
-					Object.keys(object[key]).forEach(key2 => {
-						// second level of the nested object is always a string, nearly always truthy
-						if(key2 != "so-long-and-thanks-for-all-the-fish"){
-							assert.ok(object[key][key2]);
-							assert.equal(typeof object[key][key2], "string");
-						}
-					});
+	describe('#GET /api/getFactorioLocale', function() {
+		it('should get the basegame factorio locale', async function() {
+			let res = await get('/api/getFactorioLocale');
+			let object = res.body;
+			// test that it is actual factorio locale, copied from getFactorioLocale.spec.js
+			assert.equal(typeof object, "object");
+			assert.equal(object["entity-name"]["fish"], "Fish");
+			assert.equal(object["entity-name"]["small-lamp"], "Lamp");
+			Object.keys(object).forEach(key => {
+				// first level of the nested object is always an object
+				assert.equal(typeof object[key], "object");
+
+				Object.keys(object[key]).forEach(key2 => {
+					// second level of the nested object is always a string, nearly always truthy
+					if(key2 != "so-long-and-thanks-for-all-the-fish") {
+						assert.ok(object[key][key2], `${key}.${key2} is falsy (${object[key][key2]})`);
+						assert.equal(typeof object[key][key2], "string");
+					}
 				});
-				done();
 			});
 		});
 	});
 	// describe("#GET /api/")
 	parallel("#GET static website data", function() {
 		this.timeout(6000);
-		it("sends some HTML when accessing /", function(done){
-			this.timeout(6000);
-			request(app).get("/").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /nodes",function(done){
-			this.timeout(6000);
-			request(app).get("/nodes").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /settings",function(done){
-			this.timeout(6000);
-			request(app).get("/settings").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /nodeDetails",function(done){
-			this.timeout(6000);
-			request(app).get("/nodeDetails").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /remoteMap",function(done){
-			this.timeout(6000);
-			request(app).get("/remoteMap").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					// there should be 1 error regarding complaining about me using ES6 modules before they are fully supported
-					assert(result.messages.length === 2, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
+
+		let paths = ["/", "/nodes", "/settings", "/nodeDetails", "/remoteMap"];
+		for (let path of paths) {
+			it(`sends some HTML when accessing ${path}`, () => getValidate(path));
+		}
 	});
 	let persistentMaster = request(app);
-	describe("#POST /api/place", function(){
-		it("adds an itemStack to the masters inventory", function(done){
-			persistentMaster.post("/api/place")
-			.send({
+	describe("#POST /api/place", function() {
+		it("adds an itemStack to the masters inventory", function() {
+			return persistentMaster.post("/api/place").send({
 				name:"steel-plate",
 				count:20,
 				instanceName:"unitTest"
-			})
-			.end(function(err,res){
-				assert(!err)
+			}).set(
+				"X-Access-Token", token
+			).expect(200).then(function(res) {
 				assert.equal(res.text, "success", "something went wrong with the request")
-				done();
 			});
 		});
 	});
-	describe("#GET /api/inventory", function(){
-		it("returns the masters current inventory", function(done){
-			persistentMaster.get("/api/inventory").end(function(err,res){
-				assert.equal(res.statusCode, 200);
+	describe("#GET /api/inventory", function() {
+		it("returns the masters current inventory", function() {
+			return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
 				let inventory = JSON.parse(res.text);
 				assert.equal(typeof inventory, "object", "Inventory should be an object");
 				assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
-				
+
 				let contains20SteelPlate = false;
 				inventory.forEach(itemStack => {
 					if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
 				});
 				assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
-				done();
 			});
 		});
 	});
-	parallel("#POST /api/remove", function(){
-		it("returns an itemStack of how many items were removed", function(done){
-			persistentMaster.post("/api/remove")
-			.send({
+	describe("#POST /api/remove", function() {
+		it("returns an itemStack of how many items were removed", function() {
+			return persistentMaster.post("/api/remove").send({
 				name:"steel-plate",
 				count:10
-			})
-			.end(function(err,res){
-				assert(!err);
-				assert.equal(res.body.count, 10, "Something is wrong with the response, maybe the format changed slightly or you didn't have enough steel?");
-				
-				done();
+			}).set(
+				"X-Access-Token", token
+			).expect(200).then(function(res) {
+				assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
 			});
 		});
-		it("returns an empty itemStack if you try to request addItem or removeItem", function(done){
-			let casesRan = 0;
-			persistentMaster.post("/api/remove")
-			.send({
-				name:"addItem",
-				count:10
-			})
-			.end(function(err,res){
-				assert(!err);
-				assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0");
-				
-				isDone();
-			});
-			
-			persistentMaster.post("/api/remove")
-			.send({
-				name:"removeItem",
-				count:10
-			})
-			.end(function(err,res){
-				assert(!err);
-				assert.equal(res.text, '{"name":"removeItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0");
-				
-				isDone();
-			});
-			
-			function isDone(){
-				if(++casesRan == 2) done();
+		it("returns an empty itemStack if you try to request addItem or removeItem", function() {
+			function remove() {
+				return persistentMaster.post("/api/remove").send({
+					name:"addItem",
+					count:10
+				}).set(
+					"X-Access-Token", token
+				).expect(200).then(function(res) {
+					assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0");
+				});
 			}
+
+			return Promise.all([remove(), remove()]);
 		});
-		it("returns an empty itemStack if you don't have any of the item you request", function(done){
-			persistentMaster.post("/api/remove")
-			.send({
+		it("returns an empty itemStack if you don't have any of the item you request", function() {
+			return persistentMaster.post("/api/remove").send({
 				name:"imaginaryItem",
 				count:999999
-			})
-			.end(function(err,res){
-				assert(!err);//{"name":"imaginaryItem","count":0}
+			}).set(
+				"X-Access-Token", token
+			).expect(200).then(function(res) {
 				assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
 				assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
 				assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
-				
-				done();
 			});
 		});
 	});

static/nav.html

@@ -14,8 +14,8 @@
     <%# Very ugly ejs code to hide the Players button if the playerManager plugin isn`t installed %>
     <%
     sidebarPluginButtonsHtml = "";
-    for(let o in masterPlugins){
-    let plug = masterPlugins[o];
+    for(let o in locals.masterPlugins){
+    let plug = locals.masterPlugins[o];
     if(plug.main
     && plug.main.ui
     && typeof plug.main.ui.sidebar == "object"

static/nodes.html

@@ -40,8 +40,8 @@
                                 </ul>
 							${findInArray("modName", "remoteMap", slave.mods).length ? "<a href='/remoteMap?instanceID="+id+"'>Remote map</a>" : ""}
 						`
-						for(let o in masterPlugins){
-							let plug = masterPlugins[o];
+						for(let o in locals.masterPlugins){
+							let plug = locals.masterPlugins[o];
 							if(plug.main
 							&& plug.main.ui
 							&& typeof plug.main.ui.slaveListing == "object"