From 442472e412418bc8beb78a34e2d90175d0097b28 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Aug 2019 10:06:56 +0200 Subject: [PATCH 1/4] Actually fix masterPlugins visibility The masterPlugins array should not be a global. The ejs templates already have it provided through locals.masterPlugins and plugins get it through onLoadFinish. Make masterPlugins local to master.js again and fix the ejs templates to use locals.masterPlugins. --- master.js | 2 +- static/nav.html | 4 ++-- static/nodes.html | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/master.js b/master.js index db1ef173..38463281 100644 --- a/master.js +++ b/master.js @@ -1012,7 +1012,7 @@ async function getPlugins(){ } -masterPlugins = await getPlugins(); +var masterPlugins = await getPlugins(); io.attach(server); diff --git a/static/nav.html b/static/nav.html index 44f90a62..0787e5d6 100644 --- a/static/nav.html +++ b/static/nav.html @@ -14,8 +14,8 @@ <%# Very ugly ejs code to hide the Players button if the playerManager plugin isn`t installed %> <% sidebarPluginButtonsHtml = ""; - for(let o in masterPlugins){ - let plug = masterPlugins[o]; + for(let o in locals.masterPlugins){ + let plug = locals.masterPlugins[o]; if(plug.main && plug.main.ui && typeof plug.main.ui.sidebar == "object" diff --git a/static/nodes.html b/static/nodes.html index 6fa29a3e..e544dafe 100644 --- a/static/nodes.html +++ b/static/nodes.html @@ -40,8 +40,8 @@ ${findInArray("modName", "remoteMap", slave.mods).length ? "Remote map" : ""} ` - for(let o in masterPlugins){ - let plug = masterPlugins[o]; + for(let o in locals.masterPlugins){ + let plug = locals.masterPlugins[o]; if(plug.main && plug.main.ui && typeof plug.main.ui.slaveListing == "object" From a828f50e57ea3620861edd315b6e91e203a36819 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 2 Aug 2019 22:23:04 +0200 Subject: [PATCH 2/4] Refactor master.spec.js Ensure exceptions are properly passed down to the unit test framework with promises, check the return status of requests, use config to generate an access token, reduce code duplication and reformat to be consistent. --- master.spec.js | 215 ++++++++++++++++++------------------------------- 1 file changed, 77 insertions(+), 138 deletions(-) diff --git a/master.spec.js b/master.spec.js index 56d8ccc5..7ad872d5 100644 --- a/master.spec.js +++ b/master.spec.js @@ -1,183 +1,122 @@ -var app = require('./master.js'), - assert = require('assert'), - request = require('supertest'); +const app = require('./master.js'); +const config = require('./config.json'); +const assert = require('assert').strict; +const request = require('supertest'); const validateHTML = require('html5-validator'); -var parallel = require('mocha.parallel'); +const parallel = require('mocha.parallel'); +const jwt = require("jsonwebtoken"); +const token = + jwt.sign( { id: "api" }, config.masterAuthSecret, { expiresIn: 600 }); + +async function get(path) { + return await request(app).get(path).expect(200); +} + +async function getValidate(path) { + let res = await get(path); + let validation = await validateHTML(res.text); + let filtered = validation.messages.filter(msg => msg.type !== "info"); + assert(filtered.length === 0, "there are HTML errors on the page, please fix: "+JSON.stringify(validation.messages, null, 4)); +} + describe('Master server endpoint testing', function() { - describe('#GET /api/getFactorioLocale', function() { - it('should get the basegame factorio locale', function(done) { - request(app).get('/api/getFactorioLocale').end(function(err, res) { - assert.equal(res.statusCode, 200); - - let object = res.body; - - // test that it is actual factorio locale, copied from getFactorioLocale.spec.js - assert.equal(typeof object, "object"); - assert.equal(object["entity-name"]["fish"], "Fish"); - assert.equal(object["entity-name"]["small-lamp"], "Lamp"); - Object.keys(object).forEach(key => { - // first level of the nested object is always an object - assert.equal(typeof object[key], "object"); - - Object.keys(object[key]).forEach(key2 => { - // second level of the nested object is always a string, nearly always truthy - if(key2 != "so-long-and-thanks-for-all-the-fish"){ - assert.ok(object[key][key2]); - assert.equal(typeof object[key][key2], "string"); - } - }); + describe('#GET /api/getFactorioLocale', function() { + it('should get the basegame factorio locale', async function() { + let res = await get('/api/getFactorioLocale'); + let object = res.body; + // test that it is actual factorio locale, copied from getFactorioLocale.spec.js + assert.equal(typeof object, "object"); + assert.equal(object["entity-name"]["fish"], "Fish"); + assert.equal(object["entity-name"]["small-lamp"], "Lamp"); + Object.keys(object).forEach(key => { + // first level of the nested object is always an object + assert.equal(typeof object[key], "object"); + + Object.keys(object[key]).forEach(key2 => { + // second level of the nested object is always a string, nearly always truthy + if(key2 != "so-long-and-thanks-for-all-the-fish") { + assert.ok(object[key][key2], `${key}.${key2} is falsy (${object[key][key2]})`); + assert.equal(typeof object[key][key2], "string"); + } }); - done(); }); }); }); // describe("#GET /api/") parallel("#GET static website data", function() { this.timeout(6000); - it("sends some HTML when accessing /", function(done){ - this.timeout(6000); - request(app).get("/").end(function(err,res){ - assert.equal(res.statusCode, 200); - validateHTML(res.text).then(result => { - assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages)); - done(); - }); - }); - }); - it("sends some HTML when accessing /nodes",function(done){ - this.timeout(6000); - request(app).get("/nodes").end(function(err,res){ - assert.equal(res.statusCode, 200); - validateHTML(res.text).then(result => { - assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages)); - done(); - }); - }); - }); - it("sends some HTML when accessing /settings",function(done){ - this.timeout(6000); - request(app).get("/settings").end(function(err,res){ - assert.equal(res.statusCode, 200); - validateHTML(res.text).then(result => { - assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages)); - done(); - }); - }); - }); - it("sends some HTML when accessing /nodeDetails",function(done){ - this.timeout(6000); - request(app).get("/nodeDetails").end(function(err,res){ - assert.equal(res.statusCode, 200); - validateHTML(res.text).then(result => { - assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages)); - done(); - }); - }); - }); - it("sends some HTML when accessing /remoteMap",function(done){ - this.timeout(6000); - request(app).get("/remoteMap").end(function(err,res){ - assert.equal(res.statusCode, 200); - validateHTML(res.text).then(result => { - // there should be 1 error regarding complaining about me using ES6 modules before they are fully supported - assert(result.messages.length === 2, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages)); - done(); - }); - }); - }); + + let paths = ["/", "/nodes", "/settings", "/nodeDetails", "/remoteMap"]; + for (let path of paths) { + it(`sends some HTML when accessing ${path}`, () => getValidate(path)); + } }); let persistentMaster = request(app); - describe("#POST /api/place", function(){ - it("adds an itemStack to the masters inventory", function(done){ - persistentMaster.post("/api/place") - .send({ + describe("#POST /api/place", function() { + it("adds an itemStack to the masters inventory", function() { + return persistentMaster.post("/api/place").send({ name:"steel-plate", count:20, instanceName:"unitTest" - }) - .end(function(err,res){ - assert(!err) + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { assert.equal(res.text, "success", "something went wrong with the request") - done(); }); }); }); - describe("#GET /api/inventory", function(){ - it("returns the masters current inventory", function(done){ - persistentMaster.get("/api/inventory").end(function(err,res){ - assert.equal(res.statusCode, 200); + describe("#GET /api/inventory", function() { + it("returns the masters current inventory", function() { + return persistentMaster.get("/api/inventory").expect(200).then(function(res) { let inventory = JSON.parse(res.text); assert.equal(typeof inventory, "object", "Inventory should be an object"); assert(inventory.length >= 1, "There should be at least 1 entry in the inventory"); - + let contains20SteelPlate = false; inventory.forEach(itemStack => { if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true; }); assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory") - done(); }); }); }); - parallel("#POST /api/remove", function(){ - it("returns an itemStack of how many items were removed", function(done){ - persistentMaster.post("/api/remove") - .send({ + describe("#POST /api/remove", function() { + it("returns an itemStack of how many items were removed", function() { + return persistentMaster.post("/api/remove").send({ name:"steel-plate", count:10 - }) - .end(function(err,res){ - assert(!err); - assert.equal(res.body.count, 10, "Something is wrong with the response, maybe the format changed slightly or you didn't have enough steel?"); - - done(); + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`); }); }); - it("returns an empty itemStack if you try to request addItem or removeItem", function(done){ - let casesRan = 0; - persistentMaster.post("/api/remove") - .send({ - name:"addItem", - count:10 - }) - .end(function(err,res){ - assert(!err); - assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0"); - - isDone(); - }); - - persistentMaster.post("/api/remove") - .send({ - name:"removeItem", - count:10 - }) - .end(function(err,res){ - assert(!err); - assert.equal(res.text, '{"name":"removeItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0"); - - isDone(); - }); - - function isDone(){ - if(++casesRan == 2) done(); + it("returns an empty itemStack if you try to request addItem or removeItem", function() { + function remove() { + return persistentMaster.post("/api/remove").send({ + name:"addItem", + count:10 + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0"); + }); } + + return Promise.all([remove(), remove()]); }); - it("returns an empty itemStack if you don't have any of the item you request", function(done){ - persistentMaster.post("/api/remove") - .send({ + it("returns an empty itemStack if you don't have any of the item you request", function() { + return persistentMaster.post("/api/remove").send({ name:"imaginaryItem", count:999999 - }) - .end(function(err,res){ - assert(!err);//{"name":"imaginaryItem","count":0} + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for"); assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere"); assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object"); - - done(); }); }); }); From a3fc1dda8cdd44b2fac87ebad17e1bd973a3f596 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 3 Aug 2019 12:14:04 +0200 Subject: [PATCH 3/4] Refactor master.js Move server initialization code to startServer and call it only when master.js is being runned directly. Also refactor the server listen step to work properly when https is configured and only listen after plugins have completed loading. Remove the async function wrapper around the master.js code. This is just dumb and makes the code untestable. --- master.js | 128 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 57 deletions(-) diff --git a/master.js b/master.js index 38463281..e0a16dad 100644 --- a/master.js +++ b/master.js @@ -14,15 +14,6 @@ node master.js // const updater = require("./updater.js"); // updater.update().then(console.log); -(async ()=>{ - -// Set the process title, shows up as the title of the CMD window on windows -// and as the process name in ps/top on linux. -process.title = "clusterioMaster"; - -// add better stack traces on promise rejection -process.on('unhandledRejection', r => console.log(r)); - // configgy stuff debug = false; @@ -55,24 +46,6 @@ const getFactorioLocale = require("lib/getFactorioLocale"); // homemade express middleware for token auth const authenticate = require("lib/authenticate")(config); -/** Sync */ -function randomStringAsBase64Url(size) { - return base64url(crypto.randomBytes(size)); -} -if (!fs.existsSync("secret-api-token.txt")) { - config.masterAuthSecret = randomStringAsBase64Url(256); - fs.writeFileSync("config.json",JSON.stringify(config, null, 4)); - fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { - expiresIn: 86400*365 // expires in 1 year - })); - console.log("Generated new master authentication private key!"); - process.exit(0); -} -// write an auth token to file -fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { - expiresIn: 86400*365 // expires in 1 year -})); - const express = require("express"); const compression = require('compression'); const cookieParser = require('cookie-parser'); @@ -174,9 +147,6 @@ const prometheusMasterInventoryGauge = new Prometheus.Gauge({ help: 'Amount of items stored on master', labelNames: ["itemName"], }); -setInterval(()=>{ - fs.writeFileSync("database/items.json", JSON.stringify(db.items)); -},config.autosaveInterval || 60000); /** GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format. @memberof clusterioMaster @@ -266,8 +236,6 @@ db.items.removeItem = function(object) { }; // store slaves and inventory in a .json full of JSON data -process.on('SIGINT', shutdown); // ctrl + c -process.on('SIGHUP', shutdown); // terminal closed async function shutdown() { console.log('Ctrl-C...'); let exitStartTime = Date.now(); @@ -764,28 +732,9 @@ app.get("/api/getFactorioLocale", function(req,res){ res.send(factorioLocale); }); }); -if(args.sslPort || config.sslPort){ - try{ - var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt')); - var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key')); - - httpsServer = require("https").createServer({ - key: privateKey, - cert: certificate - }, app).listen(args.sslPort || config.sslPort); - } catch(e){ - console.error(e) - } -} -if(args.masterPort || config.masterPort){ - server = require("http").Server(app); - server.listen(args.masterPort || config.masterPort || 8080, function () { - console.log("Listening on port %s...", server.address().port); - }); -} /* Websockets for remoteMap */ -var io = require("socket.io")(); +var io = require("socket.io")({}); const ioMetrics = require("socket.io-prometheus"); ioMetrics(io); @@ -975,8 +924,14 @@ io.on('connection', function (socket) { }); // handle plugins on the master -async function getPlugins(){ +var masterPlugins = []; +async function pluginManagement(){ let startPluginLoad = Date.now(); + masterPlugins = await getPlugins(); + console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms"); +} + +async function getPlugins(){ const pluginManager = require("lib/manager/pluginManager.js")(config); let plugins = []; let pluginsToLoad = await pluginManager.getPlugins(); @@ -1007,15 +962,74 @@ async function getPlugins(){ let plugin = plugins[i]; if(plugin.main.onLoadFinish && typeof plugin.main.onLoadFinish == "function") await plugin.main.onLoadFinish({plugins}); } - console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms"); return plugins; } +async function startServer() { + // Set the process title, shows up as the title of the CMD window on windows + // and as the process name in ps/top on linux. + process.title = "clusterioMaster"; -var masterPlugins = await getPlugins(); + // add better stack traces on promise rejection + process.on('unhandledRejection', r => console.log(r)); + /** Sync */ + function randomStringAsBase64Url(size) { + return base64url(crypto.randomBytes(size)); + } -io.attach(server); + if (!fs.existsSync("secret-api-token.txt")) { + config.masterAuthSecret = randomStringAsBase64Url(256); + fs.writeFileSync("config.json",JSON.stringify(config, null, 4)); + fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { + expiresIn: 86400*365 // expires in 1 year + })); + console.log("Generated new master authentication private key!"); + process.exit(0); + } + // write an auth token to file + fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { + expiresIn: 86400*365 // expires in 1 year + })); + setInterval(()=>{ + fs.writeFileSync("database/items.json", JSON.stringify(db.items)); + },config.autosaveInterval || 60000); + process.on('SIGINT', shutdown); // ctrl + c + process.on('SIGHUP', shutdown); // terminal closed + + // Make sure we're actually going to listen on a port + let httpPort = args.masterPort || config.masterPort; + let httpsPort = args.sslPort || config.sslPort; + if (!httpPort && !httpsPort) { + console.error("Error: at least one of httpPort and sslPort must be configured"); + process.exit(1); + } + + // Load plugins + await pluginManagement(); + + // Only start listening for connections after all plugins have loaded + if (httpPort) { + let httpServer = require("http").Server(app); + httpServer.listen(httpPort); + console.log("Listening for HTTP on port %s...", httpServer.address().port); + } + + if (httpsPort) { + var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt')); + var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key')); + + let httpsServer = require("https").createServer({ + key: privateKey, + cert: certificate + }, app) + httpsServer.listen(httpsPort); + console.log("Listening for HTTPS on port %s...", httpsServer.address().port); + } +} module.exports = app; -})(); + +if (module === require.main) { + startServer(); +} From 8898e923de894ecc912488de589624928d40abec Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 21 Aug 2019 01:21:23 +0200 Subject: [PATCH 4/4] Fix travis test integration Add config.json.test as the the config for running the tests on travis and add script command to copy it to config.json there. Should fix the unit test for master failing at loading the config. --- .travis.yml | 1 + config.json.test | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 config.json.test diff --git a/.travis.yml b/.travis.yml index 93725d69..9b88efc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,6 @@ before_install: - sudo apt-get install -y curl git tar - curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 - tar -xvf factorio.tar.gz + - cp -n config.json.test config.json script: - npm run cover \ No newline at end of file diff --git a/config.json.test b/config.json.test new file mode 100644 index 00000000..3ea35cbd --- /dev/null +++ b/config.json.test @@ -0,0 +1,47 @@ +{ + "factorioDirectory": "./factorio", + "instanceDirectory": "./instances", + "databaseDirectory": "./database", + + "publicIP": "localhost", + + "masterIP": "localhost", + + "masterPort": 8080, + + "sslPort": 443, + "sslCert": "database/certificates/cert.crt", + "sslPrivKey": "database/certificates/cert.key", + "masterAuthToken": "", + "masterAuthSecret": "TestSecretDoNotUse", + + "username": "", + "token": "", + "factorio_version": "0.16.41", + + "verify_user_identity": true, + "auto_pause": false, + "allow_commands": "admins-only", + "game_password": "", + "description": "This server runs Clusterio", + "visibility": { + "public": true, + "lan": true + }, + + "itemStats": { + "maxEntries":60, + "entriesPerSecond": 0.01666666667 + }, + "logItemTransfers": false, + "disableFairItemDistribution": true, + "useNeuralNetDoleDivider": false, + "uploadModsToMaster": true, + "msBetweenCommands": 10, + "allowRemoteCommandExecution": true, + "enableCrossServerShout": true, + "mirrorAllChat": false, + "autosaveInterval": 600000, + "disablePrometheusPushgateway": false, + "disableImportsOfEverythingExceptElectricity": false +}