From 0b0b2bc1a91d387c01e5d2d275d6a76509d87c36 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Wed, 25 Apr 2018 12:44:31 +0200 Subject: [PATCH 01/92] Token auth for all destructive endpoints --- client.js | 29 +++++++++++++++++++++++------ lib/authenticate.js | 15 +++++++++++++++ master.js | 42 +++++++++++++++--------------------------- 3 files changed, 53 insertions(+), 33 deletions(-) create mode 100644 lib/authenticate.js diff --git a/client.js b/client.js index 839635f9..b4b3d8ec 100644 --- a/client.js +++ b/client.js @@ -42,6 +42,19 @@ const command = process.argv[2]; // and as the process name in ps/top on linux. process.title = "clusterioClient "+instance; +// make sure we have the master access token (can't write to master without it since clusterio 2.0) +if(!config.masterAuthToken || typeof config.masterAuthToken !== "string"){ + console.log("ERROR invalid config!"); + console.log("Master server now needs an access token for write operations. As clusterio slaves depends \ + upon this, please add your token to config.js in the field named masterAuthToken. \ + You can retrieve your auth token from the master in secret-api-token.txt after running it once."); +} +const needleOptionsWithTokenAuthHeader = { + headers: { + 'x-access-token': config.masterAuthToken + }, +}; + var instanceInfo = {}; // function to handle sending commands into the game @@ -604,7 +617,7 @@ function instanceManagement() { } else { payload.mac = mac console.log("Registered our precense with master "+config.masterIP+" at " + payload.time); - needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, function (err, response, body) { + needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { if (err && err.code != "ECONNRESET"){ console.error("We got problems, something went wrong when contacting master"); console.error(err); @@ -637,7 +650,7 @@ function instanceManagement() { modName: modHashes[i].modName, hash: modHashes[i].hash, } - needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, function (err, response, body) { + needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { if(err) console.log(new Error("Unable to contact master server! Please check your config.json.")); if(response && body && body == "found") { console.log("master has mod"); @@ -647,7 +660,11 @@ function instanceManagement() { console.log("Sending mod: " + mod); // Send mods master says it wants // response.body is a string which is a modName.zip - var req = request.post("http://"+config.masterIP + ":" + config.masterPort + '/api/uploadMod', function (err, resp, body) { + var req = request.post({url: "http://"+config.masterIP + ":" + config.masterPort + '/api/uploadMod', + headers: { + "x-access-token": config.masterAuthToken, + }, + }, function (err, resp, body) { if (err) { console.log(new Error("Unable to contact master server! Please check your config.json.")); } else { @@ -705,7 +722,7 @@ function instanceManagement() { } } console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]); - needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: instanceconfig.unique,data: payload}, function (err, response, body) { + needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: instanceconfig.unique,data: payload}, needleOptionsWithTokenAuthHeader, function (err, response, body) { // we did it, keep going }); } @@ -737,7 +754,7 @@ function instanceManagement() { count: g[1], instanceName: instance, // name of instance instanceID: instanceconfig.unique, // a hash computed from the randomly generated rcon password - }, function (err, resp, body) { + }, needleOptionsWithTokenAuthHeader, function (err, resp, body) { if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]); if(config.logItemTransfers){ if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`); @@ -794,7 +811,7 @@ function instanceManagement() { // request our items, one item at a time for (let i = 0; i < Object.keys(preparedPackage).length; i++) { // console.log(preparedPackage[Object.keys(preparedPackage)[i]]); - needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], function (err, response, body) { + needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) { if (response && response.body && typeof response.body == "object") { // buffer confirmed orders confirmedOrders[confirmedOrders.length] = { diff --git a/lib/authenticate.js b/lib/authenticate.js new file mode 100644 index 00000000..b13404c5 --- /dev/null +++ b/lib/authenticate.js @@ -0,0 +1,15 @@ +const jwt = require('jsonwebtoken'); + +const config = require("./../config"); + +var middleware = function (req, res, next) { + var token = req.headers['x-access-token']; + if(!token) return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path}); + + jwt.verify(token, config.masterAuthSecret, function(err, decoded) { + if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path}); + next(); + }); +} + +module.exports = {middleware}; diff --git a/master.js b/master.js index d8cb67da..f7831405 100644 --- a/master.js +++ b/master.js @@ -28,6 +28,9 @@ var config = require('./config'); const getFactorioLocale = require("./lib/getFactorioLocale"); const stringUtils = require("./lib/stringUtils"); +// homemade express middleware for token auth +const authenticate = require("./lib/authenticate"); + // Library for create folder recursively if it does not exist const mkdirp = require("mkdirp"); mkdirp.sync("./database"); @@ -37,8 +40,13 @@ const deepmerge = require("deepmerge"); const path = require("path"); const fs = require("fs"); const nedb = require("nedb"); + const jwt = require('jsonwebtoken'); const bcrypt = require('bcryptjs'); +// write an auth token to file +fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { + expiresIn: 86400*365 // expires in 1 year +})); const express = require("express"); const ejs = require("ejs"); @@ -224,7 +232,7 @@ POST World ID management. Slaves post here to tell the server they exist @instance @alias /api/getID */ -app.post("/api/getID", function(req,res) { +app.post("/api/getID", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); // request.body should be an object @@ -253,7 +261,7 @@ POST Allows you to add metadata related to slaves for other tools, like owner da @instance @alias /api/editSlaveMeta */ -app.post("/api/editSlaveMeta", function(req,res) { +app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); // request.body should be an object @@ -292,22 +300,6 @@ app.post("/api/getSlaveMeta", function (req, res) { res.send('{"INVALID REQUEST":1}'); } }); -// TODO: Remove deprecated function, also update researchSync to not use it -/** -POST DEPRECATED Get metadata from all slaves. This is already covered by /api/slaves, please use that one instead. -@memberof clusterioMaster -@instance -@alias /api/getSlavesMeta -*/ -app.post("/api/getSlavesMeta", function (req, res) { - if(req.body.password){ - let metas = {}; - res.send(JSON.stringify(Object.keys(slaves).map(key => metas[key] = slaves[key].meta))); - } else { - res.status(400); - res.send('{"INVALID REQUEST":1}'); - } -}); // mod management // should handle uploading and checking if mods are uploaded /** @@ -341,7 +333,7 @@ POST endpoint for uploading mods to the master server. Required for automatic mo @instance @alias /api/uploadMod */ -app.post("/api/uploadMod", function(req,res) { +app.post("/api/uploadMod", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); if (!req.files) { @@ -398,7 +390,7 @@ POST endpoint for storing items in master's inventory. @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name] @returns {string} status either "success" or "failure" */ -app.post("/api/place", function(req, res) { +app.post("/api/place", authenticate.middleware, function(req, res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); res.header("Access-Control-Allow-Origin", "*"); @@ -465,7 +457,7 @@ POST endpoint to remove items from DB when client orders items. @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages. */ _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. -app.post("/api/remove", function(req, res) { +app.post("/api/remove", authenticate.middleware, function(req, res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings @@ -540,7 +532,7 @@ Gives no response @param {object} circuitFrame @param {number} circuitFrame.time */ -app.post("/api/setSignal", function(req,res) { +app.post("/api/setSignal", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); if(typeof req.body == "object" && req.body.time){ @@ -625,7 +617,7 @@ course the timeSeries data. @param {object} JSON {timestamp: Date.now(), instanceID: "string", data: {"item": number}} @returns {string} failure */ -app.post("/api/logStats", function(req,res) { +app.post("/api/logStats", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); if(typeof req.body == "object" && req.body.instanceID && req.body.timestamp && req.body.data) { @@ -793,10 +785,6 @@ Requires x-access-token header to be set. Find you api token in secret-api-token @param {object} JSON {instanceID:19412312, command:"/c game.print('hello')"} @returns {object} Status {auth: bool, message: "Informative error", data:{}} */ -// write an auth token to file -fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { - expiresIn: 86400*365 // expires in 1 year -})); app.post("/api/runCommand", (req,res) => { var token = req.headers['x-access-token']; if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' }); From a26189947df9f69c4026515e003f7aadd8aa947f Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Wed, 25 Apr 2018 12:50:33 +0200 Subject: [PATCH 02/92] Token auth in inventoryImports --- sharedPlugins/inventoryImports/interfacingFunctions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sharedPlugins/inventoryImports/interfacingFunctions.js b/sharedPlugins/inventoryImports/interfacingFunctions.js index 6a9469c6..2b4aaca0 100644 --- a/sharedPlugins/inventoryImports/interfacingFunctions.js +++ b/sharedPlugins/inventoryImports/interfacingFunctions.js @@ -76,7 +76,7 @@ function handleInventory(json, config, callback){ } //console.log("Requesting: "+JSON.stringify(request)); // request the difference from master - needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', request, {json:true}, function (err, response, body) { + needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', request, {json:true,headers: {'x-access-token': config.masterAuthToken}}, function (err, response, body) { if (!err && response && response.body && (typeof response.body == "object" || isJson(response.body))) { // confirmed orders, already removed from master inventory //response.body.name: response.body.count @@ -103,7 +103,7 @@ function handleInventory(json, config, callback){ for(let i = 0; i < stackArray.length; i++){ if(stackArray[i].count > 0){ // console.log("Returning overflow: " + JSON.stringify(stackArray[i])); - needle.post(config.masterIP + ":" + config.masterPort + '/api/place', stackArray[i], function (err, resp, body) { + needle.post(config.masterIP + ":" + config.masterPort + '/api/place', stackArray[i], {headers: {'x-access-token': config.masterAuthToken}}, function (err, resp, body) { // console.log(body); }); } From 20e66cbd40114d86232efec38dad7bd1390ca972 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Wed, 25 Apr 2018 12:55:03 +0200 Subject: [PATCH 03/92] Token auth in researchSync and UPSdisplay --- sharedPlugins/UPSdisplay/index.js | 2 +- sharedPlugins/researchSync/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sharedPlugins/UPSdisplay/index.js b/sharedPlugins/UPSdisplay/index.js index a226b38a..5618783e 100644 --- a/sharedPlugins/UPSdisplay/index.js +++ b/sharedPlugins/UPSdisplay/index.js @@ -17,7 +17,7 @@ class UPSdisplay { let UPS = Math.round(ticksInPeriod / (timePeriod/1000)); // console.log("UPS: " + UPS); try{ - needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS}}, function(err, resp) { + needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS}}, {headers: {'x-access-token': this.config.masterAuthToken}}, function(err, resp) { // success? }); } catch (err){ diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js index bdae8729..e198e310 100644 --- a/sharedPlugins/researchSync/index.js +++ b/sharedPlugins/researchSync/index.js @@ -83,7 +83,7 @@ class ResearchSync { instanceID: this.config.unique, password: this.config.clientPassword, meta: {research: this.research} - }, function (err, resp) { + }, {headers: {'x-access-token': this.config.masterAuthToken}}, function (err, resp) { // success? }); }); From ebf2fd22042afb33454f2b74f9af6fda4869519d Mon Sep 17 00:00:00 2001 From: psihius Date: Thu, 26 Apr 2018 15:40:47 +0300 Subject: [PATCH 04/92] Added cluster wide command execution + fixed some checks failing due to type mismatch --- master.js | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/master.js b/master.js index f7831405..d57154d3 100644 --- a/master.js +++ b/master.js @@ -793,19 +793,35 @@ app.post("/api/runCommand", (req,res) => { if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' }); // validate request.body - let body = req.body; - if(body.instanceID - && Object.keys(wsSlaves).includes(body.instanceID) - && body.command - && typeof body.command == "string" - && body.command[0] == "/"){ - // execute command - wsSlaves[body.instanceID].runCommand(body.command, data => { - res.status(200).send({auth: true, message: "success", data}); - }); - } else { - res.status(400).send({auth: true, message: "Error: invalid request.body"}); - } + let body = req.body; + + if ( + body.broadcast + && body.command + && typeof body.command == "string" + && body.command[0] == "/") + { + let instanceResponses = []; + for (let instanceID in wsSlaves) { + // skip loop if the property is from prototype + if (!wsSlaves.hasOwnProperty(instanceID)) continue; + wsSlaves[instanceID].runCommand(body.command); + } + res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"}); + } else if ( + body.instanceID + && wsSlaves[body.instanceID] + && body.command + && typeof body.command == "string" + && body.command[0] == "/") + { + // execute command + wsSlaves[body.instanceID].runCommand(body.command, data => { + res.status(200).send({auth: true, message: "success", data}); + }); + } else { + res.status(400).send({auth: true, message: "Error: invalid request.body"}); + } }); }); /** From ce0deef83bb45dca02727543c1b543dab28e5b47 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Thu, 26 Apr 2018 18:33:29 +0200 Subject: [PATCH 05/92] Websockets for listening to factorio logs and global chat plugin You can now !shout things across the cluster --- .gitignore | 1 + client.js | 4 ++++ master.js | 16 +++++++++++++++- sharedPlugins/globalChat/config.js | 10 ++++++++++ sharedPlugins/globalChat/index.js | 24 ++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 sharedPlugins/globalChat/config.js create mode 100644 sharedPlugins/globalChat/index.js diff --git a/.gitignore b/.gitignore index 3949e538..0270b57f 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,4 @@ sharedPlugins/* !sharedPlugins/remoteMap/ !sharedPlugins/remoteCommands/ !sharedPlugins/researchSync/ +!sharedPlugins/globalChat/ diff --git a/client.js b/client.js index b4b3d8ec..90d0b351 100644 --- a/client.js +++ b/client.js @@ -469,6 +469,10 @@ function instanceManagement() { }, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat. socket, // socket.io connection to master (and ES6 destructuring, yay) }); + if(plugins[I].factorioOutput && typeof plugins[I].factorioOutput === "function"){ + // when factorio logs a line, send it to the plugin. This includes things like autosaves, chat, errors etc + serverprocess.stdout.on("data", data => plugins[I].factorioOutput(data.toString())); + } if(pluginConfig.scriptOutputFileSubscription && typeof pluginConfig.scriptOutputFileSubscription == "string"){ if(global.subscribedFiles[pluginConfig.scriptOutputFileSubscription]) { // please choose a unique file to subscribe to. If you need plugins to share this interface, set up a direct communication diff --git a/master.js b/master.js index d57154d3..4c110aeb 100644 --- a/master.js +++ b/master.js @@ -940,6 +940,16 @@ class wsSlave { delete this.commandsWaitingForReturn[resp.commandID]; } }); + + this.socket.on("gameChat", data => { + if(typeof data === "object"){ + data.timestamp = Date.now(); + if(!global.wsChatRecievers) global.wsChatRecievers = []; + global.wsChatRecievers.forEach(socket => { + socket.emit("gameChat", data); + }); + } + }); } runCommand(command, callback){ let commandID = Math.random().toString(); @@ -988,7 +998,11 @@ io.on('connection', function (socket) { console.log("SOCKET | Created new wsSlave: "+ data.instanceID); } }); - + socket.on("registerChatReciever", function(data){ + prometheusWsUsageCounter.labels("registerChatReciever").inc(); + if(!global.wsChatRecievers) global.wsChatRecievers = []; + global.wsChatRecievers.push(socket); + }); }); module.exports = app; diff --git a/sharedPlugins/globalChat/config.js b/sharedPlugins/globalChat/config.js new file mode 100644 index 00000000..ca394648 --- /dev/null +++ b/sharedPlugins/globalChat/config.js @@ -0,0 +1,10 @@ +/* + Clusterio plugin to allow for chat between instances. +*/ +module.exports = { + // Name of package. For display somewhere I guess. + name: "globalChat", + version: "1.0.0", + binary: "nodePackage", + description: "Clusterio plugin to allow for chat between instances.", +} \ No newline at end of file diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js new file mode 100644 index 00000000..283a6184 --- /dev/null +++ b/sharedPlugins/globalChat/index.js @@ -0,0 +1,24 @@ +module.exports = class remoteCommands { + constructor(mergedConfig, messageInterface, extras){ + this.messageInterface = messageInterface; + this.config = mergedConfig; + this.socket = extras.socket; + + this.socket.emit("registerChatReciever"); + this.socket.on("gameChat", data => { + if(data.instanceID && data.instanceID != this.config.unique.toString()){ + if(data.data.includes("[CHAT]") && data.data.toLowerCase().includes("!shout")){ + let chatMessage = "["+data.instanceID+"]"+ data.data.trim().split(" ").shift().shift().shift().join(" "); + this.messageInterface("/c game.print('"+chatMessage+"')"); + } + } + }); + } + factorioOutput(data){ + // console.log("GOT CHAT!"+data) + this.socket.emit("gameChat", { + instanceID: this.config.unique.toString(), + data, + }); + } +} From 61e944b061d0ff4e7b6a8ee7bec2ece60756cbd3 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Thu, 26 Apr 2018 18:53:06 +0200 Subject: [PATCH 06/92] Attempted to add https support --- master.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/master.js b/master.js index 4c110aeb..3d9376ae 100644 --- a/master.js +++ b/master.js @@ -40,6 +40,7 @@ const deepmerge = require("deepmerge"); const path = require("path"); const fs = require("fs"); const nedb = require("nedb"); +const https = require("https"); const jwt = require('jsonwebtoken'); const bcrypt = require('bcryptjs'); @@ -840,10 +841,21 @@ app.get("/api/getFactorioLocale", function(req,res){ httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); }); -var server = require("http").Server(app); -server.listen(config.masterPort || 8080, function () { - console.log("Listening on port %s...", server.address().port); -}); +if(config.masterPort == 443){ + var privateKey = fs.readFileSync( 'privatekey.pem' ); + var certificate = fs.readFileSync( 'certificate.pem' ); + + server = https.createServer({ + key: privateKey, + cert: certificate + }, app).listen(443); +} else { + server = require("http").Server(app); + server.listen(config.masterPort || 8080, function () { + console.log("Listening on port %s...", server.address().port); + }); +} + /* Websockets for remoteMap */ var io = require("socket.io")(server); var slaveMappers = {}; From 2e1f7b7f6daff0f5eedfb2eb519c3728428fec5b Mon Sep 17 00:00:00 2001 From: Arvids Godjuks Date: Thu, 26 Apr 2018 23:58:26 +0300 Subject: [PATCH 07/92] README.md update: adjusted command Adjusted pm2 start command to have correct instance names in the process list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a67c918..2c50731b 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ To create a new instance (its own save, set of mods and config files) To launch an instance with pm2 - pm2 start --name slave client.js -- start [instancename] + pm2 start --name [instancename] client.js -- start [instancename] use `nano config.json` to change settings. From 1b00fbb328a68857b493bc73e8343807020e2663 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 27 Apr 2018 06:22:32 +0200 Subject: [PATCH 08/92] Fixed crash when master was not there and we still got some sort of response --- client.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/client.js b/client.js index 90d0b351..3f0db6e3 100644 --- a/client.js +++ b/client.js @@ -847,19 +847,19 @@ function instanceManagement() { if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver try { var inventory = JSON.parse(response.body); + var inventoryFrame = {}; + for (let i = 0; i < inventory.length; i++) { + inventoryFrame[inventory[i].name] = Number(inventory[i].count); + if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){ + inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice + } + } + inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); + // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); + messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); } catch (e){ console.log(e); } - var inventoryFrame = {}; - for (let i = 0; i < inventory.length; i++) { - inventoryFrame[inventory[i].name] = Number(inventory[i].count); - if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){ - inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice - } - } - inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); - // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); } }); }, 1000); From dc19457471bed40270572b95b08c50c78d22cc72 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 27 Apr 2018 21:23:36 +0200 Subject: [PATCH 09/92] Update to master to support server wide alerts --- master.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/master.js b/master.js index 4c110aeb..b23a721b 100644 --- a/master.js +++ b/master.js @@ -950,6 +950,16 @@ class wsSlave { }); } }); + + this.socket.on("alert", alert => { + if(typeof alert === "object"){ + alert.timestamp = Date.now(); + if(!global.wsAlertRecievers) global.wsAlertRecievers = []; + global.wsAlertRecievers.forEach(socket => { + socket.emit("alert", alert); + }); + } + }); } runCommand(command, callback){ let commandID = Math.random().toString(); @@ -1003,6 +1013,11 @@ io.on('connection', function (socket) { if(!global.wsChatRecievers) global.wsChatRecievers = []; global.wsChatRecievers.push(socket); }); + socket.on("registerAlertReciever", function(data){ + prometheusWsUsageCounter.labels("registerAlertReciever").inc(); + if(!global.wsAlertRecievers) global.wsAlertRecievers = []; + global.wsAlertRecievers.push(socket); + } }); module.exports = app; From 1bd87c4cc93297c75dd7b7ce6dd0cb87ace224d7 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 27 Apr 2018 21:58:03 +0200 Subject: [PATCH 10/92] Fixed crash on !shout --- client.js | 4 +++- sharedPlugins/globalChat/index.js | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 3f0db6e3..5247ae95 100644 --- a/client.js +++ b/client.js @@ -504,7 +504,9 @@ function instanceManagement() { } for(let i = 0; i < stuff.length; i++) { if(stuff[i] && !stuff[i].includes('\u0000\u0000')) { - plugins[I].scriptOutput(stuff[i]); + try{ + plugins[I].scriptOutput(stuff[i]); + }catch(e){console.log(e)} } } }, diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index 283a6184..aa6cf3f9 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -8,7 +8,12 @@ module.exports = class remoteCommands { this.socket.on("gameChat", data => { if(data.instanceID && data.instanceID != this.config.unique.toString()){ if(data.data.includes("[CHAT]") && data.data.toLowerCase().includes("!shout")){ - let chatMessage = "["+data.instanceID+"]"+ data.data.trim().split(" ").shift().shift().shift().join(" "); + let message = data.data.trim().split(" "); + message.shift(); + message.shift(); + message.shift(); + message.join(" "); + let chatMessage = "["+data.instanceID+"]"+ message; this.messageInterface("/c game.print('"+chatMessage+"')"); } } From e38aed08bf9a42340c6ce6ec0b7e4b77df9e67e6 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 28 Apr 2018 02:19:42 +0200 Subject: [PATCH 11/92] Sanitizing !shout chat --- master.js | 2 +- sharedPlugins/globalChat/index.js | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/master.js b/master.js index b23a721b..a36d6429 100644 --- a/master.js +++ b/master.js @@ -1017,7 +1017,7 @@ io.on('connection', function (socket) { prometheusWsUsageCounter.labels("registerAlertReciever").inc(); if(!global.wsAlertRecievers) global.wsAlertRecievers = []; global.wsAlertRecievers.push(socket); - } + }); }); module.exports = app; diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index aa6cf3f9..d121efaf 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -12,9 +12,11 @@ module.exports = class remoteCommands { message.shift(); message.shift(); message.shift(); - message.join(" "); - let chatMessage = "["+data.instanceID+"]"+ message; - this.messageInterface("/c game.print('"+chatMessage+"')"); + let chatMessage = "["+data.instanceID+"]"+ message.join(" "); + while(chatMessage.includes("'")){ + chatMessage = chatMessage.replace("'", ""); + } + this.messageInterface("/silent-command game.print('"+chatMessage.replace("!shout ","")+"')"); } } }); From 2aff63e4a1e48fb49bdf1a401b1c052bf99aba25 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 28 Apr 2018 03:10:57 +0200 Subject: [PATCH 12/92] Strip all non a-z 0-9 characters from !shout commands --- sharedPlugins/globalChat/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index d121efaf..0e4422d4 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -13,9 +13,7 @@ module.exports = class remoteCommands { message.shift(); message.shift(); let chatMessage = "["+data.instanceID+"]"+ message.join(" "); - while(chatMessage.includes("'")){ - chatMessage = chatMessage.replace("'", ""); - } + chatMessage = chatMessage.replace(/[^a-zA-Z0-9]+/g, ""); this.messageInterface("/silent-command game.print('"+chatMessage.replace("!shout ","")+"')"); } } From 6a7042c0aab9045bb1e06fa2b5c5ebcd5f7dd5d1 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 28 Apr 2018 14:16:34 +0200 Subject: [PATCH 13/92] Improved performance of /api/getFactorioLocale with caching --- lib/getFactorioLocale.js | 26 +++++++++++++++++--------- master.js | 2 +- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/getFactorioLocale.js b/lib/getFactorioLocale.js index 7a28d58c..72d84dd0 100644 --- a/lib/getFactorioLocale.js +++ b/lib/getFactorioLocale.js @@ -6,6 +6,8 @@ Module for grabbing factorio locale from a local factorio install const fs = require("fs"); const ini = require("ini"); +const cache = {}; + /** Gets raw factorio locale from the .ini file. Not available outside of module through exports. @@ -32,14 +34,20 @@ Gets factorios locale as an object, does not respect mods. */ module.exports.asObject = function getLocaleAsObject(factorioDirectory, languageCode, callback){ if(typeof factorioDirectory != "string" || typeof languageCode != "string" || typeof callback != "function"){ - throw "Error: wrong parameters provided"; + throw new Error("Error: wrong parameters provided"); + } + + if(!cache[languageCode]){ + getLocale(factorioDirectory, languageCode, (err, rawLocale) => { + if(err){ + callback(err); + } else { + let processedLocale = ini.parse(rawLocale); + cache[languageCode] = processedLocale; + callback(undefined, processedLocale); + } + }); + } else { + callback(undefined, cache[languageCode]); } - getLocale(factorioDirectory, languageCode, (err, rawLocale) => { - if(err){ - callback(err); - } else { - let processedLocale = ini.parse(rawLocale) - callback(undefined, processedLocale); - } - }); } diff --git a/master.js b/master.js index a36d6429..fa47c882 100644 --- a/master.js +++ b/master.js @@ -782,7 +782,7 @@ Requires x-access-token header to be set. Find you api token in secret-api-token @memberof clusterioMaster @instance @alias api/runCommand -@param {object} JSON {instanceID:19412312, command:"/c game.print('hello')"} +@param {object} JSON {instanceID:19412312, broadcast:false, command:"/c game.print('hello')"} @returns {object} Status {auth: bool, message: "Informative error", data:{}} */ app.post("/api/runCommand", (req,res) => { From fc1f3bd077afbe866e7bc79d118915c041439c75 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 28 Apr 2018 14:47:53 +0200 Subject: [PATCH 14/92] Attempted fix for command corruption --- client.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/client.js b/client.js index 5247ae95..d78140f3 100644 --- a/client.js +++ b/client.js @@ -831,11 +831,21 @@ function instanceManagement() { } // if we got some confirmed orders // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); - sadas = JSON.stringify(confirmedOrders); - confirmedOrders = []; - // send our RCON command with whatever we got - messageInterface("/silent-command remote.call('clusterio', 'importMany', '" + sadas + "')"); - sadas = null; + let itemCount = confirmedOrders.length; + if(itemCount > 0){ + let numberOfFragments = Math.floor(itemCount*20/2000); // 20 is approximate length of items, 2000 is maximum length we want to send at a time + let exportedCount = 0; + let exportFragment = []; + while(exportedCount < itemCount){ + exportFragment[exportFragment.length] = confirmedOrders[exportedCount]; + exportedCount++; + if(exportFragment.length > 100 || exportedCount >= itemCount){ + messageInterface("/silent-command remote.call('clusterio', 'importMany', '" + JSON.stringify(exportFragment) + "')"); + exportFragment = []; + } + } + confirmedOrders = []; + } } }, 1000); // COMBINATOR SIGNALS --------------------------------------------------------- From 02ab34dbf4789a617c28fb22f480d5f3848b6579 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 28 Apr 2018 15:04:15 +0200 Subject: [PATCH 15/92] Don't remove all spaces from !shouts --- sharedPlugins/globalChat/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index 0e4422d4..cd77f600 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -13,7 +13,7 @@ module.exports = class remoteCommands { message.shift(); message.shift(); let chatMessage = "["+data.instanceID+"]"+ message.join(" "); - chatMessage = chatMessage.replace(/[^a-zA-Z0-9]+/g, ""); + chatMessage = chatMessage.replace(/[^a-zA-Z0-9]+/g, " "); this.messageInterface("/silent-command game.print('"+chatMessage.replace("!shout ","")+"')"); } } From 1184e6446757db98aadaa611c4ad52cdbd02f478 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 28 Apr 2018 19:51:23 +0200 Subject: [PATCH 16/92] Attempt to be even more long command friendly --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index d78140f3..f2c86308 100644 --- a/client.js +++ b/client.js @@ -833,7 +833,7 @@ function instanceManagement() { // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); let itemCount = confirmedOrders.length; if(itemCount > 0){ - let numberOfFragments = Math.floor(itemCount*20/2000); // 20 is approximate length of items, 2000 is maximum length we want to send at a time + let numberOfFragments = Math.floor(itemCount*50/2000); // 50 is approximate length of items, 2000 is maximum length we want to send at a time let exportedCount = 0; let exportFragment = []; while(exportedCount < itemCount){ From f10d6bbfb5f303f9103928c9bf83d274138e108c Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Sun, 29 Apr 2018 10:29:35 +1000 Subject: [PATCH 17/92] tried to fix desyncs --- client.js | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/client.js b/client.js index f2c86308..ffffebd2 100644 --- a/client.js +++ b/client.js @@ -831,21 +831,18 @@ function instanceManagement() { } // if we got some confirmed orders // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); - let itemCount = confirmedOrders.length; - if(itemCount > 0){ - let numberOfFragments = Math.floor(itemCount*50/2000); // 50 is approximate length of items, 2000 is maximum length we want to send at a time - let exportedCount = 0; - let exportFragment = []; - while(exportedCount < itemCount){ - exportFragment[exportFragment.length] = confirmedOrders[exportedCount]; - exportedCount++; - if(exportFragment.length > 100 || exportedCount >= itemCount){ - messageInterface("/silent-command remote.call('clusterio', 'importMany', '" + JSON.stringify(exportFragment) + "')"); - exportFragment = []; - } - } - confirmedOrders = []; + let cmd="{"; + for(let i=0;i1000) + { + messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"}"+ "')"); + cmd="{"; + } } + messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"}"+ "')"); + confirmedOrders=[]; } }, 1000); // COMBINATOR SIGNALS --------------------------------------------------------- From 594c994560704faf6f866b1a5bcb2c0df7fcee31 Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Sun, 29 Apr 2018 13:52:11 +1000 Subject: [PATCH 18/92] Tried to fix desyncs further based on factorio networking stuff --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index ffffebd2..956734b2 100644 --- a/client.js +++ b/client.js @@ -835,7 +835,7 @@ function instanceManagement() { for(let i=0;i1000) + if(cmd.length>400)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS { messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"}"+ "')"); cmd="{"; From bcd1ec79cb7310d8e923d64aefded763e9f97c5b Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Sun, 29 Apr 2018 18:51:08 +1000 Subject: [PATCH 19/92] fixed to get items --- client.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client.js b/client.js index 956734b2..617cace3 100644 --- a/client.js +++ b/client.js @@ -820,9 +820,7 @@ function instanceManagement() { needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) { if (response && response.body && typeof response.body == "object") { // buffer confirmed orders - confirmedOrders[confirmedOrders.length] = { - [response.body.name]: response.body.count, - } + confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count} if(config.logItemTransfers){ console.log(`Imported ${response.body.count} ${response.body.name} from master`); } From a67a1d3d1dd43462baa42e6521a6bc10f2a3510d Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Sun, 29 Apr 2018 19:30:44 +1000 Subject: [PATCH 20/92] fixed brakets --- client.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 617cace3..5e56eacd 100644 --- a/client.js +++ b/client.js @@ -829,17 +829,17 @@ function instanceManagement() { } // if we got some confirmed orders // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); - let cmd="{"; + let cmd="["; for(let i=0;i400)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS { - messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"}"+ "')"); - cmd="{"; + messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')"); + cmd="["; } } - messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"}"+ "')"); + messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')"); confirmedOrders=[]; } }, 1000); From c5b55f27b73439bae67894ea7b8dd3cec6a3916c Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Sun, 29 Apr 2018 19:42:19 +1000 Subject: [PATCH 21/92] fixed brakets v2 --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index 5e56eacd..05c81ff1 100644 --- a/client.js +++ b/client.js @@ -832,7 +832,7 @@ function instanceManagement() { let cmd="["; for(let i=0;i400)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS { messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')"); From f15ce3fb2309f67f1a1b6979914260bb594d36f6 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 29 Apr 2018 21:45:22 +0200 Subject: [PATCH 22/92] No longer auto install clusterio 1.0.0 --- lib/npmPostinstall.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/npmPostinstall.js b/lib/npmPostinstall.js index 8106fad4..187407bf 100644 --- a/lib/npmPostinstall.js +++ b/lib/npmPostinstall.js @@ -5,7 +5,7 @@ var fs = require('fs'); var modManager = require("./manager/modManager.js"); -modManager.addMod("clusterio", "shared"); +// modManager.addMod("clusterio", "shared"); function makeDirectory(path){ return new Promise((resolve, reject) => { From 66b64eae1bb7e5bc8fce85e3bb10add582f1207a Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Mon, 30 Apr 2018 18:37:08 +1000 Subject: [PATCH 23/92] started fixing receiveInventory --- client.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 05c81ff1..2583f38c 100644 --- a/client.js +++ b/client.js @@ -828,7 +828,7 @@ function instanceManagement() { }); } // if we got some confirmed orders - // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); + console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); let cmd="["; for(let i=0;i400)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS + { + messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')"); + cmd="{"; + } + } + messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");*/ } catch (e){ console.log(e); } } }); - }, 1000); + }, 1500); // Make sure world has its worldID setTimeout(function(){ messageInterface("/silent-command remote.call('clusterio','setWorldID',"+instanceconfig.unique+")") From 5b59abb8e35c87491fc4d49a18812aadce7b6fbe Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Mon, 30 Apr 2018 18:38:48 +1000 Subject: [PATCH 24/92] changeds timings --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index 2583f38c..d21da8ca 100644 --- a/client.js +++ b/client.js @@ -880,7 +880,7 @@ function instanceManagement() { } } }); - }, 1500); + }, 1550); // Make sure world has its worldID setTimeout(function(){ messageInterface("/silent-command remote.call('clusterio','setWorldID',"+instanceconfig.unique+")") From 57f7dec9566d6f529c62591cc346690edae8b090 Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Mon, 30 Apr 2018 18:41:02 +1000 Subject: [PATCH 25/92] recomments stuff --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index d21da8ca..9bac857e 100644 --- a/client.js +++ b/client.js @@ -828,7 +828,7 @@ function instanceManagement() { }); } // if we got some confirmed orders - console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); + //console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); let cmd="["; for(let i=0;i Date: Mon, 30 Apr 2018 19:32:43 +1000 Subject: [PATCH 26/92] fixed bug fixed so that it doesnt send when cmd=="[" --- client.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 9bac857e..9fe74c22 100644 --- a/client.js +++ b/client.js @@ -829,6 +829,7 @@ function instanceManagement() { } // if we got some confirmed orders //console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); + //if (!(confirmedOrders.length>0)){return;} let cmd="["; for(let i=0;i Date: Mon, 30 Apr 2018 12:09:01 +0200 Subject: [PATCH 27/92] Added more websocket utilization metrics --- .gitignore | 1 + master.js | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 0270b57f..9752a88d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ mod_cache.json static/pictures/spritesheet.js static/pictures/spritesheet.png static/external +saves # weird pattern to ignore sharedPlugins except not ignore some core modules !sharedPlugins/ diff --git a/master.js b/master.js index fa47c882..5bece902 100644 --- a/master.js +++ b/master.js @@ -935,6 +935,7 @@ class wsSlave { // handle command return values this.commandsWaitingForReturn = {}; this.socket.on("runCommandReturnValue", resp => { + prometheusWsUsageCounter.labels('runCommandReturnValue').inc(); if(resp.commandID && resp.body && this.commandsWaitingForReturn[resp.commandID] && this.commandsWaitingForReturn[resp.commandID].callback && typeof this.commandsWaitingForReturn[resp.commandID].callback == "function"){ this.commandsWaitingForReturn[resp.commandID].callback(resp.body); delete this.commandsWaitingForReturn[resp.commandID]; @@ -942,6 +943,7 @@ class wsSlave { }); this.socket.on("gameChat", data => { + prometheusWsUsageCounter.labels('gameChat').inc(); if(typeof data === "object"){ data.timestamp = Date.now(); if(!global.wsChatRecievers) global.wsChatRecievers = []; @@ -952,6 +954,7 @@ class wsSlave { }); this.socket.on("alert", alert => { + prometheusWsUsageCounter.labels('alert').inc(); if(typeof alert === "object"){ alert.timestamp = Date.now(); if(!global.wsAlertRecievers) global.wsAlertRecievers = []; From 0afc3d7e78b456ce74c1500820f97f0a9f8fd2f2 Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Mon, 30 Apr 2018 20:44:29 +1000 Subject: [PATCH 28/92] split receiveInventory into multiple commands --- client.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client.js b/client.js index 9fe74c22..c3d20cb1 100644 --- a/client.js +++ b/client.js @@ -863,8 +863,8 @@ function instanceManagement() { } inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); - /*let cmd="{"; + //messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); + let cmd="{"; for (var key in inventoryFrame) { cmd+='"'+key+'"'+":"+inventoryFrame[key]+","; @@ -874,7 +874,7 @@ function instanceManagement() { cmd="{"; } } - messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");*/ + messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')"); } catch (e){ console.log(e); } From f4e03899e796bb199f9e26d811bdb3b36ca01615 Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Mon, 30 Apr 2018 21:28:45 +1000 Subject: [PATCH 29/92] fixed bug --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index c3d20cb1..f3fce050 100644 --- a/client.js +++ b/client.js @@ -874,7 +874,7 @@ function instanceManagement() { cmd="{"; } } - messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')"); + if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");} } catch (e){ console.log(e); } From d6369ba29550cc9928cfadee0ef762b3d301ef8b Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Mon, 30 Apr 2018 21:46:45 +1000 Subject: [PATCH 30/92] reverted stuff to prev commit reverted stuff back to prev commit until the mod has been updated --- client.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client.js b/client.js index f3fce050..b2a74fed 100644 --- a/client.js +++ b/client.js @@ -863,8 +863,8 @@ function instanceManagement() { } inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - //messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); - let cmd="{"; + messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); + /*let cmd="{"; for (var key in inventoryFrame) { cmd+='"'+key+'"'+":"+inventoryFrame[key]+","; @@ -874,7 +874,7 @@ function instanceManagement() { cmd="{"; } } - if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");} + if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");}*/ } catch (e){ console.log(e); } From 9c9db7882ca78a0cb6910cc2199b0f11067189bd Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Mon, 30 Apr 2018 22:10:13 +1000 Subject: [PATCH 31/92] enabled inventoryFrame command splitting --- client.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client.js b/client.js index b2a74fed..cbedfcf8 100644 --- a/client.js +++ b/client.js @@ -863,8 +863,7 @@ function instanceManagement() { } inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); - /*let cmd="{"; + let cmd="{"; for (var key in inventoryFrame) { cmd+='"'+key+'"'+":"+inventoryFrame[key]+","; @@ -874,7 +873,7 @@ function instanceManagement() { cmd="{"; } } - if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");}*/ + if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");} } catch (e){ console.log(e); } From 47430b5d289bc1f8d5d0489caa874390c092c0f5 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 30 Apr 2018 14:30:01 +0200 Subject: [PATCH 32/92] Remove super exessive logging on master Also added better prometheus endpoint logging --- master.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master.js b/master.js index 5bece902..7e24a9a6 100644 --- a/master.js +++ b/master.js @@ -699,7 +699,7 @@ app.post("/api/getStats", function(req,res) { } else if(req.body.statistic == "remove"){ console.log(`sending remove data for instanceID ${req.body.instanceID} ${req.body.itemName}`); // Gather data - console.log(sentItemStatisticsBySlaveID) + // console.log(sentItemStatisticsBySlaveID) let itemStats = sentItemStatisticsBySlaveID[req.body.instanceID]; console.log(itemStats) if(typeof itemStats == "object"){ From 968a306c5a6f7154dcb6b9457d544d7df064068d Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 30 Apr 2018 23:05:55 +0200 Subject: [PATCH 33/92] Fixed issue where slaves would not register when ran on VM withou MAC --- client.js | 24 ++++++++++++------------ master.js | 8 ++++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/client.js b/client.js index cbedfcf8..3712cdfe 100644 --- a/client.js +++ b/client.js @@ -619,20 +619,20 @@ function instanceManagement() { function callback(err, mac) { if (err) { + mac = "unknown"; console.log("##### getMac crashed, but we don't really give a shit because we are probably closing down #####"); - } else { - payload.mac = mac - console.log("Registered our precense with master "+config.masterIP+" at " + payload.time); - needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { - if (err && err.code != "ECONNRESET"){ - console.error("We got problems, something went wrong when contacting master"); - console.error(err); - } else if (response && response.body) { - // In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care. - console.log(response.body); - } - }); } + payload.mac = mac; + console.log("Registered our precense with master "+config.masterIP+" at " + payload.time); + needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { + if (err && err.code != "ECONNRESET"){ + console.error("We got problems, something went wrong when contacting master"); + console.error(err); + } else if (response && response.body) { + // In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care. + console.log(response.body); + } + }); } if(global.mac){ callback(undefined, global.mac); diff --git a/master.js b/master.js index 7e24a9a6..8250a171 100644 --- a/master.js +++ b/master.js @@ -291,6 +291,8 @@ POST Get metadata from a single slave. For whenever you find the data returned b @alias /api/getSlaveMeta */ app.post("/api/getSlaveMeta", function (req, res) { + endpointHitCounter.labels(req.route.path).inc(); + let reqStartTime = Date.now(); console.log("body", req.body); if(req.body && req.body.instanceID && req.body.password){ console.log("returning meta for ", req.body.instanceID); @@ -299,6 +301,7 @@ app.post("/api/getSlaveMeta", function (req, res) { res.status(400); res.send('{"INVALID REQUEST":1}'); } + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); // mod management // should handle uploading and checking if mods are uploaded @@ -786,6 +789,8 @@ Requires x-access-token header to be set. Find you api token in secret-api-token @returns {object} Status {auth: bool, message: "Informative error", data:{}} */ app.post("/api/runCommand", (req,res) => { + endpointHitCounter.labels(req.route.path).inc(); + let reqStartTime = Date.now(); var token = req.headers['x-access-token']; if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' }); @@ -808,6 +813,7 @@ app.post("/api/runCommand", (req,res) => { wsSlaves[instanceID].runCommand(body.command); } res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"}); + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } else if ( body.instanceID && wsSlaves[body.instanceID] @@ -818,9 +824,11 @@ app.post("/api/runCommand", (req,res) => { // execute command wsSlaves[body.instanceID].runCommand(body.command, data => { res.status(200).send({auth: true, message: "success", data}); + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); } else { res.status(400).send({auth: true, message: "Error: invalid request.body"}); + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } }); }); From 1e0c2759688fbff25162ee0dd7ad8e284ca6216d Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 30 Apr 2018 23:34:09 +0200 Subject: [PATCH 34/92] Use rcon instead of stdin on linux as well --- client.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 3712cdfe..2533059a 100644 --- a/client.js +++ b/client.js @@ -64,7 +64,7 @@ function messageInterface(command, callback) { command = command.toString('utf8'); } - if(process.platform == "linux" && typeof command == "string" && serverprocess) { + if(false && process.platform == "linux" && typeof command == "string" && serverprocess) { /* to send to stdin, use: serverprocess.stdin.write("/c command;\n") @@ -381,7 +381,7 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ }); // connect to the server with rcon - if(process.platform != "linux"){ + if(true || process.platform != "linux"){ // IP, port, password client = new Rcon({ host: 'localhost', From 6514462a1db6300fce2f524faf51e273c4c9535a Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 30 Apr 2018 23:47:31 +0200 Subject: [PATCH 35/92] Increased max command size for imports --- client.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 2533059a..fcda464c 100644 --- a/client.js +++ b/client.js @@ -834,13 +834,14 @@ function instanceManagement() { for(let i=0;i400)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS + if(cmd.length>8000)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS { messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')"); cmd="["; } } - if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')");}confirmedOrders=[]; + if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')");} + confirmedOrders=[]; } }, 1000); // COMBINATOR SIGNALS --------------------------------------------------------- From c7e2308a33aeac1b7f23a4428729d1989328f264 Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Tue, 1 May 2018 08:23:49 +1000 Subject: [PATCH 36/92] Added commandBuffer --- client.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index fcda464c..1cbfa590 100644 --- a/client.js +++ b/client.js @@ -56,9 +56,9 @@ const needleOptionsWithTokenAuthHeader = { }; var instanceInfo = {}; - +var commandBuffer=[]; // function to handle sending commands into the game -function messageInterface(command, callback) { +function messageInterfaceInternal(command, callback) { // try to save us if you send a buffer instead of string if(typeof command == "object") { command = command.toString('utf8'); @@ -84,6 +84,8 @@ function messageInterface(command, callback) { } } } +function messageInterface(command, callback) {commandBuffer.push([command,callback]);} + // handle commandline parameters if (!command || command == "help" || command == "--help") { @@ -643,6 +645,8 @@ function instanceManagement() { } }); + // messageInterface Management + setInterval(function(){let command=commandBuffer.pop();if(command){messageInterfaceInternal(command[0],command[1])}},10); // Mod uploading and management ----------------------------------------------- // get mod names and hashes From 1fede265648ffcd6260e6f07b6bf42ef9cf8a4d3 Mon Sep 17 00:00:00 2001 From: Valansch Date: Tue, 1 May 2018 12:22:19 +0200 Subject: [PATCH 37/92] Shout displays instance name now --- sharedPlugins/globalChat/index.js | 41 +++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index cd77f600..44d480f7 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -3,24 +3,45 @@ module.exports = class remoteCommands { this.messageInterface = messageInterface; this.config = mergedConfig; this.socket = extras.socket; - + this.instances = {}; this.socket.emit("registerChatReciever"); this.socket.on("gameChat", data => { if(data.instanceID && data.instanceID != this.config.unique.toString()){ - if(data.data.includes("[CHAT]") && data.data.toLowerCase().includes("!shout")){ - let message = data.data.trim().split(" "); - message.shift(); - message.shift(); - message.shift(); - let chatMessage = "["+data.instanceID+"]"+ message.join(" "); - chatMessage = chatMessage.replace(/[^a-zA-Z0-9]+/g, " "); - this.messageInterface("/silent-command game.print('"+chatMessage.replace("!shout ","")+"')"); + if(data.data.includes("[CHAT]") && (data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){ + let words = data.data.trim().split(" "); + words.shift(); + words.shift(); + words.shift(); + let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout", ""); + let instance = this.instances[data.instanceID]; + if (!instance) { + let me = this; //to pass 'this' down to the callback + require('needle').get(this.config.masterIP + '/api/slaves', function (err, response) { + if(err || response.statusCode != 200) { + console.log("Unable to get JSON master/api/slaves, master might be unaccessible"); + } else if (response && response.body) { + if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} + try { + for (let index in response.body) + me.instances[index] = response.body[index]["instanceName"]; + } catch (e){ + console.log(e); + return null; + } + instance = me.instances[data.instanceID] + if (!instance) + instance = data.instanceID; //somehow the master doesn't know the instance + me.messageInterface("/silent-command game.print('[" + instance + "] " + message +"')"); + } + }); + } else { + this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')"); + } } } }); } factorioOutput(data){ - // console.log("GOT CHAT!"+data) this.socket.emit("gameChat", { instanceID: this.config.unique.toString(), data, From be73f2f3a591f6dd65e0b3635c05b98eda1e2113 Mon Sep 17 00:00:00 2001 From: psihius Date: Tue, 1 May 2018 21:36:38 +0300 Subject: [PATCH 38/92] Changed command splitting logic from json data to lua tables --- client.js | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/client.js b/client.js index 1cbfa590..7f66dfdd 100644 --- a/client.js +++ b/client.js @@ -646,8 +646,8 @@ function instanceManagement() { }); // messageInterface Management - setInterval(function(){let command=commandBuffer.pop();if(command){messageInterfaceInternal(command[0],command[1])}},10); - + setInterval(function(){let command=commandBuffer.pop();if(command){messageInterfaceInternal(command[0],command[1])}},50); + // Mod uploading and management ----------------------------------------------- // get mod names and hashes // string: instance, function: callback @@ -832,19 +832,21 @@ function instanceManagement() { }); } // if we got some confirmed orders - //console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); + // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); //if (!(confirmedOrders.length>0)){return;} - let cmd="["; + let cmd="local t={"; for(let i=0;i8000)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS + cmd+='["'+confirmedOrders[i].name+'"]='+confirmedOrders[i].count+','; + if(cmd.length>320) // Factorio max packet size is 508 { - messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')"); - cmd="["; + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); + cmd="local t={"; } } - if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'importMany', '"+cmd.slice(0, -1)+"]"+ "')");} + if (!(cmd==="local t={")){ + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); + } confirmedOrders=[]; } }, 1000); @@ -868,17 +870,22 @@ function instanceManagement() { } inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - let cmd="{"; - for (var key in inventoryFrame) + let first = true; + let cmd="local s={"; + for (let key in inventoryFrame) { - cmd+='"'+key+'"'+":"+inventoryFrame[key]+","; - if(cmd.length>400)//450 //ITS SMALL SO FACTORIO DOESNT SPILT IT INTO MULTIPLE PACKETS + cmd+='["'+key+'"]='+inventoryFrame[key]+","; + if(first && cmd.length>300 || !first && cmd.length > 320) // Factorio max packet size is 508 { - messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')"); - cmd="{"; + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); + cmd="local s={"; + first = false; } } - if (!(cmd=="[")){messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '"+cmd.slice(0, -1)+"}"+ "')");} + if (!(cmd==="local s={")){ + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); + } + messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')"); } catch (e){ console.log(e); } From 6816db0628840e2a4ba1384c74478b47974f7dc5 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Wed, 2 May 2018 12:39:33 +0200 Subject: [PATCH 39/92] More detailed Websocket usage statistics --- master.js | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/master.js b/master.js index 8250a171..96408b97 100644 --- a/master.js +++ b/master.js @@ -96,7 +96,7 @@ const prometheusConnectedInstancesCounter = new Prometheus.Gauge({ const prometheusWsUsageCounter = new Prometheus.Counter({ name: prometheusPrefix+'websocket_usage_counter', help: 'Websocket traffic', - labelNames: ["connectionType"], + labelNames: ["connectionType", "instanceID"], }); const prometheusProductionGauge = new Prometheus.Gauge({ name: prometheusPrefix+'production_gauge', @@ -862,14 +862,17 @@ class slaveMapper { this.lastBeat = Date.now(); this.socket.on("heartbeat", () => { + prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc(); // we aren't ready to die yet apparently this.lastBeat = Date.now(); }); this.socket.on("sendChunk", function(data){ + prometheusWsUsageCounter.labels('sendChunk', this.instanceID).inc(); mapRequesters[data.requesterID].socket.emit("displayChunk", data); }); // slaveMapper sent us an entity update, process this.socket.on("sendEntity", entity => { + prometheusWsUsageCounter.labels('sendEntity', this.instanceID).inc(); Object.keys(mapRequesters).forEach(requesterName => { let requester = mapRequesters[requesterName]; @@ -890,12 +893,12 @@ class mapRequester { this.lastBeat = Date.now(); this.socket.on("heartbeat", () => { - prometheusWsUsageCounter.labels('heartbeat').inc(); + prometheusWsUsageCounter.labels('heartbeat', "other").inc(); // we aren't ready to die yet apparently this.lastBeat = Date.now(); }); this.socket.on("requestChunk", loc => { - prometheusWsUsageCounter.labels('requestChunk').inc(); + prometheusWsUsageCounter.labels('requestChunk', "other").inc(); loc.requesterID = this.requesterID; let instanceID = loc.instanceID || this.instanceID; if(slaveMappers[instanceID] && typeof loc.x == "number" && typeof loc.y == "number"){ @@ -903,7 +906,7 @@ class mapRequester { } }); this.socket.on("requestEntity", req => { - prometheusWsUsageCounter.labels('requestEntity').inc(); + prometheusWsUsageCounter.labels('requestEntity', "other").inc(); req.requesterID = this.requesterID; let instanceID = req.instanceID || this.instanceID; if(slaveMappers[instanceID] && typeof req.x == "number" && typeof req.y == "number"){ @@ -911,7 +914,7 @@ class mapRequester { } }); this.socket.on("placeEntity", req => { - prometheusWsUsageCounter.labels('placeEntity').inc(); + prometheusWsUsageCounter.labels('placeEntity', "other").inc(); req.requesterID = this.requesterID; let instanceID = req.instanceID || this.instanceID; if(slaveMappers[instanceID]){ @@ -929,11 +932,11 @@ class wsSlave { this.lastBeat = Date.now(); this.socket.on("heartbeat", () => { - prometheusWsUsageCounter.labels('heartbeat').inc(); + prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc(); this.lastBeat = Date.now(); }); this.socket.on("combinatorSignal", circuitFrameWithMeta => { - prometheusWsUsageCounter.labels('combinatorSignal').inc(); + prometheusWsUsageCounter.labels('combinatorSignal', this.instanceID).inc(); if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object"){ Object.keys(wsSlaves).forEach(instanceID => { wsSlaves[instanceID].socket.emit("processCombinatorSignal", circuitFrameWithMeta); @@ -943,7 +946,7 @@ class wsSlave { // handle command return values this.commandsWaitingForReturn = {}; this.socket.on("runCommandReturnValue", resp => { - prometheusWsUsageCounter.labels('runCommandReturnValue').inc(); + prometheusWsUsageCounter.labels('runCommandReturnValue', this.instanceID).inc(); if(resp.commandID && resp.body && this.commandsWaitingForReturn[resp.commandID] && this.commandsWaitingForReturn[resp.commandID].callback && typeof this.commandsWaitingForReturn[resp.commandID].callback == "function"){ this.commandsWaitingForReturn[resp.commandID].callback(resp.body); delete this.commandsWaitingForReturn[resp.commandID]; @@ -951,7 +954,7 @@ class wsSlave { }); this.socket.on("gameChat", data => { - prometheusWsUsageCounter.labels('gameChat').inc(); + prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc(); if(typeof data === "object"){ data.timestamp = Date.now(); if(!global.wsChatRecievers) global.wsChatRecievers = []; @@ -962,7 +965,7 @@ class wsSlave { }); this.socket.on("alert", alert => { - prometheusWsUsageCounter.labels('alert').inc(); + prometheusWsUsageCounter.labels('alert', this.instanceID).inc(); if(typeof alert === "object"){ alert.timestamp = Date.now(); if(!global.wsAlertRecievers) global.wsAlertRecievers = []; @@ -998,13 +1001,13 @@ io.on('connection', function (socket) { /* initial processing for remoteMap */ socket.on('registerSlaveMapper', function (data) { - prometheusWsUsageCounter.labels('registerSlaveMapper').inc(); + prometheusWsUsageCounter.labels('registerSlaveMapper', "other").inc(); slaveMappers[data.instanceID] = new slaveMapper(data.instanceID, socket); console.log("remoteMap | SOCKET registered map provider for "+data.instanceID); }); socket.on('registerMapRequester', function(data){ // data {instanceID:""} - prometheusWsUsageCounter.labels('registerMapRequester').inc(); + prometheusWsUsageCounter.labels('registerMapRequester', "other").inc(); let requesterID = Math.random().toString(); mapRequesters[requesterID] = new mapRequester(requesterID, socket, data.instanceID); socket.emit("mapRequesterReady", true); @@ -1013,19 +1016,19 @@ io.on('connection', function (socket) { /* Websockets for send and recieve combinators */ socket.on("registerSlave", function(data) { - prometheusWsUsageCounter.labels('registerSlave').inc(); + prometheusWsUsageCounter.labels('registerSlave', "other").inc(); if(data && data.instanceID){ wsSlaves[data.instanceID] = new wsSlave(data.instanceID, socket); console.log("SOCKET | Created new wsSlave: "+ data.instanceID); } }); socket.on("registerChatReciever", function(data){ - prometheusWsUsageCounter.labels("registerChatReciever").inc(); + prometheusWsUsageCounter.labels("registerChatReciever", "other").inc(); if(!global.wsChatRecievers) global.wsChatRecievers = []; global.wsChatRecievers.push(socket); }); socket.on("registerAlertReciever", function(data){ - prometheusWsUsageCounter.labels("registerAlertReciever").inc(); + prometheusWsUsageCounter.labels("registerAlertReciever", "other").inc(); if(!global.wsAlertRecievers) global.wsAlertRecievers = []; global.wsAlertRecievers.push(socket); }); From a0e98ecb0078a36bcf48a80d47a1e636291f3019 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Wed, 2 May 2018 20:37:55 +0200 Subject: [PATCH 40/92] Format numbers with , (1,000) --- static/storage.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/static/storage.js b/static/storage.js index ad74f1af..dc138d7a 100644 --- a/static/storage.js +++ b/static/storage.js @@ -16,6 +16,9 @@ function drawcontents(data) { //update existing rows or create new ones data.forEach(function(item, i) { + // format count to have .s in it for large number readability (1,000,000) + item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); + let row = rows[i]; if(!row) { row = document.createElement('tr'); From c02bb6d58ffa4ceb9e070dd1c4df2c0e9f4c1bd1 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 4 May 2018 17:50:09 +0200 Subject: [PATCH 41/92] #85 made instance directory configurable in config.json --- client.js | 24 ++++++++++++------------ config.json | 1 + lib/getInstances.js | 3 ++- lib/manager/modManager.js | 6 +++--- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/client.js b/client.js index 7f66dfdd..4e5b15ac 100644 --- a/client.js +++ b/client.js @@ -25,8 +25,8 @@ const configManager = require("./lib/manager/configManager.js"); var config = require('./config'); var global = {}; -if (!fs.existsSync("./instances/")) { - fs.mkdirSync("instances"); +if (!fs.existsSync(config.instanceDirectory)) { + fs.mkdirSync(config.instanceDirectory); } if (!fs.existsSync("./sharedPlugins/")) { fs.mkdirSync("sharedPlugins"); @@ -35,7 +35,7 @@ if (!fs.existsSync("./sharedMods/")) { fs.mkdirSync("sharedMods"); } const instance = process.argv[3]; -const instancedirectory = './instances/' + instance; +const instancedirectory = config.instanceDirectory + '/' + instance; const command = process.argv[2]; // Set the process title, shows up as the title of the CMD window on windows @@ -99,7 +99,7 @@ if (!command || command == "help" || command == "--help") { console.error("node client.js manage"); process.exit(1); } else if (command == "list") { - let instanceNames = fileOps.getDirectoriesSync("./instances/"); + let instanceNames = fileOps.getDirectoriesSync(config.instanceDirectory); instanceNames.unshift("Name:"); let longestInstanceName = 0; // determine longest instance name @@ -122,7 +122,7 @@ if (!command || command == "help" || command == "--help") { if(instance.includes("Name:")){ factorioPort = "Port:" } else { - factorioPort = require("./instances/"+instance+"/config").factorioPort; + factorioPort = require(config.instanceDirectory+"/"+instance+"/config").factorioPort; } factorioPorts.push(factorioPort); }); @@ -201,8 +201,8 @@ if (!command || command == "help" || command == "--help") { if (!process.argv[3]) { console.error("Usage: node client.js delete [instance]"); process.exit(1); - } else if (typeof process.argv[3] == "string" && fs.existsSync("./instances/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") { - fileOps.deleteFolderRecursiveSync("./instances/" + process.argv[3]); + } else if (typeof process.argv[3] == "string" && fs.existsSync(config.instanceDirectory+"/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") { + fileOps.deleteFolderRecursiveSync(config.instanceDirectory+"/" + process.argv[3]); console.log("Deleted instance " + process.argv[3]); process.exit(1); } else { @@ -241,7 +241,7 @@ if (!command || command == "help" || command == "--help") { // fs.symlinkSync('../../../sharedMods', instancedirectory + "/mods", 'junction') // This is broken because it can only take a file as first argument, not a folder fs.writeFileSync(instancedirectory + "/config.ini", "[path]\r\n\ read-data=__PATH__executable__/../../data\r\n\ -write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ +write-data=__PATH__executable__/../../../"+config.instanceDirectory+"/" + instance + "\r\n\ "); // this line is probably not needed anymore but Im not gonna remove it @@ -288,7 +288,7 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ console.log("Instance created!"); } else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)) { // Exit if no instance specified (it should be, just a safeguard); - if(instancedirectory != "./instances/undefined"){ + if(instancedirectory != config.instanceDirectory+"/undefined"){ var instanceconfig = require(instancedirectory + '/config'); instanceconfig.unique = stringUtils.hashCode(instanceconfig.clientPassword); if(process.env.FACTORIOPORT){ @@ -682,7 +682,7 @@ function instanceManagement() { } }); var form = req.form(); - form.append('file', fs.createReadStream("./instances/"+instance+"/mods/"+mod)); + form.append('file', fs.createReadStream(config.instanceDirectory+"/"+instance+"/mods/"+mod)); } else { console.log("Not sending mod: " + mod + " to master because config.uploadModsToMaster is not enabled") } @@ -967,7 +967,7 @@ function hashMods(instanceName, callback) { } let hashedMods = []; /*let mods = fs.readdirSync("./sharedMods/")*/ - let instanceMods = fs.readdirSync("./instances/"+instanceName+"/mods/"); + let instanceMods = fs.readdirSync(config.instanceDirectory+"/"+instanceName+"/mods/"); if(instanceMods.length == 0){ callback({}); // there are no mods installed, return an empty object } @@ -977,7 +977,7 @@ function hashMods(instanceName, callback) { } } for(let i=0; i 0){ if(term.includes("mods.factorio.com/mods/")){ // https://mods.factorio.com/mods/Danielv123/ChooChoo @@ -186,7 +186,7 @@ module.exports.removeMod = function removeMod(term, instance, force = false){ if(instance == "shared"){ // modDirectory has to end in / var modDirectory = "sharedMods/"; } else { - var modDirectory = "instances/"+instance+"/instanceMods/"; + var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/"; } if(term && typeof term == "string" && term.length > 0){ let mods = fs.readdirSync(modDirectory); From b49a37e4381644a86c6f0817ef75f4f5f641e950 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 5 May 2018 12:47:16 +0200 Subject: [PATCH 42/92] Use compression on master --- master.js | 2 ++ package.json | 1 + 2 files changed, 3 insertions(+) diff --git a/master.js b/master.js index 96408b97..ed86924a 100644 --- a/master.js +++ b/master.js @@ -49,6 +49,7 @@ fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAu })); const express = require("express"); +const compression = require('compression'); const ejs = require("ejs"); // Required for express post requests const bodyParser = require("body-parser"); @@ -58,6 +59,7 @@ var app = express(); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); app.use(fileUpload()); +app.use(compression()); // dynamic HTML generations with EJS app.set('views', path.join(__dirname, 'static')); diff --git a/package.json b/package.json index 76666614..e277b755 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "bcryptjs": "^2.4.3", "body-parser": "^1.18.2", "codecov": "^2.3.1", + "compression": "^1.7.2", "deepmerge": "^1.5.2", "ejs": "^2.5.7", "express": "^4.16.2", From a6d98db6a9ac1d81949f502760fa27e629af5f76 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 5 May 2018 13:52:36 +0200 Subject: [PATCH 43/92] Fix outdated leveldown dependency --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e277b755..15762233 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "html5-validator": "^1.0.7", "ini": "^1.3.5", "jsonwebtoken": "^8.2.1", - "leveldown": "^1.9.0", + "leveldown": "^3.0.1", "linvodb3": "^3.25.1", "luamin": "^1.0.4", "mkdirp": "^0.5.1", From f5e908eb5d78e42187d2fbfaab4f9155b72f7002 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 5 May 2018 13:52:36 +0200 Subject: [PATCH 44/92] Revert "Fix outdated leveldown dependency" This reverts commit a6d98db6a9ac1d81949f502760fa27e629af5f76. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 15762233..e277b755 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "html5-validator": "^1.0.7", "ini": "^1.3.5", "jsonwebtoken": "^8.2.1", - "leveldown": "^3.0.1", + "leveldown": "^1.9.0", "linvodb3": "^3.25.1", "luamin": "^1.0.4", "mkdirp": "^0.5.1", From f580f060648dea188a4db1a975ba2e0082f25652 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 6 May 2018 10:21:06 +0200 Subject: [PATCH 45/92] Prometheus now tracks import/export statistics --- master.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/master.js b/master.js index ed86924a..b381055e 100644 --- a/master.js +++ b/master.js @@ -105,6 +105,16 @@ const prometheusProductionGauge = new Prometheus.Gauge({ help: 'Items produced by instance', labelNames: ["instanceID", "itemName"], }); +const prometheusExportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'export_gauge', + help: 'Items exported by instance', + labelNames: ["instanceID", "itemName"], +}); +const prometheusImportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'import_gauge', + help: 'Items imported by instance', + labelNames: ["instanceID", "itemName"], +}); const prometheusPlayerCountGauge = new Prometheus.Gauge({ name: prometheusPrefix+'player_count_gauge', help: 'Amount of players connected to this cluster', @@ -438,11 +448,13 @@ app.post("/api/place", authenticate.middleware, function(req, res) { key:req.body.name, value:req.body.count, }); + prometheusExportGauge.labels(x.instanceID, req.body.name).inc(req.body.count); // save items we get db.items.addItem({ name:req.body.name, count:req.body.count }); + // Attempt confirming res.send("success"); } else { @@ -514,6 +526,8 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } + + prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count); res.send({count: object.count, name: object.name}); httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } else { From f5d1847975050e4b6954ba0b4d5a67e8f1923426 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 6 May 2018 10:21:06 +0200 Subject: [PATCH 46/92] Revert "Prometheus now tracks import/export statistics" This reverts commit f580f060648dea188a4db1a975ba2e0082f25652. --- master.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/master.js b/master.js index b381055e..ed86924a 100644 --- a/master.js +++ b/master.js @@ -105,16 +105,6 @@ const prometheusProductionGauge = new Prometheus.Gauge({ help: 'Items produced by instance', labelNames: ["instanceID", "itemName"], }); -const prometheusExportGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'export_gauge', - help: 'Items exported by instance', - labelNames: ["instanceID", "itemName"], -}); -const prometheusImportGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'import_gauge', - help: 'Items imported by instance', - labelNames: ["instanceID", "itemName"], -}); const prometheusPlayerCountGauge = new Prometheus.Gauge({ name: prometheusPrefix+'player_count_gauge', help: 'Amount of players connected to this cluster', @@ -448,13 +438,11 @@ app.post("/api/place", authenticate.middleware, function(req, res) { key:req.body.name, value:req.body.count, }); - prometheusExportGauge.labels(x.instanceID, req.body.name).inc(req.body.count); // save items we get db.items.addItem({ name:req.body.name, count:req.body.count }); - // Attempt confirming res.send("success"); } else { @@ -526,8 +514,6 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } - - prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count); res.send({count: object.count, name: object.name}); httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } else { From ce24a40d8c888b14133b4f985aadfd51e3ee03b4 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 6 May 2018 10:23:55 +0200 Subject: [PATCH 47/92] Prometheus now tracks import/export stats PROPERLY --- master.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/master.js b/master.js index ed86924a..fcb0bbc9 100644 --- a/master.js +++ b/master.js @@ -105,6 +105,16 @@ const prometheusProductionGauge = new Prometheus.Gauge({ help: 'Items produced by instance', labelNames: ["instanceID", "itemName"], }); +const prometheusExportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'export_gauge', + help: 'Items exported by instance', + labelNames: ["instanceID", "itemName"], +}); +const prometheusImportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'import_gauge', + help: 'Items imported by instance', + labelNames: ["instanceID", "itemName"], +}); const prometheusPlayerCountGauge = new Prometheus.Gauge({ name: prometheusPrefix+'player_count_gauge', help: 'Amount of players connected to this cluster', @@ -438,11 +448,13 @@ app.post("/api/place", authenticate.middleware, function(req, res) { key:req.body.name, value:req.body.count, }); + prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); // save items we get db.items.addItem({ name:req.body.name, count:req.body.count }); + // Attempt confirming res.send("success"); } else { @@ -514,6 +526,8 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } + + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); res.send({count: object.count, name: object.name}); httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } else { From 6dd25cb1e8d9485209723de3fc88bfd00767a019 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 6 May 2018 10:32:11 +0200 Subject: [PATCH 48/92] Don't assume playercount is 0 just because a command fails --- client.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/client.js b/client.js index 7f66dfdd..f176d98e 100644 --- a/client.js +++ b/client.js @@ -594,19 +594,7 @@ function instanceManagement() { // delete when we are done fs.unlink(instancedirectory + "/script-output/tempfile.txt", function(){}); } - // if we actually got anything from the file, proceed to categorize it - if (data && data[0]) { - while (data[0]) { - let q = data[0].split(" "); - // delete array element - data.splice(0,1); - if(q[0] == "connected_players" && Number(q[1]) != NaN) { - instanceInfo.playerCount = q[1]; - } - } - } else { - instanceInfo.playerCount = 0; - } + var payload = { time: Date.now(), rconPort: instanceconfig.clientPort, @@ -615,10 +603,21 @@ function instanceManagement() { unique: instanceconfig.unique, publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config mods:modHashes, - playerCount: instanceInfo.playerCount || 0, instanceName: instance, } + // if we actually got anything from the file, proceed to categorize it + if (data && data[0]) { + while (data[0]) { + let q = data[0].split(" "); + // delete array element + data.splice(0,1); + if(q[0] == "connected_players" && Number(q[1]) != NaN) { + payload.playerCount = q[1]; + } + } + } + function callback(err, mac) { if (err) { mac = "unknown"; From cf1a97ac9f488518040b7bfd94cb37986b3f6642 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 6 May 2018 14:30:03 +0200 Subject: [PATCH 49/92] Added caching to /api/slaves (default 5s) --- master.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/master.js b/master.js index fcb0bbc9..f67a4c6e 100644 --- a/master.js +++ b/master.js @@ -373,17 +373,25 @@ GET endpoint for getting information about all our slaves @instance @alias /api/slaves */ +let slaveCache = { + timestamp: Date.now(), +}; app.get("/api/slaves", function(req, res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); - // filter out the rcon password because thats kindof not a safe thing to share - for(key in copyOfSlaves){ - copyOfSlaves[key].rconPassword = "hidden"; + if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){ + let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); + // filter out the rcon password because thats kindof not a safe thing to share + for(key in copyOfSlaves){ + copyOfSlaves[key].rconPassword = "hidden"; + } + slaveCache.cache = copyOfSlaves; + slaveCache.timestamp = Date.now(); } - res.send(copyOfSlaves); + + res.send(slaveCache.cache); httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); var recievedItemStatisticsBySlaveID = {}; From ac55b1c442d4898dcc202f40b14b50fea729894e Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 6 May 2018 20:45:52 +0200 Subject: [PATCH 50/92] Prometheus track dole factor for imports --- master.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/master.js b/master.js index f67a4c6e..6ca8f282 100644 --- a/master.js +++ b/master.js @@ -115,6 +115,11 @@ const prometheusImportGauge = new Prometheus.Gauge({ help: 'Items imported by instance', labelNames: ["instanceID", "itemName"], }); +const prometheusDoleFactorGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'dole_factor_gauge', + help: 'The current dole division factor for this item', + labelNames: ["itemName"], +}); const prometheusPlayerCountGauge = new Prometheus.Gauge({ name: prometheusPrefix+'player_count_gauge', help: 'Amount of players connected to this cluster', @@ -535,6 +540,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } + prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); res.send({count: object.count, name: object.name}); httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); From 0b81fb63513d3e39b38c7edc2b70854bd3cb479b Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 7 May 2018 18:11:46 +0200 Subject: [PATCH 51/92] Made factorio command ratelimiting configurable --- client.js | 2 +- config.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index f176d98e..41dcb28e 100644 --- a/client.js +++ b/client.js @@ -645,7 +645,7 @@ function instanceManagement() { }); // messageInterface Management - setInterval(function(){let command=commandBuffer.pop();if(command){messageInterfaceInternal(command[0],command[1])}},50); + setInterval(function(){let command=commandBuffer.pop();if(command){messageInterfaceInternal(command[0],command[1])}},config.msBetweenCommands || 50); // Mod uploading and management ----------------------------------------------- // get mod names and hashes diff --git a/config.json b/config.json index 1589b042..01aba037 100644 --- a/config.json +++ b/config.json @@ -37,5 +37,6 @@ }, "logItemTransfers": false, "uploadModsToMaster": true, + "msBetweenCommands": 10, "allowRemoteCommandExecution": false } From 973f0b0e0a4acf29471aea4110a2d5785821153f Mon Sep 17 00:00:00 2001 From: Stefan Asbeck Date: Tue, 8 May 2018 22:04:02 +0200 Subject: [PATCH 52/92] Create new random masterAuthSecret if secret-api-token.txt doesn't exist. --- master.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/master.js b/master.js index 6ca8f282..7e0a1ad6 100644 --- a/master.js +++ b/master.js @@ -43,6 +43,18 @@ const nedb = require("nedb"); const jwt = require('jsonwebtoken'); const bcrypt = require('bcryptjs'); + +var crypto = require('crypto'); +var base64url = require('base64url'); + +/** Sync */ +function randomStringAsBase64Url(size) { + return base64url(crypto.randomBytes(size)); +} +if (!fs.existsSync("secret-api-token.txt")) { + config.masterAuthSecret = randomStringAsBase64Url(256); + fs.writeFileSync("config.json",JSON.stringify(config, null, 4)); +} // write an auth token to file fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { expiresIn: 86400*365 // expires in 1 year From 1d9b6a26a679d3b89dd1a17e3dbae815cf5440a2 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Thu, 10 May 2018 10:33:26 +0200 Subject: [PATCH 53/92] Attempted fix to #97 --- master.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/master.js b/master.js index 7e0a1ad6..ae3ec2a4 100644 --- a/master.js +++ b/master.js @@ -69,7 +69,11 @@ const fileUpload = require('express-fileupload'); var app = express(); app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ extended: true })); +app.use(bodyParser.urlencoded({ + parameterLimit: 100000, + limit: '10mb', + extended: true +})); app.use(fileUpload()); app.use(compression()); From ce6dc3af283f751d2b5096432737f1e4c1f51577 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Thu, 10 May 2018 14:15:03 +0200 Subject: [PATCH 54/92] messageInterface now takes a second arg for rcon return values (from rcon.print()) --- client.js | 42 +++++++++++++++++++++++------------------- package.json | 2 +- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/client.js b/client.js index 41dcb28e..0c3dcfe2 100644 --- a/client.js +++ b/client.js @@ -6,7 +6,7 @@ const path = require('path'); const syncRequest = require('sync-request'); const request = require("request"); const ncp = require('ncp').ncp; -const Rcon = require('simple-rcon'); +const Rcon = require('rcon-client').Rcon; const hashFiles = require('hash-files'); const _ = require('underscore'); const deepmerge = require("deepmerge"); @@ -73,14 +73,16 @@ function messageInterfaceInternal(command, callback) { if(typeof callback == "function"){ callback(); } - } else if(typeof command == "string" && client && client.exec && typeof client.exec == "function") { + } else if(typeof command == "string" && client && client.send && typeof client.send == "function") { try { - client.exec(command+"\n", callback); + client.send(command+"\n").then(str => { + if(typeof callback == "function") callback(str) + }); } catch (err) { console.log(err); - } - if(typeof callback == "function"){ - callback(); + if(typeof callback == "function"){ + callback(); + } } } } @@ -385,12 +387,7 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ // connect to the server with rcon if(true || process.platform != "linux"){ // IP, port, password - client = new Rcon({ - host: 'localhost', - port: Number(process.env.RCONPORT) || instanceconfig.clientPort, - password: instanceconfig.clientPassword, - timeout: 0 - }); + client = new Rcon(); // check the logfile to see if the RCON interface is running as there is no way to continue without it // we read the log every 2 seconds and stop looping when we start connecting to factorio @@ -398,23 +395,30 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ fs.readFile(instancedirectory+"/factorio-current.log", function (err, data) { // if (err) console.log(err); if(data && data.indexOf('Starting RCON interface') > 0){ - client.connect(); + client.connect({ + host: 'localhost', + port: Number(process.env.RCONPORT) || instanceconfig.clientPort, + password: instanceconfig.clientPassword, + timeout: 5000 + }); } else { setTimeout(function(){ checkRcon(); - },2000); + },5000); } }); } setTimeout(checkRcon, 5000); - client.on('authenticated', function () { + client.onDidAuthenticate(() => { console.log('Clusterio | RCON Authenticated!'); instanceManagement(); // start using rcons - }).on('connected', function () { + }); + client.onDidConnect(() => { console.log('Clusterio | RCON Connected!'); // getID(); - }).on('disconnected', function () { + }); + client.onDidDisconnect(() => { console.log('Clusterio | RCON Disconnected!'); process.exit(0); // exit because RCON disconnecting is undefined behaviour and we rather just wanna restart now }); @@ -462,11 +466,11 @@ function instanceManagement() { if(pluginConfig.binary == "nodePackage"){ // require index.js.main() of plugin and execute it as a class let pluginClass = require("./sharedPlugins/" + pluginDirectories[I] + "/index.js"); - plugins[I] = new pluginClass(combinedConfig, function(data){ + plugins[I] = new pluginClass(combinedConfig, function(data, callback){ if(data.toString('utf8')[0] != "/") { log("Stdout: " + data.toString('utf8')); } else { - messageInterface(data.toString('utf8')); + messageInterface(data.toString('utf8'), callback); } }, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat. socket, // socket.io connection to master (and ES6 destructuring, yay) diff --git a/package.json b/package.json index 3b0b8ac7..9247dfbd 100644 --- a/package.json +++ b/package.json @@ -48,10 +48,10 @@ "node-factorio-api": "^0.3.1-alpha", "node-forge": "^0.7.1", "prom-client": "^10.2.3", + "rcon-client": "^3.0.2", "request": "^2.83.0", "right-pad": "^1.0.1", "rmdir-sync": "^1.0.1", - "simple-rcon": "^0.3.1", "single-line-log": "^1.1.2", "socket.io": "^2.0.4", "socket.io-client": "^2.0.4", From 165cfffecb1005cdede65a2a1f6a8dc4e46824b0 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 13 May 2018 09:58:25 +0200 Subject: [PATCH 55/92] Use rcon.print instead of writefile for playercount reports --- client.js | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/client.js b/client.js index 0c3dcfe2..3f050152 100644 --- a/client.js +++ b/client.js @@ -591,14 +591,7 @@ function instanceManagement() { setInterval(getID, 10000); getID(); function getID() { - messageInterface("/silent-command game.write_file('tempfile.txt', 'connected_players ' .. #game.connected_players .. '\\n', true, 0)", function(err) {setTimeout(function(){ - // get array of lines in file - if(fs.existsSync(instancedirectory + "/script-output/tempfile.txt")) { - var data = fs.readFileSync(instancedirectory + "/script-output/tempfile.txt", "utf8").split("\n"); - // delete when we are done - fs.unlink(instancedirectory + "/script-output/tempfile.txt", function(){}); - } - + messageInterface("/silent-command rcon.print(#game.connected_players)", function(playerCount) { var payload = { time: Date.now(), rconPort: instanceconfig.clientPort, @@ -608,18 +601,7 @@ function instanceManagement() { publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config mods:modHashes, instanceName: instance, - } - - // if we actually got anything from the file, proceed to categorize it - if (data && data[0]) { - while (data[0]) { - let q = data[0].split(" "); - // delete array element - data.splice(0,1); - if(q[0] == "connected_players" && Number(q[1]) != NaN) { - payload.playerCount = q[1]; - } - } + playerCount, } function callback(err, mac) { @@ -644,7 +626,7 @@ function instanceManagement() { } else { getMac(callback); } - },1000)}); + }); } }); From e73c5055aff6d5a1bba20a3c80d599fc1d807e42 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 21 May 2018 00:00:42 +0200 Subject: [PATCH 56/92] !info to see servername (issue #101) --- sharedPlugins/globalChat/index.js | 72 ++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index 44d480f7..246d1904 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -1,11 +1,14 @@ +const needle = require("needle"); + module.exports = class remoteCommands { constructor(mergedConfig, messageInterface, extras){ this.messageInterface = messageInterface; this.config = mergedConfig; this.socket = extras.socket; + console.log(mergedConfig) this.instances = {}; this.socket.emit("registerChatReciever"); - this.socket.on("gameChat", data => { + this.socket.on("gameChat", async data => { if(data.instanceID && data.instanceID != this.config.unique.toString()){ if(data.data.includes("[CHAT]") && (data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){ let words = data.data.trim().split(" "); @@ -13,38 +16,55 @@ module.exports = class remoteCommands { words.shift(); words.shift(); let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout", ""); - let instance = this.instances[data.instanceID]; - if (!instance) { - let me = this; //to pass 'this' down to the callback - require('needle').get(this.config.masterIP + '/api/slaves', function (err, response) { - if(err || response.statusCode != 200) { - console.log("Unable to get JSON master/api/slaves, master might be unaccessible"); - } else if (response && response.body) { - if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} - try { - for (let index in response.body) - me.instances[index] = response.body[index]["instanceName"]; - } catch (e){ - console.log(e); - return null; - } - instance = me.instances[data.instanceID] - if (!instance) - instance = data.instanceID; //somehow the master doesn't know the instance - me.messageInterface("/silent-command game.print('[" + instance + "] " + message +"')"); - } - }); - } else { - this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')"); - } + + let instance = await this.getInstanceName(data.instanceID); + + this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')"); } } }); } - factorioOutput(data){ + async factorioOutput(data){ + try{ this.socket.emit("gameChat", { instanceID: this.config.unique.toString(), data, }); + if(data.includes("[CHAT]") && data.includes("!info")){ + let infoMessage = [ + "=== Server info ===", + "Name: "+await this.getInstanceName(this.config.unique)+" ("+this.config.unique+")", + ]; + infoMessage.forEach((line, i) => { + setTimeout(()=>this.messageInterface("/silent-command game.print('"+line+"')"),i*200); + }); + } + }catch(e){console.log(e)} + } + getInstanceName(instanceID){ + return new Promise((resolve, reject) => { + let instance = this.instances[instanceID]; + if(!instance){ + needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => { + if(err || response.statusCode != 200) { + console.log("Unable to get JSON master/api/slaves, master might be unaccessible"); + } else if (response && response.body) { + if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} + try { + for (let index in response.body) + this.instances[index] = response.body[index].instanceName; + } catch (e){ + console.log(e); + return null; + } + instance = this.instances[instanceID] + if (!instance) instance = instanceID; //somehow the master doesn't know the instance + resolve(instance); + } + }); + } else { + resolve(instance); + } + }); } } From 0ee2c5c2f6256e12478d436aad73c99399209548 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 26 May 2018 12:50:15 +0200 Subject: [PATCH 57/92] Resolves #103 --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 9247dfbd..98fdc942 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "app-root-path": "^2.0.1", "as-table": "^1.0.31", "averaged-timeseries": "^1.4.0", + "base64url": "^3.0.0", "bcryptjs": "^2.4.3", "body-parser": "^1.18.2", "codecov": "^3.0.1", From e6c706a83e62625aa4de62c7bd9bd19c9be9d28b Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 26 May 2018 13:16:59 +0200 Subject: [PATCH 58/92] globalChat no longer breaks when the master is restarted --- sharedPlugins/globalChat/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index 246d1904..52de1c11 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -7,7 +7,9 @@ module.exports = class remoteCommands { this.socket = extras.socket; console.log(mergedConfig) this.instances = {}; - this.socket.emit("registerChatReciever"); + + this.socket.on("hello", () => this.socket.emit("registerChatReciever")); + this.socket.on("gameChat", async data => { if(data.instanceID && data.instanceID != this.config.unique.toString()){ if(data.data.includes("[CHAT]") && (data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){ From e87c702cf238c247e436d937b94b52712f2a6948 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 19 May 2018 13:59:57 +0200 Subject: [PATCH 59/92] Added plugin functionality to master --- master.js | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/master.js b/master.js index ae3ec2a4..c6756f6d 100644 --- a/master.js +++ b/master.js @@ -27,6 +27,7 @@ var config = require('./config'); // homebrew modules const getFactorioLocale = require("./lib/getFactorioLocale"); const stringUtils = require("./lib/stringUtils"); +const fileOps = require("_app/fileOps"); // homemade express middleware for token auth const authenticate = require("./lib/authenticate"); @@ -1080,4 +1081,62 @@ io.on('connection', function (socket) { }); }); +// handle plugins on the master +pluginManagement(); +async function pluginManagement(){ + let startPluginLoad = Date.now(); + let plugins = await getPlugins("sharedPlugins"); + console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms"); +} +function getPlugins(pluginDirectory){ + return new Promise((resolve, reject) => { + let filesLeft = 0; + let plugins = []; + + function checkLoadComplete(plugin){ + if(!filesLeft){ + resolve(plugins); + } + } + fs.readdir(pluginDirectory, function(err, files){ + if(err){ + reject(err); + } else { + files.forEach(file => { + let pluginStartedLoading = Date.now(); // just for logging + filesLeft++; + let pluginPath = path.join(pluginDirectory, file); + fs.stat(pluginPath, function(err, stats){ + if(err){ + reject(err); + checkLoadComplete(); + } else if(stats.isDirectory()){ + let pluginConfig = require(path.resolve(pluginPath, "config")); + --filesLeft; + if(pluginConfig.masterPlugin){ + let masterPlugin = require(path.resolve(pluginPath, pluginConfig.masterPlugin)); + plugins.push(new masterPlugin({ + config, + pluginConfig, + path: pluginPath, + socketio: io, + express: app, + })); + + console.log("Loaded plugin "+pluginConfig.name+" in "+(Date.now() - pluginStartedLoading)+"ms"); + checkLoadComplete(); + } else { + checkLoadComplete(); + } + } else { + checkLoadComplete(); + } + }); + }); + } + }); + }); +} + + module.exports = app; From 88e8ab454fa42c034ea4b79ba8fef15636a433e1 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 19 May 2018 20:23:30 +0200 Subject: [PATCH 60/92] Possibility of having master-only plugins --- client.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 3f050152..a4862c5d 100644 --- a/client.js +++ b/client.js @@ -522,7 +522,7 @@ function instanceManagement() { } } console.log("Clusterio | Loaded plugin " + pluginDirectories[i]); - } else if(pluginConfig.binary != "nodePackage"){ + } else if(pluginConfig.binary != "nodePackage" && pluginConfig.binary){ // handle as fragile executable plugin let args = pluginConfig.args || []; plugins[I]=child_process.spawn(pluginConfig.binary, args, { @@ -583,6 +583,8 @@ function instanceManagement() { plugins[i].on('close', (code) => { log(`child process exited with code ${code}`); }); + } else { + // This plugin doesn't have a client portion } } From da6f96a34bd447814ce8562a615b52c34107a368 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Thu, 31 May 2018 21:15:34 +0200 Subject: [PATCH 61/92] UPSdisplay now uses rcon return value instead of game.write_file --- client.js | 4 ++-- sharedPlugins/UPSdisplay/index.js | 25 ++++++++++++------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/client.js b/client.js index a4862c5d..126074c1 100644 --- a/client.js +++ b/client.js @@ -290,8 +290,8 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ console.log("Instance created!"); } else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)) { // Exit if no instance specified (it should be, just a safeguard); - if(instancedirectory != "./instances/undefined"){ - var instanceconfig = require(instancedirectory + '/config'); + if(instancedirectory != config.instanceDirectory+"/undefined"){ + var instanceconfig = require(path.resolve(instancedirectory,'config')); instanceconfig.unique = stringUtils.hashCode(instanceconfig.clientPassword); if(process.env.FACTORIOPORT){ instanceconfig.factorioPort = process.env.FACTORIOPORT; diff --git a/sharedPlugins/UPSdisplay/index.js b/sharedPlugins/UPSdisplay/index.js index 5618783e..e8928abb 100644 --- a/sharedPlugins/UPSdisplay/index.js +++ b/sharedPlugins/UPSdisplay/index.js @@ -3,12 +3,20 @@ const needle = require("needle"); class UPSdisplay { - constructor(slaveConfig, msgInterface){ + constructor(slaveConfig, msgInterface, extras){ this.config = slaveConfig; this.messageInterface = msgInterface; + this.socket = extras.socket; + this.historicalTicks = []; setInterval(() => { - this.messageInterface("/silent-command game.write_file('UPSdisplay.txt', game.tick, true, 0)"); + this.messageInterface("/silent-command rcon.print(game.tick)", data => { + this.tick = Number(data); + this.historicalTicks[this.historicalTicks.length] = {tick:Number(data), timestamp:Date.now()}; + if(this.historicalTicks.length > 30){ + this.historicalTicks.shift(); // delete last element in array (position 0); + } + }); },1000); setInterval(() => { if(this.historicalTicks && this.historicalTicks[0]){ @@ -17,24 +25,15 @@ class UPSdisplay { let UPS = Math.round(ticksInPeriod / (timePeriod/1000)); // console.log("UPS: " + UPS); try{ - needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS}}, {headers: {'x-access-token': this.config.masterAuthToken}}, function(err, resp) { + needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS, tick:this.tick}}, {headers: {'x-access-token': this.config.masterAuthToken}}, function(err, resp) { // success? }); } catch (err){ console.log(err); } } - }, 10000); + }, 5000); this.messageInterface("/silent-command game.print('UPSdisplay enabled')"); } - scriptOutput(data){ - if (data !== null && !isNaN(data)){ - // console.log("Tick: " + data); - this.historicalTicks[this.historicalTicks.length] = {tick:Number(data), timestamp:Date.now()}; - if(this.historicalTicks.length > 30){ - this.historicalTicks.shift(); // delete last element in array (position 0); - } - } - } } module.exports = UPSdisplay; \ No newline at end of file From 812265d0fdbe7ca98c99b6df5c774e2fe83260fc Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Tue, 22 May 2018 08:15:03 +0200 Subject: [PATCH 62/92] messageInterface now returns a promise as well --- client.js | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/client.js b/client.js index 126074c1..9f6750cc 100644 --- a/client.js +++ b/client.js @@ -57,8 +57,16 @@ const needleOptionsWithTokenAuthHeader = { var instanceInfo = {}; var commandBuffer=[]; +// messageInterface Management +setInterval(function(){ + let command=commandBuffer.shift(); + if(command){ + messageInterfaceInternal(command[0], command[1], command[2], command[3]); + } +},config.msBetweenCommands || 50); + // function to handle sending commands into the game -function messageInterfaceInternal(command, callback) { +async function messageInterfaceInternal(command, callback, resolve, reject) { // try to save us if you send a buffer instead of string if(typeof command == "object") { command = command.toString('utf8'); @@ -73,20 +81,26 @@ function messageInterfaceInternal(command, callback) { if(typeof callback == "function"){ callback(); } + resolve(); } else if(typeof command == "string" && client && client.send && typeof client.send == "function") { try { - client.send(command+"\n").then(str => { - if(typeof callback == "function") callback(str) - }); + let str = await client.send(command+"\n"); + if(typeof callback == "function") callback(str) + resolve(str) } catch (err) { console.log(err); if(typeof callback == "function"){ callback(); } + reject(err) } } } -function messageInterface(command, callback) {commandBuffer.push([command,callback]);} +function messageInterface(command, callback) { + return new Promise((resolve,reject) => { + commandBuffer.push([command,callback, resolve, reject]); + }); +} // handle commandline parameters @@ -466,11 +480,12 @@ function instanceManagement() { if(pluginConfig.binary == "nodePackage"){ // require index.js.main() of plugin and execute it as a class let pluginClass = require("./sharedPlugins/" + pluginDirectories[I] + "/index.js"); - plugins[I] = new pluginClass(combinedConfig, function(data, callback){ + plugins[I] = new pluginClass(combinedConfig, async function(data, callback){ if(data.toString('utf8')[0] != "/") { log("Stdout: " + data.toString('utf8')); + return true; } else { - messageInterface(data.toString('utf8'), callback); + return messageInterface(data.toString('utf8'), callback); } }, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat. socket, // socket.io connection to master (and ES6 destructuring, yay) @@ -631,10 +646,6 @@ function instanceManagement() { }); } }); - - // messageInterface Management - setInterval(function(){let command=commandBuffer.pop();if(command){messageInterfaceInternal(command[0],command[1])}},config.msBetweenCommands || 50); - // Mod uploading and management ----------------------------------------------- // get mod names and hashes // string: instance, function: callback @@ -650,7 +661,7 @@ function instanceManagement() { needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { if(err) console.log(new Error("Unable to contact master server! Please check your config.json.")); if(response && body && body == "found") { - console.log("master has mod"); + console.log("master has mod "+modHashes[i].modName); } else if (response && body && typeof body == "string") { let mod = response.body; if(config.uploadModsToMaster){ From f72ea7357e8629a5c2060c53c1cc170cd8c72afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Sun, 3 Jun 2018 17:52:21 +0200 Subject: [PATCH 63/92] Removed debug line that put auth info in the logs --- sharedPlugins/globalChat/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index 52de1c11..29c33a7a 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -5,7 +5,6 @@ module.exports = class remoteCommands { this.messageInterface = messageInterface; this.config = mergedConfig; this.socket = extras.socket; - console.log(mergedConfig) this.instances = {}; this.socket.on("hello", () => this.socket.emit("registerChatReciever")); From 226e157f690fa79d97faaed7897b684d93ae90ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Wed, 6 Jun 2018 12:54:11 +0200 Subject: [PATCH 64/92] Added onExit handler to masterPlugins (see description) To use the onExit handler, simply add a method called onExit to your exported class. If it returns a promise, the master will wait for it to resolve. --- master.js | 31 +++++++++++++++++++++++-------- package.json | 4 ++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/master.js b/master.js index c6756f6d..6037a450 100644 --- a/master.js +++ b/master.js @@ -242,8 +242,9 @@ db.items.removeItem = function(object) { } // store slaves and inventory in a .json full of JSON data -process.on('SIGINT', function () { +process.on('SIGINT', async function () { console.log('Ctrl-C...'); + let exitStartTime = Date.now(); // set insane limit to slave length, if its longer than this we are probably being ddosed or something if(slaves && Object.keys(slaves).length < 2000000){ console.log("saving to slaves.json"); @@ -257,6 +258,15 @@ process.on('SIGINT', function () { } else if(slaves) { console.log("Item database too large, not saving ("+Object.keys(slaves).length+")"); } + for(let i in masterPlugins){ + let plugin = masterPlugins[i]; + if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ + let startTime = Date.now(); + await plugin.main.onExit(); + console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms"); + } + } + console.log("Exited in "+(Date.now()-exitStartTime)+"ms"); process.exit(2); }); @@ -1083,11 +1093,13 @@ io.on('connection', function (socket) { // handle plugins on the master pluginManagement(); +masterPlugins = []; async function pluginManagement(){ let startPluginLoad = Date.now(); - let plugins = await getPlugins("sharedPlugins"); + masterPlugins = await getPlugins("sharedPlugins"); console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms"); } + function getPlugins(pluginDirectory){ return new Promise((resolve, reject) => { let filesLeft = 0; @@ -1115,13 +1127,16 @@ function getPlugins(pluginDirectory){ --filesLeft; if(pluginConfig.masterPlugin){ let masterPlugin = require(path.resolve(pluginPath, pluginConfig.masterPlugin)); - plugins.push(new masterPlugin({ - config, + plugins.push({ + main:new masterPlugin({ + config, + pluginConfig, + path: pluginPath, + socketio: io, + express: app, + }), pluginConfig, - path: pluginPath, - socketio: io, - express: app, - })); + }); console.log("Loaded plugin "+pluginConfig.name+" in "+(Date.now() - pluginStartedLoading)+"ms"); checkLoadComplete(); diff --git a/package.json b/package.json index 98fdc942..f40f7cb2 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,8 @@ "ejs": "^2.5.7", "express": "^4.16.2", "express-fileupload": "0.0.6", - "follow-redirects": "^1.4.1", - "fs-extra": "^1.0.0", + "follow-redirects": "^1.5.0", + "fs-extra": "^6.0.1", "getmac": "^1.4.0", "hash-files": "^1.1.1", "html5-validator": "^1.0.7", From 17b5967b0f9d3aadc0d522417c2f2ad5f382d830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Wed, 6 Jun 2018 12:57:17 +0200 Subject: [PATCH 65/92] Added mention about masterAuthToken in the readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2c50731b..3c95657a 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ downloads and installs nodejs, pm2, git and clusterio. To specify a version, cha To download the mod for all its non vanilla features and items, (optional) node client.js download + +Add the master API key, found in the masters install directory (secret-api-token.txt) to your config.json masterAuthToken field To create a new instance (its own save, set of mods and config files) From 034db541529f28d75b2ee3227dc4b4f41e6dba53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Wed, 6 Jun 2018 21:33:08 +0200 Subject: [PATCH 66/92] Added lots of web interface related stuff for external plugin --- master.js | 11 +++++--- static/head.html | 16 ++++++------ static/main.js | 63 +++++++++++++++++++++++++++++++-------------- static/sidebar.html | 9 +++++-- 4 files changed, 66 insertions(+), 33 deletions(-) diff --git a/master.js b/master.js index 6037a450..e9f18df0 100644 --- a/master.js +++ b/master.js @@ -83,10 +83,15 @@ app.set('views', path.join(__dirname, 'static')); app.set('view engine', 'html'); app.engine('html', ejs.renderFile); -var routes = require("./routes.js"); -routes(app); - +// give ejs access to some interesting information +app.use(function(req, res, next){ + res.locals.res = res; + res.locals.req = req; + res.locals.masterPlugins = masterPlugins; + next(); +}); +require("./routes.js")(app); // Set folder to serve static content from (the website) app.use(express.static('static')); // mod downloads diff --git a/static/head.html b/static/head.html index c503fe6b..943c3324 100644 --- a/static/head.html +++ b/static/head.html @@ -1,9 +1,9 @@ Clusterio - - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/static/main.js b/static/main.js index a1b805f1..0f3d43bd 100644 --- a/static/main.js +++ b/static/main.js @@ -49,6 +49,24 @@ debugTools = { } } +let cachedInstanceNames = {}; +async function getInstanceName(instanceID){ + let instance = cachedInstanceNames[instanceID]; + if(!instance){ + let slaves = await getJSON("/api/slaves"); + try { + for(id in slaves){ + cachedInstanceNames[id] = slaves[id].instanceName; + } + } catch(e){ + console.log("Error while looking for instance name!"); + console.log(e); + } + return cachedInstanceNames[instanceID] || instanceID; + } else { + return cachedInstanceNames[instanceID]; + } +} var g = {} contents = { "iron-plate":100, @@ -65,22 +83,27 @@ function djb2(str){ } // callback(err, json) function getJSON(url, callback) { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, true); - xhr.responseType = 'json'; - xhr.onload = function() { - var status = xhr.status; - if (status == 200) { - callback(null, xhr.response); - } else { - callback(status); - } - }; - // triggers if connection is refused - xhr.onerror = function(e){ - callback(e); - }; - xhr.send(); + return new Promise((resolve, reject) => { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.responseType = 'json'; + xhr.onload = function() { + var status = xhr.status; + if (status == 200) { + if(callback) callback(null, xhr.response); + resolve(xhr.response); + } else { + if(callback) callback(status); + resolve(status); + } + }; + // triggers if connection is refused + xhr.onerror = function(e){ + if(callback) callback(e); + reject(e); + }; + xhr.send(); + }); }; // callback(err, json) function postJSON(url, data, callback) { @@ -89,12 +112,12 @@ function postJSON(url, data, callback) { xhr.setRequestHeader('Content-Type', 'application/json; charset=UTF-8'); xhr.responseType = 'json'; xhr.onload = function() { - var status = xhr.status; - if (status == 200) { + var status = xhr.status; + if (status == 200) { callback(null, xhr.response); - } else { + } else { callback(status); - } + } }; // triggers if connection is refused xhr.onerror = function(e){ diff --git a/static/sidebar.html b/static/sidebar.html index b24a4d2e..84f60cde 100644 --- a/static/sidebar.html +++ b/static/sidebar.html @@ -8,10 +8,15 @@ -