diff --git a/.gitignore b/.gitignore index 0270b57f..9752a88d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ mod_cache.json static/pictures/spritesheet.js static/pictures/spritesheet.png static/external +saves # weird pattern to ignore sharedPlugins except not ignore some core modules !sharedPlugins/ diff --git a/README.md b/README.md index 3a67c918..3c95657a 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ downloads and installs nodejs, pm2, git and clusterio. To specify a version, cha To download the mod for all its non vanilla features and items, (optional) node client.js download + +Add the master API key, found in the masters install directory (secret-api-token.txt) to your config.json masterAuthToken field To create a new instance (its own save, set of mods and config files) @@ -97,7 +99,7 @@ To create a new instance (its own save, set of mods and config files) To launch an instance with pm2 - pm2 start --name slave client.js -- start [instancename] + pm2 start --name [instancename] client.js -- start [instancename] use `nano config.json` to change settings. diff --git a/client.js b/client.js index 90d0b351..9f6750cc 100644 --- a/client.js +++ b/client.js @@ -6,7 +6,7 @@ const path = require('path'); const syncRequest = require('sync-request'); const request = require("request"); const ncp = require('ncp').ncp; -const Rcon = require('simple-rcon'); +const Rcon = require('rcon-client').Rcon; const hashFiles = require('hash-files'); const _ = require('underscore'); const deepmerge = require("deepmerge"); @@ -56,15 +56,23 @@ const needleOptionsWithTokenAuthHeader = { }; var instanceInfo = {}; +var commandBuffer=[]; +// messageInterface Management +setInterval(function(){ + let command=commandBuffer.shift(); + if(command){ + messageInterfaceInternal(command[0], command[1], command[2], command[3]); + } +},config.msBetweenCommands || 50); // function to handle sending commands into the game -function messageInterface(command, callback) { +async function messageInterfaceInternal(command, callback, resolve, reject) { // try to save us if you send a buffer instead of string if(typeof command == "object") { command = command.toString('utf8'); } - if(process.platform == "linux" && typeof command == "string" && serverprocess) { + if(false && process.platform == "linux" && typeof command == "string" && serverprocess) { /* to send to stdin, use: serverprocess.stdin.write("/c command;\n") @@ -73,17 +81,27 @@ function messageInterface(command, callback) { if(typeof callback == "function"){ callback(); } - } else if(typeof command == "string" && client && client.exec && typeof client.exec == "function") { + resolve(); + } else if(typeof command == "string" && client && client.send && typeof client.send == "function") { try { - client.exec(command+"\n", callback); + let str = await client.send(command+"\n"); + if(typeof callback == "function") callback(str) + resolve(str) } catch (err) { console.log(err); - } - if(typeof callback == "function"){ - callback(); + if(typeof callback == "function"){ + callback(); + } + reject(err) } } } +function messageInterface(command, callback) { + return new Promise((resolve,reject) => { + commandBuffer.push([command,callback, resolve, reject]); + }); +} + // handle commandline parameters if (!command || command == "help" || command == "--help") { @@ -286,8 +304,8 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ console.log("Instance created!"); } else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)) { // Exit if no instance specified (it should be, just a safeguard); - if(instancedirectory != "./instances/undefined"){ - var instanceconfig = require(instancedirectory + '/config'); + if(instancedirectory != config.instanceDirectory+"/undefined"){ + var instanceconfig = require(path.resolve(instancedirectory,'config')); instanceconfig.unique = stringUtils.hashCode(instanceconfig.clientPassword); if(process.env.FACTORIOPORT){ instanceconfig.factorioPort = process.env.FACTORIOPORT; @@ -381,14 +399,9 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ }); // connect to the server with rcon - if(process.platform != "linux"){ + if(true || process.platform != "linux"){ // IP, port, password - client = new Rcon({ - host: 'localhost', - port: Number(process.env.RCONPORT) || instanceconfig.clientPort, - password: instanceconfig.clientPassword, - timeout: 0 - }); + client = new Rcon(); // check the logfile to see if the RCON interface is running as there is no way to continue without it // we read the log every 2 seconds and stop looping when we start connecting to factorio @@ -396,23 +409,30 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ fs.readFile(instancedirectory+"/factorio-current.log", function (err, data) { // if (err) console.log(err); if(data && data.indexOf('Starting RCON interface') > 0){ - client.connect(); + client.connect({ + host: 'localhost', + port: Number(process.env.RCONPORT) || instanceconfig.clientPort, + password: instanceconfig.clientPassword, + timeout: 5000 + }); } else { setTimeout(function(){ checkRcon(); - },2000); + },5000); } }); } setTimeout(checkRcon, 5000); - client.on('authenticated', function () { + client.onDidAuthenticate(() => { console.log('Clusterio | RCON Authenticated!'); instanceManagement(); // start using rcons - }).on('connected', function () { + }); + client.onDidConnect(() => { console.log('Clusterio | RCON Connected!'); // getID(); - }).on('disconnected', function () { + }); + client.onDidDisconnect(() => { console.log('Clusterio | RCON Disconnected!'); process.exit(0); // exit because RCON disconnecting is undefined behaviour and we rather just wanna restart now }); @@ -460,11 +480,12 @@ function instanceManagement() { if(pluginConfig.binary == "nodePackage"){ // require index.js.main() of plugin and execute it as a class let pluginClass = require("./sharedPlugins/" + pluginDirectories[I] + "/index.js"); - plugins[I] = new pluginClass(combinedConfig, function(data){ + plugins[I] = new pluginClass(combinedConfig, async function(data, callback){ if(data.toString('utf8')[0] != "/") { log("Stdout: " + data.toString('utf8')); + return true; } else { - messageInterface(data.toString('utf8')); + return messageInterface(data.toString('utf8'), callback); } }, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat. socket, // socket.io connection to master (and ES6 destructuring, yay) @@ -504,7 +525,9 @@ function instanceManagement() { } for(let i = 0; i < stuff.length; i++) { if(stuff[i] && !stuff[i].includes('\u0000\u0000')) { - plugins[I].scriptOutput(stuff[i]); + try{ + plugins[I].scriptOutput(stuff[i]); + }catch(e){console.log(e)} } } }, @@ -514,7 +537,7 @@ function instanceManagement() { } } console.log("Clusterio | Loaded plugin " + pluginDirectories[i]); - } else if(pluginConfig.binary != "nodePackage"){ + } else if(pluginConfig.binary != "nodePackage" && pluginConfig.binary){ // handle as fragile executable plugin let args = pluginConfig.args || []; plugins[I]=child_process.spawn(pluginConfig.binary, args, { @@ -575,6 +598,8 @@ function instanceManagement() { plugins[i].on('close', (code) => { log(`child process exited with code ${code}`); }); + } else { + // This plugin doesn't have a client portion } } @@ -583,26 +608,7 @@ function instanceManagement() { setInterval(getID, 10000); getID(); function getID() { - messageInterface("/silent-command game.write_file('tempfile.txt', 'connected_players ' .. #game.connected_players .. '\\n', true, 0)", function(err) {setTimeout(function(){ - // get array of lines in file - if(fs.existsSync(instancedirectory + "/script-output/tempfile.txt")) { - var data = fs.readFileSync(instancedirectory + "/script-output/tempfile.txt", "utf8").split("\n"); - // delete when we are done - fs.unlink(instancedirectory + "/script-output/tempfile.txt", function(){}); - } - // if we actually got anything from the file, proceed to categorize it - if (data && data[0]) { - while (data[0]) { - let q = data[0].split(" "); - // delete array element - data.splice(0,1); - if(q[0] == "connected_players" && Number(q[1]) != NaN) { - instanceInfo.playerCount = q[1]; - } - } - } else { - instanceInfo.playerCount = 0; - } + messageInterface("/silent-command rcon.print(#game.connected_players)", function(playerCount) { var payload = { time: Date.now(), rconPort: instanceconfig.clientPort, @@ -611,37 +617,35 @@ function instanceManagement() { unique: instanceconfig.unique, publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config mods:modHashes, - playerCount: instanceInfo.playerCount || 0, instanceName: instance, + playerCount, } function callback(err, mac) { if (err) { + mac = "unknown"; console.log("##### getMac crashed, but we don't really give a shit because we are probably closing down #####"); - } else { - payload.mac = mac - console.log("Registered our precense with master "+config.masterIP+" at " + payload.time); - needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { - if (err && err.code != "ECONNRESET"){ - console.error("We got problems, something went wrong when contacting master"); - console.error(err); - } else if (response && response.body) { - // In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care. - console.log(response.body); - } - }); } + payload.mac = mac; + console.log("Registered our precense with master "+config.masterIP+" at " + payload.time); + needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { + if (err && err.code != "ECONNRESET"){ + console.error("We got problems, something went wrong when contacting master"); + console.error(err); + } else if (response && response.body) { + // In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care. + console.log(response.body); + } + }); } if(global.mac){ callback(undefined, global.mac); } else { getMac(callback); } - },1000)}); + }); } }); - - // Mod uploading and management ----------------------------------------------- // get mod names and hashes // string: instance, function: callback @@ -657,7 +661,7 @@ function instanceManagement() { needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { if(err) console.log(new Error("Unable to contact master server! Please check your config.json.")); if(response && body && body == "found") { - console.log("master has mod"); + console.log("master has mod "+modHashes[i].modName); } else if (response && body && typeof body == "string") { let mod = response.body; if(config.uploadModsToMaster){ @@ -818,9 +822,7 @@ function instanceManagement() { needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) { if (response && response.body && typeof response.body == "object") { // buffer confirmed orders - confirmedOrders[confirmedOrders.length] = { - [response.body.name]: response.body.count, - } + confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count} if(config.logItemTransfers){ console.log(`Imported ${response.body.count} ${response.body.name} from master`); } @@ -829,11 +831,21 @@ function instanceManagement() { } // if we got some confirmed orders // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); - sadas = JSON.stringify(confirmedOrders); - confirmedOrders = []; - // send our RCON command with whatever we got - messageInterface("/silent-command remote.call('clusterio', 'importMany', '" + sadas + "')"); - sadas = null; + //if (!(confirmedOrders.length>0)){return;} + let cmd="local t={"; + for(let i=0;i320) // Factorio max packet size is 508 + { + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); + cmd="local t={"; + } + } + if (!(cmd==="local t={")){ + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); + } + confirmedOrders=[]; } }, 1000); // COMBINATOR SIGNALS --------------------------------------------------------- @@ -847,22 +859,37 @@ function instanceManagement() { if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver try { var inventory = JSON.parse(response.body); + var inventoryFrame = {}; + for (let i = 0; i < inventory.length; i++) { + inventoryFrame[inventory[i].name] = Number(inventory[i].count); + if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){ + inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice + } + } + inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); + // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); + let first = true; + let cmd="local s={"; + for (let key in inventoryFrame) + { + cmd+='["'+key+'"]='+inventoryFrame[key]+","; + if(first && cmd.length>300 || !first && cmd.length > 320) // Factorio max packet size is 508 + { + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); + cmd="local s={"; + first = false; + } + } + if (!(cmd==="local s={")){ + messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); + } + messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')"); } catch (e){ console.log(e); } - var inventoryFrame = {}; - for (let i = 0; i < inventory.length; i++) { - inventoryFrame[inventory[i].name] = Number(inventory[i].count); - if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){ - inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice - } - } - inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); - // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); } }); - }, 1000); + }, 1550); // Make sure world has its worldID setTimeout(function(){ messageInterface("/silent-command remote.call('clusterio','setWorldID',"+instanceconfig.unique+")") diff --git a/config.json b/config.json index aba81cbc..01aba037 100644 --- a/config.json +++ b/config.json @@ -17,6 +17,8 @@ "username": "", "__comment4": "Token from your factorio.com profile page. Required for MP and mod manager services.", "token": "", + "factorio_version": "0.16.41", + "__comment7": "Used for communication with the mod portal, not needed if you manage mods by hand", "__comment5": "General user settings", "verify_user_identity": false, @@ -35,5 +37,6 @@ }, "logItemTransfers": false, "uploadModsToMaster": true, + "msBetweenCommands": 10, "allowRemoteCommandExecution": false } diff --git a/lib/getFactorioLocale.js b/lib/getFactorioLocale.js index 7a28d58c..72d84dd0 100644 --- a/lib/getFactorioLocale.js +++ b/lib/getFactorioLocale.js @@ -6,6 +6,8 @@ Module for grabbing factorio locale from a local factorio install const fs = require("fs"); const ini = require("ini"); +const cache = {}; + /** Gets raw factorio locale from the .ini file. Not available outside of module through exports. @@ -32,14 +34,20 @@ Gets factorios locale as an object, does not respect mods. */ module.exports.asObject = function getLocaleAsObject(factorioDirectory, languageCode, callback){ if(typeof factorioDirectory != "string" || typeof languageCode != "string" || typeof callback != "function"){ - throw "Error: wrong parameters provided"; + throw new Error("Error: wrong parameters provided"); + } + + if(!cache[languageCode]){ + getLocale(factorioDirectory, languageCode, (err, rawLocale) => { + if(err){ + callback(err); + } else { + let processedLocale = ini.parse(rawLocale); + cache[languageCode] = processedLocale; + callback(undefined, processedLocale); + } + }); + } else { + callback(undefined, cache[languageCode]); } - getLocale(factorioDirectory, languageCode, (err, rawLocale) => { - if(err){ - callback(err); - } else { - let processedLocale = ini.parse(rawLocale) - callback(undefined, processedLocale); - } - }); } diff --git a/lib/manager/modManager.js b/lib/manager/modManager.js index 1f8d53a8..fec45562 100644 --- a/lib/manager/modManager.js +++ b/lib/manager/modManager.js @@ -36,7 +36,7 @@ modManager.listMods("shared"); */ module.exports.listMods = function(instance){ if(instance && instances.includes(instance)){ - factorioAPI.init(false, 'instances/'+instance+'/instanceMods', 'instances/'+instance+'/saves', '0.16.15'); + factorioAPI.init(false, 'instances/'+instance+'/instanceMods', 'instances/'+instance+'/saves', config.factorio_version); factorioAPI.authenticate({ username: config.username, token: config.token, @@ -65,7 +65,7 @@ module.exports.listMods = function(instance){ }); } else { // list shared mods in /sharedMods/ - factorioAPI.init(false, 'sharedMods/', '', '0.16.15'); + factorioAPI.init(false, 'sharedMods/', '', config.factorio_version); factorioAPI.authenticate({ username: config.username, token: config.token, @@ -106,6 +106,7 @@ module.exports.findMods = function(searchTerm){ name:ellipse(mod.name, 30), downloads:mod.downloads_count, updated: moment(mod.updated_at).format("DD.MM.YY"), + latest_version: mod.latest_release.version, owner:mod.owner, //license: mod.license_name, } @@ -123,15 +124,15 @@ module.exports.addMod = function(term, instance){ if(term && term.length > 0){ if(term.includes("mods.factorio.com/mods/")){ // https://mods.factorio.com/mods/Danielv123/ChooChoo // treat it as a link download - factorioAPI.init(false, modDirectory, "", "0.16.15"); + factorioAPI.init(false, modDirectory, "", config.factorio_version); factorioAPI.authenticate({ username: config.username, token: config.token, require_ownership: false }).then(() => { factorioAPI.searchMods(term.split("/")[term.split("/").length-1]).then(body => { - console.log("Im gonna download: "+body[0].name); - factorioAPI.downloadMod(body[0]).then(()=>{ + console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version); + factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{ console.log("Mod downloaded"); }); }); @@ -166,15 +167,15 @@ module.exports.addMod = function(term, instance){ } } else { // treat it as a search and download the top result - factorioAPI.init(false, modDirectory, "", "0.16.15"); + factorioAPI.init(false, modDirectory, "", config.factorio_version); factorioAPI.authenticate({ username: config.username, token: config.token, require_ownership: false }).then(() => { factorioAPI.searchMods(term).then(body => { - console.log("Im gonna download: "+body[0].name); - factorioAPI.downloadMod(body[0]).then(()=>{ + console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version); + factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{ console.log("Mod downloaded"); }); }); diff --git a/lib/npmPostinstall.js b/lib/npmPostinstall.js index 8106fad4..187407bf 100644 --- a/lib/npmPostinstall.js +++ b/lib/npmPostinstall.js @@ -5,7 +5,7 @@ var fs = require('fs'); var modManager = require("./manager/modManager.js"); -modManager.addMod("clusterio", "shared"); +// modManager.addMod("clusterio", "shared"); function makeDirectory(path){ return new Promise((resolve, reject) => { diff --git a/master.js b/master.js index 3d9376ae..24bdfa8e 100644 --- a/master.js +++ b/master.js @@ -27,6 +27,7 @@ var config = require('./config'); // homebrew modules const getFactorioLocale = require("./lib/getFactorioLocale"); const stringUtils = require("./lib/stringUtils"); +const fileOps = require("_app/fileOps"); // homemade express middleware for token auth const authenticate = require("./lib/authenticate"); @@ -44,12 +45,25 @@ const https = require("https"); const jwt = require('jsonwebtoken'); const bcrypt = require('bcryptjs'); + +var crypto = require('crypto'); +var base64url = require('base64url'); + +/** Sync */ +function randomStringAsBase64Url(size) { + return base64url(crypto.randomBytes(size)); +} +if (!fs.existsSync("secret-api-token.txt")) { + config.masterAuthSecret = randomStringAsBase64Url(256); + fs.writeFileSync("config.json",JSON.stringify(config, null, 4)); +} // write an auth token to file fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { expiresIn: 86400*365 // expires in 1 year })); const express = require("express"); +const compression = require('compression'); const ejs = require("ejs"); // Required for express post requests const bodyParser = require("body-parser"); @@ -57,18 +71,28 @@ const fileUpload = require('express-fileupload'); var app = express(); app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ extended: true })); +app.use(bodyParser.urlencoded({ + parameterLimit: 100000, + limit: '10mb', + extended: true +})); app.use(fileUpload()); +app.use(compression()); // dynamic HTML generations with EJS app.set('views', path.join(__dirname, 'static')); app.set('view engine', 'html'); app.engine('html', ejs.renderFile); -var routes = require("./routes.js"); -routes(app); - +// give ejs access to some interesting information +app.use(function(req, res, next){ + res.locals.res = res; + res.locals.req = req; + res.locals.masterPlugins = masterPlugins; + next(); +}); +require("./routes.js")(app); // Set folder to serve static content from (the website) app.use(express.static('static')); // mod downloads @@ -97,13 +121,28 @@ const prometheusConnectedInstancesCounter = new Prometheus.Gauge({ const prometheusWsUsageCounter = new Prometheus.Counter({ name: prometheusPrefix+'websocket_usage_counter', help: 'Websocket traffic', - labelNames: ["connectionType"], + labelNames: ["connectionType", "instanceID"], }); const prometheusProductionGauge = new Prometheus.Gauge({ name: prometheusPrefix+'production_gauge', help: 'Items produced by instance', labelNames: ["instanceID", "itemName"], }); +const prometheusExportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'export_gauge', + help: 'Items exported by instance', + labelNames: ["instanceID", "itemName"], +}); +const prometheusImportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'import_gauge', + help: 'Items imported by instance', + labelNames: ["instanceID", "itemName"], +}); +const prometheusDoleFactorGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'dole_factor_gauge', + help: 'The current dole division factor for this item', + labelNames: ["itemName"], +}); const prometheusPlayerCountGauge = new Prometheus.Gauge({ name: prometheusPrefix+'player_count_gauge', help: 'Amount of players connected to this cluster', @@ -209,8 +248,9 @@ db.items.removeItem = function(object) { } // store slaves and inventory in a .json full of JSON data -process.on('SIGINT', function () { +process.on('SIGINT', async function () { console.log('Ctrl-C...'); + let exitStartTime = Date.now(); // set insane limit to slave length, if its longer than this we are probably being ddosed or something if(slaves && Object.keys(slaves).length < 2000000){ console.log("saving to slaves.json"); @@ -224,6 +264,15 @@ process.on('SIGINT', function () { } else if(slaves) { console.log("Item database too large, not saving ("+Object.keys(slaves).length+")"); } + for(let i in masterPlugins){ + let plugin = masterPlugins[i]; + if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ + let startTime = Date.now(); + await plugin.main.onExit(); + console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms"); + } + } + console.log("Exited in "+(Date.now()-exitStartTime)+"ms"); process.exit(2); }); @@ -292,6 +341,8 @@ POST Get metadata from a single slave. For whenever you find the data returned b @alias /api/getSlaveMeta */ app.post("/api/getSlaveMeta", function (req, res) { + endpointHitCounter.labels(req.route.path).inc(); + let reqStartTime = Date.now(); console.log("body", req.body); if(req.body && req.body.instanceID && req.body.password){ console.log("returning meta for ", req.body.instanceID); @@ -300,6 +351,7 @@ app.post("/api/getSlaveMeta", function (req, res) { res.status(400); res.send('{"INVALID REQUEST":1}'); } + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); // mod management // should handle uploading and checking if mods are uploaded @@ -359,17 +411,25 @@ GET endpoint for getting information about all our slaves @instance @alias /api/slaves */ +let slaveCache = { + timestamp: Date.now(), +}; app.get("/api/slaves", function(req, res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); - // filter out the rcon password because thats kindof not a safe thing to share - for(key in copyOfSlaves){ - copyOfSlaves[key].rconPassword = "hidden"; + if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){ + let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); + // filter out the rcon password because thats kindof not a safe thing to share + for(key in copyOfSlaves){ + copyOfSlaves[key].rconPassword = "hidden"; + } + slaveCache.cache = copyOfSlaves; + slaveCache.timestamp = Date.now(); } - res.send(copyOfSlaves); + + res.send(slaveCache.cache); httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); var recievedItemStatisticsBySlaveID = {}; @@ -434,11 +494,13 @@ app.post("/api/place", authenticate.middleware, function(req, res) { key:req.body.name, value:req.body.count, }); + prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); // save items we get db.items.addItem({ name:req.body.name, count:req.body.count }); + // Attempt confirming res.send("success"); } else { @@ -510,6 +572,9 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } + + prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); res.send({count: object.count, name: object.name}); httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } else { @@ -700,7 +765,7 @@ app.post("/api/getStats", function(req,res) { } else if(req.body.statistic == "remove"){ console.log(`sending remove data for instanceID ${req.body.instanceID} ${req.body.itemName}`); // Gather data - console.log(sentItemStatisticsBySlaveID) + // console.log(sentItemStatisticsBySlaveID) let itemStats = sentItemStatisticsBySlaveID[req.body.instanceID]; console.log(itemStats) if(typeof itemStats == "object"){ @@ -783,10 +848,12 @@ Requires x-access-token header to be set. Find you api token in secret-api-token @memberof clusterioMaster @instance @alias api/runCommand -@param {object} JSON {instanceID:19412312, command:"/c game.print('hello')"} +@param {object} JSON {instanceID:19412312, broadcast:false, command:"/c game.print('hello')"} @returns {object} Status {auth: bool, message: "Informative error", data:{}} */ app.post("/api/runCommand", (req,res) => { + endpointHitCounter.labels(req.route.path).inc(); + let reqStartTime = Date.now(); var token = req.headers['x-access-token']; if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' }); @@ -809,6 +876,7 @@ app.post("/api/runCommand", (req,res) => { wsSlaves[instanceID].runCommand(body.command); } res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"}); + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } else if ( body.instanceID && wsSlaves[body.instanceID] @@ -819,9 +887,11 @@ app.post("/api/runCommand", (req,res) => { // execute command wsSlaves[body.instanceID].runCommand(body.command, data => { res.status(200).send({auth: true, message: "success", data}); + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); } else { res.status(400).send({auth: true, message: "Error: invalid request.body"}); + httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); } }); }); @@ -866,14 +936,17 @@ class slaveMapper { this.lastBeat = Date.now(); this.socket.on("heartbeat", () => { + prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc(); // we aren't ready to die yet apparently this.lastBeat = Date.now(); }); this.socket.on("sendChunk", function(data){ + prometheusWsUsageCounter.labels('sendChunk', this.instanceID).inc(); mapRequesters[data.requesterID].socket.emit("displayChunk", data); }); // slaveMapper sent us an entity update, process this.socket.on("sendEntity", entity => { + prometheusWsUsageCounter.labels('sendEntity', this.instanceID).inc(); Object.keys(mapRequesters).forEach(requesterName => { let requester = mapRequesters[requesterName]; @@ -894,12 +967,12 @@ class mapRequester { this.lastBeat = Date.now(); this.socket.on("heartbeat", () => { - prometheusWsUsageCounter.labels('heartbeat').inc(); + prometheusWsUsageCounter.labels('heartbeat', "other").inc(); // we aren't ready to die yet apparently this.lastBeat = Date.now(); }); this.socket.on("requestChunk", loc => { - prometheusWsUsageCounter.labels('requestChunk').inc(); + prometheusWsUsageCounter.labels('requestChunk', "other").inc(); loc.requesterID = this.requesterID; let instanceID = loc.instanceID || this.instanceID; if(slaveMappers[instanceID] && typeof loc.x == "number" && typeof loc.y == "number"){ @@ -907,7 +980,7 @@ class mapRequester { } }); this.socket.on("requestEntity", req => { - prometheusWsUsageCounter.labels('requestEntity').inc(); + prometheusWsUsageCounter.labels('requestEntity', "other").inc(); req.requesterID = this.requesterID; let instanceID = req.instanceID || this.instanceID; if(slaveMappers[instanceID] && typeof req.x == "number" && typeof req.y == "number"){ @@ -915,7 +988,7 @@ class mapRequester { } }); this.socket.on("placeEntity", req => { - prometheusWsUsageCounter.labels('placeEntity').inc(); + prometheusWsUsageCounter.labels('placeEntity', "other").inc(); req.requesterID = this.requesterID; let instanceID = req.instanceID || this.instanceID; if(slaveMappers[instanceID]){ @@ -933,11 +1006,11 @@ class wsSlave { this.lastBeat = Date.now(); this.socket.on("heartbeat", () => { - prometheusWsUsageCounter.labels('heartbeat').inc(); + prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc(); this.lastBeat = Date.now(); }); this.socket.on("combinatorSignal", circuitFrameWithMeta => { - prometheusWsUsageCounter.labels('combinatorSignal').inc(); + prometheusWsUsageCounter.labels('combinatorSignal', this.instanceID).inc(); if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object"){ Object.keys(wsSlaves).forEach(instanceID => { wsSlaves[instanceID].socket.emit("processCombinatorSignal", circuitFrameWithMeta); @@ -947,6 +1020,7 @@ class wsSlave { // handle command return values this.commandsWaitingForReturn = {}; this.socket.on("runCommandReturnValue", resp => { + prometheusWsUsageCounter.labels('runCommandReturnValue', this.instanceID).inc(); if(resp.commandID && resp.body && this.commandsWaitingForReturn[resp.commandID] && this.commandsWaitingForReturn[resp.commandID].callback && typeof this.commandsWaitingForReturn[resp.commandID].callback == "function"){ this.commandsWaitingForReturn[resp.commandID].callback(resp.body); delete this.commandsWaitingForReturn[resp.commandID]; @@ -954,6 +1028,7 @@ class wsSlave { }); this.socket.on("gameChat", data => { + prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc(); if(typeof data === "object"){ data.timestamp = Date.now(); if(!global.wsChatRecievers) global.wsChatRecievers = []; @@ -962,6 +1037,17 @@ class wsSlave { }); } }); + + this.socket.on("alert", alert => { + prometheusWsUsageCounter.labels('alert', this.instanceID).inc(); + if(typeof alert === "object"){ + alert.timestamp = Date.now(); + if(!global.wsAlertRecievers) global.wsAlertRecievers = []; + global.wsAlertRecievers.forEach(socket => { + socket.emit("alert", alert); + }); + } + }); } runCommand(command, callback){ let commandID = Math.random().toString(); @@ -989,13 +1075,13 @@ io.on('connection', function (socket) { /* initial processing for remoteMap */ socket.on('registerSlaveMapper', function (data) { - prometheusWsUsageCounter.labels('registerSlaveMapper').inc(); + prometheusWsUsageCounter.labels('registerSlaveMapper', "other").inc(); slaveMappers[data.instanceID] = new slaveMapper(data.instanceID, socket); console.log("remoteMap | SOCKET registered map provider for "+data.instanceID); }); socket.on('registerMapRequester', function(data){ // data {instanceID:""} - prometheusWsUsageCounter.labels('registerMapRequester').inc(); + prometheusWsUsageCounter.labels('registerMapRequester', "other").inc(); let requesterID = Math.random().toString(); mapRequesters[requesterID] = new mapRequester(requesterID, socket, data.instanceID); socket.emit("mapRequesterReady", true); @@ -1004,17 +1090,85 @@ io.on('connection', function (socket) { /* Websockets for send and recieve combinators */ socket.on("registerSlave", function(data) { - prometheusWsUsageCounter.labels('registerSlave').inc(); + prometheusWsUsageCounter.labels('registerSlave', "other").inc(); if(data && data.instanceID){ wsSlaves[data.instanceID] = new wsSlave(data.instanceID, socket); console.log("SOCKET | Created new wsSlave: "+ data.instanceID); } }); socket.on("registerChatReciever", function(data){ - prometheusWsUsageCounter.labels("registerChatReciever").inc(); + prometheusWsUsageCounter.labels("registerChatReciever", "other").inc(); if(!global.wsChatRecievers) global.wsChatRecievers = []; global.wsChatRecievers.push(socket); }); + socket.on("registerAlertReciever", function(data){ + prometheusWsUsageCounter.labels("registerAlertReciever", "other").inc(); + if(!global.wsAlertRecievers) global.wsAlertRecievers = []; + global.wsAlertRecievers.push(socket); + }); }); +// handle plugins on the master +pluginManagement(); +masterPlugins = []; +async function pluginManagement(){ + let startPluginLoad = Date.now(); + masterPlugins = await getPlugins("sharedPlugins"); + console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms"); +} + +function getPlugins(pluginDirectory){ + return new Promise((resolve, reject) => { + let filesLeft = 0; + let plugins = []; + + function checkLoadComplete(plugin){ + if(!filesLeft){ + resolve(plugins); + } + } + fs.readdir(pluginDirectory, function(err, files){ + if(err){ + reject(err); + } else { + files.forEach(file => { + let pluginStartedLoading = Date.now(); // just for logging + filesLeft++; + let pluginPath = path.join(pluginDirectory, file); + fs.stat(pluginPath, function(err, stats){ + if(err){ + reject(err); + checkLoadComplete(); + } else if(stats.isDirectory()){ + let pluginConfig = require(path.resolve(pluginPath, "config")); + --filesLeft; + if(pluginConfig.masterPlugin){ + let masterPlugin = require(path.resolve(pluginPath, pluginConfig.masterPlugin)); + plugins.push({ + main:new masterPlugin({ + config, + pluginConfig, + path: pluginPath, + socketio: io, + express: app, + }), + pluginConfig, + }); + + console.log("Loaded plugin "+pluginConfig.name+" in "+(Date.now() - pluginStartedLoading)+"ms"); + checkLoadComplete(); + } else { + checkLoadComplete(); + } + } else { + checkLoadComplete(); + } + }); + }); + } + }); + }); +} + + module.exports = app; diff --git a/package.json b/package.json index 76666614..f40f7cb2 100644 --- a/package.json +++ b/package.json @@ -21,15 +21,17 @@ "app-root-path": "^2.0.1", "as-table": "^1.0.31", "averaged-timeseries": "^1.4.0", + "base64url": "^3.0.0", "bcryptjs": "^2.4.3", "body-parser": "^1.18.2", - "codecov": "^2.3.1", + "codecov": "^3.0.1", + "compression": "^1.7.2", "deepmerge": "^1.5.2", "ejs": "^2.5.7", "express": "^4.16.2", "express-fileupload": "0.0.6", - "follow-redirects": "^1.4.1", - "fs-extra": "^1.0.0", + "follow-redirects": "^1.5.0", + "fs-extra": "^6.0.1", "getmac": "^1.4.0", "hash-files": "^1.1.1", "html5-validator": "^1.0.7", @@ -47,10 +49,10 @@ "node-factorio-api": "^0.3.1-alpha", "node-forge": "^0.7.1", "prom-client": "^10.2.3", + "rcon-client": "^3.0.2", "request": "^2.83.0", "right-pad": "^1.0.1", "rmdir-sync": "^1.0.1", - "simple-rcon": "^0.3.1", "single-line-log": "^1.1.2", "socket.io": "^2.0.4", "socket.io-client": "^2.0.4", diff --git a/sharedPlugins/UPSdisplay/index.js b/sharedPlugins/UPSdisplay/index.js index 5618783e..e8928abb 100644 --- a/sharedPlugins/UPSdisplay/index.js +++ b/sharedPlugins/UPSdisplay/index.js @@ -3,12 +3,20 @@ const needle = require("needle"); class UPSdisplay { - constructor(slaveConfig, msgInterface){ + constructor(slaveConfig, msgInterface, extras){ this.config = slaveConfig; this.messageInterface = msgInterface; + this.socket = extras.socket; + this.historicalTicks = []; setInterval(() => { - this.messageInterface("/silent-command game.write_file('UPSdisplay.txt', game.tick, true, 0)"); + this.messageInterface("/silent-command rcon.print(game.tick)", data => { + this.tick = Number(data); + this.historicalTicks[this.historicalTicks.length] = {tick:Number(data), timestamp:Date.now()}; + if(this.historicalTicks.length > 30){ + this.historicalTicks.shift(); // delete last element in array (position 0); + } + }); },1000); setInterval(() => { if(this.historicalTicks && this.historicalTicks[0]){ @@ -17,24 +25,15 @@ class UPSdisplay { let UPS = Math.round(ticksInPeriod / (timePeriod/1000)); // console.log("UPS: " + UPS); try{ - needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS}}, {headers: {'x-access-token': this.config.masterAuthToken}}, function(err, resp) { + needle.post(this.config.masterIP+':'+this.config.masterPort+'/api/editSlaveMeta', {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS, tick:this.tick}}, {headers: {'x-access-token': this.config.masterAuthToken}}, function(err, resp) { // success? }); } catch (err){ console.log(err); } } - }, 10000); + }, 5000); this.messageInterface("/silent-command game.print('UPSdisplay enabled')"); } - scriptOutput(data){ - if (data !== null && !isNaN(data)){ - // console.log("Tick: " + data); - this.historicalTicks[this.historicalTicks.length] = {tick:Number(data), timestamp:Date.now()}; - if(this.historicalTicks.length > 30){ - this.historicalTicks.shift(); // delete last element in array (position 0); - } - } - } } module.exports = UPSdisplay; \ No newline at end of file diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index 283a6184..29c33a7a 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -1,24 +1,71 @@ +const needle = require("needle"); + module.exports = class remoteCommands { constructor(mergedConfig, messageInterface, extras){ this.messageInterface = messageInterface; this.config = mergedConfig; this.socket = extras.socket; + this.instances = {}; - this.socket.emit("registerChatReciever"); - this.socket.on("gameChat", data => { + this.socket.on("hello", () => this.socket.emit("registerChatReciever")); + + this.socket.on("gameChat", async data => { if(data.instanceID && data.instanceID != this.config.unique.toString()){ - if(data.data.includes("[CHAT]") && data.data.toLowerCase().includes("!shout")){ - let chatMessage = "["+data.instanceID+"]"+ data.data.trim().split(" ").shift().shift().shift().join(" "); - this.messageInterface("/c game.print('"+chatMessage+"')"); + if(data.data.includes("[CHAT]") && (data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){ + let words = data.data.trim().split(" "); + words.shift(); + words.shift(); + words.shift(); + let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout", ""); + + let instance = await this.getInstanceName(data.instanceID); + + this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')"); } } }); } - factorioOutput(data){ - // console.log("GOT CHAT!"+data) + async factorioOutput(data){ + try{ this.socket.emit("gameChat", { instanceID: this.config.unique.toString(), data, }); + if(data.includes("[CHAT]") && data.includes("!info")){ + let infoMessage = [ + "=== Server info ===", + "Name: "+await this.getInstanceName(this.config.unique)+" ("+this.config.unique+")", + ]; + infoMessage.forEach((line, i) => { + setTimeout(()=>this.messageInterface("/silent-command game.print('"+line+"')"),i*200); + }); + } + }catch(e){console.log(e)} + } + getInstanceName(instanceID){ + return new Promise((resolve, reject) => { + let instance = this.instances[instanceID]; + if(!instance){ + needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => { + if(err || response.statusCode != 200) { + console.log("Unable to get JSON master/api/slaves, master might be unaccessible"); + } else if (response && response.body) { + if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} + try { + for (let index in response.body) + this.instances[index] = response.body[index].instanceName; + } catch (e){ + console.log(e); + return null; + } + instance = this.instances[instanceID] + if (!instance) instance = instanceID; //somehow the master doesn't know the instance + resolve(instance); + } + }); + } else { + resolve(instance); + } + }); } } diff --git a/static/head.html b/static/head.html index c503fe6b..943c3324 100644 --- a/static/head.html +++ b/static/head.html @@ -1,9 +1,9 @@ Clusterio - - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/static/main.js b/static/main.js index a1b805f1..0f3d43bd 100644 --- a/static/main.js +++ b/static/main.js @@ -49,6 +49,24 @@ debugTools = { } } +let cachedInstanceNames = {}; +async function getInstanceName(instanceID){ + let instance = cachedInstanceNames[instanceID]; + if(!instance){ + let slaves = await getJSON("/api/slaves"); + try { + for(id in slaves){ + cachedInstanceNames[id] = slaves[id].instanceName; + } + } catch(e){ + console.log("Error while looking for instance name!"); + console.log(e); + } + return cachedInstanceNames[instanceID] || instanceID; + } else { + return cachedInstanceNames[instanceID]; + } +} var g = {} contents = { "iron-plate":100, @@ -65,22 +83,27 @@ function djb2(str){ } // callback(err, json) function getJSON(url, callback) { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, true); - xhr.responseType = 'json'; - xhr.onload = function() { - var status = xhr.status; - if (status == 200) { - callback(null, xhr.response); - } else { - callback(status); - } - }; - // triggers if connection is refused - xhr.onerror = function(e){ - callback(e); - }; - xhr.send(); + return new Promise((resolve, reject) => { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.responseType = 'json'; + xhr.onload = function() { + var status = xhr.status; + if (status == 200) { + if(callback) callback(null, xhr.response); + resolve(xhr.response); + } else { + if(callback) callback(status); + resolve(status); + } + }; + // triggers if connection is refused + xhr.onerror = function(e){ + if(callback) callback(e); + reject(e); + }; + xhr.send(); + }); }; // callback(err, json) function postJSON(url, data, callback) { @@ -89,12 +112,12 @@ function postJSON(url, data, callback) { xhr.setRequestHeader('Content-Type', 'application/json; charset=UTF-8'); xhr.responseType = 'json'; xhr.onload = function() { - var status = xhr.status; - if (status == 200) { + var status = xhr.status; + if (status == 200) { callback(null, xhr.response); - } else { + } else { callback(status); - } + } }; // triggers if connection is refused xhr.onerror = function(e){ diff --git a/static/sidebar.html b/static/sidebar.html index b24a4d2e..84f60cde 100644 --- a/static/sidebar.html +++ b/static/sidebar.html @@ -8,10 +8,15 @@ -