From 04743551753ec5fec9b5b0edc03967417cd9bd5b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 20 Nov 2019 22:34:10 +0100 Subject: [PATCH 01/16] Fix README port from 1.2.3 Fix the new instructions for installing clusterio on Windows not being corrected to install the master branch when it was ported over. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 549b63bb..8a0f2b36 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ reboot when you are done, then proceed to the next steps. *reboots matter* 1. Open PowerShell or Command prompt in the directory you want to install to and run the following commands. - git clone -b 1.2.x https://github.com/clusterio/factorioClusterio + git clone -b master https://github.com/clusterio/factorioClusterio cd factorioClusterio npm install --only=production copy config.json.dist config.json From 1d26212e1d1ed68816dc17833c54bcc16875f88a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 24 Oct 2019 21:44:41 +0200 Subject: [PATCH 02/16] Remove port and password arguments to client.js Remove undocumented command line options set the ports and password used in the instance. These are not compatible with remote management. --- CHANGELOG.md | 3 +++ Dockerfile | 2 +- client.js | 26 ++++++++++---------------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fa3e217..3ab3ceef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,9 @@ Version 2.0.0 /api/inventory, /api/inventoryAsObject and the web interface view for the storage page into the clusterioMod plugin. If you disable this plugin then these things will not be available. +- Removed undocumented --port and --rcon-port arguments to client. +- Removed undocumented FACTORIOPORT and RCONPORT enviroment variable + handling in client. ### Breaking Changes diff --git a/Dockerfile b/Dockerfile index 040399de..dc217a35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ VOLUME /factorioClusterio/instances VOLUME /factorioClusterio/sharedMods VOLUME /factorioClusterio/sharedPlugins -CMD RCONPORT="$RCONPORT" FACTORIOPORT="$FACTORIOPORT" MODE="$MODE" node $MODE start $INSTANCE +CMD MODE="$MODE" node $MODE start $INSTANCE diff --git a/client.js b/client.js index bbf4987f..ab5b976e 100644 --- a/client.js +++ b/client.js @@ -270,7 +270,7 @@ async function symlinkMods(instance, sharedMods, logger) { } } -async function createInstance(config, args, instance) { +async function createInstance(config, instance) { console.log(`Creating ${instance.path()}`); await fs.ensureDir(instance.path()); await fs.ensureDir(instance.path("script-output")); @@ -278,10 +278,10 @@ async function createInstance(config, args, instance) { await symlinkMods(instance, "sharedMods", console); let instconf = { "id": Math.random() * 2 ** 31 | 0, - "factorioPort": args.port || process.env.FACTORIOPORT || null, - "clientPort": args["rcon-port"] || process.env.RCONPORT || null, + "factorioPort": null, + "clientPort": null, "__comment_clientPassword": "This is the rcon password. Will be randomly generated if null.", - "clientPassword": args["rcon-password"] || null, + "clientPassword": null, "info": {} } console.log("Clusterio | Created instance with settings:") @@ -335,7 +335,7 @@ async function createInstance(config, args, instance) { console.log("Clusterio | Successfully created instance"); } -async function startInstance(config, args, instance) { +async function startInstance(config, instance) { var instanceconfig = JSON.parse(await fs.readFile(instance.path("config.json"))); if (typeof instanceconfig.id !== "number" || isNaN(instanceconfig.id)) { throw new Error(`${instance.path("config.json")} is missing id`); @@ -343,12 +343,6 @@ async function startInstance(config, args, instance) { // Temporary measure for backwards compatibility instanceconfig.unique = instanceconfig.id; - if (process.env.FACTORIOPORT) { - instanceconfig.factorioPort = process.env.FACTORIOPORT; - } - if (process.env.RCONPORT) { - instanceconfig.rconPort = process.env.RCONPORT; - } console.log("Deleting .tmp.zip files"); let savefiles = fs.readdirSync(instance.path("saves")); for(i = 0; i < savefiles.length; i++){ @@ -419,9 +413,9 @@ async function startInstance(config, args, instance) { await factorio.patch(instance.path("saves", latestSave), modules); let options = { - gamePort: args.port || Number(process.env.FACTORIOPORT) || instanceconfig.factorioPort, - rconPort: args["rcon-port"] || Number(process.env.RCONPORT) || instanceconfig.clientPort, - rconPassword: args["rcon-password"] || instanceconfig.clientPassword, + gamePort: instanceconfig.factorioPort, + rconPort: instanceconfig.clientPort, + rconPassword: instanceconfig.clientPassword, }; let server = new factorio.FactorioServer(path.join("factorio", "data"), instance.path(), options); @@ -526,9 +520,9 @@ async function startClient() { console.error("Usage: node client start [instanceName]"); process.exit(0); } else if (command == "start" && !fs.existsSync(instance.path())) { - await createInstance(config, args, instance); + await createInstance(config, instance); } else if (command == "start" && fs.existsSync(instance.path())) { - await startInstance(config, args, instance); + await startInstance(config, instance); } else { console.error("Invalid arguments, quitting."); process.exit(1); From f3171c2feb1fede99ebcff505798275152b977d9 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 17 Nov 2019 23:11:23 +0100 Subject: [PATCH 03/16] Remove support for uploading mods to master Remove the uploadMod and checkMod APIs as these have become more or less obsolete after Factorio implementd syncing mods to the server in game. --- CHANGELOG.md | 2 ++ client.js | 42 ------------------------------------- config.json.dist | 1 - master.js | 54 ------------------------------------------------ 4 files changed, 2 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab3ceef..05d74489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,8 @@ Version 2.0.0 - Removed getLua and getCommand from lib/clusterTools. If you need to run more than the most trivial of code in commands use the save patcher and add in a remote interface. +- Removed mod uploading and distributing from the HTTP interface. Breaks + the old unmaintained and no longer needed factorioClusterioClient. Version 1.2.3 diff --git a/client.js b/client.js index ab5b976e..325eef0c 100644 --- a/client.js +++ b/client.js @@ -704,48 +704,6 @@ async function instanceManagement(config, instance, instanceconfig, server) { }); } }); - // Mod uploading and management ----------------------------------------------- - // get mod names and hashes - // string: instance, function: callback - - setTimeout(function(){hashMods(instance, uploadMods)}, 5000); - function uploadMods(modHashes) { - // [{modName:string,hash:string}, ... ] - for(let i=0;i { - if(file == req.body.modName) { - found = true; - } - }); - if(!found) { - // we don't have mod, plz send - res.send(req.body.modName); - } else { - res.send("found"); - } - res.end(); -}); -/** -POST endpoint for uploading mods to the master server. Required for automatic mod downloads with factorioClusterioClient (electron app, see seperate repo) -@memberof clusterioMaster -@instance -@alias /api/uploadMod -*/ -app.post("/api/uploadMod", authenticate.middleware, function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - if (req.files && req.files.file) { - // console.log(req.files.file); - req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err) { - if (err) { - res.status(500).send(err); - } else { - res.send('File uploaded!'); - console.log("Uploaded mod: " + req.files.file.name); - } - }); - } else { - res.send('No files were uploaded.'); - - } -}); /** Prepare the slaveCache for both /api/slaves calls */ @@ -671,12 +623,6 @@ async function startServer() { config.databaseDirectory = args.databaseDirectory || config.databaseDirectory || "./database"; await fs.ensureDir(config.databaseDirectory); - const masterModFolder = path.join(config.databaseDirectory, "masterMods"); - await fs.ensureDir(masterModFolder); - - // mod downloads - app.use(express.static(masterModFolder)); - await loadDatabase(config.databaseDirectory); authenticate.setAuthSecret(config.masterAuthSecret); From 2f3b4c1aa8707471355ddf5d29c93259f65d8563 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 18 Nov 2019 13:03:36 +0100 Subject: [PATCH 04/16] Remove redundant API call in globalChat Remove call to getInstanceName which in turn calls the rather expensive /api/slaves in the globalChat plugin. --- CHANGELOG.md | 1 + sharedPlugins/globalChat/index.js | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05d74489..ff0d2d82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Version 2.0.0 - Removed undocumented --port and --rcon-port arguments to client. - Removed undocumented FACTORIOPORT and RCONPORT enviroment variable handling in client. +- Removed redundant call to /api/slaves in globalChat plugin. ### Breaking Changes diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index f520ec85..da9246fc 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -1,12 +1,9 @@ -const needle = require("needle"); - module.exports = class remoteCommands { constructor(mergedConfig, messageInterface, extras){ this.messageInterface = messageInterface; this.config = mergedConfig; this.socket = extras.socket; this.instances = {}; - this.getInstanceName = require("lib/clusterTools/getInstanceName")(mergedConfig); this.socket.on("hello", () => this.socket.emit("registerChatReciever")); @@ -25,11 +22,10 @@ module.exports = class remoteCommands { words.shift(); let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout ","").replace(/\[gps=.*?\]/g, "").replace(/\[train=.*?\]/g, ""); - let instance = await this.getInstanceName(data.instanceID); if(data.data.includes("[gps=")){ //Nothing }else{ - this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')"); + this.messageInterface("/silent-command game.print('[" + data.instanceName + "] " + message + "')"); } } } @@ -39,6 +35,7 @@ module.exports = class remoteCommands { try{ this.socket.emit("gameChat", { instanceID: this.config.unique.toString(), + instanceName: this.config.name, data, }); if(data.includes("[CHAT]") && data.includes("!info")){ From d2d058e7d67cd5f760ee9d62771bc366eb4bd86e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 18 Nov 2019 14:05:25 +0100 Subject: [PATCH 05/16] Remove the broken serverManager plugin The remote management interface will completely change with the implementation of #201. --- .gitignore | 4 - CHANGELOG.md | 1 + sharedPlugins/serverManager/config.js | 13 -- sharedPlugins/serverManager/index.js | 34 ---- .../serverManager/js/api-endpoints.js | 88 ---------- sharedPlugins/serverManager/js/ui.js | 19 --- sharedPlugins/serverManager/masterPlugin.js | 151 ------------------ sharedPlugins/serverManager/static/index.js | 23 --- .../serverManager/static/serverManager.css | 19 --- sharedPlugins/serverManager/test/missing.js | 5 - sharedPlugins/serverManager/views/index.ejs | 38 ----- 11 files changed, 1 insertion(+), 394 deletions(-) delete mode 100644 sharedPlugins/serverManager/config.js delete mode 100644 sharedPlugins/serverManager/index.js delete mode 100644 sharedPlugins/serverManager/js/api-endpoints.js delete mode 100644 sharedPlugins/serverManager/js/ui.js delete mode 100644 sharedPlugins/serverManager/masterPlugin.js delete mode 100644 sharedPlugins/serverManager/static/index.js delete mode 100644 sharedPlugins/serverManager/static/serverManager.css delete mode 100644 sharedPlugins/serverManager/test/missing.js delete mode 100644 sharedPlugins/serverManager/views/index.ejs diff --git a/.gitignore b/.gitignore index cca9e56a..a5a56e68 100644 --- a/.gitignore +++ b/.gitignore @@ -52,10 +52,6 @@ sharedPlugins/* !sharedPlugins/researchSync/ !sharedPlugins/globalChat/ !sharedPlugins/clusterioMod/ -!sharedPlugins/serverManager/ -sharedPlugins/serverManager/database/* -sharedPlugins/serverManager/config.json -sharedPlugins/serverManager/secret-api-token.txt !sharedPlugins/statisticsExporter/ # File used for marking plugins as disabled diff --git a/CHANGELOG.md b/CHANGELOG.md index ff0d2d82..cff654d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Version 2.0.0 - Removed undocumented FACTORIOPORT and RCONPORT enviroment variable handling in client. - Removed redundant call to /api/slaves in globalChat plugin. +- Removed broken serverManager plugin. ### Breaking Changes diff --git a/sharedPlugins/serverManager/config.js b/sharedPlugins/serverManager/config.js deleted file mode 100644 index 9b1f8277..00000000 --- a/sharedPlugins/serverManager/config.js +++ /dev/null @@ -1,13 +0,0 @@ -/* - Clusterio plugin for remote server management -*/ -module.exports = { - name: "serverManager", - version: "1.0.0", - binary: "nodePackage", - description: "Server management tools for remote config edits and web based server management", - masterPlugin: "masterPlugin.js", - dependencies: [ - "playerManager", - ] -} diff --git a/sharedPlugins/serverManager/index.js b/sharedPlugins/serverManager/index.js deleted file mode 100644 index 3208b305..00000000 --- a/sharedPlugins/serverManager/index.js +++ /dev/null @@ -1,34 +0,0 @@ -const needle = require("needle"); -const fs = require("fs-extra"); - -const pluginConfig = require("./config"); - -module.exports = class remoteCommands { - constructor(mergedConfig, messageInterface, extras){ - this.messageInterface = messageInterface; - this.config = mergedConfig; - this.socket = extras.socket; - const pluginManager = require("lib/manager/pluginManager"); - // const modManager = require("lib/manager/modManager"); - this.socket.on("hello", () => { - this.socket.emit("registerServerManager"); - }); - this.socket.on("serverManagerGetStatus", async data => { - - }); - this.socket.on("serverManagerGetMods", async (data, callback) => { - // TODO: Implement this function separately from listMods (which console.logs it) - // let mods = modManager.getMods(this.config) - // callback(mods); - }); - this.socket.on("serverManagerGetPlugins", async (data, callback) => { - callback(await pluginManager.getPlugins()); - }); - this.socket.on("serverManagerEnablePlugin", async (data, callback) => { - callback(await pluginManager.enablePlugin(data.name, data.instanceID /* Currently ignored */)); - }); - this.socket.on("serverManagerDisablePlugin", async (data, callback) => { - callback(await pluginManager.disablePlugin(data.name, data.instanceID /* Currently ignored */)); - }); - } -} diff --git a/sharedPlugins/serverManager/js/api-endpoints.js b/sharedPlugins/serverManager/js/api-endpoints.js deleted file mode 100644 index 40cd7c7e..00000000 --- a/sharedPlugins/serverManager/js/api-endpoints.js +++ /dev/null @@ -1,88 +0,0 @@ -module.exports = that => { - /** - - Enable a plugin by deleting the DISABLED file - - Use socket.io to update masters list of plugins for this server - - Give standard JSON object response - */ - that.app.post("/api/serverManager/enablePlugin", async (req, res) => { - if((req.body.token || req.cookies.token) - && req.body.instanceID - && typeof req.body.instanceID == "string" - && req.body.pluginName - && typeof req.body.pluginName == "string" - && that.playerManager){ - let perms = await that.playerManager.main.getPermissions((req.body.token || req.cookies.token)); - if(perms.cluster.includes("enablePlugin") - || (perms.instance[req.body.instanceID] - && perms.instance[req.body.instanceID].includes("enablePlugin"))){ - that.serverSockets[req.body.instanceID].socket.emit("serverManagerEnablePlugin", { - name: req.body.pluginName, - instanceID: req.body.instanceID, - }, async resp => { - if(resp.ok){ - await updateData("plugins", req.body.instanceID); - } - res.send(resp); - }); - } else { - res.send({ - ok:false, - msg:"Insufficient permissions", - }); - } - } else { - res.send({ - ok:false, - msg:"Invalid request body", - }); - } - }); - /** - - Disable a plugin by adding an empty file named DISABLED to the plugins install pathname - - Use socket.io to update masters lost of plugins for this server - - Give standard JSON object response - */ - that.app.post("/api/serverManager/disablePlugin", async (req, res) => { - if((req.body.token || req.cookies.token) - && req.body.instanceID - && typeof req.body.instanceID == "string" - && req.body.pluginName - && typeof req.body.pluginName == "string" - && that.playerManager){ - let perms = await that.playerManager.main.getPermissions((req.body.token || req.cookies.token)); - if(perms.cluster.includes("disablePlugin") - || (perms.instance[req.body.instanceID] - && perms.instance[req.body.instanceID].includes("disablePlugin"))){ - that.serverSockets[req.body.instanceID].socket.emit("serverManagerDisablePlugin", { - name: req.body.pluginName, - instanceID: req.body.instanceID, - }, async resp => { - if(resp.ok){ - await updateData("plugins", req.body.instanceID); - } - res.send(resp); - }); - } else { - res.send({ - ok:false, - msg:"Insufficient permissions", - }); - } - } else { - res.send({ - ok:false, - msg:"Invalid request body", - }); - } - }); - async function updateData(type, instanceID){ - // Retrieve the latest ${type} from the server and store it - if(type == "plugins"){ - that.serverSockets[instanceID].socket.emit("serverManagerGetPlugins", {}, data => { - that.serverSockets[instanceID].plugins = data; - }); - } - } -} diff --git a/sharedPlugins/serverManager/js/ui.js b/sharedPlugins/serverManager/js/ui.js deleted file mode 100644 index a7d23289..00000000 --- a/sharedPlugins/serverManager/js/ui.js +++ /dev/null @@ -1,19 +0,0 @@ -let masterPlugin; - -module.exports = { - injectThis: function(x){masterPlugin = x; return this}, - ui: { - slaveListing: [ - { - name:"ServerMangerButton", - getHtml: ({slave}) => { - let html = ""; - if(masterPlugin.serverSockets[slave.unique]){ - let socket = masterPlugin.serverSockets[slave.unique]; - return `Server manager` - } - }, - } - ], - } -} diff --git a/sharedPlugins/serverManager/masterPlugin.js b/sharedPlugins/serverManager/masterPlugin.js deleted file mode 100644 index 6c458389..00000000 --- a/sharedPlugins/serverManager/masterPlugin.js +++ /dev/null @@ -1,151 +0,0 @@ -const fs = require("fs-extra"); -const path = require("path"); -const Express = require("express"); -const ejs = require("ejs"); -const bcrypt = require("bcrypt-promise"); -const crypto = require('crypto'); -const base64url = require('base64url'); -const sanitizer = require('sanitizer'); - - -class masterPlugin { - constructor({config, pluginConfig, pluginPath, socketio, express}){ - this.config = config; - this.pluginConfig = pluginConfig; - this.pluginPath = pluginPath; - this.io = socketio; - this.app = express; - - // load databases - // const database = getDatabaseSync("database/playerManager.json"); - // this.whitelist = getDatabaseSync("database/whitelist.json").whitelist || []; - // this.banlist = getDatabaseSync("database/banlist.json").banlist || []; - - // initialize web API - require("./js/api-endpoints")(this); - - // expose UI elements embedded in the master - this.ui = require("./js/ui").injectThis(this).ui; - - this.serverSockets = {}; - - this.io.on("connection", socket => { - let instanceID = "unknown"; - let isServerManager = false; - socket.on("registerSlave", data => { - if(data.instanceID && !isNaN(Number(data.instanceID))){ - instanceID = data.instanceID; - if(isServerManager){ - this.serverSockets[instanceID] = {socket}; - validServerManager({server: this.serverSockets[instanceID]}); - } - } - }); - socket.on("registerServerManager", () => { - isServerManager = true; - if(instanceID !== "unknown"){ - this.serverSockets[instanceID] = {socket}; - validServerManager({server: this.serverSockets[instanceID]}); - } - console.log("Registered serverManager socket") - - socket.on("disconnect", () => { - if(this.serverSockets[instanceID]) delete this.serverSockets[instanceID]; - console.log(`serverManager ${instanceID} disconnected, ${(Object.keys(this.serverSockets).length)} left`); - }); - }); - socket.on("gameChat", async data => { - // let chatLine = data.data.replace(/(\r\n\t|\n|\r\t)/gm, "").replace("\r", ""); - // if(typeof chatLine == "string") this.handleChatLine(chatLine, instanceID); - }); - }); - let validServerManager = ({server}) => { - server.socket.emit("serverManagerGetPlugins", {}, data => { - server.plugins = data; - }); - } - - // I can't seem to get express static pages + ejs rendering to work properly, so I write my own thing. - let pages = [ - { - addr: "/serverManager", - path: "serverManager/views/index", - render: true - }, - ] - pages.forEach(page => { - this.app.get(page.addr, async (req,res) => { - if(page.render){ - if((req.query.token || req.cookies.token) && this.playerManager){ - var permissions = await this.playerManager.main.getPermissions((req.query.token || req.cookies.token)); - } else { - var permissions = {}; - } - res.render(page.path, { - permissions, - serverManager: this, - req: req, - }); - } else { - res.send(await fs.readFile(page.path)); - } - }); - }); - this.app.use('/serverManager', Express.static(path.join(__dirname, 'static'))); - } - findInArray(key, value, array){ - let indexes = []; - for(let i in array){ - if(array[i][key] && array[i][key] === value) indexes.push(i); - } - return indexes; - } - async onExit(){ - // await saveDatabase("database/banlist.json", {banlist: this.banlist}); - return; - } - async onLoadFinish({plugins}){ - this.masterPlugins = plugins; - plugins.forEach(plugin => { - if(plugin.pluginConfig.name == "playerManager"){ - this.playerManager = plugin; - } - }); - } - async onPlayerManagerGetPermissions({permissions, user}){ - if(user - && user.admin){ - permissions.cluster.push("addPlugin"); - permissions.cluster.push("removePlugin"); - permissions.cluster.push("enablePlugin"); - permissions.cluster.push("disablePlugin"); - permissions.cluster.push("readConfig"); - permissions.cluster.push("writeConfig"); - } - return permissions; - } -} -module.exports = masterPlugin; - -function getDatabaseSync(path){ - let db; - try { - db = JSON.parse(fs.readFileSync(path, "utf8")); - } catch(e){ - db = {}; - } - return db; -} -async function saveDatabase(path, database){ - if(!path){ - throw new Error("No path provided!"); - } else if(!database){ - throw new Error("No database provided!"); - } else { - try { - await fs.writeFile(path, JSON.stringify(database, null, 4)); - } catch(e){ - throw new Error("Unable to write to database! "+path); - } - } -} diff --git a/sharedPlugins/serverManager/static/index.js b/sharedPlugins/serverManager/static/index.js deleted file mode 100644 index 478ccfcc..00000000 --- a/sharedPlugins/serverManager/static/index.js +++ /dev/null @@ -1,23 +0,0 @@ -// Handle plugin management -let pluginStatuses = document.querySelectorAll(".pluginStatus"); -pluginStatuses.forEach(pluginStatus => { - pluginStatus.addEventListener("click", async ()=>{ - if(pluginStatus.style.backgroundColor == "green"){ - let status = await postJSON("/api/serverManager/disablePlugin", { - instanceID: getParameterByName("instanceID"), - pluginName: pluginStatus.parentElement.children[1].children[0].innerText, - }); - if(status.ok){ - pluginStatus.style.backgroundColor = "red"; - } - } else { - let status = await postJSON("/api/serverManager/enablePlugin", { - instanceID: getParameterByName("instanceID"), - pluginName: pluginStatus.parentElement.children[1].children[0].innerText, - }); - if(status.ok){ - pluginStatus.style.backgroundColor = "green"; - } - } - }); -}); \ No newline at end of file diff --git a/sharedPlugins/serverManager/static/serverManager.css b/sharedPlugins/serverManager/static/serverManager.css deleted file mode 100644 index 99b0142b..00000000 --- a/sharedPlugins/serverManager/static/serverManager.css +++ /dev/null @@ -1,19 +0,0 @@ -#mods { - width: calc(( 100% ) / 4); - float:left; -} -#plugins { - width:calc((( 100% ) - (( 100% ) / 4))/2); - float:left; -} -ul { - list-style: none; - padding:0px; -} -#plugins > ul > li > div { - float:left; -} -#plugins .pluginStatus { - width:40px; - height: 40px; -} \ No newline at end of file diff --git a/sharedPlugins/serverManager/test/missing.js b/sharedPlugins/serverManager/test/missing.js deleted file mode 100644 index 9993114b..00000000 --- a/sharedPlugins/serverManager/test/missing.js +++ /dev/null @@ -1,5 +0,0 @@ -// These are libraries that does not have any units test, they are -// required here in order to track coverage for them. - -require('../index'); -require('../masterPlugin'); diff --git a/sharedPlugins/serverManager/views/index.ejs b/sharedPlugins/serverManager/views/index.ejs deleted file mode 100644 index 54fc72fc..00000000 --- a/sharedPlugins/serverManager/views/index.ejs +++ /dev/null @@ -1,38 +0,0 @@ - - - <%- include("head") %> - - - - - - <%- include("sidebar") %> -
-
- <% - html = ""; - if(req.query - && req.query.instanceID - && locals.slaves.get(req.query.instanceID) - && locals.slaves.get(req.query.instanceID).mods - && serverManager.serverSockets[req.query.instanceID]){ - let slave = locals.slaves.get(req.query.instanceID); - html += ` -
- ${slave.mods.map(mod => mod.name).reduce((acc, modname) => acc + "
  • " + modname + "
  • ")} -
    -
      ` - let server = serverManager.serverSockets[req.query.instanceID]; - if(server.plugins) server.plugins.forEach(plugin => { - html += `
    • ${plugin.name}

      ${plugin.description}

    • ` - }); - html += `
    ` - } - %> - <%- html %> -
    -
    - - From d38e9a53c4161c9cfc3ff7d12d2558a3bd60b4f5 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 18 Nov 2019 14:19:12 +0100 Subject: [PATCH 06/16] Remove broken combinator signaling Remove badly implemented and broken remote combinator signaling. --- CHANGELOG.md | 1 + master.js | 11 +------ sharedPlugins/clusterioMod/index.js | 51 ----------------------------- 3 files changed, 2 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cff654d4..75e994cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Version 2.0.0 handling in client. - Removed redundant call to /api/slaves in globalChat plugin. - Removed broken serverManager plugin. +- Removed broken remote combinator signaling. ### Breaking Changes diff --git a/master.js b/master.js index c0795f1b..454f6191 100644 --- a/master.js +++ b/master.js @@ -1,7 +1,6 @@ /** Clusterio master server. Facilitates communication between slaves through -a webserver, storing data related to slaves like production graphs and -combinator signals. +a webserver, storing data related to slaves like production graphs. @module clusterioMaster @author Danielv123 @@ -424,14 +423,6 @@ class wsSlave { prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc(); this.lastBeat = Date.now(); }); - this.socket.on("combinatorSignal", circuitFrameWithMeta => { - prometheusWsUsageCounter.labels('combinatorSignal', this.instanceID).inc(); - if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object"){ - Object.keys(wsSlaves).forEach(instanceID => { - wsSlaves[instanceID].socket.emit("processCombinatorSignal", circuitFrameWithMeta); - }); - } - }); // handle command return values this.commandsWaitingForReturn = {}; this.socket.on("runCommandReturnValue", resp => { diff --git a/sharedPlugins/clusterioMod/index.js b/sharedPlugins/clusterioMod/index.js index 64a11ce4..846c781d 100644 --- a/sharedPlugins/clusterioMod/index.js +++ b/sharedPlugins/clusterioMod/index.js @@ -36,7 +36,6 @@ module.exports = class remoteCommands { ensureFileSync(instancedirectory + "/script-output/output.txt"); ensureFileSync(instancedirectory + "/script-output/orders.txt"); - ensureFileSync(instancedirectory + "/script-output/txbuffer.txt"); // flow/production statistics ------------------------------------------------------------ @@ -247,56 +246,6 @@ module.exports = class remoteCommands { setTimeout(function(){ messageInterface("/silent-command remote.call('clusterio','setWorldID',"+that.config.unique+")") }, 20000); - /* REMOTE SIGNALLING - * send any signals the slave has been told to send - * Fetch combinator signals from the server - */ - this.socket.on("processCombinatorSignal", circuitFrameWithMeta => { - if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object" && circuitFrameWithMeta.frame && Array.isArray(circuitFrameWithMeta.frame)){ - messageInterface("/silent-command remote.call('clusterio', 'receiveFrame', '"+JSON.stringify(circuitFrameWithMeta.frame)+"')"); - } - }); - // get outbound frames from file and send to master - // get array of lines in file, each line should correspond to a JSON encoded frame - let signals = fs.readFileSync(instancedirectory + "/script-output/txbuffer.txt", "utf8").split("\n"); - // if we actually got anything from the file, proceed and reset file - let readingTxBufferSoon = false; - let txBufferClearCounter = 0; - fs.watch(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (eventType, filename) => { - if(!readingTxBufferSoon){ // use a 100ms delay to avoid messing with rapid sequential writes from factorio (I think that might be a problem maybe?) - readingTxBufferSoon = true; - setTimeout(()=>{ - txBufferClearCounter++; - fs.readFile(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (err, signals) => { - signals = signals.split("\n"); - if (signals[0]) { - //if(txBufferClearCounter > 500){ - fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", ""); - // txBufferClearCounter = 0; - //} - - // loop through all our frames - for (let i = 0; i < signals.length; i++) { - if (signals[i] && objectOps.isJSON(signals[i])) { - // signals[i] is a JSON array called a "frame" of signals. We timestamp it for storage on master - // then we unpack and RCON in this.frame to the game later. - let framepart = JSON.parse(signals[i]); - let doneframe = { - time: Date.now(), - frame: framepart, // thats our array of objects(single signals); - } - // send to master using socket.io, opened at the top of instanceManagement() - this.socket.emit("combinatorSignal", doneframe); - } else { - // console.log("Invalid jsony: "+typeof signals[i]) - } - } - } - }); - readingTxBufferSoon = false; - },100); - } - }); } async factorioOutput(data){ From ca2967e25519968f8a35ca44945ee0f8defb0c09 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 18 Nov 2019 14:26:33 +0100 Subject: [PATCH 07/16] Fix clusterioMod item autosaving Fix the autosave code being broken by the move in e4aae88. --- sharedPlugins/clusterioMod/masterPlugin.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sharedPlugins/clusterioMod/masterPlugin.js b/sharedPlugins/clusterioMod/masterPlugin.js index 7151b257..efbd75ce 100644 --- a/sharedPlugins/clusterioMod/masterPlugin.js +++ b/sharedPlugins/clusterioMod/masterPlugin.js @@ -51,11 +51,8 @@ class masterPlugin { async onLoadFinish() { this.items = await loadDatabase(this.config); - this.autosaveId = setInterval(async function() { - let file = path.join(this.config.databaseDirectory, "items.json"); - console.log(`autosaving ${file}`); - let content = JSON.stringify(db.items.serialize()); - await fs.outputFile(file, content); + this.autosaveId = setInterval(async () => { + await saveDatabase(this.config, this.items); },this.config.autosaveInterval || 60000); routes.addApiRoutes( From 27fc68da0f615bf621c085da817f35c8aa0d788a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 18 Nov 2019 16:38:56 +0100 Subject: [PATCH 08/16] Remove remoteCommands plugin and interface Remove the half baked remote rcon command interface in preparation for a better version based on a new socket.io communication layer. --- .gitignore | 1 - CHANGELOG.md | 2 + CLI tools/cluster_rcon.js | 40 -------------- master.js | 58 -------------------- sharedPlugins/remoteCommands/config.js | 13 ----- sharedPlugins/remoteCommands/index.js | 25 --------- sharedPlugins/remoteCommands/test/missing.js | 4 -- 7 files changed, 2 insertions(+), 141 deletions(-) delete mode 100644 CLI tools/cluster_rcon.js delete mode 100644 sharedPlugins/remoteCommands/config.js delete mode 100644 sharedPlugins/remoteCommands/index.js delete mode 100644 sharedPlugins/remoteCommands/test/missing.js diff --git a/.gitignore b/.gitignore index a5a56e68..3ba5e9a3 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,6 @@ saves sharedPlugins/* !sharedPlugins/inventoryImports/ !sharedPlugins/UPSdisplay/ -!sharedPlugins/remoteCommands/ !sharedPlugins/researchSync/ !sharedPlugins/globalChat/ !sharedPlugins/clusterioMod/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 75e994cb..02b70141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,8 @@ Version 2.0.0 add in a remote interface. - Removed mod uploading and distributing from the HTTP interface. Breaks the old unmaintained and no longer needed factorioClusterioClient. +- Removed the remoteCommands plugin and the old runCommand interface. Breaks + playerManager and external tools sending commands. Version 1.2.3 diff --git a/CLI tools/cluster_rcon.js b/CLI tools/cluster_rcon.js deleted file mode 100644 index becb0e08..00000000 --- a/CLI tools/cluster_rcon.js +++ /dev/null @@ -1,40 +0,0 @@ -const needle = require("needle"); -const cli = require("cli"); -const config = require("./../config.json"); - -let options = cli.parse() -cli.setUsage(`node cluster_rcon /c game.player.force = "player"`) -if(process.argv.length < 3){ - cli.getUsage() - process.exit(1) -} - -const token = config.masterAuthToken -let commandArray = process.argv; -commandArray.shift(); -commandArray.shift(); -const needleOptionsWithTokenAuthHeader = { - headers: { - 'x-access-token': token, - }, -}; - -let command = commandArray.join(" "); -let killBiters = `/silent-command local surface=game.surfaces[1] for key, entity in pairs(surface.find_entities_filtered({force="enemy"})) do entity.destroy() end`; - -(async ()=>{ - needle.post(config.masterURL+"/api/runCommand", { - broadcast:true, - command, - }, needleOptionsWithTokenAuthHeader, (err, resp) => { - console.log(resp.body) - }); - // setInterval(()=>{ - // needle.post(config.masterURL+"/api/runCommand", { - // broadcast:true, - // command: killBiters, - // }, needleOptionsWithTokenAuthHeader, (err, resp) => { - // console.log(resp.body) - // }); - // }, 1000*60*5); -})(); diff --git a/master.js b/master.js index 454f6191..48a0686a 100644 --- a/master.js +++ b/master.js @@ -340,50 +340,6 @@ app.get("/api/modmeta", async function(req, res) { }); -/** -POST endpoint for running commands on slaves. -Requires x-access-token header to be set. Find you api token in secret-api-token.txt on the master (after running it once) - -@memberof clusterioMaster -@instance -@alias api/runCommand -@param {object} JSON {instanceID:19412312, broadcast:false, command:"/c game.print('hello')"} -@returns {object} Status {auth: bool, message: "Informative error", data:{}} -*/ -app.post("/api/runCommand", authenticate.middleware, function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - // validate request.body - let body = req.body; - - if ( - body.broadcast - && body.command - && typeof body.command == "string" - && body.command[0] == "/") - { - let instanceResponses = []; - for (let instanceID in wsSlaves) { - // skip loop if the property is from prototype - if (!wsSlaves.hasOwnProperty(instanceID)) continue; - wsSlaves[instanceID].runCommand(body.command); - } - res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"}); - } else if ( - body.instanceID - && wsSlaves[body.instanceID] - && body.command - && typeof body.command == "string" - && body.command[0] == "/") - { - // execute command - wsSlaves[body.instanceID].runCommand(body.command, data => { - res.status(200).send({auth: true, message: "success", data}); - }); - } else { - res.status(400).send({auth: true, message: "Error: invalid request.body"}); - } -}); - var localeCache; /** GET endpoint. Returns factorio's base locale as a JSON object. @@ -423,15 +379,6 @@ class wsSlave { prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc(); this.lastBeat = Date.now(); }); - // handle command return values - this.commandsWaitingForReturn = {}; - this.socket.on("runCommandReturnValue", resp => { - prometheusWsUsageCounter.labels('runCommandReturnValue', this.instanceID).inc(); - if(resp.commandID && resp.body && this.commandsWaitingForReturn[resp.commandID] && this.commandsWaitingForReturn[resp.commandID].callback && typeof this.commandsWaitingForReturn[resp.commandID].callback == "function"){ - this.commandsWaitingForReturn[resp.commandID].callback(resp.body); - delete this.commandsWaitingForReturn[resp.commandID]; - } - }); this.socket.on("gameChat", data => { prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc(); @@ -455,11 +402,6 @@ class wsSlave { } }); } - runCommand(command, callback){ - let commandID = Math.random().toString(); - this.socket.emit("runCommand", {command, commandID}); - if(commandID) this.commandsWaitingForReturn[commandID] = {callback, timestamp: Date.now()}; - } } io.use((socket, next) => { diff --git a/sharedPlugins/remoteCommands/config.js b/sharedPlugins/remoteCommands/config.js deleted file mode 100644 index 07a83322..00000000 --- a/sharedPlugins/remoteCommands/config.js +++ /dev/null @@ -1,13 +0,0 @@ -/* - Config for clusterio remote command plugin. - Allows you to execute commands on slaves with a web API. - - Careful, may allow for RCE vulnerabilities. -*/ -module.exports = { - // Name of package. For display somewhere I guess. - name: "remoteCommands", - version: "1.0.0", - binary: "nodePackage", - description: "Config for clusterio remote command plugin. Allows you to execute commands on slaves with a web API.", -} \ No newline at end of file diff --git a/sharedPlugins/remoteCommands/index.js b/sharedPlugins/remoteCommands/index.js deleted file mode 100644 index 62a7df20..00000000 --- a/sharedPlugins/remoteCommands/index.js +++ /dev/null @@ -1,25 +0,0 @@ -module.exports = class remoteCommands { - constructor(mergedConfig, messageInterface, extras){ - this.messageInterface = messageInterface; - this.config = mergedConfig; - this.socket = extras.socket; - - if(this.config.allowRemoteCommandExecution){ - this.socket.on("runCommand", data => { - let {commandID} = data; - this.messageInterface(data.command); - if(commandID){ - this.socket.emit("runCommandReturnValue", { - commandID, - body: { - info: "Command was sent to factorio", - }, - }); - } - }); - } - } - scriptOutput(data){ - // we don't return anything yet - } -} diff --git a/sharedPlugins/remoteCommands/test/missing.js b/sharedPlugins/remoteCommands/test/missing.js deleted file mode 100644 index 923d82bf..00000000 --- a/sharedPlugins/remoteCommands/test/missing.js +++ /dev/null @@ -1,4 +0,0 @@ -// These are libraries that does not have any units test, they are -// required here in order to track coverage for them. - -require('../index'); From fcae3ef0c11cfa7c1e759309378ba0cdb061cf6d Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 18 Nov 2019 19:53:09 +0100 Subject: [PATCH 09/16] Use detected Factorio version Remove Factorio_version from the config file and use the detected version of Factorio instead. --- CHANGELOG.md | 2 ++ config.json.dist | 4 +--- lib/factorio/server.js | 7 +++++++ lib/manager/modManager.js | 8 ++++---- test/lib/factorio/server.js | 6 ++++++ 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02b70141..98c61578 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ Version 2.0.0 - Removed redundant call to /api/slaves in globalChat plugin. - Removed broken serverManager plugin. - Removed broken remote combinator signaling. +- Removed factorio_version from config. The version installed is auto + detected and used instead. ### Breaking Changes diff --git a/config.json.dist b/config.json.dist index 3029a6a7..c162cbed 100644 --- a/config.json.dist +++ b/config.json.dist @@ -28,9 +28,7 @@ "username": "", "__comment4": "Token from your factorio.com profile page. Required for MP and mod manager services.", "token": "", - "__comment7": "Used for communication with the mod portal, not needed if you manage mods by hand", - "factorio_version": "0.16.41", - + "__comment5": "General user settings", "verify_user_identity": true, "auto_pause": false, diff --git a/lib/factorio/server.js b/lib/factorio/server.js index 70f93d55..a6ba17c4 100644 --- a/lib/factorio/server.js +++ b/lib/factorio/server.js @@ -443,6 +443,13 @@ class FactorioServer extends events.EventEmitter { this._state = "init"; } + /** + * The version of Factorio that was detected + */ + get version() { + return this._version; + } + _attachStdio() { this._server.stdout.on('data', chunk => { this._stdout.data(chunk); }); this._server.stdout.on('close', () => { this._stdout.end(); }); diff --git a/lib/manager/modManager.js b/lib/manager/modManager.js index dc5da90b..4d472135 100644 --- a/lib/manager/modManager.js +++ b/lib/manager/modManager.js @@ -38,7 +38,7 @@ async function listMods(config, instance) { return new Promise((resolve, reject) => { const instances = require("./../getInstances")(config).sync(); if (instances.includes(instance.name)) { - factorioAPI.init(false, instance.path("instanceMods"), instance.path("saves"), config.factorio_version); + factorioAPI.init(false, instance.path("instanceMods"), instance.path("saves"), instance.server.version); factorioAPI.authenticate({ username: config.username, token: config.token, @@ -68,7 +68,7 @@ async function listMods(config, instance) { }); } else { // list shared mods in /sharedMods/ - factorioAPI.init(false, 'sharedMods/', '', config.factorio_version); + factorioAPI.init(false, 'sharedMods/', '', instance.server.version); factorioAPI.authenticate({ username: config.username, token: config.token, @@ -140,7 +140,7 @@ async function addMod(config, term, instance) { if (term && term.length > 0) { if (term.includes("mods.factorio.com/mods/")) { // https://mods.factorio.com/mods/Danielv123/ChooChoo // treat it as a link download - factorioAPI.init(false, modDirectory, "", config.factorio_version); + factorioAPI.init(false, modDirectory, "", instance.server.version); factorioAPI.authenticate({ username: config.username, token: config.token, @@ -185,7 +185,7 @@ async function addMod(config, term, instance) { } } else { // treat it as a search and download the top result - factorioAPI.init(false, modDirectory, "", config.factorio_version); + factorioAPI.init(false, modDirectory, "", instance.server.version); factorioAPI.authenticate({ username: config.username, token: config.token, diff --git a/test/lib/factorio/server.js b/test/lib/factorio/server.js index c87dbc31..33ef05a5 100644 --- a/test/lib/factorio/server.js +++ b/test/lib/factorio/server.js @@ -204,5 +204,11 @@ describe("lib/factorio/server", function() { await assert.rejects(server.init(), new Error("Expected state new but state is init")); }); }); + + describe(".version", function() { + it("should return the version detected", function() { + assert.equal(server.version, "0.1.1"); + }); + }); }); }); From 65f75d86e6774f6a2dd890070f610e5b98da45db Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 19 Nov 2019 00:24:20 +0100 Subject: [PATCH 10/16] Add id based database load and save functions Add functions to save and load a Map of objects indexed by the id property in the objects. --- lib/database.js | 79 +++++++++++++++++++++++++++++++++-- test/json/array_null.json | 1 + test/json/array_object.json | 1 + test/json/load_array_map.json | 1 + test/json/save_map.json | 10 +++++ test/lib/database.js | 75 ++++++++++++++++++++++++++++++++- 6 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 test/json/array_null.json create mode 100644 test/json/array_object.json create mode 100644 test/json/load_array_map.json create mode 100644 test/json/save_map.json diff --git a/lib/database.js b/lib/database.js index 69f7ae93..43a67f27 100644 --- a/lib/database.js +++ b/lib/database.js @@ -31,9 +31,10 @@ function mapToObject(map) { } /** - * Load JSON file as a Map + * Load JSON object file as a Map * - * Loads the JSON file specified by path an converts it into a JavaScript Map. + * Loads the JSON file specified by path containing an object with + * properties as key value pairs and converts it into a JavaScript Map. * Returns an empty Map if the file does not exist. * * @param {string} filePath - The path to the JSON file to load. @@ -71,7 +72,7 @@ async function loadJsonAsMap(filePath) { } /** - * Save a Map as a JSON file + * Save a Map as a JSON object file * * Save the content of a key-value Map with only string keys as a JSON file. * If the directory the file is to be saved into doesn't exist it will be @@ -87,6 +88,76 @@ async function saveMapAsJson(filePath, map) { await fs.outputFile(filePath, JSON.stringify(obj, null, 4)); } +/** + * Load JSON array file as a Map + * + * Loads the JSON file specified by path containing an array of objects that + * each have an id property into a Map from the id to the object. + * + * @param {string} filePath - The path to the JSON file to load. + * @throws {Error} If JSON file did not contain an array. + * @throws {Error} If there are objects in the array without an id property. + * @throws {Error} If an error occured reading the file. + */ +async function loadJsonArrayAsMap(filePath) { + try { + let content = await fs.readFile(filePath); + let parsed = JSON.parse(content); + + if (parsed === null) { + throw new Error(`Expected array but got null`); + } + + if (typeof parsed !== "object") { + throw new Error(`Expected array but got ${typeof parsed}`); + } + + if (!(parsed instanceof Array)) { + throw new Error(`Expected array but got object`); + } + + let map = new Map(); + for (let element of parsed) { + if (element === null || element instanceof Array || typeof element !== "object") { + throw new Error(`Expected all elements to be objects`); + } + + if (element.id === undefined) { + throw new Error(`Expected all elements to have an id property`); + } + + map.set(element.id, element); + } + return map; + + } catch (err) { + if (err.code == "ENOENT") { + // File does not exist, return empty map + return new Map(); + + } else { + throw err; + } + } +} + +/** + * Save a Map as a JSON array file + * + * Save the values of a Map with objects containing an id property that is + * equal to the key the objects are stored at into a JSON file. + * + * @param {string} filePath - The path to the JSON file that will be saved. + * @param {Map} map - + * Map with objects containing an id property equal to the key they are + * stored under. + * @throws {Error} if an error occured writing to the file. + */ +async function saveMapAsJsonArray(filePath, map) { + await fs.outputFile(filePath, JSON.stringify([...map.values()], null, 4)); +} + + function checkName(name) { if (typeof name !== "string") { @@ -227,5 +298,7 @@ module.exports = { mapToObject, loadJsonAsMap, saveMapAsJson, + loadJsonArrayAsMap, + saveMapAsJsonArray, ItemDatabase, } diff --git a/test/json/array_null.json b/test/json/array_null.json new file mode 100644 index 00000000..62864b31 --- /dev/null +++ b/test/json/array_null.json @@ -0,0 +1 @@ +[null] diff --git a/test/json/array_object.json b/test/json/array_object.json new file mode 100644 index 00000000..93d51406 --- /dev/null +++ b/test/json/array_object.json @@ -0,0 +1 @@ +[{}] diff --git a/test/json/load_array_map.json b/test/json/load_array_map.json new file mode 100644 index 00000000..996277e8 --- /dev/null +++ b/test/json/load_array_map.json @@ -0,0 +1 @@ +[{"id":"a"},{"id":"b"}] diff --git a/test/json/save_map.json b/test/json/save_map.json new file mode 100644 index 00000000..7c1a6687 --- /dev/null +++ b/test/json/save_map.json @@ -0,0 +1,10 @@ +[ + [ + "c", + {} + ], + [ + "d", + "foo" + ] +] \ No newline at end of file diff --git a/test/lib/database.js b/test/lib/database.js index 8956fed3..ad78688c 100644 --- a/test/lib/database.js +++ b/test/lib/database.js @@ -60,7 +60,7 @@ describe("lib/database", function() { describe("saveMapAsJson()", function() { it("should save a mapping as JSON", async function() { - let testFile = 'test/json/save_map.json'; + let testFile = 'test/temp/save_map.json'; async function deleteTestFile() { try { await fs.unlink(testFile); @@ -81,7 +81,80 @@ describe("lib/database", function() { await fs.readFile(testFile, {encoding: 'utf-8'}), '{\n "c": {},\n "d": "foo"\n}' ); + }); + }); + + describe("loadJsonArrayAsMap()", function() { + let badTypes = ['null', 'object', 'number', 'string', 'boolean']; + + for (let type of badTypes) { + it(`should reject on ${type} JSON`, async function() { + await assert.rejects( + database.loadJsonArrayAsMap(`test/json/${type}.json`), + new Error(`Expected array but got ${type}`) + ); + }); + } + + it("should reject on null element", async function() { + await assert.rejects( + database.loadJsonArrayAsMap('test/json/array_null.json'), + new Error("Expected all elements to be objects") + ); + }); + + it("should reject on element missing id", async function() { + await assert.rejects( + database.loadJsonArrayAsMap('test/json/array_object.json'), + new Error("Expected all elements to have an id property") + ); + }); + + it("should work on empty array JSON", async function() { + assert.deepEqual( + await database.loadJsonArrayAsMap('test/json/array.json'), + new Map() + ); + }); + + it("should work on array JSON", async function() { + assert.deepEqual( + await database.loadJsonArrayAsMap('test/json/load_array_map.json'), + new Map([['a', {id: 'a'}], ['b', {id: 'b'}]]) + ); + }); + + it("should give an empty Map for non-existant file", async function() { + assert.deepEqual( + await database.loadJsonArrayAsMap('test/json/does-not-exist'), + new Map() + ); + }); + }); + + describe("saveMapAsJsonArray()", function() { + it("should save a mapping as JSON", async function() { + let testFile = 'test/temp/save_array_map.json'; + async function deleteTestFile() { + try { + await fs.unlink(testFile); + } catch (err) { + /* istanbul ignore if */ + if (err.code !== 'ENOENT') { + throw err; + } + } + } + await deleteTestFile(); + await database.saveMapAsJsonArray( + testFile, new Map([['c', {id: 'c'}], ['d', {id: 'd'}]]) + ); + + assert.equal( + await fs.readFile(testFile, {encoding: 'utf-8'}), + '[\n {\n "id": "c"\n },\n {\n "id": "d"\n }\n]' + ); }); }); From bc41f7afa61ef75b93a82ddf020db3734e8bc338 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 19 Nov 2019 00:40:02 +0100 Subject: [PATCH 11/16] Clean up generateSSLcert test Remove dependency to fileOps in generateSSLcert test file and clean up the code to follow a higher standard. --- test/lib/generateSSLcert.js | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/lib/generateSSLcert.js b/test/lib/generateSSLcert.js index dbe3d1e3..c4356262 100644 --- a/test/lib/generateSSLcert.js +++ b/test/lib/generateSSLcert.js @@ -1,27 +1,32 @@ -var assert = require("assert"); -var fs = require("fs-extra"); +const assert = require("assert"); +const fs = require("fs-extra"); +const path = require("path"); -var fileOps = require("lib/fileOps"); -var generateSSLcert = require("lib/generateSSLcert") +const generateSSLcert = require("lib/generateSSLcert") describe("generateSSLcert.js(options)", ()=>{ + let testDir = "test/temp/certTest"; + before(async function() { + await fs.remove(testDir); + }); + it("Creates a folder with a .crt and .key file in it", async function() { - /* istanbul ignore if */ if (process.env.FAST_TEST) this.skip(); - let certPath = "lib/certTestPlsDelete/cert.crt"; - let privKeyPath = "lib/certTestPlsDelete/cert.key"; - assert(!await fs.exists(certPath)); // if it already exsits, the test might pass because of leftover data (which is bad) - + this.timeout(1000); + if (process.env.FAST_TEST) { + this.skip(); + } + + let certPath = path.join(testDir, "cert.crt"); + let privKeyPath = path.join(testDir, "cert.key"); + await generateSSLcert({ bits: 512, // This is too small for real world usage, but faster to test sslCertPath: certPath, sslPrivKeyPath: privKeyPath, doLogging: false, }); - - assert(await fs.exists(certPath)); // private key - assert(await fs.exists(privKeyPath)); // certificate file - - // remove certificates and data from the test - fileOps.deleteFolderRecursiveSync("lib/certTestPlsDelete"); - }).timeout(10000); + + assert(await fs.exists(certPath), "cert file was not created"); + assert(await fs.exists(privKeyPath), "key file was not created"); + }); }); From 5858d978575352fd333eb7592ece412861c6ecaa Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 20 Nov 2019 20:50:18 +0100 Subject: [PATCH 12/16] Remove playerManager specific CLI tool Remove the playerManager plugin specific CLI tool delete_players.js --- CHANGELOG.md | 1 + CLI tools/delete_player.js | 19 ------------------- package.json | 1 - 3 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 CLI tools/delete_player.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 98c61578..8949c5e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Version 2.0.0 - Removed broken remote combinator signaling. - Removed factorio_version from config. The version installed is auto detected and used instead. +- Removed the playerManager specific command CLI tools/delete_player.js. ### Breaking Changes diff --git a/CLI tools/delete_player.js b/CLI tools/delete_player.js deleted file mode 100644 index 1adcdff6..00000000 --- a/CLI tools/delete_player.js +++ /dev/null @@ -1,19 +0,0 @@ -const needle = require("needle") -const config = require("./../config.json") -const cli = require("cli") - -let options = cli.parse({ - name: ["n", "Name of the player to delete, case sensitive", "string"], - token: [false, "Factorio auth token", "string", config.masterAuthToken] -}) -if(!options.name){ - cli.getUsage() - process.exit(1) -} - -needle.post(config.masterURL+"/api/playerManager/deletePlayer", { - name: options.name, - token: options.token -}, (err, resp) => { - console.log(resp.body) -}); diff --git a/package.json b/package.json index 6b84b2d7..50d271e5 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "bcrypt": "^3.0.6", "bcrypt-promise": "^2.0.0", "body-parser": "^1.18.2", - "cli": "^1.0.1", "cli-interact": "^0.1.9", "compression": "^1.7.2", "cookie-parser": "^1.4.3", From 7a2abaa5db5dd7bd436598c6f8c8cc4de4903205 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 20 Nov 2019 21:45:59 +0100 Subject: [PATCH 13/16] Daemonize slave Rework client.js to support running multiple instances at once and be managed remotely and implement the neccessary communication infrastructure for this. Due to circular dependencies on the implementation of this the master branch is significantly broken after this. --- .gitignore | 2 + CHANGELOG.md | 21 + Dockerfile | 2 + README.md | 154 ++- client.js | 912 ++++++++++-------- clusterctl.js | 382 ++++++++ config.json.dist | 10 - docs/socket.md | 115 +++ lib/clusterTools/getInstanceName.js | 40 - lib/clusterTools/index.js | 16 - lib/errors.js | 20 + lib/factorio/server.js | 2 +- lib/getInstances.js | 17 - lib/link/index.js | 4 + lib/link/link.js | 399 ++++++++ lib/link/messages.js | 429 ++++++++ lib/manager/modManager.js | 3 +- lib/schema.js | 144 +++ master.js | 419 ++++---- package.json | 4 +- sharedPlugins/UPSdisplay/index.js | 1 + sharedPlugins/clusterioMod/index.js | 3 - sharedPlugins/globalChat/index.js | 6 +- .../inventoryImports/interfacingFunctions.js | 5 - test/client.js | 4 +- test/integration/server.js | 6 +- test/lib/link/link.js | 337 +++++++ test/lib/link/messages.js | 151 +++ test/missing.js | 3 +- 29 files changed, 2811 insertions(+), 800 deletions(-) create mode 100644 clusterctl.js create mode 100644 docs/socket.md delete mode 100644 lib/clusterTools/getInstanceName.js delete mode 100644 lib/clusterTools/index.js create mode 100644 lib/errors.js delete mode 100644 lib/getInstances.js create mode 100644 lib/link/index.js create mode 100644 lib/link/link.js create mode 100644 lib/link/messages.js create mode 100644 lib/schema.js create mode 100644 test/lib/link/link.js create mode 100644 test/lib/link/messages.js diff --git a/.gitignore b/.gitignore index 3ba5e9a3..84992512 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,8 @@ Factorio* /database database/items.db config.json +config-control.json +config-slave.json package-lock.json secret-api-token.txt mod_cache.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 8949c5e5..88a54ff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ Version 2.0.0 event_loader lib into the savegame. Regular freeplay games can now be used with Clusterio and will be compatible without having to convert them to Hotpatch. +- Daemonized slaves. Slaves now have the ability to run multiple Factorio + instances, starting and stopping them individually. To manage this the + local command line interface has been replaced with a remote interface on + the master server that can be accessed through the clusterctl cli tool. ### Changes @@ -50,6 +54,10 @@ Version 2.0.0 - Removed factorio_version from config. The version installed is auto detected and used instead. - Removed the playerManager specific command CLI tools/delete_player.js. +- Creating an instance, creating a save for an instance and starting an + instance is now three separate commands. +- Removed oddball limits to slaves.json size. +- Moved slave specific configuration into its own configuration file. ### Breaking Changes @@ -74,6 +82,19 @@ Version 2.0.0 the old unmaintained and no longer needed factorioClusterioClient. - Removed the remoteCommands plugin and the old runCommand interface. Breaks playerManager and external tools sending commands. +- Removed broken serverManager plugin. +- Removed fields info, time, rconPort, rconPassword, serverPort, unique, + mods, instanceName, playerCount, mac, and meta from the slaves in the + slave database. +- Removed getInstanceName and lib/clusterTools. Breaks playerManager, + and discordChat. +- Removed the /api/rconPasswords, and /api/slaves endpoints. Breaks web + interface, trainTeleports, and discordChat. +- Removed the /api/getSlaveMeta and /api/editSlaveMeta endpoints. Breaks + researchSync, and UPSdisplay. +- Removed the hello event from the socket connection handshake. Breaks + playerManager, trainTeleports, serverSelect, and discordChat +- Changed the format of database/slaves.json. Version 1.2.3 diff --git a/Dockerfile b/Dockerfile index dc217a35..b3ba3120 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +# Warning: This is currently broken as a result of +# changes done in the 2.0 refactoring. FROM node:10 RUN apt-get update && apt install git curl tar -y RUN mkdir factorioClusterio diff --git a/README.md b/README.md index 8a0f2b36..2f63056f 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Discord for development/support/play: https://discord.gg/5XuDkje ## Important notice +**WARNING: The master branch is currently broken, do not use it!** + This is the development branch for factorioClusterio 2.0 which is currently undergoing heavy restructuring and refactoring. Expect plugins and existing installations to frequently break when using this branch. If you don't want to be an alpha tester for 2.0 please use the stable @@ -30,17 +32,15 @@ the [1.2.x branch][1.2.x] for instructions on how to install the stable version. ### Table of contents * [Introduction & methodology](#introduction) - * [Ubuntu setup](#ubuntu-setup) - * [Windows setup](#windows-setup) - +* [Running Clusterio](#running-clusterio) + * [Master Server](#master-server) + * [Slaves](#slaves) + * [Instances](#instances) * [Optional plugins](#Plugins) - * [Common problems](#Common-problems) -* [Command cheatsheet](#cheatsheet) - ## Introduction Features: @@ -123,39 +123,15 @@ Optional step (if you want to use pm2): Now you need to edit the `config.json` file. If you skip this step nothing will work. Pretty much all the blank fields should be filled in, except on the master where a few can be omitted. -* You get the `masterAuthToken` from `secret-api-token.txt` in the master install dir after running the master twice. - * You get your factorio matchmaking token from factorio.com * The `masterAuthSecret` should never be touched unless you want to invalidate everyones authentication tokens - -**Master** - - node master - -Or with pm2 (it's recommened to run it without pm2 first): - - pm2 start master --name master - - -**Server Host** - -To download the mod for all its non vanilla features and items, (optional, but very recommended) - - node client manage shared mods add clusterio - -To create a new instance (its own save, set of mods and config files) - - node client start [instancename] - -To launch an instance with pm2 - - pm2 start --name [instancename] client -- start [instancename] - -use `nano config.json` to change settings. - **Ubuntu with Docker** +The Docker support for Clusterio is curently broken. If you need it +open an issue about it. + + ## Windows setup @@ -219,35 +196,83 @@ reboot when you are done, then proceed to the next steps. *reboots matter* 4. Run `node master` to generate the athentication token into secret-api-token.txt -5. Run `node master` again to start the master server. - **Server Host** 1. Do step 1 and 2 of the Master section above *OR* use the same folder that was created in that section. -2. Open `config.json` with a text editor and configure as desired. You will need to set masterAuthToken to string found in secret-api-token.txt on the master server. - -3. Optionally run the command `node client manage shared mods add clusterio` to add the clusterio mod (needed for item teleports.) - -4. Run `node client start [instancename]` to create a new instance. Repeat it again to start the instance. - -To connect to a master server running on a remote machine, open config.json with your favourite text editor (notepad++). You can also set it up to use the official server browser. - Change `masterURL `to something like `http://203.0.113.33:8080` (provided by master server owner) Change `masterAuthToken` to the value found in `secret-api-token.txt` on the master server Repeat step 4 for more servers on one machine. You should be able to find its port by looking at the slave section on master:8080 (the web interface) -**GameClient** -Fancy game client that does the following steps automatically, but is really old so be warned: [clusterioClient](https://github.com/Danielv123/factorioClusterioClient) +## Running Clusterio -1. Download the same version of the mod as the slave is running from [the mod portal](https://mods.factorio.com/mods/Danielv123/clusterio) or [github](https://github.com/Danielv123/factorioClusterioMod +After following the installation instructions you can use the following +commands to run Clusterio. -2. Drop it into ./factorio/mods -3. Run factorio and connect to slave as a normal MP game. You will find the port number to connect to at http://[masterAddress]:8080 +### Master Server + +It's necessary to run the master server in order for anything to work. +Once you've completed the setup run the following command to start it +up: + + node master + +Or with pm2 (it's recommened to run it without pm2 first): + + pm2 start master --name master + + +### Slaves + +Slaves connect to the master server and are managed remotely from the +master server. In order for slaves to connect to the master server they +need an authentication token from the master server. This token is +written to secret-api-token.txt on the master server the first time it +is started. + +To set up the configuration for a new local slave run the following. + + node client create-config --name "Local" --token "" + +This will write a new `config-slave.json` file in the current directory +(you can change the location with the `--config` option) with the name +and token provided. If you are connecting to a remote master server you +will also need to pass the `--url ` option. + +You can edit the config of a slave with the `node client edit-config` +command. Use `node client edit-config --help` for more information. + +Once the config is set up run the slave with + + node client start + + +### Instances + +Instances are created, managed and started from the master server. For +now the only interface available is the `clusterctl` command line tool +included in Clusterio. You can run this tool from any slave, or the +master server without having to set up a config, if you want to manage +the cluster from somewhere else see `node clusterctl create-config`. + +The basic operations to start a new instance is the following + + node clusterctl create-instance --slave "Local" --name "My instance" + node clusterctl create-save --instance "My instance" + node clusterctl start-instance --instance "My instance" + +The first line creates the files and database entries for a new +instance. The second line creates a new savegame for the instance. You +could also upload your own save to the instance directory instead. And +finally the third line starts the instance. + +There are many more commands available with clusterctl. See +`node clusterctl --help` for more information. + ## Plugins Here are the known Clusterio plugins in the wild: @@ -257,7 +282,7 @@ Here are the known Clusterio plugins in the wild: ## Common problems -### Cannot find module: `/../../config` +### Cannot find module: `./config` Copy your config.json.dist to config.json and configure it. @@ -288,32 +313,3 @@ TLDR: the tested fix is: ### Other fixes for other potential problems: Sometimes the install fails. Try `node ./lib/npmPostinstall` to complete it. - - -## Cheatsheet - -**To create a new instance/start it** - - node client start [instanceName] - -**Other instance management tools:** -``` -node client delete [instanceName] -node client list -``` -**To update clusterio to the latest version:** - -1. Download the latest zip version of factorio for your platform manually from factorio.com. Place it in the project root folder and call it "factorio" (folder name is specified in config.json) - -2. Grab the latest version of the repo - -``` -git pull - -npm install --only=production -``` - -3. Download the latest version of the factorioClusterioMod from its github repo -``` -node client manage shared mods add clusterio -``` diff --git a/client.js b/client.js index 325eef0c..91af5f39 100644 --- a/client.js +++ b/client.js @@ -5,18 +5,19 @@ const child_process = require('child_process'); const path = require('path'); const request = require("request"); const deepmerge = require("deepmerge"); -const getMac = require('getmac').getMac; const ioClient = require("socket.io-client"); -const asTable = require("as-table").configure({delimiter: ' | '}); const util = require("util"); +const yargs = require("yargs"); +const version = require("./package").version; // internal libraries const objectOps = require("lib/objectOps"); const fileOps = require("lib/fileOps"); const pluginManager = require("lib/manager/pluginManager"); -const modManager = require("lib/manager/modManager"); const hashFile = require('lib/hash').hashFile; const factorio = require("lib/factorio"); +const schema = require("lib/schema"); +const link = require("lib/link"); // Uhm... var global = {}; @@ -25,9 +26,200 @@ var global = {}; * Keeps track of the runtime parameters of an instance */ class Instance { - constructor(dir, name) { - this._name = name; + constructor(dir, factorioDir, instanceConfig) { this._dir = dir; + + // This is expected to change with the config system rewrite + this.config = { + id: instanceConfig.id, + name: instanceConfig.name, + gamePort: instanceConfig.factorioPort, + rconPort: instanceConfig.clientPort, + rconPassword: instanceConfig.clientPassword, + } + + let serverOptions = { + gamePort: this.config.gamePort, + rconPort: this.config.rconPort, + rconPassword: this.config.rconPassword, + }; + + this.server = new factorio.FactorioServer( + path.join(factorioDir, "data"), this._dir, serverOptions + ); + } + + async init() { + await this.server.init(); + } + + static async create(id, instanceDir, factorioDir, options) { + let instanceConfig = { + id, + name: options.name, + factorioPort: null, + clientPort: null, + clientPassword: null, + } + + let instance = new this(instanceDir, factorioDir, instanceConfig); + await instance.init(); + console.log(`Creating ${instance.path()}`); + await fs.ensureDir(instance.path()); + await fs.ensureDir(instance.path("script-output")); + + await symlinkMods(instance, "sharedMods", console); + console.log("Clusterio | Created instance with settings:") + console.log(instanceConfig); + + // save instance config + await fs.outputFile(instance.path("config.json"), JSON.stringify(instanceConfig, null, 4)); + + let serverSettings = await instance.server.exampleSettings(); + let gameName = "Clusterio instance: " + instance.name; + if (options.username) { + gameName = options.username + "'s clusterio " + instance.name; + } + + let overrides = { + "name": gameName, + "description": options.description, + "tags": ["clusterio"], + "visibility": options.visibility, + "username": options.username, + "token": options.token, + "game_password": options.game_password, + "require_user_verification": options.verify_user_identity, + "allow_commands": options.allow_commands, + "auto_pause": options.auto_pause, + }; + + for (let [key, value] of Object.entries(overrides)) { + if (!Object.hasOwnProperty.call(serverSettings, key)) { + throw Error(`Expected server settings to have a ${key} property`); + } + serverSettings[key] = value; + } + + await fs.writeFile(instance.path("server-settings.json"), JSON.stringify(serverSettings, null, 4)); + + return instance; + } + + async createSave() { + console.log("Creating save ....."); + + // XXX Move to constructor and send to master + this.server.on('output', function(output) { + console.log("Fact: " + output.message); + }); + + await this.server.create("world"); + console.log("Clusterio | Successfully created save"); + } + + async start(slaveConfig, socket) { + console.log("Deleting .tmp.zip files"); + let savefiles = fs.readdirSync(this.path("saves")); + for(let i = 0; i < savefiles.length; i++){ + if(savefiles[i].substr(savefiles[i].length - 8, 8) == ".tmp.zip") { + fs.unlinkSync(this.path("saves", savefiles[i])); + } + } + console.log("Clusterio | Rotating old logs..."); + // clean old log file to avoid crash + try{ + let logPath = this.path("factorio-current.log"); + let stat = await fs.stat(logPath); + console.log(stat) + console.log(stat.isFile()) + if(stat.isFile()){ + let logFilename = `factorio-${Math.floor(Date.parse(stat.mtime)/1000)}.log`; + await fs.rename(logPath, this.path(logFilename)); + console.log(`Log rotated as ${logFilename}`); + } + }catch(e){} + + await symlinkMods(this, "sharedMods", console); + + // Spawn factorio server + let latestSave = await fileOps.getNewestFile(this.path("saves")); + if (latestSave === null) { + throw new Error( + "Your savefile seems to be missing. This might because you created an\n"+ + "instance without having factorio installed and configured properly.\n"+ + "Try installing factorio and adding your savefile to\n"+ + "instances/[instancename]/saves/" + ); + } + + // Patch save with lua modules from plugins + console.log("Clusterio | Patching save"); + + // For now it's assumed that all files in the lua folder of a plugin is + // to be patched in under the name of the plugin and loaded for all + // plugins that are not disabled. This will most likely change in the + // future when the plugin refactor is done. + let modules = []; + for (let pluginName of await fs.readdir("sharedPlugins")) { + let pluginDir = path.join("sharedPlugins", pluginName); + if (await fs.pathExists(path.join(pluginDir, "DISABLED"))) { + continue; + } + + if (!await fs.pathExists(path.join(pluginDir, "lua"))) { + continue; + } + + let module = { + "name": pluginName, + "files": [], + }; + + for (let fileName of await fs.readdir(path.join(pluginDir, "lua"))) { + module["files"].push({ + path: pluginName+"/"+fileName, + content: await fs.readFile(path.join(pluginDir, "lua", fileName)), + load: true, + }); + } + + modules.push(module); + } + await factorio.patch(this.path("saves", latestSave), modules); + + this.server.on('output', (output) => { + console.log("Fact: " + output.message); + }); + + this.server.on('rcon-ready', () => { + console.log("Clusterio | RCON connection established"); + // Temporary measure for backwards compatibility + let compatConfig = { + id: this.config.id, + unique: this.config.id, + name: this.config.name, + + // FactorioServer.init may have generated a random port or password + // if they were null. + factorioPort: this.server.gamePort, + clientPort: this.server.rconPort, + clientPassword: this.server.rconPassword, + } + instanceManagement(slaveConfig, this, compatConfig, this.server, socket); // XXX async function + }); + + await this.server.start(latestSave); + } + + /** + * Stop the instance + */ + async stop() { + // XXX this needs more thought to it + if (this.server._state === "running") { + await this.server.stop(); + } } /** @@ -36,7 +228,7 @@ class Instance { * This should not be used for filesystem paths. See .path() for that. */ get name() { - return this._name; + return this.config.name; } /** @@ -52,6 +244,169 @@ class Instance { } } +/** + * Handles running the slave + * + * Connects to the master server over the socket.io connection and manages + * intsances. + */ +class Slave extends link.Client { + // I don't like God classes, but the alternative of putting all this state + // into global variables is not much better. + constructor(slaveConfig) { + super('slave', slaveConfig.masterURL, slaveConfig.masterAuthToken); + link.attachAllMessages(this); + this.config = { + id: slaveConfig.id, + name: slaveConfig.name, + instancesDir: slaveConfig.instanceDirectory, + factorioDir: slaveConfig.factorioDirectory, + masterUrl: slaveConfig.masterURL, + masterToken: slaveConfig.masterAuthToken, + publicAddress: slaveConfig.publicIP, + } + } + + register() { + console.log("SOCKET | registering slave"); + this.send('register_slave', { + agent: 'Clusterio Slave', + version, + id: this.config.id, + name: this.config.name, + }); + } + + async _findNewInstanceDir(name) { + try { + checkFilename(name) + } catch (err) { + throw new Error(`Instance name ${err.message}`); + } + + // For now add dashes until an unused directory name is found + let dir = path.join(this.config.instancesDir, name); + while (await fs.pathExists(dir)) { + dir += '-'; + } + + return dir; + } + + /** + * Looks up the instance for an instance request handler + * + * Called by the handler for InstanceRequest. + */ + async forwardInstanceRequest(message, handler) { + let instance = this.instances.get(message.data.instance_id); + if (!instance) { + throw new errors.RequestError(`Instance with ID ${instanceId} does not exist`); + } + + return await handler.call(this, instance, message); + } + + async createInstanceRequestHandler(message) { + let { id, options } = message.data; + if (this.instances.has(id)) { + throw new Error(`Instance with ID ${id} already exists`); + } + + let instanceDir = await this._findNewInstanceDir(options.name); + // XXX: race condition on multiple simultanious calls + let instance = await Instance.create(id, instanceDir, this.config.factorioDir, options); + this.instances.set(id, instance); + this.updateInstances(); + } + + async createSaveRequestHandler(instance, message) { + await instance.createSave(); + } + + async startInstanceRequestHandler(instance, message) { + await instance.start(this.config, this.socket); + } + + async stopInstanceRequestHandler(instance, message) { + await instance.stop(); + } + + async deleteInstanceRequestHandler(instance, message) { + await instance.stop(); + this.instances.delete(instance.config.id); + + fs.remove(instance.path()); + this.updateInstances(); + } + + async sendRconRequestHandler(instance, message) { + let result = await instance.server.sendRcon(message.data.command); + return { result }; + } + + async findInstances() { + let instances = new Map(); + for (let entry of await fs.readdir(this.config.instancesDir, { withFileTypes: true })) { + if (entry.isDirectory()) { + let instanceConfig; + let configPath = path.join(this.config.instancesDir, entry.name, "config.json"); + try { + instanceConfig = JSON.parse(await fs.readFile(configPath)); + } catch (err) { + if (err.code === "ENOENT") { + continue; // Ignore folders without config.json + } + + console.error(`Error occured while parsing ${configPath}: ${err}`); + } + + // XXX should probably validate the entire config with a JSON Schema. + if (typeof instanceConfig.id !== "number" || isNaN(instanceConfig.id)) { + console.error(`${configPath} is missing id`); + continue; + } + + if (typeof instanceConfig.name !== "string") { + console.error(`${configPath} is missing name`); + continue; + } + + let instancePath = path.join(this.config.instancesDir, entry.name); + console.log(`found instance ${instanceConfig.name} in ${instancePath}`); + let instance = new Instance(instancePath, this.config.factorioDir, instanceConfig); + await instance.init(); + instances.set(instanceConfig.id, instance); + } + } + + return instances; + } + + updateInstances() { + let list = []; + for (let instance of this.instances.values()) { + list.push({ + id: instance.config.id, + name: instance.config.name, + }); + } + link.events.updateInstances.send(this, { instances: list }); + } + + async start() { + this.instances = await this.findInstances(); + await this.connect(); + this.updateInstances(); + } + + async stop() { + for (let instance of this.instances.values()) { + await instance.stop(); + } + } +} + function checkFilename(name) { // All of these are bad in Windows only, except for /, . and .. // See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file @@ -92,121 +447,6 @@ function checkFilename(name) { } } -function needleOptionsWithTokenAuthHeader(config) { - return { - compressed: true, - headers: { - 'x-access-token': config.masterAuthToken - }, - }; -} - -function printUsage() { - console.error("Usage: "); - console.error("node client start [instance name]"); - console.error("node client list"); - console.error("node client delete [instance]"); - console.error("To download the latest version of the Clusterio lua mod, do"); - console.error("node client manage shared mods download clusterio"); - console.error("To download a clusterio plugin, do"); - console.error("node client manage shared plugins install https://github.com/Danielv123/playerManager"); - console.error("For more management options, do"); - console.error("node client manage"); -} - -async function listInstances(config) { - let instanceNames = fileOps.getDirectoriesSync(config.instanceDirectory); - let instances = []; - for (instance of instanceNames) { - let cfg = path.resolve(config.instanceDirectory, instance, 'config'); - let port = require(cfg).factorioPort; - instances.push({ - "Name": instance, - "Port": port.toString(), - }); - } - - console.log(asTable(instances)); -} - -async function manage(config, instance) { - // console.log("Launching mod manager"); - //const fullUsage = 'node client manage [instance, "shared"] ["mods", "config"] ...'; - function usage(instance, tool, action){ - if(tool && tool == "mods"){ - console.log('node client manage '+instance.name+' '+tool+' ["list", "search", "add", "remove"]'); - } else if(tool && tool == "plugins") { - console.log(`node client manage ${instance.name} ${tool} ["list", "add", "remove"]`); - } else { - console.log('node client manage '+(instance && instance.name || '[instance, "shared"]') +' '+ (tool || '["mods", "plugins"]') + ' ...'); - } - } - const tool = process.argv[4] || ""; - const action = process.argv[5] || ""; - if (instance !== undefined) { - if(tool == "mods"){ - (async function(){try{ - // allow managing mods - if(action == "list"){ - console.log(await modManager.listMods(config, instance)); - } else if(action == "search"){ - console.log(await modManager.findMods(config, process.argv[6])); - } else if(action == "add" || action == "download"){ - await modManager.addMod(config, process.argv[6], instance); - } else if(action == "remove" || action == "rm" || action == "delete"){ - await modManager.removeMod(config, process.argv[6], instance); - } else { - usage(instance, tool); - } - process.exit(0); - }catch(e){ - console.log("Got error from modManager:") - console.log(e); - }})(); - } else if(tool == "plugins"){ - (async function(){try{ - if(action == "list"){ - await pluginManager.listPlugins(); - } else if(action == "add" || action == "install" || action == "download"){ - let status = await pluginManager.addPlugin(process.argv[6]); - } else if(action == "remove" || action == "uninstall" || action == "delete"){ - let status = await pluginManager.removePlugin(process.argv[6]); - if(status && status.msg) console.log(status.msg); - } else if(action == "enable"){ - let status = await pluginManager.enablePlugin(process.argv[6]); - if(status && status.msg) console.log(status.msg); - } else if(action == "disable"){ - let status = await pluginManager.disablePlugin(process.argv[6]); - if(status && status.msg) console.log(status.msg); - } - process.exit(0); - }catch(e){ - console.error(e); - process.exit(1); - }})(); - } else { - usage(instance); - } - } else { - console.log('Usage:'); - usage(instance); - } -} - -async function deleteInstance(instance) { - if (instance === undefined) { - console.error("Usage: node client delete [instance]"); - process.exit(1); - } else if (fs.existsSync(instance.path())) { - fileOps.deleteFolderRecursiveSync(instance.path()); // TODO: Check if this can cause i-craft users to format their server by using wrong paths - console.log("Deleted instance " + instance.name); - process.exit(0); - } else { - console.error("Instance not found: " + instance.name); - process.exit(0); - } -} - /** * Create and update symlinks for shared mods in an instance * @@ -270,228 +510,103 @@ async function symlinkMods(instance, sharedMods, logger) { } } -async function createInstance(config, instance) { - console.log(`Creating ${instance.path()}`); - await fs.ensureDir(instance.path()); - await fs.ensureDir(instance.path("script-output")); - - await symlinkMods(instance, "sharedMods", console); - let instconf = { - "id": Math.random() * 2 ** 31 | 0, - "factorioPort": null, - "clientPort": null, - "__comment_clientPassword": "This is the rcon password. Will be randomly generated if null.", - "clientPassword": null, - "info": {} - } - console.log("Clusterio | Created instance with settings:") - console.log(instconf); - - // create instance config - fs.writeFileSync(instance.path("config.json"), JSON.stringify(instconf, null, 4)); - - let server = new factorio.FactorioServer(path.join("factorio", "data"), instance.path(), {}); - await server.init(); - - let serverSettings = await server.exampleSettings(); - let name = "Clusterio instance: " + instance.name; - if (config.username) { - name = config.username + "'s clusterio " + instance.name; - } - - let overrides = { - "name": name, - "description": config.description, - "tags": ["clusterio"], - "max_players": "20", - "visibility": config.visibility, - "username": config.username, - "token": config.token, - "game_password": config.game_password, - "require_user_verification": config.verify_user_identity, - "allow_commands": config.allow_commands, - "autosave_interval": 10, - "autosave_slots": 5, - "afk_autokick_interval": 0, - "auto_pause": config.auto_pause, - }; - - for (let [name, value] of Object.entries(overrides)) { - if (!Object.hasOwnProperty.call(serverSettings, name)) { - throw Error(`Expected server settings to have a ${name} property`); - } - serverSettings[name] = value; - } - - await fs.writeFile(instance.path("server-settings.json"), JSON.stringify(serverSettings, null, 4)); - console.log("Server settings: ", serverSettings); - console.log("Creating save ....."); - - server.on('output', function(output) { - console.log("Fact: " + output.message); - }); - - await server.create("world"); - console.log("Clusterio | Successfully created instance"); -} - -async function startInstance(config, instance) { - var instanceconfig = JSON.parse(await fs.readFile(instance.path("config.json"))); - if (typeof instanceconfig.id !== "number" || isNaN(instanceconfig.id)) { - throw new Error(`${instance.path("config.json")} is missing id`); - } - // Temporary measure for backwards compatibility - instanceconfig.unique = instanceconfig.id; - - console.log("Deleting .tmp.zip files"); - let savefiles = fs.readdirSync(instance.path("saves")); - for(i = 0; i < savefiles.length; i++){ - if(savefiles[i].substr(savefiles[i].length - 8, 8) == ".tmp.zip") { - fs.unlinkSync(instance.path("saves", savefiles[i])); - } - } - console.log("Clusterio | Rotating old logs..."); - // clean old log file to avoid crash - try{ - let logPath = instance.path("factorio-current.log"); - let stat = await fs.stat(logPath); - console.log(stat) - console.log(stat.isFile()) - if(stat.isFile()){ - let logFilename = `factorio-${Math.floor(Date.parse(stat.mtime)/1000)}.log`; - await fs.rename(logPath, instance.path(logFilename)); - console.log(`Log rotated as ${logFilename}`); - } - }catch(e){} - - await symlinkMods(instance, "sharedMods", console); - - // Spawn factorio server - let latestSave = await fileOps.getNewestFile(instance.path("saves")); - if (latestSave === null) { - throw new Error( - "Your savefile seems to be missing. This might because you created an\n"+ - "instance without having factorio installed and configured properly.\n"+ - "Try installing factorio and adding your savefile to\n"+ - "instances/[instancename]/saves/" - ); - } - - // Patch save with lua modules from plugins - console.log("Clusterio | Patching save"); - - // For now it's assumed that all files in the lua folder of a plugin is - // to be patched in under the name of the plugin and loaded for all - // plugins that are not disabled. This will most likely change in the - // future when the plugin refactor is done. - let modules = []; - for (let pluginName of await fs.readdir("sharedPlugins")) { - let pluginDir = path.join("sharedPlugins", pluginName); - if (await fs.pathExists(path.join(pluginDir, "DISABLED"))) { - continue; - } - - if (!await fs.pathExists(path.join(pluginDir, "lua"))) { - continue; - } - - let module = { - "name": pluginName, - "files": [], - }; - - for (let fileName of await fs.readdir(path.join(pluginDir, "lua"))) { - module["files"].push({ - path: pluginName+"/"+fileName, - content: await fs.readFile(path.join(pluginDir, "lua", fileName)), - load: true, - }); - } - - modules.push(module); - } - await factorio.patch(instance.path("saves", latestSave), modules); - - let options = { - gamePort: instanceconfig.factorioPort, - rconPort: instanceconfig.clientPort, - rconPassword: instanceconfig.clientPassword, - }; - - let server = new factorio.FactorioServer(path.join("factorio", "data"), instance.path(), options); - await server.init(); - - // FactorioServer.init may have generated a random port or password - // if they were null. - instanceconfig.factorioPort = server.gamePort - instanceconfig.clientPort = server.rconPort - instanceconfig.clientPassword = server.rconPassword - - server.on('output', function(output) { - console.log("Fact: " + output.message); - }); - - server.on('rcon-ready', function() { - console.log("Clusterio | RCON connection established"); - instanceManagement(config, instance, instanceconfig, server); // XXX async function - }); - - let secondSigint = false - process.on('SIGINT', () => { - if (secondSigint) { - console.log("Caught second interrupt, terminating immediately"); - process.exit(1); - } - - secondSigint = true; - console.log("Caught interrupt signal, shutting down"); - server.stop().then(() => { - // There's currently no shutdown mechanism for instance plugins so - // they keep the event loop alive. - process.exit(); - }); - }); - - await server.start(latestSave); -} - async function startClient() { - // argument parsing - const args = require('minimist')(process.argv.slice(2)); - - const command = process.argv[2]; - // add better stack traces on promise rejection process.on('unhandledRejection', r => console.log(r)); - console.log(`Requiring config from ${args.config || './config'}`); - const config = require(args.config || './config'); + // argument parsing + const args = yargs + .scriptName("client") + .usage("$0 [options]") + .option('config', { + nargs: 1, + describe: "slave config file to use", + default: 'config-slave.json', + type: 'string', + }) + .command('create-config', "Create slave config", (yargs) => { + yargs.options({ + 'name': { describe: "Name of the slave", nargs: 1, type: 'string', demandOption: true }, + 'url': { describe: "Master URL", nargs: 1, type: 'string', default: "http://localhost:8080/" }, + 'token': { describe: "Master token", nargs: 1, type: 'string', demandOption: true }, + 'ip': { describe: "Public facing IP", nargs: 1, type: 'string', default: "localhost" }, + 'instances-dir': { describe: "Instances directory", nargs: 1, type: 'string', default: "instances" }, + 'factorio-dir': { describe: "Factorio directory", nargs: 1, type: 'string', default: "factorio" }, + 'id': { + describe: "Numeric id of the slave", + nargs: 1, + type: 'number', + default: Math.random() * 2**31 | 0, + defaultDescription: "random id", + }, + }); + }) + .command('edit-config', "Edit slave config", (yargs) => { + yargs.options({ + 'name': { describe: "Set name of the slave", nargs: 1, type: 'string' }, + 'url': { describe: "Set master URL", nargs: 1, type: 'string' }, + 'token': { describe: "Set master token", nargs: 1, type: 'string' }, + 'ip': { describe: "Set public facing IP", nargs: 1, type: 'string' }, + 'instances-dir': { describe: "Set instances directory", nargs: 1, type: 'string' }, + 'factorio-dir': { describe: "Set Factorio directory", nargs: 1, type: 'string' }, + 'id': { describe: "Set id of the slave", nargs: 1, type: 'number' }, + }); + }) + .command('show-config', "Show slave config") + .command('start', "Start slave") + .demandCommand(1, "You need to specify a command to run") + .strict() + .argv + ; - await fs.ensureDir(config.instanceDirectory); + let command = args._[0]; + + if (command === "create-config") { + await fs.outputFile(args.config, JSON.stringify({ + name: args.name, + masterURL: args.url, + masterAuthToken: args.token, + publicIP: args.ip, + instanceDirectory: args.instancesDir, + factorioDirectory: args.factorioDir, + id: args.id, + }, null, 4), { flag: 'wx' }); + return; + + } else if (command == "edit-config") { + let slaveConfig = JSON.parse(await fs.readFile(args.config)); + if ('name' in args) slaveConfig.name = args.name; + if ('url' in args) slaveConfig.masterURL = args.url; + if ('token' in args) slaveConfig.masterAuthToken = args.token; + if ('ip' in args) slaveConfig.publicIP = args.ip; + if ('instancesDir' in args) slaveConfig.instanceDirectory = args.instancesDir; + if ('factorioDir' in args) slaveConfig.factorioDirectory = args.factorioDir; + if ('id' in args) slaveConfig.id = args.id; + await fs.outputFile(args.config, JSON.stringify(slaveConfig, null, 4)); + return; + + } else if (command == "show-config") { + let slaveConfig = JSON.parse(await fs.readFile(args.config)); + console.log(slaveConfig); + return; + } + + // If we get here the command was start + + // handle commandline parameters + console.log(`Loading ${args.config}`); + const slaveConfig = JSON.parse(await fs.readFile(args.config)); + + await fs.ensureDir(slaveConfig.instanceDirectory); await fs.ensureDir("sharedPlugins"); await fs.ensureDir("sharedMods"); - let instance; - if (process.argv[3] !== undefined) { - let name = process.argv[3]; - try { - checkFilename(name); - } catch (err) { - throw new Error(`Instance name ${err.message}`); - } - let dir = path.join(config.instanceDirectory, name); - instance = new Instance(dir, name); - } - // Set the process title, shows up as the title of the CMD window on windows // and as the process name in ps/top on linux. - if (instance) { - process.title = "clusterioClient "+instance.name; - } + process.title = "clusterioClient"; - - // make sure we have the master access token (can't write to master without it since clusterio 2.0) - if(!config.masterAuthToken || typeof config.masterAuthToken !== "string"){ + // make sure we have the master access token + if(!slaveConfig.masterAuthToken || typeof slaveConfig.masterAuthToken !== "string"){ console.error("ERROR invalid config!"); console.error( "Master server now needs an access token for write operations. As clusterio\n"+ @@ -503,52 +618,64 @@ async function startClient() { return; } - // handle commandline parameters - if (!command || command == "help" || command == "--help") { - printUsage(); - process.exit(1); - } else if (command == "list") { - await listInstances(config); - process.exit(0); + // make sure url ends with / + if (!slaveConfig.masterURL.endsWith("/")) { + console.error("ERROR invalid config!"); + console.error("masterURL (set with --url) must end with '/'"); + process.exitCode = 1; + return; + } + + let slave = new Slave(slaveConfig); + + // Handle interrupts + let secondSigint = false + process.on('SIGINT', () => { + if (secondSigint) { + console.log("Caught second interrupt, terminating immediately"); + process.exit(1); + } + + secondSigint = true; + console.log("Caught interrupt signal, shutting down"); + slave.stop().then(() => { + // There's currently no shutdown mechanism for instance plugins so + // they keep the event loop alive. + process.exit(); + }); + }); + + await slave.start(); + + /* } else if (command == "manage"){ await manage(config, instance); // process.exit(0); } else if (command == "delete") { await deleteInstance(instance); - } else if (command == "start" && instance === undefined) { - console.error("ERROR: No instanceName provided!"); - console.error("Usage: node client start [instanceName]"); - process.exit(0); - } else if (command == "start" && !fs.existsSync(instance.path())) { - await createInstance(config, instance); - } else if (command == "start" && fs.existsSync(instance.path())) { - await startInstance(config, instance); - } else { - console.error("Invalid arguments, quitting."); - process.exit(1); - } + */ } // ensure instancemanagement only ever runs once var _instanceInitialized; -async function instanceManagement(config, instance, instanceconfig, server) { +async function instanceManagement(slaveConfig, instance, instanceconfig, server, socket) { if (_instanceInitialized) return; _instanceInitialized = true; - console.log("Started instanceManagement();"); + let compatUrl = slaveConfig.masterUrl; + if (compatUrl.endsWith("/")) { + compatUrl = compatUrl.slice(0, -1); + } + let compatConfig = { + name: slaveConfig.name, + masterURL: compatUrl, + masterAuthToken: slaveConfig.masterToken, + publicIP: slaveConfig.publicAddress, + instanceDirectory: slaveConfig.instancesDir, + factorioDirectory: slaveConfig.factorioDir, + } - /* Open websocket connection to master */ - var socket = ioClient(config.masterURL+"?token="+config.masterAuthToken); - socket.on("error", err => { - console.log("SOCKET | Error: ", err); - }); - socket.on("hello", data => { - console.log("SOCKET | registering slave!"); - socket.emit("registerSlave", { - instanceID: instanceconfig.unique, - }); - }); - setInterval(B=> socket.emit("heartbeat"), 10000); + console.log("Started instanceManagement();"); // load plugins and execute onLoad event let pluginsToLoad = await pluginManager.getPlugins(); @@ -556,7 +683,7 @@ async function instanceManagement(config, instance, instanceconfig, server) { for(let i = 0; i < pluginsToLoad.length; i++){ let pluginLoadStarted = Date.now(); - let combinedConfig = deepmerge(instanceconfig,config,{clone:true}); + let combinedConfig = deepmerge(instanceconfig,compatConfig,{clone:true}); combinedConfig.instanceName = instance.name; let pluginConfig = pluginsToLoad[i]; @@ -651,59 +778,6 @@ async function instanceManagement(config, instance, instanceconfig, server) { // this plugin doesn't have a client portion. Maybe it runs on the master only? } } - - // world IDs ------------------------------------------------------------------ - hashMods(instance, function(modHashes){ - setInterval(getID, 10000); - getID(); - function getID() { - server.sendRcon("/silent-command rcon.print(#game.connected_players)").then((playerCount) => { - var payload = { - time: Date.now(), - rconPort: instanceconfig.clientPort, - rconPassword: instanceconfig.clientPassword, - serverPort: instanceconfig.factorioPort, - unique: instanceconfig.unique, - publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config - mods:modHashes, - instanceName: instance.name, - info: instanceconfig.info, - }; - if(playerCount){ - payload.playerCount = playerCount.replace(/(\r\n\t|\n|\r\t)/gm, ""); - } else { - payload.playerCount = 0; - } - - function callback(err, mac) { - if (err) { - mac = "unknown"; - console.log("##### getMac crashed, but we don't really give a shit because we are probably closing down #####"); - } - global.mac = mac; - payload.mac = mac; - // console.log("Registered our presence with master "+config.masterURL+" at " + payload.time); - needle.post(config.masterURL + '/api/getID', payload, needleOptionsWithTokenAuthHeader(config), function (err, response, body) { - if (err && err.code != "ECONNRESET"){ - console.error("We got problems, something went wrong when contacting master "+config.masterURL+" at " + payload.time); - console.error(err); - } else if (response && response.body) { - // In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care. - if(response.body !== "ok") { - console.log("Got no \"ok\" while registering our precense with master "+config.masterURL+" at " + payload.time); - console.log(response.body); - } - } - }); - } - if(global.mac){ - callback(undefined, global.mac); - } else { - getMac(callback); - } - }); - } - }); } // END OF INSTANCE START --------------------------------------------------------------------- // string, function diff --git a/clusterctl.js b/clusterctl.js new file mode 100644 index 00000000..0b98b534 --- /dev/null +++ b/clusterctl.js @@ -0,0 +1,382 @@ +const io = require("socket.io-client"); +const jwt = require("jsonwebtoken"); +const fs = require("fs-extra"); +const yargs = require("yargs"); +const version = require("./package").version; +const asTable = require("as-table").configure({delimiter: ' | '}); + +const link = require("lib/link"); +const errors = require("lib/errors"); + + +/** + * Represens a command that can be runned by clusterctl + */ +class Command { + constructor({ definition, handler }) { + this.name = definition[0]; + this._definition = definition; + this._handler = handler; + } + + register(yargs) { + yargs.command(...this._definition); + } + + async run(args, control) { + await this._handler.call(this, args, control); + } +} + +/** + * Resolve a string to an instance ID + * + * Resolves a string with either an instance name or an id into an integer + * with the instance ID. + * + * @param link - link to master server to query instance on. + * @param instance - string with name or id of instance. + * @returns {number} instance ID. + */ +async function resolveInstance(control, instanceName) { + let instanceId; + if (/^-?\d+$/.test(instanceName)) { + instanceId = parseInt(instanceName, 10); + } else { + let response = await link.requests.listInstances.send(control); + for (let instance of response.list) { + if (instance.name === instanceName) { + instanceId = instance.id; + break; + } + } + + if (instanceId === undefined) { + throw new errors.CommandError(`No instance named ${instanceName}`); + } + } + + return instanceId; +} + +let commands = []; +commands.push(new Command({ + definition: ["list-slaves", "List slaves connected to the master"], + handler: async function(args, control) { + let response = await link.requests.listSlaves.send(control); + console.log(asTable(response.list)); + }, +})); + +commands.push(new Command({ + definition: ["list-instances", "List instances known to the master"], + handler: async function(args, control) { + let response = await link.requests.listInstances.send(control); + console.log(asTable(response.list)); + } +})); + +commands.push(new Command({ + definition: ['create-instance', "Create an instance", (yargs) => { + yargs.options({ + 'name': { describe: "Name of the instance", nargs: 1, type: 'string', demandOption: true }, + 'slave': { describe: "Slave to create on", nargs: 1, type: 'string', demandOption: true }, + }); + }], + handler: async function(args, control) { + let slaveId; + if (/^-?\d+$/.test(args.slave)) { + slaveId = parseInt(args.slave, 10); + } else { + let response = await link.requests.listSlaves.send(control); + for (let slave of response.list) { + if (slave.name === args.slave) { + slaveId = slave.id; + break; + } + } + + if (slaveId === undefined) { + throw new errors.CommandError(`No slave named ${args.slave}`); + } + } + let response = await link.requests.createInstanceCommand.send(control, { + name: args.name, + slave_id: slaveId, + }); + console.log(response); + }, +})); + +commands.push(new Command({ + definition: ['create-save', "Create a new save on an instance", (yargs) => { + yargs.options({ + 'instance': { describe: "Instance to create on", nargs: 1, type: 'string', demandOption: true }, + }); + }], + handler: async function(args, control) { + let response = await link.requests.createSave.send(control, { + instance_id: await resolveInstance(control, args.instance), + }); + console.log(response); + }, +})); + +commands.push(new Command({ + definition: ['start-instance', "Start instance", (yargs) => { + yargs.options({ + 'instance': { describe: "Instance to start", nargs: 1, type: 'string', demandOption: true }, + }); + }], + handler: async function(args, control) { + let response = await link.requests.startInstance.send(control, { + instance_id: await resolveInstance(control, args.instance), + }); + console.log(response); + }, +})); + +commands.push(new Command({ + definition: ['stop-instance', "Stop instance", (yargs) => { + yargs.options({ + 'instance': { describe: "Instance to stop", nargs: 1, type: 'string', demandOption: true }, + }); + }], + handler: async function(args, control) { + let response = await link.requests.stopInstance.send(control, { + instance_id: await resolveInstance(control, args.instance), + }); + console.log(response); + }, +})); + +commands.push(new Command({ + definition: ['delete-instance', "Delete instance", (yargs) => { + yargs.options({ + 'instance': { describe: "Instance to delete", nargs: 1, type: 'string', demandOption: true }, + }); + }], + handler: async function(args, control) { + let response = await link.requests.deleteInstance.send(control, { + instance_id: await resolveInstance(control, args.instance), + }); + console.log(response); + }, +})); + +commands.push(new Command({ + definition: ['send-rcon', "Send RCON command", (yargs) => { + yargs.options({ + 'instance': { describe: "Instance to sent to", nargs: 1, type: 'string', demandOption: true }, + 'command': { describe: "command to send", nargs: 1, type: 'string', demandOption: true }, + }); + }], + handler: async function(args, control) { + let response = await link.requests.sendRcon.send(control, { + instance_id: await resolveInstance(control, args.instance), + command: args.command + }) + + // Factorio includes a newline in it's response output. + process.stdout.write(response.result); + }, +})); + +// Convert to mapping from name to command instance +commands = new Map([...commands.map(command => [command.name, command])]); + + +/** + * Handles running the control + * + * Connects to the master server over the socket.io connection and sends + * commands to it. + */ +class Control extends link.Client { + + // I don't like God classes, but the alternative of putting all this state + // into global variables is not much better. + constructor(controlConfig) { + super('control', controlConfig.url, controlConfig.token); + link.attachAllMessages(this); + } + + register() { + console.log("SOCKET | registering control"); + this.send('register_control', { + agent: 'clusterctl', + version: version, + }); + } +} + +async function findCredentials() { + // Try looking in the default master config location + try { + let masterConfig = JSON.parse(await fs.readFile("config.json")); + let token = jwt.sign({ id: "api" }, masterConfig.masterAuthSecret, { expiresIn: 600 }); + + let url; + if (masterConfig.sslPort) { + url = `https://localhost:${masterConfig.sslPort}`; + } else { + url = `http://localhost:${masterConfig.masterPort}`; + } + + return { url, token }; + + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + // Try looking in the default slave config location + // In the future it's likely that tokens will be assigned more granular + // authorizations, meaning that you cannot use a slave token to control + // the cluster. + try { + let slaveConfig = JSON.parse(await fs.readFile("config-slave.json")); + return { + url: slaveConfig.masterURL, + token: slaveConfig.masterAuthToken, + } + + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + throw new errors.StartupError("Couldn't find credentials to connect to the master server with"); +} + + +async function startControl() { + yargs + .scriptName("clusterctl") + .usage("$0 [options]") + .option('config', { + nargs: 1, + describe: "config file to get credentails from", + default: "config-control.json", + defaultDescription: "auto", + type: 'string', + }) + .command('create-config', "Create control config", (yargs) => { + yargs.options({ + 'url': { describe: "Master URL", nargs: 1, type: 'string', default: "http://localhost:8080/" }, + 'token': { describe: "Master token", nargs: 1, type: 'string', demandOption: true }, + }); + }) + .command('edit-config', "Edit control config", (yargs) => { + yargs.options({ + 'url': { describe: "Set master URL", nargs: 1, type: 'string' }, + 'token': { describe: "Set master token", nargs: 1, type: 'string' }, + }); + }) + .command('show-config', "Show control config") + ; + + for (let command of commands.values()) { + command.register(yargs); + } + + const args = yargs.demandCommand(1, "You need to specify a command to run") + .strict() + .argv + ; + + let commandName = args._[0]; + if (commandName === "create-config") { + await fs.outputFile(args.config, JSON.stringify({ + url: args.url, + token: args.token, + }, null, 4), { flag: 'wx' }); + return; + + } else if (commandName === "edit-config") { + let controlConfig = JSON.parse(await fs.readFile(args.config)); + if ('url' in args) controlConfig.url = args.url; + if ('token' in args) controlConfig.token = args.token; + await fs.outputFile(args.config, JSON.stringify(controlConfig, null, 4)); + return; + + } else if (commandName === "show-config") { + let controlConfig = JSON.parse(await fs.readFile(args.config)); + console.log(controlConfig); + return; + } + + // The remaining commands require connecting to the master server. + let controlConfig; + try { + controlConfig = JSON.parse(await fs.readFile(args.config)); + } catch (err) { + if (err.code === "ENOENT") { + controlConfig = await findCredentials(); + } else { + throw err; + } + } + + let control = new Control(controlConfig); + await control.connect(); + + if (commands.has(commandName)) { + command = commands.get(commandName); + + try { + await command.run(args, control) + + } catch (err) { + if (err instanceof errors.CommandError) { + console.error(`Error running command: ${err.message}`); + process.exitCode = 1; + + } else if (err instanceof errors.RequestError) { + console.error(`Error sending request: ${err.message}`); + process.exitCode = 1; + + } else { + control.close("error"); + throw err; + } + } + + control.close("quit"); + return; // ?? + } + + //XXX control.close("done"); +} + + +if (module === require.main) { + console.warn(` ++==========================================================+ +I WARNING: This is the development branch for the 2.0 I +I version of clusterio. Expect things to break. I ++==========================================================+ +` + ); + startControl().catch(err => { + if (!(err instanceof errors.StartupError)) { + console.error(` ++----------------------------------------------------------------+ +| Unexpected error occured while starting control, please report | +| it to https://github.com/clusterio/factorioClusterio/issues | ++----------------------------------------------------------------+` + ); + } else { + console.error(` ++-------------------------------+ +| Unable to to start clusterctl | ++-------------------------------+` + ); + } + + console.error(err); + process.exitCode = 1; + }); +} diff --git a/config.json.dist b/config.json.dist index c162cbed..5481789e 100644 --- a/config.json.dist +++ b/config.json.dist @@ -1,15 +1,8 @@ { "__comment": "Where factorio folder. Should be in the same folder as this config file.", "factorioDirectory": "./factorio", - "instanceDirectory": "./instances", "databaseDirectory": "./database", - "__comment1": "IP players should connect to to join your servers, should be portforwarded and stuff", - "publicIP": "localhost", - - "__comment2": "URL the master web server runs at, can be ignored on the master system. Must not end with a slash", - "masterURL": "http://localhost:8080", - "__commentMasterPort": "HTTP port for the master to listen to", "masterPort": 8080, @@ -18,9 +11,6 @@ "sslCert": "database/certificates/cert.crt", "sslPrivKey": "database/certificates/cert.key", - "__commentMasterAuthToken": "Authentication token to connect to the master server. Find it in secret-api-token.txt on the master.", - "masterAuthToken": "", - "__commentAuthSecret": "This is a long random string used for generating authentication tokens. Do not share. Regenerates when secret-api-token.txt is deleted.", "masterAuthSecret": "", diff --git a/docs/socket.md b/docs/socket.md new file mode 100644 index 00000000..49827e87 --- /dev/null +++ b/docs/socket.md @@ -0,0 +1,115 @@ +Socket.io protocol +================== + +This document describes the protocol used between the master and +slaves, and unless you're developing Clusterio it's probably not going +to be very useful for you. + +The protocol is built on the [socket.io](https://socket.io/) library +which lets mulitple streams be multiplexed in many different ways over +the same connection and abstracts away reconnection logic. +Unfortunately I've been unable to figure what the message delivery +guarantees of the library is, so whether or not messages can be lost is +still an unknown, I assume that messages are delivered as long as a +connection is maintained or manages to reconnect. + +While there are many mechanisms to multiplex streams, the core +communication between the master server and slaves is done entirely by +sending single POJO payloads over the `message` event. There's a +shortcut for sending these via the `.send` method of the socket.io +sockets. + +These messages contain the following three properties: +- seq - integer - The sequence number of the message +- type - string - The type of message +- data - object - Message data + + +Handshake +--------- + +Upon connecting to the socket.io endpoint the server will respond with a +`hello` message containing the master server version as the data. The +client is then expected to send a `register_slave` or `register_control` +message back depending on what kind of client it is. If the register +operation is successful the master will reply with a `ready` message +with an empty data payload. + +**Master `hello` data** + +- version - string - The version of the master, e.g. "2.0.0". + +**Slave `register_slave` response data** + +- agent - string - Human readable identifying the software connecting. +- version - string - The protocol version of the slave, e.g. "2.0.0". +- name - string - Name of the slave. +- id - integer - ID of the slave. + +**Control `register_control repsonse data** + +- agent - string - Human readable identifying the software connecting. +- version - string - The protocol version of the control, e.g. "2.0.0". + +**Example handshake** +```js +// Server hello +({ + type: 'hello', + data: { + version: "2.0.0", + } +}) + +// Client response +({ + type: 'register_slave', + data: { + agent: 'Clusterio Slave', + version: "2.0.0", + name: "Foo", + id: 123, + } +}) + +// Server repsonse +({ + type: 'ready', + data: {} +}) +``` + + +Requests +-------- + +Requests are messages which expects a response in return. They function +a lot like HTTP requests except that both parties of a connection can +inniate a request that is expected to be responded to. By convention +message types ending in `*_request` is a request, and is expected to be +replied to with a corresponding `*_response` message. If an error +occured while processing the request an error response containing an +error property in the data payload is sent instead. + +**`*_response` data** + +- seq - integer - The seq number of the request message. +- additional properties may be present depending on the type of request. + +**`*_resopnse` error data** +- seq - integer - The seq number of the request message. +- error - string - Human readable text message describing the error. + +See [lib/link/requests.js](lib/link/requests.js) for the recognized +requests and their implementation. + + +Closing the connection +---------------------- + +The party that innitates the shut down of the connection should send a +close message containing the reason for closing before disconnecting. + +**`close` data** + +- reason - string - Reason for closing the connection diff --git a/lib/clusterTools/getInstanceName.js b/lib/clusterTools/getInstanceName.js deleted file mode 100644 index c0ba7e4e..00000000 --- a/lib/clusterTools/getInstanceName.js +++ /dev/null @@ -1,40 +0,0 @@ -const needle = require("needle"); - -module.exports = function(config){ - const instances = []; - - if(config){ - return function getInstanceName(instanceID){ - return new Promise((resolve, reject) => { - let instance = instances[instanceID]; - if(!instance){ - needle.get(config.masterURL + '/api/slaves', { compressed: true }, (err, response) => { - if(err || response.statusCode != 200) { - console.log("Unable to get JSON master/api/slaves, master might be unaccessible"); - } else if (response && response.body) { - if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); reject(new Error())} - try { - for (let index in response.body) - instances[index] = response.body[index].instanceName; - } catch (e){ - console.log(e); - return null; - } - instance = instances[instanceID] - if (!instance) instance = instanceID; //somehow the master doesn't know the instance - resolve(instance); - } - }); - } else { - resolve(instance); - } - }); - } - } else { - return async function thisAintGonnaWork(){ - let err = new Error("getInstanceName from lib/clusterTools needs to be called with {config} to work"); - console.log(err); - throw err; - } - } -} diff --git a/lib/clusterTools/index.js b/lib/clusterTools/index.js deleted file mode 100644 index 09351555..00000000 --- a/lib/clusterTools/index.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - -Functions frequently needed from different places in the cluster. Ex: - * Getting an instance's displayname from their ID - -*/ - -module.exports = ({config} = {}) => ({ - /** - async function to get an instances displayname from their instanceID - - arg1 is the instanceID - Promise resolves with a string, either the displayName or instanceID if not found. - */ - getInstanceName: require("./getInstanceName")(config), -}) diff --git a/lib/errors.js b/lib/errors.js new file mode 100644 index 00000000..a213a7a9 --- /dev/null +++ b/lib/errors.js @@ -0,0 +1,20 @@ +/** + * Errors thrown in Clusterio + * @module + */ + +class CommandError extends Error { } +class RequestError extends Error { } + +// Signal for messages that fail validation +class InvalidMessage extends Error { } + +// Errror class for known errors occuring during startup +class StartupError extends Error { } + +module.exports = { + CommandError, + RequestError, + InvalidMessage, + StartupError, +} diff --git a/lib/factorio/server.js b/lib/factorio/server.js index a6ba17c4..b93e9b73 100644 --- a/lib/factorio/server.js +++ b/lib/factorio/server.js @@ -612,7 +612,7 @@ class FactorioServer extends events.EventEmitter { * @returns {string} response from server. */ async sendRcon(message) { - this._check(["init", "running"]) + this._check(["running"]) if (!this._rconReady) { await events.once(this, 'rcon-ready'); } diff --git a/lib/getInstances.js b/lib/getInstances.js deleted file mode 100644 index 54a4b6ec..00000000 --- a/lib/getInstances.js +++ /dev/null @@ -1,17 +0,0 @@ -const fileOps = require("lib/fileOps"); -// const config = require("./../config"); - -module.exports = function(config) { - return { - sync: function(){ - let instances; - try { - instances = fileOps.getDirectoriesSync(config.instanceDirectory); - } catch (e) { - // there are no instances created yet, probably missing the folder - instances = []; - } - return instances; - }, - } -} \ No newline at end of file diff --git a/lib/link/index.js b/lib/link/index.js new file mode 100644 index 00000000..c8641a97 --- /dev/null +++ b/lib/link/index.js @@ -0,0 +1,4 @@ +module.exports = { + ...require('lib/link/link'), + ...require('lib/link/messages'), +}; diff --git a/lib/link/link.js b/lib/link/link.js new file mode 100644 index 00000000..a6398d21 --- /dev/null +++ b/lib/link/link.js @@ -0,0 +1,399 @@ +"use strict"; +const events = require("events"); +const io = require("socket.io-client"); + +const schema = require("lib/schema"); +const errors = require("lib/errors"); +const messages = require("lib/link/messages"); + +// Some definitions for the terminology used here: +// link: Either side of a master - client connection +// connection: the master server side of a link +// client: the side that is not the master server of a link +// message: the complete object sent using the 'message' event +// data: the data property of a message, essentially the payload. +// type: the type property of a message, identifies what to expect in data. + +/** + * Common interface for server and client connections + */ +class Link { + constructor(source, target, socket) { + this.source = source; + this.target = target; + this.socket = socket; + + this._seq = 1; + this._waiters = new Map(); + this._handlers = new Map(); + this._validators = new Map(); + } + + /** + * Process a received message on the link + * + * Validates and invokes the handler and/or waiters for a message that has + * been received. An unhandled message is considered to be an error. + * + * @throws {InvalidMessage} + * if validation failed, is missing, the message is invalid, or no + * handlers/waiters were invoked to handle the message. + */ + processMessage(message) { + if (!schema.message(message)) { + throw new errors.InvalidMessage("Malformed message"); + } + + let validator = this._validators.get(message.type); + if (!validator) { + throw new errors.InvalidMessage(`No validator for ${message.type}`); + } + + if (!validator(message)) { + // TODO better logging of errors encountered validating + console.error(validator.errors); + throw new errors.InvalidMessage(`Validation failed for ${message.type}`); + } + + let hadHandlers = this._processHandler(message); + let hadWaiters = this._processWaiters(message); + + if (!hadWaiters && !hadHandlers) { + // XXX console.error(`Unhandled message: ${JSON.stringify(message, null, 4)}`); + throw new errors.InvalidMessage(`Unhandled message ${message.type}`); + } + } + + /** + * Invoke the handler for this message if it exists + * + * @returns {Boolean} true if a handler was invoked. + */ + _processHandler(message) { + let handler = this._handlers.get(message.type); + if (handler) { + handler(message); + return true; + } else { + return false; + } + } + + /** + * Invokes the waiters for this message if any + * + * @returns {Boolean} true if a waiter was triggered. + */ + _processWaiters(message) { + let { seq, type, data } = message; + let waiters = this._waiters.get(type); + if (!waiters || !waiters.length) { + return false; + } + + let matched = []; + waitersLoop: for (let index = 0; index < waiters.length; index++) { + let waiter = waiters[index]; + for (let [key, value] of Object.entries(waiter.data)) { + if (data[key] !== value) { + continue waitersLoop; + } + } + + waiter.resolve(message); + matched.push(index); + } + + matched.reverse(); + for (let index of matched) { + waiters.splice(index, 1); + } + + return matched.length > 0; + } + + /** + * Set handler for a message type + * + * Set function called upon receiving a message of the given type. + * + * @param {string} type - Type of message to listen to + * @param {function} handler - + * Callack to invoke on messages of the given type. + * @param {function} validator - + * Function validating the message have the correct format. + */ + setHandler(type, handler, validator) { + if (this._handlers.has(type)) { + throw new Error(`${type} already has a handler`); + } + + if (!validator) { + throw new Error("validator is required"); + } + + this.setValidator(type, validator); + this._handlers.set(type, handler); + } + + /** + * Set validator for a message type + * + * Set function called upon receiving a message of the given type in + * order to validate the message. The function should return true if + * the message is valid and false otherwise. + * + * @param {string} type - Type of message to validate + * @param {function} validator - Validation function. + */ + setValidator(type, validator) { + if (this._validators.has(type)) { + throw new Error(`${type} already has a validator`); + } + + this._validators.set(type, validator); + } + + /** + * Send a message over the link + * + * @returns the sequence number of the message sent + */ + send(type, data) { + this.socket.send({ seq: this._seq, type, data }); + return this._seq++; + } + + /** + * Wait for a message matching given type and data + * + * Waits for a message over the link that matches the type given as well + * as any properties specified in data. For example specifying { foo: + * 21 } as the data will cause it to wait until a message with the right + * type and a foo property value of 21 in the data payload. + */ + async waitFor(type, data) { + if (!this._validators.has(type)) { + throw new Error(`no validator for ${type}`); + } + + return new Promise((resolve, reject) => { + if (!this._waiters.has(type)) { + this._waiters.set(type, [{ data, resolve }]); + } else { + this._waiters.get(type).push({ data, resolve }); + } + }); + } + + /** + * Close the connection with the given reason. + * + * Sends a close message and disconnects from the socket. + */ + close(reason) { + this.send('close', { reason }); + this.disconnect(); + } + + + /** + * Disconnect from the socket + * + * Abstract function that subclases override to implement disconnection + * from the socket. + */ + disconnect() { + throw new Error("Abstract function"); + } +} + +class Connection extends Link { + constructor(target, socket) { + super('master', target, socket); + + this._seq = 3; // XXX this is private to Link + this.socket.on('message', payload => { + try { + this.processMessage(payload); + } catch (err) { + if (err instanceof errors.InvalidMessage) { + this.close(`Invalid message: ${err.message}`); + } else { + throw err; + } + } + }); + this.socket.on('disconnect', this.disconnect.bind(this)); + + this.setHandler('close', payload => { + let address = this.socket.handshake.address + console.log(`SOCKET | received close from control ${address}: ${payload.data.reason}`); + this.socket.disconnect(true); + }, schema.close); + + socket.send({ seq: 2, type: 'ready', data: {} }); + } + + disconnect() { + this.socket.disconnect(true); + } +} + + +class Client extends Link { + constructor(source, url, token) { + super(source, 'master', null); + + this._url = url; + this._token = token; + + this._state = "new"; + this._events = new events.EventEmitter(); + + this.setHandler('close', message => { + console.log(`SOCKET | received close from server: ${message.data.reason}`); + this.socket.close(); + return; // XXX what now? + }, schema.close); + } + + _check(expectedStates) { + if (!expectedStates.includes(this._state)) { + throw new Error( + `Expected state ${expectedStates} but state is ${this._state}` + ); + } + } + + async connect() { + this._check(["new"]); + this._state = "handshake"; + + // Open socket.io connection to master + console.log(`SOCKET | connecting to ${this._url}`) + let url = new URL(this._url); + url.searchParams.append('token', this._token); + + // I really have to question the choice of having the path connected + // to be passed as separate parameter instead of the namespace. + let urlPath = url.pathname + "socket.io"; + url.pathname = ""; + this.socket = io(url.href, { + path: urlPath, + reconnectionAttempts: 2, + + // For now we do not verify TLS certificates since the default setup is + // to create a self-signed certificate. + rejectUnauthorized: false, + }); + + _attachSocketHandlers(); + + // Wait for the connection to be ready + await events.once(this._events, 'ready'); + } + + _attachSocketHandlers() { + this.socket.on("error", err => { + console.error("SOCKET | Error:", err); + this.socket.close(); + }); + + /*this.socket.on("connect_error", err => { + console.log("SOCKET | Connect error:", err); + });*/ + this.socket.on("connect_timeout", () => { + console.log("SOCKET | Connect timeout"); + }); + this.socket.on("reconnect", attempt => { + console.log(`SOCKET | Reconnect no. ${attempt} succeeded`); + }); + this.socket.on("reconnect_attempt", () => { + console.log("SOCKET | Attempting reconnect"); + }); + this.socket.on("reconnecting", attempt => { + console.log(`SOCKET | Attempting reconnect no. ${attempt}`); + }); + /*this.socket.on("reconnect_error", err => { + console.log("SOCKET | Reconnect error:", err); + });*/ + this.socket.on("reconnect_failed", () => { + console.log("SOCKET | Reconnecting failed"); + }); + /*this.socket.on("ping", () => { + console.log("SOCKET | Ping"); + }); + this.socket.on("pong", (ms) => { + // XXX Latency might be useful + console.log(`SOCKET | Pong latency ${ms}ms`); + });*/ + this.socket.on("connect", () => { + console.log("SOCKET | connected"); + }); + + // Handle messages + this.socket.on("message", message => { + if (this._state === "handshake") { + this._processHandshake(message); + } else if (this._state === "ready") { + try { + this.processMessage(message); + } catch (err) { + if (err instanceof errors.InvalidMessage) { + this.close(`Invalid message: ${err.message}`); + } else { + throw err; + } + } + } else { + throw new Error(`Received message in unexpected state ${this._state}`); + } + }) + } + + disconnect(reason) { + this.socket.close(); + // XXX what now? + } + + _processHandshake(message) { + if (!schema.serverHandshake(message)) { + console.log("SOCKET | closing after received invalid handshake:", message); + this.close("Invalid handshake"); + return; + } + + let { seq, type, data } = message; + if (type == 'hello') { + console.log(`SOCKET | received hello from master version ${data.version}`); + this.register(); + + } else if (type === 'ready') { + console.log("SOCKET | received ready from master"); + this._state = "ready"; + this._events.emit('ready'); + + } else if (type === 'close') { + console.log(`SOCKET | received close from server: ${data.reason}`); + this._events.emit('error', new Error(`server closed during handshake: ${data.reason}`)); + this.socket.close(); + } + } + + /** + * Register the link with the server + * + * This function is expected to be overriden by sub-classes and send the + * register message over the socket when invoked. + */ + register() { + throw Error("Abstract function"); + } +} + +module.exports = { + Link, + Connection, + Client, +} diff --git a/lib/link/messages.js b/lib/link/messages.js new file mode 100644 index 00000000..fd94ef59 --- /dev/null +++ b/lib/link/messages.js @@ -0,0 +1,429 @@ +"use strict"; +const schema = require("lib/schema"); +const errors = require("lib/errors"); + +/** + * Represents a request sent over the link + * + * A request-response message that can be innitated by either party of the + * link. + */ +class Request { + constructor({ + type, sources, targets, requestProperties = {}, responseProperties = {} + }) { + this.type = type; + this.sources = sources; + this.targets = targets; + + this.requestType = type + '_request'; + this.responseType = type + '_response'; + + this._requestValidator = schema.compile({ + $schema: "http://json-schema.org/draft-07/schema#", + properties: { + "type": { const: this.requestType }, + "data": { + additionalProperties: false, + required: Object.keys(requestProperties), + properties: requestProperties, + }, + }, + }); + + this._responseValidator = schema.compile({ + $schema: "http://json-schema.org/draft-07/schema#", + properties: { + "type": { const: this.responseType }, + "data": { + anyOf: [ + { + additionalProperties: false, + required: ["seq", ...Object.keys(responseProperties)], + properties: { + "seq": { type: "integer" }, + ...responseProperties, + }, + }, + { + additionalProperties: false, + required: ["seq", "error"], + properties: { + "seq": { type: "integer" }, + "error": { type: "string" }, + }, + }, + ], + }, + } + }); + } + + /** + * Attach to a link + * + * Set a the validator for the response on the given link if it's a + * source and sets the handler for the request if it is a target. + * + * Does nothing if the link is neither a source nor a target for this + * request. + * + * @param {Link} link - The link to attach to. + * @param {Function} handler - + * Async function to invoke with link set as this to handle the + * request request. Only used on the target side. + * @throws {Error} if the handler is needed and not defined. + */ + attach(link, handler) { + // Source side. + if (this.sources.includes(link.source) && this.targets.includes(link.target)) { + link.setValidator(this.responseType, this._responseValidator); + } + + // Target side. Note: Source and target is reversed here. + if (this.targets.includes(link.source) && this.sources.includes(link.target)) { + if (!handler) { + throw new Error(`Missing handler for ${this.requestType} on ${link.target}-${link.source} link`); + } + + link.setHandler(this.requestType, message => { + handler.call(link, message).then(response => { + if (response === undefined) { + // XXX Should we allow implicit responses like this? + response = {}; + } + + if (Object.prototype.hasOwnProperty.call(response, 'seq')) { + throw new errors.RequestError("response contains reserved property 'seq'"); + } + link.send(this.responseType, { seq: message.seq, ...response }); + + }).catch(err => { + if (!(err instanceof errors.RequestError)) { + console.error(`Unexpected error while responding to ${this.requestType}`, err); + } + link.send(this.responseType, { seq: message.seq, error: err.message }); + }); + }, this._requestValidator); + } + } + + /** + * Send request over the given link + * + * Sends the given data over the link as the request payload and waits + * for the response. + * + * @returns {object} response data + */ + async send(link, data = {}) { + // XXX validate link target/source? + let seq = link.send(this.requestType, data); + let responseMessage = await link.waitFor(this.responseType, { seq }); + if (responseMessage.data.error) { + throw new errors.RequestError(responseMessage.data.error); + } + return responseMessage.data; + } +} + +/** + * Represents a request sent to the master that is forwarded to an instance + * + * This is a two part request with a master part and a slave part. The + * master part is done by the forwardInstanceRequest method on the master + * side of link which finds the slave the requuested instance is on and + * forwards it there using the foward method. + * + * The slave part is done by the forwardInstanceRequest on the slave side of + * the link and finds the instance and then calls the handler with it and + * the message. + */ +class InstanceRequest { + constructor({ + type, sources, requestProperties = {}, responseProperties = {} + }) { + this._masterRequest = new Request({ + type: type + '_master', + sources, + targets: ['master'], + requestProperties: { + "instance_id": { type: "integer" }, + ...requestProperties, + }, + responseProperties, + }); + this._slaveRequest = new Request({ + type: type + '_slave', + sources: ['master'], + targets: ['slave'], + requestProperties: { + "instance_id": { type: "integer" }, + ...requestProperties, + }, + responseProperties, + }); + } + + /** + * Attach to a link + * + * Attach master/slave part of the instance request. If this is the + * slave side the handler argument is required. + * + * Does nothing if the link is neither a source nor a target for the + * slave or master request. + * + * @param {Link} link - The link to attach to. + * @param {Function} handler - + * Async function to to handle the instance request. It is passed on + * to forwardInstanceRequest on the slave side. + * @throws {Error} if the handler is needed and not defined. + */ + attach(link, handler) { + this._masterRequest.attach(link, async (message) => { + return await link.forwardInstanceRequest(message, this); + }); + this._slaveRequest.attach(link, async (message) => { + return await link.forwardInstanceRequest(message, handler); + }); + } + + /** + * Forward the message over the link + * + * This method is expected to be called by forwardInstanceRequest on the + * master with the slave link that the instance is on. + */ + async forward(link, message) { + let response = await this._slaveRequest.send(link, message.data); + delete response.seq; + return response; + } + + /** + * Send request over the given link + * + * Sends the given data over the link to the master and waits for it to + * forward the request to the slave with the instance on and return the + * result of it. + * + * @returns {object} response data + */ + async send(link, data = {}) { + return await this._masterRequest.send(link, data); + } +} + +let requests = {} + +// Management requests +requests.listSlaves = new Request({ + type: 'list_slaves', + sources: ['control'], + targets: ['master'], + responseProperties: { + "list": { + type: "array", + items: { + additionalProperties: false, + required: ["agent", "version", "name", "id", "connected"], + properties: { + "agent": { type: "string" }, + "version": { type: "string" }, + "name": { type: "string" }, + "id": { type: "integer" }, + "connected": { type: "boolean" }, + }, + }, + }, + }, +}); + +requests.listInstances = new Request({ + type: 'list_instances', + sources: ['control'], + targets: ['master'], + responseProperties: { + "list": { + type: "array", + items: { + additionalProperties: false, + required: ["name", "id", "slave_id"], + properties: { + "name": { type: "string" }, + "id": { type: "integer" }, + "slave_id": { type: "integer" }, + }, + }, + }, + }, +}); + +requests.createInstanceCommand = new Request({ + type: 'create_instance', + sources: ['control'], + targets: ['master'], + requestProperties: { + "name": { type: "string" }, + "slave_id": { type: "integer" }, + }, +}); + +requests.createInstance = new Request({ + type: 'create_instance', + sources: ['master'], + targets: ['slave'], + requestProperties: { + "id": { type: "integer" }, + "options": { + additionalProperties: false, + required: [ + "name", "description", "visibility", "username", "token", "game_password", + "verify_user_identity", "allow_commands", "auto_pause", + ], + properties: { + "name": { type: "string" }, + "description": { type: "string" }, + "visibility": { type: "object" }, + "username": { type: "string" }, + "token": { type: "string" }, + "game_password": { type: "string" }, + "verify_user_identity": { type: "boolean" }, + "allow_commands": { type: "string" }, + "auto_pause": { type: "boolean" }, + }, + }, + }, +}); + + +requests.createSave = new InstanceRequest({ + type: 'create_save', + sources: ['control'], +}); + +requests.startInstance = new InstanceRequest({ + type: 'start_instance', + sources: ['control'], +}); + +requests.stopInstance = new InstanceRequest({ + type: 'stop_instance', + sources: ['control'], +}); + +requests.deleteInstance = new InstanceRequest({ + type: 'delete_instance', + sources: ['control'], +}); + +requests.sendRcon = new InstanceRequest({ + type: 'send_rcon', + sources: ['control'], + requestProperties: { + "command": { type: "string" }, + }, + responseProperties: { + "result": { type: "string" }, + } +}); + + +class Event { + constructor({ type, sources, targets, eventProperties = {} }) { + this.type = type; + this.sources = sources; + this.targets = targets; + + this.eventType = type + '_event'; + this._eventValidator = schema.compile({ + $schema: "http://json-schema.org/draft-07/schema#", + properties: { + "type": { const: this.eventType }, + "data": { + additionalProperties: false, + required: Object.keys(eventProperties), + properties: eventProperties, + }, + }, + }); + } + + attach(link, handler) { + // Target side. Note: Source and target is reversed here. + if (this.targets.includes(link.source) && this.sources.includes(link.target)) { + if (!handler) { + throw new Error(`Missing handler for ${this.eventType} on ${link.target}-${link.source} link`); + } + + link.setHandler(this.eventType, message => { + // XXX Should event handlers even be allowed to be async? + handler.call(link, message).catch(err => { + console.error(`Unexpected error while handling ${this.eventType}`, err); + }); + }, this._eventValidator); + } + } + + /** + * Send event over the given link + * + * Sends the given event data over the link. + */ + send(link, data = {}) { + link.send(this.eventType, data); + } +} + +let events = {}; +events.updateInstances = new Event({ + type: 'update_instances', + sources: ['slave'], + targets: ['master'], + eventProperties: { + "instances": { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["id", "name"], + properties: { + "id": { type: "integer" }, + "name": { type: "string" }, + }, + }, + }, + }, +}); + + +/** + * Attaches all requests and events to the given link + * + * Loops through all requests and events defined and attaches all of + * them to the link. For requests the handler used is looked up on the + * link instance as the name of the request + 'RequestHandler'. For + * events the the handler used is looked up on the link instance as the + * name of the event + 'EventHandler'. + */ +function attachAllMessages(link) { + for (let [name, request] of Object.entries(requests)) { + request.attach(link, link[name + 'RequestHandler']); + } + + for (let [name, event] of Object.entries(events)) { + event.attach(link, link[name + 'EventHandler']); + } +} + +module.exports = { + Request, + InstanceRequest, + Event, + + attachAllMessages, + + requests, + events, +} diff --git a/lib/manager/modManager.js b/lib/manager/modManager.js index 4d472135..cef0a6b4 100644 --- a/lib/manager/modManager.js +++ b/lib/manager/modManager.js @@ -36,8 +36,7 @@ modManager.listMods(config, instance); */ async function listMods(config, instance) { return new Promise((resolve, reject) => { - const instances = require("./../getInstances")(config).sync(); - if (instances.includes(instance.name)) { + if (true) { factorioAPI.init(false, instance.path("instanceMods"), instance.path("saves"), instance.server.version); factorioAPI.authenticate({ username: config.username, diff --git a/lib/schema.js b/lib/schema.js new file mode 100644 index 00000000..8b49c9cb --- /dev/null +++ b/lib/schema.js @@ -0,0 +1,144 @@ +const ajv = new (require('ajv'))({ + verbose: true, + format: "full", + extendRefs: "fail", + strictDefaluts: true, + strictKeywords: true, +}) + + +const closeSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + title: "Close Message", + description: "Format for a close messages on a link", + properties: { + "type": { const: "close" }, + "data": { + additionalProperties: false, + required: ["reason"], + properties: { + "reason": { type: "string" }, + } + }, + }, +} + +const close = ajv.compile(closeSchema); + +/** + * The base format for any message sent over the link + */ +const messageSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + title: "Message", + description: "Format for messages on a link", + type: "object", + additionalProperties: false, + required: ["seq", "type", "data"], + properties: { + "seq": { type: "integer" }, + "type": { type: "string" }, + "data": { type: "object" }, + }, +} + +const message = ajv.compile(messageSchema); + +const serverActions = [ + {}, +]; + +const actionResponse = { + additionalProperties: false, + required: ["action", "id", "status", "message"], + properties: { + "action": { enum: ["TODO"] }, + "id": { type: "integer" }, + "status": { enum: ["started", "done", "error"] }, + "message": { type: "string" }, + }, +}; + +const serverHandshake = ajv.compile({ + $schema: "http://json-schema.org/draft-07/schema#", + title: "Server Handshake", + description: "Message sent from the server to a client during the handshake", + allOf: [ messageSchema ], + anyOf: [ + { + properties: { + "type": { const: "hello" }, + "data": { + additionalProperties: false, + required: ["version"], + properties: { + "version": { type: "string" }, + } + }, + }, + }, + { + properties: { + "type": { const: "ready" }, + "data": { + additionalProperties: false, + /* XXX anything here? + required: ["version"], + properties: { + "version": { type: "string" }, + }*/ + }, + }, + }, + closeSchema, + ], +}); + +const clientHandshake = ajv.compile({ + $schema: "http://json-schema.org/draft-07/schema#", + title: "Client Message", + description: "Message sent from a client to the server", + allOf: [ messageSchema ], + anyOf: [ + { + properties: { + "type": { const: "register_slave" }, + "data": { + additionalProperties: false, + required: ["agent", "version", "name", "id"], + properties: { + "agent": { type: "string" }, + "version": { type: "string" }, + "name": { type: "string" }, + "id": { type: "integer" }, + }, + }, + }, + }, + { + properties: { + "type": { const: "register_control" }, + "data": { + additionalProperties: false, + required: ["agent", "version"], + properties: { + "agent": { type: "string" }, + "version": { type: "string" }, + } + }, + }, + }, + closeSchema, + ], +}); + + +module.exports = { + compile: ajv.compile.bind(ajv), + + message, + close, + + serverHandshake, + clientHandshake, +} diff --git a/master.js b/master.js index 48a0686a..52b9a468 100644 --- a/master.js +++ b/master.js @@ -24,6 +24,7 @@ const crypto = require('crypto'); const base64url = require('base64url'); const moment = require("moment"); const request = require("request"); +const version = require("./package").version; // constants let config = {}; @@ -33,6 +34,9 @@ const generateSSLcert = require("lib/generateSSLcert"); const pluginManager = require("lib/manager/pluginManager"); const database = require("lib/database"); const factorio = require("lib/factorio"); +const schema = require("lib/schema"); +const link = require("lib/link"); +const errors = require("lib/errors"); // homemade express middleware for token auth const authenticate = require("lib/authenticate"); @@ -155,10 +159,22 @@ function registerMoreMetrics(){ // set up database const db = {}; -async function loadDatabase(databaseDirectory) { - let slavesPath = path.resolve(databaseDirectory, "slaves.json"); - console.log(`Loading ${slavesPath}`); - db.slaves = await database.loadJsonAsMap(slavesPath); +/** + * Load Map from JSON file in the database directory. + */ +async function loadMap(databaseDirectory, file) { + let databasePath = path.resolve(databaseDirectory, file); + console.log(`Loading ${databasePath}`); + return await database.loadJsonArrayAsMap(databasePath); +} + +/** + * Save Map to JSON file in the database directory. + */ +async function saveMap(databaseDirectory, file, map) { + let databasePath = path.resolve(config.databaseDirectory, file); + console.log(`Saving ${databasePath}`); + await database.saveMapAsJsonArray(databasePath, map); } @@ -167,14 +183,8 @@ async function shutdown() { console.log('Ctrl-C...'); let exitStartTime = Date.now(); try { - // set insane limit to slave length, if its longer than this we are probably being ddosed or something - if (db.slaves && db.slaves.size < 2000000) { - let file = path.resolve(config.databaseDirectory, "slaves.json"); - console.log(`writing ${file}`); - await database.saveMapAsJson(file, db.slaves); - } else if (db.slaves) { - console.error(`Slave database too large, not saving (${db.slaves.size}`); - } + await saveMap(config.databaseDirectory, "slaves.json", db.slaves); + await saveMap(config.databaseDirectory, "instances.json", db.instances); for(let i in masterPlugins){ let plugin = masterPlugins[i]; @@ -193,135 +203,6 @@ async function shutdown() { } } -/** -POST World ID management. Slaves post here to tell the server they exist -@memberof clusterioMaster -@instance -@alias /api/getID -*/ -app.post("/api/getID", authenticate.middleware, function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - // request.body should be an object - // {rconPort, rconPassword, serverPort, mac, time} - // time us a unix timestamp we can use to check for how long the server has been unresponsive - // we should save that somewhere and give appropriate response - if(req.body){ - if (!db.slaves.has(String(req.body.unique))) { - db.slaves.set(String(req.body.unique), {}); - } - let slave = db.slaves.get(String(req.body.unique)); - for(let k in req.body){ - if(k != "meta" && req.body.hasOwnProperty(k)){ - slave[k] = req.body[k]; - } - } - // console.log("Slave registered: " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP + " with name " + slave.instanceName); - } -}); -/** -POST Allows you to add metadata related to slaves for other tools, like owner data, descriptions or statistics. -@memberof clusterioMaster -@instance -@alias /api/editSlaveMeta -*/ -app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - // request.body should be an object - // {instanceID, pass, meta:{x,y,z}} - - if(req.body && req.body.instanceID && req.body.meta){ - if (db.slaves.has(String(req.body.instanceID))) { - let slave = db.slaves.get(String(req.body.instanceID)); - if(!slave.meta) { - slave.meta = {}; - } - slave.meta = deepmerge(slave.meta, req.body.meta, {clone:true}); - let metaPortion = JSON.stringify(req.body.meta); - if(metaPortion.length > 50) { - metaPortion = metaPortion.substring(0,20) + "..."; - } - // console.log("Updating slave "+slave.instanceName+": " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP, metaPortion); - res.sendStatus(200); - } else { - res.send("ERROR: Invalid instanceID or password") - } - } -}); - -/** -POST Get metadata from a single slave. For whenever you find the data returned by /api/slaves overkill. -@memberof clusterioMaster -@instance -@alias /api/getSlaveMeta -*/ -app.post("/api/getSlaveMeta", function (req, res) { - endpointHitCounter.labels(req.route.path).inc(); - console.log("body", req.body); - if(req.body && req.body.instanceID){ - if (db.slaves.has(String(req.body.instanceID))) { - let slave = db.slaves.get(String(req.body.instanceID)); - console.log("returning meta for ", req.body.instanceID); - if(!slave.meta) { - slave.meta = {}; - } - res.send(JSON.stringify(slave.meta)) - } else { - res.status(404); - res.send('{"status": 404, "info": "Slave not registered"}') - } - } else { - res.status(400); - res.send('{"INVALID_REQUEST":1}'); - } -}); -/** -Prepare the slaveCache for both /api/slaves calls -*/ -let slaveCache = { - timestamp: Date.now(), -}; -function getSlaves() { - if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000) { - let copyOfSlaves = JSON.parse(JSON.stringify(database.mapToObject(db.slaves))); - slaveCache.cache = copyOfSlaves; - slaveCache.timestamp = Date.now(); - } - return slaveCache; -} -/** -GET endpoint for getting information about all our slaves -@memberof clusterioMaster -@instance -@alias /api/slaves -*/ -app.get("/api/slaves", function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let slaveCache = getSlaves(); - for(key in slaveCache.cache) { - slaveCache.cache[key].rconPassword = "hidden"; - } - res.send(slaveCache.cache); -}); -/** -POST endpoint for getting the rconPasswords of all our slaves -@memberof clusterioMaster -@instance -@alias /api/rconPasswords -*/ -app.post("/api/rconPasswords", authenticate.middleware, function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let slaveCache = getSlaves(); - let validKeys = ['rconPort', 'rconPassword', 'publicIP']; - for(key in slaveCache.cache) { - slaveCache.cache[key] = {'publicIP': slaveCache.cache[key].publicIP, 'rconPort': slaveCache.cache[key].rconPort, 'rconPassword': slaveCache.cache[key].rconPassword}; - } - res.send(slaveCache.cache); -}); - // wrap a request in an promise async function downloadPage(url) { return new Promise((resolve, reject) => { @@ -361,49 +242,170 @@ app.get("/api/getFactorioLocale", function(req,res){ } }); -/* Websockets for plugins */ +// socket.io connection for slaves var io = require("socket.io")({}); const ioMetrics = require("socket.io-prometheus"); ioMetrics(io); -/* Websockets for send and recieve combinators */ -var wsSlaves = {}; -class wsSlave { - constructor(instanceID, socket){ - this.instanceID = instanceID; - this.socket = socket; - this.lastBeat = Date.now(); +class BaseConnection extends link.Connection { + constructor(target, socket) { + super(target, socket); + link.attachAllMessages(this); + } - this.socket.on("heartbeat", () => { - prometheusWsUsageCounter.labels('heartbeat', this.instanceID).inc(); - this.lastBeat = Date.now(); - }); + async forwardInstanceRequest(message, request) { + let instance = db.instances.get(message.data.instance_id); + if (!instance) { + throw new errors.RequestError(`Instance with ID ${instance_id} does not exist`); + } - this.socket.on("gameChat", data => { - prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc(); - if(typeof data === "object"){ - data.timestamp = Date.now(); - if(!global.wsChatRecievers) global.wsChatRecievers = []; - global.wsChatRecievers.forEach(socket => { - socket.emit("gameChat", data); - }); + let connection = slaveConnections.get(instance.slaveId); + if (!connection) { + throw new errors.RequestError("Slave containing instance is not connected"); + } + + return await request.forward(connection, message); + } +} + +let controlConnections = new Array(); +class ControlConnection extends BaseConnection { + constructor(registerData, socket) { + super('control', socket) + + this._agent = registerData.agent; + this._version = registerData.version; + + socket.on('disconnect', () => { + let index = controlConnections.indexOf(this); + if (index !== -1) { + controlConnections.splice(index, 1); } }); + } - this.socket.on("alert", alert => { - prometheusWsUsageCounter.labels('alert', this.instanceID).inc(); - if(typeof alert === "object"){ - alert.timestamp = Date.now(); - if(!global.wsAlertRecievers) global.wsAlertRecievers = []; - global.wsAlertRecievers.forEach(socket => { - socket.emit("alert", alert); - }); + async listSlavesRequestHandler(message) { + let list = []; + for (let slave of db.slaves.values()) { + list.push({ + agent: slave.agent, + version: slave.version, + id: slave.id, + name: slave.name, + connected: slaveConnections.has(slave.id), + }); + } + return { list }; + } + + async listInstancesRequestHandler(message) { + let list = []; + for (let instance of db.instances.values()) { + list.push({ + id: instance.id, + name: instance.name, + slave_id: instance.slaveId, + }); + } + return { list }; + } + + // XXX should probably add a hook for slave reuqests? + async createInstanceCommandRequestHandler(message) { + let { slave_id, name } = message.data; + let connection = slaveConnections.get(slave_id); + if (!connection) { + throw new errors.RequestError("Slave is not connected"); + } + + await link.requests.createInstance.send(connection, { + id: Math.random() * 2**31 | 0, // TODO: add id option + options: { + 'name': name, + 'description': config.description, + 'visibility': config.visibility, + 'username': config.username, + 'token': config.token, + 'game_password': config.game_password, + 'verify_user_identity': config.verify_user_identity, + 'allow_commands': config.allow_commands, + 'auto_pause': config.auto_pause, } }); } } +var slaveConnections = new Map(); +class SlaveConnection extends BaseConnection { + constructor(registerData, socket) { + super('slave', socket); + + this._agent = registerData.agent; + this._id = registerData.id; + this._name = registerData.name; + this._version = registerData.version; + + db.slaves.set(this._id, { + agent: this._agent, + id: this._id, + name: this._name, + version: this._version, + }); + + socket.on('message', () => { + prometheusWsUsageCounter.labels('message', "other").inc(); + }); + + socket.on('disconnect', () => { + if (slaveConnections.get(this._id) === this) { + slaveConnections.delete(this._id); + } + + // XXX for backwards compatibility only + if (global.wsChatRecievers) { + global.wsChatRecievers.delete(this._id); + } + }); + + // XXX for backwards compatibility only + socket.on("gameChat", data => { + prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc(); + if(typeof data === "object"){ + data.timestamp = Date.now(); + if(!global.wsChatRecievers) global.wsChatRecievers = new Map(); + for (let slave of global.wsChatRecievers.values()) { + slave.socket.emit("gameChat", data); + } + } + }); + + socket.on("registerChatReciever", data => { + prometheusWsUsageCounter.labels("registerChatReciever", "other").inc(); + if(!global.wsChatRecievers) global.wsChatRecievers = new Map(); + global.wsChatRecievers.set(this._id, this); + }); + } + + async updateInstancesEventHandler(message) { + // Prune any instances previously had by the slave + for (let id of [...db.instances.keys()]) { + if (db.instances.get(id).slaveId === this._id) { + db.instances.delete(id); + } + } + + for (let instance of message.data.instances) { + db.instances.set(instance.id, { + id: instance.id, + name: instance.name, + slaveId: this._id, + }); + } + } +} + +// Authentication middleware for socket.io connections io.use((socket, next) => { let token = socket.handshake.query.token; authenticate.check(token).then((result) => { @@ -416,42 +418,65 @@ io.use((socket, next) => { }); }); +/** + * Returns true if value is a signed 32-bit integer + * + * @param value - value to test. + * @returns {Boolean} + * true if value is an integer between -2**31 and 2**31-1. + */ +function isInteger(value) { + return value | 0 === value; +} + + io.on('connection', function (socket) { - // cleanup dead sockets from disconnected people - let terminatedConnections = 0; - let currentConnections = Object.keys(wsSlaves).length; - [wsSlaves].forEach(list => { - Object.keys(list).forEach(connectionID => { - let connection = list[connectionID]; - if(connection.lastBeat < (Date.now() - 30000)){ - terminatedConnections++; - delete list[connectionID]; + console.log(`SOCKET | new connection from ${socket.handshake.address}`); + + // Start connection handshake + socket.send({ seq: 1, type: 'hello', data: { version }}); + + // Handle socket handshake + socket.once('message', (payload) => { + prometheusWsUsageCounter.labels('message', "other").inc(); + if (!schema.clientHandshake(payload)) { + console.log(`SOCKET | closing ${socket.handshake.address} after receiving invalid handshake`, payload); + socket.send({ type: 'close', data: { + reason: "Invalid handshake", + }}); + socket.disconnect(true); + return; + } + + let { seq, type, data } = payload; + if (type === "register_slave") { + let connection = slaveConnections.get(data.id); + if (connection) { + console.log(`SOCKET | disconecting existing connection for slave ${data.id}`); + connection.close("Registered from another connection"); } - }); - }); - if(terminatedConnections > 0) console.log("SOCKET | There are currently "+currentConnections+" websocket connections, deleting "+terminatedConnections+" on timeout"); - // tell our friend that we are listening - setTimeout(()=>socket.emit('hello', { hello: 'world' }), 5000); + console.log(`SOCKET | registered slave ${data.id} version ${data.version}`); + slaveConnections.set(data.id, new SlaveConnection(data, socket)); - /* Websockets for send and recieve combinators */ - socket.on("registerSlave", function(data) { - prometheusWsUsageCounter.labels('registerSlave', "other").inc(); - if(data && data.instanceID){ - wsSlaves[data.instanceID] = new wsSlave(data.instanceID, socket); - console.log("SOCKET | Created new wsSlave: "+ data.instanceID); + } else if (type === "register_control") { + console.log(`SOCKET | registered control from ${socket.handshake.address}`); + controlConnections.push(new ControlConnection(data, socket)); + + } else if (type === "close") { + console.log(`SOCKET | received close from ${socket.handshake.address}: ${data.reason}`); + socket.disconnect(true); + return; } }); - socket.on("registerChatReciever", function(data){ - prometheusWsUsageCounter.labels("registerChatReciever", "other").inc(); - if(!global.wsChatRecievers) global.wsChatRecievers = []; - global.wsChatRecievers.push(socket); - }); + + // XXX TODO + /* socket.on("registerAlertReciever", function(data){ prometheusWsUsageCounter.labels("registerAlertReciever", "other").inc(); if(!global.wsAlertRecievers) global.wsAlertRecievers = []; global.wsAlertRecievers.push(socket); - }); + });*/ }); // handle plugins on the master @@ -556,7 +581,9 @@ async function startServer() { config.databaseDirectory = args.databaseDirectory || config.databaseDirectory || "./database"; await fs.ensureDir(config.databaseDirectory); - await loadDatabase(config.databaseDirectory); + db.slaves = await loadMap(config.databaseDirectory, "slaves.json"); + db.instances = await loadMap(config.databaseDirectory, "instances.json"); + authenticate.setAuthSecret(config.masterAuthSecret); // Make sure we're actually going to listen on a port diff --git a/package.json b/package.json index 50d271e5..1d1fd4ff 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,6 @@ "express-prometheus-request-metrics": "github:burnash/express-prometheus-request-metrics", "follow-redirects": "^1.5.0", "fs-extra": "^8.1.0", - "getmac": "^1.4.0", "ini": "^1.3.5", "jsonwebtoken": "^8.2.1", "jszip": "^3.2.2", @@ -67,7 +66,8 @@ "socket.io-prometheus": "^0.2.1", "sync-request": "^4.0.1", "tail": "^2.0.2", - "typescript-collections": "^1.3.2" + "typescript-collections": "^1.3.2", + "yargs": "^14.2.0" }, "devDependencies": { "html5-validator": "^1.2.1", diff --git a/sharedPlugins/UPSdisplay/index.js b/sharedPlugins/UPSdisplay/index.js index a373e21f..f15a4ea2 100644 --- a/sharedPlugins/UPSdisplay/index.js +++ b/sharedPlugins/UPSdisplay/index.js @@ -7,6 +7,7 @@ class UPSdisplay { this.config = slaveConfig; this.messageInterface = msgInterface; this.socket = extras.socket; + this.slave = extras.slave; this.historicalTicks = []; setInterval(() => { diff --git a/sharedPlugins/clusterioMod/index.js b/sharedPlugins/clusterioMod/index.js index 846c781d..d9b1d9a9 100644 --- a/sharedPlugins/clusterioMod/index.js +++ b/sharedPlugins/clusterioMod/index.js @@ -21,9 +21,6 @@ module.exports = class remoteCommands { this.messageInterface = messageInterface; this.config = mergedConfig; this.socket = extras.socket; - this.socket.on("hello", () => { - - }); const that = this; const instance = process.argv[3]; const instancedirectory = this.config.instanceDirectory + '/' + instance; diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index da9246fc..6eb03bf3 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -4,9 +4,9 @@ module.exports = class remoteCommands { this.config = mergedConfig; this.socket = extras.socket; this.instances = {}; - - this.socket.on("hello", () => this.socket.emit("registerChatReciever")); - + + this.socket.emit("registerChatReciever"); + this.socket.on("gameChat", async data => { if(data.instanceID && data.instanceID != this.config.unique.toString()){ if(data.data.includes("[CHAT]") diff --git a/sharedPlugins/inventoryImports/interfacingFunctions.js b/sharedPlugins/inventoryImports/interfacingFunctions.js index c8493c67..a7d78b87 100644 --- a/sharedPlugins/inventoryImports/interfacingFunctions.js +++ b/sharedPlugins/inventoryImports/interfacingFunctions.js @@ -140,11 +140,6 @@ function pollInventories(outputFile){ // Warning: This command is not in sync with lua commands.lua return `/silent-command local a='{"players":{'local b=false;for c,d in pairs(game.players)do if d.connected and d.character then if b then a=a..','else b=true end;local e=game.players[c].get_inventory(defines.inventory.character_main).get_contents()local g={}for h=1,game.players[c].force.character_logistic_slot_count do g[h]=game.players[c].character.get_request_slot(h)end;a=a..'"'..c..'":{"inventory":{'local i=false;for j,k in pairs(e)do if i then a=a..','else i=true end;a=a..'"'..j..'":'..k end;a=a..'},"requestSlots":{'i=false;for j,k in pairs(g)do if i then a=a..','else i=true end;a=a..'"'..k['name']..'":'..k['count']end;a=a..'}}'end end;a=a.."}}"game.write_file("${outputFile}",a,true,0)` - /* - needle.post(config.masterURL+'/api/editSlaveMeta', {instanceID: config.unique, password: config.clientPassword, meta: {UPS:UPS}}, function(err, resp) { - // success? - }); - */ } module.exports = { diff --git a/test/client.js b/test/client.js index 9bc5b6d4..1e9bda30 100644 --- a/test/client.js +++ b/test/client.js @@ -6,7 +6,7 @@ const client = require("../client"); describe("Client testing", function() { describe("class Instance", function() { - let instance = new client._Instance("dir", "foo") + let instance = new client._Instance("dir", "factorioDir", { name:"foo" }) it("should give the name on .name", function() { assert.equal(instance.name, "foo"); }) @@ -81,7 +81,7 @@ describe("Client testing", function() { fs.outputFileSync(path.join(testDir, "shared", "mod_a.zip"), "a"); fs.outputFileSync(path.join(testDir, "shared", "mod_b.zip"), "b"); fs.outputFileSync(path.join(testDir, "shared", "mod.dat"), "c"); - let instance = new client._Instance(path.join(testDir, "instance"), "test"); + let instance = new client._Instance(path.join(testDir, "instance"), "factorioDir", { name: "test" }); fs.outputFileSync(instance.path("mods", "mod_i.zip"), "i"); let discardingLogger = { diff --git a/test/integration/server.js b/test/integration/server.js index 65d2c169..9c8de5ee 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -50,9 +50,9 @@ describe("Integration of lib/factorio/server", function() { it("contains the settings used by Clusterio", async function() { let keysUsed = new Set([ - "name", "description", "tags", "max_players", "visibility", "username", "token", - "game_password", "require_user_verification", "allow_commands", "autosave_interval", - "autosave_slots", "afk_autokick_interval", "auto_pause", + "name", "description", "tags", "visibility", "username", "token", + "game_password", "require_user_verification", "allow_commands", + "auto_pause", ]); for (let key of Object.keys(settings)) { diff --git a/test/lib/link/link.js b/test/lib/link/link.js new file mode 100644 index 00000000..44f494f1 --- /dev/null +++ b/test/lib/link/link.js @@ -0,0 +1,337 @@ +const assert = require('assert').strict; +const events = require('events'); + +const link = require("lib/link"); +const errors = require("lib/errors"); +const schema = require("lib/schema"); + + +class MockSocket { + constructor() { + this.sentMessages = []; + this.events = new Map(); + this.handshake = { address: "socket.test" }; + } + + send(message) { + this.sentMessages.push(message); + } + + on(event, fn) { + this.events.set(event, fn); + } + + disconnect() { + this.disconnectCalled = true; + } + + close() { + this.closeCalled = true; + } +}; + +describe("lib/link/link", function() { + describe("class Link", function() { + let testLink = new link.Link('source', 'target', new MockSocket); + testLink.setValidator('simple', schema.compile({ + properties: { + "data": { + type: 'object', + properties: { + "string": { type: 'string' }, + }, + }, + }, + })); + + describe(".setValidator()", function() { + it("should set the validator", function() { + testLink.setValidator('test', () => true); + assert(testLink._validators.has('test'), "validator was not added"); + }); + it("should throw on duplicated validator", function() { + assert.throws( + () => testLink.setValidator('test', () => true), + new Error("test already has a validator") + ); + }); + }); + + describe(".setHandler()", function() { + it("should throw if validator is not passed", function() { + assert.throws( + () => testLink.setHandler('no_validator_test', (message) => {}), + new Error("validator is required") + ); + }); + it("should set the handler", function() { + let handler = (message) => {}; + testLink.setHandler('handler_test', handler, (message) => true); + assert.equal(testLink._handlers.get('handler_test'), handler); + }); + it("should throw on duplicated handler", function() { + assert.throws( + () => testLink.setHandler('handler_test', (message) => {}), + new Error("handler_test already has a handler") + ); + }); + }); + + describe(".processMessage()", function() { + it("should throw on invalid message", function() { + assert.throws( + () => testLink.processMessage({ data: "invalid message" }), + new errors.InvalidMessage("Malformed message") + ); + }); + it("should throw on message without validator", function() { + assert.throws( + () => testLink.processMessage({ seq: 1, type: 'no_validator', data: {} }), + new errors.InvalidMessage("No validator for no_validator") + ); + }); + it("should throw on message failing validation", function() { + assert.throws( + () => testLink.processMessage({ seq: 1, type: 'simple', data: { string: 1 }}), + new errors.InvalidMessage("Validation failed for simple") + ); + }); + it("should throw on unhandled message", function() { + assert.throws( + () => testLink.processMessage({ seq: 1, type: 'simple', data: { string: "a" }}), + new errors.InvalidMessage("Unhandled message simple") + ); + }); + }); + + describe("._processHandler()", function() { + it("should call the handler", function() { + let handled = []; + testLink.setHandler('handled', (message) => handled.push(message), (message) => true) + testLink._processHandler({ seq: 1, type: 'handled', data: { test: "foo" } }); + assert.deepEqual(handled, [{ seq: 1, type: 'handled', data: { test: "foo" } }]); + }); + it("should return true if handled", function() { + let result = testLink._processHandler({ seq: 1, type: 'handled', data: {} }); + assert.equal(result, true); + }); + it("should return false if not handled", function() { + let result = testLink._processHandler({ seq: 1, type: 'not_handled', data: {} }); + assert.equal(result, false); + }); + }); + + describe(".disconnect()", function() { + it("is abstract", function() { + assert.throws( + () => testLink.disconnect(), + new Error("Abstract function") + ); + }); + }); + + describe(".send()", function() { + it("calls send on the socket", function() { + let seq = testLink.send('test', { test: true }); + assert.deepEqual(testLink.socket.sentMessages, [{ seq, type: 'test', data: { test: true }}]); + testLink.socket.sentMessages = []; + }); + }); + + describe("._processWaiters()", function() { + it("should call waiters", async function() { + let waiter = new Promise(resolve => { + testLink._waiters.set('test', [{ resolve, data: {} }]); + }); + assert( + testLink._processWaiters({ type: 'test', seq: 4, data: {} }), + "Waiter was not processed" + ); + assert.deepEqual(await waiter, { type: 'test', seq: 4, data: {} }); + }); + it("should filter on data", async function() { + let waiter = new Promise(resolve => { + testLink._waiters.set('test', [{ resolve, data: { a: 1, b: 2 } }]); + }); + assert( + !testLink._processWaiters({ type: 'test', seq: 4, data: {} }), + "Incorrectly matched empty message" + ); + assert( + !testLink._processWaiters({ type: 'test', seq: 4, data: { a: 1, b: 0 } }), + "Incorrectly matched message with wrong data" + ); + assert( + testLink._processWaiters({ type: 'test', seq: 4, data: { a: 1, b: 2, c: 3 } }), + "Did not match correct message" + ); + assert.deepEqual(await waiter, { type: 'test', seq: 4, data: { a: 1, b: 2, c: 3 } }); + }); + }); + + describe(".waitFor()", function() { + testLink.setValidator('waiter', (message) => true); + it("should throw on missing validator", async function() { + await assert.rejects( + testLink.waitFor('no_validator', {}), + new Error("no validator for no_validator") + ); + }); + it("should wait for a given message", async function() { + let result = testLink.waitFor('waiter', {}); + assert( + testLink._processWaiters({ type: 'waiter', seq: 5, data: {} }), + "Did not proccess waiter" + ); + assert.deepEqual(await result, { type: 'waiter', seq: 5, data: {} }); + }); + it("should handle multiple waiters", async function() { + let result1 = testLink.waitFor('waiter', {}); + let result2 = testLink.waitFor('waiter', {}); + assert( + testLink._processWaiters({ type: 'waiter', seq: 5, data: {} }), + "Did not proccess waiter" + ); + assert.deepEqual(await result1, { type: 'waiter', seq: 5, data: {} }); + assert.deepEqual(await result2, { type: 'waiter', seq: 5, data: {} }); + }); + }); + + describe(".close()", function() { + it("should send close and call disconnect", function() { + let called = false; + testLink.disconnect = () => { called = true; } + testLink.close("test reason"); + assert.deepEqual( + testLink.socket.sentMessages, + [{ seq: testLink._seq - 1, type: 'close', data: { reason: "test reason" } }] + ) + assert(called, ".disconnect() was not called"); + }); + }); + }); + + describe("class Connection", function() { + let testConnection = new link.Connection('target', new MockSocket()); + it("should send ready after constructor", function() { + assert.deepEqual( + testConnection.socket.sentMessages, + [{seq: 2, type: 'ready', data: {}}] + ); + testConnection.socket.sentMessages = []; + }); + it("should disconnect when receiving a close message", function() { + testConnection.socket.disconnectCalled = false; + testConnection.socket.events.get('message')({ + seq: 1, type: 'close', data: { reason: "test close" } + }); + assert(testConnection.socket.disconnectCalled, "Disconnect was not called"); + }); + it("should close when receiving an invalid message", function() { + testConnection.socket.events.get('message')({ invalid: true }); + assert.deepEqual( + testConnection.socket.sentMessages, + [{seq: testConnection._seq - 1, type: 'close', data: { reason: "Invalid message: Malformed message" }}] + ); + testConnection.socket.sentMessages = []; + }); + + describe(".disconnect()", function() { + it("should call disconnect on the socket", function() { + testConnection.socket.disconnectCalled = false; + testConnection.disconnect(); + assert(testConnection.socket.disconnectCalled, "Disconnect was not called"); + }); + }); + }); + + describe("class Client", function() { + let testClient = new link.Client('source', 'url', 'token'); + testClient.socket = new MockSocket(); + describe(".disconnect()", function() { + it("should call close on the socket", function() { + testClient.socket.closeCalled = false; + testClient.disconnect(); + assert(testClient.socket.closeCalled, "Close was not called"); + }); + }); + + describe(".register()", function() { + it("is abstract", function() { + assert.throws( + () => testClient.register(), + new Error("Abstract function") + ); + }); + }); + + describe(".processHandshake()", function() { + it("should close on invalid message", function() { + testClient._processHandshake({ data: "invalid message" }), + assert.deepEqual( + testClient.socket.sentMessages, + [{seq: testClient._seq - 1, type: 'close', data: { + reason: "Invalid handshake" } + }] + ); + testClient.socket.sentMessages = []; + }); + it("should call register on hello", function() { + let called = false; + testClient.register = () => { called = true; }; + testClient._processHandshake({ seq: 1, type: 'hello', data: { version: "test" }}), + assert(called, "register was not called"); + }); + it("should emit ready on ready", async function() { + let result = events.once(testClient._events, 'ready'); + testClient._processHandshake({ seq: 1, type: 'ready', data: {}}), + await result; + }); + it("should emit error on close", async function() { + let result = events.once(testClient._events, 'ready'); + testClient._processHandshake({ seq: 1, type: 'close', data: { reason: "test" }}), + await assert.rejects(result, new Error("server closed during handshake: test")); + }); + }); + + describe("._attachSocketHandlers()", function() { + it("should attach handlers", function() { + testClient._attachSocketHandlers(); + assert(testClient.socket.events.size > 0, "No handlers were attached"); + }); + it("should throw on message received in invalid state", function() { + testClient._state = "new"; + assert.throws( + () => testClient.socket.events.get('message')(), + new Error("Received message in unexpected state new") + ); + }); + it("should call _processHandshake on message in handshake state", function() { + testClient._state = "handshake"; + let called = false; + testClient._processHandshake = () => { called = true; }; + testClient.socket.events.get('message')(); + assert(called, "_processHandshake was not called"); + }); + it("should call processMessage on message in ready state", function() { + testClient._state = "ready"; + let called = false; + testClient.processMessage = () => { called = true; }; + testClient.socket.events.get('message')(); + assert(called, "processMessage was not called"); + delete testClient.processMessage; + }); + it("should close on invalid message in ready state", function() { + testClient._state = "ready"; + testClient.socket.events.get('message')({ data: "invalid message"}); + assert.deepEqual( + testClient.socket.sentMessages, + [{seq: testClient._seq - 1, type: 'close', data: { + reason: "Invalid message: Malformed message" } + }] + ); + testClient.socket.sentMessages = []; + }); + }); + }); +}); diff --git a/test/lib/link/messages.js b/test/lib/link/messages.js new file mode 100644 index 00000000..461c013d --- /dev/null +++ b/test/lib/link/messages.js @@ -0,0 +1,151 @@ +const assert = require('assert').strict; + +const link = require('lib/link'); +const errors = require('lib/errors'); + + +describe("lib/link/messages", function() { + let testSourceLink = new link.Link('source', 'target', null); + let testTargetLink = new link.Link('target', 'source', null); + + let lastTargetSent; + testTargetLink.send = (type, data) => { lastTargetSent = { type, data }; }; + + describe("class Request", function() { + let testRequest = new link.Request({ + type: 'test', + sources: 'source', + targets: 'target', + }); + + describe(".attach()", function() { + it("should attach validator to a source link", function() { + testRequest.attach(testSourceLink); + assert(testSourceLink._validators.has('test_response'), "Validator was not set"); + }); + it("should throw if missing handler on target link", function() { + assert.throws( + () => testRequest.attach(testTargetLink), + new Error("Missing handler for test_request on source-target link") + ); + }); + + let handlerResult; + it("should attach handler to a target link", function() { + testRequest.attach(testTargetLink, async (message) => handlerResult ); + assert(testTargetLink._handlers.has('test_request'), "Handler was not set"); + }); + it("should throw on handler returning object with seq", async function() { + handlerResult = { seq: 3 }; + testTargetLink._handlers.get('test_request')({ seq: 2 }); + let result = await new Promise(resolve => { + testTargetLink.send = (type, data) => resolve({ type, data }); + }); + assert.deepEqual(result, { + type: 'test_response', + data: { + error: "response contains reserved property 'seq'", + seq: 2, + }, + }); + }); + it("should send result of calling the handler", async function() { + handlerResult = { test: "handler" }; + testTargetLink._handlers.get('test_request')({ seq: 2 }); + let result = await new Promise(resolve => { + testTargetLink.send = (type, data) => resolve({ type, data }); + }); + assert.deepEqual(result, { + type: 'test_response', + data: { + test: "handler", + seq: 2, + }, + }); + }); + it("should implicitly send an empty object on empty return", async function() { + handlerResult = undefined; + testTargetLink._handlers.get('test_request')({ seq: 2 }); + let result = await new Promise(resolve => { + testTargetLink.send = (type, data) => resolve({ type, data }); + }); + assert.deepEqual(result, { + type: 'test_response', + data: { + seq: 2, + }, + }); + }); + }); + + describe(".send()", function() { + let request; + testSourceLink.send = (type, data) => { + request = { type, data }; + }; + it("should send request with send and use waitFor to get response", async function() { + testSourceLink.waitFor = (type, condition) => { + return { data: { type, request }}; + }; + assert.deepEqual( + await testRequest.send(testSourceLink, { test: "request" }), + { type: 'test_response', request: { type: 'test_request', data: {test: "request" }}} + ); + }); + it("should throw error response", async function() { + testSourceLink.waitFor = (type, condition) => { + return { data: { error: "test error" }}; + }; + assert.rejects( + testRequest.send(testSourceLink), + new errors.RequestError("test error") + ); + }); + }); + }); + + describe("class InstanceRequest", function() { + }); + + describe("class Event", function() { + let testEvent = new link.Event({ + type: 'test', + sources: 'source', + targets: 'target', + }); + + describe(".attach()", function() { + it("should throw if missing handler on target link", function() { + assert.throws( + () => testEvent.attach(testTargetLink), + new Error("Missing handler for test_event on source-target link") + ); + }); + let called = false; + it("should attach handler and validator to a target link", function() { + testEvent.attach(testTargetLink, async (message) => { called = true; }); + assert(testTargetLink._validators.has('test_event'), "Validator was not set"); + }); + it("should call the attached event handler", function() { + testTargetLink._handlers.get("test_event")(); + assert(called, "Handler was not called"); + }); + }); + + describe(".send()", function() { + it("should send the event over the link", async function() { + let sentData = []; + testSourceLink.send = (type, data) => { sentData.push({ type, data }); }; + + testEvent.send(testSourceLink, { test: "event" }); + assert.deepEqual(sentData, [{ type: "test_event", data: { test: "event" }}]); + }); + }); + }); + + describe("attachAllMessages()", function() { + it("does not throw", function() { + link.attachAllMessages(testSourceLink); + }) + }); +}); diff --git a/test/missing.js b/test/missing.js index edb63b13..318eca58 100644 --- a/test/missing.js +++ b/test/missing.js @@ -1,10 +1,9 @@ // These are libraries that does not have any units test, they are // required here in order to track coverage for them. -require('lib/clusterTools'); -require('lib/getInstances'); require('lib/manager/instanceManager'); require('lib/manager/modManager'); require('lib/manager/pluginManager'); require('lib/npmPostinstall.js'); require('lib/typedef'); +require('../clusterctl.js'); From 554cab4fb04fb051d081aea1a7baf471bb3f6a98 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 20 Nov 2019 22:27:19 +0100 Subject: [PATCH 14/16] Add SIGKILL detection to lib/factorio/server Check if Factorio terminated as a result of SIGKILL and throw an error suggesting the system may be low on memory as a cause. Also print exit code if the unexpected shutdown occured without a signal. --- lib/errors.js | 4 ++++ lib/factorio/server.js | 18 +++++++++++++++++- test/integration/server.js | 30 ++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/lib/errors.js b/lib/errors.js index a213a7a9..9e9676f9 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -12,9 +12,13 @@ class InvalidMessage extends Error { } // Errror class for known errors occuring during startup class StartupError extends Error { } +// Errors outside of our control. +class EnvironmentError extends Error { } + module.exports = { CommandError, RequestError, InvalidMessage, StartupError, + EnvironmentError, } diff --git a/lib/factorio/server.js b/lib/factorio/server.js index b93e9b73..0f45921a 100644 --- a/lib/factorio/server.js +++ b/lib/factorio/server.js @@ -14,6 +14,8 @@ const crypto = require("crypto"); const ini = require("ini"); const rconClient = require("rcon-client"); +const errors = require("lib/errors"); + /** * Determines the version of Factorio the datadir is pointing to by @@ -461,7 +463,21 @@ class FactorioServer extends events.EventEmitter { this._server.on('exit', (code, signal) => { if (this._state !== "stopping") { if (this._unexpected.length === 0) { - this.emit('error', new Error("Factorio server unexpectedly shut down")); + if (signal === 'SIGKILL') { + this.emit('error', new errors.EnvironmentError( + "Factorio server was unexpectedly killed, is the system low on memory?" + )); + + } else { + let msg; + if (code !== null) { + msg = `Factorio server unexpectedly shut down with code ${code}` + } else { + msg = `Factorio server was unexpectedly shut by down by signal ${signal}`; + } + + this.emit('error', new Error(msg)); + } } else if (this._unexpected.length === 1) { this.emit('error', new Error(this._unexpected[0])); diff --git a/test/integration/server.js b/test/integration/server.js index 9c8de5ee..0c135ec8 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -4,6 +4,7 @@ const fs = require("fs-extra"); const path = require("path"); const factorio = require("lib/factorio"); +const errors = require("lib/errors"); describe("Integration of lib/factorio/server", function() { @@ -182,5 +183,34 @@ describe("Integration of lib/factorio/server", function() { await server.stop(); }); }); + + describe(".start() termination detection", function() { + it("should tell if Factorio got killed", async function() { + // This does not work on Windows + if (process.platform == "win32") { + this.skip(); + } + slowTest(this); + log(".start() for kill detection"); + + let startPromise = server.start("test.zip"); + server.once('output', () => server._server.kill('SIGKILL')); + + await assert.rejects( + startPromise, + new errors.EnvironmentError("Factorio server was unexpectedly killed, is the system low on memory?") + ); + }); + + it("should tell if Factorio unexpectedly closed with a code", async function() { + slowTest(this); + log(".start() for unexpected close detection"); + + await assert.rejects( + server.start("does-not-exist.zip"), + new errors.EnvironmentError("Factorio server unexpectedly shut down with code 1") + ); + }); + }); }); }); From f2c30060460c7c51e2330894f08a172c8b100aae Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 20 Nov 2019 22:31:00 +0100 Subject: [PATCH 15/16] Remove obsolete fileOps code Remove sync functions that should not be used as well as of functions that provide functionality that is already implemented by fs-extra. --- lib/fileOps/fileOps.js | 74 ------------------------------------- test/lib/fileOps/fileOps.js | 68 ---------------------------------- 2 files changed, 142 deletions(-) diff --git a/lib/fileOps/fileOps.js b/lib/fileOps/fileOps.js index 1bcbe039..556220bf 100644 --- a/lib/fileOps/fileOps.js +++ b/lib/fileOps/fileOps.js @@ -25,80 +25,6 @@ async function getNewestFile(directory) { return newestFile; } -// get names of all directories in directory -var getDirectoriesSync = function(srcpath) { - return fs.readdirSync(srcpath).filter(function (file) { - return fs.statSync(path.join(srcpath, file)).isDirectory(); - }); -} -// get names of all files in a directory -var getFileNamesSync = function(srcpath) { - return fs.readdirSync(srcpath).filter(function (file) { - return !fs.statSync(path.join(srcpath, file)).isDirectory(); - }); -} -// does what it says on the tin -var deleteFolderRecursiveSync = function(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function (file, index) { - var curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { // recurse - deleteFolderRecursiveSync(curPath); - } else { // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - return true; - } else { - return false; - } -}; -var deleteFolderRecursive = function(path){ - return new Promise((resolve, reject) => { - fs.readdir(path, (err, files) => { - if(err){ - reject(err); - } else { - let thingsToDelete = files.length; - files.forEach(file => { - let curPath = path + "/" + file; - fs.stat(curPath, (err, stat) => { - if(!err){ - if(stat.isDirectory()){ - // recurse - deleteFolderRecursive(curPath).then(() => { - whenFilesDeleted(); - }).catch(e => console.log(e)); - } else { - fs.unlink(curPath, () => { - // console.log("Deleted "+curPath); - whenFilesDeleted(); - }); - } - } else { - resolve(false); - } - }); - }); - whenFilesDeleted(); - function whenFilesDeleted(){ - if(!thingsToDelete--){// var-- instead of --var because we are calling the function once too much and need the delay - fs.rmdir(path, () => { - resolve(); - // console.log("Deleted "+path); - }); - } - } - } - }); - }); -} - module.exports = { getNewestFile, - getDirectoriesSync, - deleteFolderRecursiveSync, - deleteFolderRecursive, - getFileNamesSync, } diff --git a/test/lib/fileOps/fileOps.js b/test/lib/fileOps/fileOps.js index 74aadfee..1ed8608c 100644 --- a/test/lib/fileOps/fileOps.js +++ b/test/lib/fileOps/fileOps.js @@ -16,40 +16,6 @@ describe("fileOps.js", function(){ before(setupTestingEnv); - describe("fileOps.getDirectoriesSync()", function(){ - it("gets an array of directory names from a path", function(){ - let directories = fileOps.getDirectoriesSync(path.join(baseDir, "test")); - - // because there are no arrays for some reason - assert(typeof directories == "object"); - - // they are sorted alphabetically because thats the way it is - assert(directories[0] == "another folder" && directories[1] == "folder"); - }); - it("does not return any filenames", function(){ - let directories = fileOps.getDirectoriesSync(path.join(baseDir, "test")); - - // this is a function to return directories, not files. So yeah. - assert(!directories.includes("file.txt") && !directories.includes("another file.txt")); - }); - it("throws when lacking first argument", function(){ - assert.throws(function(){ - let files = fileOps.getDirectoriesSync(); - }, Error); - }); - }); - describe("fileOps.getFileNamesSync(path)", function(){ - it("gets names of files in directory", function(){ - let files = fileOps.getFileNamesSync(path.join(baseDir, "test")); - assert(files.length > 0); - assert(files.includes("file.txt") && files.includes("another file.txt")); - }); - it("throws when lacking first argument", function(){ - assert.throws(function(){ - let files = fileOps.getFileNamesSync(); - }, Error); - }); - }); describe("fileOps.getNewestFile()", function(){ it("returns a string in a directory with files", async function() { let newest = await fileOps.getNewestFile(path.join(baseDir, "test")); @@ -60,38 +26,4 @@ describe("fileOps.js", function(){ assert.equal(newest, null); }); }); - describe("fileOps.deleteFolderRecursiveSync(path)", function(){ - it("deletes a folder and all files and folders in it", function(){ - let x = fileOps.deleteFolderRecursiveSync(path.join(baseDir, "test")); - assert.throws(function(){ - fs.statSync(path.join(baseDir, "test")); - }, Error); - assert(x); - // give it some extra time because its sometimes very slow - }).timeout(5000); - it("returns false if you give it an invalid path", function(){ - let x = fileOps.deleteFolderRecursiveSync("This isn't a real path"); - assert(!x); - }); - }); - describe("fileOps.deleteFolderRecursive(path)", function(){ - it("deletes a folder and all files and folders in it, just like sync version", async function(){ - await setupTestingEnv(); - assert(fs.statSync(path.join(baseDir, "test"))); - let x = await fileOps.deleteFolderRecursive(path.join(baseDir, "test")); - assert.throws(function(){ - fs.statSync(path.join(baseDir, "test")); - }, Error); - assert(x === undefined, "Promise version of this function does not have a return value upon success"); - return true; - }).timeout(5000); - it("throws like a promise does if you give it an invalid path", async function(){ - let x = await fileOps.deleteFolderRecursive("Look this path is fake!") - .catch(e => { - assert(e); - assert.equal(e.code, "ENOENT"); - return true; - }); - }); - }); }); From 1dea5fddbe775ba0fb48beae56cbe3264165f141 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 22 Nov 2019 21:36:20 +0100 Subject: [PATCH 16/16] Forward Factorio logs to clusterctl Add instance output event that is forwarded by the master server to control connections that are subscribing to them. Add code subscribing to the relevant instance output when dealing with instances in clusterctl. --- client.js | 18 +++++----- clusterctl.js | 81 ++++++++++++++++++++++++++++++++++++++++++-- lib/link/messages.js | 37 ++++++++++++++++++++ master.js | 15 ++++++++ 4 files changed, 139 insertions(+), 12 deletions(-) diff --git a/client.js b/client.js index 91af5f39..3a215c87 100644 --- a/client.js +++ b/client.js @@ -109,11 +109,6 @@ class Instance { async createSave() { console.log("Creating save ....."); - // XXX Move to constructor and send to master - this.server.on('output', function(output) { - console.log("Fact: " + output.message); - }); - await this.server.create("world"); console.log("Clusterio | Successfully created save"); } @@ -188,10 +183,6 @@ class Instance { } await factorio.patch(this.path("saves", latestSave), modules); - this.server.on('output', (output) => { - console.log("Fact: " + output.message); - }); - this.server.on('rcon-ready', () => { console.log("Clusterio | RCON connection established"); // Temporary measure for backwards compatibility @@ -317,6 +308,7 @@ class Slave extends link.Client { // XXX: race condition on multiple simultanious calls let instance = await Instance.create(id, instanceDir, this.config.factorioDir, options); this.instances.set(id, instance); + this.hookInstance(instance); this.updateInstances(); } @@ -376,6 +368,7 @@ class Slave extends link.Client { console.log(`found instance ${instanceConfig.name} in ${instancePath}`); let instance = new Instance(instancePath, this.config.factorioDir, instanceConfig); await instance.init(); + this.hookInstance(instance); // XXX this is the wrong place for this instances.set(instanceConfig.id, instance); } } @@ -383,6 +376,13 @@ class Slave extends link.Client { return instances; } + hookInstance(instance) { + instance.server.on('output', (output) => { + link.events.instanceOutput.send(this, { instance_id: instance.config.id, output }) + }); + } + + updateInstances() { let list = []; for (let instance of this.instances.values()) { diff --git a/clusterctl.js b/clusterctl.js index 0b98b534..9195e774 100644 --- a/clusterctl.js +++ b/clusterctl.js @@ -4,11 +4,71 @@ const fs = require("fs-extra"); const yargs = require("yargs"); const version = require("./package").version; const asTable = require("as-table").configure({delimiter: ' | '}); +const chalk = require("chalk"); const link = require("lib/link"); const errors = require("lib/errors"); +/** + * Format a parsed Factorio output message with colors + * + * Formats a parsed Factorio output from lib/factorio into a readable + * colorized output using terminal escape codes that can be printed. + */ +function formatOutputColored(output) { + let time = ""; + if (output.format === 'seconds') { + time = chalk.dim(output.time.padStart(8)) + ' '; + } else if (output.format === 'date') { + time = chalk.dim(output.time) + ' '; + } + + let info = ""; + if (output.type === 'log') { + let level = output.level; + if (level === "Info") { + level = chalk.bold.blueBright(level); + } else if (output.level === "Warning") { + level = chalk.bold.yellowBright(level); + } else if (output.level === "Error") { + level = chalk.bold.redBright(level); + } + + info = level + ' ' + chalk.dim(output.file) + ': '; + + } else if (output.type === 'action') { + info = '[' + chalk.yellow(output.action) + '] '; + } + + return time + info + output.message; +} + +/** + * Format a parsed Factorio output message + * + * Formats a parsed Factorio output from lib/factorio into a readable + * output string that can be printed + */ +function formatOutput(output) { + let time = ""; + if (output.format === 'seconds') { + time = output.time.padStart(8) + ' '; + } else if (output.format === 'date') { + time = output.time + ' '; + } + + let info = ""; + if (output.type === 'log') { + info = output.level + ' ' + output.file + ': '; + + } else if (output.type === 'action') { + info = '[' + output.action + '] '; + } + + return time + info + output.message; +} + /** * Represens a command that can be runned by clusterctl */ @@ -115,8 +175,10 @@ commands.push(new Command({ }); }], handler: async function(args, control) { + let instanceId = await resolveInstance(control, args.instance); + await link.requests.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] }); let response = await link.requests.createSave.send(control, { - instance_id: await resolveInstance(control, args.instance), + instance_id: instanceId, }); console.log(response); }, @@ -129,8 +191,10 @@ commands.push(new Command({ }); }], handler: async function(args, control) { + let instanceId = await resolveInstance(control, args.instance); + await link.requests.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] }); let response = await link.requests.startInstance.send(control, { - instance_id: await resolveInstance(control, args.instance), + instance_id: instanceId, }); console.log(response); }, @@ -143,8 +207,10 @@ commands.push(new Command({ }); }], handler: async function(args, control) { + let instanceId = await resolveInstance(control, args.instance); + await link.requests.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] }); let response = await link.requests.stopInstance.send(control, { - instance_id: await resolveInstance(control, args.instance), + instance_id: instanceId, }); console.log(response); }, @@ -208,6 +274,15 @@ class Control extends link.Client { version: version, }); } + + async instanceOutputEventHandler(message) { + let { instance_id, output } = message.data; + if (process.platform === "win32") { + console.log(formatOutput(output)); + } else { + console.log(formatOutputColored(output)); + } + } } async function findCredentials() { diff --git a/lib/link/messages.js b/lib/link/messages.js index fd94ef59..d030a6ef 100644 --- a/lib/link/messages.js +++ b/lib/link/messages.js @@ -260,6 +260,18 @@ requests.listInstances = new Request({ }, }); +requests.setInstanceOutputSubscriptions = new Request({ + type: 'set_instance_output_subscriptions', + sources: ['control'], + targets: ['master'], + requestProperties: { + "instance_ids": { + type: "array", + items: { type: "integer" }, + }, + }, +}); + requests.createInstanceCommand = new Request({ type: 'create_instance', sources: ['control'], @@ -397,6 +409,31 @@ events.updateInstances = new Event({ }, }); +events.instanceOutput = new Event({ + type: 'instance_output', + sources: ['slave', 'master'], + targets: ['master', 'control'], + eventProperties: { + "instance_id": { type: "integer" }, + "output": { + type: "object", + additionalProperties: false, + required: ["source", "received", "format", "type", "message"], + properties: { + "source": { type: "string" }, + "received": { type: "number" }, + "format": { type: "string" }, + "time": { type: "string" }, + "level": { type: "string" }, + "file": { type: "string" }, + "type": { type: "string" }, + "action": { type: "string" }, + "message": { type: "string" }, + }, + }, + }, +}); + /** * Attaches all requests and events to the given link diff --git a/master.js b/master.js index 52b9a468..497d8dc2 100644 --- a/master.js +++ b/master.js @@ -283,6 +283,8 @@ class ControlConnection extends BaseConnection { controlConnections.splice(index, 1); } }); + + this.instanceOutputSubscriptions = new Set(); } async listSlavesRequestHandler(message) { @@ -334,6 +336,10 @@ class ControlConnection extends BaseConnection { } }); } + + async setInstanceOutputSubscriptionsRequestHandler(message) { + this.instanceOutputSubscriptions = new Set(message.data.instance_ids); + } } var slaveConnections = new Map(); @@ -403,6 +409,15 @@ class SlaveConnection extends BaseConnection { }); } } + + async instanceOutputEventHandler(message) { + let { instance_id, output } = message.data; + for (let controlConnection of controlConnections) { + if (controlConnection.instanceOutputSubscriptions.has(instance_id)) { + link.events.instanceOutput.send(controlConnection, message.data); + } + } + } } // Authentication middleware for socket.io connections