From 9ce4f064ad5c17984b397d09c1e5011ef2967c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Tue, 24 Jul 2018 22:59:40 +0200 Subject: [PATCH 01/86] Add favicon to master (I think, just a quick-fix) --- static/favicon.ico | Bin 0 -> 3262 bytes static/head.html | 1 + 2 files changed, 1 insertion(+) create mode 100644 static/favicon.ico diff --git a/static/favicon.ico b/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..b70c1b1c8027e7d3e8ee9a6caa805aa009a8a684 GIT binary patch literal 3262 zcmcgvYi|_S8J@l0Z?iYg?9S}W?#|BMcfGq_+iQDwy{s>5urM};)POM<+nAdj6Qp3; zk_sHPfmW*8C0C$=~QRGlV!GARnr}oJHsI zKF>MlJ!hWxm>^X6$8iMyZzFD43BpGZL>xbK5q12e{zsW^?n9}}|5Yj4`H!>)(y2t!A`5l@3uIAW3 ztxI3jrY?>&x+*p80P{reMK@cLcg=JIf=)@2l;09+v00bN)*obcat0|<&+>vAQF|qt* zjIUzi%B*=$x6MT7w> z1iY=S4U`^_$L)5zwiue|#1I_d24_G31RNyNE)XdBED#x-3qpt8VF3uKLQc~(;NT%i z(&=>C?RMK1Lld1CQf7fO!$?yA4iZp-P(E(yD&pvO4AeS0I&drEagmEAm=KTcd%a#P z8B|uQ)gqf&Hl>bEWfr(mG#%h#;fQqNASsv2qk}cjD)+p$?_#Z5EtN{`?Rhv@*owt3 z_ya+#&E^SOZFaDlOeW(NL(^n3+pIQ+!y(0vPB^%NlL`xBJe$pgNPAwc&|{Lb2@Q zvVAj``p^F{xo|Vo{2tYJMws46pZ?HWoU{8GJCkz6X^ZGGi=;&=E{ou_i2+Nxym#%j z{PH`A*_(9VO0cmiF8m>R;yqGmTD%;|=SKG|4i61@g+a3?Qa!nzJNCNK-RZDd5L-1t zluo{#S@{z?wAg=nt$O`Cy;pblT$w;YrTWY9>hHckeC<^KcQ@kuud6gJoCl8E?v7T@ z-HA=V%G93@S9Z0A2kZ5EE?-X1-yT_eKRx|3!kjZYS-sUu5bo92|B*QMZm@rz9j#M? zLfcT3Y6&P(Wl#|JPi?>aLFV`^u603WW{i%2-saaj3QynoS88!HT$n0#H$dLg)04?$ zlVX^yE@e*L4z+%0;acA0;MApC#Z$M1$*X}(hn_U)Tm~I!)VobO6w-*M)~VN#ZgKo4 znG)b~LX()eo}T_yV(zv4Gaq2HiG#lhmG{RE{5G2DgD64H z6@Gkrfi_PY6D6GvFU3V?-pnNX-adV}i>K3+R|Tqv41Ic6+D!Ly6E{tEp3ZHL&ivlx zbU`tKv>${DKem7hWDx4;_xtd{LI#Xlwa;$lJ+99#onc7_?XW&XuhN)lt(6t$J~KFY zqt$0J8mtJL%LVlV6$n9##n);ze9>q^qcDVT5w`#eU@#ivls7?nKfSOV@i+~*`&)LQ z))+Llh%oc1kxZD#ILHB!u8-Rd5>SB!C5C8!O38+8_TIA4yu_H)xkR8Z4 zlopx^RYZue9-yEKIhJ|jnfZE34iW!^%i(j;a8nNkj%)XpKuZJ6)0Ra$jkbnw= zpp_H!I&!FT1V|P*o%>smWX-6zvx99^=Ell#6J5&9zT!zXNJdDU{EsWrX(efp!|nd! z0SAc^e9$V{9|;;c6lrej79fmRk_6P1uU(TJ_OdWlT-+L|8JTHu|ne8s9 zE8O6`b@8AAA!wDS1+qU9G`4^=hoeCNl_)tKb`!}H=GapgUX32vU|KKOxuzpL65e$s zwelBFc8|?V`;tA-6NMZ=z(FG4oC;`_cZ-}~sz}htp}4o$pfZJrv|D52so}ZO(yj2= ziz zTP>|@#wIQi#)!$ysZHKgW9HzCe^1T5>0xp{pOo$8{Kx + \ No newline at end of file From d913d59d72234d16e1b3e599744830423c56211c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Tue, 24 Jul 2018 23:00:42 +0200 Subject: [PATCH 02/86] Handle instance creation failing gracefully upon lacking internet --- client.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index d55d4f50..3b93c1e9 100644 --- a/client.js +++ b/client.js @@ -232,7 +232,7 @@ if (!command || command == "help" || command == "--help") { console.error("Usage: node client.js delete [instance]"); process.exit(1); } else if (typeof process.argv[3] == "string" && fs.existsSync(config.instanceDirectory+"/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") { - fileOps.deleteFolderRecursiveSync(config.instanceDirectory+"/" + process.argv[3]); + fileOps.deleteFolderRecursiveSync(path.resolve(config.instanceDirectory, process.argv[3])); // TODO: Check if this can cause i-craft users to format their server by using wrong paths console.log("Deleted instance " + process.argv[3]); process.exit(0); } else { @@ -355,6 +355,14 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n console.log("Instance created!") process.exit(0); } + if(data.includes("Downloading from auth server failed")){ + console.error("Instance creation failed, unable to establish auth server connection."); + console.error("Deleting broken instance..."); + + fileOps.deleteFolderRecursiveSync(path.resolve(config.instanceDirectory, process.argv[3])); + + process.exit(0); + } }); } else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)){ // Exit if no instance specified (it should be, just a safeguard); From d72a45cc662a136680d25dbcf89ae162ea628b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Tue, 24 Jul 2018 23:01:51 +0200 Subject: [PATCH 03/86] Added autosaveInterval to the config.json.dist --- config.json.dist | 4 +++- master.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config.json.dist b/config.json.dist index ade686b6..ca4c19de 100644 --- a/config.json.dist +++ b/config.json.dist @@ -51,5 +51,7 @@ "msBetweenCommands": 10, "allowRemoteCommandExecution": true, "enableCrossServerShout": true, - "mirrorAllChat": false + "mirrorAllChat": false, + "__comment7": "Autosave interval for items in the master storage in milliseconds. Defaults to one minute when set to 0 or not set." + "autosaveInterval": 600000 } diff --git a/master.js b/master.js index 81cfd915..6f4c3cd8 100644 --- a/master.js +++ b/master.js @@ -426,7 +426,7 @@ app.post("/api/uploadMod", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); let reqStartTime = Date.now(); if (req.files && req.files.file) { - console.log(req.files.file); + // console.log(req.files.file); req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err) { if (err) { res.status(500).send(err); From 958cdfa15a57da4b8746efe4df52cc42777d2426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Tue, 24 Jul 2018 23:14:02 +0200 Subject: [PATCH 04/86] Fixed node client.js manage * mods * --- client.js | 4 +++- lib/manager/modManager.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 3b93c1e9..079e84e5 100644 --- a/client.js +++ b/client.js @@ -18,7 +18,6 @@ const ioClient = require("socket.io-client"); const objectOps = require("./lib/objectOps.js"); const fileOps = require("_app/fileOps"); const stringUtils = require("./lib/stringUtils.js"); -const modManager = require("./lib/manager/modManager.js"); const configManager = require("./lib/manager/configManager.js"); // argument parsing @@ -183,6 +182,9 @@ if (!command || command == "help" || command == "--help") { if(instance){ if(tool == "mods"){ (async function(){try{ + // do require down here to reduce master load time + const modManager = require("./lib/manager/modManager.js")(config); + // allow managing mods if(action == "list"){ console.log(await modManager.listMods(instance)); diff --git a/lib/manager/modManager.js b/lib/manager/modManager.js index cb259be2..2b338d28 100644 --- a/lib/manager/modManager.js +++ b/lib/manager/modManager.js @@ -34,7 +34,7 @@ module.exports = function(config){ @param {string} instance - Name of the instance or "shared" @example - var modManager = require("modManager"); + var modManager = require("modManager")(config); modManager.listMods("testInstance"); // console.logs table of mods installed on instance modManager.listMods("shared"); From 8abeb26dcef9046358a160389be5784267ef2564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Wed, 25 Jul 2018 01:14:43 +0200 Subject: [PATCH 05/86] Commandline tools to install/uninstall plugins from github --- client.js | 22 +++++++++++- lib/manager/pluginManager.js | 67 ++++++++++++++++++++++++++++++++++++ package.json | 5 +-- 3 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 lib/manager/pluginManager.js diff --git a/client.js b/client.js index 079e84e5..97df1cde 100644 --- a/client.js +++ b/client.js @@ -118,6 +118,8 @@ if (!command || command == "help" || command == "--help") { console.error("node client.js delete [instance]"); console.error("To download the latest version of the Clusterio lua mod, do"); console.error("node client.js manage shared mods download clusterio"); + console.error("To download a clusterio plugin, do"); + console.error("node client.js manage shared plugins install https://github.com/Danielv123/playerManager"); console.error("For more management options, do"); console.error("node client.js manage"); process.exit(1); @@ -173,8 +175,10 @@ if (!command || command == "help" || command == "--help") { console.log('node client.js manage '+instance+' '+tool+' ["list", "search", "add", "remove", "update"]'); } else if(tool && tool == "config") { console.log('node client.js manage '+instance+' '+tool+' ["list", "edit"]'); + } else if(tool && tool == "plugins") { + console.log(`node client.js manage ${instance} ${tool} ["list", "add", "remove"]`); } else { - console.log('node client.js manage '+(instance || '[instance, "shared"]') +' '+ (tool || '["mods", "config"]') + ' ...'); + console.log('node client.js manage '+(instance || '[instance, "shared"]') +' '+ (tool || '["mods", "config", "plugins"]') + ' ...'); } } const tool = process.argv[4] || ""; @@ -221,6 +225,22 @@ if (!command || command == "help" || command == "--help") { } else { usage(instance, tool); } + } else if(tool == "plugins"){ + (async function(){try{ + const pluginManager = require("./lib/manager/pluginManager.js")(config); + if(action == "list"){ + console.log(await pluginManager.listPlugins()) + } else if(action == "add" || action == "install" || action == "download"){ + let status = await pluginManager.addPlugin(process.argv[6]); + } else if(action == "remove" || action == "uninstall" || action == "delete"){ + let status = await pluginManager.removePlugin(process.argv[6]); + if(status && status.msg) console.log(status.msg); + } + process.exit(0); + }catch(e){ + console.error(e); + process.exit(1); + }})(); } else { usage(instance); } diff --git a/lib/manager/pluginManager.js b/lib/manager/pluginManager.js new file mode 100644 index 00000000..3eff4035 --- /dev/null +++ b/lib/manager/pluginManager.js @@ -0,0 +1,67 @@ +const fs = require("fs-extra"); +const path = require("path"); +const git = require("simple-git")("./sharedPlugins"); + + +module.exports = function(config){ + if(!config) config = require("./../../config"); + + return { + listPlugins: async function(){ + let plugins = await this.getPlugins(); + plugins.forEach(plug => { + console.log(`${plug.name}`.padEnd(22).substr(0,22)+` - ${plug.description}`.substr(0,80-22)); + }); + }, + getPlugins: async function(){ + let directories = await fs.readdir("./sharedPlugins"); + let returnArray = []; + for(let i in directories){ + let dir = directories[i]; + let pluginContents = await fs.readdir(path.join("./sharedPlugins", dir)); + // console.log(pluginContents) + if(pluginContents.includes("config.js") || pluginContents.includes("config.json")){ + try{ + let pluginConfig = require(path.join("./../../sharedPlugins", dir, "config")); + pluginConfig.pluginPath = path.join("./sharedPlugins", dir); + returnArray.push(pluginConfig); + }catch(e){ + console.log(e) + } + } + } + return returnArray; + }, + addPlugin: function(path){ + return new Promise((resolve, reject) => { + if(path.includes("git")){ // we are decently sure this is a git repo + git.clone(path, data => resolve(data)) + } + }); + }, + removePlugin: async function(name){ + let plugins = await this.getPlugins(); + for(let i in plugins){ + let plug = plugins[i]; + if(plug.name.includes(name)){ + // console.log(`Removing plugin ${plug.name}`); + await fs.remove(plug.pluginPath); + return { + ok:true, + msg:`Deleted plugin ${plug.name}` + } + } + } + return { + ok:false, + msg:`Plugin matching "${name}" not found`, + } + }, + enablePlugin: function(){ + + }, + disablePlugin: function(){ + + } + } +} diff --git a/package.json b/package.json index 82f22118..9e1bc980 100644 --- a/package.json +++ b/package.json @@ -61,12 +61,13 @@ "right-pad": "^1.0.1", "rmdir-sync": "^1.0.1", "sanitizer": "^0.1.3", + "simple-git": "^1.96.0", "single-line-log": "^1.1.2", + "snyk": "^1.88.0", "socket.io": "^2.0.4", "socket.io-client": "^2.0.4", "sync-request": "^4.0.1", - "underscore": "^1.8.3", - "snyk": "^1.88.0" + "underscore": "^1.8.3" }, "devDependencies": { "documentation": "^5.4.0", From 329f721b11ab1a419677fc42bf990156f666d897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Wed, 25 Jul 2018 02:09:00 +0200 Subject: [PATCH 06/86] Fixed invalid json in config.json.dist --- config.json.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.dist b/config.json.dist index ca4c19de..5a180bde 100644 --- a/config.json.dist +++ b/config.json.dist @@ -52,6 +52,6 @@ "allowRemoteCommandExecution": true, "enableCrossServerShout": true, "mirrorAllChat": false, - "__comment7": "Autosave interval for items in the master storage in milliseconds. Defaults to one minute when set to 0 or not set." + "__comment7": "Autosave interval for items in the master storage in milliseconds. Defaults to one minute when set to 0 or not set.", "autosaveInterval": 600000 } From 49124021a895392dbdfc49f3eabf27db3573732d Mon Sep 17 00:00:00 2001 From: Gerkiz <39602942+Gerkiz@users.noreply.github.com> Date: Wed, 25 Jul 2018 21:47:03 +0200 Subject: [PATCH 07/86] Added docker support Docker doesn't really do anything with the '/quit'. However, changing from 'quit' to 'serverprocess.kill("SIGINT");' solved this issue. The node exits with: '[dumb-init] Child exited with status 0. Goodbye.' --- client.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client.js b/client.js index 97df1cde..2d3e8592 100644 --- a/client.js +++ b/client.js @@ -433,8 +433,8 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n fs.copySync(path.join(instancedirectory, "instanceMods"), path.join(instancedirectory, "mods")); process.on('SIGINT', function () { - console.log("Caught interrupt signal, sending /quit"); - messageInterface("/quit"); + console.log("Caught interrupt signal, sending ^C"); + serverprocess.kill("SIGINT"); }); // Spawn factorio server @@ -1082,4 +1082,4 @@ function hashMods(instanceName, callback) { } }); } -} \ No newline at end of file +} From 4fed43bad6de0241da22ece40a34c77f84d5b69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Sat, 28 Jul 2018 13:45:05 +0200 Subject: [PATCH 08/86] [big change] Moved core clusterio Lua mod functionality into its own plugin --- .gitignore | 1 + client.js | 258 +------------- sharedPlugins/clusterioMod/config.js | 11 + sharedPlugins/clusterioMod/index.js | 395 +++++++++++++++++++++ sharedPlugins/clusterioMod/masterPlugin.js | 85 +++++ static/head.html | 2 +- 6 files changed, 494 insertions(+), 258 deletions(-) create mode 100644 sharedPlugins/clusterioMod/config.js create mode 100644 sharedPlugins/clusterioMod/index.js create mode 100644 sharedPlugins/clusterioMod/masterPlugin.js diff --git a/.gitignore b/.gitignore index 8e02f843..e9ddfb74 100644 --- a/.gitignore +++ b/.gitignore @@ -50,5 +50,6 @@ sharedPlugins/* !sharedPlugins/remoteCommands/ !sharedPlugins/researchSync/ !sharedPlugins/globalChat/ +!sharedPlugins/clusterioMod/ \.cache-require-paths\.json diff --git a/client.js b/client.js index 2d3e8592..35097d3a 100644 --- a/client.js +++ b/client.js @@ -229,7 +229,7 @@ if (!command || command == "help" || command == "--help") { (async function(){try{ const pluginManager = require("./lib/manager/pluginManager.js")(config); if(action == "list"){ - console.log(await pluginManager.listPlugins()) + await pluginManager.listPlugins(); } else if(action == "add" || action == "install" || action == "download"){ let status = await pluginManager.addPlugin(process.argv[6]); } else if(action == "remove" || action == "uninstall" || action == "delete"){ @@ -780,262 +780,6 @@ function instanceManagement() { }); } } - - // flow/production statistics ------------------------------------------------------------ - oldFlowStats = false - setInterval(function(){ - fs.readFile(instancedirectory + "/script-output/flows.txt", {encoding: "utf8"}, function(err, data) { - if(!err && data) { - let timestamp = Date.now(); - data = data.split("\n"); - let flowStats = []; - for(let i = 0; i < data.length; i++) { - // try catch to remove any invalid json - try{ - flowStats[flowStats.length] = JSON.parse(data[i]); - } catch (e) { - // console.log(" invalid json: " + i); - // some lines of JSON are invalid but don't worry, we just filter em out - } - } - // fluids - let flowStat1 = flowStats[flowStats.length-1].flows.player.input_counts - // items - let flowStat2 = flowStats[flowStats.length-2].flows.player.input_counts - // merge fluid and item flows - let totalFlows = {}; - for(let key in flowStat1) totalFlows[key] = flowStat1[key]; - for(let key in flowStat2) totalFlows[key] = flowStat2[key]; - if(oldFlowStats && totalFlows && oldTimestamp) { - let payload = objectOps.deepclone(totalFlows); - // change from total reported to per time unit - for(let key in oldFlowStats) { - // get production per minute - payload[key] = Math.floor((payload[key] - oldFlowStats[key])/(timestamp - oldTimestamp)*60000); - if(payload[key] < 0) { - payload[key] = 0; - } - } - for(let key in payload) { - if(payload[key] == '0') { - delete payload[key]; - } - } - console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]); - needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: instanceconfig.unique,data: payload}, needleOptionsWithTokenAuthHeader, function (err, response, body) { - // we did it, keep going - }); - } - oldTimestamp = timestamp; - oldFlowStats = totalFlows; - fs.writeFileSync(instancedirectory + "/script-output/flows.txt", ""); - } - }); - // we don't need to update stats quickly as that could be expensive - }, 60000*5); - - // provide items -------------------------------------------------------------- - // trigger when something happens to output.txt - fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) { - // get array of lines in file - let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n"); - // if you found anything, reset the file - if (items[0]) { - fs.writeFileSync(instancedirectory + "/script-output/output.txt", ""); - } - for (let i = 0; i < items.length; i++) { - if (items[i]) { - let g = items[i].split(" "); - g[0] = g[0].replace("\u0000", ""); - // console.log("exporting " + JSON.stringify(g)); - // send our entity and count to the master for him to keep track of - needle.post(config.masterIP + ":" + config.masterPort + '/api/place', { - name: g[0], - count: g[1], - instanceName: instance, // name of instance - instanceID: instanceconfig.unique, // a hash computed from the randomly generated rcon password - }, needleOptionsWithTokenAuthHeader, function (err, resp, body) { - if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]); - if(config.logItemTransfers){ - if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`); - } - }); - } - } - }); - // request items -------------------------------------------------------------- - setInterval(function () { - // get array of lines in file - let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n"); - // if we actually got anything from the file, proceed and reset file - if (items[0]) { - fs.writeFileSync(instancedirectory + "/script-output/orders.txt", ""); - // prepare a package of all our requested items in a more tranfer friendly format - var preparedPackage = {}; - for (let i = 0; i < items.length; i++) { - (function (i) { - if (items[i]) { - items[i] = items[i].split(" "); - items[i][0] = items[i][0].replace("\u0000", ""); - items[i][0] = items[i][0].replace(",", ""); - if (preparedPackage[items[i][0]]) { - // if we have buffered some already, sum the new items - if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") { - preparedPackage[items[i][0]] = { - "name": items[i][0], - "count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]), - "instanceName":instance, - "instanceID":instanceconfig.unique, - }; - // else just add em in without summing - } else if (typeof Number(items[i][1]) == "number") { - preparedPackage[items[i][0]] = { - "name": items[i][0], - "count": Number(items[i][1]), - "instanceName":instance, - "instanceID":instanceconfig.unique, - }; - } - // this condition will NEVER be triggered but we know how that goes - } else if (typeof Number(items[i][1]) == "number") { - preparedPackage[items[i][0]] = { - "name": items[i][0], - "count": Number(items[i][1]), - "instanceName":instance, - "instanceID":instanceconfig.unique, - }; - } - } - })(i); - } - // request our items, one item at a time - for (let i = 0; i < Object.keys(preparedPackage).length; i++) { - // console.log(preparedPackage[Object.keys(preparedPackage)[i]]); - needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) { - if (response && response.body && typeof response.body == "object") { - // buffer confirmed orders - confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count} - if(config.logItemTransfers){ - console.log(`Imported ${response.body.count} ${response.body.name} from master`); - } - } - }); - } - // if we got some confirmed orders - // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); - //if (!(confirmedOrders.length>0)){return;} - let cmd="local t={"; - for(let i=0;i320) // Factorio max packet size is 508 - { - messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); - cmd="local t={"; - } - } - if (!(cmd==="local t={")){ - messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); - } - confirmedOrders=[]; - } - }, 1000); - // COMBINATOR SIGNALS --------------------------------------------------------- - // get inventory from Master and RCON it to our slave - setInterval(function () { - needle.get(config.masterIP + ":" + config.masterPort + '/api/inventory', function (err, response, body) { - if(err){ - console.error("Unable to get JSON master/api/inventory, master might be unaccessible"); - } else if (response && response.body) { - // Take the inventory we (hopefully) got and turn it into the format LUA accepts - if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver - try { - var inventory = JSON.parse(response.body); - var inventoryFrame = {}; - for (let i = 0; i < inventory.length; i++) { - inventoryFrame[inventory[i].name] = Number(inventory[i].count); - if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){ - inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice - } - } - inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); - // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - let first = true; - let cmd="local s={"; - for (let key in inventoryFrame) - { - cmd+='["'+key+'"]='+inventoryFrame[key]+","; - if(first && cmd.length>300 || !first && cmd.length > 320) // Factorio max packet size is 508 - { - messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); - cmd="local s={"; - first = false; - } - } - if (!(cmd==="local s={")){ - messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); - } - messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')"); - } catch (e){ - console.log(e); - } - } - }); - }, 1550); - // Make sure world has its worldID - setTimeout(function(){ - messageInterface("/silent-command remote.call('clusterio','setWorldID',"+instanceconfig.unique+")") - }, 20000); - /* REMOTE SIGNALLING - * send any signals the slave has been told to send - * Fetch combinator signals from the server - */ - socket.on("processCombinatorSignal", circuitFrameWithMeta => { - if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object" && circuitFrameWithMeta.frame && Array.isArray(circuitFrameWithMeta.frame)){ - messageInterface("/silent-command remote.call('clusterio', 'receiveFrame', '"+JSON.stringify(circuitFrameWithMeta.frame)+"')"); - } - }); - // get outbound frames from file and send to master - // get array of lines in file, each line should correspond to a JSON encoded frame - let signals = fs.readFileSync(instancedirectory + "/script-output/txbuffer.txt", "utf8").split("\n"); - // if we actually got anything from the file, proceed and reset file - let readingTxBufferSoon = false; - let txBufferClearCounter = 0; - fs.watch(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (eventType, filename) => { - if(!readingTxBufferSoon){ // use a 100ms delay to avoid messing with rapid sequential writes from factorio (I think that might be a problem maybe?) - readingTxBufferSoon = true; - setTimeout(()=>{ - txBufferClearCounter++; - fs.readFile(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (err, signals) => { - signals = signals.split("\n"); - if (signals[0]) { - //if(txBufferClearCounter > 500){ - fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", ""); - // txBufferClearCounter = 0; - //} - - // loop through all our frames - for (let i = 0; i < signals.length; i++) { - if (signals[i] && objectOps.isJSON(signals[i])) { - // signals[i] is a JSON array called a "frame" of signals. We timestamp it for storage on master - // then we unpack and RCON in this.frame to the game later. - let framepart = JSON.parse(signals[i]); - let doneframe = { - time: Date.now(), - frame: framepart, // thats our array of objects(single signals); - } - // send to master using socket.io, opened at the top of instanceManagement() - socket.emit("combinatorSignal", doneframe); - } else { - // console.log("Invalid jsony: "+typeof signals[i]) - } - } - } - }); - readingTxBufferSoon = false; - },100); - } - }); } // END OF INSTANCE START --------------------------------------------------------------------- // string, function diff --git a/sharedPlugins/clusterioMod/config.js b/sharedPlugins/clusterioMod/config.js new file mode 100644 index 00000000..01c46abf --- /dev/null +++ b/sharedPlugins/clusterioMod/config.js @@ -0,0 +1,11 @@ +/* + Clusterio plugin to allow for chat between instances. +*/ +module.exports = { + // Name of package. For display somewhere I guess. + name: "clusterioMod", + version: "1.0.0", + binary: "nodePackage", + description: "Server side handling of functionality in the clusterio lua mod", + masterPlugin: "masterPlugin.js", +} diff --git a/sharedPlugins/clusterioMod/index.js b/sharedPlugins/clusterioMod/index.js new file mode 100644 index 00000000..fec5da1a --- /dev/null +++ b/sharedPlugins/clusterioMod/index.js @@ -0,0 +1,395 @@ +const needle = require("needle"); +const fs = require("fs-extra"); + +const objectOps = require("./../../lib/objectOps.js"); +const fileOps = require("_app/fileOps"); +const stringUtils = require("./../../lib/stringUtils.js"); + +const pluginConfig = require("./config"); +const COMPRESS_LUA = false; + +module.exports = class remoteCommands { + constructor(mergedConfig, messageInterface, extras){ + this.messageInterface = messageInterface; + this.config = mergedConfig; + this.socket = extras.socket; + this.socket.on("hello", () => { + + }); + const that = this; + const instance = process.argv[3]; + const instancedirectory = this.config.instanceDirectory + '/' + instance; + + const needleOptionsWithTokenAuthHeader = { + headers: { + 'x-access-token': this.config.masterAuthToken + }, + }; + + (async ()=>{ + let hotpatchInstallStatus = await this.checkHotpatchInstallation(); + this.messageInterface("Hotpach installation status: "+hotpatchInstallStatus); + if(hotpatchInstallStatus){ + // let mainCode = await this.getSafeLua("sharedPlugins/playerManager/lua/playerTracking.lua"); + // if(mainCode) var returnValue = await messageInterface("/silent-command remote.call('hotpatch', 'update', '"+pluginConfig.name+"', '"+pluginConfig.version+"', '"+mainCode+"')"); + } + })().catch(e => console.log(e)); + + // flow/production statistics ------------------------------------------------------------ + var oldFlowStats = false; + var oldTimestamp; + var oldFlowStats; + setInterval(function(){ + fs.readFile(instancedirectory + "/script-output/flows.txt", {encoding: "utf8"}, function(err, data) { + if(!err && data) { + let timestamp = Date.now(); + data = data.split("\n"); + let flowStats = []; + for(let i = 0; i < data.length; i++) { + // try catch to remove any invalid json + try{ + flowStats[flowStats.length] = JSON.parse(data[i]); + } catch (e) { + // console.log(" invalid json: " + i); + // some lines of JSON are invalid but don't worry, we just filter em out + } + } + // fluids + let flowStat1 = flowStats[flowStats.length-1].flows.player.input_counts + // items + let flowStat2 = flowStats[flowStats.length-2].flows.player.input_counts + // merge fluid and item flows + let totalFlows = {}; + for(let key in flowStat1) totalFlows[key] = flowStat1[key]; + for(let key in flowStat2) totalFlows[key] = flowStat2[key]; + if(oldFlowStats && totalFlows && oldTimestamp) { + let payload = objectOps.deepclone(totalFlows); + // change from total reported to per time unit + for(let key in oldFlowStats) { + // get production per minute + payload[key] = Math.floor((payload[key] - oldFlowStats[key])/(timestamp - oldTimestamp)*60000); + if(payload[key] < 0) { + payload[key] = 0; + } + } + for(let key in payload) { + if(payload[key] == '0') { + delete payload[key]; + } + } + console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]); + needle.post(that.config.masterIP + ":" + that.config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: that.config.unique, data: payload}, needleOptionsWithTokenAuthHeader, function (err, response, body) { + // we did it, keep going + }); + } + oldTimestamp = timestamp; + oldFlowStats = totalFlows; + fs.writeFileSync(instancedirectory + "/script-output/flows.txt", ""); + } + }); + // we don't need to update stats quickly as that could be expensive + }, 60000*5); + + // provide items -------------------------------------------------------------- + // trigger when something happens to output.txt + fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) { + // get array of lines in file + let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n"); + // if you found anything, reset the file + if (items[0]) { + fs.writeFileSync(instancedirectory + "/script-output/output.txt", ""); + } + for (let i = 0; i < items.length; i++) { + if (items[i]) { + let g = items[i].split(" "); + g[0] = g[0].replace("\u0000", ""); + // console.log("exporting " + JSON.stringify(g)); + // send our entity and count to the master for him to keep track of + needle.post(that.config.masterIP + ":" + that.config.masterPort + '/api/place', { + name: g[0], + count: g[1], + instanceName: instance, // name of instance + instanceID: that.config.unique, // a hash computed from the randomly generated rcon password + }, needleOptionsWithTokenAuthHeader, function (err, resp, body) { + if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]); + if(that.config.logItemTransfers){ + if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`); + } + }); + } + } + }); + + // request items -------------------------------------------------------------- + setInterval(function () { + // get array of lines in file + let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n"); + // if we actually got anything from the file, proceed and reset file + if (items[0]) { + fs.writeFileSync(instancedirectory + "/script-output/orders.txt", ""); + // prepare a package of all our requested items in a more transfer friendly format + var preparedPackage = {}; + for (let i = 0; i < items.length; i++) { + (function (i) { + if (items[i]) { + items[i] = items[i].split(" "); + items[i][0] = items[i][0].replace("\u0000", ""); + items[i][0] = items[i][0].replace(",", ""); + if (preparedPackage[items[i][0]]) { + // if we have buffered some already, sum the new items + if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") { + preparedPackage[items[i][0]] = { + "name": items[i][0], + "count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]), + "instanceName":instance, + "instanceID":that.config.unique, + }; + // else just add em in without summing + } else if (typeof Number(items[i][1]) == "number") { + preparedPackage[items[i][0]] = { + "name": items[i][0], + "count": Number(items[i][1]), + "instanceName":instance, + "instanceID":that.config.unique, + }; + } + // this condition will NEVER be triggered but we know how that goes + } else if (typeof Number(items[i][1]) == "number") { + preparedPackage[items[i][0]] = { + "name": items[i][0], + "count": Number(items[i][1]), + "instanceName":instance, + "instanceID":that.config.unique, + }; + } + } + })(i); + } + // request our items, one item at a time + for (let i = 0; i < Object.keys(preparedPackage).length; i++) { + // console.log(preparedPackage[Object.keys(preparedPackage)[i]]); + needle.post(that.config.masterIP + ":" + that.config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) { + if (response && response.body && typeof response.body == "object") { + // buffer confirmed orders + confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count} + if(that.config.logItemTransfers){ + console.log(`Imported ${response.body.count} ${response.body.name} from master`); + } + } + }); + } + // if we got some confirmed orders + // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); + //if (!(confirmedOrders.length>0)){return;} + let cmd="local t={"; + for(let i=0;i320) // Factorio max packet size is 508 + { + that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); + cmd="local t={"; + } + } + if (!(cmd==="local t={")){ + that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')"); + } + confirmedOrders=[]; + } + }, 1000); + // COMBINATOR SIGNALS --------------------------------------------------------- + // get inventory from Master and RCON it to our slave + setInterval(() => { + needle.get(that.config.masterIP + ":" + that.config.masterPort + '/api/inventory', function (err, response, body) { + if(err){ + console.error("Unable to get JSON master/api/inventory, master might be unaccessible"); + } else if (response && response.body) { + // Take the inventory we (hopefully) got and turn it into the format LUA accepts + if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver + try { + var inventory = JSON.parse(response.body); + var inventoryFrame = {}; + for (let i = 0; i < inventory.length; i++) { + inventoryFrame[inventory[i].name] = Number(inventory[i].count); + if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){ + inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice + } + } + inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); + // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); + let first = true; + let cmd="local s={"; + for (let key in inventoryFrame) + { + cmd+='["'+key+'"]='+inventoryFrame[key]+","; + if(first && cmd.length>300 || !first && cmd.length > 320) // Factorio max packet size is 508 + { + that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); + cmd="local s={"; + first = false; + } + } + if (!(cmd==="local s={")){ + that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); + } + that.messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')"); + } catch (e){ + console.log(e); + } + } + }); + }, 1550); + // Make sure world has its worldID + setTimeout(function(){ + messageInterface("/silent-command remote.call('clusterio','setWorldID',"+that.config.unique+")") + }, 20000); + /* REMOTE SIGNALLING + * send any signals the slave has been told to send + * Fetch combinator signals from the server + */ + this.socket.on("processCombinatorSignal", circuitFrameWithMeta => { + if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object" && circuitFrameWithMeta.frame && Array.isArray(circuitFrameWithMeta.frame)){ + messageInterface("/silent-command remote.call('clusterio', 'receiveFrame', '"+JSON.stringify(circuitFrameWithMeta.frame)+"')"); + } + }); + // get outbound frames from file and send to master + // get array of lines in file, each line should correspond to a JSON encoded frame + let signals = fs.readFileSync(instancedirectory + "/script-output/txbuffer.txt", "utf8").split("\n"); + // if we actually got anything from the file, proceed and reset file + let readingTxBufferSoon = false; + let txBufferClearCounter = 0; + fs.watch(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (eventType, filename) => { + if(!readingTxBufferSoon){ // use a 100ms delay to avoid messing with rapid sequential writes from factorio (I think that might be a problem maybe?) + readingTxBufferSoon = true; + setTimeout(()=>{ + txBufferClearCounter++; + fs.readFile(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (err, signals) => { + signals = signals.split("\n"); + if (signals[0]) { + //if(txBufferClearCounter > 500){ + fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", ""); + // txBufferClearCounter = 0; + //} + + // loop through all our frames + for (let i = 0; i < signals.length; i++) { + if (signals[i] && objectOps.isJSON(signals[i])) { + // signals[i] is a JSON array called a "frame" of signals. We timestamp it for storage on master + // then we unpack and RCON in this.frame to the game later. + let framepart = JSON.parse(signals[i]); + let doneframe = { + time: Date.now(), + frame: framepart, // thats our array of objects(single signals); + } + // send to master using socket.io, opened at the top of instanceManagement() + socket.emit("combinatorSignal", doneframe); + } else { + // console.log("Invalid jsony: "+typeof signals[i]) + } + } + } + }); + readingTxBufferSoon = false; + },100); + } + }); + } + async getCommand(file){ + this.commandCache = this.commandCache || {}; + if(!this.commandCache[file]){ + try{ + let command = (await fs.readFile(file)).toString(); + this.commandCache[file] = command.replace(/(\r\n\t|\n|\r\t)/gm, " "); // remove newlines + return this.commandCache[file]; + } catch(e){ + console.log("Unable to get command from file!"); + console.log(e) + } + } else if(typeof this.commandCache[file] == "string"){ + return this.commandCache[file]; + } else { + throw new Error("Command not found"); + } + } + async factorioOutput(data){ + + } + getInstanceName(instanceID){ + return new Promise((resolve, reject) => { + let instance = this.instances[instanceID]; + if(!instance){ + needle.get(this.config.masterIP+":"+this.config.masterPort+ '/api/slaves', (err, response) => { + if(err || response.statusCode != 200) { + console.log("Unable to get JSON master/api/slaves, master might be unaccessible"); + } else if (response && response.body) { + if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} + try { + for (let index in response.body) + this.instances[index] = response.body[index].instanceName; + } catch (e){ + console.log(e); + return null; + } + instance = this.instances[instanceID] + if (!instance) instance = instanceID; //somehow the master doesn't know the instance + resolve(instance); + } + }); + } else { + resolve(instance); + } + }); + } + async getSafeLua(filePath){ + return new Promise((resolve, reject) => { + fs.readFile(filePath, "utf8", (err, contents) => { + if(err){ + reject(err); + } else { + // split content into lines + contents = contents.split(/\r?\n/); + + // join those lines after making them safe again + contents = contents.reduce((acc, val) => { + val = val.replace(/\\/g ,'\\\\'); + // remove leading and trailing spaces + val = val.trim(); + // escape single quotes + val = val.replace(/'/g ,'\\\''); + + // remove single line comments + let singleLineCommentPosition = val.indexOf("--"); + let multiLineCommentPosition = val.indexOf("--[["); + + if(multiLineCommentPosition === -1 && singleLineCommentPosition !== -1) { + val = val.substr(0, singleLineCommentPosition); + } + + return acc + val + '\\n'; + }, ""); // need the "" or it will not process the first row, potentially leaving a single line comment in that disables the whole code + + // console.log(contents); + + // this takes about 46 ms to minify train_stop_tracking.lua in my tests on an i3 + if(COMPRESS_LUA) contents = require("luamin").minify(contents); + + resolve(contents); + } + }); + }); + } + async checkHotpatchInstallation(){ + let yn = await this.messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end"); + yn = yn.replace(/(\r\n\t|\n|\r\t)/gm, ""); + if(yn == "true"){ + return true; + } else if(yn == "false"){ + return false; + } + } +} +async function sleep(s){ + return new Promise((resolve, reject) => { + setTimeout(resolve, s*1000); + }); +} diff --git a/sharedPlugins/clusterioMod/masterPlugin.js b/sharedPlugins/clusterioMod/masterPlugin.js new file mode 100644 index 00000000..9123a1c5 --- /dev/null +++ b/sharedPlugins/clusterioMod/masterPlugin.js @@ -0,0 +1,85 @@ +const fs = require("fs-extra"); +const path = require("path"); +const Express = require("express"); +const ejs = require("ejs"); + +const pmSockets = []; + +class masterPlugin { + constructor({config, pluginConfig, pluginPath, socketio, express}){ + this.config = config; + this.pluginConfig = pluginConfig; + this.pluginPath = pluginPath; + this.io = socketio; + this.app = express; + + // load databases + this.items = getDatabaseSync("database/items.json"); + + this.clients = {}; + this.slaves = {}; + + // initialize web API + // require("./js/api-endpoints.js")(this); + + // expose UI elements embedded in the master + // this.ui = require("./js/ui.js").ui; + + this.io.on("connection", socket => { + + }); + } + async broadcastCommand(command){ + let returnValues = []; + for(let instanceID in this.slaves){ + let slave = this.slaves[instanceID]; + slave.emit("runCommand", { + // commandID:Math.random(), + command, + }); + }; + return returnValues; + } + findInArray(key, value, array){ + let indexes = []; + for(let i in array){ + if(array[i][key] && array[i][key] === value) indexes.push(i); + } + return indexes; + } + async onExit(){ + // await saveDatabase("database/items.json", this.items); + return; + } +} +module.exports = masterPlugin; + +function arrayRemoveDuplicates(array){ + let newArray = []; + array.forEach(value => { + if(!newArray.includes(value)) newArray.push(value); + }); + return newArray; +} +function getDatabaseSync(path){ + let db; + try { + db = JSON.parse(fs.readFileSync(path, "utf8")); + } catch(e){ + db = {}; + } + return db; +} +async function saveDatabase(path, database){ + if(!path){ + throw new Error("No path provided!"); + } else if(!database){ + throw new Error("No database provided!"); + } else { + try { + await fs.writeFile(path, JSON.stringify(database, null, 4)); + } catch(e){ + throw new Error("Unable to write to database! "+path); + } + } +} diff --git a/static/head.html b/static/head.html index f2a4b8e0..5b33e429 100644 --- a/static/head.html +++ b/static/head.html @@ -7,4 +7,4 @@ - \ No newline at end of file + \ No newline at end of file From 9461978a6edd48262ae8dc29b0814206dc0a1861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Mon, 30 Jul 2018 13:30:03 +0200 Subject: [PATCH 09/86] Reduced log spam --- client.js | 2 +- master.js | 4 ++-- sharedPlugins/inventoryImports/index.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 35097d3a..3acc337d 100644 --- a/client.js +++ b/client.js @@ -716,7 +716,7 @@ function instanceManagement() { console.log("##### getMac crashed, but we don't really give a shit because we are probably closing down #####"); } payload.mac = mac; - console.log("Registered our presence with master "+config.masterIP+" at " + payload.time); + // console.log("Registered our presence with master "+config.masterIP+" at " + payload.time); needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) { if (err && err.code != "ECONNRESET"){ console.error("We got problems, something went wrong when contacting master"+config.masterIP+" at " + payload.time); diff --git a/master.js b/master.js index 6f4c3cd8..f66663d9 100644 --- a/master.js +++ b/master.js @@ -337,7 +337,7 @@ app.post("/api/getID", authenticate.middleware, function(req,res) { slaves[req.body.unique][k] = req.body[k]; } } - console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName); + // console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName); } httpRequestDurationMilliseconds.labels(req.route.path).observe(Date.now()-reqStartTime); }); @@ -362,7 +362,7 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) { slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true}); let metaPortion = JSON.stringify(req.body.meta); if(metaPortion.length > 50) metaPortion = metaPortion.substring(0,20) + "..."; - console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion); + // console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion); } else { res.send("ERROR: Invalid instanceID or password") } diff --git a/sharedPlugins/inventoryImports/index.js b/sharedPlugins/inventoryImports/index.js index ebec81df..40dd929e 100644 --- a/sharedPlugins/inventoryImports/index.js +++ b/sharedPlugins/inventoryImports/index.js @@ -12,7 +12,7 @@ class inventoryImports { this.messageInterface = messageInterface; messageInterface("/silent-command game.print('InventoryImports enabled')"); setInterval(function(){ - messageInterface("Polling inventories\n") + // messageInterface("Polling inventories\n") messageInterface(functions.pollInventories(pluginConfig.scriptOutputFileSubscription)); }, 10000); } From cac481307e89108a7565701112e4063f250ae3c0 Mon Sep 17 00:00:00 2001 From: Valansch Date: Wed, 1 Aug 2018 13:54:37 +0200 Subject: [PATCH 10/86] Fixed typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec419e9d..2601d6a2 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Master and all slaves: tar -xf factorio.tar.gz npm install cp config.json.dist config.json - node ./lib/npmPostinstall.json + node ./lib/npmPostinstall.js downloads and installs nodejs, pm2, git and clusterio. To specify a version, change "latest" in the link to a version number like 0.14.21. From 8518cd54c680a3f730a91cfe67a7415c3b9efcea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Fri, 10 Aug 2018 15:50:46 +0200 Subject: [PATCH 11/86] Introduce more tools for use by plugins --- master.js | 3 ++ .../clusterTools/checkHotpatchInstallation.js | 9 ++++ node_modules/_app/clusterTools/getCommand.js | 16 ++++++++ .../_app/clusterTools/getInstanceName.js | 40 ++++++++++++++++++ node_modules/_app/clusterTools/getLua.js | 40 ++++++++++++++++++ node_modules/_app/clusterTools/index.js | 41 +++++++++++++++++++ package.json | 2 +- 7 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 node_modules/_app/clusterTools/checkHotpatchInstallation.js create mode 100644 node_modules/_app/clusterTools/getCommand.js create mode 100644 node_modules/_app/clusterTools/getInstanceName.js create mode 100644 node_modules/_app/clusterTools/getLua.js create mode 100644 node_modules/_app/clusterTools/index.js diff --git a/master.js b/master.js index f66663d9..c49195ed 100644 --- a/master.js +++ b/master.js @@ -38,6 +38,7 @@ const jwt = require('jsonwebtoken'); const bcrypt = require('bcryptjs'); const crypto = require('crypto'); const base64url = require('base64url'); +const moment = require("moment"); // constants const config = require(args.config || './config'); @@ -99,6 +100,8 @@ app.use(function(req, res, next){ res.locals.res = res; res.locals.req = req; res.locals.masterPlugins = masterPlugins; + res.locals.slaves = slaves; + res.locals.moment = moment; next(); }); diff --git a/node_modules/_app/clusterTools/checkHotpatchInstallation.js b/node_modules/_app/clusterTools/checkHotpatchInstallation.js new file mode 100644 index 00000000..4ffb825f --- /dev/null +++ b/node_modules/_app/clusterTools/checkHotpatchInstallation.js @@ -0,0 +1,9 @@ +module.exports = async function checkHotpatchInstallation(messageInterface){ + let yn = await messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end"); + yn = yn.replace(/(\r\n\t|\n|\r\t)/gm, ""); + if(yn == "true"){ + return true; + } else if(yn == "false"){ + return false; + } +} diff --git a/node_modules/_app/clusterTools/getCommand.js b/node_modules/_app/clusterTools/getCommand.js new file mode 100644 index 00000000..8d5afc11 --- /dev/null +++ b/node_modules/_app/clusterTools/getCommand.js @@ -0,0 +1,16 @@ +const fs = require("fs-extra"); + +const getLua = require("./getLua.js"); + +let commandCache; +module.exports = async function getCommand(file, COMPRESS = true){ + commandCache = commandCache || {}; + if(!commandCache[file]){ + commandCache[file] = await getLua(file); + return commandCache[file]; + } else if(typeof commandCache[file] == "string"){ + return commandCache[file]; + } else { + throw new Error("Command not found"); + } +} diff --git a/node_modules/_app/clusterTools/getInstanceName.js b/node_modules/_app/clusterTools/getInstanceName.js new file mode 100644 index 00000000..018242a7 --- /dev/null +++ b/node_modules/_app/clusterTools/getInstanceName.js @@ -0,0 +1,40 @@ +const needle = require("needle"); + +module.exports = function(config){ + const instances = []; + + if(config){ + return function getInstanceName(instanceID){ + return new Promise((resolve, reject) => { + let instance = instances[instanceID]; + if(!instance){ + needle.get(config.masterIP+":"+config.masterPort+ '/api/slaves', (err, response) => { + if(err || response.statusCode != 200) { + console.log("Unable to get JSON master/api/slaves, master might be unaccessible"); + } else if (response && response.body) { + if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); reject(new Error())} + try { + for (let index in response.body) + instances[index] = response.body[index].instanceName; + } catch (e){ + console.log(e); + return null; + } + instance = instances[instanceID] + if (!instance) instance = instanceID; //somehow the master doesn't know the instance + resolve(instance); + } + }); + } else { + resolve(instance); + } + }); + } + } else { + return async function thisAintGonnaWork(){ + let err = new Error("getInstanceName from _app/clusterTools needs to be called with {config} to work"); + console.log(err); + throw err; + } + } +} diff --git a/node_modules/_app/clusterTools/getLua.js b/node_modules/_app/clusterTools/getLua.js new file mode 100644 index 00000000..a54c8dac --- /dev/null +++ b/node_modules/_app/clusterTools/getLua.js @@ -0,0 +1,40 @@ +const fs = require("fs-extra"); + +module.exports = function getLua(filePath, COMPRESS_LUA = true){ + return new Promise((resolve, reject) => { + fs.readFile(filePath, "utf8", (err, contents) => { + if(err){ + reject(err); + } else { + // split content into lines + contents = contents.split(/\r?\n/); + + // join those lines after making them safe again + contents = contents.reduce((acc, val) => { + val = val.replace(/\\/g ,'\\\\'); + // remove leading and trailing spaces + val = val.trim(); + // escape single quotes + val = val.replace(/'/g ,'\\\''); + + // remove single line comments + let singleLineCommentPosition = val.indexOf("--"); + let multiLineCommentPosition = val.indexOf("--[["); + + if(multiLineCommentPosition === -1 && singleLineCommentPosition !== -1) { + val = val.substr(0, singleLineCommentPosition); + } + + return acc + val + '\\n'; + }, ""); // need the "" or it will not process the first row, potentially leaving a single line comment in that disables the whole code + + // console.log(contents); + + // this takes about 46 ms to minify train_stop_tracking.lua in my tests on an i3 6100u + if(COMPRESS_LUA) contents = require("luamin").minify(contents); + + resolve(contents); + } + }); + }); +} diff --git a/node_modules/_app/clusterTools/index.js b/node_modules/_app/clusterTools/index.js new file mode 100644 index 00000000..737b2d1d --- /dev/null +++ b/node_modules/_app/clusterTools/index.js @@ -0,0 +1,41 @@ +/** + +Functions frequently needed from different places in the cluster. Ex: + * Loading Lua scripts + * Checking if hotpatch is installed + * Getting an instance's displayname from their ID + +*/ + +module.exports = {config} => { + /** + async function to get Lua from a file, remove comments, minify and return. + + arg1 is the file path + arg2 is whether or not to compress the Lua. Makes debugging harder, but reduces bandwidth when many players are online. Recommended for polling commands. Defaults to true. + Promise resolves with a string of Lua. + */ + getLua: require("./getLua.js"), + /** + async function to check whether hotpatch is installed in a server. + + First argument is the servers async messageInterface for rcon return values + Promise resolves with boolean. + */ + checkHotpatchInstallation: require("./checkHotpatchInstallation.js"), + /** + async function to get an instances displayname from their instanceID + + arg1 is the instanceID + Promise resolves with a string, either the displayName or instanceID if not found. + */ + getInstanceName: require("./getInstanceName.js")(config), + /** + async function same as getLua, but using an in memory cache to lower FS operations. + + arg1 is the file path + arg2 is whether or not to compress, default true + Promise resolves with a string of Lua. + */ + getCommand: require("./getCommand.js"), +} \ No newline at end of file diff --git a/package.json b/package.json index 9e1bc980..f2dc4e69 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "minimist": "^1.2.0", "mkdirp": "^0.5.1", "mkdirp-promise": "^5.0.1", - "moment": "^2.20.1", + "moment": "^2.22.2", "ncp": "^2.0.0", "nedb": "^1.8.0", "needle": "^2.2.1", From da268d57e7ea5faa2950a643568a28cfc5283859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Fri, 10 Aug 2018 15:51:41 +0200 Subject: [PATCH 12/86] Serverside rendering of /slaves for more plugin functionality --- static/nodes.html | 46 +++++++++++++++++++++++++++++++++++++++++++-- static/sidebar.html | 10 ++++++---- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/static/nodes.html b/static/nodes.html index b519c797..031a901c 100644 --- a/static/nodes.html +++ b/static/nodes.html @@ -1,7 +1,7 @@ <%- include("head.html") %> - + @@ -10,7 +10,49 @@

Slaves

- + <% + function findInArray(key, value, array){ + let indexes = []; + for(let i in array){ + if(array[i][key] && array[i][key].includes(value)) indexes.push(i); + } + return indexes; + } + html = "

Current connections:

"; + for(let id in slaves){ + let slave = slaves[id]; + html += ` +
+ +

${slave.instanceName}

+ +

ID: ${id}

+

Last seen: ${moment(Number(slave.time)).format("DD.MM.YY, hh:mm")}

+

Online players: ${slave.playerCount}

+

IP: ${slave.publicIP}:${slave.serverPort}

+

Host: ${slave.mac}

+ ${findInArray("modName", "remoteMap", slave.mods).length ? "Remote map" : ""} + ` + for(let o in masterPlugins){ + let plug = masterPlugins[o]; + if(plug.main + && plug.main.ui + && typeof plug.main.ui.slaveListing == "object" + && Array.isArray(plug.main.ui.slaveListing)){ + let slaveListingEntries = plug.main.ui.slaveListing; + for(let i in slaveListingEntries){ + let button = slaveListingEntries[i]; + if(button.getHtml && typeof button.getHtml == "function"){ + html += button.getHtml({slave}); + } + } + } + } + html += ` +
` + } + %> + <%- html %>
diff --git a/static/sidebar.html b/static/sidebar.html index f51e6443..82be0ab8 100644 --- a/static/sidebar.html +++ b/static/sidebar.html @@ -15,19 +15,21 @@ <%# Very ugly ejs code to hide the Players button if the playerManager plugin isn`t installed %> <% sidebarPluginButtonsHtml = ""; - masterPlugins.forEach(plug => { + for(let o in masterPlugins){ + let plug = masterPlugins[o]; if(plug.main && plug.main.ui && typeof plug.main.ui.sidebar == "object" && Array.isArray(plug.main.ui.sidebar)){ let sidebarEntries = plug.main.ui.sidebar; - sidebarEntries.forEach(button => { + for(let i in sidebarEntries){ + let button = sidebarEntries[i]; if(button.getHtml && typeof button.getHtml == "function"){ sidebarPluginButtonsHtml += button.getHtml(); } - }); + } } - }); + } %> <%- sidebarPluginButtonsHtml %>
` - if(permissions && permissions.cluster.includes("readConfig")){ + if(permissions && permissions.cluster && permissions.cluster.includes("readConfig")){ if(server.config){ html += `