diff --git a/.gitignore b/.gitignore index 9506bc24..9b288416 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,6 @@ Factorio* /instances /sharedMods /database -/sharedPlugins \ No newline at end of file +/sharedPlugins +database/items.db +config.json \ No newline at end of file diff --git a/client.js b/client.js index 1ebe80fd..6570e837 100644 --- a/client.js +++ b/client.js @@ -9,8 +9,12 @@ var ncp = require('ncp').ncp; var Rcon = require('simple-rcon'); var hashFiles = require('hash-files'); var _ = require('underscore'); +var deepmerge = require("deepmerge"); + +// internal libraries +var objectOps = require("./lib/objectOps.js"); +var fileOps = require("./lib/fileOps.js"); -var libomega = require("./libomega.js"); // require config.json var config = require('./config'); var global = {}; @@ -29,21 +33,6 @@ const instancedirectory = './instances/' + instance; const command = process.argv[2]; var instanceInfo = {}; -// Functions -function deleteFolderRecursive(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function (file, index) { - var curPath = path + "/" + file; - if (fs.lstatSync(curPath).isDirectory()) { // recurse - deleteFolderRecursive(curPath); - } else { // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -}; - // simple string hasher String.prototype.hashCode = function () { var hash = 0; @@ -86,27 +75,27 @@ function messageInterface(command, callback) { // handle commandline parameters if (!command || command == "help" || command == "--help") { - console.error("Usage: ") - console.error("node client.js start [instance name]") - console.error("node client.js list") - console.error("node client.js delete [instance]") - console.error("To download the latest version of the Clusterio lua mod, do") - console.error("node client.js download") - process.exit(1) + console.error("Usage: "); + console.error("node client.js start [instance name]"); + console.error("node client.js list"); + console.error("node client.js delete [instance]"); + console.error("To download the latest version of the Clusterio lua mod, do"); + console.error("node client.js download"); + process.exit(1); } else if (command == "list") { - console.dir(getDirectories("./instances/")); - process.exit(1) + console.dir(fileOps.getDirectoriesSync("./instances/")); + process.exit(1); } else if (command == "delete") { if (!process.argv[3]) { console.error("Usage: node client.js delete [instance]"); - process.exit(1) + process.exit(1); } else if (typeof process.argv[3] == "string" && fs.existsSync("./instances/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") { - deleteFolderRecursive("./instances/" + process.argv[3]); - console.log("Deleted instance " + process.argv[3]) - process.exit(1) + fileOps.deleteFolderRecursiveSync("./instances/" + process.argv[3]); + console.log("Deleted instance " + process.argv[3]); + process.exit(1); } else { console.error("Instance not found: " + process.argv[3]); - process.exit(1) + process.exit(1); } } else if (command == "download") { console.log("Downloading mods..."); @@ -119,7 +108,7 @@ if (!command || command == "help" || command == "--help") { var file = fs.createWriteStream("sharedMods/"+name); var request = https.get(url, function(response) { response.pipe(file); - console.log("Downloaded "+name) + console.log("Downloaded "+name); }); } } else if (command == "start" && typeof instance == "string" && instance != "/" && !fs.existsSync(instancedirectory)) { @@ -138,14 +127,14 @@ read-data=__PATH__executable__/../../data\r\n\ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ "); - fs.copySync('sharedMods', instancedirectory + "/mods") + fs.copySync('sharedMods', instancedirectory + "/mods"); let instconf = { "factorioPort": process.env.FACTORIOPORT | Math.floor(Math.random() * 65535), "clientPort": process.env.RCONPORT | Math.floor(Math.random() * 65535), "clientPassword": Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8), } - console.log("Clusterio | Moving shared mods...") - console.log(instconf) + console.log("Clusterio | Moving shared mods..."); + console.log(instconf); // create instance config fs.writeFileSync(instancedirectory + "/config.json", JSON.stringify(instconf, null, 4)); @@ -178,10 +167,10 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ '-c', instancedirectory + '/config.ini', '--create', instancedirectory + '/saves/save.zip', ] - ) - console.log("Instance created!") + ); + console.log("Instance created!"); } else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)) { - // Exit if no instance specified (it should be, just a safeguard) + // Exit if no instance specified (it should be, just a safeguard); if(instancedirectory != "./instances/undefined"){ var instanceconfig = require(instancedirectory + '/config'); instanceconfig.unique = instanceconfig.clientPassword.hashCode(); @@ -192,7 +181,7 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ instanceconfig.rconPort = process.env.RCONPORT; } } else { - process.exit(1) + process.exit(1); } console.log("Deleting .tmp.zip files"); let savefiles = fs.readdirSync(instancedirectory + "/saves/"); @@ -208,12 +197,12 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ // clean old log file to avoid crash // file exists, delete so we don't get in trouble try { - fs.unlinkSync(instancedirectory+'/factorio-current.log') + fs.unlinkSync(instancedirectory+'/factorio-current.log'); } catch (err){ if(err){ console.log(err); } else { - console.log("Clusterio | Deleting old logs...") + console.log("Clusterio | Deleting old logs..."); } } @@ -227,8 +216,9 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ }); // Spawn factorio server - //var serverprocess = child_process.exec(commandline) - getNewestFile(instancedirectory + "/saves/", fs.readdirSync(instancedirectory + "/saves/"),function(latestSave) { + //var serverprocess = child_process.exec(commandline); + fileOps.getNewestFile(instancedirectory + "/saves/", fs.readdirSync(instancedirectory + "/saves/"),function(err, latestSave) { + if(err) throw err; // implicit global serverprocess = child_process.spawn( './' + config.factorioDirectory + '/bin/x64/factorio', [ @@ -241,7 +231,7 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ ], { 'stdio': ['pipe', 'pipe', 'pipe'] } - ) + ); serverprocess.on('close', (code) => { console.log(`child process exited with code ${code}`); @@ -260,11 +250,11 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ if(process.platform == "linux"){ console.log('Factorio: ' + chunk); } - }) + }); serverprocess.stderr.on('data', (chunk) => { console.log('ERR: ' + chunk); - }) + }); // connect to the server with rcon // IP, port, password @@ -305,29 +295,32 @@ write-data=__PATH__executable__/../../../instances/" + instance + "\r\n\ // set some globals confirmedOrders = []; lastSignalCheck = Date.now(); - }) + }); } + +// ensure instancemanagement only runs once _.once(instanceManagement); function instanceManagement() { - console.log("Started instanceManagement();") + console.log("Started instanceManagement();"); // load plugins and execute onLoad event - let pluginDirectories = getDirectories("./sharedPlugins/"); + let pluginDirectories = fileOps.getDirectoriesSync("./sharedPlugins/"); let plugins = []; for(i=0; i { - // log("Stdout: " + data); - messageInterface(data.toString('utf8')); + if(data.toString('utf8')[0] != "/") { + log("Stdout: " + data.toString('utf8')) + } else { + messageInterface(data.toString('utf8')); + } }); plugins[i].stderr.on("data", (data) => { log("STDERR: " + data); - }) + }); plugins[i].on('close', (code) => { log(`child process exited with code ${code}`); }); @@ -376,7 +380,7 @@ function instanceManagement() { // world IDs ------------------------------------------------------------------ hashMods(instance, function(modHashes){ setInterval(getID, 10000); - getID() + getID(); function getID() { messageInterface("/silent-command game.write_file('tempfile.txt', 'connected_players ' .. #game.connected_players .. '\\n', true, 0)", function(err) {setTimeout(function(){ // get array of lines in file @@ -390,7 +394,7 @@ function instanceManagement() { while (data[0]) { let q = data[0].split(" "); // delete array element - data.splice(0,1) + data.splice(0,1); if(q[0] == "connected_players" && Number(q[1]) != NaN) { instanceInfo.playerCount = q[1]; } @@ -419,14 +423,14 @@ function instanceManagement() { console.error(err); } else if (response && response.body) { // In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care. - console.log(response.body) + console.log(response.body); } }); - }) - },1000) - }) + }); + },1000); + }); } - }) + }); // Mod uploading and management ----------------------------------------------- @@ -444,10 +448,10 @@ function instanceManagement() { needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, function (err, response, body) { if(err) throw err // Unable to contact master server! Please check your config.json. if(response && body && body == "found") { - console.log("master has mod") + console.log("master has mod"); } else if (response && body && typeof body == "string") { let mod = response.body; - console.log("Sending mod: " + mod) + console.log("Sending mod: " + mod); // Send mods master says it wants // response.body is a string which is a modName.zip var req = request.post("http://"+config.masterIP + ":" + config.masterPort + '/api/uploadMod', function (err, resp, body) { @@ -491,7 +495,8 @@ function instanceManagement() { for(let key in flowStat1) totalFlows[key] = flowStat1[key]; for(let key in flowStat2) totalFlows[key] = flowStat2[key]; if(oldFlowStats && totalFlows && oldTimestamp) { - let payload = libomega.clone(totalFlows); + // todo: Migrate this to JSON.parse(JSON.stringify()) + let payload = objectOps.clone(totalFlows); // change from total reported to per time unit for(let key in oldFlowStats) { // get production per minute @@ -503,15 +508,15 @@ function instanceManagement() { console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]); needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, slaveID: instanceconfig.unique,data: payload}, function (err, response, body) { // we did it, keep going - }) + }); } oldTimestamp = timestamp; oldFlowStats = totalFlows; fs.writeFileSync(instancedirectory + "/script-output/flows.txt", ""); } - }) + }); // we don't need to update stats quickly as that could be expensive - }, 60000) + }, 60000); // provide items -------------------------------------------------------------- // trigger when something happens to output.txt @@ -520,7 +525,7 @@ function instanceManagement() { let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n"); // if you found anything, reset the file if (items[0]) { - fs.writeFileSync(instancedirectory + "/script-output/output.txt", "") + fs.writeFileSync(instancedirectory + "/script-output/output.txt", ""); } for (let i = 0; i < items.length; i++) { if (items[i]) { @@ -536,7 +541,7 @@ function instanceManagement() { }); } } - }) + }); // request items -------------------------------------------------------------- setInterval(function () { // get array of lines in file @@ -574,7 +579,7 @@ function instanceManagement() { } else if (typeof Number(items[i][1]) == "number") { preparedPackage[items[i][0]] = { "name": items[i][0], - "count": Number(items[i][1]) + "count": Number(items[i][1]), }; } } @@ -582,7 +587,7 @@ function instanceManagement() { } // request our items, one item at a time for (let i = 0; i < Object.keys(preparedPackage).length; i++) { - console.log(preparedPackage[Object.keys(preparedPackage)[i]]) + console.log(preparedPackage[Object.keys(preparedPackage)[i]]); needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], function (err, response, body) { if (response && response.body && typeof response.body == "object") { // buffer confirmed orders @@ -594,20 +599,20 @@ function instanceManagement() { } // if we got some confirmed orders // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); - sadas = JSON.stringify(confirmedOrders) + sadas = JSON.stringify(confirmedOrders); confirmedOrders = []; // send our RCON command with whatever we got messageInterface("/silent-command remote.call('clusterio', 'importMany', '" + sadas + "')"); sadas = null; } - }, 3000) + }, 3000); // COMBINATOR SIGNALS --------------------------------------------------------- // get inventory from Master and RCON it to our slave setInterval(function () { needle.get(config.masterIP + ":" + config.masterPort + '/api/inventory', function (err, response, body) { if (response && response.body) { // Take the inventory we (hopefully) got and turn it into the format LUA accepts - // console.log(response.body) + // console.log(response.body); var inventory = response.body; var inventoryFrame = {}; for (let i = 0; i < inventory.length; i++) { @@ -618,7 +623,7 @@ function instanceManagement() { messageInterface("/silent-command remote.call('clusterio', 'receiveInventory', '" + JSON.stringify(inventoryFrame) + "')"); } }); - }, 1000) + }, 1000); // REMOTE SIGNALLING // send any signals the slave has been told to send setInterval(function () { @@ -652,12 +657,12 @@ function instanceManagement() { if (signals[i]) { // signals[i] is a JSON array called a "frame" of signals. We timestamp it for storage on master // then we unpack and RCON in this.frame to the game later. - let framepart = JSON.parse(signals[i]) + let framepart = JSON.parse(signals[i]); let doneframe = { time: Date.now(), - frame: framepart, // thats our array of objects(single signals) + frame: framepart, // thats our array of objects(single signals); } - // console.log(doneframe) + // console.log(doneframe); needle.post(config.masterIP + ":" + config.masterPort + '/api/setSignal', doneframe, function (err, response, body) { if (response && response.body) { // In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care. @@ -667,16 +672,9 @@ function instanceManagement() { })(i); } } - }, 1000) + }, 1000); } // END OF INSTANCE START --------------------------------------------------------------------- -// get all directories in folder -function getDirectories(srcpath) { - return fs.readdirSync(srcpath).filter(function (file) { - return fs.statSync(path.join(srcpath, file)).isDirectory(); - }); -} - // string, function // returns [{modName:string,hash:string}, ... ] function hashMods(instanceName, callback) { @@ -692,12 +690,12 @@ function hashMods(instanceName, callback) { if(hashedMods.length == /*mods.length*/ + instanceMods.length) { callback(hashedMods); } - //console.log(modname) + //console.log(modname); } let hashedMods = []; var i = 0; /*let mods = fs.readdirSync("./sharedMods/")*/ - let instanceMods = fs.readdirSync("./instances/"+instanceName+"/mods/") + let instanceMods = fs.readdirSync("./instances/"+instanceName+"/mods/"); for(o=0;o newest.mtime.getTime()) { - newest = { file : file, mtime : stats.mtime }; - } - if (checked == files.length) { - callback(newest); - } - }); - })(dir + file); - } - }); - } diff --git a/database/items.db b/database/items.db deleted file mode 100644 index e69de29b..00000000 diff --git a/lib/fileOps.js b/lib/fileOps.js new file mode 100644 index 00000000..65250fcf --- /dev/null +++ b/lib/fileOps.js @@ -0,0 +1,79 @@ +var fs = require("fs"); +var path = require("path"); + +// gets newest file in a directory +// dir is directory +// files is array of filenames +// callback(err, filename string); +var getNewestFile = function(dir, files, callback) { + if (!callback) throw new Error("Missing callback"); + if (!files || (files && files.length === 0)) { + callback(new Error("No files")); + return; + } + // we are appending the file name to the directory, without the + // "/" we would get paths like /folderfile.txt instead of /folder/file.txt + if(dir[dir.length-1] != "/"){ + dir += "/"; + } + var newest = { file: files[0] }; + var checked = 0; + fs.stat(dir + newest.file, function(err, stats) { + if(err) { + callback(err); + } else { + newest.mtime = stats.mtime; + for (var i = 0; i < files.length; i++) { + var file = files[i]; + (function(file) { + fs.stat(file, function(err, stats) { + ++checked; + if (stats.mtime.getTime() > newest.mtime.getTime()) { + newest = { file : file, mtime : stats.mtime }; + } + if (checked == files.length) { + callback(undefined,newest); + } + }); + })(dir + file); + } + } + }); +} + +// get names of all directories in directory +var getDirectoriesSync = function(srcpath) { + return fs.readdirSync(srcpath).filter(function (file) { + return fs.statSync(path.join(srcpath, file)).isDirectory(); + }); +} +// get names of all files in a directory +var getFileNamesSync = function(srcpath) { + return fs.readdirSync(srcpath).filter(function (file) { + return !fs.statSync(path.join(srcpath, file)).isDirectory(); + }); +} +// does what it says on the tin +var deleteFolderRecursiveSync = function(path) { + if (fs.existsSync(path)) { + fs.readdirSync(path).forEach(function (file, index) { + var curPath = path + "/" + file; + if (fs.lstatSync(curPath).isDirectory()) { // recurse + deleteFolderRecursiveSync(curPath); + } else { // delete file + fs.unlinkSync(curPath); + } + }); + fs.rmdirSync(path); + return true; + } else { + return false; + } +}; + +module.exports = { + getNewestFile:getNewestFile, + getDirectoriesSync:getDirectoriesSync, + deleteFolderRecursiveSync:deleteFolderRecursiveSync, + getFileNamesSync:getFileNamesSync, +} diff --git a/lib/fileOps.spec.js b/lib/fileOps.spec.js new file mode 100644 index 00000000..77ffb249 --- /dev/null +++ b/lib/fileOps.spec.js @@ -0,0 +1,106 @@ +var assert = require('assert'); +var fileOps = require('./fileOps'); +var fs = require('fs'); + +// create folders for testing +if (!fs.existsSync("lib/test")){ + fs.mkdirSync("lib/test"); +} +if (!fs.existsSync("lib/test/folder")){ + fs.mkdirSync("lib/test/folder"); +} +if (!fs.existsSync("lib/test/another folder")){ + fs.mkdirSync("lib/test/another folder"); +} +if (!fs.existsSync("lib/test/file.txt")){ + fs.writeFileSync("lib/test/file.txt", "contents"); +} +if (!fs.existsSync("lib/test/another file.txt")){ + fs.writeFileSync("lib/test/another file.txt", "more contents"); +} +// we need this xfile to test a branch in fileOps.getNewestFile()s propagation +if (!fs.existsSync("lib/test/xfile.txt")){ + fs.writeFileSync("lib/test/xfile.txt", "contents"); +} + +describe("fileOps.js", function(){ + describe("fileOps.getDirectoriesSync()", function(){ + it("gets an array of directory names from a path", function(){ + let directories = fileOps.getDirectoriesSync("lib/test"); + + // because there are no arrays for some reason + assert(typeof directories == "object"); + + // they are sorted alphabetically because thats the way it is + assert(directories[0] == "another folder" && directories[1] == "folder"); + }); + it("does not return any filenames", function(){ + let directories = fileOps.getDirectoriesSync("lib/test"); + + // this is a function to return directories, not files. So yeah. + assert(!directories.includes("file.txt") && !directories.includes("another file.txt")); + }); + it("throws when lacking first argument", function(){ + assert.throws(function(){ + let files = fileOps.getDirectoriesSync(); + }, Error); + }); + }); + describe("fileOps.getFileNamesSync(path)", function(){ + it("gets names of files in directory", function(){ + let files = fileOps.getFileNamesSync("lib/test"); + assert(files.length > 0); + assert(files.includes("file.txt") && files.includes("another file.txt")); + }); + it("throws when lacking first argument", function(){ + assert.throws(function(){ + let files = fileOps.getFileNamesSync(); + }, Error); + }); + }); + describe("fileOps.getNewestFile()", function(){ + it("gets the newest file in a directory async", function(done){ + fileOps.getNewestFile("lib/test", fileOps.getFileNamesSync("lib/test"), function(err, newest){ + assert(newest && typeof newest.file == "string"); + done(); + }); + }); + it("throws if callback is missing", function(){ + assert.throws(function(){ + fileOps.getNewestFile("lib/test", fileOps.getFileNamesSync("lib/test")); + }, Error); + }); + it("executes callback(err) if no files are provided", function(done){ + fileOps.getNewestFile("lib/test", [], function(err, newest){ + assert(err && typeof err == "object"); + done(); + }); + }); + it("executes callback(err) if launched without a directory", function(done){ + fileOps.getNewestFile("", fileOps.getFileNamesSync("lib/test"), function(err, newest){ + assert(err); + done(); + }); + }); + it("executes callback(err) when you do stupid shit", function(done){ + fileOps.getNewestFile({thisWontWork: "but not crash either"}, {lolz:"yeah..."}, function(err, newest){ + assert(err); + // ENOENT: no such file or directory, stat 'C:\**\factorioClusterio\lib\test\undefined' + done(); + }); + }); + }); + describe("fileOps.deleteFolderRecursiveSync(path)", function(){ + it("deletes a folder and all files and folders in it", function(){ + let x = fileOps.deleteFolderRecursiveSync("lib/test"); + assert.throws(function(){ + fs.statSync("lib/test"); + }, Error); + assert(x); + }); + it("returns false if you give it an invalid path", function(){ + let x = fileOps.deleteFolderRecursiveSync("This isn't a real path"); + assert(!x); + }); + }); +}); diff --git a/libomega.js b/lib/objectOps.js similarity index 86% rename from libomega.js rename to lib/objectOps.js index e7ef0a36..3bf9e988 100644 --- a/libomega.js +++ b/lib/objectOps.js @@ -1,4 +1,4 @@ -function copyChildren(d){ +var copyChildren = function(d){ var copied = clone(d); copy = []; if(copied.children){ @@ -38,6 +38,11 @@ var clone = function(obj) { return copy; } +var deepclone = function(obj){ + return JSON.parse(JSON.stringify(obj)); +} + module.exports = { clone:clone, + deepclone:deepclone, }; \ No newline at end of file diff --git a/lib/objectOps.spec.js b/lib/objectOps.spec.js new file mode 100644 index 00000000..2e16ee64 --- /dev/null +++ b/lib/objectOps.spec.js @@ -0,0 +1,41 @@ +var assert = require('assert'); +var objectOps = require('./objectOps'); + +describe("objectOps.js", function(){ + describe("objectOps.clone()", function(){ + it("clones javascript objects", function(){ + var object1 = {hello:"world", cat:{legs:4, name:"Kitty", colors:["brown", "yellow", "purple"]}}; + var object2 = {}; + object2 = objectOps.clone(object1); + + assert.equal(object2.cat.name, object1.cat.name) + assert.equal(object1.hello, object2.hello) + }); + it("does not deep clone objects", function(){ + var obj1 = {hello:"world", cat:{legs:4, name:"Kitty", colors:["brown", "yellow", "purple"]}}; + var obj2 = {}; + obj2 = objectOps.clone(obj1); + + obj1.cat.colors = "black"; + assert.equal(obj1.cat.colors, "black"); + assert.notEqual(obj2.cat.colors[1], "yellow"); + }); + }); + describe("objectOps.deepclone()", function(){ + it("deep clones javascript objects", function(){ + var obj1 = {hello:"world", cat:{legs:4, name:"Kitty", colors:["brown", "yellow", "purple"]}}; + var obj2 = {}; + obj2 = objectOps.deepclone(obj1); + + obj1.cat.colors = "black"; + assert.equal(obj1.cat.colors, "black"); + assert.equal(obj2.cat.colors[1], "yellow"); + }); + it("throws on non JSON parameters", function(){ + x = function(){}; + assert.throws(function(){ + y = objectOps.deepclone(x) + }); + }); + }); +}); \ No newline at end of file diff --git a/master.js b/master.js index d6713f59..0867dce5 100644 --- a/master.js +++ b/master.js @@ -1,3 +1,6 @@ +// configgy stuff +debug = false; + // constants const masterModFolder = "./database/masterMods/" @@ -5,9 +8,10 @@ const masterModFolder = "./database/masterMods/" const mkdirp = require("mkdirp"); mkdirp.sync("./database"); mkdirp.sync(masterModFolder); -const path = require("path") -const fs = require("fs") -var nedb = require("nedb") +const deepmerge = require("deepmerge"); +const path = require("path"); +const fs = require("fs"); +var nedb = require("nedb"); // require config.json var config = require('./config'); @@ -47,7 +51,7 @@ db.signals = new Datastore({ filename: 'database/signals.db', autoload: true, in db.signals.ensureIndex({ fieldName: 'time', expireAfterSeconds: 3600 }, function (err) {}); // production chart database -db.flows = new Datastore({ filename: "database/flows.db", autoload: true}) +db.flows = new Datastore({ filename: "database/flows.db", autoload: true}); db.flows.ensureIndex({ fieldName: "slaveID", expireAfterSeconds: 2592000}); // expire after 30 days // db.slaves = new Datastore({ filename: 'database/slaves.db', autoload: true, inMemoryOnly: false}); @@ -64,9 +68,34 @@ db.items.additem = function(object) { db.items.insert(object); console.log('Item created!'); } - }) + }); } -var slaves = {} + +// store slaves in a .json full of JSON data +process.on('SIGINT', function () { + console.log('Ctrl-C...'); + // set insane limit to slave length, if its longer than this we are probably being ddosed or something + if(slaves && Object.keys(slaves).length < 50000){ + fs.writeFileSync("database/slaves.json", JSON.stringify(slaves)); + console.log("saving to slaves.json"); + } else if(slaves) { + console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")"); + } + process.exit(2); +}); +var slaves = {}; +(function(){ + let x; + try{ + x = fs.statSync("database/slaves.json"); + } catch (e){ + + } + if(x){ + console.log("loading slaves from database/slaves.json"); + slaves = JSON.parse(fs.readFileSync("database/slaves.json")); + } +})() // world ID management // slaves post here to tell the server they exist app.post("/api/getID", function(req,res) { @@ -75,14 +104,42 @@ app.post("/api/getID", function(req,res) { // time us a unix timestamp we can use to check for how long the server has been unresponsive // we should save that somewhere and give appropriate response if(req.body){ - slaves[req.body.unique] = req.body; - console.log("Slave: " + req.body.mac + " : " + req.body.serverPort+" at " + req.body.publicIP); + if(debug){ + console.log(req.body) + } + if(!slaves[req.body.unique]){ + slaves[req.body.unique] = {}; + } + for(k in req.body){ + if(k != "meta" && req.body.hasOwnProperty(k)){ + slaves[req.body.unique][k] = req.body[k]; + } + } + console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName); + } +}); + +app.post("/api/editSlaveMeta", function(req,res) { + // request.body should be an object + // {slaveID, pass, meta:{x,y,z}} + + if(req.body && req.body.slaveID && req.body.password && req.body.meta){ + // check for editing permissions + if(slaves[req.body.slaveID] && slaves[req.body.slaveID].rconPassword == req.body.password){ + if(!slaves[req.body.slaveID].meta){ + slaves[req.body.slaveID].meta = {}; + slaves[req.body.slaveID].meta = deepmerge(slaves[req.body.slaveID].meta, req.body.meta, {clone:true}) + } + console.log("Updating slave: " + slaves[req.body.slaveID].mac + " : " + slaves[req.body.slaveID].serverPort+" at " + slaves[req.body.slaveID].publicIP); + } else { + res.send("ERROR: Invalid slaveID or password") + } } }); // mod management // should handle uploading and checking if mods are uploaded app.post("/api/checkMod", function(req,res) { - let files = fs.readdirSync(masterModFolder) + let files = fs.readdirSync(masterModFolder); let found = false; files.forEach(file => { if(file == req.body.modName) { @@ -91,24 +148,24 @@ app.post("/api/checkMod", function(req,res) { }); if(!found) { // we don't have mod, plz send - res.send(req.body.modName) + res.send(req.body.modName); } else { - res.send("found") + res.send("found"); } - res.end() + res.end(); }); app.post("/api/uploadMod", function(req,res) { if (!req.files) { res.send('No files were uploaded.'); return; } else { - console.log(req.files.file) + console.log(req.files.file); req.files.file.mv('./database/masterMods/'+req.files.file.name, function(err) { if (err) { res.status(500).send(err); } else { res.send('File uploaded!'); - console.log("Uploaded mod: " + req.files.file.name) + console.log("Uploaded mod: " + req.files.file.name); } }); } @@ -117,12 +174,12 @@ app.post("/api/uploadMod", function(req,res) { app.get("/api/slaves", function(req, res) { res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let copyOfSlaves = slaves + let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); // filter out the rcon password because thats kindof not a safe thing to share - for(key in copyOfSlaves) { + for(key in copyOfSlaves){ copyOfSlaves[key].rconPassword = "hidden"; } - res.send(copyOfSlaves) + res.send(copyOfSlaves); }); // endpoint to send items to @@ -131,7 +188,7 @@ app.post("/api/place", function(req, res) { res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); console.log("added: " + req.body.name + " " + req.body.count); // save items we get - db.items.additem(req.body) + db.items.additem(req.body); // Attempt confirming res.end("success"); }); @@ -172,14 +229,14 @@ app.post("/api/remove", function(req, res) { console.log('failure ' + object.name); } } - }) + }); }); // circuit stuff app.post("/api/setSignal", function(req,res) { if(typeof req.body == "object" && req.body.time){ db.signals.insert(req.body); - // console.log("signal set") + // console.log("signal set"); } }); @@ -190,7 +247,7 @@ app.post("/api/readSignal", function(req,res) { db.signals.find({time:{$gte: req.body.since}}, function (err, docs) { // $gte means greater than or equal to, meaning we only get entries newer than the timestamp res.send(docs); - // console.log(docs) + // console.log(docs); }); }); @@ -215,18 +272,18 @@ app.post("/api/logStats", function(req,res) { timestamp: req.body.timestamp, data: req.body.data, }); - console.log("inserted: " + req.body.slaveID + " | " + req.body.timestamp) + console.log("inserted: " + req.body.slaveID + " | " + req.body.timestamp); } else { console.log("error invalid timestamp " + req.body.timestamp); - res.send(failure) + res.send(failure); } } else { - res.send("failure") + res.send("failure"); } -}) +}); // {slaveID: string, fromTime: Date, toTime, Date} app.post("/api/getStats", function(req,res) { - console.log(req.body) + console.log(req.body); if(typeof req.body == "object" && req.body.slaveID) { // if not specified, get stats for last 24 hours if(!req.body.fromTime) { @@ -235,18 +292,18 @@ app.post("/api/getStats", function(req,res) { if(!req.body.toTime) { req.body.toTime = Date.now(); } - console.log("Looking... " + JSON.stringify(req.body)) + console.log("Looking... " + JSON.stringify(req.body)); db.flows.find({ slaveID: req.body.slaveID, }, function(err, docs) { let entries = docs.filter(function (el) { return el.timestamp <= req.body.toTime && el.timestamp >= req.body.fromTime; }); - // console.log(entries) - res.send(entries) - }) + // console.log(entries); + res.send(entries); + }); } -}) +}); // endpoint for getting the chartjs library app.get("/chart.js", function(req, res) { diff --git a/package.json b/package.json index 46f5a301..3dfc8462 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,13 @@ { - "name": "factorio-clusterio", + "name": "clusterio", "version": "1.0.0", - "description": "Factorio cluster???", + "description": "Interserver communication for factorio", "main": "master.js", "scripts": { - "test": "nodemon master.js", - "start": "echo Run node client.js start [instance] to start slaves && node master.js" + "test": "npm run test-unit", + "test-unit": "mocha --recursive \"./{,!(node_modules)/**/}*.spec.js\" || echo .", + "start": "echo Run node client.js start [instance] to start slaves && node master.js", + "cover": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- lib/*.js -R spec" }, "keywords": [ "factorio" @@ -16,6 +18,7 @@ "body-parser": "^1.15.2", "canvasjs": "^1.8.1", "child_process": "^1.0.2", + "deepmerge": "^1.3.2", "ejs": "^2.5.6", "express": "^4.14.0", "express-fileupload": "0.0.6", @@ -31,8 +34,15 @@ "needle": "^1.1.0", "portfinder": "^1.0.9", "request": "^2.79.0", + "semver": "^5.3.0", "simple-rcon": "^0.3.1", "sync-request": "^4.0.1", "underscore": "^1.8.3" + }, + "devDependencies": { + "chai": "^3.5.0", + "istanbul": "^0.4.5", + "mocha": "^3.4.1", + "mock-fs": "^4.3.0" } } diff --git a/routes.js b/routes.js index 45f05954..efd330b0 100644 --- a/routes.js +++ b/routes.js @@ -8,4 +8,7 @@ module.exports = function(app) { app.get("/settings", function(req, res) { res.render("settings"); }); + app.get("/nodeDetails", function(req, res) { + res.render("nodeDetails"); + }); }; \ No newline at end of file diff --git a/sharedPlugins/UPSdisplay/config.js b/sharedPlugins/UPSdisplay/config.js new file mode 100644 index 00000000..8b2e6d16 --- /dev/null +++ b/sharedPlugins/UPSdisplay/config.js @@ -0,0 +1,18 @@ +/* + Example config file for example Clusterio plugin + Binary will be spawned using child_process.spawn and get input from stdin and text from stdout will be ran as + commands in the factorio server. +*/ + +module.exports = { + // Name of package. For display somewhere I guess. + name: "UPSdisplay", + version: "1.0.0", + // Binary entrypoint for plugin. Don't let it crash. Stdout is sent to game as server chat (run /c commands from here for interface) + // Make sure its cross platform somehow. + binary: "node", + arguments: ["index.js"], + description: "Internal clusterio plugin to get the worlds UPS and display it on master", + // We'll send everything in this file to your stdin. Beware. + scriptOutputFileSubscription: "UPSdisplay.txt", +} \ No newline at end of file diff --git a/sharedPlugins/UPSdisplay/index.js b/sharedPlugins/UPSdisplay/index.js new file mode 100644 index 00000000..05dfab0f --- /dev/null +++ b/sharedPlugins/UPSdisplay/index.js @@ -0,0 +1,38 @@ +const needle = require("needle"); + +console.log("/silent-command game.print('UPSdisplay enabled')\n"); + +setInterval(function(){ + console.log("/silent-command game.write_file('UPSdisplay.txt', game.tick, true, 0)\n"); +},1000); +var config = {}; +process.stdin.setEncoding('utf8'); +let historicalTicks = []; +process.stdin.on('readable', () => { + var chunk = process.stdin.read(); + if (chunk !== null && !isNaN(chunk) && !chunk.includes("{")){ + // console.log("Tick: " + chunk); + historicalTicks[historicalTicks.length] = {tick:Number(chunk), timestamp:Date.now()}; + if(historicalTicks.length > 30){ + historicalTicks.shift(); // delete last element in array (position 0); + } + } else { + config = JSON.parse(chunk); + } +}); +process.stdin.on('end', () => { + process.stdout.write('end'); +}); + +// Post average UPS +setInterval(function(){ + if(historicalTicks && historicalTicks[0]){ + let timePeriod = historicalTicks[historicalTicks.length-1].timestamp - historicalTicks[0].timestamp; + let ticksInPeriod = historicalTicks[historicalTicks.length-1].tick - historicalTicks[0].tick; + let UPS = Math.round(ticksInPeriod / (timePeriod/1000)); + // console.log("UPS: " + UPS); + needle.post(config.masterIP+':'+config.masterPort+'/api/editSlaveMeta', {slaveID: config.unique, password: config.clientPassword, meta: {UPS:UPS}}, function(err, resp) { + // success? + }); + } +}, 10000); \ No newline at end of file diff --git a/static/head.html b/static/head.html index 7ce98e7d..52dfa7b2 100644 --- a/static/head.html +++ b/static/head.html @@ -1,4 +1,6 @@ Clusterio + + \ No newline at end of file diff --git a/static/index.ejs b/static/index.ejs index f8f0ebb9..03a05b22 100644 --- a/static/index.ejs +++ b/static/index.ejs @@ -9,6 +9,7 @@

Storage

+
diff --git a/static/nodeDetails.css b/static/nodeDetails.css new file mode 100644 index 00000000..1983b1db --- /dev/null +++ b/static/nodeDetails.css @@ -0,0 +1,54 @@ +#header .subbar h6 { + display:inline; + font-weight:normal; + color: #BBB +} +#header { + background-color: #222; + color: #DDD; + font-family: 'Roboto', sans-serif; + padding-left:16px; +} +#displayBody { + margin-left: 16px; +} +#hero { + background-color: #444; + height:400px; +} + +#header h1 { + margin-left: 10px; + display:inline-block; +} +#header .subbar h6 { + margin-right: 16px; +} +#onlineIndicator { + background-color:red; + height:20px; + width:20px; + display:inline-block; + border-radius:16px; +} +.productionGraph { + background-color:black; +} +#leftHeroContainer { + float:left; +} +ul { + width:300px; + list-style:none; + max-height:100px; + margin:0; + overflow:auto; + padding:0; + text-indent:10px; +} +li{ + line-height:25px; +} +li:nth-child(even){ + background:#ccc; +} \ No newline at end of file diff --git a/static/nodeDetails.ejs b/static/nodeDetails.ejs new file mode 100644 index 00000000..32ce63eb --- /dev/null +++ b/static/nodeDetails.ejs @@ -0,0 +1,21 @@ + + + <%- include("head.html") %> + + + + + + <%- include("sidebar.html") %> +
+
+

Slave details

+
+ +
+
+
+ +
+
+ \ No newline at end of file diff --git a/static/nodeDetails.js b/static/nodeDetails.js new file mode 100644 index 00000000..7bfaad78 --- /dev/null +++ b/static/nodeDetails.js @@ -0,0 +1,238 @@ +// string, object, function(object) +function post(url, data, callback) { + console.log("POST " + url + JSON.stringify(data)) + var xhr = new XMLHttpRequest(); + xhr.open("POST", url, true); + xhr.setRequestHeader("Content-type", "application/json"); + xhr.onreadystatechange = function () { + if (xhr.readyState == 4 && xhr.status == 200) { + var json = JSON.parse(xhr.responseText); + callback(json); + } + } + xhr.send(JSON.stringify(data)); +} +// callback(err, json) +function getJSON(url, callback) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.responseType = 'json'; + xhr.onload = function() { + var status = xhr.status; + if (status == 200) { + callback(null, xhr.response); + } else { + callback(status); + } + }; + // triggers if connection is refused + xhr.onerror = function(e){ + callback(e); + }; + xhr.send(); +}; +// function to sort arrays of objects after a keys value +function sortByKey(array, key) { + return array.sort(function(a, b) { + var x = Number(a[key]); var y = Number(b[key]); + return ((x < y) ? 1 : ((x > y) ? -1 : 0)); + }); +} +function getParameterByName(name, url) { + if (!url) url = window.location.href; + name = name.replace(/[\[\]]/g, "\\$&"); + var regex = new RegExp("[?&]" + name + "(=([^&#]*)|&|#|$)"), + results = regex.exec(url); + if (!results) return null; + if (!results[2]) return ''; + return decodeURIComponent(results[2].replace(/\+/g, " ")); +} + + + + +/// code below here +// update the online indicator dot +setInterval(function(){ + getJSON("/api/slaves", function(err, slaveData){ + let indicator = document.querySelector("#onlineIndicator"); + if(err){ + // our request threw an error, most likely master is unavailable so we show yellow indicator + indicator.style.backgroundColor = "yellow"; + indicator.title = "Master server unavailable"; + // compare with Math.floor(x/100000) allows us to check if they are within 10s of each other + } else if(Math.floor(slaveData[getParameterByName("slaveID")].time/100000) == Math.floor(Date.now()/100000)){ + indicator.style.backgroundColor = "green"; + indicator.title = "Slave is online"; + } else { + indicator.style.backgroundColor = "red"; + indicator.title = "Slave is offline"; + } + }); +}, 1000); + +function populateSlaveInfo(){ + let slaveID = getParameterByName("slaveID"); + if(!slaveID){ + throw "We need slaveID! GIVE US SLAVEID! ALL HAIL OUR NEW OVERLORDS!" + } + getJSON("/api/slaves", function(err, slaveData){ + /* + {"-1530507550":{ + "time":"1495475926375", + "rconPort":"8714", + "rconPassword":"hidden", + "serverPort":"14345", + "unique":"-1530507550", + "publicIP":"localhost", + "mods":[ + { + "modName":"clusterio_1.0.0.zip", + "hash":"341feb6c60918c83f7f3a6a14a4a308aef18dda6" + },{ + "modName":"clusterio_1.0.1.zip", + "hash":"aee7a4b67ece19bddf236312fe408a5d4b67248d" + },{ + "modName":"clusterio_1.1.0.zip", + "hash":"18e2f7e402d716e3dd1c1458b7fab2d0bb29eccf" + }], + "playerCount":"0", + "instanceName":"t", + "mac":"44-85-00-D2-7E-D6"} + } + */ + let slave = slaveData[slaveID]; + let HTML = "" // end of header + // left container + HTML += "
" + HTML += "

Last seen: "+moment(Number(slave.time)).fromNow()+"

" + HTML += "

Online players: "+slave.playerCount+"

" + HTML += "
" // end of displayBody + + // list mods and other metadata + HTML += "

Mods

    " + for(let i = 0; i < slave.mods.length; i++){ + HTML += "
  • "+slave.mods[i].modName+"
  • " + } + HTML += "
" + + HTML += "
" // end of left container + + // chart + HTML += '
' + + document.querySelector("#hero").innerHTML = HTML; + + // slaveDetails + HTML = ""; + if(slave.meta) + for(let key in slave.meta){ + let t = slave.meta[key]; + if(typeof t == "string"){ + HTML += "

"+key+": "+t+"

"; + } + } + + document.querySelector("#body > #details").innerHTML = HTML; + + // make production graph + makeGraph(slave.unique, slave.unique) + + }); +} + +populateSlaveInfo(); + +// ID of slave, ID of canvasjs div without # +function makeGraph(slaveID, selector) { + post("api/getStats", {slaveID: slaveID}, function(data){ + //console.log("Building chart " + slaveID + " with this data:") + //console.log(data) + if(data.length > 0) { + // find keys + let itemNames = []; + for(let key in data[data.length - 1].data) { + itemNames[itemNames.length] = key + } + let chartData = []; + for(let o = 0; o < itemNames.length; o++) { + if(itemNames[o] != "water"){ + chartData[chartData.length] = generateLineChartArray(data, itemNames[o]); + } + } + //console.log(chartData) + drawChart(selector, chartData) + } + // callback(data); + }) +} + +function generateLineChartArray(data, nameKey) { + let chartData = []; + for(let i = 0; i < data.length; i++) { + // only show recent data + if(data[i].timestamp > Date.now() - (24*60*60*1000)){ + let y = data[i].data[nameKey] + if(!data[i].data[nameKey]) { + y = 0; + } else if(y < 0) { + y = 0; + } + chartData[chartData.length] = { + x: new Date(data[i].timestamp), + y: Number(y) + } + } + } + chartData = sortByKey(chartData, "x"); + let xyz = {}; + xyz.name = nameKey; + xyz.type = "line"; + if(nameKey == "copper-wire"||nameKey == "iron-plate"||nameKey == "copper-plate"||nameKey == "electronic-circuit"||nameKey == "steel-plate"||nameKey == "advanced-circuit"||nameKey == "crude-oil"||nameKey == "petroleum-gas"){ + xyz.showInLegend = true; + } + xyz.dataPoints = chartData; + return xyz; +} + +var chartsByID = {}; +function drawChart(selector, chartData) { + // selector is ID of element, ex "chartContainer" or "-123199123" + console.log(chartData) + chartsByID[selector] = new CanvasJS.Chart(selector, { + title:{ + text: "Production graph" + }, + toolTip:{ + content: "{name}: {y}" + }, + zoomEnabled: true, + axisY:{ + includeZero: true, + }, + legend: { + cursor: "pointer", + itemclick: function (e) { + //console.log("legend click: " + e.dataPointIndex); + //console.log(e); + if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) { + e.dataSeries.visible = false; + } else { + e.dataSeries.visible = true; + } + e.chart.render(); + } + }, + data: chartData + }); + chart = chartsByID[selector]; + // console.log(chart) + chart.render(); +} \ No newline at end of file diff --git a/static/slaveDisplay.js b/static/slaveDisplay.js index 0045fad7..959491b5 100644 --- a/static/slaveDisplay.js +++ b/static/slaveDisplay.js @@ -19,7 +19,7 @@ setTimeout(function() { } HTML += "
"; HTML += '
'; - HTML += "

" + slaveData[key].instanceName + "

ID: " + slaveData[key].unique + "

Last seen: "+seenDate+"

Online players: "+slaveData[key].playerCount+"

IP: "+slaveData[key].publicIP +":"+ slaveData[key].serverPort+"

" + HTML += "

" + slaveData[key].instanceName + "

ID: " + slaveData[key].unique + "

Last seen: "+seenDate+"

Online players: "+slaveData[key].playerCount+"

IP: "+slaveData[key].publicIP +":"+ slaveData[key].serverPort+"

" HTML += "

Host: "+slaveData[key].mac+"

"; HTML += ""; HTML += ""; diff --git a/static/storage.js b/static/storage.js index 83e5930f..97d184b7 100644 --- a/static/storage.js +++ b/static/storage.js @@ -11,7 +11,10 @@ function drawcontents(data) { } else { img = "https://wiki.factorio.com/images/" + capitalizeFirstLetter(data[i].name) + ".png"; } - result = result + "" + data[i].name + "" + data[i].count + ""; + var searchField = document.querySelector("#search"); + if(!searchField.value || data[i].name.includes(searchField.value)) { + result = result + "" + data[i].name + "" + data[i].count + ""; + } } result = result + "" document.getElementById("contents").innerHTML = result; diff --git a/static/style.css b/static/style.css index 42d8de37..5156f4f4 100644 --- a/static/style.css +++ b/static/style.css @@ -1,11 +1,30 @@ * { margin:0px; + color: #FFF; +} +body { + /* same as http://www.lua-api.factorio.com/latest */ + background-color: rgb(38, 38, 38); +} +button { + border-radius: 0; + border: 1px solid transparent; + border-color: #FF7200; + border-width:1px; + cursor: pointer; } - #logo { - background-color: yellow; + /* using same colors as factorios official ones, taken from + http://www.lua-api.factorio.com */ + background-color: #262626; + color: #C8C8C8; } - +/* search field */ +#storage > input { + background-color:black; + width:300px; +} +/* Navbar on the right */ #sidenav { position:fixed; left:0px; @@ -13,12 +32,28 @@ background-color:darkgrey; height:100%; } +#sidenav a p { + color: #FF7200; + transition:0.5s; +} +#sidenav a:hover p { + color: #FFF; +} +a { + color: #FF7200; + text-decoration: none; +} +a:hover { + color: #FFF; +} +/* ensure body does not overlap navbar on the right */ #body { margin-left:200px; } #body > #production { display:block; } +/* Inventory display table */ #contents { width:300px; } @@ -38,7 +73,7 @@ float:left; width:100%; /*height:110px;*/ - background-color:#EEE; + background-color:#444; /*border-radius: 10px; margin-left:5px; margin-right:5px;*/ @@ -46,8 +81,18 @@ .slaveBox > p { margin:5px; } -.slaveBox > h2 { +.slaveBox h2 { margin:5px; + color: inherit; + transition:0.2s; +} +.slaveBox > button { + background-color: transparent; + color: #FF7200; + transition: 0.2s; +} +.slaveBox > button:hover { + color: #FFF; } .productionGraph { float:right;