From 8766529e5b4569ad06374b73ed13e5377bc4ca88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Mon, 22 Oct 2018 11:06:21 +0200 Subject: [PATCH 1/3] Removed linvodb/leveldown dependency --- master.js | 190 +-------------------- package.json | 3 - sharedPlugins/clusterioMod/masterPlugin.js | 2 - 3 files changed, 6 insertions(+), 189 deletions(-) diff --git a/master.js b/master.js index 438cbda2..3c8bb14b 100644 --- a/master.js +++ b/master.js @@ -10,6 +10,10 @@ combinator signals. node master.js */ +// Attempt updating +// const updater = require("./updater.js"); +// updater.update().then(console.log); + (async ()=>{ // Set the process title, shows up as the title of the CMD window on windows @@ -35,7 +39,6 @@ const averagedTimeSeries = require("averaged-timeseries"); const deepmerge = require("deepmerge"); const path = require("path"); const fs = require("fs-extra"); -const nedb = require("nedb"); const jwt = require('jsonwebtoken'); const bcrypt = require('bcryptjs'); const crypto = require('crypto'); @@ -230,24 +233,7 @@ function registerMoreMetrics(){ } // set up database -var Datastore = require('nedb'); -db = {}; -var LinvoDB = require("linvodb3"); - -// TODO: This breaks. databaseDirectory is either "database" or "./database". `LinvoDB.dbPath = "./database/linvodb/";` is the old working version. -LinvoDB.dbPath = path.resolve(config.databaseDirectory, "linvodb"); -// LinvoDB.dbPath = "./database/linvodb/"; - -// database for items in system -// db.items = new Datastore({ filename: 'database/items.db', autoload: true }); - -// in memory database for combinator signals -db.signals = new Datastore({ filename: path.resolve(config.databaseDirectory, '/signals.db'), autoload: true, inMemoryOnly: true}); -db.signals.ensureIndex({ fieldName: 'time', expireAfterSeconds: 3600 }, function (err) {}); - -// production chart database -db.flows = new LinvoDB("flows", {}, {}); - +const db = {}; (function(){ try{ @@ -618,43 +604,6 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { } }); -// circuit stuff -/** -POST endpoint to send and store circuit frames on the master. -Gives no response - -@memberof clusterioMaster -@instance -@alias api/setSignal -@param {object} circuitFrame -@param {number} circuitFrame.time -*/ -app.post("/api/setSignal", authenticate.middleware, function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - if(typeof req.body == "object" && req.body.time){ - db.signals.insert(req.body); - // console.log("signal set"); - } -}); -/** -POST endpoint to read database of circuit signals sent to master - -@memberof clusterioMaster -@instance -@alias api/readSignal -@returns {object} circuitFrame -*/ -app.post("/api/readSignal", function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - // request.body should be an object - // {since:UNIXTIMESTAMP,} - // we should send an array of all signals since then - db.signals.find({time:{$gte: req.body.since}}, function (err, docs) { - // $gte means greater than or equal to, meaning we only get entries newer than the timestamp - res.send(docs); - // console.log(docs); - }); -}); /** GET endpoint to read the masters current inventory of items. @@ -695,7 +644,7 @@ app.get("/api/inventoryAsObject", function(req, res) { // post flowstats here for production graphs // {timestamp: Date, instanceID: string, data: {"item":number}} -/** +/** --- TODO: Replace this with a simpler Prometheus exporter. Requires changes to the mod as well. POST endpoint to log production graph statistics. Should contain a timestamp gathered from Date.now(), a instanceID (also reffered to as "unique") and of course the timeSeries data. @@ -730,134 +679,7 @@ app.post("/api/logStats", authenticate.middleware, function(req,res) { res.send("failure"); } }); -// {instanceID: string, fromTime: Date, toTime, Date} -/** -POST endpoint to get timeSeries statistics stored on the master. Can give production -graphs and other IO statistics. -@memberof clusterioMaster -@instance -@alias api/getStats -@param {object} JSON -@returns {object} - with statistics -*/ -app.post("/api/getStats", function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - if(typeof req.body == "string"){ - req.body = JSON.parse(req.body); - } - // console.log(req.body); - if(typeof req.body == "object" && req.body.instanceID && req.body.statistic === undefined) { - // if not specified, get stats for last 24 hours - if(!req.body.fromTime) { - req.body.fromTime = Date.now() - 86400000 // 24 hours in MS - } - if(!req.body.toTime) { - req.body.toTime = Date.now(); - } - console.log("Looking... " + JSON.stringify(req.body)); - db.flows.find({ - instanceID: req.body.instanceID, - }, function(err, docs) { - let entries = docs.filter(function (el) { - return el.timestamp <= req.body.toTime && el.timestamp >= req.body.fromTime; - }); - // console.log(entries); - res.send(entries); - }); - } else if(typeof req.body == "object" && req.body.instanceID && typeof req.body.instanceID == "string" && req.body.itemName){ - if(req.body.statistic == "place"){ - console.log(`sending place data for instanceID ${req.body.instanceID} ${req.body.itemName}`); - // Gather data - //console.log(recievedItemStatisticsBySlaveID) - let itemStats = recievedItemStatisticsBySlaveID[req.body.instanceID]; - if(itemStats === undefined){ - res.send({statusForDebugging:"no data available"}); - return false; - } - let data = itemStats.get(config.itemStats.maxEntries, req.body.itemName); - - res.send({ - maxEntries:config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - data: data, - }); - } else if(req.body.statistic == "remove"){ - console.log(`sending remove data for instanceID ${req.body.instanceID} ${req.body.itemName}`); - // Gather data - // console.log(sentItemStatisticsBySlaveID) - let itemStats = sentItemStatisticsBySlaveID[req.body.instanceID]; - console.log(itemStats) - if(typeof itemStats == "object"){ - let data = itemStats.get(config.itemStats.maxEntries, req.body.itemName); - //console.log(itemStats.get(config.itemStats.maxEntries, req.body.itemName)); - res.send({ - maxEntries:config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - data: data, - }); - } else { - res.send({statusForDebugging:"no data available"}); - } - } - } -}); - -/** -POST endpoint. Modified version of api/getStats which feeds the web interface production graphs -at /nodes. It is like getStats but it does not report items which were not -produced at the moment of recording. (This is to save space, the 0-items were -making up about 92% of the response body weight.) - -@memberof clusterioMaster -@instance -@alias api/getTimelineStats -@param {object} JSON {instanceID:1941029, fromTime: ???, toTime: ???} -@returns {object[]} timeseries where each entry is a set point in time. -*/ -app.post("/api/getTimelineStats", function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - console.log(req.body); - if(typeof req.body == "object" && req.body.instanceID) { - // if not specified, get stats for last 24 hours - if(!req.body.fromTime) { - req.body.fromTime = Date.now() - 86400000 // 24 hours in MS - } - if(!req.body.toTime) { - req.body.toTime = Date.now(); - } - console.log("Looking... " + JSON.stringify(req.body)); - db.flows.find({ - instanceID: req.body.instanceID, - }, function(err, docs) { - if(err) { - console.error(err); - return; - } - - //Filter out all the entries outside the time range. - docs = docs.filter(function (el) { - return el.timestamp <= req.body.toTime && el.timestamp >= req.body.fromTime; - }); - - //Filter out all the elements that weren't produced. - docs.forEach(el => { - for(let key in el.data) { - if(el.data[key] === '0') { - // Set value to undefined instead of using the delete keyword. - // This is because the delete keyword is super duper slow. - // https://stackoverflow.com/questions/208105/how-do-i-remove-a-property-from-a-javascript-object - - //delete el.data[key]; - el.data[key] = undefined; - } - } - }); - - res.send(docs); - }); - } -}); /** POST endpoint for running commands on slaves. Requires x-access-token header to be set. Find you api token in secret-api-token.txt on the master (after running it once) diff --git a/package.json b/package.json index 833197ff..4fb90032 100644 --- a/package.json +++ b/package.json @@ -44,15 +44,12 @@ "html5-validator": "^1.0.7", "ini": "^1.3.5", "jsonwebtoken": "^8.2.1", - "leveldown": "^1.9.0", - "linvodb3": "^3.25.1", "luamin": "^1.0.4", "minimist": "^1.2.0", "mkdirp": "^0.5.1", "mkdirp-promise": "^5.0.1", "moment": "^2.22.2", "ncp": "^2.0.0", - "nedb": "^1.8.0", "needle": "^2.2.1", "node-factorio-api": "^0.3.1-alpha", "node-forge": "^0.7.1", diff --git a/sharedPlugins/clusterioMod/masterPlugin.js b/sharedPlugins/clusterioMod/masterPlugin.js index 9123a1c5..e02ca4b0 100644 --- a/sharedPlugins/clusterioMod/masterPlugin.js +++ b/sharedPlugins/clusterioMod/masterPlugin.js @@ -3,8 +3,6 @@ const path = require("path"); const Express = require("express"); const ejs = require("ejs"); -const pmSockets = []; - class masterPlugin { constructor({config, pluginConfig, pluginPath, socketio, express}){ this.config = config; From aaf4d8b6f1d4b1039b335cd77b79f312a52ade3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Tue, 23 Oct 2018 11:10:26 +0200 Subject: [PATCH 2/3] Removed graphing functionality that utilized linvodb --- README.md | 34 +++++++++++++++++++++++++--------- master.js | 46 ++-------------------------------------------- 2 files changed, 27 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 2405643e..03de0c11 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,12 @@ Discord for development/support/play: https://discord.gg/5XuDkje * [Windows setup](#windows-setup) +* [Optional plugins](#Plugins) + * [Common problems](#Common-problems) * [Command cheatsheet](#cheatsheet) -* [Optional plugins](#Plugins) - ## Introduction Features: @@ -36,7 +36,7 @@ Features: - Inventory combinator to display item levels in the cluster (and epoch time) -- Reporting of graphs and UPS on master interface +- Reporting of graphs and UPS on master interface (Also has extensive Prometheus reporting) Optional extras (see [Plugins](#Plugins)) @@ -205,6 +205,12 @@ Fancy game client that does the following steps automatically, but is really old 3. Run factorio and connect to slave as a normal MP game. You will find the port number to connect to at http://[masterAddress]:8080 +## Plugins +Here are the known Clusterio plugins in the wild: +1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta) +2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook. +3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers. + ## Common problems ### Cannot find module: `/../../config` @@ -227,6 +233,22 @@ if that does not suffice do If it still doesn't work, come over to our support discord linked at the top of this readme. +### Portforwarding doesn't work on the master server when running under WSL + +If you follow the ubuntu guide on WSL (Windows Subsystem for Linux, Bash on Ubuntu on Windows specifically), you will find that the website works on localhost and on your local ip, but not on the global ip. This is also true when you correctly port-forwarded the correct ports. Even when routing this server through nginx in WSL, the issue persists. Then, on a hunch, I tried to run nginx from windows itself and found that this DID work. It came to me that the only usage difference between the 2 versions of nginx is that I got a Windows Firewall popup. + +TLDR: the tested fix is: + +- open your windows firewall and go to advanced settings + +- click on inbound rules and click on new rule... + +- select port and click next > + +- select TCP and select specific local ports and type in the ports that you want to open (comma separated) and click next > 3 times + +- give the rule a name (like 'web server' or something), give it a description (optionally) and click finish + ### Other fixes for other potential problems: Sometimes the install fails. Try `node ./lib/npmPostinstall.js` to complete it. @@ -259,9 +281,3 @@ npm install ``` node client.js manage shared mods add clusterio ``` - -## Plugins -Here are the known Clusterio plugins in the wild: -1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta) -2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook. -3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers. diff --git a/master.js b/master.js index 3c8bb14b..0e72e40c 100644 --- a/master.js +++ b/master.js @@ -148,11 +148,6 @@ const prometheusWsUsageCounter = new Prometheus.Counter({ help: 'Websocket traffic', labelNames: ["connectionType", "instanceID"], }); -const prometheusProductionGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'production_gauge', - help: 'Items produced by instance', - labelNames: ["instanceID", "itemName"], -}); const prometheusExportGauge = new Prometheus.Gauge({ name: prometheusPrefix+'export_gauge', help: 'Items exported by instance', @@ -202,7 +197,7 @@ app.get('/metrics', (req, res) => { res.set('Content-Type', Prometheus.register.contentType); /// gather some static metrics - reisterMoreMetrics(); + registerMoreMetrics(); res.end(Prometheus.register.metrics()); }); @@ -642,44 +637,6 @@ app.get("/api/inventoryAsObject", function(req, res) { res.send(JSON.stringify(db.items)); }); -// post flowstats here for production graphs -// {timestamp: Date, instanceID: string, data: {"item":number}} -/** --- TODO: Replace this with a simpler Prometheus exporter. Requires changes to the mod as well. -POST endpoint to log production graph statistics. Should contain a timestamp -gathered from Date.now(), a instanceID (also reffered to as "unique") and of -course the timeSeries data. - -@memberof clusterioMaster -@instance -@alias api/logStats -@param {object} JSON {timestamp: Date.now(), instanceID: "string", data: {"item": number}} -@returns {string} failure -*/ -app.post("/api/logStats", authenticate.middleware, function(req,res) { - endpointHitCounter.labels(req.route.path).inc(); - if(typeof req.body == "object" && req.body.instanceID && req.body.timestamp && req.body.data) { - if(Number(req.body.timestamp) != NaN){ - req.body.timestamp = Number(req.body.timestamp); - db.flows.insert({ - instanceID: req.body.instanceID, - timestamp: req.body.timestamp, - data: req.body.data, - }); - try{ - Object.keys(req.body.data).forEach(itemName => { - prometheusProductionGauge.labels(req.body.instanceID, itemName).inc(Number(req.body.data[itemName]) || 0); - }); - }catch(e){console.log(e)}; - console.log("inserted: " + req.body.instanceID + " | " + req.body.timestamp); - } else { - console.log("error invalid timestamp " + req.body.timestamp); - res.send("failure"); - } - } else { - res.send("failure"); - } -}); - /** POST endpoint for running commands on slaves. Requires x-access-token header to be set. Find you api token in secret-api-token.txt on the master (after running it once) @@ -974,6 +931,7 @@ async function getPlugins(){ path: pluginConfig.pluginPath, socketio: io, express: app, + Prometheus, }), pluginConfig, }); From 52d35ab95572a59c7c197a3144b8666bb01bd1d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Tue, 23 Oct 2018 11:10:59 +0200 Subject: [PATCH 3/3] Added new plugin for reporting production/kills/construction statistics to Prometheus --- .../Lua/exportStatistics.lua | 19 ++++++ sharedPlugins/statisticsExporter/config.js | 10 ++++ sharedPlugins/statisticsExporter/index.js | 40 +++++++++++++ .../statisticsExporter/masterPlugin.js | 58 +++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 sharedPlugins/statisticsExporter/Lua/exportStatistics.lua create mode 100644 sharedPlugins/statisticsExporter/config.js create mode 100644 sharedPlugins/statisticsExporter/index.js create mode 100644 sharedPlugins/statisticsExporter/masterPlugin.js diff --git a/sharedPlugins/statisticsExporter/Lua/exportStatistics.lua b/sharedPlugins/statisticsExporter/Lua/exportStatistics.lua new file mode 100644 index 00000000..c50196b5 --- /dev/null +++ b/sharedPlugins/statisticsExporter/Lua/exportStatistics.lua @@ -0,0 +1,19 @@ +local statisticsData = "[" +for _, force in pairs(game.forces) do + statisticsData = statisticsData .. '{"forceName":"'..force.name..'","data":{' + local statistics = {"item_production_statistics", "fluid_production_statistics", "kill_count_statistics", "entity_build_count_statistics"} + for key, stat in pairs(statistics) do + statisticsData = statisticsData..'"'..stat..'":{"input":{' + for name, value in pairs(force[stat].input_counts) do + statisticsData = statisticsData..'"'..name..'":'..value..',' + end + statisticsData = statisticsData..'},"output":{' + for name, value in pairs(force[stat].output_counts) do + statisticsData = statisticsData..'"'..name..'":'..value..',' + end + statisticsData = statisticsData .. '}},' + end + statisticsData = statisticsData..'}},' +end +statisticsData = statisticsData..']' +rcon.print(statisticsData) diff --git a/sharedPlugins/statisticsExporter/config.js b/sharedPlugins/statisticsExporter/config.js new file mode 100644 index 00000000..bfa220de --- /dev/null +++ b/sharedPlugins/statisticsExporter/config.js @@ -0,0 +1,10 @@ +/* + Clusterio plugin for gathering ingame statistics +*/ +module.exports = { + name: "statisticsExporter", + version: "1.0.0", + binary: "nodePackage", + description: "Gathers ingame stats such as production, consumption, kills and buildings placed", + masterPlugin: "masterPlugin.js" +} diff --git a/sharedPlugins/statisticsExporter/index.js b/sharedPlugins/statisticsExporter/index.js new file mode 100644 index 00000000..bc2579fb --- /dev/null +++ b/sharedPlugins/statisticsExporter/index.js @@ -0,0 +1,40 @@ +const needle = require("needle"); +const fs = require("fs-extra"); +const path = require("path"); + +const clusterTools = require("_app/clusterTools")(); +const pluginConfig = require("./config"); + +module.exports = class remoteCommands { + constructor(mergedConfig, messageInterface, extras){ + this.messageInterface = messageInterface; + this.config = mergedConfig; + + const needleOptionsWithTokenAuthHeader = { + headers: { + 'x-access-token': this.config.masterAuthToken + }, + }; + + // Gather and submit statistics every 15 seconds + setInterval(async () => { + let stats = await this.getStats(); + stats.forEach(force => { + needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, { + instanceID: this.config.unique, + force: force.forceName, + data: force.data, + }, needleOptionsWithTokenAuthHeader, (err, resp) => { + if(!err && resp.body && resp.body.ok == false) console.log(resp.body); + }); + }); + }, 15000); + } + async getStats(){ + let command = fs.readFileSync(path.join(__dirname, "Lua/exportStatistics.lua"), "utf8")//clusterTools.getLua(path.join(__dirname, "Lua/exportStatistics.lua"), false); + let string = await this.messageInterface(`/silent-command ${command}`); + var data; + eval(`data = ${string}`); + return data; + } +} diff --git a/sharedPlugins/statisticsExporter/masterPlugin.js b/sharedPlugins/statisticsExporter/masterPlugin.js new file mode 100644 index 00000000..89b1a97b --- /dev/null +++ b/sharedPlugins/statisticsExporter/masterPlugin.js @@ -0,0 +1,58 @@ +const fs = require("fs-extra"); +const path = require("path"); + +class masterPlugin { + constructor({config, socketio, express, Prometheus}){ + const authenticate = require("./../../lib/authenticate")(config); + + this.config = config; + this.io = socketio; + const app = express; + + const prometheusStatisticsGauge = new Prometheus.Gauge({ + name: 'clusterio_statistics_gauge', + help: 'Items produced/built/killed by a force', + labelNames: ["instanceID", "force", "statistic", "direction", "itemName"], + }); + + /** --- TODO: Replace this with a simpler Prometheus exporter. Requires changes to the mod as well. + POST endpoint to log production graph statistics. Should contain an instanceID (also reffered to as "unique") + and of course the timeSeries data organized by category. + + @memberof clusterioMaster + @instance + @alias api/logStats + @param {object} JSON {instanceID: "string", force: "string" data: {"item_production_statistics": {"input":{"iron-plate":150}, "output":{}}, ...}} + @returns {string} failure + */ + app.post("/api/logStats", authenticate.middleware, function(req,res) { + // endpointHitCounter.labels(req.route.path).inc(); + if(typeof req.body == "object" + && req.body.instanceID + && typeof req.body.instanceID == "string" + && req.body.instanceID.length < 100 + && req.body.data + && typeof req.body.data == "object" + && !Array.isArray(req.body.data)){ + try{ + Object.keys(req.body.data).forEach(statisticName => { + Object.keys(req.body.data[statisticName]).forEach(direction => { + let statHash = req.body.data[statisticName][direction]; + Object.keys(statHash).forEach(item => { + let value = Number(statHash[item]); + if(value) prometheusStatisticsGauge.labels(req.body.instanceID, req.body.force, statisticName, direction, item).set(value); + }); + }); + }); + res.send({ok:true, msg:`Statistics successfully reported at ${Date.now()}`}); + }catch(e){ + res.send({ok:false, msg:"Statistics reporter crashed, see sharedPlugins/statisticsExporter/masterPlugin.js"}); + console.log(e) + }; + } else { + res.send("failure"); + } + }); + } +} +module.exports = masterPlugin;