From fa4c10dce23de8165d1b148cf5a46c2277b219c1 Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Fri, 12 Jul 2019 16:22:26 +0300 Subject: [PATCH 01/19] Code refactor --- .gitignore | 3 + sharedPlugins/researchSync/dumpResearch.lua | 9 +- sharedPlugins/researchSync/enableResearch.lua | 2 +- sharedPlugins/researchSync/index.js | 133 ++++++++++-------- 4 files changed, 86 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index d12f04bb..a7049f1f 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ sharedPlugins/scenario_loader/scenarios sharedPlugins/*/DISABLED \.cache-require-paths\.json + +# IDE +.idea/ diff --git a/sharedPlugins/researchSync/dumpResearch.lua b/sharedPlugins/researchSync/dumpResearch.lua index 2ecfb17b..e7e43cf0 100644 --- a/sharedPlugins/researchSync/dumpResearch.lua +++ b/sharedPlugins/researchSync/dumpResearch.lua @@ -1,9 +1,14 @@ -/silent-command local data = {}; for _, tech in pairs(game.forces["player"].technologies) do - table.insert(data, tech.name .. ":" .. tostring(tech.researched) .. ":" .. tostring(tech.level)); + local tech_data = { + tech.name, + tostring(tech.researched), + tostring(tech.level), + } + tech_data = table.concat(tech_data, ':') + table.insert(data, tech_data); end game.write_file("researchSync.txt", table.concat(data, "\n"), false, 0) diff --git a/sharedPlugins/researchSync/enableResearch.lua b/sharedPlugins/researchSync/enableResearch.lua index 13004d13..22a00652 100644 --- a/sharedPlugins/researchSync/enableResearch.lua +++ b/sharedPlugins/researchSync/enableResearch.lua @@ -1,4 +1,4 @@ -/silent-command + if game.forces['player'].technologies['{tech_name}'] then if game.forces['player'].technologies['{tech_name}'].researched ~= {tech_researched} then game.forces['player'].technologies['{tech_name}'].researched = {tech_researched} diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js index e198e310..7ab9c488 100644 --- a/sharedPlugins/researchSync/index.js +++ b/sharedPlugins/researchSync/index.js @@ -1,13 +1,24 @@ const fs = require('fs'); const needle = require("needle"); +function format_tech(tech) { + return `${!!tech.researched} at level ${tech.level}` +} + + +function log_tech_unlocking(key, current, prev) { + console.log( + `Unlocking ${key}: ${format_tech(current)}, was ${format_tech(prev)}` + ); +} + class ResearchSync { constructor(slaveConfig, messageInterface, extras = {}){ this.config = slaveConfig; this.messageInterface = messageInterface; this.functions = this.loadFunctions(); - + this.research = {}; setInterval(() => { @@ -22,61 +33,62 @@ class ResearchSync { } doSync() { - needle.get(this.config.masterIP + ':' + this.config.masterPort + '/api/slaves', (err, resp, slaveData) => { - if (err){ + const slaves_data_url = `${this.config.masterIP}:${this.config.masterPort}/api/slaves` + needle.get(slaves_data_url, (err, resp, slavesData) => { + if (err) { this.messageInterface("Unable to post JSON master/api/slaves, master might be unreachable"); return false; } - if (resp.statusCode !== 200){ + if (resp.statusCode !== 200) { this.messageInterface("got error when calling slaves", resp.statusCode, resp.body); return; } let needResearch = {}; + for (let slave_data of Object.values(slavesData)) { + if (slave_data.unique === this.config.unique.toString()) + continue; - Object.keys(slaveData).forEach(instanceKey => { - if (slaveData[instanceKey].unique === this.config.unique.toString()) { - return; + if (!slave_data.meta || !slave_data.meta.research) + continue; + + let researchList = slave_data.meta.research; + for (let [researchName, research] of Object.entries(researchList)) { + if (isNaN(research.researched) || isNaN(research.level)) + continue; + + if (needResearch[researchName]) { + if (needResearch[researchName].researched === 0) { + needResearch[researchName].researched = parseInt(research.researched); + } + if (needResearch[researchName].level < parseInt(research.level)) { + needResearch[researchName].level = parseInt(research.level); + } + } + else { + needResearch[researchName] = { + researched: parseInt(researchList[researchName].researched), + level: parseInt(researchList[researchName].level) + }; + } } - if (!slaveData[instanceKey].hasOwnProperty('meta') || !slaveData[instanceKey].meta.hasOwnProperty('research')) { - return; - } - let researchList = slaveData[instanceKey].meta.research; - if(researchList){ - Object.keys(researchList).forEach(researchName => { - if (isNaN(researchList[researchName].researched) || isNaN(researchList[researchName].level)) { - return; - } - if (needResearch.hasOwnProperty(researchName)) { - if (needResearch[researchName].researched === 0) { - needResearch[researchName].researched = parseInt(researchList[researchName].researched); - } - if (needResearch[researchName].level < parseInt(researchList[researchName].level)) { - needResearch[researchName].level = parseInt(researchList[researchName].level); - } - } - else { - needResearch[researchName] = {researched: parseInt(researchList[researchName].researched), level: parseInt(researchList[researchName].level)}; - } - }); - } - }); + } let difference = this.filterResearchDiff(this.research, needResearch); Object.keys(difference).forEach((key) => { - if (this.research[key]) { - let command = this.functions.enableResearch; - while(command.includes("{tech_name}")){ - command = command.replace("{tech_name}", key); - command = command.replace("{tech_researched}", difference[key].researched); - command = command.replace("{tech_level}", difference[key].level); - } - this.messageInterface(command); - console.log('Unlocking '+ key + ': ' + (difference[key].researched === 0 ? 'false' : 'true') + ' and level ' + difference[key].level + ', was '+ (this.research[key].researched === 0 ? 'false' : 'true') + ' at level ' + this.research[key].level); - this.messageInterface("Unlocking research: " + key + " at research state = " + (difference[key].researched === 0 ? 'false' : 'true') + ' and level ' + difference[key].level); - this.research[key] = difference[key]; - } + if (!this.research[key]) + return + let command = this.functions.enableResearch; + command = command.replace(/{tech_name}/g, key); + command = command.replace(/{tech_researched}/g, difference[key].researched); + command = command.replace(/{tech_level}/g, difference[key].level); + this.messageInterface(command); + log_tech_unlocking(key, difference[key], this.research[key]) + this.messageInterface( + `Unlocking research: ${key} with research state ${format_tech(difference[key])}` + ); + this.research[key] = difference[key]; }); needle.post(this.config.masterIP + ':' + this.config.masterPort + '/api/editSlaveMeta', { @@ -91,15 +103,20 @@ class ResearchSync { filterResearchDiff(localResearch, remoteResearch) { let diff = {}; - Object.keys(localResearch).forEach((key) => { - if (remoteResearch.hasOwnProperty(key)) { - if ((localResearch[key].researched === 0 && localResearch[key].researched !== remoteResearch[key].researched) || localResearch[key].level < remoteResearch[key].level) { - if (!isNaN(remoteResearch[key].researched) && !isNaN(remoteResearch[key].level)) { - diff[key] = {researched: remoteResearch[key].researched, level: remoteResearch[key].level}; - } - } + for (let key in localResearch) { + if (!remoteResearch[key]) + continue + if (isNaN(remoteResearch[key].researched) || isNaN(remoteResearch[key].level)) + continue + + if (remoteResearch[key].researched > localResearch[key].researched + || remoteResearch[key].level > localResearch[key].level) { + diff[key] = { + researched: remoteResearch[key].researched, + level: remoteResearch[key].level + }; } - }); + } return diff; } @@ -111,15 +128,15 @@ class ResearchSync { } loadFunc(path) { - return fs.readFileSync("sharedPlugins/researchSync/" + path,'utf-8').replace(/\r?\n|\r/g,' '); + let command = fs.readFileSync("sharedPlugins/researchSync/" + path,'utf-8') + command = command.replace(/\r?\n|\r/g,' ') + command = '/silent-command ' + command + return command; } - scriptOutput(data){ - let kv = data.split(":"); - let name = kv[0]; - let researched = ('true' !== kv[1] - ? 0 - : 1); - let level = parseInt(kv[2]); + scriptOutput(data) { + let [name, researched, level] = data.split(":") + researched = +(researched === 'true'); + level = parseInt(level); if (!isNaN(level) && !isNaN(researched)) { this.research[name] = {researched: researched, level: level}; } From 98c6795f8f7429293efe55e19ec316318a90264e Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Sun, 14 Jul 2019 23:29:15 +0300 Subject: [PATCH 02/19] Make researchSync work for any research on any node --- sharedPlugins/researchSync/dumpResearch.lua | 10 +- sharedPlugins/researchSync/index.js | 258 ++++++++++++------ sharedPlugins/researchSync/index.spec.js | 6 +- sharedPlugins/researchSync/updateProgress.lua | 28 ++ 4 files changed, 208 insertions(+), 94 deletions(-) create mode 100644 sharedPlugins/researchSync/updateProgress.lua diff --git a/sharedPlugins/researchSync/dumpResearch.lua b/sharedPlugins/researchSync/dumpResearch.lua index e7e43cf0..178365fc 100644 --- a/sharedPlugins/researchSync/dumpResearch.lua +++ b/sharedPlugins/researchSync/dumpResearch.lua @@ -1,11 +1,19 @@ local data = {}; +local force = game.forces["player"]; -for _, tech in pairs(game.forces["player"].technologies) do +for _, tech in pairs(force.technologies) do + local progress; + if tech == force.current_research then + progress = force.research_progress + else + progress = force.get_saved_technology_progress(tech.name); + end local tech_data = { tech.name, tostring(tech.researched), tostring(tech.level), + tostring(progress), } tech_data = table.concat(tech_data, ':') table.insert(data, tech_data); diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js index 7ab9c488..c1fb9528 100644 --- a/sharedPlugins/researchSync/index.js +++ b/sharedPlugins/researchSync/index.js @@ -6,24 +6,22 @@ function format_tech(tech) { } -function log_tech_unlocking(key, current, prev) { - console.log( - `Unlocking ${key}: ${format_tech(current)}, was ${format_tech(prev)}` - ); -} - - class ResearchSync { constructor(slaveConfig, messageInterface, extras = {}){ this.config = slaveConfig; this.messageInterface = messageInterface; - this.functions = this.loadFunctions(); + this.functions = { + dumpResearch: this.loadFunc("dumpResearch.lua"), + enableResearch: this.loadFunc("enableResearch.lua"), + updateProgress: this.loadFunc("updateProgress.lua") + } this.research = {}; + this.prev_research = {}; setInterval(() => { this.pollResearch(); - setTimeout(this.doSync.bind(this), 2000); + setTimeout(this.request_cluster_data.bind(this), 2000); }, extras.researchSyncPollInterval || 5000); } @@ -32,99 +30,154 @@ class ResearchSync { this.messageInterface(this.functions.dumpResearch); } - doSync() { + request_cluster_data() { const slaves_data_url = `${this.config.masterIP}:${this.config.masterPort}/api/slaves` - needle.get(slaves_data_url, (err, resp, slavesData) => { - if (err) { - this.messageInterface("Unable to post JSON master/api/slaves, master might be unreachable"); - return false; - } - if (resp.statusCode !== 200) { - this.messageInterface("got error when calling slaves", resp.statusCode, resp.body); - return; - } + needle.get(slaves_data_url, this.sync_researches.bind(this)) + } - let needResearch = {}; - for (let slave_data of Object.values(slavesData)) { - if (slave_data.unique === this.config.unique.toString()) - continue; + sync_researches(err, resp, slaves_data) { + if (err) { + this.messageInterface("Unable to post JSON master/api/slaves, master might be unreachable"); + return false; + } + if (resp.statusCode !== 200) { + this.messageInterface("got error when calling slaves", resp.statusCode, resp.body); + return; + } - if (!slave_data.meta || !slave_data.meta.research) - continue; + slaves_data = Object.values(slaves_data) + slaves_data = slaves_data.filter( + slave_data => slave_data.unique !== this.config.unique.toString() + && slave_data.meta && slave_data.meta.research + ) - let researchList = slave_data.meta.research; - for (let [researchName, research] of Object.entries(researchList)) { - if (isNaN(research.researched) || isNaN(research.level)) - continue; + let cluster_techs = this.get_cluster_techs(slaves_data) + this.recount_cluster_research_progress(slaves_data, cluster_techs) - if (needResearch[researchName]) { - if (needResearch[researchName].researched === 0) { - needResearch[researchName].researched = parseInt(research.researched); - } - if (needResearch[researchName].level < parseInt(research.level)) { - needResearch[researchName].level = parseInt(research.level); - } - } - else { - needResearch[researchName] = { - researched: parseInt(researchList[researchName].researched), - level: parseInt(researchList[researchName].level) - }; - } - } - } + let to_research = this.filter_researched_techs(cluster_techs) + let to_update_progress = this.filter_updated_techs(cluster_techs, to_research) - let difference = this.filterResearchDiff(this.research, needResearch); + this.research_technologies(to_research) + this.update_technologies_progress(to_update_progress) - Object.keys(difference).forEach((key) => { - if (!this.research[key]) - return - let command = this.functions.enableResearch; - command = command.replace(/{tech_name}/g, key); - command = command.replace(/{tech_researched}/g, difference[key].researched); - command = command.replace(/{tech_level}/g, difference[key].level); - this.messageInterface(command); - log_tech_unlocking(key, difference[key], this.research[key]) - this.messageInterface( - `Unlocking research: ${key} with research state ${format_tech(difference[key])}` - ); - this.research[key] = difference[key]; - }); - - needle.post(this.config.masterIP + ':' + this.config.masterPort + '/api/editSlaveMeta', { - instanceID: this.config.unique, - password: this.config.clientPassword, - meta: {research: this.research} - }, {headers: {'x-access-token': this.config.masterAuthToken}}, function (err, resp) { - // success? - }); + needle.post(this.config.masterIP + ':' + this.config.masterPort + '/api/editSlaveMeta', { + instanceID: this.config.unique, + password: this.config.clientPassword, + meta: {research: this.research} + }, {headers: {'x-access-token': this.config.masterAuthToken}}, function (err, resp) { + // success? }); } - filterResearchDiff(localResearch, remoteResearch) { - let diff = {}; - for (let key in localResearch) { - if (!remoteResearch[key]) - continue - if (isNaN(remoteResearch[key].researched) || isNaN(remoteResearch[key].level)) - continue + get_cluster_techs(slavesData) { + let researches = {} + for (let slave_data of slavesData) { + let node_researches = slave_data.meta.research + for (let [name, research] of Object.entries(node_researches)) { + if (isNaN(research.researched) || isNaN(research.level)) + continue - if (remoteResearch[key].researched > localResearch[key].researched - || remoteResearch[key].level > localResearch[key].level) { - diff[key] = { - researched: remoteResearch[key].researched, - level: remoteResearch[key].level - }; + if (researches[name]) { + if (researches[name].researched === 0) { + researches[name].researched = parseInt(research.researched) + } + if (researches[name].level < parseInt(research.level)) { + researches[name].level = parseInt(research.level) + } + } else { + researches[name] = { + researched: parseInt(research.researched), + level: parseInt(research.level) + } + } } } - return diff; + return researches } - loadFunctions() { - return { - dumpResearch: this.loadFunc("dumpResearch.lua"), - enableResearch: this.loadFunc("enableResearch.lua"), - }; + recount_cluster_research_progress(slaves_data, cluster_researches) { + for (let research of Object.values(cluster_researches)) + research.progress = research.contribution + + for (let slave_data of slaves_data) + for (let [name, research] of Object.entries(slave_data.meta.research)) + if (!isNaN(cluster_researches[name].progress) && !isNaN(research.contribution)) + cluster_researches[name].progress += research.contribution + + for (let [name, research] of Object.entries(cluster_researches)) { + if (research.progress > 1) { + research.progress = null + research.researched = 1 + if (this.research[name].level >= research.level) + research.level = this.research[name].level + 1 + } + } + } + + filter_researched_techs(cluster_researches) { + let local_researches = this.research + let result = {}; + for (let key in local_researches) { + if (!cluster_researches[key]) + continue + if (isNaN(cluster_researches[key].researched) || isNaN(cluster_researches[key].level)) + continue + + if (cluster_researches[key].researched > local_researches[key].researched + || cluster_researches[key].level > local_researches[key].level) { + result[key] = cluster_researches[key] + } + } + return result; + } + + filter_updated_techs(cluster_techs, to_research) { + let local_techs = this.research + let result = {} + for (let name in local_techs) { + if (name in to_research || !cluster_techs[name]) + continue + if (isNaN(cluster_techs[name].progress)) + continue + if (local_techs[name].progress < cluster_techs[name].progress) + result[name] = cluster_techs[name] + } + return result + } + + research_technologies(to_research) { + for (let [name, tech] of Object.entries(to_research)) { + if (!this.research[name]) + continue + let command = this.functions.enableResearch; + command = command.replace(/{tech_name}/g, key); + command = command.replace(/{tech_researched}/g, tech.researched); + command = command.replace(/{tech_level}/g, tech.level); + this.messageInterface(command); + console.log( + `Unlocking ${name}: ${format_tech(tech)}, was ${format_tech(this.research[name])}` + ); + this.messageInterface( + `Unlocking research: ${name} with research state ${format_tech(tech)}` + ); + this.research[key] = tech; + } + } + + update_technologies_progress(to_update) { + for (let [name, tech] of Object.entries(to_update)) { + if (!this.research[name]) + continue + let command = this.functions.updateProgress + command = command.replace(/{tech_name}/g, name) + command = command.replace(/{last_check_progress}/g, this.research[name].progress) + command = command.replace(/{new_progress}/g, tech.progress) + this.messageInterface(command); + console.log( + `Updating ${name}: progress += ${tech.progress - this.research[name].progress}` + ); + this.research[name].progress = tech.progress + } } loadFunc(path) { @@ -134,11 +187,36 @@ class ResearchSync { return command; } scriptOutput(data) { - let [name, researched, level] = data.split(":") + let [name, researched, level, progress] = data.split(":") researched = +(researched === 'true'); level = parseInt(level); - if (!isNaN(level) && !isNaN(researched)) { - this.research[name] = {researched: researched, level: level}; + if (progress === 'nil') + progress = null + else + progress = parseFloat(progress) + + if (isNaN(level) || isNaN(researched)) + return + this.prev_research[name] = this.research[name] + if (!this.prev_research[name]) { + this.prev_research[name] = { + researched: null, + level: null, + progress: null, + contribution: 0, + } + } + this.research[name] = { + researched, + level, + progress, + contribution: this.prev_research[name].contribution + } + if (this.prev_research[name].progress && this.research[name].progress) { + // this.prev_research[name].progress gets updated to overall cluster progress + // therefore contribution should be own research progress change over sync interval + let contribution = this.research[name].progress - this.prev_research[name].progress + this.research[name].contribution += contribution } } } diff --git a/sharedPlugins/researchSync/index.spec.js b/sharedPlugins/researchSync/index.spec.js index eae51abc..372bf01e 100644 --- a/sharedPlugins/researchSync/index.spec.js +++ b/sharedPlugins/researchSync/index.spec.js @@ -9,7 +9,7 @@ describe("researchSync/index.js", ()=>{ assert.equal(typeof researchSync, "function"); }); describe("class researchSync()", ()=>{ - it(".filterResearchDiff(object1, object2) returns research that needs enabling on the local instance", ()=>{ + it(".filter_researched_techs(object1, object2) returns research that needs enabling on the local instance", ()=>{ let reSync = new researchSync({}, function(){}); let obj1 = { @@ -24,9 +24,9 @@ describe("researchSync/index.js", ()=>{ c:{researched: false, level: 0}, e:{researched: false, level: 50}, }; - assert(typeof reSync.filterResearchDiff == "function"); + assert(typeof reSync.filter_researched_techs == "function"); - let diffResult = reSync.filterResearchDiff(obj1, obj2); + let diffResult = reSync.filter_researched_techs(obj1, obj2); assert(diffResult.a === undefined); diff --git a/sharedPlugins/researchSync/updateProgress.lua b/sharedPlugins/researchSync/updateProgress.lua new file mode 100644 index 00000000..68b7d4ec --- /dev/null +++ b/sharedPlugins/researchSync/updateProgress.lua @@ -0,0 +1,28 @@ + +local function get_tech_progress(tech) + if tech == force.current_research then + return force.research_progress + else + return force.get_saved_technology_progress(tech.name); + end +end +local function set_tech_progress(tech, progress) + if tech == force.current_research then + force.research_progress = progress + else + force.set_saved_technology_progress(tech.name, progress); + end +end + +local force = game.force['player'] +local tech_name = {tech_name} +local tech = force.technologies[tech_name] +local last_check_progress = {last_check_progress} +local new_progress = {new_progress} + +local progress_delta = get_tech_progress(tech) - last_check_progress +new_progress = new_progress + progress_delta +if new_progress > 1 then + new_progress = 1 +end +set_tech_progress(tech, new_progress) From 3b814eeea41776e1bbc624808ac7335e82bdb2cf Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Mon, 15 Jul 2019 11:34:02 +0300 Subject: [PATCH 03/19] Fix a bug causing all slaves data gets cleared on master startup (#165) --- master.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master.js b/master.js index 276af857..cba39301 100644 --- a/master.js +++ b/master.js @@ -235,7 +235,7 @@ const db = {}; (function(){ try{ - let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json")); + let x = fs.statSync(path.resolve(config.databaseDirectory, "slaves.json")); console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`); slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json"))); } catch (e){ From 4a0956d17461c2cd283258a6254d135512a3606c Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Mon, 15 Jul 2019 11:26:03 +0300 Subject: [PATCH 04/19] Fix a bug causing all slaves data gets cleared on master startup Cherry-picked from fix-slaves-data-cleanup-on-startup --- master.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master.js b/master.js index 276af857..cba39301 100644 --- a/master.js +++ b/master.js @@ -235,7 +235,7 @@ const db = {}; (function(){ try{ - let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json")); + let x = fs.statSync(path.resolve(config.databaseDirectory, "slaves.json")); console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`); slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json"))); } catch (e){ From e1b510b4417c9ed3bebfc16c05fcb94601644532 Mon Sep 17 00:00:00 2001 From: veladon Date: Mon, 15 Jul 2019 12:46:38 +0100 Subject: [PATCH 05/19] fix for stats exporter console errors --- sharedPlugins/statisticsExporter/index.js | 16 +++++++++++----- sharedPlugins/statisticsExporter/masterPlugin.js | 15 +++++++++------ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/sharedPlugins/statisticsExporter/index.js b/sharedPlugins/statisticsExporter/index.js index bc2579fb..93062f9e 100644 --- a/sharedPlugins/statisticsExporter/index.js +++ b/sharedPlugins/statisticsExporter/index.js @@ -20,13 +20,19 @@ module.exports = class remoteCommands { setInterval(async () => { let stats = await this.getStats(); stats.forEach(force => { - needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, { + // I have no idea why I need to do JSON.stringify before passing it to needle.post, + // but if you do forceData: force.data, it doesn't work... + let request = { instanceID: this.config.unique, force: force.forceName, - data: force.data, - }, needleOptionsWithTokenAuthHeader, (err, resp) => { - if(!err && resp.body && resp.body.ok == false) console.log(resp.body); - }); + forceData: JSON.stringify(force.data), + }; + needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, + request, + needleOptionsWithTokenAuthHeader, (err, resp) => { + if(!err && resp.body && resp.body.ok == false) console.log("Error calling /api/logStats: " + resp.body); + } + ); }); }, 15000); } diff --git a/sharedPlugins/statisticsExporter/masterPlugin.js b/sharedPlugins/statisticsExporter/masterPlugin.js index 7a5da91e..1dae44bd 100644 --- a/sharedPlugins/statisticsExporter/masterPlugin.js +++ b/sharedPlugins/statisticsExporter/masterPlugin.js @@ -26,18 +26,19 @@ class masterPlugin { @returns {string} failure */ app.post("/api/logStats", authenticate.middleware, function(req,res) { + let data = JSON.parse(req.body.forceData); // endpointHitCounter.labels(req.route.path).inc(); if(typeof req.body == "object" && req.body.instanceID && typeof req.body.instanceID == "string" && req.body.instanceID.length < 100 - && req.body.data - && typeof req.body.data == "object" - && !Array.isArray(req.body.data)){ + && data + && typeof data == "object" + && !Array.isArray(data)){ try{ - Object.keys(req.body.data).forEach(statisticName => { - Object.keys(req.body.data[statisticName]).forEach(direction => { - let statHash = req.body.data[statisticName][direction]; + Object.keys(data).forEach(statisticName => { + Object.keys(data[statisticName]).forEach(direction => { + let statHash = data[statisticName][direction]; Object.keys(statHash).forEach(item => { let value = Number(statHash[item]); if(value) prometheusStatisticsGauge.labels(req.body.instanceID, req.body.force, statisticName, direction, item).set(value); @@ -50,6 +51,8 @@ class masterPlugin { console.log(e) }; } else { + console.log(`/api/logStats: Invalid request. instanceID: ${req.body.instanceID}, dataIsArray: ` + + `${Array.isArray(data)}, ${typeof req.body.instanceID}, ${typeof data}`); res.send({ ok:false, msg:"Invalid request parameters" From 233410add70702e894ebe6a6d30a2e26f0b0bfbf Mon Sep 17 00:00:00 2001 From: veladon Date: Mon, 15 Jul 2019 12:49:49 +0100 Subject: [PATCH 06/19] fix for stats exporter console errors (#166) --- sharedPlugins/statisticsExporter/index.js | 16 +++++++++++----- sharedPlugins/statisticsExporter/masterPlugin.js | 15 +++++++++------ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/sharedPlugins/statisticsExporter/index.js b/sharedPlugins/statisticsExporter/index.js index bc2579fb..93062f9e 100644 --- a/sharedPlugins/statisticsExporter/index.js +++ b/sharedPlugins/statisticsExporter/index.js @@ -20,13 +20,19 @@ module.exports = class remoteCommands { setInterval(async () => { let stats = await this.getStats(); stats.forEach(force => { - needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, { + // I have no idea why I need to do JSON.stringify before passing it to needle.post, + // but if you do forceData: force.data, it doesn't work... + let request = { instanceID: this.config.unique, force: force.forceName, - data: force.data, - }, needleOptionsWithTokenAuthHeader, (err, resp) => { - if(!err && resp.body && resp.body.ok == false) console.log(resp.body); - }); + forceData: JSON.stringify(force.data), + }; + needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, + request, + needleOptionsWithTokenAuthHeader, (err, resp) => { + if(!err && resp.body && resp.body.ok == false) console.log("Error calling /api/logStats: " + resp.body); + } + ); }); }, 15000); } diff --git a/sharedPlugins/statisticsExporter/masterPlugin.js b/sharedPlugins/statisticsExporter/masterPlugin.js index 7a5da91e..1dae44bd 100644 --- a/sharedPlugins/statisticsExporter/masterPlugin.js +++ b/sharedPlugins/statisticsExporter/masterPlugin.js @@ -26,18 +26,19 @@ class masterPlugin { @returns {string} failure */ app.post("/api/logStats", authenticate.middleware, function(req,res) { + let data = JSON.parse(req.body.forceData); // endpointHitCounter.labels(req.route.path).inc(); if(typeof req.body == "object" && req.body.instanceID && typeof req.body.instanceID == "string" && req.body.instanceID.length < 100 - && req.body.data - && typeof req.body.data == "object" - && !Array.isArray(req.body.data)){ + && data + && typeof data == "object" + && !Array.isArray(data)){ try{ - Object.keys(req.body.data).forEach(statisticName => { - Object.keys(req.body.data[statisticName]).forEach(direction => { - let statHash = req.body.data[statisticName][direction]; + Object.keys(data).forEach(statisticName => { + Object.keys(data[statisticName]).forEach(direction => { + let statHash = data[statisticName][direction]; Object.keys(statHash).forEach(item => { let value = Number(statHash[item]); if(value) prometheusStatisticsGauge.labels(req.body.instanceID, req.body.force, statisticName, direction, item).set(value); @@ -50,6 +51,8 @@ class masterPlugin { console.log(e) }; } else { + console.log(`/api/logStats: Invalid request. instanceID: ${req.body.instanceID}, dataIsArray: ` + + `${Array.isArray(data)}, ${typeof req.body.instanceID}, ${typeof data}`); res.send({ ok:false, msg:"Invalid request parameters" From 0a65946bf4003d1d61dc6e1edb9ed9960910b7cd Mon Sep 17 00:00:00 2001 From: veladon Date: Mon, 15 Jul 2019 13:47:53 +0100 Subject: [PATCH 07/19] fix for /api/logstats --- sharedPlugins/statisticsExporter/masterPlugin.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sharedPlugins/statisticsExporter/masterPlugin.js b/sharedPlugins/statisticsExporter/masterPlugin.js index 1dae44bd..689dd2eb 100644 --- a/sharedPlugins/statisticsExporter/masterPlugin.js +++ b/sharedPlugins/statisticsExporter/masterPlugin.js @@ -26,7 +26,10 @@ class masterPlugin { @returns {string} failure */ app.post("/api/logStats", authenticate.middleware, function(req,res) { - let data = JSON.parse(req.body.forceData); + let data = req.body.forceData; + if(typeof req.body.forceData == "string") { + data = JSON.parse(req.body.forceData); + } // endpointHitCounter.labels(req.route.path).inc(); if(typeof req.body == "object" && req.body.instanceID From 251ec696a37b6725cfb3e008c341dd092215ddfd Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Mon, 15 Jul 2019 18:29:53 +0300 Subject: [PATCH 08/19] Fixes && debugging --- sharedPlugins/researchSync/index.js | 105 ++++++++++++++---- sharedPlugins/researchSync/updateProgress.lua | 19 ++-- 2 files changed, 93 insertions(+), 31 deletions(-) diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js index c1fb9528..e23c79b9 100644 --- a/sharedPlugins/researchSync/index.js +++ b/sharedPlugins/researchSync/index.js @@ -18,16 +18,47 @@ class ResearchSync { this.research = {}; this.prev_research = {}; - - setInterval(() => { - this.pollResearch(); - setTimeout(this.request_cluster_data.bind(this), 2000); - }, extras.researchSyncPollInterval || 5000); + this.initial_request_own_data( + () => this.setup_sync_task(extras) + ) } - pollResearch() { - // this.messageInterface("Polling Research\n") + initial_request_own_data(callback) { + const url = `${this.config.masterIP}:${this.config.masterPort}/api/getSlaveMeta` + const data = { + instanceID: this.config.unique, + password: this.config.clientPassword, + } + const options = { + headers: {'x-access-token': this.config.masterAuthToken}, + json: true + } + needle.post(url, data, options, (err, res, techs) => { + if (err) { + console.log(`Can't get own slave data:`) + console.error(err) + return + } + if (res.statusCode !== 200) { + console.log(`Can't get own slave data:`) + console.error(`status code ${res.statusCode}, ${res.body}`) + return + } + techs = JSON.parse(techs) + if (typeof techs.research === 'object') + this.research = techs.research + callback() + }) + } + + setup_sync_task(extras) { + const timeout = extras.researchSyncPollInterval || 5000 + setInterval(() => this.sync_task(), timeout); + } + + sync_task() { this.messageInterface(this.functions.dumpResearch); + setTimeout(this.request_cluster_data.bind(this), 2000); } request_cluster_data() { @@ -60,12 +91,17 @@ class ResearchSync { this.research_technologies(to_research) this.update_technologies_progress(to_update_progress) + this.print_own_contribution() + needle.post(this.config.masterIP + ':' + this.config.masterPort + '/api/editSlaveMeta', { instanceID: this.config.unique, password: this.config.clientPassword, meta: {research: this.research} - }, {headers: {'x-access-token': this.config.masterAuthToken}}, function (err, resp) { - // success? + }, {headers: {'x-access-token': this.config.masterAuthToken}, json: true}, function (err, resp) { + if (err) + console.error(err) + else + console.log('sent') }); } @@ -79,16 +115,13 @@ class ResearchSync { if (researches[name]) { if (researches[name].researched === 0) { - researches[name].researched = parseInt(research.researched) + researches[name].researched = research.researched } - if (researches[name].level < parseInt(research.level)) { - researches[name].level = parseInt(research.level) + if (researches[name].level < research.level) { + researches[name].level = research.level } } else { - researches[name] = { - researched: parseInt(research.researched), - level: parseInt(research.level) - } + researches[name] = research } } } @@ -96,8 +129,8 @@ class ResearchSync { } recount_cluster_research_progress(slaves_data, cluster_researches) { - for (let research of Object.values(cluster_researches)) - research.progress = research.contribution + for (let [name, research] of Object.entries(cluster_researches)) + research.progress = this.research[name].contribution for (let slave_data of slaves_data) for (let [name, research] of Object.entries(slave_data.meta.research)) @@ -108,6 +141,8 @@ class ResearchSync { if (research.progress > 1) { research.progress = null research.researched = 1 + this.research[name].contribution = 0 + research.contribution = 0 if (this.research[name].level >= research.level) research.level = this.research[name].level + 1 } @@ -149,8 +184,10 @@ class ResearchSync { for (let [name, tech] of Object.entries(to_research)) { if (!this.research[name]) continue + this.research[name].contribution = 0 + this.research[name].progress = null let command = this.functions.enableResearch; - command = command.replace(/{tech_name}/g, key); + command = command.replace(/{tech_name}/g, name); command = command.replace(/{tech_researched}/g, tech.researched); command = command.replace(/{tech_level}/g, tech.level); this.messageInterface(command); @@ -160,7 +197,7 @@ class ResearchSync { this.messageInterface( `Unlocking research: ${name} with research state ${format_tech(tech)}` ); - this.research[key] = tech; + this.research[name] = tech; } } @@ -168,22 +205,38 @@ class ResearchSync { for (let [name, tech] of Object.entries(to_update)) { if (!this.research[name]) continue + let progress = this.research[name].progress + if (progress === null) + progress = 'nil' let command = this.functions.updateProgress command = command.replace(/{tech_name}/g, name) - command = command.replace(/{last_check_progress}/g, this.research[name].progress) + + command = command.replace(/{last_check_progress}/g, progress) command = command.replace(/{new_progress}/g, tech.progress) this.messageInterface(command); console.log( - `Updating ${name}: progress += ${tech.progress - this.research[name].progress}` + `Updating ${name}: ${this.research[name].progress} += ${tech.progress - this.research[name].progress}` ); this.research[name].progress = tech.progress } } - loadFunc(path) { + print_own_contribution() { + if (!this.prev_research) + return + for (let [name, tech] of Object.entries(this.research)) { + if (!this.prev_research[name]) + continue + let diff = this.research[name].contribution - this.prev_research[name].contribution + if (Math.abs(diff) > Number.EPSILON * 1000) + console.log(`Own research ${name}: ${this.research[name].progress} += ${diff}`) + } + } + + loadFunc(path, silent=true) { let command = fs.readFileSync("sharedPlugins/researchSync/" + path,'utf-8') command = command.replace(/\r?\n|\r/g,' ') - command = '/silent-command ' + command + command = (silent ? '/silent-command ' : '/c ') + command return command; } scriptOutput(data) { @@ -218,6 +271,10 @@ class ResearchSync { let contribution = this.research[name].progress - this.prev_research[name].progress this.research[name].contribution += contribution } + if (Math.abs(this.research[name].contribution) < Number.EPSILON * 1000) { + // if contribution should be 0 but because of floating-point precision is e.g. 2.2564e-18 + this.research[name].contribution = 0 + } } } diff --git a/sharedPlugins/researchSync/updateProgress.lua b/sharedPlugins/researchSync/updateProgress.lua index 68b7d4ec..0d07bb84 100644 --- a/sharedPlugins/researchSync/updateProgress.lua +++ b/sharedPlugins/researchSync/updateProgress.lua @@ -1,12 +1,12 @@ -local function get_tech_progress(tech) +local function get_tech_progress(force, tech) if tech == force.current_research then return force.research_progress else return force.get_saved_technology_progress(tech.name); end end -local function set_tech_progress(tech, progress) +local function set_tech_progress(force, tech, progress) if tech == force.current_research then force.research_progress = progress else @@ -14,15 +14,20 @@ local function set_tech_progress(tech, progress) end end -local force = game.force['player'] -local tech_name = {tech_name} +local force = game.forces['player'] +local tech_name = '{tech_name}' local tech = force.technologies[tech_name] local last_check_progress = {last_check_progress} local new_progress = {new_progress} -local progress_delta = get_tech_progress(tech) - last_check_progress -new_progress = new_progress + progress_delta +local progress_delta = get_tech_progress(force, tech) +if last_check_progress ~= nil then + progress_delta = progress_delta - last_check_progress +end +if progress_delta ~= nil then + new_progress = new_progress + progress_delta +end if new_progress > 1 then new_progress = 1 end -set_tech_progress(tech, new_progress) +set_tech_progress(force, tech, new_progress) From fec3cf5ddf2bf38050fd34836ea130de1d19eead Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Mon, 15 Jul 2019 22:49:39 +0300 Subject: [PATCH 09/19] Fix for migation to new researchSync version --- sharedPlugins/researchSync/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js index d8cb5887..dbb563c0 100644 --- a/sharedPlugins/researchSync/index.js +++ b/sharedPlugins/researchSync/index.js @@ -156,7 +156,8 @@ class ResearchSync { for (let slave_data of slaves_data) for (let [name, research] of Object.entries(slave_data.meta.research)) - if (!isNaN(cluster_researches[name].progress) && !isNaN(research.contribution)) + if (cluster_researches[name] + && !isNaN(cluster_researches[name].progress) && !isNaN(research.contribution)) cluster_researches[name].progress += research.contribution for (let [name, research] of Object.entries(cluster_researches)) { From d48a824a69565ba8d049ff96c5cb960ef2f6fbcd Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Wed, 17 Jul 2019 00:31:15 +0300 Subject: [PATCH 10/19] Fix researchSync that it counts progress only from techs on same level --- sharedPlugins/researchSync/index.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js index dbb563c0..d48f0565 100644 --- a/sharedPlugins/researchSync/index.js +++ b/sharedPlugins/researchSync/index.js @@ -154,11 +154,19 @@ class ResearchSync { for (let [name, research] of Object.entries(cluster_researches)) research.progress = this.research[name].contribution - for (let slave_data of slaves_data) - for (let [name, research] of Object.entries(slave_data.meta.research)) - if (cluster_researches[name] - && !isNaN(cluster_researches[name].progress) && !isNaN(research.contribution)) + for (let slave_data of slaves_data) { + for (let [name, research] of Object.entries(slave_data.meta.research)) { + if (!cluster_researches[name]) + continue + if (isNaN(cluster_researches[name].progress) + || isNaN(research.contribution) + || isNaN(research.level) + || isNaN(cluster_researches[name].level)) + continue + if (cluster_researches[name].level === research.level) cluster_researches[name].progress += research.contribution + } + } for (let [name, research] of Object.entries(cluster_researches)) { if (research.progress > 1) { From 8672afe9409ba15160557ab39abfad9bcd8fdbcd Mon Sep 17 00:00:00 2001 From: MCRcortex Date: Wed, 17 Jul 2019 22:04:29 +1000 Subject: [PATCH 11/19] added res.sendStatus(200); to stop client errors throwing according to the docs "The methods on the response object (res) in the following table can send a response to the client, and terminate the request-response cycle. If none of these methods are called from a route handler, the client request will be left hanging." Last sentence is the critical one. Sending a status code 200 should fix the issue --- master.js | 1 + 1 file changed, 1 insertion(+) diff --git a/master.js b/master.js index 358e911f..b87b7d27 100644 --- a/master.js +++ b/master.js @@ -362,6 +362,7 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) { let metaPortion = JSON.stringify(req.body.meta); if(metaPortion.length > 50) metaPortion = metaPortion.substring(0,20) + "..."; // console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion); + res.sendStatus(200); } else { res.send("ERROR: Invalid instanceID or password") } From 50370bf858d2c53dcde657c36a28433b1e78d051 Mon Sep 17 00:00:00 2001 From: Master-Guy <566429+Master-Guy@users.noreply.github.com> Date: Wed, 17 Jul 2019 18:35:02 +0200 Subject: [PATCH 12/19] 1/2 Add "info" field between client and master Adds a "info" field to config.json and allows the client to send the contents of this info field to the master, so it also shows up (i.e. with /api/slaves) --- client.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client.js b/client.js index ce67a6a8..f1fe4e41 100644 --- a/client.js +++ b/client.js @@ -673,6 +673,7 @@ async function instanceManagement(instanceconfig) { publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config mods:modHashes, instanceName: instance, + info: instanceconfig.info, } if(playerCount){ payload.playerCount = playerCount.replace(/(\r\n\t|\n|\r\t)/gm, ""); From c9cbc48d95142a50ab26cd8e4564e554cd4d4013 Mon Sep 17 00:00:00 2001 From: Master-Guy <566429+Master-Guy@users.noreply.github.com> Date: Wed, 17 Jul 2019 18:40:01 +0200 Subject: [PATCH 13/19] 2/2 Add "info" field between client and master Adds a "info" field to config.json and allows the client to send the contents of this info field to the master, so it also shows up (i.e. with /api/slaves) --- client.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client.js b/client.js index f1fe4e41..2f755780 100644 --- a/client.js +++ b/client.js @@ -317,6 +317,7 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n "clientPort": args["rcon-port"] || process.env.RCONPORT || Math.floor(Math.random() * 65535), "__comment_clientPassword": "This is the rcon password. Its also used for making an instanceID. Make sure its unique and not blank.", "clientPassword": args["rcon-password"] || Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8), + "info": {}, } console.log("Clusterio | Created instance with settings:") console.log(instconf); From 6ac8c0711f20d516c56f5e5c4602018009fe1d5d Mon Sep 17 00:00:00 2001 From: Master-Guy <566429+Master-Guy@users.noreply.github.com> Date: Wed, 17 Jul 2019 19:23:53 +0200 Subject: [PATCH 14/19] JSON fix Small fix for JSON --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index 2f755780..9f4427b6 100644 --- a/client.js +++ b/client.js @@ -317,7 +317,7 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n "clientPort": args["rcon-port"] || process.env.RCONPORT || Math.floor(Math.random() * 65535), "__comment_clientPassword": "This is the rcon password. Its also used for making an instanceID. Make sure its unique and not blank.", "clientPassword": args["rcon-password"] || Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8), - "info": {}, + "info": {} } console.log("Clusterio | Created instance with settings:") console.log(instconf); From 2831c07000581d7564e8f03f8f84ef49666c4ffd Mon Sep 17 00:00:00 2001 From: TechnoStrife Date: Wed, 17 Jul 2019 23:21:57 +0300 Subject: [PATCH 15/19] Add logging to researchSync plugin --- sharedPlugins/researchSync/index.js | 90 +++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js index d48f0565..d9554f04 100644 --- a/sharedPlugins/researchSync/index.js +++ b/sharedPlugins/researchSync/index.js @@ -1,24 +1,79 @@ const fs = require('fs'); +const path = require('path') const needle = require("needle"); +function time() { + let d = new Date() + return `${d.getUTCHours()}:${d.getUTCMinutes()}:${d.getUTCSeconds()}.${d.getUTCMilliseconds()}` +} + class ResearchSync { constructor(slaveConfig, messageInterface, extras = {}){ - this.config = slaveConfig; - this.messageInterface = messageInterface; + this.config = slaveConfig + this.messageInterface = messageInterface + this.functions = { dumpResearch: this.loadFunc("dumpResearch.lua"), enableResearch: this.loadFunc("enableResearch.lua"), updateProgress: this.loadFunc("updateProgress.lua") } - this.research = {}; - this.prev_research = {}; + this.log_folder = './logs' + if (!fs.existsSync(this.log_folder)) + fs.mkdirSync(this.log_folder); + this.log_file = path.join(this.log_folder, `${this.config.unique}-research.log`) + this.get_node_name_and_move_log() + + this.research = {} + this.prev_research = {} this.initial_request_own_data( () => this.setup_sync_task(extras) ) } + log(data) { + try { + console.log(`researchSync: ${data}`) + fs.appendFileSync(this.log_file, `${time()}: ${data}\n`); + } catch (e) { + console.error(e) + } + } + + error(data) { + try { + console.error(`researchSync: ${data}`) + fs.appendFileSync(this.log_file, `${time()}: ${data}\n`); + } catch (e) { + console.error(e) + } + } + + get_node_name_and_move_log() { + const url = `${this.config.masterIP}:${this.config.masterPort}/api/slaves` + needle.get(url, (err, res, slaves_data) => { + if (err) + return this.error(err) + + slaves_data = Object.values(slaves_data) + let node_data = slaves_data.find( + slave_data => slave_data.unique === this.config.unique.toString() + ) + if (node_data === undefined) + return this.error('slave was not found') + let node_name = node_data.instanceName + let log_file = path.join(this.log_folder, `${node_name}-research.log`) + if (fs.existsSync(log_file)) { + fs.appendFileSync(log_file, fs.readFileSync(this.log_file)); + fs.unlinkSync(this.log_file) + } else { + fs.renameSync(this.log_file, log_file) + } + this.log_file = log_file + }) + } + initial_request_own_data(callback) { const url = `${this.config.masterIP}:${this.config.masterPort}/api/getSlaveMeta` const data = { @@ -31,12 +86,12 @@ class ResearchSync { } needle.post(url, data, options, (err, res, techs) => { if (err) { - console.log(`Can't get own slave data:`) - console.error(err) + this.log(`Can't get own slave data:`) + this.error(err) return } if (res.statusCode === 404) { - console.log('researchSync: slave is not registered yet. Delaying for 5 secs') + this.log('slave is not registered yet. Delaying for 5 secs') setTimeout( () => this.initial_request_own_data(callback), 5000 @@ -44,14 +99,14 @@ class ResearchSync { return } if (res.statusCode !== 200) { - console.log(`Can't get own slave data:`) - console.error(`status code ${res.statusCode}, ${res.body}`) + this.log(`Can't get own slave data:`) + this.error(`status code ${res.statusCode}, ${res.body}`) return } techs = JSON.parse(techs) if (typeof techs.research === 'object') this.research = techs.research - console.log('researchSync: techs imported from master') + this.log('techs imported from master') callback() }) } @@ -103,13 +158,10 @@ class ResearchSync { instanceID: this.config.unique, password: this.config.clientPassword, meta: {research: this.research} - }, {headers: {'x-access-token': this.config.masterAuthToken}, json: true}, function (err, resp) { - // it doesn't get called + }, {headers: {'x-access-token': this.config.masterAuthToken}, json: true}, (err, resp) => { if (err) - console.error(err) - else - console.log('sent') - }); + this.error(err) + }) } clear_contribution_to_researched_techs() { @@ -234,7 +286,7 @@ class ResearchSync { let log_message = tech.infinite ? `Unlocking infinite research ${name} at level ${this.research[name].level}` : `Unlocking research ${name}` - console.log(log_message); + this.log(log_message); this.messageInterface(log_message); this.research[name] = tech; } @@ -252,7 +304,7 @@ class ResearchSync { command = command.replace(/{last_check_progress}/g, progress) command = command.replace(/{new_progress}/g, tech.progress) this.messageInterface(command); - console.log( + this.log( `Updating ${name}: ${this.research[name].progress} += ${tech.progress - this.research[name].progress}` ); this.research[name].progress = tech.progress @@ -267,7 +319,7 @@ class ResearchSync { continue let diff = this.research[name].contribution - this.prev_research[name].contribution if (Math.abs(diff) > Number.EPSILON * 1000) - console.log(`Own research ${name}: ${this.research[name].progress} += ${diff}`) + this.log(`Own research ${name}: ${this.research[name].progress} += ${diff}`) } } From ee6ab6ab721367df1f8076c75984a1e7842662e4 Mon Sep 17 00:00:00 2001 From: AreYouScared Date: Fri, 19 Jul 2019 06:05:03 -0400 Subject: [PATCH 16/19] Remove GPS pings in crosschat (#172) * Remove GPS pings in crosschat Cross-server gps pings are annoying, lets fix that * Lets just do it the right way <3 Should be better * Take 2 Let's try that again --- sharedPlugins/globalChat/index.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js index 65fe7738..99622368 100644 --- a/sharedPlugins/globalChat/index.js +++ b/sharedPlugins/globalChat/index.js @@ -22,12 +22,15 @@ module.exports = class remoteCommands { words.shift(); words.shift(); words.shift(); - let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout", ""); + let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout ","").replace(/\[gps=.*?\]/g, "").replace(/\[train=.*?\]/g, ""); - let instance = await this.getInstanceName(data.instanceID); - - this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')"); - } + let instance = await this.getInstanceName(data.instanceID); + if(data.data.includes("[gps=")){ + //Nothing + }else{ + this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')"); + } + } } }); } From 49611a2062d70b20ffd45663c901929015601546 Mon Sep 17 00:00:00 2001 From: Master-Guy <566429+Master-Guy@users.noreply.github.com> Date: Fri, 19 Jul 2019 14:37:49 +0200 Subject: [PATCH 17/19] /api/slaves POST (optimized) (#176) * Adds a /api/slaves POST call that requires authentication. It also returns the rconPasswords, unlike the GET call. --- master.js | 55 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/master.js b/master.js index b87b7d27..7c4725e0 100644 --- a/master.js +++ b/master.js @@ -440,37 +440,64 @@ app.post("/api/uploadMod", authenticate.middleware, function(req,res) { } }); /** +Prepare the slaveCache for both /api/slaves calls +*/ +let slaveCache = { + timestamp: Date.now(), +}; +function getSlaves() { + if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000) { + let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); + slaveCache.cache = copyOfSlaves; + slaveCache.timestamp = Date.now(); + } + return slaveCache; +} +/** GET endpoint for getting information about all our slaves @memberof clusterioMaster @instance @alias /api/slaves */ -let slaveCache = { - timestamp: Date.now(), -}; app.get("/api/slaves", function(req, res) { endpointHitCounter.labels(req.route.path).inc(); res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){ - let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); - // filter out the rcon password because thats kindof not a safe thing to share - for(key in copyOfSlaves){ - copyOfSlaves[key].rconPassword = "hidden"; - } - slaveCache.cache = copyOfSlaves; - slaveCache.timestamp = Date.now(); + let slaveCache = getSlaves(); + for(key in slaveCache) { + slaveCache[key].rconPassword = "hidden"; } - res.send(slaveCache.cache); }); -var recievedItemStatisticsBySlaveID = {}; -var sentItemStatisticsBySlaveID = {}; +/** +POST endpoint for getting information about all our slaves, requires auth, responds including RCON passwords +@memberof clusterioMaster +@instance +@alias /api/slaves +*/ +app.post("/api/slaves", function(req, res) { + let token = req.headers['x-access-token']; + if(!token) { + return res.status(401).send({ auth: false, message: 'No token provided.' }); + } + jwt.verify(token, config.masterAuthSecret, function(err, decoded) { + if(err) { + return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' }); + } + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + let slaveCache = getSlaves(); + res.send(slaveCache.cache); + }); +}); /* var recievedItemStatistics = new averagedTimeSeries({ maxEntries: config.itemStats.maxEntries, entriesPerSecond: config.itemStats.entriesPerSecond, }, console.log);*/ +var recievedItemStatisticsBySlaveID = {}; +var sentItemStatisticsBySlaveID = {}; // /** POST endpoint for storing items in master's inventory. From d909b4aa47bd41fe05a53b378a96c36586e8e236 Mon Sep 17 00:00:00 2001 From: Master-Guy <566429+Master-Guy@users.noreply.github.com> Date: Sat, 20 Jul 2019 14:33:54 +0200 Subject: [PATCH 18/19] Fix for /issues/177 (#178) /api/editSlaveMeta: Removed unneeded rcon check /api/getSlaveMeta: Removed useless password check /api/checkMod: Added authenticate.middleware /api/slaves: Renamed to /api/rconPasswords and changed to authenticate.middleware /api/runCommand: Changed to authenticate.middleware --- master.js | 107 ++++++++++++++++++++++++------------------------------ 1 file changed, 47 insertions(+), 60 deletions(-) diff --git a/master.js b/master.js index 7c4725e0..4ae98038 100644 --- a/master.js +++ b/master.js @@ -352,15 +352,16 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) { // request.body should be an object // {instanceID, pass, meta:{x,y,z}} - if(req.body && req.body.instanceID && req.body.password && req.body.meta){ - // check for editing permissions - if(slaves[req.body.instanceID] && slaves[req.body.instanceID].rconPassword == req.body.password){ - if(!slaves[req.body.instanceID].meta){ + if(req.body && req.body.instanceID && req.body.meta){ + if(slaves[req.body.instanceID]){ + if(!slaves[req.body.instanceID].meta) { slaves[req.body.instanceID].meta = {}; } slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true}); let metaPortion = JSON.stringify(req.body.meta); - if(metaPortion.length > 50) metaPortion = metaPortion.substring(0,20) + "..."; + if(metaPortion.length > 50) { + metaPortion = metaPortion.substring(0,20) + "..."; + } // console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion); res.sendStatus(200); } else { @@ -378,7 +379,7 @@ POST Get metadata from a single slave. For whenever you find the data returned b app.post("/api/getSlaveMeta", function (req, res) { endpointHitCounter.labels(req.route.path).inc(); console.log("body", req.body); - if(req.body && req.body.instanceID && req.body.password){ + if(req.body && req.body.instanceID){ if (slaves[req.body.instanceID]) { console.log("returning meta for ", req.body.instanceID); res.send(JSON.stringify(slaves[req.body.instanceID].meta)) @@ -399,7 +400,7 @@ POST Check if a mod has been uploaded to the master before. Only checks against @instance @alias /api/checkMod */ -app.post("/api/checkMod", function(req,res) { +app.post("/api/checkMod", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); let files = fs.readdirSync(masterModFolder); let found = false; @@ -470,26 +471,19 @@ app.get("/api/slaves", function(req, res) { res.send(slaveCache.cache); }); /** -POST endpoint for getting information about all our slaves, requires auth, responds including RCON passwords +POST endpoint for getting the rconPasswords of all our slaves @memberof clusterioMaster @instance -@alias /api/slaves +@alias /api/rconPasswords */ -app.post("/api/slaves", function(req, res) { - let token = req.headers['x-access-token']; - if(!token) { - return res.status(401).send({ auth: false, message: 'No token provided.' }); - } - jwt.verify(token, config.masterAuthSecret, function(err, decoded) { - if(err) { - return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' }); - } - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let slaveCache = getSlaves(); - res.send(slaveCache.cache); - }); +app.post("/api/rconPasswords", authenticate.middleware, function(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + let slaveCache = getSlaves(); + let serverInfo = []; + serverInfo[key] = {id: key, rconPassword: slaveCache[key].rconPassword}; + res.send(serverInfo); }); /* var recievedItemStatistics = new averagedTimeSeries({ @@ -730,45 +724,38 @@ Requires x-access-token header to be set. Find you api token in secret-api-token @param {object} JSON {instanceID:19412312, broadcast:false, command:"/c game.print('hello')"} @returns {object} Status {auth: bool, message: "Informative error", data:{}} */ -app.post("/api/runCommand", (req,res) => { +app.post("/api/runCommand", authenticate.middleware, function(req,res) { endpointHitCounter.labels(req.route.path).inc(); - var token = req.headers['x-access-token']; - if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' }); - - jwt.verify(token, config.masterAuthSecret, function(err, decoded) { - if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' }); - - // validate request.body - let body = req.body; + // validate request.body + let body = req.body; - if ( - body.broadcast - && body.command - && typeof body.command == "string" - && body.command[0] == "/") - { - let instanceResponses = []; - for (let instanceID in wsSlaves) { - // skip loop if the property is from prototype - if (!wsSlaves.hasOwnProperty(instanceID)) continue; - wsSlaves[instanceID].runCommand(body.command); - } - res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"}); - } else if ( - body.instanceID - && wsSlaves[body.instanceID] - && body.command - && typeof body.command == "string" - && body.command[0] == "/") - { - // execute command - wsSlaves[body.instanceID].runCommand(body.command, data => { - res.status(200).send({auth: true, message: "success", data}); - }); - } else { - res.status(400).send({auth: true, message: "Error: invalid request.body"}); + if ( + body.broadcast + && body.command + && typeof body.command == "string" + && body.command[0] == "/") + { + let instanceResponses = []; + for (let instanceID in wsSlaves) { + // skip loop if the property is from prototype + if (!wsSlaves.hasOwnProperty(instanceID)) continue; + wsSlaves[instanceID].runCommand(body.command); } - }); + res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"}); + } else if ( + body.instanceID + && wsSlaves[body.instanceID] + && body.command + && typeof body.command == "string" + && body.command[0] == "/") + { + // execute command + wsSlaves[body.instanceID].runCommand(body.command, data => { + res.status(200).send({auth: true, message: "success", data}); + }); + } else { + res.status(400).send({auth: true, message: "Error: invalid request.body"}); + } }); /** GET endpoint. Returns factorio's base locale as a JSON object. From e2775adcc3a4c26777f4d40bb7b5d50139212d1d Mon Sep 17 00:00:00 2001 From: Master-Guy <566429+Master-Guy@users.noreply.github.com> Date: Sat, 20 Jul 2019 15:52:47 +0200 Subject: [PATCH 19/19] /api/rconPasswords fix & /api/slaves security fix (#179) Fix for /api/slaves to not send rconPasswords Fix for /api/rconPasswords to work properly and send partial information --- master.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/master.js b/master.js index 4ae98038..f92e3dd8 100644 --- a/master.js +++ b/master.js @@ -465,8 +465,8 @@ app.get("/api/slaves", function(req, res) { res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); let slaveCache = getSlaves(); - for(key in slaveCache) { - slaveCache[key].rconPassword = "hidden"; + for(key in slaveCache.cache) { + slaveCache.cache[key].rconPassword = "hidden"; } res.send(slaveCache.cache); }); @@ -481,9 +481,11 @@ app.post("/api/rconPasswords", authenticate.middleware, function(req, res) { res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); let slaveCache = getSlaves(); - let serverInfo = []; - serverInfo[key] = {id: key, rconPassword: slaveCache[key].rconPassword}; - res.send(serverInfo); + let validKeys = ['rconPort', 'rconPassword', 'publicIP']; + for(key in slaveCache.cache) { + slaveCache.cache[key] = {'publicIP': slaveCache.cache[key].publicIP, 'rconPort': slaveCache.cache[key].rconPort, 'rconPassword': slaveCache.cache[key].rconPassword}; + } + res.send(slaveCache.cache); }); /* var recievedItemStatistics = new averagedTimeSeries({