Merge pull request #9 from Danielv123/master

sync

.gitignore

@@ -64,3 +64,6 @@ sharedPlugins/scenario_loader/scenarios
 sharedPlugins/*/DISABLED
 
 \.cache-require-paths\.json
+
+# IDE
+.idea/

client.js

@@ -317,6 +317,7 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
 		"clientPort": args["rcon-port"] || process.env.RCONPORT || Math.floor(Math.random() * 65535),
 		"__comment_clientPassword": "This is the rcon password. Its also used for making an instanceID. Make sure its unique and not blank.",
 		"clientPassword": args["rcon-password"] || Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8),
+		"info": {}
 	}
 	console.log("Clusterio | Created instance with settings:")
 	console.log(instconf);
@@ -673,6 +674,7 @@ async function instanceManagement(instanceconfig) {
 					publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config
 					mods:modHashes,
 					instanceName: instance,
+					info: instanceconfig.info,
 				}
 				if(playerCount){
 					payload.playerCount = playerCount.replace(/(\r\n\t|\n|\r\t)/gm, "");

master.js

@@ -235,7 +235,7 @@ const db = {};
 
 (function(){
 	try{
-		let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json"));
+		let x = fs.statSync(path.resolve(config.databaseDirectory, "slaves.json"));
 		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
 		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json")));
 	} catch (e){
@@ -352,16 +352,18 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 	// request.body should be an object
 	// {instanceID, pass, meta:{x,y,z}}
 	
-	if(req.body && req.body.instanceID && req.body.password && req.body.meta){
-		// check for editing permissions
-		if(slaves[req.body.instanceID] && slaves[req.body.instanceID].rconPassword == req.body.password){
-			if(!slaves[req.body.instanceID].meta){
+	if(req.body && req.body.instanceID && req.body.meta){
+		if(slaves[req.body.instanceID]){
+			if(!slaves[req.body.instanceID].meta) {
 				slaves[req.body.instanceID].meta = {};
 			}
 			slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
 			let metaPortion = JSON.stringify(req.body.meta);
-			if(metaPortion.length > 50) metaPortion = metaPortion.substring(0,20) + "...";
+			if(metaPortion.length > 50) {
+                metaPortion = metaPortion.substring(0,20) + "...";
+            }
 			// console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion);
+			res.sendStatus(200);
 		} else {
 			res.send("ERROR: Invalid instanceID or password")
 		}
@@ -377,9 +379,14 @@ POST Get metadata from a single slave. For whenever you find the data returned b
 app.post("/api/getSlaveMeta", function (req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	console.log("body", req.body);
-    if(req.body && req.body.instanceID && req.body.password){
-    	console.log("returning meta for ", req.body.instanceID);
-    	res.send(JSON.stringify(slaves[req.body.instanceID].meta))
+    if(req.body && req.body.instanceID){
+    	if (slaves[req.body.instanceID]) {
+			console.log("returning meta for ", req.body.instanceID);
+			res.send(JSON.stringify(slaves[req.body.instanceID].meta))
+		} else {
+    		res.status(404)
+			res.send('{"status": 404, "info": "Slave not registered"}')
+		}
 	} else {
     	res.status(400);
     	res.send('{"INVALID REQUEST":1}');
@@ -393,7 +400,7 @@ POST Check if a mod has been uploaded to the master before. Only checks against
 @instance
 @alias /api/checkMod
 */
-app.post("/api/checkMod", function(req,res) {
+app.post("/api/checkMod", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	let files = fs.readdirSync(masterModFolder);
 	let found = false;
@@ -434,37 +441,59 @@ app.post("/api/uploadMod", authenticate.middleware, function(req,res) {
     }
 });
 /**
+Prepare the slaveCache for both /api/slaves calls
+*/
+let slaveCache = {
+	timestamp: Date.now(),
+};
+function getSlaves() {
+	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000) {
+		let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
+		slaveCache.cache = copyOfSlaves;
+		slaveCache.timestamp = Date.now();
+	}
+	return slaveCache;
+}
+/**
 GET endpoint for getting information about all our slaves
 @memberof clusterioMaster
 @instance
 @alias /api/slaves
 */
-let slaveCache = {
-	timestamp: Date.now(),
-};
 app.get("/api/slaves", function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){
-		let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
-		// filter out the rcon password because thats kindof not a safe thing to share
-		for(key in copyOfSlaves){
-			copyOfSlaves[key].rconPassword = "hidden";
-		}
-		slaveCache.cache = copyOfSlaves;
-		slaveCache.timestamp = Date.now();
+	let slaveCache = getSlaves();
+	for(key in slaveCache.cache) {
+		slaveCache.cache[key].rconPassword = "hidden";
 	}
-	
 	res.send(slaveCache.cache);
 });
-var recievedItemStatisticsBySlaveID = {};
-var sentItemStatisticsBySlaveID = {};
+/**
+POST endpoint for getting the rconPasswords of all our slaves
+@memberof clusterioMaster
+@instance
+@alias /api/rconPasswords
+*/
+app.post("/api/rconPasswords", authenticate.middleware, function(req, res) {
+    endpointHitCounter.labels(req.route.path).inc();
+    res.header("Access-Control-Allow-Origin", "*");
+    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+    let slaveCache = getSlaves();
+    let validKeys = ['rconPort', 'rconPassword', 'publicIP'];
+    for(key in slaveCache.cache) {
+        slaveCache.cache[key] = {'publicIP': slaveCache.cache[key].publicIP, 'rconPort': slaveCache.cache[key].rconPort, 'rconPassword': slaveCache.cache[key].rconPassword};
+    }
+    res.send(slaveCache.cache);
+});
 /*
 var recievedItemStatistics = new averagedTimeSeries({
 	maxEntries: config.itemStats.maxEntries,
 	entriesPerSecond: config.itemStats.entriesPerSecond,
 }, console.log);*/
+var recievedItemStatisticsBySlaveID = {};
+var sentItemStatisticsBySlaveID = {};
 // 
 /**
 POST endpoint for storing items in master's inventory.
@@ -697,45 +726,38 @@ Requires x-access-token header to be set. Find you api token in secret-api-token
 @param {object} JSON {instanceID:19412312, broadcast:false, command:"/c game.print('hello')"}
 @returns {object} Status {auth: bool, message: "Informative error", data:{}}
 */
-app.post("/api/runCommand", (req,res) => {
+app.post("/api/runCommand", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	var token = req.headers['x-access-token'];
-	if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' });
-	
-	jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
-		if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' });
-		
-		// validate request.body
-        let body = req.body;
+    // validate request.body
+    let body = req.body;
 
-        if (
-            body.broadcast
-            && body.command
-            && typeof body.command == "string"
-            && body.command[0] == "/")
-        {
-        	let instanceResponses = [];
-            for (let instanceID in wsSlaves) {
-                // skip loop if the property is from prototype
-                if (!wsSlaves.hasOwnProperty(instanceID)) continue;
-                wsSlaves[instanceID].runCommand(body.command);
-            }
-            res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"});
-        } else if (
-            body.instanceID
-            && wsSlaves[body.instanceID]
-            && body.command
-            && typeof body.command == "string"
-            && body.command[0] == "/")
-        {
-            // execute command
-            wsSlaves[body.instanceID].runCommand(body.command, data => {
-                res.status(200).send({auth: true, message: "success", data});
-            });
-        } else {
-            res.status(400).send({auth: true, message: "Error: invalid request.body"});
+    if (
+        body.broadcast
+        && body.command
+        && typeof body.command == "string"
+        && body.command[0] == "/")
+    {
+        let instanceResponses = [];
+        for (let instanceID in wsSlaves) {
+            // skip loop if the property is from prototype
+            if (!wsSlaves.hasOwnProperty(instanceID)) continue;
+            wsSlaves[instanceID].runCommand(body.command);
         }
-	});
+        res.status(200).send({auth: true, message: "success", response: "Cluster wide messaging initiated"});
+    } else if (
+        body.instanceID
+        && wsSlaves[body.instanceID]
+        && body.command
+        && typeof body.command == "string"
+        && body.command[0] == "/")
+    {
+        // execute command
+        wsSlaves[body.instanceID].runCommand(body.command, data => {
+            res.status(200).send({auth: true, message: "success", data});
+        });
+    } else {
+        res.status(400).send({auth: true, message: "Error: invalid request.body"});
+    }
 });
 /**
 GET endpoint. Returns factorio's base locale as a JSON object.

sharedPlugins/globalChat/index.js

@@ -22,12 +22,15 @@ module.exports = class remoteCommands {
 					words.shift();
 					words.shift();
 					words.shift();
-					let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout", "");
+					                let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout ","").replace(/\[gps=.*?\]/g, "").replace(/\[train=.*?\]/g, "");
 					
-					let instance = await this.getInstanceName(data.instanceID);
-					
-					this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')");
-				}
+                    let instance = await this.getInstanceName(data.instanceID);
+                    if(data.data.includes("[gps=")){
+                    	//Nothing
+                    }else{
+                    	this.messageInterface("/silent-command game.print('[" + instance + "] " + message + "')");
+                    }
+                }
 			}
 		});
 	}

sharedPlugins/researchSync/dumpResearch.lua

@@ -1,13 +1,24 @@
-/silent-command
 
 local data = {};
+local force = game.forces["player"];
 
-for _, tech in pairs(game.forces["player"].technologies) do
-    if tech.enabled == true and tech.prototype.max_level == 4294967295 then
-        table.insert(data, tech.name .. ":" .. tostring(tech.researched) .. ":" .. tostring(tech.level) .. ":true");
+for _, tech in pairs(force.technologies) do
+    local progress;
+    if tech == force.current_research then
+        progress = force.research_progress
     else
-        table.insert(data, tech.name .. ":" .. tostring(tech.researched) .. ":" .. tostring(tech.level) .. ":false");
+        progress = force.get_saved_technology_progress(tech.name);
     end
+    local infinite = tech.enabled == true and tech.prototype.max_level == 4294967295;
+    local tech_data = {
+        tech.name,
+        tostring(tech.researched),
+        tostring(tech.level),
+        tostring(progress),
+        tostring(infinite),
+    }
+    tech_data = table.concat(tech_data, ':')
+    table.insert(data, tech_data);
 end
 
 game.write_file("researchSync.txt", table.concat(data, "\n"), false, 0)

sharedPlugins/researchSync/enableResearch.lua

@@ -1,4 +1,4 @@
-/silent-command
+
 if game.forces['player'].technologies['{tech_name}'] then
     if 1 == {tech_infinite} then
         game.forces['player'].technologies['{tech_name}'].level = {tech_level}

sharedPlugins/researchSync/index.js

@@ -1,162 +1,372 @@
-const fs     = require('fs');
+const fs = require('fs');
+const path = require('path')
 const needle = require("needle");
 
+
+function time() {
+    let d = new Date()
+    return `${d.getUTCHours()}:${d.getUTCMinutes()}:${d.getUTCSeconds()}.${d.getUTCMilliseconds()}`
+}
+
 class ResearchSync {
-    constructor(slaveConfig, messageInterface, extras = {}) {
-        this.config           = slaveConfig;
-        this.messageInterface = messageInterface;
-        this.functions        = this.loadFunctions();
+    constructor(slaveConfig, messageInterface, extras = {}){
+        this.config = slaveConfig
+        this.messageInterface = messageInterface
 
-        this.research = {};
+        this.functions = {
+            dumpResearch: this.loadFunc("dumpResearch.lua"),
+			enableResearch: this.loadFunc("enableResearch.lua"),
+            updateProgress: this.loadFunc("updateProgress.lua")
+        }
 
-        setInterval(
-            () => {
-                this.pollResearch();
-                setTimeout(this.doSync.bind(this), 5000);
-            },
-            extras.researchSyncPollInterval || 30000
-        );
+        this.log_folder = './logs'
+        if (!fs.existsSync(this.log_folder))
+            fs.mkdirSync(this.log_folder);
+        this.log_file = path.join(this.log_folder, `${this.config.unique}-research.log`)
+        this.get_node_name_and_move_log()
+
+        this.research = {}
+        this.prev_research = {}
+        this.initial_request_own_data(
+            () => this.setup_sync_task(extras)
+        )
     }
 
-    pollResearch() {
-        // this.messageInterface("Polling Research\n")
-        this.messageInterface(this.functions.dumpResearch);
+    log(data) {
+        try {
+            console.log(`researchSync: ${data}`)
+            fs.appendFileSync(this.log_file, `${time()}: ${data}\n`);
+        } catch (e) {
+            console.error(e)
+        }
     }
 
-    doSync() {
-        needle.get(this.config.masterIP + ':' + this.config.masterPort + '/api/slaves', (err, resp, slaveData) => {
+    error(data) {
+        try {
+            console.error(`researchSync: ${data}`)
+            fs.appendFileSync(this.log_file, `${time()}: ${data}\n`);
+        } catch (e) {
+            console.error(e)
+        }
+    }
+
+    get_node_name_and_move_log() {
+        const url = `${this.config.masterIP}:${this.config.masterPort}/api/slaves`
+        needle.get(url, (err, res, slaves_data) => {
+            if (err)
+                return this.error(err)
+
+            slaves_data = Object.values(slaves_data)
+            let node_data = slaves_data.find(
+                slave_data => slave_data.unique === this.config.unique.toString()
+            )
+            if (node_data === undefined)
+                return this.error('slave was not found')
+            let node_name = node_data.instanceName
+            let log_file = path.join(this.log_folder, `${node_name}-research.log`)
+            if (fs.existsSync(log_file)) {
+                fs.appendFileSync(log_file, fs.readFileSync(this.log_file));
+                fs.unlinkSync(this.log_file)
+            } else {
+                fs.renameSync(this.log_file, log_file)
+            }
+            this.log_file = log_file
+        })
+    }
+
+    initial_request_own_data(callback) {
+        const url = `${this.config.masterIP}:${this.config.masterPort}/api/getSlaveMeta`
+        const data = {
+            instanceID: this.config.unique,
+            password: this.config.clientPassword,
+        }
+        const options = {
+            headers: {'x-access-token': this.config.masterAuthToken},
+            json: true
+        }
+        needle.post(url, data, options, (err, res, techs) => {
             if (err) {
-                this.messageInterface("Unable to post JSON master/api/slaves, master might be unreachable");
-                return false;
+                this.log(`Can't get own slave data:`)
+                this.error(err)
+                return
             }
-            if (resp.statusCode !== 200) {
-                this.messageInterface("got error when calling slaves", resp.statusCode, resp.body);
-                return;
+            if (res.statusCode === 404) {
+                this.log('slave is not registered yet. Delaying for 5 secs')
+                setTimeout(
+                    () => this.initial_request_own_data(callback),
+                    5000
+                )
+                return
             }
-
-            let needResearch = {};
-
-            Object.keys(slaveData).forEach(instanceKey => {
-                if (slaveData[instanceKey].unique === this.config.unique.toString()) {
-                    return;
-                }
-                if (!slaveData[instanceKey].hasOwnProperty('meta')
-                    || !slaveData[instanceKey].meta.hasOwnProperty('research')
-                ) {
-                    return;
-                }
-                let researchList = slaveData[instanceKey].meta.research;
-                if (researchList) {
-                    Object.keys(researchList).forEach(researchName => {
-                        if (
-                            isNaN(researchList[researchName].researched)
-                            || isNaN(researchList[researchName].level)
-                            || isNaN(researchList[researchName].infinite))
-                        {
-                            return;
-                        }
-                        if (needResearch.hasOwnProperty(researchName)) {
-                            if (
-                                needResearch[researchName].infinite === 1
-                                && needResearch[researchName].level < parseInt(researchList[researchName].level)
-                            ) {
-                                needResearch[researchName].level = parseInt(researchList[researchName].level);
-                            } else if (
-                                needResearch[researchName].researched === 0
-                                && parseInt(researchList[researchName].researched) === 1
-                            ) {
-                                needResearch[researchName].researched = 1
-                            }
-                        } else {
-                            needResearch[researchName] = {
-                                researched: parseInt(researchList[researchName].researched),
-                                level     : parseInt(researchList[researchName].level),
-                                infinite  : parseInt(researchList[researchName].infinite)
-                            };
-                        }
-                    });
-                }
-            });
-
-            let difference = this.filterResearchDiff(this.research, needResearch);
-
-            let count = Object.keys(difference).length;
-            let counter = 1;
-            Object.keys(difference).forEach((key) => {
-                if (this.research[key]) {
-                    let command = this.functions.enableResearch;
-                    while (command.includes("{tech_name}")) {
-                        command = command.replace("{tech_name}", key);
-                        command = command.replace("{tech_researched}", difference[key].researched);
-                        command = command.replace("{tech_level}", difference[key].level);
-                        command = command.replace("{tech_infinite}", difference[key].infinite);
-                        command = command.replace("{notify}", count > 1 || count === counter ? 'false' : 'true');
-                    }
-                    this.messageInterface(command);
-                    let log_message = '';
-                    if (difference[key].infinite === 1) {
-                        log_message = 'Unlocking infinite research ' + key + 'at level ' + this.research[key].level;
-                    }  else {
-                        log_message = 'Unlocking research ' + key;
-                    }
-                    console.log(log_message);
-                    this.messageInterface(log_message);
-                    this.research[key] = difference[key];
-                    ++counter;
-                }
-            });
-
-            needle.post(this.config.masterIP + ':' + this.config.masterPort + '/api/editSlaveMeta', {
-                instanceID: this.config.unique,
-                password  : this.config.clientPassword,
-                meta      : {research: this.research}
-            }, {headers: {'x-access-token': this.config.masterAuthToken}}, function (err, resp) {
-                // success?
-            });
-        });
+            if (res.statusCode !== 200) {
+                this.log(`Can't get own slave data:`)
+                this.error(`status code ${res.statusCode}, ${res.body}`)
+                return
+            }
+            techs = JSON.parse(techs)
+            if (typeof techs.research === 'object')
+                this.research = techs.research
+            this.log('techs imported from master')
+            callback()
+        })
     }
 
-    filterResearchDiff(localResearch, remoteResearch) {
-        let diff = {};
-        Object.keys(localResearch).forEach((key) => {
-            if (remoteResearch.hasOwnProperty(key)) {
-                if (
-                    (localResearch[key].infinite === 0 && localResearch[key].researched === 0 && remoteResearch[key].researched === 1)
-                    ||
-                    (localResearch[key].infinite === 1 && localResearch[key].level < remoteResearch[key].level)
-                ) {
-                    if (!isNaN(remoteResearch[key].researched) && !isNaN(remoteResearch[key].level) && !isNaN(remoteResearch[key].infinite)) {
-                        diff[key] = {researched: remoteResearch[key].researched, level: remoteResearch[key].level, infinite: remoteResearch[key].infinite};
+    setup_sync_task(extras) {
+        const timeout = extras.researchSyncPollInterval || 5000
+        setInterval(() => this.sync_task(), timeout);
+    }
+
+    sync_task() {
+        this.messageInterface(this.functions.dumpResearch);
+        setTimeout(this.request_cluster_data.bind(this), 2000);
+    }
+
+    request_cluster_data() {
+        const slaves_data_url = `${this.config.masterIP}:${this.config.masterPort}/api/slaves`
+        needle.get(slaves_data_url, this.sync_researches.bind(this))
+    }
+
+    sync_researches(err, resp, slaves_data) {
+        if (err) {
+            this.messageInterface("Unable to post JSON master/api/slaves, master might be unreachable");
+            return false;
+        }
+        if (resp.statusCode !== 200) {
+            this.messageInterface("got error when calling slaves", resp.statusCode, resp.body);
+            return;
+        }
+
+        slaves_data = Object.values(slaves_data)
+        slaves_data = slaves_data.filter(
+            slave_data => slave_data.unique !== this.config.unique.toString()
+                && slave_data.meta && slave_data.meta.research
+        )
+
+        this.clear_contribution_to_researched_techs()
+        let cluster_techs = this.get_cluster_techs(slaves_data)
+        this.recount_cluster_research_progress(slaves_data, cluster_techs)
+
+        let to_research = this.filter_researched_techs(cluster_techs)
+        let to_update_progress = this.filter_updated_techs(cluster_techs, to_research)
+
+        this.research_technologies(to_research)
+        this.update_technologies_progress(to_update_progress)
+
+        this.print_own_contribution()
+
+        needle.post(this.config.masterIP + ':' + this.config.masterPort + '/api/editSlaveMeta', {
+            instanceID: this.config.unique,
+            password: this.config.clientPassword,
+            meta: {research: this.research}
+        }, {headers: {'x-access-token': this.config.masterAuthToken}, json: true}, (err, resp) => {
+            if (err)
+                this.error(err)
+        })
+    }
+
+    clear_contribution_to_researched_techs() {
+        // if between updates tech was researched
+        if (!this.prev_research)
+            return
+        for (let [name, research] of Object.entries(this.research)) {
+            let researched
+            if (research.infinite)
+                researched = this.prev_research[name].level < research.level
+            else
+                researched = this.prev_research[name].researched < research.researched
+
+            if (researched)
+                research.contribution = 0
+        }
+    }
+
+    get_cluster_techs(slavesData) {
+        let cluster_techs = {}
+        for (let slave_data of slavesData) {
+            let node_researches = slave_data.meta.research
+            for (let [name, node_tech] of Object.entries(node_researches)) {
+                if (isNaN(node_tech.researched) || isNaN(node_tech.level) || isNaN(node_tech.infinite))
+                    continue
+
+                if (cluster_techs[name]) {
+                    if (cluster_techs[name].infinite === 1 && cluster_techs[name].level < node_tech.level) {
+                        cluster_techs[name].level = node_tech.level
+                    } else if (node_tech.researched > cluster_techs[name].researched) {
+                        cluster_techs[name].researched = 1
                     }
+                } else {
+                    cluster_techs[name] = node_tech
                 }
             }
-        });
-        return diff;
+        }
+        return cluster_techs
     }
 
-    loadFunctions() {
-        return {
-            dumpResearch  : this.loadFunc("dumpResearch.lua"),
-            enableResearch: this.loadFunc("enableResearch.lua"),
-        };
+    recount_cluster_research_progress(slaves_data, cluster_researches) {
+        for (let [name, research] of Object.entries(cluster_researches))
+            research.progress = this.research[name].contribution
+
+        for (let slave_data of slaves_data) {
+            for (let [name, research] of Object.entries(slave_data.meta.research)) {
+                if (!cluster_researches[name])
+                    continue
+                if (isNaN(cluster_researches[name].progress)
+                    || isNaN(research.contribution)
+                    || isNaN(research.level)
+                    || isNaN(cluster_researches[name].level))
+                    continue
+                if (cluster_researches[name].level === research.level)
+                    cluster_researches[name].progress += research.contribution
+            }
+        }
+
+        for (let [name, research] of Object.entries(cluster_researches)) {
+            if (research.progress > 1) {
+                research.progress = null
+                research.researched = 1
+                this.research[name].contribution = 0
+                research.contribution = 0
+                if (this.research[name].level >= research.level)
+                    research.level = this.research[name].level + 1
+            }
+        }
     }
 
-    loadFunc(path) {
-        return fs.readFileSync("sharedPlugins/researchSync/" + path, 'utf-8').replace(/\r?\n|\r/g, ' ');
+    filter_researched_techs(cluster_researches) {
+        let local_researches = this.research
+        let result = {};
+        for (let key in local_researches) {
+            if (!cluster_researches[key])
+                continue
+            if (isNaN(cluster_researches[key].researched) || isNaN(cluster_researches[key].level))
+                continue
+
+            let researched
+            if (local_researches[key].infinite)
+                researched = local_researches[key].level < cluster_researches[key].level
+            else
+                researched = local_researches[key].researched < cluster_researches[key].researched
+
+            if (researched)
+                result[key] = cluster_researches[key]
+        }
+        return result;
     }
 
+    filter_updated_techs(cluster_techs, to_research) {
+        let local_techs = this.research
+        let result = {}
+        for (let name in local_techs) {
+            if (name in to_research || !cluster_techs[name])
+                continue
+            if (isNaN(cluster_techs[name].progress))
+                continue
+            if (local_techs[name].progress < cluster_techs[name].progress)
+                result[name] = cluster_techs[name]
+        }
+        return result
+    }
+
+    research_technologies(to_research) {
+        for (let name of Object.keys(to_research))
+            if (!this.research[name])
+                delete to_research[name]
+
+        const notify = Object.keys(to_research).length === 1
+        for (let [name, tech] of Object.entries(to_research)) {
+            this.research[name].contribution = 0
+            this.research[name].progress = null
+            let command = this.functions.enableResearch;
+            command = command.replace(/{tech_name}/g, name);
+            command = command.replace(/{tech_researched}/g, tech.researched);
+            command = command.replace(/{tech_level}/g, tech.level);
+            command = command.replace(/{tech_infinite}/g, tech.infinite);
+            command = command.replace(/{notify}/g, notify);
+            this.messageInterface(command);
+            let log_message = tech.infinite
+                ? `Unlocking infinite research ${name} at level ${this.research[name].level}`
+                : `Unlocking research ${name}`
+            this.log(log_message);
+            this.messageInterface(log_message);
+            this.research[name] = tech;
+        }
+    }
+
+    update_technologies_progress(to_update) {
+        for (let [name, tech] of Object.entries(to_update)) {
+            if (!this.research[name])
+                continue
+            let progress = this.research[name].progress
+            if (progress === null)
+                progress = 'nil'
+            let command = this.functions.updateProgress
+            command = command.replace(/{tech_name}/g, name)
+            command = command.replace(/{last_check_progress}/g, progress)
+            command = command.replace(/{new_progress}/g, tech.progress)
+            this.messageInterface(command);
+            this.log(
+                `Updating ${name}: ${this.research[name].progress} += ${tech.progress - this.research[name].progress}`
+            );
+            this.research[name].progress = tech.progress
+        }
+    }
+
+    print_own_contribution() {
+        if (!this.prev_research)
+            return
+        for (let [name, tech] of Object.entries(this.research)) {
+            if (!this.prev_research[name])
+                continue
+            let diff = this.research[name].contribution - this.prev_research[name].contribution
+            if (Math.abs(diff) > Number.EPSILON * 1000)
+                this.log(`Own research ${name}: ${this.research[name].progress} += ${diff}`)
+        }
+    }
+
+    loadFunc(path, silent=true) {
+        let command = fs.readFileSync("sharedPlugins/researchSync/" + path,'utf-8')
+        command = command.replace(/\r?\n|\r/g,' ')
+        command = (silent ? '/silent-command ' : '/c ') + command
+        return command;
+    }
     scriptOutput(data) {
-        let kv         = data.split(":");
-        let name       = kv[0];
-        let researched = ('true' !== kv[1]
-            ? 0
-            : 1);
-        let level      = parseInt(kv[2]);
-        let infinite   = ('true' !== kv[3]
-            ? 0
-            : 1);
-        if (!isNaN(level) && !isNaN(researched)) {
-            this.research[name] = {researched: researched, level: level, infinite: infinite};
+        let [name, researched, level, progress, infinite] = data.split(":")
+        researched = +(researched === 'true')
+        infinite = +(infinite === 'true')
+        level = parseInt(level);
+        if (progress === 'nil')
+            progress = null
+        else
+            progress = parseFloat(progress)
+
+        if (isNaN(level) || isNaN(researched))
+            return
+        this.prev_research[name] = this.research[name]
+        if (!this.prev_research[name]) {
+            this.prev_research[name] = {
+                researched: null,
+                level: null,
+                progress: null,
+                contribution: 0,
+                infinite,
+            }
+        }
+        this.research[name] = {
+            researched,
+            level,
+            progress,
+            contribution: this.prev_research[name].contribution,
+            infinite
+        }
+        if (this.prev_research[name].progress && this.research[name].progress) {
+            // this.prev_research[name].progress gets updated to overall cluster progress
+            // therefore contribution should be own research progress change over sync interval
+            let contribution = this.research[name].progress - this.prev_research[name].progress
+            this.research[name].contribution += contribution
+        }
+        if (Math.abs(this.research[name].contribution) < Number.EPSILON * 1000) {
+            // if contribution should be 0 but because of floating-point precision is e.g. 2.2564e-18
+            this.research[name].contribution = 0
         }
     }
 }

sharedPlugins/researchSync/index.spec.js

@@ -9,7 +9,7 @@ describe("researchSync/index.js", ()=>{
 		assert.equal(typeof researchSync, "function");
 	});
 	describe("class researchSync()", ()=>{
-		it(".filterResearchDiff(object1, object2) returns research that needs enabling on the local instance", ()=>{
+		it(".filter_researched_techs(object1, object2) returns research that needs enabling on the local instance", ()=>{
 			let reSync = new researchSync({}, function(){});
 			
 			let obj1 = {
@@ -24,9 +24,9 @@ describe("researchSync/index.js", ()=>{
 				c:{researched: false, level: 0},
 				e:{researched: false, level: 50},
 			};
-			assert(typeof reSync.filterResearchDiff == "function");
+			assert(typeof reSync.filter_researched_techs == "function");
 			
-			let diffResult = reSync.filterResearchDiff(obj1, obj2);
+			let diffResult = reSync.filter_researched_techs(obj1, obj2);
 			
 			assert(diffResult.a === undefined);
 			

sharedPlugins/researchSync/updateProgress.lua

@@ -0,0 +1,33 @@
+
+local function get_tech_progress(force, tech)
+    if tech == force.current_research then
+        return force.research_progress
+    else
+        return force.get_saved_technology_progress(tech.name);
+    end
+end
+local function set_tech_progress(force, tech, progress)
+    if tech == force.current_research then
+        force.research_progress = progress
+    else
+        force.set_saved_technology_progress(tech.name, progress);
+    end
+end
+
+local force = game.forces['player']
+local tech_name = '{tech_name}'
+local tech = force.technologies[tech_name]
+local last_check_progress = {last_check_progress}
+local new_progress = {new_progress}
+
+local progress_delta = get_tech_progress(force, tech)
+if last_check_progress ~= nil then
+    progress_delta = progress_delta - last_check_progress
+end
+if progress_delta ~= nil then
+    new_progress = new_progress + progress_delta
+end
+if new_progress > 1 then
+    new_progress = 1
+end
+set_tech_progress(force, tech, new_progress)

sharedPlugins/statisticsExporter/index.js

@@ -20,13 +20,19 @@ module.exports = class remoteCommands {
 		setInterval(async () => {
 			let stats = await this.getStats();
 			stats.forEach(force => {
-				needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, {
+				// I have no idea why I need to do JSON.stringify before passing it to needle.post,
+				// but if you do forceData: force.data, it doesn't work...
+				let request = {
 					instanceID: this.config.unique,
 					force: force.forceName,
-					data: force.data,
-				}, needleOptionsWithTokenAuthHeader, (err, resp) => {
-					if(!err && resp.body && resp.body.ok == false) console.log(resp.body);
-				});
+					forceData: JSON.stringify(force.data),
+				};
+				needle.post(`${mergedConfig.masterIP}:${mergedConfig.masterPort}/api/logStats`, 
+					request, 
+					needleOptionsWithTokenAuthHeader, (err, resp) => {
+						if(!err && resp.body && resp.body.ok == false) console.log("Error calling /api/logStats: " + resp.body);
+					}
+				);
 			});
 		}, 15000);
 	}

sharedPlugins/statisticsExporter/masterPlugin.js

@@ -26,18 +26,22 @@ class masterPlugin {
 		@returns {string} failure
 		*/
 		app.post("/api/logStats", authenticate.middleware, function(req,res) {
+			let data = req.body.forceData;
+			if(typeof req.body.forceData == "string") {
+				data = JSON.parse(req.body.forceData);
+			}
 			// endpointHitCounter.labels(req.route.path).inc();
 			if(typeof req.body == "object"
 			&& req.body.instanceID
 			&& typeof req.body.instanceID == "string"
 			&& req.body.instanceID.length < 100
-			&& req.body.data
-			&& typeof req.body.data == "object"
-			&& !Array.isArray(req.body.data)){
+			&& data
+			&& typeof data == "object"
+			&& !Array.isArray(data)){
 				try{
-					Object.keys(req.body.data).forEach(statisticName => {
-						Object.keys(req.body.data[statisticName]).forEach(direction => {
-							let statHash = req.body.data[statisticName][direction];
+					Object.keys(data).forEach(statisticName => {
+						Object.keys(data[statisticName]).forEach(direction => {
+							let statHash = data[statisticName][direction];
 							Object.keys(statHash).forEach(item => {
 								let value = Number(statHash[item]);
 								if(value) prometheusStatisticsGauge.labels(req.body.instanceID, req.body.force, statisticName, direction, item).set(value);
@@ -50,6 +54,8 @@ class masterPlugin {
 					console.log(e)
 				};
 			} else {
+				console.log(`/api/logStats: Invalid request. instanceID: ${req.body.instanceID}, dataIsArray: `
+					+ `${Array.isArray(data)}, ${typeof req.body.instanceID}, ${typeof data}`);
 				res.send({
 					ok:false,
 					msg:"Invalid request parameters"