Merge branch 'neuralDoleDivision'

config.json.dist

@@ -47,7 +47,9 @@
 		"entriesPerSecond": 0.01666666667
 	},
 	"logItemTransfers": false,
+	"__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.",
 	"disableFairItemDistribution": false,
+	"useNeuralNetDoleDivider": true,
 	"uploadModsToMaster": true,
 	"msBetweenCommands": 10,
 	"allowRemoteCommandExecution": true,

lib/dole_nn_base.js

@@ -0,0 +1,139 @@
+//Sigmoids...
+
+function sigmoid0(x)
+{
+	return x/(1+Math.abs(x));
+}
+function sigmoid1(x)
+{
+	if (x>=0.0) {
+		return 1.0
+	} else {
+		return 0.0
+	}
+}
+function sigmoid2(x)
+{
+	if (x>=0.0){
+		return x
+	} else {
+		return 0.0
+	}
+}
+function sigmoid3(x)
+{
+	return x;
+}
+
+//DOSE - Doses items for item X slave Y
+//Numreq - number of items that are requested for item X for slave Y
+//Instore - number of items for item X in master gauge (currently)
+//store_last_tick - Number of items for item X in master gauge at time of last tick
+//dole - Current dole for item X
+//carry - carry for item X slave Y
+//prev_req - Numreq for previous request for item X slave Y
+
+//Returns: Array of 3 elements
+//0.Number of items to give in that dose
+//1.New dole for item X
+//2.New carry for item X slave Y
+function Dose(numreq,instore,store_last_tick,dole,carry,prev_req)
+{
+	var inputs=[numreq,instore,store_last_tick,dole,carry,prev_req];
+	var internal=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0];
+	var outputs=[0,0];
+	//NN: Please don't touch :), code is generated
+	
+internal[1]=internal[8]*-1.643964648;
+internal[1]=internal[1]+internal[3]*1.475263;
+internal[1]=internal[1]+internal[10]*1.137161493;
+internal[1]=internal[1]+1.0*0.581692636;
+internal[1]=internal[1]+inputs[3]*-0.7429554462;
+internal[1]=internal[1]+inputs[1]*-4.860008717;
+internal[1]=internal[1]+inputs[0]*0.8682486415;
+internal[1]=internal[1]+inputs[5]*-2.332767725;
+internal[1]=sigmoid0(internal[1]);
+internal[2]=internal[1]*0.8746165037;
+internal[2]=sigmoid0(internal[2]);
+internal[3]=inputs[0]*-5.502003193;
+internal[3]=internal[3]+inputs[4]*0.2463098913;
+internal[3]=internal[3]+inputs[5]*2.313313961;
+internal[3]=internal[3]+1.0*-0.3230940998;
+internal[3]=internal[3]+inputs[3]*-0.7781001329;
+internal[3]=internal[3]+inputs[2]*7.295908451;
+internal[3]=sigmoid0(internal[3]);
+internal[6]=internal[1]*1.406680346;
+internal[6]=internal[6]+inputs[3]*0.5068955421;
+internal[6]=internal[6]+inputs[1]*-0.4729067087;
+internal[6]=internal[6]+inputs[2]*0.8344552517;
+internal[6]=internal[6]+inputs[0]*2.739265203;
+internal[6]=sigmoid0(internal[6]);
+internal[7]=inputs[0]*-0.9354650974;
+internal[7]=internal[7]+inputs[5]*1.853131294;
+internal[7]=internal[7]+inputs[2]*-0.5912416577;
+internal[7]=internal[7]+inputs[1]*1.000927806;
+internal[7]=sigmoid0(internal[7]);
+internal[8]=1.0*-2.179960728;
+internal[8]=internal[8]+inputs[1]*1.66548419;
+internal[8]=internal[8]+inputs[0]*1.550180197;
+internal[8]=sigmoid0(internal[8]);
+internal[10]=inputs[2]*1.491255403;
+internal[10]=internal[10]+1.0*1.035691857;
+internal[10]=sigmoid0(internal[10]);
+outputs[0]=inputs[2]*0.5093510151;
+outputs[0]=outputs[0]+inputs[4]*0.9384511113;
+outputs[0]=outputs[0]+outputs[1]*-3.595440626;
+outputs[0]=sigmoid1(outputs[0]);
+outputs[1]=inputs[2]*1.534054637;
+outputs[1]=outputs[1]+internal[2]*-2.732098818;
+outputs[1]=outputs[1]+outputs[2]*1.163635135;
+outputs[1]=outputs[1]+internal[1]*-0.2005333602;
+outputs[1]=outputs[1]+internal[6]*0.6757783294;
+outputs[1]=sigmoid0(outputs[1]);
+outputs[2]=1.0*-0.9311643243;
+outputs[2]=outputs[2]+inputs[1]*-1.85346806;
+outputs[2]=outputs[2]+inputs[2]*-1.232177019;
+outputs[2]=outputs[2]+inputs[0]*1.937278271;
+outputs[2]=outputs[2]+internal[7]*-0.728112638;
+outputs[2]=sigmoid0(outputs[2]);
+
+	if (outputs[0]<0) outputs[0]=0;
+	outputs[0]=outputs[0]*numreq;
+	var res1=Math.trunc(outputs[0]);
+	if (res1>numreq) res1=numreq;
+	if (res1>instore) res1=instore;
+	if (res1<0) res1=0;
+	carry=outputs[2]+outputs[0]-res1;
+	return [res1,outputs[1],carry];
+}
+
+//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
+//Instore - number of items for item X in master gauge (currently)
+//dole - Current dole for item X
+//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
+
+//Returns: Array of 2 elements
+//0.New dole for item X
+//1.Guessed percentage average of items supplied in last 10 ticks
+function Tick(instore,dole,store_last_tick)
+{
+	var inputs=[instore,dole,store_last_tick];
+	var internal=[0,0,0,0,0];
+	var outputs=[0,0];
+	//NN: Please don't touch :), code is generated
+	
+outputs[0]=1.0*0.5381062031;
+outputs[0]=sigmoid1(outputs[0]);
+outputs[1]=1.0*-0.5653554201;
+outputs[1]=outputs[1]+inputs[0]*0.65881145;
+outputs[1]=outputs[1]+inputs[1]*0.6852278709;
+outputs[1]=sigmoid0(outputs[1]);
+
+
+	avg=outputs[1];
+	if (avg>1) avg=1;
+	if (avg<0) avg=0;
+	return [outputs[0],avg];
+}
+
+module.exports = {Dose,Tick}

master.js

@@ -283,29 +283,35 @@ process.on('SIGHUP', shutdown); // terminal closed
 async function shutdown() {
 	console.log('Ctrl-C...');
 	let exitStartTime = Date.now();
-	// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-	if(slaves && Object.keys(slaves).length < 2000000){
-		console.log("saving to slaves.json");
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
-	} else if(slaves) {
-		console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
-	}
-	if(db.items && Object.keys(db.items).length < 50000){
-		console.log("saving to items.json");
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
-	} else if(slaves) {
-		console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
-	}
-	for(let i in masterPlugins){
-		let plugin = masterPlugins[i];
-		if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
-			let startTime = Date.now();
-			await plugin.main.onExit();
-			console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
+	try {
+		// set insane limit to slave length, if its longer than this we are probably being ddosed or something
+		if(slaves && Object.keys(slaves).length < 2000000){
+			console.log("saving to slaves.json");
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
+		} else if(slaves) {
+			console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
 		}
+		if(db.items && Object.keys(db.items).length < 50000){
+			console.log("saving to items.json");
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
+		} else if(slaves) {
+			console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
+		}
+		for(let i in masterPlugins){
+			let plugin = masterPlugins[i];
+			if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
+				let startTime = Date.now();
+				await plugin.main.onExit();
+				console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
+			}
+		}
+		console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
+		process.exit(0);
+	} catch(e) {
+		console.log(e)
+		console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms")
+		process.exit(1)
 	}
-	console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
-	process.exit(0);
 }
 
 /**
@@ -528,6 +534,10 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 		res.send("failure");
 	}
 });
+const routes_api_remove = require("./routes/api/remove.js")
+const neuralDole = new routes_api_remove.neuralDole({
+	items: db.items,
+})
 /**
 POST endpoint to remove items from DB when client orders items.
 
@@ -539,11 +549,8 @@ POST endpoint to remove items from DB when client orders items.
 @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name]
 @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
 */
-_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 app.post("/api/remove", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
-	const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// save items we get
@@ -567,6 +574,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		}
 		res.send({name:object.name, count:0});
 	} else if(config.disableFairItemDistribution){
+		// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
 		let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item));
 		db.items.removeItem({count: numberToRemove, name: object.name});
 		res.send({count: numberToRemove, name: object.name});
@@ -587,45 +595,27 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		});
 		//console.log(sentItemStatistics.data)
 		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+	} else if(config.useNeuralNetDoleDivider){
+		// use fancy neural net to calculate a "fair" dole division rate.
+		neuralDole.divider({
+			res,
+			object,
+			config,
+			sentItemStatisticsBySlaveID,
+			prometheusImportGauge
+		})
 	} else {
-		const originalCount = Number(object.count) || 0;
-		object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
-		object.count = Math.round(object.count);
-		if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
-		
-		// Update existing items if item name already exists
-		if(Number(item) > Number(object.count)) {
-			//If successful, increase dole
-			_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
-			if(config.logItemTransfers){
-				console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
-			}
-			if(db.items.removeItem({count: object.count, name: object.name})){
-				let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-				if(sentItemStatistics === undefined){
-					sentItemStatistics = new averagedTimeSeries({
-						maxEntries: config.itemStats.maxEntries,
-						entriesPerSecond: config.itemStats.entriesPerSecond,
-						mergeMode: "add",
-					}, console.log);
-				}
-				sentItemStatistics.add({
-					key:object.name,
-					value:object.count,
-				});
-				//console.log(sentItemStatistics.data)
-				sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-			}
-			
-			prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
-			prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
-			res.send({count: object.count, name: object.name});
-		} else {
-			// if we didn't have enough, attempt giving out a smaller amount next time
-			_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
-			res.send({name:object.name, count:0});
-			//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
-		}
+		// Use dole division. Makes it really slow to drain out the last little bit.
+		routes_api_remove.doleDivider({
+			item,
+			object,
+			db,
+			sentItemStatisticsBySlaveID,
+			config,
+			prometheusDoleFactorGauge,
+			prometheusImportGauge,
+			req,res,
+		})
 	}
 });
 

routes/api/remove.js

@@ -0,0 +1,146 @@
+const averagedTimeSeries = require("averaged-timeseries")
+
+const doleNN = require("./../../lib/dole_nn_base.js")
+
+class neuralDole {
+    constructor({
+        items,
+    }){
+        // Set some storage variables for the dole divider
+        this.items = items
+        this.itemsLastTick = JSON.parse(JSON.stringify(items))
+        this.dole = {}
+        this.carry = {}
+        this.lastRequest = {}
+
+        setInterval(()=>{
+            for(let name in this.items){
+                // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties.
+                if(typeof this.items[name] !== "number") continue
+
+                let count = this.items[name]
+                let magicData = doleNN.Tick(
+                    count,
+                    this.dole[name],
+                    this.itemsLastTick[name]
+                )
+                this.dole[name] = magicData[0]
+                // TODO handle magicData[1] for graphing for our users
+            }
+            this.itemsLastTick = JSON.parse(JSON.stringify(this.items))
+        }, 1000)
+    }
+    divider({
+        res,
+        object,
+        config,
+        sentItemStatisticsBySlaveID,
+        prometheusImportGauge
+    }){
+        let magicData = doleNN.Dose(
+            object.count, // numReq
+            this.items[object.name],
+            this.itemsLastTick[object.name] || 0,
+            this.dole[object.name],
+            this.carry[object.name] || 0,
+            this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0
+        )
+        this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
+        //0.Number of items to give in that dose
+        //1.New dole for item X
+        //2.New carry for item X slave Y
+        this.dole[object.name] = magicData[1]
+        this.carry[object.name] = magicData[2]
+        // Remove item from DB and send it
+        if(this.items.removeItem({count: magicData[0], name: object.name})){
+            if(config.logItemTransfers){
+                console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName);
+            }
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:magicData[0],
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+            prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
+
+            res.send({
+                name: object.name,
+                count: magicData[0],
+            })
+        } else {
+            res.send({
+                name: object.name,
+                count: 0,
+            })
+        }        
+    }
+}
+
+_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
+function doleDivider({
+    item,
+    object,
+    db,
+    sentItemStatisticsBySlaveID,
+    config,
+    prometheusDoleFactorGauge,
+    prometheusImportGauge,
+    req,res,
+}){
+	const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
+    const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
+    
+    const originalCount = Number(object.count) || 0;
+    object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+    object.count = Math.round(object.count);
+    if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
+    
+    // Update existing items if item name already exists
+    if(Number(item) > Number(object.count)) {
+        //If successful, increase dole
+        _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+        if(config.logItemTransfers){
+            console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
+        }
+        if(db.items.removeItem({count: object.count, name: object.name})){
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:object.count,
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+        }
+        
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
+        res.send({count: object.count, name: object.name});
+    } else {
+        // if we didn't have enough, attempt giving out a smaller amount next time
+        _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        res.send({name:object.name, count:0});
+        //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
+    }
+}
+
+module.exports = {
+    doleDivider,
+    neuralDole
+}

sharedPlugins/scenario_loader/index.js

@@ -81,7 +81,7 @@ module.exports = class remoteCommands {
 				if(returnValue) this.messageInterface(returnValue);
 				this.messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
 			}
-		})();
+		}
 	}
 	async checkHotpatchInstallation(){
 		let yn = await this.messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end");