Initial release of neural net based dole divider

lib/dole_nn_base.js

@@ -2,7 +2,7 @@
 
 function sigmoid0(x)
 {
-	return x/(1+abs(x));
+	return x/(1+Math.abs(x));
 }
 function sigmoid1(x)
 {
@@ -99,7 +99,7 @@ outputs[2]=sigmoid0(outputs[2]);
 
 	if (outputs[0]<0) outputs[0]=0;
 	outputs[0]=outputs[0]*numreq;
-	var res1=trunc(outputs[0]);
+	var res1=Math.trunc(outputs[0]);
 	if (res1>numreq) res1=numreq;
 	if (res1>instore) res1=instore;
 	if (res1<0) res1=0;

master.js

@@ -283,29 +283,35 @@ process.on('SIGHUP', shutdown); // terminal closed
 async function shutdown() {
 	console.log('Ctrl-C...');
 	let exitStartTime = Date.now();
-	// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-	if(slaves && Object.keys(slaves).length < 2000000){
-		console.log("saving to slaves.json");
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
-	} else if(slaves) {
-		console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
-	}
-	if(db.items && Object.keys(db.items).length < 50000){
-		console.log("saving to items.json");
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
-	} else if(slaves) {
-		console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
-	}
-	for(let i in masterPlugins){
-		let plugin = masterPlugins[i];
-		if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
-			let startTime = Date.now();
-			await plugin.main.onExit();
-			console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
+	try {
+		// set insane limit to slave length, if its longer than this we are probably being ddosed or something
+		if(slaves && Object.keys(slaves).length < 2000000){
+			console.log("saving to slaves.json");
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
+		} else if(slaves) {
+			console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
 		}
+		if(db.items && Object.keys(db.items).length < 50000){
+			console.log("saving to items.json");
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
+		} else if(slaves) {
+			console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
+		}
+		for(let i in masterPlugins){
+			let plugin = masterPlugins[i];
+			if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
+				let startTime = Date.now();
+				await plugin.main.onExit();
+				console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
+			}
+		}
+		console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
+		process.exit(0);
+	} catch(e) {
+		console.log(e)
+		console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms")
+		process.exit(1)
 	}
-	console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
-	process.exit(0);
 }
 
 /**
@@ -528,6 +534,10 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 		res.send("failure");
 	}
 });
+const routes_api_remove = require("./routes/api/remove.js")
+const neuralDole = new routes_api_remove.neuralDole({
+	items: db.items,
+})
 /**
 POST endpoint to remove items from DB when client orders items.
 
@@ -564,6 +574,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		}
 		res.send({name:object.name, count:0});
 	} else if(config.disableFairItemDistribution){
+		// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
 		let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item));
 		db.items.removeItem({count: numberToRemove, name: object.name});
 		res.send({count: numberToRemove, name: object.name});
@@ -585,10 +596,17 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		//console.log(sentItemStatistics.data)
 		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
 	} else if(config.useNeuralNetDoleDivider){
-		
+		// use fancy neural net to calculate a "fair" dole division rate.
+		neuralDole.divider({
+			res,
+			object,
+			config,
+			sentItemStatisticsBySlaveID,
+			prometheusImportGauge
+		})
 	} else {
-		// Use dole division
-		require("./routes/api/remove.js").doleDivider({
+		// Use dole division. Makes it really slow to drain out the last little bit.
+		routes_api_remove.doleDivider({
 			item,
 			object,
 			db,

routes/api/remove.js

@@ -2,12 +2,90 @@ const averagedTimeSeries = require("averaged-timeseries")
 
 const doleNN = require("./../../lib/dole_nn_base.js")
 
-module.exports = {
-    doleDivider
+class neuralDole {
+    constructor({
+        items,
+    }){
+        // Set some storage variables for the dole divider
+        this.items = items
+        this.itemsLastTick = JSON.parse(JSON.stringify(items))
+        this.dole = {}
+        this.carry = {}
+        this.lastRequest = {}
+
+        setInterval(()=>{
+            for(let name in this.items){
+                // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties.
+                if(typeof this.items[name] !== "number") continue
+
+                let count = this.items[name]
+                let magicData = doleNN.Tick(
+                    count,
+                    this.dole[name],
+                    this.itemsLastTick[name]
+                )
+                this.dole[name] = magicData[0]
+                // TODO handle magicData[1] for graphing for our users
+            }
+            this.itemsLastTick = JSON.parse(JSON.stringify(this.items))
+        }, 1000)
+    }
+    divider({
+        res,
+        object,
+        config,
+        sentItemStatisticsBySlaveID,
+        prometheusImportGauge
+    }){
+        let magicData = doleNN.Dose(
+            object.count, // numReq
+            this.items[object.name],
+            this.itemsLastTick[object.name] || 0,
+            this.dole[object.name],
+            this.carry[object.name] || 0,
+            this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0
+        )
+        this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
+        //0.Number of items to give in that dose
+        //1.New dole for item X
+        //2.New carry for item X slave Y
+        this.dole[object.name] = magicData[1]
+        this.carry[object.name] = magicData[2]
+        // Remove item from DB and send it
+        if(this.items.removeItem({count: magicData[0], name: object.name})){
+            if(config.logItemTransfers){
+                console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName);
+            }
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:magicData[0],
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+            prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
+
+            res.send({
+                name: object.name,
+                count: magicData[0],
+            })
+        } else {
+            res.send({
+                name: object.name,
+                count: 0,
+            })
+        }        
+    }
 }
 
 _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
-
 function doleDivider({
     item,
     object,
@@ -60,4 +138,9 @@ function doleDivider({
         res.send({name:object.name, count:0});
         //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
     }
+}
+
+module.exports = {
+    doleDivider,
+    neuralDole
 }