diff --git a/lib/dole_nn_base.js b/lib/dole_nn_base.js index faf19070..ace6650c 100644 --- a/lib/dole_nn_base.js +++ b/lib/dole_nn_base.js @@ -2,7 +2,7 @@ function sigmoid0(x) { - return x/(1+abs(x)); + return x/(1+Math.abs(x)); } function sigmoid1(x) { @@ -99,7 +99,7 @@ outputs[2]=sigmoid0(outputs[2]); if (outputs[0]<0) outputs[0]=0; outputs[0]=outputs[0]*numreq; - var res1=trunc(outputs[0]); + var res1=Math.trunc(outputs[0]); if (res1>numreq) res1=numreq; if (res1>instore) res1=instore; if (res1<0) res1=0; diff --git a/master.js b/master.js index 14a76436..c99a00bc 100644 --- a/master.js +++ b/master.js @@ -283,29 +283,35 @@ process.on('SIGHUP', shutdown); // terminal closed async function shutdown() { console.log('Ctrl-C...'); let exitStartTime = Date.now(); - // set insane limit to slave length, if its longer than this we are probably being ddosed or something - if(slaves && Object.keys(slaves).length < 2000000){ - console.log("saving to slaves.json"); - fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves)); - } else if(slaves) { - console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")"); - } - if(db.items && Object.keys(db.items).length < 50000){ - console.log("saving to items.json"); - fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items)); - } else if(slaves) { - console.log("Item database too large, not saving ("+Object.keys(slaves).length+")"); - } - for(let i in masterPlugins){ - let plugin = masterPlugins[i]; - if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ - let startTime = Date.now(); - await plugin.main.onExit(); - console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms"); + try { + // set insane limit to slave length, if its longer than this we are probably being ddosed or something + if(slaves && Object.keys(slaves).length < 2000000){ + console.log("saving to slaves.json"); + fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves)); + } else if(slaves) { + console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")"); } + if(db.items && Object.keys(db.items).length < 50000){ + console.log("saving to items.json"); + fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items)); + } else if(slaves) { + console.log("Item database too large, not saving ("+Object.keys(slaves).length+")"); + } + for(let i in masterPlugins){ + let plugin = masterPlugins[i]; + if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ + let startTime = Date.now(); + await plugin.main.onExit(); + console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms"); + } + } + console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms"); + process.exit(0); + } catch(e) { + console.log(e) + console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms") + process.exit(1) } - console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms"); - process.exit(0); } /** @@ -528,6 +534,10 @@ app.post("/api/place", authenticate.middleware, function(req, res) { res.send("failure"); } }); +const routes_api_remove = require("./routes/api/remove.js") +const neuralDole = new routes_api_remove.neuralDole({ + items: db.items, +}) /** POST endpoint to remove items from DB when client orders items. @@ -564,6 +574,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { } res.send({name:object.name, count:0}); } else if(config.disableFairItemDistribution){ + // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing. let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item)); db.items.removeItem({count: numberToRemove, name: object.name}); res.send({count: numberToRemove, name: object.name}); @@ -585,10 +596,17 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } else if(config.useNeuralNetDoleDivider){ - + // use fancy neural net to calculate a "fair" dole division rate. + neuralDole.divider({ + res, + object, + config, + sentItemStatisticsBySlaveID, + prometheusImportGauge + }) } else { - // Use dole division - require("./routes/api/remove.js").doleDivider({ + // Use dole division. Makes it really slow to drain out the last little bit. + routes_api_remove.doleDivider({ item, object, db, diff --git a/routes/api/remove.js b/routes/api/remove.js index ed7000c6..23476749 100644 --- a/routes/api/remove.js +++ b/routes/api/remove.js @@ -2,12 +2,90 @@ const averagedTimeSeries = require("averaged-timeseries") const doleNN = require("./../../lib/dole_nn_base.js") -module.exports = { - doleDivider +class neuralDole { + constructor({ + items, + }){ + // Set some storage variables for the dole divider + this.items = items + this.itemsLastTick = JSON.parse(JSON.stringify(items)) + this.dole = {} + this.carry = {} + this.lastRequest = {} + + setInterval(()=>{ + for(let name in this.items){ + // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties. + if(typeof this.items[name] !== "number") continue + + let count = this.items[name] + let magicData = doleNN.Tick( + count, + this.dole[name], + this.itemsLastTick[name] + ) + this.dole[name] = magicData[0] + // TODO handle magicData[1] for graphing for our users + } + this.itemsLastTick = JSON.parse(JSON.stringify(this.items)) + }, 1000) + } + divider({ + res, + object, + config, + sentItemStatisticsBySlaveID, + prometheusImportGauge + }){ + let magicData = doleNN.Dose( + object.count, // numReq + this.items[object.name], + this.itemsLastTick[object.name] || 0, + this.dole[object.name], + this.carry[object.name] || 0, + this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0 + ) + this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count + //0.Number of items to give in that dose + //1.New dole for item X + //2.New carry for item X slave Y + this.dole[object.name] = magicData[1] + this.carry[object.name] = magicData[2] + // Remove item from DB and send it + if(this.items.removeItem({count: magicData[0], name: object.name})){ + if(config.logItemTransfers){ + console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName); + } + let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; + if(sentItemStatistics === undefined){ + sentItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + } + sentItemStatistics.add({ + key:object.name, + value:magicData[0], + }); + //console.log(sentItemStatistics.data) + sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0); + + res.send({ + name: object.name, + count: magicData[0], + }) + } else { + res.send({ + name: object.name, + count: 0, + }) + } + } } _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. - function doleDivider({ item, object, @@ -60,4 +138,9 @@ function doleDivider({ res.send({name:object.name, count:0}); //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")"); } +} + +module.exports = { + doleDivider, + neuralDole } \ No newline at end of file