From 56cf131c7077d78bc51e54bc6d18c3978b0185ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Fri, 22 Mar 2019 19:29:56 +0100 Subject: [PATCH 1/3] Moved most of /api/remove to its own file --- config.json.dist | 2 + lib/dole_nn_base.js | 139 +++++++++++++++++++++++++++++++++++++++++++ master.js | 54 ++++------------- routes/api/remove.js | 63 ++++++++++++++++++++ 4 files changed, 217 insertions(+), 41 deletions(-) create mode 100644 lib/dole_nn_base.js create mode 100644 routes/api/remove.js diff --git a/config.json.dist b/config.json.dist index 3ca02906..d7f6a6b3 100644 --- a/config.json.dist +++ b/config.json.dist @@ -47,7 +47,9 @@ "entriesPerSecond": 0.01666666667 }, "logItemTransfers": false, + "__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.", "disableFairItemDistribution": false, + "useNeuralNetDoleDivider": true, "uploadModsToMaster": true, "msBetweenCommands": 10, "allowRemoteCommandExecution": true, diff --git a/lib/dole_nn_base.js b/lib/dole_nn_base.js new file mode 100644 index 00000000..faf19070 --- /dev/null +++ b/lib/dole_nn_base.js @@ -0,0 +1,139 @@ +//Sigmoids... + +function sigmoid0(x) +{ + return x/(1+abs(x)); +} +function sigmoid1(x) +{ + if (x>=0.0) { + return 1.0 + } else { + return 0.0 + } +} +function sigmoid2(x) +{ + if (x>=0.0){ + return x + } else { + return 0.0 + } +} +function sigmoid3(x) +{ + return x; +} + +//DOSE - Doses items for item X slave Y +//Numreq - number of items that are requested for item X for slave Y +//Instore - number of items for item X in master gauge (currently) +//store_last_tick - Number of items for item X in master gauge at time of last tick +//dole - Current dole for item X +//carry - carry for item X slave Y +//prev_req - Numreq for previous request for item X slave Y + +//Returns: Array of 3 elements +//0.Number of items to give in that dose +//1.New dole for item X +//2.New carry for item X slave Y +function Dose(numreq,instore,store_last_tick,dole,carry,prev_req) +{ + var inputs=[numreq,instore,store_last_tick,dole,carry,prev_req]; + var internal=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]; + var outputs=[0,0]; + //NN: Please don't touch :), code is generated + +internal[1]=internal[8]*-1.643964648; +internal[1]=internal[1]+internal[3]*1.475263; +internal[1]=internal[1]+internal[10]*1.137161493; +internal[1]=internal[1]+1.0*0.581692636; +internal[1]=internal[1]+inputs[3]*-0.7429554462; +internal[1]=internal[1]+inputs[1]*-4.860008717; +internal[1]=internal[1]+inputs[0]*0.8682486415; +internal[1]=internal[1]+inputs[5]*-2.332767725; +internal[1]=sigmoid0(internal[1]); +internal[2]=internal[1]*0.8746165037; +internal[2]=sigmoid0(internal[2]); +internal[3]=inputs[0]*-5.502003193; +internal[3]=internal[3]+inputs[4]*0.2463098913; +internal[3]=internal[3]+inputs[5]*2.313313961; +internal[3]=internal[3]+1.0*-0.3230940998; +internal[3]=internal[3]+inputs[3]*-0.7781001329; +internal[3]=internal[3]+inputs[2]*7.295908451; +internal[3]=sigmoid0(internal[3]); +internal[6]=internal[1]*1.406680346; +internal[6]=internal[6]+inputs[3]*0.5068955421; +internal[6]=internal[6]+inputs[1]*-0.4729067087; +internal[6]=internal[6]+inputs[2]*0.8344552517; +internal[6]=internal[6]+inputs[0]*2.739265203; +internal[6]=sigmoid0(internal[6]); +internal[7]=inputs[0]*-0.9354650974; +internal[7]=internal[7]+inputs[5]*1.853131294; +internal[7]=internal[7]+inputs[2]*-0.5912416577; +internal[7]=internal[7]+inputs[1]*1.000927806; +internal[7]=sigmoid0(internal[7]); +internal[8]=1.0*-2.179960728; +internal[8]=internal[8]+inputs[1]*1.66548419; +internal[8]=internal[8]+inputs[0]*1.550180197; +internal[8]=sigmoid0(internal[8]); +internal[10]=inputs[2]*1.491255403; +internal[10]=internal[10]+1.0*1.035691857; +internal[10]=sigmoid0(internal[10]); +outputs[0]=inputs[2]*0.5093510151; +outputs[0]=outputs[0]+inputs[4]*0.9384511113; +outputs[0]=outputs[0]+outputs[1]*-3.595440626; +outputs[0]=sigmoid1(outputs[0]); +outputs[1]=inputs[2]*1.534054637; +outputs[1]=outputs[1]+internal[2]*-2.732098818; +outputs[1]=outputs[1]+outputs[2]*1.163635135; +outputs[1]=outputs[1]+internal[1]*-0.2005333602; +outputs[1]=outputs[1]+internal[6]*0.6757783294; +outputs[1]=sigmoid0(outputs[1]); +outputs[2]=1.0*-0.9311643243; +outputs[2]=outputs[2]+inputs[1]*-1.85346806; +outputs[2]=outputs[2]+inputs[2]*-1.232177019; +outputs[2]=outputs[2]+inputs[0]*1.937278271; +outputs[2]=outputs[2]+internal[7]*-0.728112638; +outputs[2]=sigmoid0(outputs[2]); + + if (outputs[0]<0) outputs[0]=0; + outputs[0]=outputs[0]*numreq; + var res1=trunc(outputs[0]); + if (res1>numreq) res1=numreq; + if (res1>instore) res1=instore; + if (res1<0) res1=0; + carry=outputs[2]+outputs[0]-res1; + return [res1,outputs[1],carry]; +} + +//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item +//Instore - number of items for item X in master gauge (currently) +//dole - Current dole for item X +//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one) + +//Returns: Array of 2 elements +//0.New dole for item X +//1.Guessed percentage average of items supplied in last 10 ticks +function Tick(instore,dole,store_last_tick) +{ + var inputs=[instore,dole,store_last_tick]; + var internal=[0,0,0,0,0]; + var outputs=[0,0]; + //NN: Please don't touch :), code is generated + +outputs[0]=1.0*0.5381062031; +outputs[0]=sigmoid1(outputs[0]); +outputs[1]=1.0*-0.5653554201; +outputs[1]=outputs[1]+inputs[0]*0.65881145; +outputs[1]=outputs[1]+inputs[1]*0.6852278709; +outputs[1]=sigmoid0(outputs[1]); + + + avg=outputs[1]; + if (avg>1) avg=1; + if (avg<0) avg=0; + return [outputs[0],avg]; +} + +module.exports = {Dose,Tick} \ No newline at end of file diff --git a/master.js b/master.js index ea764a90..14a76436 100644 --- a/master.js +++ b/master.js @@ -539,11 +539,8 @@ POST endpoint to remove items from DB when client orders items. @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name] @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages. */ -_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. app.post("/api/remove", authenticate.middleware, function(req, res) { endpointHitCounter.labels(req.route.path).inc(); - const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings - const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); // save items we get @@ -587,45 +584,20 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { }); //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; - } else { - const originalCount = Number(object.count) || 0; - object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation; - object.count = Math.round(object.count); - if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`); + } else if(config.useNeuralNetDoleDivider){ - // Update existing items if item name already exists - if(Number(item) > Number(object.count)) { - //If successful, increase dole - _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1; - if(config.logItemTransfers){ - console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName); - } - if(db.items.removeItem({count: object.count, name: object.name})){ - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }, console.log); - } - sentItemStatistics.add({ - key:object.name, - value:object.count, - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; - } - - prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); - prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); - res.send({count: object.count, name: object.name}); - } else { - // if we didn't have enough, attempt giving out a smaller amount next time - _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2); - res.send({name:object.name, count:0}); - //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")"); - } + } else { + // Use dole division + require("./routes/api/remove.js").doleDivider({ + item, + object, + db, + sentItemStatisticsBySlaveID, + config, + prometheusDoleFactorGauge, + prometheusImportGauge, + req,res, + }) } }); diff --git a/routes/api/remove.js b/routes/api/remove.js new file mode 100644 index 00000000..ed7000c6 --- /dev/null +++ b/routes/api/remove.js @@ -0,0 +1,63 @@ +const averagedTimeSeries = require("averaged-timeseries") + +const doleNN = require("./../../lib/dole_nn_base.js") + +module.exports = { + doleDivider +} + +_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. + +function doleDivider({ + item, + object, + db, + sentItemStatisticsBySlaveID, + config, + prometheusDoleFactorGauge, + prometheusImportGauge, + req,res, +}){ + const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings + const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase + + const originalCount = Number(object.count) || 0; + object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation; + object.count = Math.round(object.count); + if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`); + + // Update existing items if item name already exists + if(Number(item) > Number(object.count)) { + //If successful, increase dole + _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1; + if(config.logItemTransfers){ + console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName); + } + if(db.items.removeItem({count: object.count, name: object.name})){ + let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; + if(sentItemStatistics === undefined){ + sentItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + } + sentItemStatistics.add({ + key:object.name, + value:object.count, + }); + //console.log(sentItemStatistics.data) + sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; + } + + prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); + res.send({count: object.count, name: object.name}); + } else { + // if we didn't have enough, attempt giving out a smaller amount next time + _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2); + prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); + res.send({name:object.name, count:0}); + //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")"); + } +} \ No newline at end of file From 94d434c8ef0f5f965060bfb60ca733199b3c39cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Sat, 23 Mar 2019 00:14:23 +0100 Subject: [PATCH 2/3] Initial release of neural net based dole divider --- lib/dole_nn_base.js | 4 +- master.js | 66 ++++++++++++++++++++------------ routes/api/remove.js | 89 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 130 insertions(+), 29 deletions(-) diff --git a/lib/dole_nn_base.js b/lib/dole_nn_base.js index faf19070..ace6650c 100644 --- a/lib/dole_nn_base.js +++ b/lib/dole_nn_base.js @@ -2,7 +2,7 @@ function sigmoid0(x) { - return x/(1+abs(x)); + return x/(1+Math.abs(x)); } function sigmoid1(x) { @@ -99,7 +99,7 @@ outputs[2]=sigmoid0(outputs[2]); if (outputs[0]<0) outputs[0]=0; outputs[0]=outputs[0]*numreq; - var res1=trunc(outputs[0]); + var res1=Math.trunc(outputs[0]); if (res1>numreq) res1=numreq; if (res1>instore) res1=instore; if (res1<0) res1=0; diff --git a/master.js b/master.js index 14a76436..c99a00bc 100644 --- a/master.js +++ b/master.js @@ -283,29 +283,35 @@ process.on('SIGHUP', shutdown); // terminal closed async function shutdown() { console.log('Ctrl-C...'); let exitStartTime = Date.now(); - // set insane limit to slave length, if its longer than this we are probably being ddosed or something - if(slaves && Object.keys(slaves).length < 2000000){ - console.log("saving to slaves.json"); - fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves)); - } else if(slaves) { - console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")"); - } - if(db.items && Object.keys(db.items).length < 50000){ - console.log("saving to items.json"); - fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items)); - } else if(slaves) { - console.log("Item database too large, not saving ("+Object.keys(slaves).length+")"); - } - for(let i in masterPlugins){ - let plugin = masterPlugins[i]; - if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ - let startTime = Date.now(); - await plugin.main.onExit(); - console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms"); + try { + // set insane limit to slave length, if its longer than this we are probably being ddosed or something + if(slaves && Object.keys(slaves).length < 2000000){ + console.log("saving to slaves.json"); + fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves)); + } else if(slaves) { + console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")"); } + if(db.items && Object.keys(db.items).length < 50000){ + console.log("saving to items.json"); + fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items)); + } else if(slaves) { + console.log("Item database too large, not saving ("+Object.keys(slaves).length+")"); + } + for(let i in masterPlugins){ + let plugin = masterPlugins[i]; + if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ + let startTime = Date.now(); + await plugin.main.onExit(); + console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms"); + } + } + console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms"); + process.exit(0); + } catch(e) { + console.log(e) + console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms") + process.exit(1) } - console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms"); - process.exit(0); } /** @@ -528,6 +534,10 @@ app.post("/api/place", authenticate.middleware, function(req, res) { res.send("failure"); } }); +const routes_api_remove = require("./routes/api/remove.js") +const neuralDole = new routes_api_remove.neuralDole({ + items: db.items, +}) /** POST endpoint to remove items from DB when client orders items. @@ -564,6 +574,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { } res.send({name:object.name, count:0}); } else if(config.disableFairItemDistribution){ + // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing. let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item)); db.items.removeItem({count: numberToRemove, name: object.name}); res.send({count: numberToRemove, name: object.name}); @@ -585,10 +596,17 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; } else if(config.useNeuralNetDoleDivider){ - + // use fancy neural net to calculate a "fair" dole division rate. + neuralDole.divider({ + res, + object, + config, + sentItemStatisticsBySlaveID, + prometheusImportGauge + }) } else { - // Use dole division - require("./routes/api/remove.js").doleDivider({ + // Use dole division. Makes it really slow to drain out the last little bit. + routes_api_remove.doleDivider({ item, object, db, diff --git a/routes/api/remove.js b/routes/api/remove.js index ed7000c6..23476749 100644 --- a/routes/api/remove.js +++ b/routes/api/remove.js @@ -2,12 +2,90 @@ const averagedTimeSeries = require("averaged-timeseries") const doleNN = require("./../../lib/dole_nn_base.js") -module.exports = { - doleDivider +class neuralDole { + constructor({ + items, + }){ + // Set some storage variables for the dole divider + this.items = items + this.itemsLastTick = JSON.parse(JSON.stringify(items)) + this.dole = {} + this.carry = {} + this.lastRequest = {} + + setInterval(()=>{ + for(let name in this.items){ + // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties. + if(typeof this.items[name] !== "number") continue + + let count = this.items[name] + let magicData = doleNN.Tick( + count, + this.dole[name], + this.itemsLastTick[name] + ) + this.dole[name] = magicData[0] + // TODO handle magicData[1] for graphing for our users + } + this.itemsLastTick = JSON.parse(JSON.stringify(this.items)) + }, 1000) + } + divider({ + res, + object, + config, + sentItemStatisticsBySlaveID, + prometheusImportGauge + }){ + let magicData = doleNN.Dose( + object.count, // numReq + this.items[object.name], + this.itemsLastTick[object.name] || 0, + this.dole[object.name], + this.carry[object.name] || 0, + this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0 + ) + this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count + //0.Number of items to give in that dose + //1.New dole for item X + //2.New carry for item X slave Y + this.dole[object.name] = magicData[1] + this.carry[object.name] = magicData[2] + // Remove item from DB and send it + if(this.items.removeItem({count: magicData[0], name: object.name})){ + if(config.logItemTransfers){ + console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName); + } + let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; + if(sentItemStatistics === undefined){ + sentItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + } + sentItemStatistics.add({ + key:object.name, + value:magicData[0], + }); + //console.log(sentItemStatistics.data) + sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0); + + res.send({ + name: object.name, + count: magicData[0], + }) + } else { + res.send({ + name: object.name, + count: 0, + }) + } + } } _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. - function doleDivider({ item, object, @@ -60,4 +138,9 @@ function doleDivider({ res.send({name:object.name, count:0}); //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")"); } +} + +module.exports = { + doleDivider, + neuralDole } \ No newline at end of file From 8357fa23ce0444dbbacafea03402c3d4bdbe4722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Sun, 24 Mar 2019 20:14:47 +0100 Subject: [PATCH 3/3] Fixed bug in scenarioLoader --- sharedPlugins/scenario_loader/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharedPlugins/scenario_loader/index.js b/sharedPlugins/scenario_loader/index.js index 5889ee4b..79503b88 100644 --- a/sharedPlugins/scenario_loader/index.js +++ b/sharedPlugins/scenario_loader/index.js @@ -81,7 +81,7 @@ module.exports = class remoteCommands { if(returnValue) this.messageInterface(returnValue); this.messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`); } - })(); + } } async checkHotpatchInstallation(){ let yn = await this.messageInterface("/silent-command if remote.interfaces['hotpatch'] then rcon.print('true') else rcon.print('false') end");