From 56cf131c7077d78bc51e54bc6d18c3978b0185ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Vest=C3=B8l?= Date: Fri, 22 Mar 2019 19:29:56 +0100 Subject: [PATCH] Moved most of /api/remove to its own file --- config.json.dist | 2 + lib/dole_nn_base.js | 139 +++++++++++++++++++++++++++++++++++++++++++ master.js | 54 ++++------------- routes/api/remove.js | 63 ++++++++++++++++++++ 4 files changed, 217 insertions(+), 41 deletions(-) create mode 100644 lib/dole_nn_base.js create mode 100644 routes/api/remove.js diff --git a/config.json.dist b/config.json.dist index 3ca02906..d7f6a6b3 100644 --- a/config.json.dist +++ b/config.json.dist @@ -47,7 +47,9 @@ "entriesPerSecond": 0.01666666667 }, "logItemTransfers": false, + "__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.", "disableFairItemDistribution": false, + "useNeuralNetDoleDivider": true, "uploadModsToMaster": true, "msBetweenCommands": 10, "allowRemoteCommandExecution": true, diff --git a/lib/dole_nn_base.js b/lib/dole_nn_base.js new file mode 100644 index 00000000..faf19070 --- /dev/null +++ b/lib/dole_nn_base.js @@ -0,0 +1,139 @@ +//Sigmoids... + +function sigmoid0(x) +{ + return x/(1+abs(x)); +} +function sigmoid1(x) +{ + if (x>=0.0) { + return 1.0 + } else { + return 0.0 + } +} +function sigmoid2(x) +{ + if (x>=0.0){ + return x + } else { + return 0.0 + } +} +function sigmoid3(x) +{ + return x; +} + +//DOSE - Doses items for item X slave Y +//Numreq - number of items that are requested for item X for slave Y +//Instore - number of items for item X in master gauge (currently) +//store_last_tick - Number of items for item X in master gauge at time of last tick +//dole - Current dole for item X +//carry - carry for item X slave Y +//prev_req - Numreq for previous request for item X slave Y + +//Returns: Array of 3 elements +//0.Number of items to give in that dose +//1.New dole for item X +//2.New carry for item X slave Y +function Dose(numreq,instore,store_last_tick,dole,carry,prev_req) +{ + var inputs=[numreq,instore,store_last_tick,dole,carry,prev_req]; + var internal=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]; + var outputs=[0,0]; + //NN: Please don't touch :), code is generated + +internal[1]=internal[8]*-1.643964648; +internal[1]=internal[1]+internal[3]*1.475263; +internal[1]=internal[1]+internal[10]*1.137161493; +internal[1]=internal[1]+1.0*0.581692636; +internal[1]=internal[1]+inputs[3]*-0.7429554462; +internal[1]=internal[1]+inputs[1]*-4.860008717; +internal[1]=internal[1]+inputs[0]*0.8682486415; +internal[1]=internal[1]+inputs[5]*-2.332767725; +internal[1]=sigmoid0(internal[1]); +internal[2]=internal[1]*0.8746165037; +internal[2]=sigmoid0(internal[2]); +internal[3]=inputs[0]*-5.502003193; +internal[3]=internal[3]+inputs[4]*0.2463098913; +internal[3]=internal[3]+inputs[5]*2.313313961; +internal[3]=internal[3]+1.0*-0.3230940998; +internal[3]=internal[3]+inputs[3]*-0.7781001329; +internal[3]=internal[3]+inputs[2]*7.295908451; +internal[3]=sigmoid0(internal[3]); +internal[6]=internal[1]*1.406680346; +internal[6]=internal[6]+inputs[3]*0.5068955421; +internal[6]=internal[6]+inputs[1]*-0.4729067087; +internal[6]=internal[6]+inputs[2]*0.8344552517; +internal[6]=internal[6]+inputs[0]*2.739265203; +internal[6]=sigmoid0(internal[6]); +internal[7]=inputs[0]*-0.9354650974; +internal[7]=internal[7]+inputs[5]*1.853131294; +internal[7]=internal[7]+inputs[2]*-0.5912416577; +internal[7]=internal[7]+inputs[1]*1.000927806; +internal[7]=sigmoid0(internal[7]); +internal[8]=1.0*-2.179960728; +internal[8]=internal[8]+inputs[1]*1.66548419; +internal[8]=internal[8]+inputs[0]*1.550180197; +internal[8]=sigmoid0(internal[8]); +internal[10]=inputs[2]*1.491255403; +internal[10]=internal[10]+1.0*1.035691857; +internal[10]=sigmoid0(internal[10]); +outputs[0]=inputs[2]*0.5093510151; +outputs[0]=outputs[0]+inputs[4]*0.9384511113; +outputs[0]=outputs[0]+outputs[1]*-3.595440626; +outputs[0]=sigmoid1(outputs[0]); +outputs[1]=inputs[2]*1.534054637; +outputs[1]=outputs[1]+internal[2]*-2.732098818; +outputs[1]=outputs[1]+outputs[2]*1.163635135; +outputs[1]=outputs[1]+internal[1]*-0.2005333602; +outputs[1]=outputs[1]+internal[6]*0.6757783294; +outputs[1]=sigmoid0(outputs[1]); +outputs[2]=1.0*-0.9311643243; +outputs[2]=outputs[2]+inputs[1]*-1.85346806; +outputs[2]=outputs[2]+inputs[2]*-1.232177019; +outputs[2]=outputs[2]+inputs[0]*1.937278271; +outputs[2]=outputs[2]+internal[7]*-0.728112638; +outputs[2]=sigmoid0(outputs[2]); + + if (outputs[0]<0) outputs[0]=0; + outputs[0]=outputs[0]*numreq; + var res1=trunc(outputs[0]); + if (res1>numreq) res1=numreq; + if (res1>instore) res1=instore; + if (res1<0) res1=0; + carry=outputs[2]+outputs[0]-res1; + return [res1,outputs[1],carry]; +} + +//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item +//Instore - number of items for item X in master gauge (currently) +//dole - Current dole for item X +//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one) + +//Returns: Array of 2 elements +//0.New dole for item X +//1.Guessed percentage average of items supplied in last 10 ticks +function Tick(instore,dole,store_last_tick) +{ + var inputs=[instore,dole,store_last_tick]; + var internal=[0,0,0,0,0]; + var outputs=[0,0]; + //NN: Please don't touch :), code is generated + +outputs[0]=1.0*0.5381062031; +outputs[0]=sigmoid1(outputs[0]); +outputs[1]=1.0*-0.5653554201; +outputs[1]=outputs[1]+inputs[0]*0.65881145; +outputs[1]=outputs[1]+inputs[1]*0.6852278709; +outputs[1]=sigmoid0(outputs[1]); + + + avg=outputs[1]; + if (avg>1) avg=1; + if (avg<0) avg=0; + return [outputs[0],avg]; +} + +module.exports = {Dose,Tick} \ No newline at end of file diff --git a/master.js b/master.js index ea764a90..14a76436 100644 --- a/master.js +++ b/master.js @@ -539,11 +539,8 @@ POST endpoint to remove items from DB when client orders items. @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name] @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages. */ -_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. app.post("/api/remove", authenticate.middleware, function(req, res) { endpointHitCounter.labels(req.route.path).inc(); - const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings - const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); // save items we get @@ -587,45 +584,20 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { }); //console.log(sentItemStatistics.data) sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; - } else { - const originalCount = Number(object.count) || 0; - object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation; - object.count = Math.round(object.count); - if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`); + } else if(config.useNeuralNetDoleDivider){ - // Update existing items if item name already exists - if(Number(item) > Number(object.count)) { - //If successful, increase dole - _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1; - if(config.logItemTransfers){ - console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName); - } - if(db.items.removeItem({count: object.count, name: object.name})){ - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }, console.log); - } - sentItemStatistics.add({ - key:object.name, - value:object.count, - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; - } - - prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); - prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); - res.send({count: object.count, name: object.name}); - } else { - // if we didn't have enough, attempt giving out a smaller amount next time - _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2); - res.send({name:object.name, count:0}); - //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")"); - } + } else { + // Use dole division + require("./routes/api/remove.js").doleDivider({ + item, + object, + db, + sentItemStatisticsBySlaveID, + config, + prometheusDoleFactorGauge, + prometheusImportGauge, + req,res, + }) } }); diff --git a/routes/api/remove.js b/routes/api/remove.js new file mode 100644 index 00000000..ed7000c6 --- /dev/null +++ b/routes/api/remove.js @@ -0,0 +1,63 @@ +const averagedTimeSeries = require("averaged-timeseries") + +const doleNN = require("./../../lib/dole_nn_base.js") + +module.exports = { + doleDivider +} + +_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. + +function doleDivider({ + item, + object, + db, + sentItemStatisticsBySlaveID, + config, + prometheusDoleFactorGauge, + prometheusImportGauge, + req,res, +}){ + const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings + const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase + + const originalCount = Number(object.count) || 0; + object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation; + object.count = Math.round(object.count); + if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`); + + // Update existing items if item name already exists + if(Number(item) > Number(object.count)) { + //If successful, increase dole + _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1; + if(config.logItemTransfers){ + console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName); + } + if(db.items.removeItem({count: object.count, name: object.name})){ + let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; + if(sentItemStatistics === undefined){ + sentItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + } + sentItemStatistics.add({ + key:object.name, + value:object.count, + }); + //console.log(sentItemStatistics.data) + sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; + } + + prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); + res.send({count: object.count, name: object.name}); + } else { + // if we didn't have enough, attempt giving out a smaller amount next time + _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2); + prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); + res.send({name:object.name, count:0}); + //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")"); + } +} \ No newline at end of file