mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-19 07:33:12 +09:00
Move clusterioMod specific code into the plugin
Move item database, tests, endpoint definitions for /api/place, /api/remove, /api/inventory, /api/inventoryAsObject, and the web interface view for the storage page into the clusterioMod plugin.
This commit is contained in:
1 parent
bc444b79c2
commit
e4aae88614
14 files changed
+524
-473
No files matched your search
@@ -0,0 +1,174 @@
|
||||
const averagedTimeSeries = require("averaged-timeseries")
|
||||
|
||||
const doleNN = require("./dole_nn_base")
|
||||
|
||||
const Prometheus = require('prom-client');
|
||||
|
||||
class neuralDole {
|
||||
|
||||
getRequestStats(itemname,samples)
|
||||
{
|
||||
var sum=0;
|
||||
samples=Math.min(samples,(this.stats[itemname] || []).length-1);
|
||||
if (samples<1) return 0.1;
|
||||
for (var i = 1; i <= samples; i++) {
|
||||
sum+=this.stats[itemname][i].req;
|
||||
}
|
||||
if (sum==0) return 0.1;
|
||||
return sum/samples;
|
||||
}
|
||||
|
||||
constructor({
|
||||
items, gaugePrefix
|
||||
}){
|
||||
// Set some storage variables for the dole divider
|
||||
this.prometheusNNDoleGauge = new Prometheus.Gauge({
|
||||
name: gaugePrefix+'nn_dole_gauge',
|
||||
help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
|
||||
labelNames: ["itemName"],
|
||||
});
|
||||
this.items = items
|
||||
this.itemsLastTick = new Map(items._items)
|
||||
this.dole = {}
|
||||
this.carry = {}
|
||||
this.lastRequest = {}
|
||||
this.stats=[]
|
||||
this.debt= {}
|
||||
|
||||
setInterval(()=>{
|
||||
for (let [name, count] of this.items._items) {
|
||||
let magicData = doleNN.Tick(
|
||||
count,
|
||||
this.dole[name],
|
||||
this.itemsLastTick.get(name),
|
||||
this.getRequestStats(name,5)
|
||||
)
|
||||
this.stats[name]=this.stats[name] || []
|
||||
this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
|
||||
if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
|
||||
|
||||
this.dole[name] = magicData[0]
|
||||
// DONE handle magicData[1] for graphing for our users
|
||||
this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
|
||||
}
|
||||
this.itemsLastTick = new Map(this.items._items)
|
||||
}, 1000)
|
||||
}
|
||||
divider({
|
||||
res,
|
||||
object,
|
||||
config,
|
||||
sentItemStatisticsBySlaveID,
|
||||
prometheusImportGauge
|
||||
}){
|
||||
let magicData = doleNN.Dose(
|
||||
object.count, // numReq
|
||||
this.items.getItemCount(object.name),
|
||||
this.itemsLastTick.get(object.name) || 0,
|
||||
this.dole[object.name],
|
||||
this.carry[object.name+" "+object.instanceID] || 0,
|
||||
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
|
||||
this.getRequestStats(object.name,5),
|
||||
this.debt[object.name+" "+object.instanceID] || 0
|
||||
)
|
||||
if ((this.stats[object.name] || []).length>0)
|
||||
{
|
||||
this.stats[object.name][0].req+=Number(object.count);
|
||||
this.stats[object.name][0].given+=Number(magicData[0]);
|
||||
}
|
||||
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
|
||||
//0.Number of items to give in that dose
|
||||
//1.New dole for item X
|
||||
//2.New carry for item X slave Y
|
||||
this.dole[object.name] = magicData[1]
|
||||
this.carry[object.name+" "+object.instanceID] = magicData[2]
|
||||
this.debt[object.name+" "+object.instanceID] = magicData[3]
|
||||
|
||||
// Remove item from DB and send it
|
||||
this.items.removeItem(object.name, magicData[0]);
|
||||
if(config.logItemTransfers){
|
||||
console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
|
||||
}
|
||||
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
|
||||
if(sentItemStatistics === undefined){
|
||||
sentItemStatistics = new averagedTimeSeries({
|
||||
maxEntries: config.itemStats.maxEntries,
|
||||
entriesPerSecond: config.itemStats.entriesPerSecond,
|
||||
mergeMode: "add",
|
||||
}, console.log);
|
||||
}
|
||||
sentItemStatistics.add({
|
||||
key:object.name,
|
||||
value:magicData[0],
|
||||
});
|
||||
//console.log(sentItemStatistics.data)
|
||||
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
|
||||
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
|
||||
|
||||
res.send({
|
||||
name: object.name,
|
||||
count: magicData[0],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
|
||||
function doleDivider({
|
||||
itemCount,
|
||||
object,
|
||||
items,
|
||||
sentItemStatisticsBySlaveID,
|
||||
config,
|
||||
prometheusDoleFactorGauge,
|
||||
prometheusImportGauge,
|
||||
req,res,
|
||||
}){
|
||||
const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
|
||||
const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
|
||||
|
||||
const originalCount = Number(object.count) || 0;
|
||||
object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
|
||||
object.count = Math.round(object.count);
|
||||
if (itemCount > 40) {
|
||||
console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`);
|
||||
}
|
||||
|
||||
// Update existing items if item name already exists
|
||||
if (itemCount > object.count) {
|
||||
//If successful, increase dole
|
||||
_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
|
||||
if (config.logItemTransfers) {
|
||||
console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
|
||||
}
|
||||
items.removeItem(object.name, object.count)
|
||||
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
|
||||
if(sentItemStatistics === undefined){
|
||||
sentItemStatistics = new averagedTimeSeries({
|
||||
maxEntries: config.itemStats.maxEntries,
|
||||
entriesPerSecond: config.itemStats.entriesPerSecond,
|
||||
mergeMode: "add",
|
||||
}, console.log);
|
||||
}
|
||||
sentItemStatistics.add({
|
||||
key:object.name,
|
||||
value:object.count,
|
||||
});
|
||||
//console.log(sentItemStatistics.data)
|
||||
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
|
||||
|
||||
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
|
||||
prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);
|
||||
res.send({count: object.count, name: object.name});
|
||||
} else {
|
||||
// if we didn't have enough, attempt giving out a smaller amount next time
|
||||
_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
|
||||
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
|
||||
res.send({name:object.name, count:0});
|
||||
//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
doleDivider,
|
||||
neuralDole
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
//Sigmoids...
|
||||
|
||||
function sigmoid0(x)
|
||||
{
|
||||
return x/(1+Math.abs(x));
|
||||
}
|
||||
function sigmoid1(x)
|
||||
{
|
||||
if (x>=0.0) {
|
||||
return 1.0
|
||||
} else {
|
||||
return 0.0
|
||||
}
|
||||
}
|
||||
function sigmoid2(x)
|
||||
{
|
||||
if (x>=0.0){
|
||||
return x
|
||||
} else {
|
||||
return 0.0
|
||||
}
|
||||
}
|
||||
function sigmoid3(x)
|
||||
{
|
||||
return x;
|
||||
}
|
||||
|
||||
//DOSE - Doses items for item X slave Y
|
||||
//Numreq - number of items that are requested for item X for slave Y
|
||||
//Instore - number of items for item X in master gauge (currently)
|
||||
//store_last_tick - Number of items for item X in master gauge at time of last tick
|
||||
//dole - Current dole for item X
|
||||
//carry - carry for item X slave Y
|
||||
//prev_req - Numreq for previous request for item X slave Y
|
||||
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
|
||||
//debt - debt for item X slave Y
|
||||
|
||||
//Returns: Array of 3 elements
|
||||
//0.Number of items to give in that dose
|
||||
//1.New dole for item X
|
||||
//2.New carry for item X slave Y
|
||||
//3.New debt
|
||||
function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj,debt)
|
||||
{
|
||||
numreq=Number(numreq);
|
||||
instore=Number(instore);
|
||||
var instore_adj=instore;
|
||||
if (instore_adj==0) instore_adj=0.1;
|
||||
|
||||
var b=Math.trunc(prev_req/instore_adj*numreq);
|
||||
var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
|
||||
var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b,numreq_total_adj,store_last_tick,debt];//5..12
|
||||
var inputs=inp1.concat(inp2);
|
||||
var internal=new Array(20).fill(0);
|
||||
var outputs=[0,0,0,0];
|
||||
//NN: Please don't touch :), code is generated
|
||||
|
||||
outputs[0]=inputs[7]*0.8058582544;
|
||||
outputs[0]=sigmoid2(outputs[0]);
|
||||
outputs[1]=sigmoid0(outputs[1]);
|
||||
outputs[2]=sigmoid0(outputs[2]);
|
||||
outputs[3]=inputs[6]*0.2294657379;
|
||||
outputs[3]=sigmoid0(outputs[3]);
|
||||
|
||||
debt=debt || 0;
|
||||
if (outputs[0]>0.99) outputs[0]=1.0;
|
||||
|
||||
var res1=Number(outputs[0])*numreq+outputs[3]+debt;
|
||||
var res_capped=Math.round(res1);
|
||||
|
||||
if (res_capped>numreq) res_capped=numreq;
|
||||
if (res_capped>instore) res_capped=instore;
|
||||
if (res_capped<0) res_capped=0;
|
||||
|
||||
res1=res1 || 0;
|
||||
res_capped=res_capped || 0;//Safeguard against wrong values
|
||||
debt=res1-res_capped;
|
||||
debt_cap=Math.round(numreq*1.5)+5;
|
||||
if (debt>debt_cap) debt=debt_cap;
|
||||
|
||||
carry=sigmoid0(outputs[2]*carry+outputs[0]) || 0;
|
||||
return [res_capped,dole+outputs[1],carry,debt];
|
||||
}
|
||||
|
||||
//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
|
||||
//Instore - number of items for item X in master gauge (currently)
|
||||
//dole - Current dole for item X
|
||||
//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
|
||||
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
|
||||
//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
|
||||
|
||||
|
||||
//Returns: Array of 2 elements
|
||||
//0.New dole for item X
|
||||
//1.Guessed percentage average of items supplied in last 10 ticks
|
||||
function Tick(instore,dole,store_last_tick,numreq_total_adj)
|
||||
{
|
||||
var instore_adj=instore;
|
||||
if (instore_adj==0) instore_adj=0.1;
|
||||
var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj,store_last_tick,numreq_total_adj];
|
||||
var internal=new Array(10).fill(0);
|
||||
var outputs=[0,0];
|
||||
//NN: Please don't touch :), code is generated
|
||||
|
||||
outputs[0]=inputs[3]*0.9364774823;
|
||||
outputs[0]=outputs[0]*1.0*0.4590721726;
|
||||
outputs[0]=sigmoid2(outputs[0]);
|
||||
outputs[1]=outputs[0]*1.828422785;
|
||||
outputs[1]=sigmoid3(outputs[1]);
|
||||
|
||||
avg=outputs[1];
|
||||
if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
|
||||
if (avg>1) avg=1;
|
||||
if (avg<0) avg=0;
|
||||
|
||||
return [sigmoid0(outputs[0]),avg];
|
||||
}
|
||||
|
||||
module.exports = {Dose,Tick}
|
||||
@@ -1,83 +1,104 @@
|
||||
"use strict";
|
||||
const fs = require("fs-extra");
|
||||
const path = require("path");
|
||||
const Express = require("express");
|
||||
const ejs = require("ejs");
|
||||
|
||||
const database = require("lib/database");
|
||||
const routes = require("./routes");
|
||||
|
||||
|
||||
class masterPlugin {
|
||||
constructor({config, pluginConfig, pluginPath, socketio, express}){
|
||||
constructor({
|
||||
config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter,
|
||||
}) {
|
||||
this.config = config;
|
||||
this.pluginConfig = pluginConfig;
|
||||
this.pluginPath = pluginPath;
|
||||
this.io = socketio;
|
||||
this.app = express;
|
||||
|
||||
// load databases
|
||||
this.items = getDatabaseSync("database/items.json");
|
||||
|
||||
this.clients = {};
|
||||
this.slaves = {};
|
||||
|
||||
// initialize web API
|
||||
// require("./js/api-endpoints.js")(this);
|
||||
|
||||
// expose UI elements embedded in the master
|
||||
// this.ui = require("./js/ui.js").ui;
|
||||
|
||||
this.io.on("connection", socket => {
|
||||
|
||||
});
|
||||
}
|
||||
async broadcastCommand(command){
|
||||
let returnValues = [];
|
||||
for(let instanceID in this.slaves){
|
||||
let slave = this.slaves[instanceID];
|
||||
slave.emit("runCommand", {
|
||||
// commandID:Math.random(),
|
||||
command,
|
||||
});
|
||||
};
|
||||
return returnValues;
|
||||
}
|
||||
findInArray(key, value, array){
|
||||
let indexes = [];
|
||||
for(let i in array){
|
||||
if(array[i][key] && array[i][key] === value) indexes.push(i);
|
||||
}
|
||||
return indexes;
|
||||
}
|
||||
async onExit(){
|
||||
// await saveDatabase("database/items.json", this.items);
|
||||
return;
|
||||
}
|
||||
}
|
||||
module.exports = masterPlugin;
|
||||
|
||||
function arrayRemoveDuplicates(array){
|
||||
let newArray = [];
|
||||
array.forEach(value => {
|
||||
if(!newArray.includes(value)) newArray.push(value);
|
||||
});
|
||||
return newArray;
|
||||
}
|
||||
function getDatabaseSync(path){
|
||||
let db;
|
||||
try {
|
||||
db = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||
} catch(e){
|
||||
db = {};
|
||||
this.express = express;
|
||||
this.db = db,
|
||||
this.Prometheus = Prometheus;
|
||||
this.prometheusPrefix = prometheusPrefix;
|
||||
this.endpointHitCounter = endpointHitCounter;
|
||||
|
||||
this.prometheusMasterInventoryGauge = new Prometheus.Gauge({
|
||||
name: prometheusPrefix+'master_inventory_gauge',
|
||||
help: 'Amount of items stored on master',
|
||||
labelNames: ["itemName"],
|
||||
});
|
||||
|
||||
this.ui = {
|
||||
sidebar: [
|
||||
{
|
||||
getHtml: () => `
|
||||
<div class="nav-item mr-1">
|
||||
<a class="nav-link align-middle" href="/clusterioMod/storage">Storage</a>
|
||||
</div>`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
onMetrics() {
|
||||
if (this.items) {
|
||||
for (let [key, count] of this.items._items) {
|
||||
this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async onLoadFinish() {
|
||||
this.items = await loadDatabase(this.config);
|
||||
this.autosaveId = setInterval(async function() {
|
||||
let file = path.join(this.config.databaseDirectory, "items.json");
|
||||
console.log(`autosaving ${file}`);
|
||||
let content = JSON.stringify(db.items.serialize());
|
||||
await fs.outputFile(file, content);
|
||||
},this.config.autosaveInterval || 60000);
|
||||
|
||||
routes.addApiRoutes(
|
||||
this.express, this.config, this.db, this.items,
|
||||
this.Prometheus, this.prometheusPrefix, this.endpointHitCounter,
|
||||
);
|
||||
|
||||
routes.addWebRoutes(this.express);
|
||||
}
|
||||
|
||||
async onExit(){
|
||||
clearInterval(this.autosaveId);
|
||||
await saveDatabase(this.config, this.items);
|
||||
}
|
||||
return db;
|
||||
}
|
||||
async function saveDatabase(path, database){
|
||||
if(!path){
|
||||
throw new Error("No path provided!");
|
||||
} else if(!database){
|
||||
throw new Error("No database provided!");
|
||||
} else {
|
||||
try {
|
||||
await fs.writeFile(path, JSON.stringify(database, null, 4));
|
||||
} catch(e){
|
||||
throw new Error("Unable to write to database! "+path);
|
||||
|
||||
async function loadDatabase(config) {
|
||||
let itemsPath = path.resolve(config.databaseDirectory, "items.json");
|
||||
console.log(`Loading ${itemsPath}`);
|
||||
try {
|
||||
let content = await fs.readFile(itemsPath);
|
||||
return new database.ItemDatabase(JSON.parse(content));
|
||||
|
||||
} catch (err) {
|
||||
if (err.code === "ENOENT") {
|
||||
console.log("Creating new item database");
|
||||
return new database.ItemDatabase();
|
||||
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDatabase(config, items) {
|
||||
if (items && items.size < 50000) {
|
||||
let file = path.resolve(config.databaseDirectory, "items.json");
|
||||
console.log(`writing ${file}`);
|
||||
let content = JSON.stringify(items.serialize());
|
||||
await fs.outputFile(file, content);
|
||||
} else if (items) {
|
||||
console.error(`Item database too large, not saving (${items.size})`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = masterPlugin;
|
||||
@@ -0,0 +1,241 @@
|
||||
var recievedItemStatisticsBySlaveID = {};
|
||||
var sentItemStatisticsBySlaveID = {};
|
||||
|
||||
const averagedTimeSeries = require("averaged-timeseries");
|
||||
const express = require("express");
|
||||
const path = require("path");
|
||||
|
||||
const authenticate = require("lib/authenticate");
|
||||
|
||||
const dole = require("./dole");
|
||||
|
||||
|
||||
function addWebRoutes(app) {
|
||||
app.use('/clusterioMod', express.static(path.join(__dirname, 'static')));
|
||||
app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); });
|
||||
}
|
||||
|
||||
function addApiRoutes(
|
||||
app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter,
|
||||
) {
|
||||
const prometheusExportGauge = new Prometheus.Gauge({
|
||||
name: prometheusPrefix+'export_gauge',
|
||||
help: 'Items exported by instance',
|
||||
labelNames: ["instanceID", "itemName"],
|
||||
});
|
||||
const prometheusImportGauge = new Prometheus.Gauge({
|
||||
name: prometheusPrefix+'import_gauge',
|
||||
help: 'Items imported by instance',
|
||||
labelNames: ["instanceID", "itemName"],
|
||||
});
|
||||
const prometheusDoleFactorGauge = new Prometheus.Gauge({
|
||||
name: prometheusPrefix+'dole_factor_gauge',
|
||||
help: 'The current dole division factor for this item',
|
||||
labelNames: ["itemName"],
|
||||
});
|
||||
|
||||
// Only initialize neural network when it's enabled, otherwise it might override gauge
|
||||
let neuralDole;
|
||||
if(config.useNeuralNetDoleDivider) {
|
||||
neuralDole=new dole.neuralDole({
|
||||
db, gaugePrefix: prometheusPrefix
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* POST endpoint for storing items in master's inventory.
|
||||
|
||||
* @memberof clusterioMaster
|
||||
* @instance
|
||||
* @alias api/place
|
||||
* @param {itemStack} itemStack the number and type of items to store (see typedef)
|
||||
* @param {string} [itemStack.instanceID="unknown"]
|
||||
* the unique/instanceID which is a numerical value for an instance
|
||||
* @param {string} [itemStack.instanceName="unknown"]
|
||||
* the name of an instance for identification in statistics, as
|
||||
* provided when launching it. ex node client start [name]
|
||||
* @returns {string} status either "success" or "failure"
|
||||
*/
|
||||
app.post("/api/place", authenticate.middleware, function(req, res) {
|
||||
endpointHitCounter.labels(req.route.path).inc();
|
||||
res.header("Access-Control-Allow-Origin", "*");
|
||||
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
|
||||
let x;
|
||||
try {
|
||||
x = JSON.parse(req.body);
|
||||
} catch (e) {
|
||||
x = req.body;
|
||||
}
|
||||
if(!x.instanceName) {
|
||||
x.instanceName = "unknown";
|
||||
}
|
||||
if(!x.instanceID) {
|
||||
x.instanceID = "unknown";
|
||||
for (let [instanceID, slave] of db.slaves) {
|
||||
if(slave.instanceName == req.body.instanceName) {
|
||||
x.instanceID = instanceID;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(x.instanceID
|
||||
&& x.instanceName
|
||||
&& !isNaN(Number(x.count))
|
||||
&& x.name
|
||||
&& typeof x.name == "string"){
|
||||
if(config.logItemTransfers){
|
||||
console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
|
||||
}
|
||||
// gather statistics
|
||||
let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
|
||||
if(recievedItemStatistics === undefined){
|
||||
recievedItemStatistics = new averagedTimeSeries({
|
||||
maxEntries: config.itemStats.maxEntries,
|
||||
entriesPerSecond: config.itemStats.entriesPerSecond,
|
||||
mergeMode: "add",
|
||||
}, console.log);
|
||||
recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
|
||||
}
|
||||
recievedItemStatistics.add({
|
||||
key:req.body.name,
|
||||
value:req.body.count,
|
||||
});
|
||||
prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
|
||||
// save items we get
|
||||
var count = Number(req.body.count);
|
||||
items.addItem(req.body.name, count);
|
||||
|
||||
// Attempt confirming
|
||||
res.send("success");
|
||||
} else {
|
||||
res.send("failure");
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST endpoint to remove items from DB when client orders items.
|
||||
*
|
||||
* @memberof clusterioMaster
|
||||
* @instance
|
||||
* @alias api/remove
|
||||
* @param {itemStack} itemStack the name of and the number of items to remove (see typedef)
|
||||
* @param {string} [itemStack.instanceID="unknown"]
|
||||
* the unique/instanceID which is a numerical value for an instance
|
||||
* @param {string} [itemStack.instanceName="unknown"]
|
||||
* the name of an instance for identification in statistics, as
|
||||
* provided when launching it. ex node client start [name]
|
||||
* @returns {itemStack}
|
||||
* the number of items actually removed, may be lower than what was
|
||||
* asked for due to shortages.
|
||||
*/
|
||||
app.post("/api/remove", authenticate.middleware, function(req, res) {
|
||||
endpointHitCounter.labels(req.route.path).inc();
|
||||
res.header("Access-Control-Allow-Origin", "*");
|
||||
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
|
||||
// save items we get
|
||||
var object = req.body;
|
||||
if(!object.instanceID) {
|
||||
object.instanceID = "unknown"
|
||||
}
|
||||
if(!object.instanceName) {
|
||||
object.instanceName = "unknown";
|
||||
}
|
||||
if (db.slaves.has(String(object.instanceID))) {
|
||||
object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
|
||||
}
|
||||
let itemCount = items.getItemCount(object.name);
|
||||
if (
|
||||
(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
|
||||
&& object.name != "electricity"
|
||||
) {
|
||||
if(config.logItemTransfers){
|
||||
console.log('failure could not find ' + object.name);
|
||||
}
|
||||
res.send({name:object.name, count:0});
|
||||
} else if(config.disableFairItemDistribution){
|
||||
// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
|
||||
let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
|
||||
items.removeItem(object.name, numberToRemove);
|
||||
res.send({count: numberToRemove, name: object.name});
|
||||
|
||||
// track statistics and do graphing things
|
||||
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
|
||||
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
|
||||
if(sentItemStatistics === undefined){
|
||||
sentItemStatistics = new averagedTimeSeries({
|
||||
maxEntries: config.itemStats.maxEntries,
|
||||
entriesPerSecond: config.itemStats.entriesPerSecond,
|
||||
mergeMode: "add",
|
||||
}, console.log);
|
||||
}
|
||||
sentItemStatistics.add({
|
||||
key:object.name,
|
||||
value:numberToRemove,
|
||||
});
|
||||
//console.log(sentItemStatistics.data)
|
||||
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
|
||||
} else if(config.useNeuralNetDoleDivider){
|
||||
// use fancy neural net to calculate a "fair" dole division rate.
|
||||
neuralDole.divider({
|
||||
res,
|
||||
object,
|
||||
config,
|
||||
sentItemStatisticsBySlaveID,
|
||||
prometheusImportGauge
|
||||
})
|
||||
} else {
|
||||
// Use dole division. Makes it really slow to drain out the last little bit.
|
||||
dole.doleDivider({
|
||||
itemCount,
|
||||
object,
|
||||
db,
|
||||
sentItemStatisticsBySlaveID,
|
||||
config,
|
||||
prometheusDoleFactorGauge,
|
||||
prometheusImportGauge,
|
||||
req,res,
|
||||
})
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET endpoint to read the masters current inventory of items.
|
||||
*
|
||||
* @memberof clusterioMaster
|
||||
* @instance
|
||||
* @alias api/inventory
|
||||
* @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
|
||||
*/
|
||||
app.get("/api/inventory", function(req, res) {
|
||||
endpointHitCounter.labels(req.route.path).inc();
|
||||
res.header("Access-Control-Allow-Origin", "*");
|
||||
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
|
||||
// Check it and send it
|
||||
var inventory = [];
|
||||
for (let [name, count] of items._items) {
|
||||
inventory.push({ name, count });
|
||||
}
|
||||
res.send(JSON.stringify(inventory));
|
||||
});
|
||||
|
||||
/**
|
||||
* GET endpoint to read the masters inventory as an object with key:value pairs
|
||||
*
|
||||
* @memberof clusterioMaster
|
||||
* @instance
|
||||
* @alias api/inventoryAsObject
|
||||
* @returns {object} JSON {"iron-plate":100, "copper-plate":5}
|
||||
*/
|
||||
app.get("/api/inventoryAsObject", function(req, res) {
|
||||
endpointHitCounter.labels(req.route.path).inc();
|
||||
res.header("Access-Control-Allow-Origin", "*");
|
||||
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
|
||||
// Check it and send it
|
||||
res.send(JSON.stringify(items.serialise()));
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
addWebRoutes,
|
||||
addApiRoutes,
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
// function to draw data we recieve from ajax requests
|
||||
_lastData = [];
|
||||
function drawcontents(data) {
|
||||
data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box.
|
||||
_lastData = data;
|
||||
|
||||
// allow searching for multiple criteria separated by space
|
||||
var searchArgs = document.querySelector("#search").value;
|
||||
searchArgs = searchArgs.trim();
|
||||
searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')';
|
||||
const search = new RegExp(searchArgs, 'i');
|
||||
data = data.filter(function(item) {
|
||||
return search.test(item.name);
|
||||
})
|
||||
|
||||
sortByKey(data, "count");
|
||||
|
||||
const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically
|
||||
const rows = table.children;
|
||||
|
||||
//update existing rows or create new ones
|
||||
data.forEach(function(item, i) {
|
||||
// format count to have .s in it for large number readability (1,000,000)
|
||||
item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
||||
|
||||
let row = rows[i];
|
||||
if(!row) {
|
||||
row = document.createElement('tr');
|
||||
row.innerHTML = "<td><img width=32 height=32></td><td class=name></td><td class=count></td>";
|
||||
table.appendChild(row);
|
||||
}
|
||||
|
||||
const img = row.querySelector('img');
|
||||
const imgName = getImageFromName(item.name);
|
||||
if(img.getAttribute('src') !== imgName) {
|
||||
img.setAttribute('src',imgName);
|
||||
}
|
||||
|
||||
const name = row.querySelector('.name');
|
||||
if(name.textContent !== item.name) {
|
||||
name.textContent = item.name;
|
||||
row.onclick = ()=>{
|
||||
makeItemGraphs(item.name);
|
||||
}
|
||||
}
|
||||
|
||||
const count = row.querySelector('.count');
|
||||
if(count.textContent !== ''+item.count) {
|
||||
count.textContent = item.count;
|
||||
}
|
||||
});
|
||||
|
||||
//remove excess rows, for example, after filtering
|
||||
while (data.length < rows.length) {
|
||||
table.removeChild(rows[data.length]);
|
||||
}
|
||||
}
|
||||
|
||||
// get cluster inventory from master
|
||||
function updateInventory() {
|
||||
var xmlhttp = new XMLHttpRequest();
|
||||
xmlhttp.onreadystatechange = function() {
|
||||
if(xmlhttp.readyState == 4 && xmlhttp.status == 200) {
|
||||
let data = JSON.parse(xmlhttp.responseText);
|
||||
drawcontents(data);
|
||||
}
|
||||
}
|
||||
xmlhttp.open("GET", "/api/inventory", true);
|
||||
xmlhttp.send();
|
||||
}
|
||||
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
|
||||
setInterval(updateInventory, 500);
|
||||
} else {
|
||||
updateInventory();
|
||||
}
|
||||
|
||||
// function to sort arrays of objects after a keys value
|
||||
function sortByKey(array, key) {
|
||||
array.sort(function(a, b) {
|
||||
return b[key] - a[key];
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelector("#search").addEventListener('input', function() {
|
||||
drawcontents();
|
||||
});
|
||||
|
||||
// Make graph of cluster IO items
|
||||
/*getJSON("/api/getStats", function(err, data){
|
||||
|
||||
}*/
|
||||
function getItemStatData(item, callback){
|
||||
var xhr = new XMLHttpRequest();
|
||||
var url = "/api/getStats";
|
||||
xhr.open("POST", url, true);
|
||||
xhr.setRequestHeader("Content-type", "application/json");
|
||||
xhr.onreadystatechange = function () {
|
||||
if (xhr.readyState === 4 && xhr.status === 200) {
|
||||
var json = JSON.parse(xhr.responseText);
|
||||
console.log(json.email + ", " + json.password);
|
||||
callback(json);
|
||||
}
|
||||
};
|
||||
var requestData = JSON.stringify({
|
||||
"statistic": "place",
|
||||
"itemName": "shit",
|
||||
});
|
||||
xhr.send(data);
|
||||
}
|
||||
|
||||
function makeItemGraphs(itemName){
|
||||
getJSON("/api/slaves", function(err, slaves){
|
||||
if(err) throw err;
|
||||
if(typeof slaves == "string"){
|
||||
slaves = JSON.parse(slaves);
|
||||
}
|
||||
let instanceIDs = Object.keys(slaves);
|
||||
let HTML = ""
|
||||
// Draw in/out chart for unregistered slaves as well
|
||||
instanceIDs.unshift("unknown");
|
||||
instanceIDs.forEach(function(instanceID){
|
||||
if(instanceID == "unknown" || Date.now() - slaves[instanceID].time < 120000 || JSON.parse(localStorage.settings)["Display offline slaves"]){
|
||||
HTML += '<div id="contentGraph'+instanceID+'" class="storageGraph" style="width: 100%;"></div>';
|
||||
["place", "remove"].forEach(statistic => {
|
||||
console.log("Making chart for "+instanceID+"'s "+statistic+" statistic");
|
||||
|
||||
setTimeout(function(){
|
||||
let xmlhttp = new XMLHttpRequest(); // new HttpRequest instance
|
||||
xmlhttp.open("POST", "/api/getStats");
|
||||
xmlhttp.setRequestHeader("Content-Type", "application/json");
|
||||
xmlhttp.onreadystatechange = function(){
|
||||
console.log(xmlhttp.status);
|
||||
if(xmlhttp.readyState === XMLHttpRequest.DONE && xmlhttp.status === 200) {
|
||||
// console.log(xmlhttp.responseText);
|
||||
if(isJSON(xmlhttp.responseText) && JSON.parse(xmlhttp.responseText).statusForDebugging === undefined){
|
||||
let slaveDisplayName = "unknown";
|
||||
try {
|
||||
slaveDisplayName = slaves[instanceID].instanceName;
|
||||
} catch(e) {}
|
||||
let whatGraphIsIt = "Item IO from ";
|
||||
// if(statistic == "remove") whatGraphIsIt = "Items taken from cluster by";
|
||||
|
||||
let chartData = JSON.parse(xmlhttp.response).data;
|
||||
switch(statistic){
|
||||
case "place":
|
||||
chartData.name = "Items exported:";
|
||||
break;
|
||||
case "remove":
|
||||
chartData.name = "Items imported:";
|
||||
break;
|
||||
}
|
||||
drawChartWhenCalledTwice(chartData, whatGraphIsIt + " " + slaveDisplayName);
|
||||
//drawChart(statistic+"contentGraph"+instanceID, [JSON.parse(xmlhttp.response).data], `${whatGraphIsIt} ${slaveDisplayName} (${instanceID}) ${itemName}`);
|
||||
// console.log(JSON.parse(xmlhttp.response).data);
|
||||
}
|
||||
}
|
||||
}
|
||||
xmlhttp.send(JSON.stringify({statistic:statistic, itemName:itemName, instanceID:instanceID}));
|
||||
},1);
|
||||
});
|
||||
let chartLines = [];
|
||||
function drawChartWhenCalledTwice(data, slaveDisplayName){
|
||||
chartLines.push(data);
|
||||
if(chartLines.length >= 2){
|
||||
drawChart("contentGraph"+instanceID, chartLines, `${slaveDisplayName} (${instanceID}) ${itemName}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
document.querySelector("#graphDisplay > #storageGraphContainer").innerHTML = HTML;
|
||||
});
|
||||
}
|
||||
let chartsByID = {};
|
||||
function drawChart(selector, chartData, title) {
|
||||
// selector is ID of element, ex "chartContainer" or "-123199123"
|
||||
console.log(chartData)
|
||||
chartsByID[selector] = new CanvasJS.Chart(selector, {
|
||||
title:{
|
||||
text: title || "Production graph"
|
||||
},
|
||||
toolTip:{
|
||||
shared: true,
|
||||
// content: "{name}: {y}",
|
||||
},
|
||||
zoomEnabled: true,
|
||||
axisY:{
|
||||
includeZero: true,
|
||||
},
|
||||
legend: {
|
||||
cursor: "pointer",
|
||||
itemclick: function (e) {
|
||||
//console.log("legend click: " + e.dataPointIndex);
|
||||
//console.log(e);
|
||||
if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
|
||||
e.dataSeries.visible = false;
|
||||
} else {
|
||||
e.dataSeries.visible = true;
|
||||
}
|
||||
e.chart.render();
|
||||
}
|
||||
},
|
||||
data: chartData
|
||||
});
|
||||
chart = chartsByID[selector];
|
||||
// console.log(chart)
|
||||
chart.render();
|
||||
return chart;
|
||||
}
|
||||
@@ -3,3 +3,5 @@
|
||||
|
||||
require('../index');
|
||||
require('../masterPlugin');
|
||||
require('../dole');
|
||||
require('../dole_nn_base');
|
||||
@@ -0,0 +1,110 @@
|
||||
const assert = require("assert").strict;
|
||||
const express = require("express");
|
||||
const bodyParser = require("body-parser");
|
||||
const request = require("supertest");
|
||||
const Prometheus = require('prom-client');
|
||||
const jwt = require("jsonwebtoken");
|
||||
|
||||
const authenticate = require('lib/authenticate');
|
||||
const database = require("lib/database");
|
||||
|
||||
const routes = require("../routes");
|
||||
|
||||
|
||||
const testSecret = "TestSecretDoNotUse";
|
||||
authenticate.setAuthSecret(testSecret);
|
||||
|
||||
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
|
||||
|
||||
describe("clusterioMod endpoints", function() {
|
||||
const app = express();
|
||||
app.use(bodyParser.json({ limit: '10mb', }));
|
||||
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
|
||||
const config = {
|
||||
itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 },
|
||||
disableFairItemDistribution: true,
|
||||
};
|
||||
const items = new database.ItemDatabase();
|
||||
const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] });
|
||||
routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge);
|
||||
|
||||
let persistentMaster = request(app);
|
||||
describe("#POST /api/place", function() {
|
||||
it("adds an itemStack to the masters inventory", function() {
|
||||
return persistentMaster.post("/api/place").send({
|
||||
name:"steel-plate",
|
||||
count:20,
|
||||
instanceName:"unitTest"
|
||||
}).set(
|
||||
"X-Access-Token", token
|
||||
).expect(200).then(function(res) {
|
||||
assert.equal(res.text, "success", "something went wrong with the request")
|
||||
});
|
||||
});
|
||||
});
|
||||
describe("#GET /api/inventory", function() {
|
||||
it("returns the masters current inventory", function() {
|
||||
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
|
||||
let inventory = JSON.parse(res.text);
|
||||
assert.equal(typeof inventory, "object", "Inventory should be an object");
|
||||
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
|
||||
|
||||
let contains20SteelPlate = false;
|
||||
inventory.forEach(itemStack => {
|
||||
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
|
||||
});
|
||||
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
|
||||
});
|
||||
});
|
||||
});
|
||||
describe("#POST /api/remove", function() {
|
||||
it("returns an itemStack of how many items were removed", function() {
|
||||
return persistentMaster.post("/api/remove").send({
|
||||
name:"steel-plate",
|
||||
count:10
|
||||
}).set(
|
||||
"X-Access-Token", token
|
||||
).expect(200).then(function(res) {
|
||||
assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
|
||||
});
|
||||
});
|
||||
it("works correctly for items named addItem or removeItem", async function() {
|
||||
function add(item) {
|
||||
return persistentMaster.post("/api/place").send({
|
||||
name: item,
|
||||
count: 10
|
||||
}).set(
|
||||
"X-Access-Token", token
|
||||
).expect(200).then(function(res) {
|
||||
assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
|
||||
})
|
||||
}
|
||||
|
||||
function remove(item) {
|
||||
return persistentMaster.post("/api/remove").send({
|
||||
name: item,
|
||||
count: 10
|
||||
}).set(
|
||||
"X-Access-Token", token
|
||||
).expect(200).then(function(res) {
|
||||
assert.deepEqual(res.body, {"name":item,"count":10});
|
||||
});
|
||||
}
|
||||
|
||||
await Promise.all([add("addItem"), add("removeItem")]);
|
||||
await Promise.all([remove("addItem"), remove("removeItem")]);
|
||||
});
|
||||
it("returns an empty itemStack if you don't have any of the item you request", function() {
|
||||
return persistentMaster.post("/api/remove").send({
|
||||
name:"imaginaryItem",
|
||||
count:999999
|
||||
}).set(
|
||||
"X-Access-Token", token
|
||||
).expect(200).then(function(res) {
|
||||
assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
|
||||
assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
|
||||
assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
<!doctype HTML>
|
||||
<head>
|
||||
<%- include("head") %>
|
||||
<script defer src="/clusterioMod/storage.js"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<%- include("nav") %>
|
||||
<div id="body" class="container">
|
||||
<div id="storage">
|
||||
<div class="page-header mt-2 row" id="banner">
|
||||
<div class="row">
|
||||
<div class="col-lg-8 col-md-7 col-sm-6">
|
||||
<h2>Storage</h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group mt-3 mb-3 row">
|
||||
<input type="text" class="form-control" id="search" placeholder="search">
|
||||
</div>
|
||||
<div id="contents" class="row w-100">
|
||||
<table class="table table-striped table-hover">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th scope="col">Icon</th>
|
||||
<th scope="col">Item ID</th>
|
||||
<th scope="col">Quantity</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody><!-- populated by js --></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div id="graphDisplay">
|
||||
<div id="storageGraphContainer">
|
||||
<div id="contentGraph" class="storageGraph" style="width: 100%;"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
|
||||
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
|
||||
crossorigin="anonymous"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
|
||||
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
|
||||
crossorigin="anonymous"></script>
|
||||
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
|
||||
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
|
||||
crossorigin="anonymous"></script>
|
||||
</body>
|
||||
Reference in new issue
Block a user