Move clusterioMod specific code into the plugin

Move item database, tests, endpoint definitions for /api/place,
/api/remove, /api/inventory, /api/inventoryAsObject, and the web
interface view for the storage page into the clusterioMod plugin.
This commit is contained in:
Hornwitser committed 2019-10-28 19:46:22 +01:00
1 parent bc444b79c2
commit e4aae88614
14 files changed
+524 -473

No files matched your search

+174
View File
@@ -0,0 +1,174 @@
const averagedTimeSeries = require("averaged-timeseries")
const doleNN = require("./dole_nn_base")
const Prometheus = require('prom-client');
class neuralDole {
getRequestStats(itemname,samples)
{
var sum=0;
samples=Math.min(samples,(this.stats[itemname] || []).length-1);
if (samples<1) return 0.1;
for (var i = 1; i <= samples; i++) {
sum+=this.stats[itemname][i].req;
}
if (sum==0) return 0.1;
return sum/samples;
}
constructor({
items, gaugePrefix
}){
// Set some storage variables for the dole divider
this.prometheusNNDoleGauge = new Prometheus.Gauge({
name: gaugePrefix+'nn_dole_gauge',
help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
labelNames: ["itemName"],
});
this.items = items
this.itemsLastTick = new Map(items._items)
this.dole = {}
this.carry = {}
this.lastRequest = {}
this.stats=[]
this.debt= {}
setInterval(()=>{
for (let [name, count] of this.items._items) {
let magicData = doleNN.Tick(
count,
this.dole[name],
this.itemsLastTick.get(name),
this.getRequestStats(name,5)
)
this.stats[name]=this.stats[name] || []
this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
this.dole[name] = magicData[0]
// DONE handle magicData[1] for graphing for our users
this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
}
this.itemsLastTick = new Map(this.items._items)
}, 1000)
}
divider({
res,
object,
config,
sentItemStatisticsBySlaveID,
prometheusImportGauge
}){
let magicData = doleNN.Dose(
object.count, // numReq
this.items.getItemCount(object.name),
this.itemsLastTick.get(object.name) || 0,
this.dole[object.name],
this.carry[object.name+" "+object.instanceID] || 0,
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
this.getRequestStats(object.name,5),
this.debt[object.name+" "+object.instanceID] || 0
)
if ((this.stats[object.name] || []).length>0)
{
this.stats[object.name][0].req+=Number(object.count);
this.stats[object.name][0].given+=Number(magicData[0]);
}
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
//0.Number of items to give in that dose
//1.New dole for item X
//2.New carry for item X slave Y
this.dole[object.name] = magicData[1]
this.carry[object.name+" "+object.instanceID] = magicData[2]
this.debt[object.name+" "+object.instanceID] = magicData[3]
// Remove item from DB and send it
this.items.removeItem(object.name, magicData[0]);
if(config.logItemTransfers){
console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
}
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
}
sentItemStatistics.add({
key:object.name,
value:magicData[0],
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
res.send({
name: object.name,
count: magicData[0],
})
}
}
_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
function doleDivider({
itemCount,
object,
items,
sentItemStatisticsBySlaveID,
config,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
}){
const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
const originalCount = Number(object.count) || 0;
object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
object.count = Math.round(object.count);
if (itemCount > 40) {
console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`);
}
// Update existing items if item name already exists
if (itemCount > object.count) {
//If successful, increase dole
_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
if (config.logItemTransfers) {
console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
}
items.removeItem(object.name, object.count)
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
}
sentItemStatistics.add({
key:object.name,
value:object.count,
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);
res.send({count: object.count, name: object.name});
} else {
// if we didn't have enough, attempt giving out a smaller amount next time
_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
res.send({name:object.name, count:0});
//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
}
}
module.exports = {
doleDivider,
neuralDole
}
+119
View File
@@ -0,0 +1,119 @@
//Sigmoids...
function sigmoid0(x)
{
return x/(1+Math.abs(x));
}
function sigmoid1(x)
{
if (x>=0.0) {
return 1.0
} else {
return 0.0
}
}
function sigmoid2(x)
{
if (x>=0.0){
return x
} else {
return 0.0
}
}
function sigmoid3(x)
{
return x;
}
//DOSE - Doses items for item X slave Y
//Numreq - number of items that are requested for item X for slave Y
//Instore - number of items for item X in master gauge (currently)
//store_last_tick - Number of items for item X in master gauge at time of last tick
//dole - Current dole for item X
//carry - carry for item X slave Y
//prev_req - Numreq for previous request for item X slave Y
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
//debt - debt for item X slave Y
//Returns: Array of 3 elements
//0.Number of items to give in that dose
//1.New dole for item X
//2.New carry for item X slave Y
//3.New debt
function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj,debt)
{
numreq=Number(numreq);
instore=Number(instore);
var instore_adj=instore;
if (instore_adj==0) instore_adj=0.1;
var b=Math.trunc(prev_req/instore_adj*numreq);
var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b,numreq_total_adj,store_last_tick,debt];//5..12
var inputs=inp1.concat(inp2);
var internal=new Array(20).fill(0);
var outputs=[0,0,0,0];
//NN: Please don't touch :), code is generated
outputs[0]=inputs[7]*0.8058582544;
outputs[0]=sigmoid2(outputs[0]);
outputs[1]=sigmoid0(outputs[1]);
outputs[2]=sigmoid0(outputs[2]);
outputs[3]=inputs[6]*0.2294657379;
outputs[3]=sigmoid0(outputs[3]);
debt=debt || 0;
if (outputs[0]>0.99) outputs[0]=1.0;
var res1=Number(outputs[0])*numreq+outputs[3]+debt;
var res_capped=Math.round(res1);
if (res_capped>numreq) res_capped=numreq;
if (res_capped>instore) res_capped=instore;
if (res_capped<0) res_capped=0;
res1=res1 || 0;
res_capped=res_capped || 0;//Safeguard against wrong values
debt=res1-res_capped;
debt_cap=Math.round(numreq*1.5)+5;
if (debt>debt_cap) debt=debt_cap;
carry=sigmoid0(outputs[2]*carry+outputs[0]) || 0;
return [res_capped,dole+outputs[1],carry,debt];
}
//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
//Instore - number of items for item X in master gauge (currently)
//dole - Current dole for item X
//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
//Returns: Array of 2 elements
//0.New dole for item X
//1.Guessed percentage average of items supplied in last 10 ticks
function Tick(instore,dole,store_last_tick,numreq_total_adj)
{
var instore_adj=instore;
if (instore_adj==0) instore_adj=0.1;
var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj,store_last_tick,numreq_total_adj];
var internal=new Array(10).fill(0);
var outputs=[0,0];
//NN: Please don't touch :), code is generated
outputs[0]=inputs[3]*0.9364774823;
outputs[0]=outputs[0]*1.0*0.4590721726;
outputs[0]=sigmoid2(outputs[0]);
outputs[1]=outputs[0]*1.828422785;
outputs[1]=sigmoid3(outputs[1]);
avg=outputs[1];
if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
if (avg>1) avg=1;
if (avg<0) avg=0;
return [sigmoid0(outputs[0]),avg];
}
module.exports = {Dose,Tick}
+88 -67
View File
@@ -1,83 +1,104 @@
"use strict";
const fs = require("fs-extra");
const path = require("path");
const Express = require("express");
const ejs = require("ejs");
const database = require("lib/database");
const routes = require("./routes");
class masterPlugin {
constructor({config, pluginConfig, pluginPath, socketio, express}){
constructor({
config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter,
}) {
this.config = config;
this.pluginConfig = pluginConfig;
this.pluginPath = pluginPath;
this.io = socketio;
this.app = express;
// load databases
this.items = getDatabaseSync("database/items.json");
this.clients = {};
this.slaves = {};
// initialize web API
// require("./js/api-endpoints.js")(this);
// expose UI elements embedded in the master
// this.ui = require("./js/ui.js").ui;
this.io.on("connection", socket => {
});
}
async broadcastCommand(command){
let returnValues = [];
for(let instanceID in this.slaves){
let slave = this.slaves[instanceID];
slave.emit("runCommand", {
// commandID:Math.random(),
command,
});
};
return returnValues;
}
findInArray(key, value, array){
let indexes = [];
for(let i in array){
if(array[i][key] && array[i][key] === value) indexes.push(i);
}
return indexes;
}
async onExit(){
// await saveDatabase("database/items.json", this.items);
return;
}
}
module.exports = masterPlugin;
function arrayRemoveDuplicates(array){
let newArray = [];
array.forEach(value => {
if(!newArray.includes(value)) newArray.push(value);
});
return newArray;
}
function getDatabaseSync(path){
let db;
try {
db = JSON.parse(fs.readFileSync(path, "utf8"));
} catch(e){
db = {};
this.express = express;
this.db = db,
this.Prometheus = Prometheus;
this.prometheusPrefix = prometheusPrefix;
this.endpointHitCounter = endpointHitCounter;
this.prometheusMasterInventoryGauge = new Prometheus.Gauge({
name: prometheusPrefix+'master_inventory_gauge',
help: 'Amount of items stored on master',
labelNames: ["itemName"],
});
this.ui = {
sidebar: [
{
getHtml: () => `
<div class="nav-item mr-1">
<a class="nav-link align-middle" href="/clusterioMod/storage">Storage</a>
</div>`,
},
],
};
}
onMetrics() {
if (this.items) {
for (let [key, count] of this.items._items) {
this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
}
}
}
async onLoadFinish() {
this.items = await loadDatabase(this.config);
this.autosaveId = setInterval(async function() {
let file = path.join(this.config.databaseDirectory, "items.json");
console.log(`autosaving ${file}`);
let content = JSON.stringify(db.items.serialize());
await fs.outputFile(file, content);
},this.config.autosaveInterval || 60000);
routes.addApiRoutes(
this.express, this.config, this.db, this.items,
this.Prometheus, this.prometheusPrefix, this.endpointHitCounter,
);
routes.addWebRoutes(this.express);
}
async onExit(){
clearInterval(this.autosaveId);
await saveDatabase(this.config, this.items);
}
return db;
}
async function saveDatabase(path, database){
if(!path){
throw new Error("No path provided!");
} else if(!database){
throw new Error("No database provided!");
} else {
try {
await fs.writeFile(path, JSON.stringify(database, null, 4));
} catch(e){
throw new Error("Unable to write to database! "+path);
async function loadDatabase(config) {
let itemsPath = path.resolve(config.databaseDirectory, "items.json");
console.log(`Loading ${itemsPath}`);
try {
let content = await fs.readFile(itemsPath);
return new database.ItemDatabase(JSON.parse(content));
} catch (err) {
if (err.code === "ENOENT") {
console.log("Creating new item database");
return new database.ItemDatabase();
} else {
throw err;
}
}
}
async function saveDatabase(config, items) {
if (items && items.size < 50000) {
let file = path.resolve(config.databaseDirectory, "items.json");
console.log(`writing ${file}`);
let content = JSON.stringify(items.serialize());
await fs.outputFile(file, content);
} else if (items) {
console.error(`Item database too large, not saving (${items.size})`);
}
}
module.exports = masterPlugin;
+241
View File
@@ -0,0 +1,241 @@
var recievedItemStatisticsBySlaveID = {};
var sentItemStatisticsBySlaveID = {};
const averagedTimeSeries = require("averaged-timeseries");
const express = require("express");
const path = require("path");
const authenticate = require("lib/authenticate");
const dole = require("./dole");
function addWebRoutes(app) {
app.use('/clusterioMod', express.static(path.join(__dirname, 'static')));
app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); });
}
function addApiRoutes(
app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter,
) {
const prometheusExportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'export_gauge',
help: 'Items exported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusImportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'import_gauge',
help: 'Items imported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusDoleFactorGauge = new Prometheus.Gauge({
name: prometheusPrefix+'dole_factor_gauge',
help: 'The current dole division factor for this item',
labelNames: ["itemName"],
});
// Only initialize neural network when it's enabled, otherwise it might override gauge
let neuralDole;
if(config.useNeuralNetDoleDivider) {
neuralDole=new dole.neuralDole({
db, gaugePrefix: prometheusPrefix
});
}
/**
* POST endpoint for storing items in master's inventory.
* @memberof clusterioMaster
* @instance
* @alias api/place
* @param {itemStack} itemStack the number and type of items to store (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {string} status either "success" or "failure"
*/
app.post("/api/place", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
let x;
try {
x = JSON.parse(req.body);
} catch (e) {
x = req.body;
}
if(!x.instanceName) {
x.instanceName = "unknown";
}
if(!x.instanceID) {
x.instanceID = "unknown";
for (let [instanceID, slave] of db.slaves) {
if(slave.instanceName == req.body.instanceName) {
x.instanceID = instanceID;
break;
}
}
}
if(x.instanceID
&& x.instanceName
&& !isNaN(Number(x.count))
&& x.name
&& typeof x.name == "string"){
if(config.logItemTransfers){
console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
}
// gather statistics
let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
if(recievedItemStatistics === undefined){
recievedItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
}
recievedItemStatistics.add({
key:req.body.name,
value:req.body.count,
});
prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
// save items we get
var count = Number(req.body.count);
items.addItem(req.body.name, count);
// Attempt confirming
res.send("success");
} else {
res.send("failure");
}
});
/**
* POST endpoint to remove items from DB when client orders items.
*
* @memberof clusterioMaster
* @instance
* @alias api/remove
* @param {itemStack} itemStack the name of and the number of items to remove (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {itemStack}
* the number of items actually removed, may be lower than what was
* asked for due to shortages.
*/
app.post("/api/remove", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// save items we get
var object = req.body;
if(!object.instanceID) {
object.instanceID = "unknown"
}
if(!object.instanceName) {
object.instanceName = "unknown";
}
if (db.slaves.has(String(object.instanceID))) {
object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
}
let itemCount = items.getItemCount(object.name);
if (
(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
&& object.name != "electricity"
) {
if(config.logItemTransfers){
console.log('failure could not find ' + object.name);
}
res.send({name:object.name, count:0});
} else if(config.disableFairItemDistribution){
// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
items.removeItem(object.name, numberToRemove);
res.send({count: numberToRemove, name: object.name});
// track statistics and do graphing things
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
}
sentItemStatistics.add({
key:object.name,
value:numberToRemove,
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
} else if(config.useNeuralNetDoleDivider){
// use fancy neural net to calculate a "fair" dole division rate.
neuralDole.divider({
res,
object,
config,
sentItemStatisticsBySlaveID,
prometheusImportGauge
})
} else {
// Use dole division. Makes it really slow to drain out the last little bit.
dole.doleDivider({
itemCount,
object,
db,
sentItemStatisticsBySlaveID,
config,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
})
}
});
/**
* GET endpoint to read the masters current inventory of items.
*
* @memberof clusterioMaster
* @instance
* @alias api/inventory
* @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
*/
app.get("/api/inventory", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
var inventory = [];
for (let [name, count] of items._items) {
inventory.push({ name, count });
}
res.send(JSON.stringify(inventory));
});
/**
* GET endpoint to read the masters inventory as an object with key:value pairs
*
* @memberof clusterioMaster
* @instance
* @alias api/inventoryAsObject
* @returns {object} JSON {"iron-plate":100, "copper-plate":5}
*/
app.get("/api/inventoryAsObject", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
res.send(JSON.stringify(items.serialise()));
});
}
module.exports = {
addWebRoutes,
addApiRoutes,
}
@@ -0,0 +1,208 @@
// function to draw data we recieve from ajax requests
_lastData = [];
function drawcontents(data) {
data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box.
_lastData = data;
// allow searching for multiple criteria separated by space
var searchArgs = document.querySelector("#search").value;
searchArgs = searchArgs.trim();
searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')';
const search = new RegExp(searchArgs, 'i');
data = data.filter(function(item) {
return search.test(item.name);
})
sortByKey(data, "count");
const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically
const rows = table.children;
//update existing rows or create new ones
data.forEach(function(item, i) {
// format count to have .s in it for large number readability (1,000,000)
item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
let row = rows[i];
if(!row) {
row = document.createElement('tr');
row.innerHTML = "<td><img width=32 height=32></td><td class=name></td><td class=count></td>";
table.appendChild(row);
}
const img = row.querySelector('img');
const imgName = getImageFromName(item.name);
if(img.getAttribute('src') !== imgName) {
img.setAttribute('src',imgName);
}
const name = row.querySelector('.name');
if(name.textContent !== item.name) {
name.textContent = item.name;
row.onclick = ()=>{
makeItemGraphs(item.name);
}
}
const count = row.querySelector('.count');
if(count.textContent !== ''+item.count) {
count.textContent = item.count;
}
});
//remove excess rows, for example, after filtering
while (data.length < rows.length) {
table.removeChild(rows[data.length]);
}
}
// get cluster inventory from master
function updateInventory() {
var xmlhttp = new XMLHttpRequest();
xmlhttp.onreadystatechange = function() {
if(xmlhttp.readyState == 4 && xmlhttp.status == 200) {
let data = JSON.parse(xmlhttp.responseText);
drawcontents(data);
}
}
xmlhttp.open("GET", "/api/inventory", true);
xmlhttp.send();
}
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
setInterval(updateInventory, 500);
} else {
updateInventory();
}
// function to sort arrays of objects after a keys value
function sortByKey(array, key) {
array.sort(function(a, b) {
return b[key] - a[key];
});
}
document.querySelector("#search").addEventListener('input', function() {
drawcontents();
});
// Make graph of cluster IO items
/*getJSON("/api/getStats", function(err, data){
}*/
function getItemStatData(item, callback){
var xhr = new XMLHttpRequest();
var url = "/api/getStats";
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-type", "application/json");
xhr.onreadystatechange = function () {
if (xhr.readyState === 4 && xhr.status === 200) {
var json = JSON.parse(xhr.responseText);
console.log(json.email + ", " + json.password);
callback(json);
}
};
var requestData = JSON.stringify({
"statistic": "place",
"itemName": "shit",
});
xhr.send(data);
}
function makeItemGraphs(itemName){
getJSON("/api/slaves", function(err, slaves){
if(err) throw err;
if(typeof slaves == "string"){
slaves = JSON.parse(slaves);
}
let instanceIDs = Object.keys(slaves);
let HTML = ""
// Draw in/out chart for unregistered slaves as well
instanceIDs.unshift("unknown");
instanceIDs.forEach(function(instanceID){
if(instanceID == "unknown" || Date.now() - slaves[instanceID].time < 120000 || JSON.parse(localStorage.settings)["Display offline slaves"]){
HTML += '<div id="contentGraph'+instanceID+'" class="storageGraph" style="width: 100%;"></div>';
["place", "remove"].forEach(statistic => {
console.log("Making chart for "+instanceID+"'s "+statistic+" statistic");
setTimeout(function(){
let xmlhttp = new XMLHttpRequest(); // new HttpRequest instance
xmlhttp.open("POST", "/api/getStats");
xmlhttp.setRequestHeader("Content-Type", "application/json");
xmlhttp.onreadystatechange = function(){
console.log(xmlhttp.status);
if(xmlhttp.readyState === XMLHttpRequest.DONE && xmlhttp.status === 200) {
// console.log(xmlhttp.responseText);
if(isJSON(xmlhttp.responseText) && JSON.parse(xmlhttp.responseText).statusForDebugging === undefined){
let slaveDisplayName = "unknown";
try {
slaveDisplayName = slaves[instanceID].instanceName;
} catch(e) {}
let whatGraphIsIt = "Item IO from ";
// if(statistic == "remove") whatGraphIsIt = "Items taken from cluster by";
let chartData = JSON.parse(xmlhttp.response).data;
switch(statistic){
case "place":
chartData.name = "Items exported:";
break;
case "remove":
chartData.name = "Items imported:";
break;
}
drawChartWhenCalledTwice(chartData, whatGraphIsIt + " " + slaveDisplayName);
//drawChart(statistic+"contentGraph"+instanceID, [JSON.parse(xmlhttp.response).data], `${whatGraphIsIt} ${slaveDisplayName} (${instanceID}) ${itemName}`);
// console.log(JSON.parse(xmlhttp.response).data);
}
}
}
xmlhttp.send(JSON.stringify({statistic:statistic, itemName:itemName, instanceID:instanceID}));
},1);
});
let chartLines = [];
function drawChartWhenCalledTwice(data, slaveDisplayName){
chartLines.push(data);
if(chartLines.length >= 2){
drawChart("contentGraph"+instanceID, chartLines, `${slaveDisplayName} (${instanceID}) ${itemName}`);
}
}
}
});
document.querySelector("#graphDisplay > #storageGraphContainer").innerHTML = HTML;
});
}
let chartsByID = {};
function drawChart(selector, chartData, title) {
// selector is ID of element, ex "chartContainer" or "-123199123"
console.log(chartData)
chartsByID[selector] = new CanvasJS.Chart(selector, {
title:{
text: title || "Production graph"
},
toolTip:{
shared: true,
// content: "{name}: {y}",
},
zoomEnabled: true,
axisY:{
includeZero: true,
},
legend: {
cursor: "pointer",
itemclick: function (e) {
//console.log("legend click: " + e.dataPointIndex);
//console.log(e);
if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
e.dataSeries.visible = false;
} else {
e.dataSeries.visible = true;
}
e.chart.render();
}
},
data: chartData
});
chart = chartsByID[selector];
// console.log(chart)
chart.render();
return chart;
}
@@ -3,3 +3,5 @@
require('../index');
require('../masterPlugin');
require('../dole');
require('../dole_nn_base');
+110
View File
@@ -0,0 +1,110 @@
const assert = require("assert").strict;
const express = require("express");
const bodyParser = require("body-parser");
const request = require("supertest");
const Prometheus = require('prom-client');
const jwt = require("jsonwebtoken");
const authenticate = require('lib/authenticate');
const database = require("lib/database");
const routes = require("../routes");
const testSecret = "TestSecretDoNotUse";
authenticate.setAuthSecret(testSecret);
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
describe("clusterioMod endpoints", function() {
const app = express();
app.use(bodyParser.json({ limit: '10mb', }));
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
const config = {
itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 },
disableFairItemDistribution: true,
};
const items = new database.ItemDatabase();
const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] });
routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge);
let persistentMaster = request(app);
describe("#POST /api/place", function() {
it("adds an itemStack to the masters inventory", function() {
return persistentMaster.post("/api/place").send({
name:"steel-plate",
count:20,
instanceName:"unitTest"
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, "success", "something went wrong with the request")
});
});
});
describe("#GET /api/inventory", function() {
it("returns the masters current inventory", function() {
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
let inventory = JSON.parse(res.text);
assert.equal(typeof inventory, "object", "Inventory should be an object");
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
let contains20SteelPlate = false;
inventory.forEach(itemStack => {
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
});
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
});
});
});
describe("#POST /api/remove", function() {
it("returns an itemStack of how many items were removed", function() {
return persistentMaster.post("/api/remove").send({
name:"steel-plate",
count:10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
});
});
it("works correctly for items named addItem or removeItem", async function() {
function add(item) {
return persistentMaster.post("/api/place").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
})
}
function remove(item) {
return persistentMaster.post("/api/remove").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.deepEqual(res.body, {"name":item,"count":10});
});
}
await Promise.all([add("addItem"), add("removeItem")]);
await Promise.all([remove("addItem"), remove("removeItem")]);
});
it("returns an empty itemStack if you don't have any of the item you request", function() {
return persistentMaster.post("/api/remove").send({
name:"imaginaryItem",
count:999999
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
});
});
});
});
@@ -0,0 +1,49 @@
<!doctype HTML>
<head>
<%- include("head") %>
<script defer src="/clusterioMod/storage.js"></script>
</head>
<body>
<%- include("nav") %>
<div id="body" class="container">
<div id="storage">
<div class="page-header mt-2 row" id="banner">
<div class="row">
<div class="col-lg-8 col-md-7 col-sm-6">
<h2>Storage</h2>
</div>
</div>
</div>
<div class="form-group mt-3 mb-3 row">
<input type="text" class="form-control" id="search" placeholder="search">
</div>
<div id="contents" class="row w-100">
<table class="table table-striped table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">Icon</th>
<th scope="col">Item ID</th>
<th scope="col">Quantity</th>
</tr>
</thead>
<tbody><!-- populated by js --></tbody>
</table>
</div>
</div>
<div id="graphDisplay">
<div id="storageGraphContainer">
<div id="contentGraph" class="storageGraph" style="width: 100%;"></div>
</div>
</div>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
crossorigin="anonymous"></script>
</body>