Port and rename clusterioMod plugin

Port the clusterioMod plugin to the new plugin system under the new
subspace_storage name.
This commit is contained in:
Hornwitser committed 2020-03-13 22:46:17 +01:00
1 parent d9fe408c42
commit 737c9924b9
17 files changed
+641 -891

No files matched your search

-10
View File
@@ -1,10 +0,0 @@
/*
Clusterio plugin to allow for chat between instances.
*/
module.exports = {
// Name of package. For display somewhere I guess.
name: "clusterioMod",
version: "1.0.0",
description: "Server side handling of functionality in the clusterio lua mod",
masterPlugin: "masterPlugin.js",
}
-143
View File
@@ -1,143 +0,0 @@
const doleNN = require("./dole_nn_base")
const Prometheus = require('prom-client');
class neuralDole {
getRequestStats(itemname,samples)
{
var sum=0;
samples=Math.min(samples,(this.stats[itemname] || []).length-1);
if (samples<1) return 0.1;
for (var i = 1; i <= samples; i++) {
sum+=this.stats[itemname][i].req;
}
if (sum==0) return 0.1;
return sum/samples;
}
constructor({
items, gaugePrefix
}){
// Set some storage variables for the dole divider
this.prometheusNNDoleGauge = new Prometheus.Gauge({
name: gaugePrefix+'nn_dole_gauge',
help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
labelNames: ["itemName"],
});
this.items = items
this.itemsLastTick = new Map(items._items)
this.dole = {}
this.carry = {}
this.lastRequest = {}
this.stats=[]
this.debt= {}
setInterval(()=>{
for (let [name, count] of this.items._items) {
let magicData = doleNN.Tick(
count,
this.dole[name],
this.itemsLastTick.get(name),
this.getRequestStats(name,5)
)
this.stats[name]=this.stats[name] || []
this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
this.dole[name] = magicData[0]
// DONE handle magicData[1] for graphing for our users
this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
}
this.itemsLastTick = new Map(this.items._items)
}, 1000)
}
divider({
res,
object,
masterConfig,
prometheusImportGauge
}){
let magicData = doleNN.Dose(
object.count, // numReq
this.items.getItemCount(object.name),
this.itemsLastTick.get(object.name) || 0,
this.dole[object.name],
this.carry[object.name+" "+object.instanceID] || 0,
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
this.getRequestStats(object.name,5),
this.debt[object.name+" "+object.instanceID] || 0
)
if ((this.stats[object.name] || []).length>0)
{
this.stats[object.name][0].req+=Number(object.count);
this.stats[object.name][0].given+=Number(magicData[0]);
}
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
//0.Number of items to give in that dose
//1.New dole for item X
//2.New carry for item X slave Y
this.dole[object.name] = magicData[1]
this.carry[object.name+" "+object.instanceID] = magicData[2]
this.debt[object.name+" "+object.instanceID] = magicData[3]
// Remove item from DB and send it
this.items.removeItem(object.name, magicData[0]);
if (masterConfig.get('log_item_transfers')) {
console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
}
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
res.send({
name: object.name,
count: magicData[0],
})
}
}
// If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
let _doleDivisionFactor = {};
function doleDivider({
itemCount,
object,
items,
masterConfig,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
}){
const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
const originalCount = Number(object.count) || 0;
object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
object.count = Math.round(object.count);
if (itemCount > 40) {
console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`);
}
// Update existing items if item name already exists
if (itemCount > object.count) {
//If successful, increase dole
_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
if (masterConfig.get('log_item_transfers')) {
console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
}
items.removeItem(object.name, object.count)
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);
res.send({count: object.count, name: object.name});
} else {
// if we didn't have enough, attempt giving out a smaller amount next time
_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
res.send({name:object.name, count:0});
//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
}
}
module.exports = {
doleDivider,
neuralDole
}
-119
View File
@@ -1,119 +0,0 @@
//Sigmoids...
function sigmoid0(x)
{
return x/(1+Math.abs(x));
}
function sigmoid1(x)
{
if (x>=0.0) {
return 1.0
} else {
return 0.0
}
}
function sigmoid2(x)
{
if (x>=0.0){
return x
} else {
return 0.0
}
}
function sigmoid3(x)
{
return x;
}
//DOSE - Doses items for item X slave Y
//Numreq - number of items that are requested for item X for slave Y
//Instore - number of items for item X in master gauge (currently)
//store_last_tick - Number of items for item X in master gauge at time of last tick
//dole - Current dole for item X
//carry - carry for item X slave Y
//prev_req - Numreq for previous request for item X slave Y
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
//debt - debt for item X slave Y
//Returns: Array of 3 elements
//0.Number of items to give in that dose
//1.New dole for item X
//2.New carry for item X slave Y
//3.New debt
function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj,debt)
{
numreq=Number(numreq);
instore=Number(instore);
var instore_adj=instore;
if (instore_adj==0) instore_adj=0.1;
var b=Math.trunc(prev_req/instore_adj*numreq);
var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b,numreq_total_adj,store_last_tick,debt];//5..12
var inputs=inp1.concat(inp2);
var internal=new Array(20).fill(0);
var outputs=[0,0,0,0];
//NN: Please don't touch :), code is generated
outputs[0]=inputs[7]*0.8058582544;
outputs[0]=sigmoid2(outputs[0]);
outputs[1]=sigmoid0(outputs[1]);
outputs[2]=sigmoid0(outputs[2]);
outputs[3]=inputs[6]*0.2294657379;
outputs[3]=sigmoid0(outputs[3]);
debt=debt || 0;
if (outputs[0]>0.99) outputs[0]=1.0;
var res1=Number(outputs[0])*numreq+outputs[3]+debt;
var res_capped=Math.round(res1);
if (res_capped>numreq) res_capped=numreq;
if (res_capped>instore) res_capped=instore;
if (res_capped<0) res_capped=0;
res1=res1 || 0;
res_capped=res_capped || 0;//Safeguard against wrong values
debt=res1-res_capped;
debt_cap=Math.round(numreq*1.5)+5;
if (debt>debt_cap) debt=debt_cap;
carry=sigmoid0(outputs[2]*carry+outputs[0]) || 0;
return [res_capped,dole+outputs[1],carry,debt];
}
//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
//Instore - number of items for item X in master gauge (currently)
//dole - Current dole for item X
//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
//Returns: Array of 2 elements
//0.New dole for item X
//1.Guessed percentage average of items supplied in last 10 ticks
function Tick(instore,dole,store_last_tick,numreq_total_adj)
{
var instore_adj=instore;
if (instore_adj==0) instore_adj=0.1;
var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj,store_last_tick,numreq_total_adj];
var internal=new Array(10).fill(0);
var outputs=[0,0];
//NN: Please don't touch :), code is generated
outputs[0]=inputs[3]*0.9364774823;
outputs[0]=outputs[0]*1.0*0.4590721726;
outputs[0]=sigmoid2(outputs[0]);
outputs[1]=outputs[0]*1.828422785;
outputs[1]=sigmoid3(outputs[1]);
avg=outputs[1];
if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
if (avg>1) avg=1;
if (avg<0) avg=0;
return [sigmoid0(outputs[0]),avg];
}
module.exports = {Dose,Tick}
-194
View File
@@ -1,194 +0,0 @@
const needle = require("needle");
const fs = require("fs-extra");
const objectOps = require("lib/objectOps");
const fileOps = require("lib/fileOps");
const pluginConfig = require("./config");
function ensureFileSync(path) {
if (!fs.existsSync(path)) {
fs.outputFileSync(path, "");
}
}
module.exports = class remoteCommands {
constructor(mergedConfig, messageInterface, extras){
// Ugly global
global.confirmedOrders = [];
this.messageInterface = messageInterface;
this.config = mergedConfig;
this.socket = extras.socket;
const that = this;
const instance = process.argv[3];
const instancedirectory = this.config.instanceDirectory + '/' + instance;
const needleOptionsWithTokenAuthHeader = {
headers: {
'x-access-token': this.config.masterAuthToken
},
};
ensureFileSync(instancedirectory + "/script-output/output.txt");
ensureFileSync(instancedirectory + "/script-output/orders.txt");
// provide items --------------------------------------------------------------
// trigger when something happens to output.txt
fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) {
// get array of lines in file
let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n");
// if you found anything, reset the file
if (items[0]) {
fs.writeFileSync(instancedirectory + "/script-output/output.txt", "");
}
for (let i = 0; i < items.length; i++) {
if (items[i]) {
let g = items[i].split(" ");
g[0] = g[0].replace("\u0000", "");
// console.log("exporting " + JSON.stringify(g));
// send our entity and count to the master for him to keep track of
needle.post(that.config.masterURL + '/api/place', {
name: g[0],
count: g[1],
instanceName: instance, // name of instance
instanceID: that.config.unique, // a hash computed from the randomly generated rcon password
}, needleOptionsWithTokenAuthHeader, function (err, resp, body) {
if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]);
if(that.config.logItemTransfers){
if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`);
}
});
}
}
});
// request items --------------------------------------------------------------
setInterval(function () {
// get array of lines in file
let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n");
// if we actually got anything from the file, proceed and reset file
if (items[0]) {
fs.writeFileSync(instancedirectory + "/script-output/orders.txt", "");
// prepare a package of all our requested items in a more transfer friendly format
var preparedPackage = {};
for (let i = 0; i < items.length; i++) {
(function (i) {
if (items[i]) {
items[i] = items[i].split(" ");
items[i][0] = items[i][0].replace("\u0000", "");
items[i][0] = items[i][0].replace(",", "");
if (preparedPackage[items[i][0]]) {
// if we have buffered some already, sum the new items
if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") {
preparedPackage[items[i][0]] = {
"name": items[i][0],
"count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]),
"instanceName":instance,
"instanceID":that.config.unique,
};
// else just add em in without summing
} else if (typeof Number(items[i][1]) == "number") {
preparedPackage[items[i][0]] = {
"name": items[i][0],
"count": Number(items[i][1]),
"instanceName":instance,
"instanceID":that.config.unique,
};
}
// this condition will NEVER be triggered but we know how that goes
} else if (typeof Number(items[i][1]) == "number") {
preparedPackage[items[i][0]] = {
"name": items[i][0],
"count": Number(items[i][1]),
"instanceName":instance,
"instanceID":that.config.unique,
};
}
}
})(i);
}
// request our items, one item at a time
for (let i = 0; i < Object.keys(preparedPackage).length; i++) {
// console.log(preparedPackage[Object.keys(preparedPackage)[i]]);
function callback(err, response, body) {
if (response && response.body && typeof response.body == "object") {
// buffer confirmed orders
confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count}
if(that.config.logItemTransfers){
console.log(`Imported ${response.body.count} ${response.body.name} from master`);
}
}
}
needle.post(
that.config.masterURL + '/api/remove',
preparedPackage[Object.keys(preparedPackage)[i]],
needleOptionsWithTokenAuthHeader,
callback
);
}
// if we got some confirmed orders
// console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders));
//if (!(confirmedOrders.length>0)){return;}
let cmd="local t={";
for(let i=0;i<confirmedOrders.length;i++)
{
cmd+='["'+confirmedOrders[i].name+'"]='+confirmedOrders[i].count+',';
}
if (!(cmd==="local t={")){
that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
}
confirmedOrders=[];
}
}, 1000);
// COMBINATOR SIGNALS ---------------------------------------------------------
// get inventory from Master and RCON it to our slave
setInterval(() => {
needle.get(that.config.masterURL + '/api/inventory', function (err, response, body) {
if(err){
console.error("Unable to get JSON master/api/inventory, master might be unaccessible");
} else if (response && response.body) {
// Take the inventory we (hopefully) got and turn it into the format LUA accepts
if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver
try {
var inventory = JSON.parse(response.body);
var inventoryFrame = {};
for (let i = 0; i < inventory.length; i++) {
inventoryFrame[inventory[i].name] = Number(inventory[i].count);
if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){
inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice
}
}
inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000);
// console.log("RCONing inventory! " + JSON.stringify(inventoryFrame));
let first = true;
let cmd="local s={";
for (let key in inventoryFrame)
{
cmd+='["'+key+'"]='+inventoryFrame[key]+",";
}
if (!(cmd==="local s={")){
that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
}
that.messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')");
} catch (e){
console.log(e);
}
}
});
}, 1550);
// Make sure world has its worldID
setTimeout(function(){
messageInterface("/silent-command remote.call('clusterio','setWorldID',"+that.config.unique+")")
}, 20000);
}
async factorioOutput(data){
}
}
async function sleep(s){
return new Promise((resolve, reject) => {
setTimeout(resolve, s*1000);
});
}
-101
View File
@@ -1,101 +0,0 @@
"use strict";
const fs = require("fs-extra");
const path = require("path");
const Express = require("express");
const ejs = require("ejs");
const database = require("lib/database");
const routes = require("./routes");
class masterPlugin {
constructor({
config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter,
}) {
this.config = config;
this.pluginConfig = pluginConfig;
this.pluginPath = pluginPath;
this.io = socketio;
this.express = express;
this.db = db,
this.Prometheus = Prometheus;
this.prometheusPrefix = prometheusPrefix;
this.endpointHitCounter = endpointHitCounter;
this.prometheusMasterInventoryGauge = new Prometheus.Gauge({
name: prometheusPrefix+'master_inventory_gauge',
help: 'Amount of items stored on master',
labelNames: ["itemName"],
});
this.ui = {
sidebar: [
{
getHtml: () => `
<div class="nav-item mr-1">
<a class="nav-link align-middle" href="/clusterioMod/storage">Storage</a>
</div>`,
},
],
};
}
onMetrics() {
if (this.items) {
for (let [key, count] of this.items._items) {
this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
}
}
}
async onLoadFinish() {
this.items = await loadDatabase(this.config);
this.autosaveId = setInterval(async () => {
await saveDatabase(this.config, this.items);
},this.config.autosaveInterval || 60000);
routes.addApiRoutes(
this.express, this.config, this.db, this.items,
this.Prometheus, this.prometheusPrefix, this.endpointHitCounter,
);
routes.addWebRoutes(this.express);
}
async onExit(){
clearInterval(this.autosaveId);
await saveDatabase(this.config, this.items);
}
}
async function loadDatabase(config) {
let itemsPath = path.resolve(config.databaseDirectory, "items.json");
console.log(`Loading ${itemsPath}`);
try {
let content = await fs.readFile(itemsPath);
return new database.ItemDatabase(JSON.parse(content));
} catch (err) {
if (err.code === "ENOENT") {
console.log("Creating new item database");
return new database.ItemDatabase();
} else {
throw err;
}
}
}
async function saveDatabase(config, items) {
if (items && items.size < 50000) {
let file = path.resolve(config.databaseDirectory, "items.json");
console.log(`writing ${file}`);
let content = JSON.stringify(items.serialize());
await fs.outputFile(file, content);
} else if (items) {
console.error(`Item database too large, not saving (${items.size})`);
}
}
module.exports = masterPlugin;
-208
View File
@@ -1,208 +0,0 @@
const express = require("express");
const path = require("path");
const authenticate = require("lib/authenticate");
const dole = require("./dole");
function addWebRoutes(app) {
app.use('/clusterioMod', express.static(path.join(__dirname, 'static')));
app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); });
}
function addApiRoutes(
app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter,
) {
const prometheusExportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'export_gauge',
help: 'Items exported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusImportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'import_gauge',
help: 'Items imported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusDoleFactorGauge = new Prometheus.Gauge({
name: prometheusPrefix+'dole_factor_gauge',
help: 'The current dole division factor for this item',
labelNames: ["itemName"],
});
// Only initialize neural network when it's enabled, otherwise it might override gauge
let neuralDole;
if(config.useNeuralNetDoleDivider) {
neuralDole=new dole.neuralDole({
db, gaugePrefix: prometheusPrefix
});
}
/**
* POST endpoint for storing items in master's inventory.
* @memberof clusterioMaster
* @instance
* @alias api/place
* @param {itemStack} itemStack the number and type of items to store (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {string} status either "success" or "failure"
*/
app.post("/api/place", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
let x;
try {
x = JSON.parse(req.body);
} catch (e) {
x = req.body;
}
if(!x.instanceName) {
x.instanceName = "unknown";
}
if(!x.instanceID) {
x.instanceID = "unknown";
for (let [instanceID, slave] of db.slaves) {
if(slave.instanceName == req.body.instanceName) {
x.instanceID = instanceID;
break;
}
}
}
if(x.instanceID
&& x.instanceName
&& !isNaN(Number(x.count))
&& x.name
&& typeof x.name == "string"){
if(config.logItemTransfers){
console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
}
// gather statistics
prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
// save items we get
var count = Number(req.body.count);
items.addItem(req.body.name, count);
// Attempt confirming
res.send("success");
} else {
res.send("failure");
}
});
/**
* POST endpoint to remove items from DB when client orders items.
*
* @memberof clusterioMaster
* @instance
* @alias api/remove
* @param {itemStack} itemStack the name of and the number of items to remove (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {itemStack}
* the number of items actually removed, may be lower than what was
* asked for due to shortages.
*/
app.post("/api/remove", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// save items we get
var object = req.body;
if(!object.instanceID) {
object.instanceID = "unknown"
}
if(!object.instanceName) {
object.instanceName = "unknown";
}
if (db.slaves.has(String(object.instanceID))) {
object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
}
let itemCount = items.getItemCount(object.name);
if (
(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
&& object.name != "electricity"
) {
if(config.logItemTransfers){
console.log('failure could not find ' + object.name);
}
res.send({name:object.name, count:0});
} else if(config.disableFairItemDistribution){
// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
items.removeItem(object.name, numberToRemove);
res.send({count: numberToRemove, name: object.name});
// track statistics and do graphing things
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
} else if(config.useNeuralNetDoleDivider){
// use fancy neural net to calculate a "fair" dole division rate.
neuralDole.divider({
res,
object,
config,
prometheusImportGauge
})
} else {
// Use dole division. Makes it really slow to drain out the last little bit.
dole.doleDivider({
itemCount,
object,
db,
config,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
})
}
});
/**
* GET endpoint to read the masters current inventory of items.
*
* @memberof clusterioMaster
* @instance
* @alias api/inventory
* @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
*/
app.get("/api/inventory", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
var inventory = [];
for (let [name, count] of items._items) {
inventory.push({ name, count });
}
res.send(JSON.stringify(inventory));
});
/**
* GET endpoint to read the masters inventory as an object with key:value pairs
*
* @memberof clusterioMaster
* @instance
* @alias api/inventoryAsObject
* @returns {object} JSON {"iron-plate":100, "copper-plate":5}
*/
app.get("/api/inventoryAsObject", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
res.send(JSON.stringify(items.serialise()));
});
}
module.exports = {
addWebRoutes,
addApiRoutes,
}
@@ -1,208 +0,0 @@
// function to draw data we recieve from ajax requests
_lastData = [];
function drawcontents(data) {
data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box.
_lastData = data;
// allow searching for multiple criteria separated by space
var searchArgs = document.querySelector("#search").value;
searchArgs = searchArgs.trim();
searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')';
const search = new RegExp(searchArgs, 'i');
data = data.filter(function(item) {
return search.test(item.name);
})
sortByKey(data, "count");
const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically
const rows = table.children;
//update existing rows or create new ones
data.forEach(function(item, i) {
// format count to have .s in it for large number readability (1,000,000)
item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
let row = rows[i];
if(!row) {
row = document.createElement('tr');
row.innerHTML = "<td><img width=32 height=32></td><td class=name></td><td class=count></td>";
table.appendChild(row);
}
const img = row.querySelector('img');
const imgName = getImageFromName(item.name);
if(img.getAttribute('src') !== imgName) {
img.setAttribute('src',imgName);
}
const name = row.querySelector('.name');
if(name.textContent !== item.name) {
name.textContent = item.name;
row.onclick = ()=>{
makeItemGraphs(item.name);
}
}
const count = row.querySelector('.count');
if(count.textContent !== ''+item.count) {
count.textContent = item.count;
}
});
//remove excess rows, for example, after filtering
while (data.length < rows.length) {
table.removeChild(rows[data.length]);
}
}
// get cluster inventory from master
function updateInventory() {
var xmlhttp = new XMLHttpRequest();
xmlhttp.onreadystatechange = function() {
if(xmlhttp.readyState == 4 && xmlhttp.status == 200) {
let data = JSON.parse(xmlhttp.responseText);
drawcontents(data);
}
}
xmlhttp.open("GET", "/api/inventory", true);
xmlhttp.send();
}
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
setInterval(updateInventory, 500);
} else {
updateInventory();
}
// function to sort arrays of objects after a keys value
function sortByKey(array, key) {
array.sort(function(a, b) {
return b[key] - a[key];
});
}
document.querySelector("#search").addEventListener('input', function() {
drawcontents();
});
// Make graph of cluster IO items
/*getJSON("/api/getStats", function(err, data){
}*/
function getItemStatData(item, callback){
var xhr = new XMLHttpRequest();
var url = "/api/getStats";
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-type", "application/json");
xhr.onreadystatechange = function () {
if (xhr.readyState === 4 && xhr.status === 200) {
var json = JSON.parse(xhr.responseText);
console.log(json.email + ", " + json.password);
callback(json);
}
};
var requestData = JSON.stringify({
"statistic": "place",
"itemName": "shit",
});
xhr.send(data);
}
function makeItemGraphs(itemName){
getJSON("/api/slaves", function(err, slaves){
if(err) throw err;
if(typeof slaves == "string"){
slaves = JSON.parse(slaves);
}
let instanceIDs = Object.keys(slaves);
let HTML = ""
// Draw in/out chart for unregistered slaves as well
instanceIDs.unshift("unknown");
instanceIDs.forEach(function(instanceID){
if(instanceID == "unknown" || Date.now() - slaves[instanceID].time < 120000 || JSON.parse(localStorage.settings)["Display offline slaves"]){
HTML += '<div id="contentGraph'+instanceID+'" class="storageGraph" style="width: 100%;"></div>';
["place", "remove"].forEach(statistic => {
console.log("Making chart for "+instanceID+"'s "+statistic+" statistic");
setTimeout(function(){
let xmlhttp = new XMLHttpRequest(); // new HttpRequest instance
xmlhttp.open("POST", "/api/getStats");
xmlhttp.setRequestHeader("Content-Type", "application/json");
xmlhttp.onreadystatechange = function(){
console.log(xmlhttp.status);
if(xmlhttp.readyState === XMLHttpRequest.DONE && xmlhttp.status === 200) {
// console.log(xmlhttp.responseText);
if(isJSON(xmlhttp.responseText) && JSON.parse(xmlhttp.responseText).statusForDebugging === undefined){
let slaveDisplayName = "unknown";
try {
slaveDisplayName = slaves[instanceID].instanceName;
} catch(e) {}
let whatGraphIsIt = "Item IO from ";
// if(statistic == "remove") whatGraphIsIt = "Items taken from cluster by";
let chartData = JSON.parse(xmlhttp.response).data;
switch(statistic){
case "place":
chartData.name = "Items exported:";
break;
case "remove":
chartData.name = "Items imported:";
break;
}
drawChartWhenCalledTwice(chartData, whatGraphIsIt + " " + slaveDisplayName);
//drawChart(statistic+"contentGraph"+instanceID, [JSON.parse(xmlhttp.response).data], `${whatGraphIsIt} ${slaveDisplayName} (${instanceID}) ${itemName}`);
// console.log(JSON.parse(xmlhttp.response).data);
}
}
}
xmlhttp.send(JSON.stringify({statistic:statistic, itemName:itemName, instanceID:instanceID}));
},1);
});
let chartLines = [];
function drawChartWhenCalledTwice(data, slaveDisplayName){
chartLines.push(data);
if(chartLines.length >= 2){
drawChart("contentGraph"+instanceID, chartLines, `${slaveDisplayName} (${instanceID}) ${itemName}`);
}
}
}
});
document.querySelector("#graphDisplay > #storageGraphContainer").innerHTML = HTML;
});
}
let chartsByID = {};
function drawChart(selector, chartData, title) {
// selector is ID of element, ex "chartContainer" or "-123199123"
console.log(chartData)
chartsByID[selector] = new CanvasJS.Chart(selector, {
title:{
text: title || "Production graph"
},
toolTip:{
shared: true,
// content: "{name}: {y}",
},
zoomEnabled: true,
axisY:{
includeZero: true,
},
legend: {
cursor: "pointer",
itemclick: function (e) {
//console.log("legend click: " + e.dataPointIndex);
//console.log(e);
if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
e.dataSeries.visible = false;
} else {
e.dataSeries.visible = true;
}
e.chart.render();
}
},
data: chartData
});
chart = chartsByID[selector];
// console.log(chart)
chart.render();
return chart;
}
@@ -1,7 +0,0 @@
// These are libraries that does not have any units test, they are
// required here in order to track coverage for them.
require('../index');
require('../masterPlugin');
require('../dole');
require('../dole_nn_base');
-109
View File
@@ -1,109 +0,0 @@
const assert = require("assert").strict;
const express = require("express");
const bodyParser = require("body-parser");
const request = require("supertest");
const Prometheus = require('prom-client');
const jwt = require("jsonwebtoken");
const authenticate = require('lib/authenticate');
const database = require("lib/database");
const routes = require("../routes");
const testSecret = "TestSecretDoNotUse";
authenticate.setAuthSecret(testSecret);
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
describe("clusterioMod endpoints", function() {
const app = express();
app.use(bodyParser.json({ limit: '10mb', }));
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
const config = {
disableFairItemDistribution: true,
};
const items = new database.ItemDatabase();
const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] });
routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge);
let persistentMaster = request(app);
describe("#POST /api/place", function() {
it("adds an itemStack to the masters inventory", function() {
return persistentMaster.post("/api/place").send({
name:"steel-plate",
count:20,
instanceName:"unitTest"
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, "success", "something went wrong with the request")
});
});
});
describe("#GET /api/inventory", function() {
it("returns the masters current inventory", function() {
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
let inventory = JSON.parse(res.text);
assert.equal(typeof inventory, "object", "Inventory should be an object");
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
let contains20SteelPlate = false;
inventory.forEach(itemStack => {
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
});
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
});
});
});
describe("#POST /api/remove", function() {
it("returns an itemStack of how many items were removed", function() {
return persistentMaster.post("/api/remove").send({
name:"steel-plate",
count:10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
});
});
it("works correctly for items named addItem or removeItem", async function() {
function add(item) {
return persistentMaster.post("/api/place").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
})
}
function remove(item) {
return persistentMaster.post("/api/remove").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.deepEqual(res.body, {"name":item,"count":10});
});
}
await Promise.all([add("addItem"), add("removeItem")]);
await Promise.all([remove("addItem"), remove("removeItem")]);
});
it("returns an empty itemStack if you don't have any of the item you request", function() {
return persistentMaster.post("/api/remove").send({
name:"imaginaryItem",
count:999999
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
});
});
});
});
@@ -1,49 +0,0 @@
<!doctype HTML>
<head>
<%- include("head") %>
<script defer src="/clusterioMod/storage.js"></script>
</head>
<body>
<%- include("nav") %>
<div id="body" class="container">
<div id="storage">
<div class="page-header mt-2 row" id="banner">
<div class="row">
<div class="col-lg-8 col-md-7 col-sm-6">
<h2>Storage</h2>
</div>
</div>
</div>
<div class="form-group mt-3 mb-3 row">
<input type="text" class="form-control" id="search" placeholder="search">
</div>
<div id="contents" class="row w-100">
<table class="table table-striped table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">Icon</th>
<th scope="col">Item ID</th>
<th scope="col">Quantity</th>
</tr>
</thead>
<tbody><!-- populated by js --></tbody>
</table>
</div>
</div>
<div id="graphDisplay">
<div id="storageGraphContainer">
<div id="contentGraph" class="storageGraph" style="width: 100%;"></div>
</div>
</div>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
crossorigin="anonymous"></script>
</body>