diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2adbb981..844e5665 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,10 @@ Version 2.0.0
time the instance is started.
- Removed the broken client download command as this would download
pre-releases.
+- Moved the item database and HTTP definitions for /api/place, /api/remove,
+ /api/inventory, /api/inventoryAsObject and the web interface view for the
+ storage page into the clusterioMod plugin. If you disable this plugin
+ then these things will not be available.
### Breaking Changes
diff --git a/master.js b/master.js
index 3acb2be3..a154a6d9 100644
--- a/master.js
+++ b/master.js
@@ -17,7 +17,6 @@ node master
// argument parsing
const args = require('minimist')(process.argv.slice(2));
-const averagedTimeSeries = require("averaged-timeseries");
const deepmerge = require("deepmerge");
const path = require("path");
const fs = require("fs-extra");
@@ -99,21 +98,6 @@ const prometheusWsUsageCounter = new Prometheus.Counter({
help: 'Websocket traffic',
labelNames: ["connectionType", "instanceID"],
});
-const prometheusExportGauge = new Prometheus.Gauge({
- name: prometheusPrefix+'export_gauge',
- help: 'Items exported by instance',
- labelNames: ["instanceID", "itemName"],
-});
-const prometheusImportGauge = new Prometheus.Gauge({
- name: prometheusPrefix+'import_gauge',
- help: 'Items imported by instance',
- labelNames: ["instanceID", "itemName"],
-});
-const prometheusDoleFactorGauge = new Prometheus.Gauge({
- name: prometheusPrefix+'dole_factor_gauge',
- help: 'The current dole division factor for this item',
- labelNames: ["itemName"],
-});
const prometheusPlayerCountGauge = new Prometheus.Gauge({
name: prometheusPrefix+'player_count_gauge',
help: 'Amount of players connected to this cluster',
@@ -124,11 +108,6 @@ const prometheusUPSGauge = new Prometheus.Gauge({
help: 'UPS of the current server',
labelNames: ["instanceID", "instanceName"],
});
-const prometheusMasterInventoryGauge = new Prometheus.Gauge({
- name: prometheusPrefix+'master_inventory_gauge',
- help: 'Amount of items stored on master',
- labelNames: ["itemName"],
-});
/**
GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format.
@memberof clusterioMaster
@@ -156,10 +135,14 @@ function registerMoreMetrics(){
if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0);
}catch(e){}
}
- // inventory
- for (let [key, count] of db.items._items) {
- prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
+
+ // plugins
+ for (let plugin of masterPlugins) {
+ if (plugin.main && plugin.main.onMetrics && typeof plugin.main.onMetrics == "function") {
+ plugin.main.onMetrics();
+ }
}
+
// Slave count
let numberOfActiveSlaves = 0;
for(let slave of db.slaves.values()){
@@ -175,26 +158,10 @@ async function loadDatabase(databaseDirectory) {
let slavesPath = path.resolve(databaseDirectory, "slaves.json");
console.log(`Loading ${slavesPath}`);
db.slaves = await database.loadJsonAsMap(slavesPath);
-
- let itemsPath = path.resolve(databaseDirectory, "items.json");
- console.log(`Loading ${itemsPath}`);
- try {
- let content = await fs.readFile(itemsPath);
- db.items = new database.ItemDatabase(JSON.parse(content));
-
- } catch (err) {
- if (err.code === "ENOENT") {
- console.log("Creating new item database");
- db.items = new database.ItemDatabase();
-
- } else {
- throw err;
- }
- }
}
-// store slaves and inventory in a .json full of JSON data
+// store slaves in a .json full of JSON data
async function shutdown() {
console.log('Ctrl-C...');
let exitStartTime = Date.now();
@@ -208,14 +175,6 @@ async function shutdown() {
console.error(`Slave database too large, not saving (${db.slaves.size}`);
}
- if (db.items && db.items.size < 50000) {
- let file = path.resolve(config.databaseDirectory, "items.json");
- console.log(`writing ${file}`);
- let content = JSON.stringify(db.items.serialize());
- await fs.outputFile(file, content);
- } else if (db.items) {
- console.error(`Item database too large, not saving (${db.items.size})`);
- }
for(let i in masterPlugins){
let plugin = masterPlugins[i];
if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
@@ -410,213 +369,14 @@ app.post("/api/rconPasswords", authenticate.middleware, function(req, res) {
res.send(slaveCache.cache);
});
-var recievedItemStatisticsBySlaveID = {};
-var sentItemStatisticsBySlaveID = {};
-//
-/**
-POST endpoint for storing items in master's inventory.
-
-@memberof clusterioMaster
-@instance
-@alias api/place
-@param {itemStack} itemStack the number and type of items to store (see typedef)
-@param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance
-@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name]
-@returns {string} status either "success" or "failure"
-*/
-app.post("/api/place", authenticate.middleware, function(req, res) {
- endpointHitCounter.labels(req.route.path).inc();
- res.header("Access-Control-Allow-Origin", "*");
- res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
- let x;
- try {
- x = JSON.parse(req.body);
- } catch (e) {
- x = req.body;
- }
- if(!x.instanceName) {
- x.instanceName = "unknown";
- }
- if(!x.instanceID) {
- x.instanceID = "unknown";
- for (let [instanceID, slave] of db.slaves) {
- if(slave.instanceName == req.body.instanceName) {
- x.instanceID = instanceID;
- break;
- }
- }
- }
- if(x.instanceID
- && x.instanceName
- && !isNaN(Number(x.count))
- && x.name
- && typeof x.name == "string"){
- if(config.logItemTransfers){
- console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
- }
- // gather statistics
- let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
- if(recievedItemStatistics === undefined){
- recievedItemStatistics = new averagedTimeSeries({
- maxEntries: config.itemStats.maxEntries,
- entriesPerSecond: config.itemStats.entriesPerSecond,
- mergeMode: "add",
- }, console.log);
- recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
- }
- recievedItemStatistics.add({
- key:req.body.name,
- value:req.body.count,
- });
- prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
- // save items we get
-
- var count = Number(req.body.count);
- db.items.addItem(req.body.name, count);
-
- // Attempt confirming
- res.send("success");
- } else {
- res.send("failure");
- }
-});
-const routes_api_remove = require("./routes/api/remove");
-let neuralDole;
-
-/**
-POST endpoint to remove items from DB when client orders items.
-
-@memberof clusterioMaster
-@instance
-@alias api/remove
-@param {itemStack} itemStack the name of and the number of items to remove (see typedef)
-@param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance
-@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name]
-@returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
-*/
-app.post("/api/remove", authenticate.middleware, function(req, res) {
- endpointHitCounter.labels(req.route.path).inc();
- res.header("Access-Control-Allow-Origin", "*");
- res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
- // save items we get
- var object = req.body;
- if(!object.instanceID) {
- object.instanceID = "unknown"
- }
- if(!object.instanceName) {
- object.instanceName = "unknown";
- }
- if (db.slaves.has(String(object.instanceID))) {
- object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
- }
- let itemCount = db.items.getItemCount(object.name);
- if (
- (config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
- && object.name != "electricity"
- ) {
- if(config.logItemTransfers){
- console.log('failure could not find ' + object.name);
- }
- res.send({name:object.name, count:0});
- } else if(config.disableFairItemDistribution){
- // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
- let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
- db.items.removeItem(object.name, numberToRemove);
- res.send({count: numberToRemove, name: object.name});
-
- // track statistics and do graphing things
- prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
- let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
- if(sentItemStatistics === undefined){
- sentItemStatistics = new averagedTimeSeries({
- maxEntries: config.itemStats.maxEntries,
- entriesPerSecond: config.itemStats.entriesPerSecond,
- mergeMode: "add",
- }, console.log);
- }
- sentItemStatistics.add({
- key:object.name,
- value:numberToRemove,
- });
- //console.log(sentItemStatistics.data)
- sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
- } else if(config.useNeuralNetDoleDivider){
- // use fancy neural net to calculate a "fair" dole division rate.
- neuralDole.divider({
- res,
- object,
- config,
- sentItemStatisticsBySlaveID,
- prometheusImportGauge
- })
- } else {
- // Use dole division. Makes it really slow to drain out the last little bit.
- routes_api_remove.doleDivider({
- itemCount,
- object,
- db,
- sentItemStatisticsBySlaveID,
- config,
- prometheusDoleFactorGauge,
- prometheusImportGauge,
- req,res,
- })
- }
-});
-
-/**
-GET endpoint to read the masters current inventory of items.
-
-@memberof clusterioMaster
-@instance
-@alias api/inventory
-@returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
-*/
-// endpoint for getting an inventory of what we got
-app.get("/api/inventory", function(req, res) {
- endpointHitCounter.labels(req.route.path).inc();
- res.header("Access-Control-Allow-Origin", "*");
- res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
- // Check it and send it
- var inventory = [];
- for (let [name, count] of db.items._items) {
- inventory.push({ name, count });
- }
- res.send(JSON.stringify(inventory));
-});
-/**
-GET endpoint to read the masters inventory as an object with key:value pairs
-
-@memberof clusterioMaster
-@instance
-@alias api/inventoryAsObject
-@returns {object} JSON {"iron-plate":100, "copper-plate":5}
-*/
-app.get("/api/inventoryAsObject", function(req, res) {
- endpointHitCounter.labels(req.route.path).inc();
- res.header("Access-Control-Allow-Origin", "*");
- res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
- // Check it and send it
- res.send(JSON.stringify(db.items.serialise()));
-});
-
-/**
- GET endpoint to read the masters inventory as an object with key:value pairs
-
- @memberof clusterioMaster
- @instance
- @alias api/modData
- @returns {object} JSON
- */
-
// wrap a request in an promise
- async function downloadPage(url) {
- return new Promise((resolve, reject) => {
- request(url, (error, response, body) => {
- resolve(body);
- });
+async function downloadPage(url) {
+ return new Promise((resolve, reject) => {
+ request(url, (error, response, body) => {
+ resolve(body);
});
- }
+ });
+}
app.get("/api/modmeta", async function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
@@ -832,7 +592,10 @@ async function getPlugins(){
path: pluginConfig.pluginPath,
socketio: io,
express: app,
+ db,
Prometheus,
+ prometheusPrefix,
+ endpointHitCounter,
}),
pluginConfig,
});
@@ -900,12 +663,6 @@ async function startServer() {
fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
expiresIn: 86400*365 // expires in 1 year
}));
- setInterval(async function() {
- let file = path.resolve(config.databaseDirectory, "items.json");
- console.log(`autosaving ${file}`);
- let content = JSON.stringify(db.items.serialize());
- await fs.outputFile(file, content);
- },config.autosaveInterval || 60000);
process.on('SIGINT', shutdown); // ctrl + c
process.on('SIGHUP', shutdown); // terminal closed
@@ -921,13 +678,6 @@ async function startServer() {
await loadDatabase(config.databaseDirectory);
authenticate.setAuthSecret(config.masterAuthSecret);
- // Only initialize neural network when it's enabled, otherwise it might override gauge
- if(config.useNeuralNetDoleDivider) {
- neuralDole=new routes_api_remove.neuralDole({
- db, gaugePrefix: prometheusPrefix
- });
- }
-
// Make sure we're actually going to listen on a port
let httpPort = args.masterPort || config.masterPort;
let httpsPort = args.sslPort || config.sslPort;
diff --git a/routes/api/remove.js b/sharedPlugins/clusterioMod/dole.js
similarity index 92%
rename from routes/api/remove.js
rename to sharedPlugins/clusterioMod/dole.js
index 382dade9..3ddbf7f7 100644
--- a/routes/api/remove.js
+++ b/sharedPlugins/clusterioMod/dole.js
@@ -1,6 +1,6 @@
const averagedTimeSeries = require("averaged-timeseries")
-const doleNN = require("lib/dole_nn_base")
+const doleNN = require("./dole_nn_base")
const Prometheus = require('prom-client');
@@ -19,7 +19,7 @@ class neuralDole {
}
constructor({
- db, gaugePrefix
+ items, gaugePrefix
}){
// Set some storage variables for the dole divider
this.prometheusNNDoleGauge = new Prometheus.Gauge({
@@ -27,8 +27,8 @@ class neuralDole {
help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
labelNames: ["itemName"],
});
- this.db = db
- this.itemsLastTick = new Map(db.items._items)
+ this.items = items
+ this.itemsLastTick = new Map(items._items)
this.dole = {}
this.carry = {}
this.lastRequest = {}
@@ -36,7 +36,7 @@ class neuralDole {
this.debt= {}
setInterval(()=>{
- for (let [name, count] of this.db.items._items) {
+ for (let [name, count] of this.items._items) {
let magicData = doleNN.Tick(
count,
this.dole[name],
@@ -51,7 +51,7 @@ class neuralDole {
// DONE handle magicData[1] for graphing for our users
this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
}
- this.itemsLastTick = new Map(this.db.items._items)
+ this.itemsLastTick = new Map(this.items._items)
}, 1000)
}
divider({
@@ -63,7 +63,7 @@ class neuralDole {
}){
let magicData = doleNN.Dose(
object.count, // numReq
- this.db.items.getItemCount(object.name),
+ this.items.getItemCount(object.name),
this.itemsLastTick.get(object.name) || 0,
this.dole[object.name],
this.carry[object.name+" "+object.instanceID] || 0,
@@ -85,9 +85,9 @@ class neuralDole {
this.debt[object.name+" "+object.instanceID] = magicData[3]
// Remove item from DB and send it
- this.db.items.removeItem(object.name, magicData[0]);
+ this.items.removeItem(object.name, magicData[0]);
if(config.logItemTransfers){
- console.log("removed: " + object.name + " " + magicData[0] + " . " + this.db.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
+ console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
}
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
@@ -116,7 +116,7 @@ _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this
function doleDivider({
itemCount,
object,
- db,
+ items,
sentItemStatisticsBySlaveID,
config,
prometheusDoleFactorGauge,
@@ -140,7 +140,7 @@ function doleDivider({
if (config.logItemTransfers) {
console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
}
- db.items.removeItem(object.name, object.count)
+ items.removeItem(object.name, object.count)
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
diff --git a/lib/dole_nn_base.js b/sharedPlugins/clusterioMod/dole_nn_base.js
similarity index 100%
rename from lib/dole_nn_base.js
rename to sharedPlugins/clusterioMod/dole_nn_base.js
diff --git a/sharedPlugins/clusterioMod/masterPlugin.js b/sharedPlugins/clusterioMod/masterPlugin.js
index e02ca4b0..7151b257 100644
--- a/sharedPlugins/clusterioMod/masterPlugin.js
+++ b/sharedPlugins/clusterioMod/masterPlugin.js
@@ -1,83 +1,104 @@
+"use strict";
const fs = require("fs-extra");
const path = require("path");
const Express = require("express");
const ejs = require("ejs");
+const database = require("lib/database");
+const routes = require("./routes");
+
+
class masterPlugin {
- constructor({config, pluginConfig, pluginPath, socketio, express}){
+ constructor({
+ config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter,
+ }) {
this.config = config;
this.pluginConfig = pluginConfig;
this.pluginPath = pluginPath;
this.io = socketio;
- this.app = express;
-
- // load databases
- this.items = getDatabaseSync("database/items.json");
-
- this.clients = {};
- this.slaves = {};
-
- // initialize web API
- // require("./js/api-endpoints.js")(this);
-
- // expose UI elements embedded in the master
- // this.ui = require("./js/ui.js").ui;
-
- this.io.on("connection", socket => {
-
- });
- }
- async broadcastCommand(command){
- let returnValues = [];
- for(let instanceID in this.slaves){
- let slave = this.slaves[instanceID];
- slave.emit("runCommand", {
- // commandID:Math.random(),
- command,
- });
- };
- return returnValues;
- }
- findInArray(key, value, array){
- let indexes = [];
- for(let i in array){
- if(array[i][key] && array[i][key] === value) indexes.push(i);
- }
- return indexes;
- }
- async onExit(){
- // await saveDatabase("database/items.json", this.items);
- return;
- }
-}
-module.exports = masterPlugin;
-function arrayRemoveDuplicates(array){
- let newArray = [];
- array.forEach(value => {
- if(!newArray.includes(value)) newArray.push(value);
- });
- return newArray;
-}
-function getDatabaseSync(path){
- let db;
- try {
- db = JSON.parse(fs.readFileSync(path, "utf8"));
- } catch(e){
- db = {};
+ this.express = express;
+ this.db = db,
+ this.Prometheus = Prometheus;
+ this.prometheusPrefix = prometheusPrefix;
+ this.endpointHitCounter = endpointHitCounter;
+
+ this.prometheusMasterInventoryGauge = new Prometheus.Gauge({
+ name: prometheusPrefix+'master_inventory_gauge',
+ help: 'Amount of items stored on master',
+ labelNames: ["itemName"],
+ });
+
+ this.ui = {
+ sidebar: [
+ {
+ getHtml: () => `
+ `,
+ },
+ ],
+ };
+ }
+
+ onMetrics() {
+ if (this.items) {
+ for (let [key, count] of this.items._items) {
+ this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
+ }
+ }
+ }
+
+ async onLoadFinish() {
+ this.items = await loadDatabase(this.config);
+ this.autosaveId = setInterval(async function() {
+ let file = path.join(this.config.databaseDirectory, "items.json");
+ console.log(`autosaving ${file}`);
+ let content = JSON.stringify(db.items.serialize());
+ await fs.outputFile(file, content);
+ },this.config.autosaveInterval || 60000);
+
+ routes.addApiRoutes(
+ this.express, this.config, this.db, this.items,
+ this.Prometheus, this.prometheusPrefix, this.endpointHitCounter,
+ );
+
+ routes.addWebRoutes(this.express);
+ }
+
+ async onExit(){
+ clearInterval(this.autosaveId);
+ await saveDatabase(this.config, this.items);
}
- return db;
}
-async function saveDatabase(path, database){
- if(!path){
- throw new Error("No path provided!");
- } else if(!database){
- throw new Error("No database provided!");
- } else {
- try {
- await fs.writeFile(path, JSON.stringify(database, null, 4));
- } catch(e){
- throw new Error("Unable to write to database! "+path);
+
+async function loadDatabase(config) {
+ let itemsPath = path.resolve(config.databaseDirectory, "items.json");
+ console.log(`Loading ${itemsPath}`);
+ try {
+ let content = await fs.readFile(itemsPath);
+ return new database.ItemDatabase(JSON.parse(content));
+
+ } catch (err) {
+ if (err.code === "ENOENT") {
+ console.log("Creating new item database");
+ return new database.ItemDatabase();
+
+ } else {
+ throw err;
}
}
}
+
+async function saveDatabase(config, items) {
+ if (items && items.size < 50000) {
+ let file = path.resolve(config.databaseDirectory, "items.json");
+ console.log(`writing ${file}`);
+ let content = JSON.stringify(items.serialize());
+ await fs.outputFile(file, content);
+ } else if (items) {
+ console.error(`Item database too large, not saving (${items.size})`);
+ }
+}
+
+module.exports = masterPlugin;
diff --git a/sharedPlugins/clusterioMod/routes.js b/sharedPlugins/clusterioMod/routes.js
new file mode 100644
index 00000000..c087093a
--- /dev/null
+++ b/sharedPlugins/clusterioMod/routes.js
@@ -0,0 +1,241 @@
+var recievedItemStatisticsBySlaveID = {};
+var sentItemStatisticsBySlaveID = {};
+
+const averagedTimeSeries = require("averaged-timeseries");
+const express = require("express");
+const path = require("path");
+
+const authenticate = require("lib/authenticate");
+
+const dole = require("./dole");
+
+
+function addWebRoutes(app) {
+ app.use('/clusterioMod', express.static(path.join(__dirname, 'static')));
+ app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); });
+}
+
+function addApiRoutes(
+ app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter,
+) {
+ const prometheusExportGauge = new Prometheus.Gauge({
+ name: prometheusPrefix+'export_gauge',
+ help: 'Items exported by instance',
+ labelNames: ["instanceID", "itemName"],
+ });
+ const prometheusImportGauge = new Prometheus.Gauge({
+ name: prometheusPrefix+'import_gauge',
+ help: 'Items imported by instance',
+ labelNames: ["instanceID", "itemName"],
+ });
+ const prometheusDoleFactorGauge = new Prometheus.Gauge({
+ name: prometheusPrefix+'dole_factor_gauge',
+ help: 'The current dole division factor for this item',
+ labelNames: ["itemName"],
+ });
+
+ // Only initialize neural network when it's enabled, otherwise it might override gauge
+ let neuralDole;
+ if(config.useNeuralNetDoleDivider) {
+ neuralDole=new dole.neuralDole({
+ db, gaugePrefix: prometheusPrefix
+ });
+ }
+
+ /**
+ * POST endpoint for storing items in master's inventory.
+
+ * @memberof clusterioMaster
+ * @instance
+ * @alias api/place
+ * @param {itemStack} itemStack the number and type of items to store (see typedef)
+ * @param {string} [itemStack.instanceID="unknown"]
+ * the unique/instanceID which is a numerical value for an instance
+ * @param {string} [itemStack.instanceName="unknown"]
+ * the name of an instance for identification in statistics, as
+ * provided when launching it. ex node client start [name]
+ * @returns {string} status either "success" or "failure"
+ */
+ app.post("/api/place", authenticate.middleware, function(req, res) {
+ endpointHitCounter.labels(req.route.path).inc();
+ res.header("Access-Control-Allow-Origin", "*");
+ res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+ let x;
+ try {
+ x = JSON.parse(req.body);
+ } catch (e) {
+ x = req.body;
+ }
+ if(!x.instanceName) {
+ x.instanceName = "unknown";
+ }
+ if(!x.instanceID) {
+ x.instanceID = "unknown";
+ for (let [instanceID, slave] of db.slaves) {
+ if(slave.instanceName == req.body.instanceName) {
+ x.instanceID = instanceID;
+ break;
+ }
+ }
+ }
+ if(x.instanceID
+ && x.instanceName
+ && !isNaN(Number(x.count))
+ && x.name
+ && typeof x.name == "string"){
+ if(config.logItemTransfers){
+ console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
+ }
+ // gather statistics
+ let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
+ if(recievedItemStatistics === undefined){
+ recievedItemStatistics = new averagedTimeSeries({
+ maxEntries: config.itemStats.maxEntries,
+ entriesPerSecond: config.itemStats.entriesPerSecond,
+ mergeMode: "add",
+ }, console.log);
+ recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
+ }
+ recievedItemStatistics.add({
+ key:req.body.name,
+ value:req.body.count,
+ });
+ prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
+ // save items we get
+ var count = Number(req.body.count);
+ items.addItem(req.body.name, count);
+
+ // Attempt confirming
+ res.send("success");
+ } else {
+ res.send("failure");
+ }
+ });
+
+ /**
+ * POST endpoint to remove items from DB when client orders items.
+ *
+ * @memberof clusterioMaster
+ * @instance
+ * @alias api/remove
+ * @param {itemStack} itemStack the name of and the number of items to remove (see typedef)
+ * @param {string} [itemStack.instanceID="unknown"]
+ * the unique/instanceID which is a numerical value for an instance
+ * @param {string} [itemStack.instanceName="unknown"]
+ * the name of an instance for identification in statistics, as
+ * provided when launching it. ex node client start [name]
+ * @returns {itemStack}
+ * the number of items actually removed, may be lower than what was
+ * asked for due to shortages.
+ */
+ app.post("/api/remove", authenticate.middleware, function(req, res) {
+ endpointHitCounter.labels(req.route.path).inc();
+ res.header("Access-Control-Allow-Origin", "*");
+ res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+ // save items we get
+ var object = req.body;
+ if(!object.instanceID) {
+ object.instanceID = "unknown"
+ }
+ if(!object.instanceName) {
+ object.instanceName = "unknown";
+ }
+ if (db.slaves.has(String(object.instanceID))) {
+ object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
+ }
+ let itemCount = items.getItemCount(object.name);
+ if (
+ (config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
+ && object.name != "electricity"
+ ) {
+ if(config.logItemTransfers){
+ console.log('failure could not find ' + object.name);
+ }
+ res.send({name:object.name, count:0});
+ } else if(config.disableFairItemDistribution){
+ // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
+ let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
+ items.removeItem(object.name, numberToRemove);
+ res.send({count: numberToRemove, name: object.name});
+
+ // track statistics and do graphing things
+ prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
+ let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+ if(sentItemStatistics === undefined){
+ sentItemStatistics = new averagedTimeSeries({
+ maxEntries: config.itemStats.maxEntries,
+ entriesPerSecond: config.itemStats.entriesPerSecond,
+ mergeMode: "add",
+ }, console.log);
+ }
+ sentItemStatistics.add({
+ key:object.name,
+ value:numberToRemove,
+ });
+ //console.log(sentItemStatistics.data)
+ sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+ } else if(config.useNeuralNetDoleDivider){
+ // use fancy neural net to calculate a "fair" dole division rate.
+ neuralDole.divider({
+ res,
+ object,
+ config,
+ sentItemStatisticsBySlaveID,
+ prometheusImportGauge
+ })
+ } else {
+ // Use dole division. Makes it really slow to drain out the last little bit.
+ dole.doleDivider({
+ itemCount,
+ object,
+ db,
+ sentItemStatisticsBySlaveID,
+ config,
+ prometheusDoleFactorGauge,
+ prometheusImportGauge,
+ req,res,
+ })
+ }
+ });
+
+ /**
+ * GET endpoint to read the masters current inventory of items.
+ *
+ * @memberof clusterioMaster
+ * @instance
+ * @alias api/inventory
+ * @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
+ */
+ app.get("/api/inventory", function(req, res) {
+ endpointHitCounter.labels(req.route.path).inc();
+ res.header("Access-Control-Allow-Origin", "*");
+ res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+ // Check it and send it
+ var inventory = [];
+ for (let [name, count] of items._items) {
+ inventory.push({ name, count });
+ }
+ res.send(JSON.stringify(inventory));
+ });
+
+ /**
+ * GET endpoint to read the masters inventory as an object with key:value pairs
+ *
+ * @memberof clusterioMaster
+ * @instance
+ * @alias api/inventoryAsObject
+ * @returns {object} JSON {"iron-plate":100, "copper-plate":5}
+ */
+ app.get("/api/inventoryAsObject", function(req, res) {
+ endpointHitCounter.labels(req.route.path).inc();
+ res.header("Access-Control-Allow-Origin", "*");
+ res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+ // Check it and send it
+ res.send(JSON.stringify(items.serialise()));
+ });
+}
+
+module.exports = {
+ addWebRoutes,
+ addApiRoutes,
+}
diff --git a/static/storage.js b/sharedPlugins/clusterioMod/static/storage.js
similarity index 99%
rename from static/storage.js
rename to sharedPlugins/clusterioMod/static/storage.js
index 0e3c68a7..0a8f561f 100644
--- a/static/storage.js
+++ b/sharedPlugins/clusterioMod/static/storage.js
@@ -65,7 +65,7 @@ function updateInventory() {
drawcontents(data);
}
}
- xmlhttp.open("GET", "api/inventory", true);
+ xmlhttp.open("GET", "/api/inventory", true);
xmlhttp.send();
}
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
diff --git a/sharedPlugins/clusterioMod/test/missing.js b/sharedPlugins/clusterioMod/test/missing.js
index 9993114b..5b080fef 100644
--- a/sharedPlugins/clusterioMod/test/missing.js
+++ b/sharedPlugins/clusterioMod/test/missing.js
@@ -3,3 +3,5 @@
require('../index');
require('../masterPlugin');
+require('../dole');
+require('../dole_nn_base');
diff --git a/sharedPlugins/clusterioMod/test/routes.js b/sharedPlugins/clusterioMod/test/routes.js
new file mode 100644
index 00000000..1e5a6e20
--- /dev/null
+++ b/sharedPlugins/clusterioMod/test/routes.js
@@ -0,0 +1,110 @@
+const assert = require("assert").strict;
+const express = require("express");
+const bodyParser = require("body-parser");
+const request = require("supertest");
+const Prometheus = require('prom-client');
+const jwt = require("jsonwebtoken");
+
+const authenticate = require('lib/authenticate');
+const database = require("lib/database");
+
+const routes = require("../routes");
+
+
+const testSecret = "TestSecretDoNotUse";
+authenticate.setAuthSecret(testSecret);
+
+const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
+
+describe("clusterioMod endpoints", function() {
+ const app = express();
+ app.use(bodyParser.json({ limit: '10mb', }));
+ app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
+ const config = {
+ itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 },
+ disableFairItemDistribution: true,
+ };
+ const items = new database.ItemDatabase();
+ const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] });
+ routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge);
+
+ let persistentMaster = request(app);
+ describe("#POST /api/place", function() {
+ it("adds an itemStack to the masters inventory", function() {
+ return persistentMaster.post("/api/place").send({
+ name:"steel-plate",
+ count:20,
+ instanceName:"unitTest"
+ }).set(
+ "X-Access-Token", token
+ ).expect(200).then(function(res) {
+ assert.equal(res.text, "success", "something went wrong with the request")
+ });
+ });
+ });
+ describe("#GET /api/inventory", function() {
+ it("returns the masters current inventory", function() {
+ return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
+ let inventory = JSON.parse(res.text);
+ assert.equal(typeof inventory, "object", "Inventory should be an object");
+ assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
+
+ let contains20SteelPlate = false;
+ inventory.forEach(itemStack => {
+ if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
+ });
+ assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
+ });
+ });
+ });
+ describe("#POST /api/remove", function() {
+ it("returns an itemStack of how many items were removed", function() {
+ return persistentMaster.post("/api/remove").send({
+ name:"steel-plate",
+ count:10
+ }).set(
+ "X-Access-Token", token
+ ).expect(200).then(function(res) {
+ assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
+ });
+ });
+ it("works correctly for items named addItem or removeItem", async function() {
+ function add(item) {
+ return persistentMaster.post("/api/place").send({
+ name: item,
+ count: 10
+ }).set(
+ "X-Access-Token", token
+ ).expect(200).then(function(res) {
+ assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
+ })
+ }
+
+ function remove(item) {
+ return persistentMaster.post("/api/remove").send({
+ name: item,
+ count: 10
+ }).set(
+ "X-Access-Token", token
+ ).expect(200).then(function(res) {
+ assert.deepEqual(res.body, {"name":item,"count":10});
+ });
+ }
+
+ await Promise.all([add("addItem"), add("removeItem")]);
+ await Promise.all([remove("addItem"), remove("removeItem")]);
+ });
+ it("returns an empty itemStack if you don't have any of the item you request", function() {
+ return persistentMaster.post("/api/remove").send({
+ name:"imaginaryItem",
+ count:999999
+ }).set(
+ "X-Access-Token", token
+ ).expect(200).then(function(res) {
+ assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
+ assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
+ assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
+ });
+ });
+ });
+});
diff --git a/sharedPlugins/clusterioMod/views/storage.ejs b/sharedPlugins/clusterioMod/views/storage.ejs
new file mode 100644
index 00000000..c274964b
--- /dev/null
+++ b/sharedPlugins/clusterioMod/views/storage.ejs
@@ -0,0 +1,49 @@
+
+
+ <%- include("head") %>
+
+
+
+
+<%- include("nav") %>
+
+
+
+
+
+
+
+
+
+
+ | Icon |
+ Item ID |
+ Quantity |
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
- | Icon |
- Item ID |
- Quantity |
-
-
-
-
-
-
Clusterio
-
<%