aperx/factorio-clusterio
1
0 Fork 0
mirror of https://github.com/clusterio/clusterio.git synced 2026-11-12 07:04:52 +09:00
Code Issues Packages Projects Releases Wiki Activity

Move clusterioMod specific code into the plugin

Browse Source 
Move item database, tests, endpoint definitions for /api/place,
/api/remove, /api/inventory, /api/inventoryAsObject, and the web
interface view for the storage page into the clusterioMod plugin.
This commit is contained in:
Hornwitser committed 2019-10-28 19:46:22 +01:00
1 parent bc444b79c2
commit e4aae88614
14 files changed
+524 -473
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

CHANGELOG.md
+4
View File
@@ -36,6 +36,10 @@ Version 2.0.0
time the instance is started. time the instance is started.
- Removed the broken client download command as this would download - Removed the broken client download command as this would download
pre-releases. pre-releases.
- Moved the item database and HTTP definitions for /api/place, /api/remove,
/api/inventory, /api/inventoryAsObject and the web interface view for the
storage page into the clusterioMod plugin. If you disable this plugin
then these things will not be available.
### Breaking Changes ### Breaking Changes
master.js
+17 -267
View File
@@ -17,7 +17,6 @@ node master
// argument parsing // argument parsing
const args = require('minimist')(process.argv.slice(2)); const args = require('minimist')(process.argv.slice(2));
const averagedTimeSeries = require("averaged-timeseries");
const deepmerge = require("deepmerge"); const deepmerge = require("deepmerge");
const path = require("path"); const path = require("path");
const fs = require("fs-extra"); const fs = require("fs-extra");
@@ -99,21 +98,6 @@ const prometheusWsUsageCounter = new Prometheus.Counter({
help: 'Websocket traffic', help: 'Websocket traffic',
labelNames: ["connectionType", "instanceID"], labelNames: ["connectionType", "instanceID"],
}); });
const prometheusExportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'export_gauge',
help: 'Items exported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusImportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'import_gauge',
help: 'Items imported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusDoleFactorGauge = new Prometheus.Gauge({
name: prometheusPrefix+'dole_factor_gauge',
help: 'The current dole division factor for this item',
labelNames: ["itemName"],
});
const prometheusPlayerCountGauge = new Prometheus.Gauge({ const prometheusPlayerCountGauge = new Prometheus.Gauge({
name: prometheusPrefix+'player_count_gauge', name: prometheusPrefix+'player_count_gauge',
help: 'Amount of players connected to this cluster', help: 'Amount of players connected to this cluster',
@@ -124,11 +108,6 @@ const prometheusUPSGauge = new Prometheus.Gauge({
help: 'UPS of the current server', help: 'UPS of the current server',
labelNames: ["instanceID", "instanceName"], labelNames: ["instanceID", "instanceName"],
}); });
const prometheusMasterInventoryGauge = new Prometheus.Gauge({
name: prometheusPrefix+'master_inventory_gauge',
help: 'Amount of items stored on master',
labelNames: ["itemName"],
});
/** /**
GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format. GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format.
@memberof clusterioMaster @memberof clusterioMaster
@@ -156,10 +135,14 @@ function registerMoreMetrics(){
if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0); if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0);
}catch(e){} }catch(e){}
} }
// inventory
for (let [key, count] of db.items._items) { // plugins
prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0); for (let plugin of masterPlugins) {
if (plugin.main && plugin.main.onMetrics && typeof plugin.main.onMetrics == "function") {
plugin.main.onMetrics();
}
} }
// Slave count // Slave count
let numberOfActiveSlaves = 0; let numberOfActiveSlaves = 0;
for(let slave of db.slaves.values()){ for(let slave of db.slaves.values()){
@@ -175,26 +158,10 @@ async function loadDatabase(databaseDirectory) {
let slavesPath = path.resolve(databaseDirectory, "slaves.json"); let slavesPath = path.resolve(databaseDirectory, "slaves.json");
console.log(`Loading ${slavesPath}`); console.log(`Loading ${slavesPath}`);
db.slaves = await database.loadJsonAsMap(slavesPath); db.slaves = await database.loadJsonAsMap(slavesPath);
let itemsPath = path.resolve(databaseDirectory, "items.json");
console.log(`Loading ${itemsPath}`);
try {
let content = await fs.readFile(itemsPath);
db.items = new database.ItemDatabase(JSON.parse(content));
} catch (err) {
if (err.code === "ENOENT") {
console.log("Creating new item database");
db.items = new database.ItemDatabase();
} else {
throw err;
}
}
} }
// store slaves and inventory in a .json full of JSON data // store slaves in a .json full of JSON data
async function shutdown() { async function shutdown() {
console.log('Ctrl-C...'); console.log('Ctrl-C...');
let exitStartTime = Date.now(); let exitStartTime = Date.now();
@@ -208,14 +175,6 @@ async function shutdown() {
console.error(`Slave database too large, not saving (${db.slaves.size}`); console.error(`Slave database too large, not saving (${db.slaves.size}`);
} }
if (db.items && db.items.size < 50000) {
let file = path.resolve(config.databaseDirectory, "items.json");
console.log(`writing ${file}`);
let content = JSON.stringify(db.items.serialize());
await fs.outputFile(file, content);
} else if (db.items) {
console.error(`Item database too large, not saving (${db.items.size})`);
}
for(let i in masterPlugins){ for(let i in masterPlugins){
let plugin = masterPlugins[i]; let plugin = masterPlugins[i];
if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
@@ -410,213 +369,14 @@ app.post("/api/rconPasswords", authenticate.middleware, function(req, res) {
res.send(slaveCache.cache); res.send(slaveCache.cache);
}); });
var recievedItemStatisticsBySlaveID = {};
var sentItemStatisticsBySlaveID = {};
//
/**
POST endpoint for storing items in master's inventory.
@memberof clusterioMaster
@instance
@alias api/place
@param {itemStack} itemStack the number and type of items to store (see typedef)
@param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance
@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name]
@returns {string} status either "success" or "failure"
*/
app.post("/api/place", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
let x;
try {
x = JSON.parse(req.body);
} catch (e) {
x = req.body;
}
if(!x.instanceName) {
x.instanceName = "unknown";
}
if(!x.instanceID) {
x.instanceID = "unknown";
for (let [instanceID, slave] of db.slaves) {
if(slave.instanceName == req.body.instanceName) {
x.instanceID = instanceID;
break;
}
}
}
if(x.instanceID
&& x.instanceName
&& !isNaN(Number(x.count))
&& x.name
&& typeof x.name == "string"){
if(config.logItemTransfers){
console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
}
// gather statistics
let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
if(recievedItemStatistics === undefined){
recievedItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
}
recievedItemStatistics.add({
key:req.body.name,
value:req.body.count,
});
prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
// save items we get
var count = Number(req.body.count);
db.items.addItem(req.body.name, count);
// Attempt confirming
res.send("success");
} else {
res.send("failure");
}
});
const routes_api_remove = require("./routes/api/remove");
let neuralDole;
/**
POST endpoint to remove items from DB when client orders items.
@memberof clusterioMaster
@instance
@alias api/remove
@param {itemStack} itemStack the name of and the number of items to remove (see typedef)
@param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance
@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name]
@returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
*/
app.post("/api/remove", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// save items we get
var object = req.body;
if(!object.instanceID) {
object.instanceID = "unknown"
}
if(!object.instanceName) {
object.instanceName = "unknown";
}
if (db.slaves.has(String(object.instanceID))) {
object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
}
let itemCount = db.items.getItemCount(object.name);
if (
(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
&& object.name != "electricity"
) {
if(config.logItemTransfers){
console.log('failure could not find ' + object.name);
}
res.send({name:object.name, count:0});
} else if(config.disableFairItemDistribution){
// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
db.items.removeItem(object.name, numberToRemove);
res.send({count: numberToRemove, name: object.name});
// track statistics and do graphing things
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
}
sentItemStatistics.add({
key:object.name,
value:numberToRemove,
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
} else if(config.useNeuralNetDoleDivider){
// use fancy neural net to calculate a "fair" dole division rate.
neuralDole.divider({
res,
object,
config,
sentItemStatisticsBySlaveID,
prometheusImportGauge
})
} else {
// Use dole division. Makes it really slow to drain out the last little bit.
routes_api_remove.doleDivider({
itemCount,
object,
db,
sentItemStatisticsBySlaveID,
config,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
})
}
});
/**
GET endpoint to read the masters current inventory of items.
@memberof clusterioMaster
@instance
@alias api/inventory
@returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
*/
// endpoint for getting an inventory of what we got
app.get("/api/inventory", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
var inventory = [];
for (let [name, count] of db.items._items) {
inventory.push({ name, count });
}
res.send(JSON.stringify(inventory));
});
/**
GET endpoint to read the masters inventory as an object with key:value pairs
@memberof clusterioMaster
@instance
@alias api/inventoryAsObject
@returns {object} JSON {"iron-plate":100, "copper-plate":5}
*/
app.get("/api/inventoryAsObject", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
res.send(JSON.stringify(db.items.serialise()));
});
/**
GET endpoint to read the masters inventory as an object with key:value pairs
@memberof clusterioMaster
@instance
@alias api/modData
@returns {object} JSON
*/
// wrap a request in an promise // wrap a request in an promise
async function downloadPage(url) { async function downloadPage(url) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(url, (error, response, body) => { request(url, (error, response, body) => {
resolve(body); resolve(body);
});
}); });
} });
}
app.get("/api/modmeta", async function(req, res) { app.get("/api/modmeta", async function(req, res) {
endpointHitCounter.labels(req.route.path).inc(); endpointHitCounter.labels(req.route.path).inc();
@@ -832,7 +592,10 @@ async function getPlugins(){
path: pluginConfig.pluginPath, path: pluginConfig.pluginPath,
socketio: io, socketio: io,
express: app, express: app,
db,
Prometheus, Prometheus,
prometheusPrefix,
endpointHitCounter,
}), }),
pluginConfig, pluginConfig,
}); });
@@ -900,12 +663,6 @@ async function startServer() {
fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
expiresIn: 86400*365 // expires in 1 year expiresIn: 86400*365 // expires in 1 year
})); }));
setInterval(async function() {
let file = path.resolve(config.databaseDirectory, "items.json");
console.log(`autosaving ${file}`);
let content = JSON.stringify(db.items.serialize());
await fs.outputFile(file, content);
},config.autosaveInterval || 60000);
process.on('SIGINT', shutdown); // ctrl + c process.on('SIGINT', shutdown); // ctrl + c
process.on('SIGHUP', shutdown); // terminal closed process.on('SIGHUP', shutdown); // terminal closed
@@ -921,13 +678,6 @@ async function startServer() {
await loadDatabase(config.databaseDirectory); await loadDatabase(config.databaseDirectory);
authenticate.setAuthSecret(config.masterAuthSecret); authenticate.setAuthSecret(config.masterAuthSecret);
// Only initialize neural network when it's enabled, otherwise it might override gauge
if(config.useNeuralNetDoleDivider) {
neuralDole=new routes_api_remove.neuralDole({
db, gaugePrefix: prometheusPrefix
});
}
// Make sure we're actually going to listen on a port // Make sure we're actually going to listen on a port
let httpPort = args.masterPort || config.masterPort; let httpPort = args.masterPort || config.masterPort;
let httpsPort = args.sslPort || config.sslPort; let httpsPort = args.sslPort || config.sslPort;
routes/api/remove.js → sharedPlugins/clusterioMod/dole.js
+11 -11
View File
@@ -1,6 +1,6 @@
const averagedTimeSeries = require("averaged-timeseries") const averagedTimeSeries = require("averaged-timeseries")
const doleNN = require("lib/dole_nn_base") const doleNN = require("./dole_nn_base")
const Prometheus = require('prom-client'); const Prometheus = require('prom-client');
@@ -19,7 +19,7 @@ class neuralDole {
} }
constructor({ constructor({
db, gaugePrefix items, gaugePrefix
}){ }){
// Set some storage variables for the dole divider // Set some storage variables for the dole divider
this.prometheusNNDoleGauge = new Prometheus.Gauge({ this.prometheusNNDoleGauge = new Prometheus.Gauge({
@@ -27,8 +27,8 @@ class neuralDole {
help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given', help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
labelNames: ["itemName"], labelNames: ["itemName"],
}); });
this.db = db this.items = items
this.itemsLastTick = new Map(db.items._items) this.itemsLastTick = new Map(items._items)
this.dole = {} this.dole = {}
this.carry = {} this.carry = {}
this.lastRequest = {} this.lastRequest = {}
@@ -36,7 +36,7 @@ class neuralDole {
this.debt= {} this.debt= {}
setInterval(()=>{ setInterval(()=>{
for (let [name, count] of this.db.items._items) { for (let [name, count] of this.items._items) {
let magicData = doleNN.Tick( let magicData = doleNN.Tick(
count, count,
this.dole[name], this.dole[name],
@@ -51,7 +51,7 @@ class neuralDole {
// DONE handle magicData[1] for graphing for our users // DONE handle magicData[1] for graphing for our users
this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0); this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
} }
this.itemsLastTick = new Map(this.db.items._items) this.itemsLastTick = new Map(this.items._items)
}, 1000) }, 1000)
} }
divider({ divider({
@@ -63,7 +63,7 @@ class neuralDole {
}){ }){
let magicData = doleNN.Dose( let magicData = doleNN.Dose(
object.count, // numReq object.count, // numReq
this.db.items.getItemCount(object.name), this.items.getItemCount(object.name),
this.itemsLastTick.get(object.name) || 0, this.itemsLastTick.get(object.name) || 0,
this.dole[object.name], this.dole[object.name],
this.carry[object.name+" "+object.instanceID] || 0, this.carry[object.name+" "+object.instanceID] || 0,
@@ -85,9 +85,9 @@ class neuralDole {
this.debt[object.name+" "+object.instanceID] = magicData[3] this.debt[object.name+" "+object.instanceID] = magicData[3]
// Remove item from DB and send it // Remove item from DB and send it
this.db.items.removeItem(object.name, magicData[0]); this.items.removeItem(object.name, magicData[0]);
if(config.logItemTransfers){ if(config.logItemTransfers){
console.log("removed: " + object.name + " " + magicData[0] + " . " + this.db.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName); console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
} }
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){ if(sentItemStatistics === undefined){
@@ -116,7 +116,7 @@ _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this
function doleDivider({ function doleDivider({
itemCount, itemCount,
object, object,
db, items,
sentItemStatisticsBySlaveID, sentItemStatisticsBySlaveID,
config, config,
prometheusDoleFactorGauge, prometheusDoleFactorGauge,
@@ -140,7 +140,7 @@ function doleDivider({
if (config.logItemTransfers) { if (config.logItemTransfers) {
console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName); console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
} }
db.items.removeItem(object.name, object.count) items.removeItem(object.name, object.count)
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){ if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({ sentItemStatistics = new averagedTimeSeries({
lib/dole_nn_base.js → sharedPlugins/clusterioMod/dole_nn_base.js
View File
File renamed without changes.
sharedPlugins/clusterioMod/masterPlugin.js
+88 -67
View File
@@ -1,83 +1,104 @@
"use strict";
const fs = require("fs-extra"); const fs = require("fs-extra");
const path = require("path"); const path = require("path");
const Express = require("express"); const Express = require("express");
const ejs = require("ejs"); const ejs = require("ejs");
const database = require("lib/database");
const routes = require("./routes");
class masterPlugin { class masterPlugin {
constructor({config, pluginConfig, pluginPath, socketio, express}){ constructor({
config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter,
}) {
this.config = config; this.config = config;
this.pluginConfig = pluginConfig; this.pluginConfig = pluginConfig;
this.pluginPath = pluginPath; this.pluginPath = pluginPath;
this.io = socketio; this.io = socketio;
this.app = express;
// load databases
this.items = getDatabaseSync("database/items.json");
this.clients = {};
this.slaves = {};
// initialize web API
// require("./js/api-endpoints.js")(this);
// expose UI elements embedded in the master
// this.ui = require("./js/ui.js").ui;
this.io.on("connection", socket => {
});
}
async broadcastCommand(command){
let returnValues = [];
for(let instanceID in this.slaves){
let slave = this.slaves[instanceID];
slave.emit("runCommand", {
// commandID:Math.random(),
command,
});
};
return returnValues;
}
findInArray(key, value, array){
let indexes = [];
for(let i in array){
if(array[i][key] && array[i][key] === value) indexes.push(i);
}
return indexes;
}
async onExit(){
// await saveDatabase("database/items.json", this.items);
return;
}
}
module.exports = masterPlugin;
function arrayRemoveDuplicates(array){ this.express = express;
let newArray = []; this.db = db,
array.forEach(value => { this.Prometheus = Prometheus;
if(!newArray.includes(value)) newArray.push(value); this.prometheusPrefix = prometheusPrefix;
}); this.endpointHitCounter = endpointHitCounter;
return newArray;
} this.prometheusMasterInventoryGauge = new Prometheus.Gauge({
function getDatabaseSync(path){ name: prometheusPrefix+'master_inventory_gauge',
let db; help: 'Amount of items stored on master',
try { labelNames: ["itemName"],
db = JSON.parse(fs.readFileSync(path, "utf8")); });
} catch(e){
db = {}; this.ui = {
sidebar: [
{
getHtml: () => `
<div class="nav-item mr-1">
<a class="nav-link align-middle" href="/clusterioMod/storage">Storage</a>
</div>`,
},
],
};
}
onMetrics() {
if (this.items) {
for (let [key, count] of this.items._items) {
this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
}
}
}
async onLoadFinish() {
this.items = await loadDatabase(this.config);
this.autosaveId = setInterval(async function() {
let file = path.join(this.config.databaseDirectory, "items.json");
console.log(`autosaving ${file}`);
let content = JSON.stringify(db.items.serialize());
await fs.outputFile(file, content);
},this.config.autosaveInterval || 60000);
routes.addApiRoutes(
this.express, this.config, this.db, this.items,
this.Prometheus, this.prometheusPrefix, this.endpointHitCounter,
);
routes.addWebRoutes(this.express);
}
async onExit(){
clearInterval(this.autosaveId);
await saveDatabase(this.config, this.items);
} }
return db;
} }
async function saveDatabase(path, database){
if(!path){ async function loadDatabase(config) {
throw new Error("No path provided!"); let itemsPath = path.resolve(config.databaseDirectory, "items.json");
} else if(!database){ console.log(`Loading ${itemsPath}`);
throw new Error("No database provided!"); try {
} else { let content = await fs.readFile(itemsPath);
try { return new database.ItemDatabase(JSON.parse(content));
await fs.writeFile(path, JSON.stringify(database, null, 4));
} catch(e){ } catch (err) {
throw new Error("Unable to write to database! "+path); if (err.code === "ENOENT") {
console.log("Creating new item database");
return new database.ItemDatabase();
} else {
throw err;
} }
} }
} }
async function saveDatabase(config, items) {
if (items && items.size < 50000) {
let file = path.resolve(config.databaseDirectory, "items.json");
console.log(`writing ${file}`);
let content = JSON.stringify(items.serialize());
await fs.outputFile(file, content);
} else if (items) {
console.error(`Item database too large, not saving (${items.size})`);
}
}
module.exports = masterPlugin;
sharedPlugins/clusterioMod/routes.js
+241
View File
@@ -0,0 +1,241 @@
var recievedItemStatisticsBySlaveID = {};
var sentItemStatisticsBySlaveID = {};
const averagedTimeSeries = require("averaged-timeseries");
const express = require("express");
const path = require("path");
const authenticate = require("lib/authenticate");
const dole = require("./dole");
function addWebRoutes(app) {
app.use('/clusterioMod', express.static(path.join(__dirname, 'static')));
app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); });
}
function addApiRoutes(
app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter,
) {
const prometheusExportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'export_gauge',
help: 'Items exported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusImportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'import_gauge',
help: 'Items imported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusDoleFactorGauge = new Prometheus.Gauge({
name: prometheusPrefix+'dole_factor_gauge',
help: 'The current dole division factor for this item',
labelNames: ["itemName"],
});
// Only initialize neural network when it's enabled, otherwise it might override gauge
let neuralDole;
if(config.useNeuralNetDoleDivider) {
neuralDole=new dole.neuralDole({
db, gaugePrefix: prometheusPrefix
});
}
/**
* POST endpoint for storing items in master's inventory.
* @memberof clusterioMaster
* @instance
* @alias api/place
* @param {itemStack} itemStack the number and type of items to store (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {string} status either "success" or "failure"
*/
app.post("/api/place", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
let x;
try {
x = JSON.parse(req.body);
} catch (e) {
x = req.body;
}
if(!x.instanceName) {
x.instanceName = "unknown";
}
if(!x.instanceID) {
x.instanceID = "unknown";
for (let [instanceID, slave] of db.slaves) {
if(slave.instanceName == req.body.instanceName) {
x.instanceID = instanceID;
break;
}
}
}
if(x.instanceID
&& x.instanceName
&& !isNaN(Number(x.count))
&& x.name
&& typeof x.name == "string"){
if(config.logItemTransfers){
console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
}
// gather statistics
let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID];
if(recievedItemStatistics === undefined){
recievedItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics;
}
recievedItemStatistics.add({
key:req.body.name,
value:req.body.count,
});
prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
// save items we get
var count = Number(req.body.count);
items.addItem(req.body.name, count);
// Attempt confirming
res.send("success");
} else {
res.send("failure");
}
});
/**
* POST endpoint to remove items from DB when client orders items.
*
* @memberof clusterioMaster
* @instance
* @alias api/remove
* @param {itemStack} itemStack the name of and the number of items to remove (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {itemStack}
* the number of items actually removed, may be lower than what was
* asked for due to shortages.
*/
app.post("/api/remove", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// save items we get
var object = req.body;
if(!object.instanceID) {
object.instanceID = "unknown"
}
if(!object.instanceName) {
object.instanceName = "unknown";
}
if (db.slaves.has(String(object.instanceID))) {
object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
}
let itemCount = items.getItemCount(object.name);
if (
(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
&& object.name != "electricity"
) {
if(config.logItemTransfers){
console.log('failure could not find ' + object.name);
}
res.send({name:object.name, count:0});
} else if(config.disableFairItemDistribution){
// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
items.removeItem(object.name, numberToRemove);
res.send({count: numberToRemove, name: object.name});
// track statistics and do graphing things
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
if(sentItemStatistics === undefined){
sentItemStatistics = new averagedTimeSeries({
maxEntries: config.itemStats.maxEntries,
entriesPerSecond: config.itemStats.entriesPerSecond,
mergeMode: "add",
}, console.log);
}
sentItemStatistics.add({
key:object.name,
value:numberToRemove,
});
//console.log(sentItemStatistics.data)
sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
} else if(config.useNeuralNetDoleDivider){
// use fancy neural net to calculate a "fair" dole division rate.
neuralDole.divider({
res,
object,
config,
sentItemStatisticsBySlaveID,
prometheusImportGauge
})
} else {
// Use dole division. Makes it really slow to drain out the last little bit.
dole.doleDivider({
itemCount,
object,
db,
sentItemStatisticsBySlaveID,
config,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
})
}
});
/**
* GET endpoint to read the masters current inventory of items.
*
* @memberof clusterioMaster
* @instance
* @alias api/inventory
* @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
*/
app.get("/api/inventory", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
var inventory = [];
for (let [name, count] of items._items) {
inventory.push({ name, count });
}
res.send(JSON.stringify(inventory));
});
/**
* GET endpoint to read the masters inventory as an object with key:value pairs
*
* @memberof clusterioMaster
* @instance
* @alias api/inventoryAsObject
* @returns {object} JSON {"iron-plate":100, "copper-plate":5}
*/
app.get("/api/inventoryAsObject", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
res.send(JSON.stringify(items.serialise()));
});
}
module.exports = {
addWebRoutes,
addApiRoutes,
}
static/storage.js → sharedPlugins/clusterioMod/static/storage.js
+1 -1
View File
@@ -65,7 +65,7 @@ function updateInventory() {
drawcontents(data); drawcontents(data);
} }
} }
xmlhttp.open("GET", "api/inventory", true); xmlhttp.open("GET", "/api/inventory", true);
xmlhttp.send(); xmlhttp.send();
} }
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) { if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
sharedPlugins/clusterioMod/test/missing.js
+2
View File
@@ -3,3 +3,5 @@
require('../index'); require('../index');
require('../masterPlugin'); require('../masterPlugin');
require('../dole');
require('../dole_nn_base');
sharedPlugins/clusterioMod/test/routes.js
+110
View File
@@ -0,0 +1,110 @@
const assert = require("assert").strict;
const express = require("express");
const bodyParser = require("body-parser");
const request = require("supertest");
const Prometheus = require('prom-client');
const jwt = require("jsonwebtoken");
const authenticate = require('lib/authenticate');
const database = require("lib/database");
const routes = require("../routes");
const testSecret = "TestSecretDoNotUse";
authenticate.setAuthSecret(testSecret);
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
describe("clusterioMod endpoints", function() {
const app = express();
app.use(bodyParser.json({ limit: '10mb', }));
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
const config = {
itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 },
disableFairItemDistribution: true,
};
const items = new database.ItemDatabase();
const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] });
routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge);
let persistentMaster = request(app);
describe("#POST /api/place", function() {
it("adds an itemStack to the masters inventory", function() {
return persistentMaster.post("/api/place").send({
name:"steel-plate",
count:20,
instanceName:"unitTest"
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, "success", "something went wrong with the request")
});
});
});
describe("#GET /api/inventory", function() {
it("returns the masters current inventory", function() {
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
let inventory = JSON.parse(res.text);
assert.equal(typeof inventory, "object", "Inventory should be an object");
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
let contains20SteelPlate = false;
inventory.forEach(itemStack => {
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
});
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
});
});
});
describe("#POST /api/remove", function() {
it("returns an itemStack of how many items were removed", function() {
return persistentMaster.post("/api/remove").send({
name:"steel-plate",
count:10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
});
});
it("works correctly for items named addItem or removeItem", async function() {
function add(item) {
return persistentMaster.post("/api/place").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
})
}
function remove(item) {
return persistentMaster.post("/api/remove").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.deepEqual(res.body, {"name":item,"count":10});
});
}
await Promise.all([add("addItem"), add("removeItem")]);
await Promise.all([remove("addItem"), remove("removeItem")]);
});
it("returns an empty itemStack if you don't have any of the item you request", function() {
return persistentMaster.post("/api/remove").send({
name:"imaginaryItem",
count:999999
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
});
});
});
});
sharedPlugins/clusterioMod/views/storage.ejs
+49
View File
@@ -0,0 +1,49 @@
<!doctype HTML>
<head>
<%- include("head") %>
<script defer src="/clusterioMod/storage.js"></script>
</head>
<body>
<%- include("nav") %>
<div id="body" class="container">
<div id="storage">
<div class="page-header mt-2 row" id="banner">
<div class="row">
<div class="col-lg-8 col-md-7 col-sm-6">
<h2>Storage</h2>
</div>
</div>
</div>
<div class="form-group mt-3 mb-3 row">
<input type="text" class="form-control" id="search" placeholder="search">
</div>
<div id="contents" class="row w-100">
<table class="table table-striped table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">Icon</th>
<th scope="col">Item ID</th>
<th scope="col">Quantity</th>
</tr>
</thead>
<tbody><!-- populated by js --></tbody>
</table>
</div>
</div>
<div id="graphDisplay">
<div id="storageGraphContainer">
<div id="contentGraph" class="storageGraph" style="width: 100%;"></div>
</div>
</div>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
crossorigin="anonymous"></script>
</body>
test/master.js
-90
View File
@@ -2,25 +2,14 @@ const assert = require('assert').strict;
const request = require('supertest'); const request = require('supertest');
const validateHTML = require('html5-validator'); const validateHTML = require('html5-validator');
const parallel = require('mocha.parallel'); const parallel = require('mocha.parallel');
const jwt = require("jsonwebtoken");
const master = require('../master'); const master = require('../master');
const authenticate = require('lib/authenticate');
const database = require('lib/database');
master._db.slaves = new Map(); master._db.slaves = new Map();
master._db.items = new database.ItemDatabase();
// Fields required in the config // Fields required in the config
master._config.factorioDirectory = "factorio"; master._config.factorioDirectory = "factorio";
master._config.itemStats = { maxEntries:60, entriesPerSecond: 1 / 60 };
master._config.disableFairItemDistribution = true;
const testSecret = "TestSecretDoNotUse";
authenticate.setAuthSecret(testSecret);
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
async function get(path) { async function get(path) {
return await request(master.app).get(path).expect(200); return await request(master.app).get(path).expect(200);
@@ -54,83 +43,4 @@ describe('Master testing', function() {
it(`sends some HTML when accessing ${path}`, () => getValidate(path)); it(`sends some HTML when accessing ${path}`, () => getValidate(path));
} }
}); });
let persistentMaster = request(master.app);
describe("#POST /api/place", function() {
it("adds an itemStack to the masters inventory", function() {
return persistentMaster.post("/api/place").send({
name:"steel-plate",
count:20,
instanceName:"unitTest"
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, "success", "something went wrong with the request")
});
});
});
describe("#GET /api/inventory", function() {
it("returns the masters current inventory", function() {
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
let inventory = JSON.parse(res.text);
assert.equal(typeof inventory, "object", "Inventory should be an object");
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
let contains20SteelPlate = false;
inventory.forEach(itemStack => {
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
});
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
});
});
});
describe("#POST /api/remove", function() {
it("returns an itemStack of how many items were removed", function() {
return persistentMaster.post("/api/remove").send({
name:"steel-plate",
count:10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
});
});
it("works correctly for items named addItem or removeItem", async function() {
function add(item) {
return persistentMaster.post("/api/place").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
})
}
function remove(item) {
return persistentMaster.post("/api/remove").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.deepEqual(res.body, {"name":item,"count":10});
});
}
await Promise.all([add("addItem"), add("removeItem")]);
await Promise.all([remove("addItem"), remove("removeItem")]);
});
it("returns an empty itemStack if you don't have any of the item you request", function() {
return persistentMaster.post("/api/remove").send({
name:"imaginaryItem",
count:999999
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
});
});
});
}); });
test/missing.js
-1
View File
@@ -2,7 +2,6 @@
// required here in order to track coverage for them. // required here in order to track coverage for them.
require('lib/clusterTools'); require('lib/clusterTools');
require('lib/dole_nn_base');
require('lib/getInstances'); require('lib/getInstances');
require('lib/manager/instanceManager'); require('lib/manager/instanceManager');
require('lib/manager/modManager'); require('lib/manager/modManager');
views/index.ejs
+1 -33
View File
@@ -1,7 +1,6 @@
<!doctype HTML> <!doctype HTML>
<head> <head>
<%- include("head") %> <%- include("head") %>
<script defer src="storage.js"></script>
</head> </head>
<body> <body>
@@ -10,40 +9,9 @@
<div id="storage"> <div id="storage">
<div class="page-header mt-2 row" id="banner"> <div class="page-header mt-2 row" id="banner">
<div class="row"> <div class="row">
<div class="col-lg-8 col-md-7 col-sm-6"> <h2>Work in progress interface</h2>
<h2>Storage</h2>
</div>
</div> </div>
</div> </div>
<div class="form-group mt-3 mb-3 row">
<input type="text" class="form-control" id="search" placeholder="search">
</div>
<div id="contents" class="row w-100">
<table class="table table-striped table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">Icon</th>
<th scope="col">Item ID</th>
<th scope="col">Quantity</th>
</tr>
</thead>
<tbody><!-- populated by js --></tbody>
</table>
</div>
</div>
<div id="graphDisplay">
<div id="storageGraphContainer">
<div id="contentGraph" class="storageGraph" style="width: 100%;"></div>
</div>
</div> </div>
</div> </div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
crossorigin="anonymous"></script>
</body> </body>
views/nav.ejs
-3
View File
@@ -5,9 +5,6 @@
<h3 class=" ml-3 nav-title">Clusterio</h3> <h3 class=" ml-3 nav-title">Clusterio</h3>
</a> </a>
</div> </div>
<div class="nav-item mr-1">
<a class="nav-link align-middle" href="/">Storage</a>
</div>
<div class="nav-item mr-1"> <div class="nav-item mr-1">
<a class="nav-link align-middle" href="/nodes">Slaves</a> <a class="nav-link align-middle" href="/nodes">Slaves</a>
</div><% </div><%