mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-17 15:33:12 +09:00
Port and rename clusterioMod plugin
Port the clusterioMod plugin to the new plugin system under the new subspace_storage name.
This commit is contained in:
1 parent
d9fe408c42
commit
737c9924b9
17 files changed
+641
-891
No files matched your search
@@ -0,0 +1,129 @@
|
||||
const doleNN = require("./dole_nn_base")
|
||||
|
||||
const prometheus = require("lib/prometheus");
|
||||
|
||||
|
||||
const prometheusNNDoleGauge = new prometheus.Gauge(
|
||||
"clusterio_nn_dole_gauge",
|
||||
"Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given",
|
||||
{ labels: ["resource"] }
|
||||
);
|
||||
const prometheusDoleFactorGauge = new prometheus.Gauge(
|
||||
"clusterio_dole_factor_gauge",
|
||||
"The current dole division factor for this resource",
|
||||
{ labels: ["resource"] }
|
||||
);
|
||||
|
||||
class neuralDole {
|
||||
|
||||
getRequestStats(itemname,samples)
|
||||
{
|
||||
var sum=0;
|
||||
samples=Math.min(samples,(this.stats[itemname] || []).length-1);
|
||||
if (samples<1) return 0.1;
|
||||
for (var i = 1; i <= samples; i++) {
|
||||
sum+=this.stats[itemname][i].req;
|
||||
}
|
||||
if (sum==0) return 0.1;
|
||||
return sum/samples;
|
||||
}
|
||||
|
||||
constructor({
|
||||
items
|
||||
}){
|
||||
// Set some storage variables for the dole divider
|
||||
this.items = items
|
||||
this.itemsLastTick = new Map(items._items)
|
||||
this.dole = {}
|
||||
this.carry = {}
|
||||
this.lastRequest = {}
|
||||
this.stats=[]
|
||||
this.debt= {}
|
||||
}
|
||||
doMagic() {
|
||||
for (let [name, count] of this.items._items) {
|
||||
let magicData = doleNN.Tick(
|
||||
count,
|
||||
this.dole[name],
|
||||
this.itemsLastTick.get(name),
|
||||
this.getRequestStats(name,5)
|
||||
)
|
||||
this.stats[name]=this.stats[name] || []
|
||||
this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
|
||||
if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
|
||||
|
||||
this.dole[name] = magicData[0]
|
||||
// DONE handle magicData[1] for graphing for our users
|
||||
prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
|
||||
}
|
||||
this.itemsLastTick = new Map(this.items._items)
|
||||
}
|
||||
divider(object) {
|
||||
let magicData = doleNN.Dose(
|
||||
object.count, // numReq
|
||||
this.items.getItemCount(object.name),
|
||||
this.itemsLastTick.get(object.name) || 0,
|
||||
this.dole[object.name],
|
||||
this.carry[object.name+" "+object.instanceID] || 0,
|
||||
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
|
||||
this.getRequestStats(object.name,5),
|
||||
this.debt[object.name+" "+object.instanceID] || 0
|
||||
)
|
||||
if ((this.stats[object.name] || []).length>0)
|
||||
{
|
||||
this.stats[object.name][0].req+=Number(object.count);
|
||||
this.stats[object.name][0].given+=Number(magicData[0]);
|
||||
}
|
||||
this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
|
||||
//0.Number of items to give in that dose
|
||||
//1.New dole for item X
|
||||
//2.New carry for item X slave Y
|
||||
this.dole[object.name] = magicData[1]
|
||||
this.carry[object.name+" "+object.instanceID] = magicData[2]
|
||||
this.debt[object.name+" "+object.instanceID] = magicData[3]
|
||||
|
||||
// Remove item from DB and send it
|
||||
this.items.removeItem(object.name, magicData[0]);
|
||||
|
||||
return magicData[0];
|
||||
}
|
||||
}
|
||||
|
||||
// If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
|
||||
let _doleDivisionFactor = {};
|
||||
function doleDivider({
|
||||
object,
|
||||
items,
|
||||
}){
|
||||
let itemCount = items.getItemCount(object.name);
|
||||
const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
|
||||
const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
|
||||
|
||||
const originalCount = Number(object.count) || 0;
|
||||
object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
|
||||
object.count = Math.round(object.count);
|
||||
if (itemCount > 40) {
|
||||
console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`);
|
||||
}
|
||||
|
||||
// Update existing items if item name already exists
|
||||
if (itemCount > object.count) {
|
||||
//If successful, increase dole
|
||||
_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
|
||||
items.removeItem(object.name, object.count)
|
||||
|
||||
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
|
||||
return object.count;
|
||||
} else {
|
||||
// if we didn't have enough, attempt giving out a smaller amount next time
|
||||
_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
|
||||
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
|
||||
return 0;
|
||||
//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
doleDivider,
|
||||
neuralDole
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
//Sigmoids...
|
||||
|
||||
function sigmoid0(x)
|
||||
{
|
||||
return x/(1+Math.abs(x));
|
||||
}
|
||||
function sigmoid1(x)
|
||||
{
|
||||
if (x>=0.0) {
|
||||
return 1.0
|
||||
} else {
|
||||
return 0.0
|
||||
}
|
||||
}
|
||||
function sigmoid2(x)
|
||||
{
|
||||
if (x>=0.0){
|
||||
return x
|
||||
} else {
|
||||
return 0.0
|
||||
}
|
||||
}
|
||||
function sigmoid3(x)
|
||||
{
|
||||
return x;
|
||||
}
|
||||
|
||||
//DOSE - Doses items for item X slave Y
|
||||
//Numreq - number of items that are requested for item X for slave Y
|
||||
//Instore - number of items for item X in master gauge (currently)
|
||||
//store_last_tick - Number of items for item X in master gauge at time of last tick
|
||||
//dole - Current dole for item X
|
||||
//carry - carry for item X slave Y
|
||||
//prev_req - Numreq for previous request for item X slave Y
|
||||
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
|
||||
//debt - debt for item X slave Y
|
||||
|
||||
//Returns: Array of 3 elements
|
||||
//0.Number of items to give in that dose
|
||||
//1.New dole for item X
|
||||
//2.New carry for item X slave Y
|
||||
//3.New debt
|
||||
function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj,debt)
|
||||
{
|
||||
numreq=Number(numreq);
|
||||
instore=Number(instore);
|
||||
var instore_adj=instore;
|
||||
if (instore_adj==0) instore_adj=0.1;
|
||||
|
||||
var b=Math.trunc(prev_req/instore_adj*numreq);
|
||||
var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
|
||||
var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b,numreq_total_adj,store_last_tick,debt];//5..12
|
||||
var inputs=inp1.concat(inp2);
|
||||
var internal=new Array(20).fill(0);
|
||||
var outputs=[0,0,0,0];
|
||||
//NN: Please don't touch :), code is generated
|
||||
|
||||
outputs[0]=inputs[7]*0.8058582544;
|
||||
outputs[0]=sigmoid2(outputs[0]);
|
||||
outputs[1]=sigmoid0(outputs[1]);
|
||||
outputs[2]=sigmoid0(outputs[2]);
|
||||
outputs[3]=inputs[6]*0.2294657379;
|
||||
outputs[3]=sigmoid0(outputs[3]);
|
||||
|
||||
debt=debt || 0;
|
||||
if (outputs[0]>0.99) outputs[0]=1.0;
|
||||
|
||||
var res1=Number(outputs[0])*numreq+outputs[3]+debt;
|
||||
var res_capped=Math.round(res1);
|
||||
|
||||
if (res_capped>numreq) res_capped=numreq;
|
||||
if (res_capped>instore) res_capped=instore;
|
||||
if (res_capped<0) res_capped=0;
|
||||
|
||||
res1=res1 || 0;
|
||||
res_capped=res_capped || 0;//Safeguard against wrong values
|
||||
debt=res1-res_capped;
|
||||
debt_cap=Math.round(numreq*1.5)+5;
|
||||
if (debt>debt_cap) debt=debt_cap;
|
||||
|
||||
carry=sigmoid0(outputs[2]*carry+outputs[0]) || 0;
|
||||
return [res_capped,dole+outputs[1],carry,debt];
|
||||
}
|
||||
|
||||
//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
|
||||
//Instore - number of items for item X in master gauge (currently)
|
||||
//dole - Current dole for item X
|
||||
//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
|
||||
//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
|
||||
//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
|
||||
|
||||
|
||||
//Returns: Array of 2 elements
|
||||
//0.New dole for item X
|
||||
//1.Guessed percentage average of items supplied in last 10 ticks
|
||||
function Tick(instore,dole,store_last_tick,numreq_total_adj)
|
||||
{
|
||||
var instore_adj=instore;
|
||||
if (instore_adj==0) instore_adj=0.1;
|
||||
var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj,store_last_tick,numreq_total_adj];
|
||||
var internal=new Array(10).fill(0);
|
||||
var outputs=[0,0];
|
||||
//NN: Please don't touch :), code is generated
|
||||
|
||||
outputs[0]=inputs[3]*0.9364774823;
|
||||
outputs[0]=outputs[0]*1.0*0.4590721726;
|
||||
outputs[0]=sigmoid2(outputs[0]);
|
||||
outputs[1]=outputs[0]*1.828422785;
|
||||
outputs[1]=sigmoid3(outputs[1]);
|
||||
|
||||
avg=outputs[1];
|
||||
if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
|
||||
if (avg>1) avg=1;
|
||||
if (avg<0) avg=0;
|
||||
|
||||
return [sigmoid0(outputs[0]),avg];
|
||||
}
|
||||
|
||||
module.exports = {Dose,Tick}
|
||||
@@ -0,0 +1,106 @@
|
||||
const link = require("lib/link");
|
||||
const config = require("lib/config");
|
||||
|
||||
class MasterConfigGroup extends config.PluginConfigGroup {}
|
||||
MasterConfigGroup.groupName = "subspace_storage";
|
||||
MasterConfigGroup.define({
|
||||
name: "autosave_interval",
|
||||
title: "Autosave Interval",
|
||||
description: "Interval the storage is autosaved at in seconds.",
|
||||
type: "number",
|
||||
initial_value: 60,
|
||||
});
|
||||
MasterConfigGroup.define({
|
||||
name: "division_method",
|
||||
title: "Division Method",
|
||||
description: "Method for dividing resource requests between instances.",
|
||||
type: "string",
|
||||
enum: ["simple", "dole", "neural_dole"],
|
||||
initial_value: "simple",
|
||||
});
|
||||
MasterConfigGroup.define({
|
||||
name: "log_item_transfers",
|
||||
title: "Log Item Trassfers",
|
||||
description: "Spam master console with item transfers done.",
|
||||
type: "boolean",
|
||||
initial_value: false,
|
||||
});
|
||||
MasterConfigGroup.finalize();
|
||||
|
||||
class InstanceConfigGroup extends config.PluginConfigGroup {}
|
||||
InstanceConfigGroup.groupName = "subspace_storage";
|
||||
InstanceConfigGroup.define({
|
||||
name: "log_item_transfers",
|
||||
title: "Log Item Trassfers",
|
||||
description: "Spam slave console with item transfers done.",
|
||||
type: "boolean",
|
||||
initial_value: false,
|
||||
});
|
||||
InstanceConfigGroup.finalize();
|
||||
|
||||
// JSON schema for subspace storage items
|
||||
const items = {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "array",
|
||||
minItems: 2,
|
||||
maxItems: 2,
|
||||
items: [
|
||||
{ type: "string" },
|
||||
{ type: "integer" },
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
name: "subspace_storage",
|
||||
title: "Subspace Storage",
|
||||
description: "Provides shared storage across instances for the Subspace Storage mod",
|
||||
version: "2.0.0-alpha",
|
||||
instanceEntrypoint: "instance",
|
||||
InstanceConfigGroup,
|
||||
|
||||
masterEntrypoint: "master",
|
||||
MasterConfigGroup,
|
||||
|
||||
messages: {
|
||||
// XXX this should be a request to be reliable
|
||||
place: new link.Event({
|
||||
type: "subspace_storage:place",
|
||||
links: ["instance-slave", "slave-master"],
|
||||
forwardTo: "master",
|
||||
eventProperties: {
|
||||
"instance_id": { type: "integer" },
|
||||
"items": items,
|
||||
},
|
||||
}),
|
||||
remove: new link.Request({
|
||||
type: "subspace_storage:remove",
|
||||
links: ["instance-slave", "slave-master"],
|
||||
forwardTo: "master",
|
||||
requestProperties: {
|
||||
"instance_id": { type: "integer" },
|
||||
"items": items,
|
||||
},
|
||||
responseProperties: {
|
||||
"items": items,
|
||||
},
|
||||
}),
|
||||
getStorage: new link.Request({
|
||||
type: "subspace_storage:get_storage",
|
||||
links: ["instance-slave", "slave-master"],
|
||||
forwardTo: "master",
|
||||
responseProperties: {
|
||||
"items": items,
|
||||
},
|
||||
}),
|
||||
updateStorage: new link.Event({
|
||||
type: "subspace_storage:update_storage",
|
||||
links: ["master-slave", "slave-instance"],
|
||||
broadcastTo: "instance",
|
||||
eventProperties: {
|
||||
"items": items,
|
||||
},
|
||||
}),
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
const fs = require("fs-extra");
|
||||
|
||||
const plugin = require("lib/plugin");
|
||||
const luaTools = require("lib/luaTools");
|
||||
|
||||
function unexpectedError(err) {
|
||||
console.log("Unexpected error in subspace_storage")
|
||||
console.log("------------------------------------")
|
||||
console.log(err);
|
||||
}
|
||||
|
||||
class InstancePlugin extends plugin.BaseInstancePlugin {
|
||||
async init() {
|
||||
this.instance.server.on("ipc-subspace_storage:output", (output) => {
|
||||
this.provideItems(output).catch(unexpectedError);
|
||||
});
|
||||
this.instance.server.on("ipc-subspace_storage:orders", (orders) => {
|
||||
this.requestItems(orders).catch(unexpectedError);
|
||||
});
|
||||
}
|
||||
|
||||
async onStart() {
|
||||
this.pingId = setInterval(() => {
|
||||
this.instance.server.sendRcon(
|
||||
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0"
|
||||
).catch(unexpectedError);
|
||||
}, 5000);
|
||||
|
||||
let response = await this.info.messages.getStorage.send(this.instance);
|
||||
let itemsJson = luaTools.escapeString(JSON.stringify(response.items));
|
||||
await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`);
|
||||
}
|
||||
|
||||
async onStop() {
|
||||
clearInterval(this.pingId);
|
||||
}
|
||||
|
||||
// provide items --------------------------------------------------------------
|
||||
async provideItems(items) {
|
||||
this.info.messages.place.send(this.instance, {
|
||||
items,
|
||||
instance_id: this.instance.config.get("instance.id"),
|
||||
});
|
||||
|
||||
if (this.instance.config.get("subspace_storage.log_item_transfers")) {
|
||||
console.log("Exported the following to master:")
|
||||
console.log(items);
|
||||
}
|
||||
}
|
||||
|
||||
// request items --------------------------------------------------------------
|
||||
async requestItems(items) {
|
||||
// Request the items all at once
|
||||
let response = await this.info.messages.remove.send(this.instance, {
|
||||
instance_id: this.instance.config.get("instance.id"),
|
||||
items,
|
||||
});
|
||||
|
||||
if (!response.items.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.instance.config.get("subspace_storage.log_item_transfers")) {
|
||||
console.log("Imported following from master:")
|
||||
console.log(response.items);
|
||||
}
|
||||
|
||||
let itemsJson = luaTools.escapeString(JSON.stringify(response.items));
|
||||
await this.instance.server.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`);
|
||||
}
|
||||
|
||||
// combinator signals ---------------------------------------------------------
|
||||
async updateStorageEventHandler(message, event) {
|
||||
let items = message.data.items;
|
||||
|
||||
// XXX this should be done on the lua side
|
||||
// Ensure counts don't overflow
|
||||
for (let item of items) {
|
||||
if (item[1] > 0x7ffffff) {
|
||||
item[1] = 0x7ffffff;
|
||||
}
|
||||
}
|
||||
|
||||
// XXX this should be moved to instance/clusterio api
|
||||
items.push(["signal-unixtime", Math.floor(Date.now()/1000)]);
|
||||
|
||||
let itemsJson = luaTools.escapeString(JSON.stringify(items));
|
||||
await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
InstancePlugin,
|
||||
};
|
||||
@@ -0,0 +1,217 @@
|
||||
"use strict";
|
||||
const fs = require("fs-extra");
|
||||
const path = require("path");
|
||||
|
||||
const database = require("lib/database");
|
||||
const plugin = require("lib/plugin");
|
||||
const prometheus = require("lib/prometheus");
|
||||
|
||||
const routes = require("./routes");
|
||||
const dole = require("./dole");
|
||||
|
||||
|
||||
const prometheusExportGauge = new prometheus.Gauge(
|
||||
"clusterio_export_gauge",
|
||||
"Resources exported by instance",
|
||||
{ labels: ["instance_id", "resource"] }
|
||||
);
|
||||
const prometheusImportGauge = new prometheus.Gauge(
|
||||
"clusterio_import_gauge",
|
||||
"Resources imported by instance",
|
||||
{ labels: ["instance_id", "resource"] }
|
||||
);
|
||||
|
||||
|
||||
class MasterPlugin extends plugin.BaseMasterPlugin {
|
||||
async init() {
|
||||
this.prometheusMasterInventoryGauge = new prometheus.Gauge(
|
||||
"clusterio_master_inventory_gauge",
|
||||
"Amount of resources stored on master",
|
||||
{ labels: ["resource"] }
|
||||
);
|
||||
|
||||
this.ui = {
|
||||
sidebar: [
|
||||
{
|
||||
getHtml: () => `
|
||||
<div class="nav-item mr-1">
|
||||
<a class="nav-link align-middle" href="/subspace_storage/storage">Storage</a>
|
||||
</div>`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
this.items = await loadDatabase(this.master.config);
|
||||
this.itemsLastUpdate = new Map(this.items._items.entries());
|
||||
this.autosaveId = setInterval(() => {
|
||||
saveDatabase(this.master.config, this.items).catch(err => {
|
||||
console.log("Unexpected error autosaving items in subspace_storage")
|
||||
console.log("-----------------------------------------------------")
|
||||
console.log(err);
|
||||
});
|
||||
}, this.master.config.get("subspace_storage.autosave_interval") * 1000);
|
||||
|
||||
this.neuralDole = new dole.neuralDole({ items: this.items });
|
||||
this.doleMagicId = setInterval(() => {
|
||||
if (this.master.config.get("subspace_storage.division_method") === "neural_dole") {
|
||||
this.neuralDole.doMagic()
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
routes.addApiRoutes(this.master.app, this.items, this.metrics.endpointHitCounter);
|
||||
routes.addWebRoutes(this.master.app);
|
||||
}
|
||||
|
||||
updateStorage() {
|
||||
let itemsToUpdate = new Map();
|
||||
for (let [name, count] of this.items._items) {
|
||||
if (this.itemsLastUpdate.get(name) === count) {
|
||||
continue;
|
||||
}
|
||||
itemsToUpdate.set(name, count);
|
||||
}
|
||||
|
||||
if (!itemsToUpdate.size) {
|
||||
return;
|
||||
}
|
||||
|
||||
itemsToUpdate = [...itemsToUpdate.entries()];
|
||||
for (let slaveConnection of this.master.slaveConnections.values()) {
|
||||
this.info.messages.updateStorage.send(slaveConnection, { items: itemsToUpdate });
|
||||
}
|
||||
this.itemsLastUpdate = new Map(this.items._items.entries());
|
||||
}
|
||||
|
||||
async getStorageRequestHandler(message) {
|
||||
return { items: [...this.items._items.entries()] };
|
||||
}
|
||||
|
||||
async placeEventHandler(message) {
|
||||
let instanceId = message.data.instance_id;
|
||||
|
||||
for (let item of message.data.items) {
|
||||
this.items.addItem(item[0], item[1]);
|
||||
prometheusExportGauge.labels(String(instanceId), item[0]).inc(item[1]);
|
||||
}
|
||||
|
||||
this.updateStorage();
|
||||
|
||||
if (this.master.config.get("subspace_storage.log_item_transfers")) {
|
||||
console.log(`Imported the following from ${message.data.instance_id}:`)
|
||||
console.log(message.data.items);
|
||||
}
|
||||
}
|
||||
|
||||
async removeRequestHandler(message) {
|
||||
let method = this.master.config.get("subspace_storage.division_method");
|
||||
let instanceId = message.data.instance_id;
|
||||
|
||||
let itemsRemoved = [];
|
||||
if (method === "simple") {
|
||||
// Give out as much items as possible until there are 0 left. This
|
||||
// might lead to one slave getting all the items and the rest nothing.
|
||||
for (let item of message.data.items) {
|
||||
let count = this.items.getItemCount(item[0]);
|
||||
let toRemove = Math.min(count, item[1]);
|
||||
if (toRemove > 0) {
|
||||
this.items.removeItem(item[0], toRemove);
|
||||
itemsRemoved.push([item[0], toRemove]);
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
let instanceConfig = this.master.db.instances.get(instanceId);
|
||||
let instanceName = instanceConfig ? instanceConfig.get("instance.name") : "unkonwn";
|
||||
|
||||
// use fancy neural net to calculate a "fair" dole division rate.
|
||||
if (method === "neural_dole") {
|
||||
for (let item of message.data.items) {
|
||||
let count = neuralDole.divider({ name: item[0], count: item[1], instanceId, instanceName });
|
||||
if (count > 0) {
|
||||
itemsRemoved.push([item[0], count]);
|
||||
}
|
||||
}
|
||||
|
||||
// Use dole division. Makes it really slow to drain out the last little bit.
|
||||
} else if (method === "dole") {
|
||||
for (let item of message.data.items) {
|
||||
let count = dole.doleDivider({
|
||||
object: { name: item[0], count: item[1], instanceId, instanceName},
|
||||
items: this.items,
|
||||
});
|
||||
if (count > 0) {
|
||||
itemsRemoved.push([item[0], count]);
|
||||
}
|
||||
}
|
||||
|
||||
// Should not be possible
|
||||
} else {
|
||||
throw Error(`Unkown division_method ${method}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (itemsRemoved.length) {
|
||||
for (let item of itemsRemoved) {
|
||||
prometheusImportGauge.labels(String(instanceId), item[0]).inc(item[1]);
|
||||
}
|
||||
|
||||
this.updateStorage();
|
||||
|
||||
if (itemsRemoved.length && this.master.config.get("subspace_storage.log_item_transfers")) {
|
||||
console.log(`Exported the following to ${instanceId}`);
|
||||
console.log(itemsRemoved);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
items: itemsRemoved,
|
||||
}
|
||||
}
|
||||
|
||||
onMetrics() {
|
||||
if (this.items) {
|
||||
for (let [key, count] of this.items._items) {
|
||||
this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async onExit() {
|
||||
clearInterval(this.autosaveId);
|
||||
clearInterval(this.doleMagicId);
|
||||
await saveDatabase(this.master.config, this.items);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadDatabase(config) {
|
||||
let itemsPath = path.resolve(config.get("master.database_directory"), "items.json");
|
||||
console.log(`Loading ${itemsPath}`);
|
||||
try {
|
||||
let content = await fs.readFile(itemsPath);
|
||||
return new database.ItemDatabase(JSON.parse(content));
|
||||
|
||||
} catch (err) {
|
||||
if (err.code === "ENOENT") {
|
||||
console.log("Creating new item database");
|
||||
return new database.ItemDatabase();
|
||||
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDatabase(masterConfig, items) {
|
||||
if (items && items.size < 50000) {
|
||||
let file = path.resolve(masterConfig.get("master.database_directory"), "items.json");
|
||||
console.log(`writing ${file}`);
|
||||
let content = JSON.stringify(items.serialize());
|
||||
await fs.outputFile(file, content);
|
||||
} else if (items) {
|
||||
console.error(`Item database too large, not saving (${items.size})`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
MasterPlugin,
|
||||
};
|
||||
@@ -0,0 +1,55 @@
|
||||
const express = require("express");
|
||||
const path = require("path");
|
||||
|
||||
const authenticate = require("lib/authenticate");
|
||||
const prometheus = require("lib/prometheus");
|
||||
|
||||
|
||||
function addWebRoutes(app) {
|
||||
app.use("/subspace_storage", express.static(path.join(__dirname, "static")));
|
||||
app.get("/subspace_storage/storage", (req, res) => { res.render("subspace_storage/views/storage"); });
|
||||
}
|
||||
|
||||
function addApiRoutes(app, items, endpointHitCounter) {
|
||||
|
||||
/**
|
||||
* GET endpoint to read the masters current inventory of items.
|
||||
*
|
||||
* @memberof clusterioMaster
|
||||
* @instance
|
||||
* @alias api/inventory
|
||||
* @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
|
||||
*/
|
||||
app.get("/api/inventory", function(req, res) {
|
||||
endpointHitCounter.labels(req.route.path).inc();
|
||||
res.header("Access-Control-Allow-Origin", "*");
|
||||
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
|
||||
// Check it and send it
|
||||
var inventory = [];
|
||||
for (let [name, count] of items._items) {
|
||||
inventory.push({ name, count });
|
||||
}
|
||||
res.send(JSON.stringify(inventory));
|
||||
});
|
||||
|
||||
/**
|
||||
* GET endpoint to read the masters inventory as an object with key:value pairs
|
||||
*
|
||||
* @memberof clusterioMaster
|
||||
* @instance
|
||||
* @alias api/inventoryAsObject
|
||||
* @returns {object} JSON {"iron-plate":100, "copper-plate":5}
|
||||
*/
|
||||
app.get("/api/inventoryAsObject", function(req, res) {
|
||||
endpointHitCounter.labels(req.route.path).inc();
|
||||
res.header("Access-Control-Allow-Origin", "*");
|
||||
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
|
||||
// Check it and send it
|
||||
res.send(JSON.stringify(items.serialise()));
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
addWebRoutes,
|
||||
addApiRoutes,
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// function to draw data we recieve from ajax requests
|
||||
_lastData = [];
|
||||
function drawcontents(data) {
|
||||
data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box.
|
||||
_lastData = data;
|
||||
|
||||
// allow searching for multiple criteria separated by space
|
||||
var searchArgs = document.querySelector("#search").value;
|
||||
searchArgs = searchArgs.trim();
|
||||
searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')';
|
||||
const search = new RegExp(searchArgs, 'i');
|
||||
data = data.filter(function(item) {
|
||||
return search.test(item.name);
|
||||
})
|
||||
|
||||
sortByKey(data, "count");
|
||||
|
||||
const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically
|
||||
const rows = table.children;
|
||||
|
||||
//update existing rows or create new ones
|
||||
data.forEach(function(item, i) {
|
||||
// format count to have .s in it for large number readability (1,000,000)
|
||||
item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
||||
|
||||
let row = rows[i];
|
||||
if(!row) {
|
||||
row = document.createElement('tr');
|
||||
row.innerHTML = "<td><img width=32 height=32></td><td class=name></td><td class=count></td>";
|
||||
table.appendChild(row);
|
||||
}
|
||||
|
||||
const img = row.querySelector('img');
|
||||
const imgName = getImageFromName(item.name);
|
||||
if(img.getAttribute('src') !== imgName) {
|
||||
img.setAttribute('src',imgName);
|
||||
}
|
||||
|
||||
const name = row.querySelector('.name');
|
||||
if(name.textContent !== item.name) {
|
||||
name.textContent = item.name;
|
||||
}
|
||||
|
||||
const count = row.querySelector('.count');
|
||||
if(count.textContent !== ''+item.count) {
|
||||
count.textContent = item.count;
|
||||
}
|
||||
});
|
||||
|
||||
//remove excess rows, for example, after filtering
|
||||
while (data.length < rows.length) {
|
||||
table.removeChild(rows[data.length]);
|
||||
}
|
||||
}
|
||||
|
||||
// get cluster inventory from master
|
||||
function updateInventory() {
|
||||
var xmlhttp = new XMLHttpRequest();
|
||||
xmlhttp.onreadystatechange = function() {
|
||||
if(xmlhttp.readyState == 4 && xmlhttp.status == 200) {
|
||||
let data = JSON.parse(xmlhttp.responseText);
|
||||
drawcontents(data);
|
||||
}
|
||||
}
|
||||
xmlhttp.open("GET", "/api/inventory", true);
|
||||
xmlhttp.send();
|
||||
}
|
||||
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
|
||||
setInterval(updateInventory, 500);
|
||||
} else {
|
||||
updateInventory();
|
||||
}
|
||||
|
||||
// function to sort arrays of objects after a keys value
|
||||
function sortByKey(array, key) {
|
||||
array.sort(function(a, b) {
|
||||
return b[key] - a[key];
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelector("#search").addEventListener('input', function() {
|
||||
drawcontents();
|
||||
});
|
||||
@@ -0,0 +1,7 @@
|
||||
// These are libraries that does not have any units test, they are
|
||||
// required here in order to track coverage for them.
|
||||
|
||||
require('../instance');
|
||||
require('../master');
|
||||
require('../dole');
|
||||
require('../dole_nn_base');
|
||||
@@ -0,0 +1,44 @@
|
||||
const assert = require("assert").strict;
|
||||
const express = require("express");
|
||||
const bodyParser = require("body-parser");
|
||||
const request = require("supertest");
|
||||
const jwt = require("jsonwebtoken");
|
||||
|
||||
const authenticate = require('lib/authenticate');
|
||||
const database = require("lib/database");
|
||||
const prometheus = require("lib/prometheus");
|
||||
|
||||
const routes = require("../routes");
|
||||
|
||||
|
||||
const testSecret = "TestSecretDoNotUse";
|
||||
authenticate.setAuthSecret(testSecret);
|
||||
|
||||
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
|
||||
|
||||
describe("subspace_storage endpoints", function() {
|
||||
const app = express();
|
||||
app.use(bodyParser.json({ limit: '10mb', }));
|
||||
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
|
||||
const items = new database.ItemDatabase();
|
||||
items.addItem("steel-plate", 20);
|
||||
const gauge = new prometheus.Gauge("test_gauge", "test", { labels: ["route"] });
|
||||
routes.addApiRoutes(app, items, gauge);
|
||||
|
||||
let persistentMaster = request(app);
|
||||
describe("#GET /api/inventory", function() {
|
||||
it("returns the masters current inventory", function() {
|
||||
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
|
||||
let inventory = JSON.parse(res.text);
|
||||
assert.equal(typeof inventory, "object", "Inventory should be an object");
|
||||
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
|
||||
|
||||
let contains20SteelPlate = false;
|
||||
inventory.forEach(itemStack => {
|
||||
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
|
||||
});
|
||||
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
<!doctype HTML>
|
||||
<head>
|
||||
<%- include("head") %>
|
||||
<script defer src="/subspace_storage/storage.js"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<%- include("nav") %>
|
||||
<div id="body" class="container">
|
||||
<div id="storage">
|
||||
<div class="page-header mt-2 row" id="banner">
|
||||
<div class="row">
|
||||
<div class="col-lg-8 col-md-7 col-sm-6">
|
||||
<h2>Storage</h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group mt-3 mb-3 row">
|
||||
<input type="text" class="form-control" id="search" placeholder="search">
|
||||
</div>
|
||||
<div id="contents" class="row w-100">
|
||||
<table class="table table-striped table-hover">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th scope="col">Icon</th>
|
||||
<th scope="col">Item ID</th>
|
||||
<th scope="col">Quantity</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody><!-- populated by js --></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
|
||||
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
|
||||
crossorigin="anonymous"></script>
|
||||
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
|
||||
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
|
||||
crossorigin="anonymous"></script>
|
||||
</body>
|
||||
Reference in new issue
Block a user