Add quality support to subspace storage

This commit is contained in:
Danielv123 committed 2025-01-22 19:46:23 +01:00
1 parent 26269df7d4
commit 4fd123abc1
5 files changed
+182 -117

No files matched your search

+37 -23
View File
@@ -164,7 +164,7 @@ function checkCount(count: unknown): asserts count is number {
* deserialized the content is verified.
*/
export class ItemDatabase {
private _items: Map<string, number> = new Map();
private _items: Map<string, { [quality: string]: number }> = new Map();
/**
* Create a new item database
@@ -177,11 +177,19 @@ export class ItemDatabase {
constructor(serialized?: object) {
// Verify the content of the serialized database
if (serialized !== undefined) {
for (let [name, count] of Object.entries(serialized)) {
for (let [name, qualities] of Object.entries(serialized)) {
checkName(name);
checkCount(count);
// Migration from pre-quality database format
if (typeof qualities === "number") {
qualities = { normal: qualities };
}
for (const quality in qualities) {
if (Object.prototype.hasOwnProperty.call(qualities, quality)) {
checkCount(qualities[quality]);
}
}
this._items.set(name, count);
this._items.set(name, qualities);
}
}
}
@@ -195,10 +203,15 @@ export class ItemDatabase {
* @returns Serialized representation of the database
*/
serialize() {
let obj: Record<string, number> = {};
for (let [name, count] of this._items) {
if (count !== 0) {
obj[name] = count;
let obj: Record<string, { [quality: string]: number }> = {};
for (let [name, qualities] of this._items) {
for (const quality in qualities) {
if (Object.prototype.hasOwnProperty.call(qualities, quality) && qualities[quality] !== 0) {
if (!obj[name]) {
obj[name] = {};
}
obj[name][quality] = qualities[quality];
}
}
}
return obj;
@@ -222,16 +235,17 @@ export class ItemDatabase {
* then 0 is returned.
*
* @param name - The name of the item to get the count of.
* @param quality - The quality of the item to get the count of.
* @returns The count of the item stored.
*/
getItemCount(name: string): number {
getItemCount(name: string, quality: string): number {
checkName(name);
if (!this._items.has(name)) {
if (!this._items.has(name) || !this._items.get(name)?.[quality]) {
return 0;
}
return this._items.get(name)!;
return this._items.get(name)![quality];
}
/**
@@ -241,16 +255,20 @@ export class ItemDatabase {
*
* @param name - The name of the item to add.
* @param count - The count of item to remove.
* @param quality - The quality of the item to add.
*/
addItem(name: string, count: number) {
addItem(name: string, count: number, quality: string) {
checkName(name);
checkCount(count);
if (this._items.has(name)) {
count += this._items.get(name)!;
if (!this._items.has(name)) {
this._items.set(name, { [quality]: count });
return;
}
this._items.set(name, count);
// Nullish coalescing operator is used to avoid NaN when the quality is not present
let currentCount = this._items.get(name)![quality] ?? 0;
this._items.get(name)![quality] = currentCount + count;
};
/**
@@ -262,24 +280,20 @@ export class ItemDatabase {
*
* @param name - The name of the item to remove.
* @param count - The count of items to remove.
* @param quality - The quality of the item to remove.
*/
removeItem(name: string, count: number) {
removeItem(name: string, count: number, quality: string) {
checkName(name);
checkCount(count);
count = -count;
if (this._items.has(name)) {
count += this._items.get(name)!;
}
this._items.set(name, count);
this.addItem(name, -count, quality);
};
/**
* Allow to iterate through the items.
* @returns array of items.
*/
getEntries(): IterableIterator<[string, number]> {
getEntries(): IterableIterator<[string, { [quality: string]: number }]> {
return this._items.entries();
}
}
+49 -32
View File
@@ -22,17 +22,17 @@ import {
const exportCounter = new Counter(
"clusterio_subspace_storage_export_total",
"Resources exported by instance",
{ labels: ["instance_id", "resource"] }
{ labels: ["instance_id", "resource", "quality"] }
);
const importCounter = new Counter(
"clusterio_subspace_storage_import_total",
"Resources imported by instance",
{ labels: ["instance_id", "resource"] }
{ labels: ["instance_id", "resource", "quality"] }
);
const controllerInventoryGauge = new Gauge(
"clusterio_subspace_storage_controller_inventory",
"Amount of resources stored on controller",
{ labels: ["resource"] }
{ labels: ["resource", "quality"] }
);
@@ -73,7 +73,7 @@ async function saveDatabase(
export class ControllerPlugin extends BaseControllerPlugin {
items!: lib.ItemDatabase;
itemUpdateRateLimiter!: lib.RateLimiter;
itemsLastUpdate!: Map<string, number>;
itemsLastUpdate: Map<string, { [quality: string]: number }> = new Map();
subscribedControlLinks!: Set<ControlConnection>;
doleMagicId!: ReturnType<typeof setInterval>;
neuralDole!: dole.NeuralDole;
@@ -91,7 +91,10 @@ export class ControllerPlugin extends BaseControllerPlugin {
}
},
});
this.itemsLastUpdate = new Map(this.items.getEntries());
this.itemsLastUpdate = new Map();
for (const [name, qualities] of this.items.getEntries()) {
this.itemsLastUpdate.set(name, { ...qualities });
}
this.neuralDole = new dole.NeuralDole({ items: this.items });
this.doleMagicId = setInterval(() => {
@@ -116,37 +119,49 @@ export class ControllerPlugin extends BaseControllerPlugin {
}
broadcastStorage() {
let itemsToUpdateMap:Map<string, number> = new Map();
for (let [name, count] of this.items.getEntries()) {
if (this.itemsLastUpdate.get(name) === count) {
continue;
let itemsToUpdate: Item[] = [];
for (const [name, qualities] of this.items.getEntries()) {
const lastQualities = this.itemsLastUpdate.get(name);
for (const [quality, count] of Object.entries(qualities)) {
if (!lastQualities || lastQualities[quality] !== count) {
itemsToUpdate.push(new Item(name, count, quality));
}
}
itemsToUpdateMap.set(name, count);
}
if (!itemsToUpdateMap.size) {
if (!itemsToUpdate.length) {
return;
}
let itemsToUpdate = [...itemsToUpdateMap.entries()];
let update = UpdateStorageEvent.fromJSON({ items: itemsToUpdate });
let update = new UpdateStorageEvent(itemsToUpdate);
this.controller.sendTo("allInstances", update);
for (let link of this.subscribedControlLinks) {
link.send(update);
}
this.itemsLastUpdate = new Map(this.items.getEntries());
this.itemsLastUpdate = new Map();
for (const [name, qualities] of this.items.getEntries()) {
this.itemsLastUpdate.set(name, { ...qualities });
}
}
async handleGetStorageRequest() {
return [...this.items.getEntries()];
const result: Item[] = [];
for (const [name, qualities] of this.items.getEntries()) {
for (const [quality, count] of Object.entries(qualities)) {
result.push(new Item(name, count, quality));
}
}
this.logger.verbose(`Sending storage update:\n${JSON.stringify(result)}`);
return result;
}
async handlePlaceEvent(request: PlaceEvent, src: lib.Address) {
let instanceId = src.id;
for (let item of request.items) {
this.items.addItem(item.name, item.count);
exportCounter.labels(String(instanceId), item.name).inc(item.count);
this.items.addItem(item.name, item.count, item.quality || "normal");
exportCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count);
}
this.updateStorage();
@@ -164,55 +179,55 @@ export class ControllerPlugin extends BaseControllerPlugin {
let itemsRemoved = [];
if (method === "simple") {
// Give out as much items as possible until there are 0 left. This
// might lead to one host getting all the items and the rest nothing.
for (let item of request.items) {
let count = this.items.getItemCount(item.name);
const quality = item.quality || "normal";
let count = this.items.getItemCount(item.name, quality);
let toRemove = Math.min(count, item.count);
if (toRemove > 0) {
this.items.removeItem(item.name, toRemove);
itemsRemoved.push(new Item(item.name, toRemove));
this.items.removeItem(item.name, toRemove, quality);
itemsRemoved.push(new Item(item.name, toRemove, quality));
}
}
} else {
let instance = this.controller.instances.get(instanceId);
let instanceName = instance ? instance.config.get("instance.name") : "unkonwn";
let instanceName = instance ? instance.config.get("instance.name") : "unknown";
// use fancy neural net to calculate a "fair" dole division rate.
if (method === "neural_dole") {
for (let item of request.items) {
const quality = item.quality || "normal";
let count = this.neuralDole.divider(
{ name: item.name, count: item.count, instanceId, instanceName }
{ name: item.name, quality, count: item.count, instanceId, instanceName }
);
if (count > 0) {
itemsRemoved.push(new Item(item.name, count));
itemsRemoved.push(new Item(item.name, count, quality));
}
}
// Use dole division. Makes it really slow to drain out the last little bit.
} else if (method === "dole") {
for (let item of request.items) {
const quality = item.quality || "normal";
let count = dole.doleDivider({
object: { name: item.name, count: item.count, instanceId, instanceName },
object: { name: item.name, quality, count: item.count, instanceId, instanceName },
items: this.items,
logItemTransfers: this.controller.config.get("subspace_storage.log_item_transfers"),
logger: this.logger,
});
if (count > 0) {
itemsRemoved.push(new Item(item.name, count));
itemsRemoved.push(new Item(item.name, count, quality));
}
}
// Should not be possible
} else {
throw Error(`Unkown division_method ${method}`);
throw Error(`Unknown division_method ${method}`);
}
}
if (itemsRemoved.length) {
for (let item of itemsRemoved) {
importCounter.labels(String(instanceId), item.name).inc(item.count);
importCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count);
}
this.updateStorage();
@@ -242,8 +257,10 @@ export class ControllerPlugin extends BaseControllerPlugin {
async onMetrics() {
if (this.items) {
for (let [key, count] of this.items.getEntries()) {
controllerInventoryGauge.labels(key).set(Number(count) || 0);
for (const [name, qualities] of this.items.getEntries()) {
for (const [quality, count] of Object.entries(qualities)) {
controllerInventoryGauge.labels(name, quality).set(Number(count) || 0);
}
}
}
}
+67 -50
View File
@@ -8,23 +8,23 @@ const prometheusNNDoleGauge = new Gauge(
"clusterio_subspace_storage_nn_dole_gauge",
"Current demand being supplied by Neural Network ; 1 means all demand covered, "+
"0.5 means about half of each supply is covered, 0 means no items are given",
{ labels: ["resource"] }
{ labels: ["resource", "quality"] }
);
const prometheusDoleFactorGauge = new Gauge(
"clusterio_subspace_storage_dole_factor_gauge",
"The current dole division factor for this resource",
{ labels: ["resource"] }
{ labels: ["resource", "quality"] }
);
export class NeuralDole {
getRequestStats(itemname: string, samples: number): number {
getRequestStats(itemname: string, quality: string, samples: number): number {
let sum = 0;
samples = Math.min(samples, (this.stats[itemname] || []).length - 1);
samples = Math.min(samples, (this.stats[itemname+quality] || []).length - 1);
if (samples < 1) {
return 0.1;
}
for (let i = 1; i <= samples; i++) {
sum += this.stats[itemname][i].req;
sum += this.stats[itemname+quality][i].req;
}
if (sum === 0) {
return 0.1;
@@ -42,55 +42,67 @@ export class NeuralDole {
constructor({ items }: { items: lib.ItemDatabase }) {
// Set some storage variables for the dole divider
this.items = items;
this.itemsLastTick = new Map(items.getEntries());
this.itemsLastTick = new Map();
for (let [name, qualities] of items.getEntries()) {
for (let [quality, count] of Object.entries(qualities)) {
this.itemsLastTick.set(name+quality, count);
}
}
}
doMagic() {
for (let [name, count] of this.items.getEntries()) {
let magicData = doleNN.tick(
count,
this.dole[name],
this.itemsLastTick.get(name) || 0,
this.getRequestStats(name, 5)
);
this.stats[name] = this.stats[name] || [];
this.stats[name].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we currently collect
if (this.stats[name].length>10) {
this.stats[name].pop(); // remove if too many samples in stats
}
for (let [name, qualities] of this.items.getEntries()) {
for (let [quality, count] of Object.entries(qualities)) {
let magicData = doleNN.tick(
count,
this.dole[name+quality],
this.itemsLastTick.get(name+quality) || 0,
this.getRequestStats(name, quality, 5)
);
this.stats[name+quality] = this.stats[name+quality] || [];
this.stats[name+quality].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we are collecting
if (this.stats[name+quality].length>10) {
this.stats[name+quality].pop(); // remove if too many samples in stats
}
this.dole[name] = magicData[0];
// DONE handle magicData[1] for graphing for our users
prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
this.dole[name+quality] = magicData[0];
// DONE handle magicData[1] for graphing for our users
prometheusNNDoleGauge.labels(name+quality).set(magicData[1] || 0);
}
}
this.itemsLastTick = new Map();
for (let [name, qualities] of this.items.getEntries()) {
for (let [quality, count] of Object.entries(qualities)) {
this.itemsLastTick.set(name+quality, count);
}
}
this.itemsLastTick = new Map(this.items.getEntries());
}
divider(object: { name:string, count:number, instanceId:number, instanceName:string }): number {
divider(object: { name:string, quality:string, count:number, instanceId:number, instanceName:string }): number {
let magicData = doleNN.dose(
object.count, // numReq
this.items.getItemCount(object.name) || 0,
this.itemsLastTick.get(object.name) || 0,
this.dole[object.name],
this.carry[`${object.name} ${object.instanceId}`] || 0,
this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] || 0,
this.getRequestStats(object.name, 5),
this.debt[`${object.name} ${object.instanceId}`] || 0
this.items.getItemCount(object.name, object.quality) || 0,
this.itemsLastTick.get(object.name+object.quality) || 0,
this.dole[object.name+object.quality],
this.carry[`${object.name} ${object.quality} ${object.instanceId}`] || 0,
this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] || 0,
this.getRequestStats(object.name, object.quality, 5),
this.debt[`${object.name} ${object.quality} ${object.instanceId}`] || 0
);
if ((this.stats[object.name] || []).length > 0) {
this.stats[object.name][0].req += Number(object.count);
this.stats[object.name][0].given += Number(magicData[0]);
if ((this.stats[object.name+object.quality] || []).length > 0) {
this.stats[object.name+object.quality][0].req += Number(object.count);
this.stats[object.name+object.quality][0].given += Number(magicData[0]);
}
this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] = object.count;
this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] = object.count;
// 0. Number of items to give in that dose
// 1. New dole for item X
// 2. New carry for item X instance Y
this.dole[object.name] = magicData[1];
this.carry[`${object.name} ${object.instanceId}`] = magicData[2];
this.debt[`${object.name} ${object.instanceId}`] = magicData[3];
this.dole[object.name+object.quality] = magicData[1];
this.carry[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[2];
this.debt[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[3];
// Remove item from DB and send it
this.items.removeItem(object.name, magicData[0]);
this.items.removeItem(object.name, magicData[0], object.quality);
return magicData[0];
}
@@ -106,13 +118,13 @@ export function doleDivider(
logItemTransfers,
logger,
} :{
object: { name:string, count:number, instanceId:number, instanceName:string },
object: { name:string, quality:string, count:number, instanceId:number, instanceName:string },
items: lib.ItemDatabase,
logItemTransfers: boolean,
logger: lib.Logger,
},
) {
let itemCount = items.getItemCount(object.name);
let itemCount = items.getItemCount(object.name, object.quality);
// lower rates will equal more dramatic swings
const doleDivisionRetardation = 10;
// a higher cap will divide the store more ways, but will take longer to
@@ -120,13 +132,14 @@ export function doleDivider(
const maxDoleDivision = 250;
const originalCount = Number(object.count) || 0;
object.count /= ((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation;
const doleFactor = _doleDivisionFactor[object.name+object.quality] || 0;
object.count /= (doleFactor + doleDivisionRetardation) / doleDivisionRetardation;
object.count = Math.round(object.count);
if (logItemTransfers) {
logger.verbose(
`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} `+
`with dole division factor ${(_doleDivisionFactor[object.name]||0)} `+
`(real=${((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation}), `+
`with dole division factor ${doleFactor} `+
`(real=${doleFactor + doleDivisionRetardation}), `+
`item is ${itemCount > object.count?"stocked":"short"}.`
);
}
@@ -134,18 +147,22 @@ export function doleDivider(
// Update existing items if item name already exists
if (itemCount > object.count) {
// If successful, increase dole
_doleDivisionFactor[object.name] = Math.max(_doleDivisionFactor[object.name] || 1, 1) - 1;
items.removeItem(object.name, object.count);
const key = object.name + object.quality;
_doleDivisionFactor[key] = Math.max(_doleDivisionFactor[key] || 1, 1) - 1;
items.removeItem(object.name, object.count, object.quality);
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
prometheusDoleFactorGauge
.labels(object.name, object.quality)
.set(_doleDivisionFactor[key] || 0);
return object.count;
}
// if we didn't have enough, attempt giving out a smaller amount next time
_doleDivisionFactor[object.name] = Math.min(
maxDoleDivision, Math.max(_doleDivisionFactor[object.name] || 1, 1) * 2
);
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
const key = object.name + object.quality;
_doleDivisionFactor[key] = Math.min(maxDoleDivision, Math.max(_doleDivisionFactor[key] || 1, 1) * 2);
prometheusDoleFactorGauge
.labels(object.name, object.quality)
.set(_doleDivisionFactor[key] || 0);
return 0;
// console.log(`failure out of ${object.name}/${object.count} from ${object.instanceID} (${object.instanceName})`);
}
+24 -8
View File
@@ -9,7 +9,7 @@ import {
Item,
} from "./messages";
type IpcItems = [string, number][];
type IpcItems = [string, { [quality: string]: number }][];
export class InstancePlugin extends BaseInstancePlugin {
pendingTasks!: Set<any>;
@@ -54,7 +54,7 @@ export class InstancePlugin extends BaseInstancePlugin {
let items = await this.instance.sendTo("controller", new GetStorageRequest());
// TODO Diff with dump of invdata produce minimal command to sync
let itemsJson = lib.escapeString(JSON.stringify(items));
await this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true);
await this.sendRcon(`/c __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true);
}
async onStop() {
@@ -78,8 +78,15 @@ export class InstancePlugin extends BaseInstancePlugin {
return;
}
const fromIpcItems = items.map(item => new Item(item[0], item[1]));
const fromIpcItems: Item[] = [];
items.forEach(item => {
// Loop over quality
for (const quality in item[1]) {
if (Object.prototype.hasOwnProperty.call(item[1], quality)) {
fromIpcItems.push(new Item(item[0], item[1][quality], quality));
}
}
});
this.instance.sendTo("controller", new PlaceEvent(fromIpcItems));
if (this.instance.config.get("subspace_storage.log_item_transfers")) {
@@ -90,8 +97,17 @@ export class InstancePlugin extends BaseInstancePlugin {
// request items --------------------------------------------------------------
async requestItems(requestItems: IpcItems) {
this.logger.info(`Requesting items: ${JSON.stringify(requestItems)}`);
// Request the items all at once
let fromIpcItems = requestItems.map(item => new Item(item[0], item[1]));
const fromIpcItems: Item[] = [];
requestItems.forEach(item => {
// Loop over quality
for (const quality in item[1]) {
if (Object.prototype.hasOwnProperty.call(item[1], quality)) {
fromIpcItems.push(new Item(item[0], item[1][quality], quality));
}
}
});
let items = await this.instance.sendTo("controller", new RemoveRequest(fromIpcItems));
if (!items.length) {
@@ -104,7 +120,7 @@ export class InstancePlugin extends BaseInstancePlugin {
}
let itemsJson = lib.escapeString(JSON.stringify(items));
await this.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`, true);
await this.sendRcon(`/c __subspace_storage__ Import("${itemsJson}")`, true);
}
// combinator signals ---------------------------------------------------------
@@ -115,10 +131,10 @@ export class InstancePlugin extends BaseInstancePlugin {
let items = event.items;
// XXX this should be moved to instance/clusterio api
items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000)));
items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000), "normal"));
let itemsJson = lib.escapeString(JSON.stringify(items));
let task = this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true);
let task = this.sendRcon(`/c __subspace_storage__ UpdateInvData("${itemsJson}")`, true);
this.pendingTasks.add(task);
await task.finally(() => { this.pendingTasks.delete(task); });
}
+5 -4
View File
@@ -1,25 +1,26 @@
import { Type, Static } from "@sinclair/typebox";
import * as lib from "@clusterio/lib";
import { type } from "os";
export class Item {
constructor(
public name: string,
public count: number
public count: number,
public quality: string = "normal"
) {
}
static jsonSchema = Type.Tuple([
Type.String(),
Type.Number(),
Type.String(),
]);
toJSON() {
return [this.name, this.count];
return [this.name, this.count, this.quality];
}
static fromJSON(json: Static<typeof Item.jsonSchema>): Item {
return new this(json[0], json[1]);
return new this(json[0], json[1], json[2]);
}
}