diff --git a/packages/lib/src/database.ts b/packages/lib/src/database.ts index 37dc64f0..4f2a0f90 100644 --- a/packages/lib/src/database.ts +++ b/packages/lib/src/database.ts @@ -164,7 +164,7 @@ function checkCount(count: unknown): asserts count is number { * deserialized the content is verified. */ export class ItemDatabase { - private _items: Map = new Map(); + private _items: Map = new Map(); /** * Create a new item database @@ -177,11 +177,19 @@ export class ItemDatabase { constructor(serialized?: object) { // Verify the content of the serialized database if (serialized !== undefined) { - for (let [name, count] of Object.entries(serialized)) { + for (let [name, qualities] of Object.entries(serialized)) { checkName(name); - checkCount(count); + // Migration from pre-quality database format + if (typeof qualities === "number") { + qualities = { normal: qualities }; + } + for (const quality in qualities) { + if (Object.prototype.hasOwnProperty.call(qualities, quality)) { + checkCount(qualities[quality]); + } + } - this._items.set(name, count); + this._items.set(name, qualities); } } } @@ -195,10 +203,15 @@ export class ItemDatabase { * @returns Serialized representation of the database */ serialize() { - let obj: Record = {}; - for (let [name, count] of this._items) { - if (count !== 0) { - obj[name] = count; + let obj: Record = {}; + for (let [name, qualities] of this._items) { + for (const quality in qualities) { + if (Object.prototype.hasOwnProperty.call(qualities, quality) && qualities[quality] !== 0) { + if (!obj[name]) { + obj[name] = {}; + } + obj[name][quality] = qualities[quality]; + } } } return obj; @@ -222,16 +235,17 @@ export class ItemDatabase { * then 0 is returned. * * @param name - The name of the item to get the count of. + * @param quality - The quality of the item to get the count of. * @returns The count of the item stored. */ - getItemCount(name: string): number { + getItemCount(name: string, quality: string): number { checkName(name); - if (!this._items.has(name)) { + if (!this._items.has(name) || !this._items.get(name)?.[quality]) { return 0; } - return this._items.get(name)!; + return this._items.get(name)![quality]; } /** @@ -241,16 +255,20 @@ export class ItemDatabase { * * @param name - The name of the item to add. * @param count - The count of item to remove. + * @param quality - The quality of the item to add. */ - addItem(name: string, count: number) { + addItem(name: string, count: number, quality: string) { checkName(name); checkCount(count); - if (this._items.has(name)) { - count += this._items.get(name)!; + if (!this._items.has(name)) { + this._items.set(name, { [quality]: count }); + return; } - this._items.set(name, count); + // Nullish coalescing operator is used to avoid NaN when the quality is not present + let currentCount = this._items.get(name)![quality] ?? 0; + this._items.get(name)![quality] = currentCount + count; }; /** @@ -262,24 +280,20 @@ export class ItemDatabase { * * @param name - The name of the item to remove. * @param count - The count of items to remove. + * @param quality - The quality of the item to remove. */ - removeItem(name: string, count: number) { + removeItem(name: string, count: number, quality: string) { checkName(name); checkCount(count); - count = -count; - if (this._items.has(name)) { - count += this._items.get(name)!; - } - - this._items.set(name, count); + this.addItem(name, -count, quality); }; /** * Allow to iterate through the items. * @returns array of items. */ - getEntries(): IterableIterator<[string, number]> { + getEntries(): IterableIterator<[string, { [quality: string]: number }]> { return this._items.entries(); } } diff --git a/plugins/subspace_storage/controller.ts b/plugins/subspace_storage/controller.ts index dfafce44..7e84b5b9 100644 --- a/plugins/subspace_storage/controller.ts +++ b/plugins/subspace_storage/controller.ts @@ -22,17 +22,17 @@ import { const exportCounter = new Counter( "clusterio_subspace_storage_export_total", "Resources exported by instance", - { labels: ["instance_id", "resource"] } + { labels: ["instance_id", "resource", "quality"] } ); const importCounter = new Counter( "clusterio_subspace_storage_import_total", "Resources imported by instance", - { labels: ["instance_id", "resource"] } + { labels: ["instance_id", "resource", "quality"] } ); const controllerInventoryGauge = new Gauge( "clusterio_subspace_storage_controller_inventory", "Amount of resources stored on controller", - { labels: ["resource"] } + { labels: ["resource", "quality"] } ); @@ -73,7 +73,7 @@ async function saveDatabase( export class ControllerPlugin extends BaseControllerPlugin { items!: lib.ItemDatabase; itemUpdateRateLimiter!: lib.RateLimiter; - itemsLastUpdate!: Map; + itemsLastUpdate: Map = new Map(); subscribedControlLinks!: Set; doleMagicId!: ReturnType; neuralDole!: dole.NeuralDole; @@ -91,7 +91,10 @@ export class ControllerPlugin extends BaseControllerPlugin { } }, }); - this.itemsLastUpdate = new Map(this.items.getEntries()); + this.itemsLastUpdate = new Map(); + for (const [name, qualities] of this.items.getEntries()) { + this.itemsLastUpdate.set(name, { ...qualities }); + } this.neuralDole = new dole.NeuralDole({ items: this.items }); this.doleMagicId = setInterval(() => { @@ -116,37 +119,49 @@ export class ControllerPlugin extends BaseControllerPlugin { } broadcastStorage() { - let itemsToUpdateMap:Map = new Map(); - for (let [name, count] of this.items.getEntries()) { - if (this.itemsLastUpdate.get(name) === count) { - continue; + let itemsToUpdate: Item[] = []; + for (const [name, qualities] of this.items.getEntries()) { + const lastQualities = this.itemsLastUpdate.get(name); + for (const [quality, count] of Object.entries(qualities)) { + if (!lastQualities || lastQualities[quality] !== count) { + itemsToUpdate.push(new Item(name, count, quality)); + } } - itemsToUpdateMap.set(name, count); } - if (!itemsToUpdateMap.size) { + if (!itemsToUpdate.length) { return; } - let itemsToUpdate = [...itemsToUpdateMap.entries()]; - let update = UpdateStorageEvent.fromJSON({ items: itemsToUpdate }); + let update = new UpdateStorageEvent(itemsToUpdate); this.controller.sendTo("allInstances", update); for (let link of this.subscribedControlLinks) { link.send(update); } - this.itemsLastUpdate = new Map(this.items.getEntries()); + + this.itemsLastUpdate = new Map(); + for (const [name, qualities] of this.items.getEntries()) { + this.itemsLastUpdate.set(name, { ...qualities }); + } } async handleGetStorageRequest() { - return [...this.items.getEntries()]; + const result: Item[] = []; + for (const [name, qualities] of this.items.getEntries()) { + for (const [quality, count] of Object.entries(qualities)) { + result.push(new Item(name, count, quality)); + } + } + this.logger.verbose(`Sending storage update:\n${JSON.stringify(result)}`); + return result; } async handlePlaceEvent(request: PlaceEvent, src: lib.Address) { let instanceId = src.id; for (let item of request.items) { - this.items.addItem(item.name, item.count); - exportCounter.labels(String(instanceId), item.name).inc(item.count); + this.items.addItem(item.name, item.count, item.quality || "normal"); + exportCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count); } this.updateStorage(); @@ -164,55 +179,55 @@ export class ControllerPlugin extends BaseControllerPlugin { let itemsRemoved = []; if (method === "simple") { - // Give out as much items as possible until there are 0 left. This - // might lead to one host getting all the items and the rest nothing. for (let item of request.items) { - let count = this.items.getItemCount(item.name); + const quality = item.quality || "normal"; + let count = this.items.getItemCount(item.name, quality); let toRemove = Math.min(count, item.count); if (toRemove > 0) { - this.items.removeItem(item.name, toRemove); - itemsRemoved.push(new Item(item.name, toRemove)); + this.items.removeItem(item.name, toRemove, quality); + itemsRemoved.push(new Item(item.name, toRemove, quality)); } } - } else { let instance = this.controller.instances.get(instanceId); - let instanceName = instance ? instance.config.get("instance.name") : "unkonwn"; + let instanceName = instance ? instance.config.get("instance.name") : "unknown"; // use fancy neural net to calculate a "fair" dole division rate. if (method === "neural_dole") { for (let item of request.items) { + const quality = item.quality || "normal"; let count = this.neuralDole.divider( - { name: item.name, count: item.count, instanceId, instanceName } + { name: item.name, quality, count: item.count, instanceId, instanceName } ); if (count > 0) { - itemsRemoved.push(new Item(item.name, count)); + itemsRemoved.push(new Item(item.name, count, quality)); } } // Use dole division. Makes it really slow to drain out the last little bit. } else if (method === "dole") { for (let item of request.items) { + const quality = item.quality || "normal"; let count = dole.doleDivider({ - object: { name: item.name, count: item.count, instanceId, instanceName }, + object: { name: item.name, quality, count: item.count, instanceId, instanceName }, items: this.items, logItemTransfers: this.controller.config.get("subspace_storage.log_item_transfers"), logger: this.logger, }); if (count > 0) { - itemsRemoved.push(new Item(item.name, count)); + itemsRemoved.push(new Item(item.name, count, quality)); } } // Should not be possible } else { - throw Error(`Unkown division_method ${method}`); + throw Error(`Unknown division_method ${method}`); } } if (itemsRemoved.length) { for (let item of itemsRemoved) { - importCounter.labels(String(instanceId), item.name).inc(item.count); + importCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count); } this.updateStorage(); @@ -242,8 +257,10 @@ export class ControllerPlugin extends BaseControllerPlugin { async onMetrics() { if (this.items) { - for (let [key, count] of this.items.getEntries()) { - controllerInventoryGauge.labels(key).set(Number(count) || 0); + for (const [name, qualities] of this.items.getEntries()) { + for (const [quality, count] of Object.entries(qualities)) { + controllerInventoryGauge.labels(name, quality).set(Number(count) || 0); + } } } } diff --git a/plugins/subspace_storage/dole.ts b/plugins/subspace_storage/dole.ts index 85e86b59..4060f424 100644 --- a/plugins/subspace_storage/dole.ts +++ b/plugins/subspace_storage/dole.ts @@ -8,23 +8,23 @@ const prometheusNNDoleGauge = new Gauge( "clusterio_subspace_storage_nn_dole_gauge", "Current demand being supplied by Neural Network ; 1 means all demand covered, "+ "0.5 means about half of each supply is covered, 0 means no items are given", - { labels: ["resource"] } + { labels: ["resource", "quality"] } ); const prometheusDoleFactorGauge = new Gauge( "clusterio_subspace_storage_dole_factor_gauge", "The current dole division factor for this resource", - { labels: ["resource"] } + { labels: ["resource", "quality"] } ); export class NeuralDole { - getRequestStats(itemname: string, samples: number): number { + getRequestStats(itemname: string, quality: string, samples: number): number { let sum = 0; - samples = Math.min(samples, (this.stats[itemname] || []).length - 1); + samples = Math.min(samples, (this.stats[itemname+quality] || []).length - 1); if (samples < 1) { return 0.1; } for (let i = 1; i <= samples; i++) { - sum += this.stats[itemname][i].req; + sum += this.stats[itemname+quality][i].req; } if (sum === 0) { return 0.1; @@ -42,55 +42,67 @@ export class NeuralDole { constructor({ items }: { items: lib.ItemDatabase }) { // Set some storage variables for the dole divider this.items = items; - this.itemsLastTick = new Map(items.getEntries()); + this.itemsLastTick = new Map(); + for (let [name, qualities] of items.getEntries()) { + for (let [quality, count] of Object.entries(qualities)) { + this.itemsLastTick.set(name+quality, count); + } + } } doMagic() { - for (let [name, count] of this.items.getEntries()) { - let magicData = doleNN.tick( - count, - this.dole[name], - this.itemsLastTick.get(name) || 0, - this.getRequestStats(name, 5) - ); - this.stats[name] = this.stats[name] || []; - this.stats[name].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we currently collect - if (this.stats[name].length>10) { - this.stats[name].pop(); // remove if too many samples in stats - } + for (let [name, qualities] of this.items.getEntries()) { + for (let [quality, count] of Object.entries(qualities)) { + let magicData = doleNN.tick( + count, + this.dole[name+quality], + this.itemsLastTick.get(name+quality) || 0, + this.getRequestStats(name, quality, 5) + ); + this.stats[name+quality] = this.stats[name+quality] || []; + this.stats[name+quality].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we are collecting + if (this.stats[name+quality].length>10) { + this.stats[name+quality].pop(); // remove if too many samples in stats + } - this.dole[name] = magicData[0]; - // DONE handle magicData[1] for graphing for our users - prometheusNNDoleGauge.labels(name).set(magicData[1] || 0); + this.dole[name+quality] = magicData[0]; + // DONE handle magicData[1] for graphing for our users + prometheusNNDoleGauge.labels(name+quality).set(magicData[1] || 0); + } + } + this.itemsLastTick = new Map(); + for (let [name, qualities] of this.items.getEntries()) { + for (let [quality, count] of Object.entries(qualities)) { + this.itemsLastTick.set(name+quality, count); + } } - this.itemsLastTick = new Map(this.items.getEntries()); } - divider(object: { name:string, count:number, instanceId:number, instanceName:string }): number { + divider(object: { name:string, quality:string, count:number, instanceId:number, instanceName:string }): number { let magicData = doleNN.dose( object.count, // numReq - this.items.getItemCount(object.name) || 0, - this.itemsLastTick.get(object.name) || 0, - this.dole[object.name], - this.carry[`${object.name} ${object.instanceId}`] || 0, - this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] || 0, - this.getRequestStats(object.name, 5), - this.debt[`${object.name} ${object.instanceId}`] || 0 + this.items.getItemCount(object.name, object.quality) || 0, + this.itemsLastTick.get(object.name+object.quality) || 0, + this.dole[object.name+object.quality], + this.carry[`${object.name} ${object.quality} ${object.instanceId}`] || 0, + this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] || 0, + this.getRequestStats(object.name, object.quality, 5), + this.debt[`${object.name} ${object.quality} ${object.instanceId}`] || 0 ); - if ((this.stats[object.name] || []).length > 0) { - this.stats[object.name][0].req += Number(object.count); - this.stats[object.name][0].given += Number(magicData[0]); + if ((this.stats[object.name+object.quality] || []).length > 0) { + this.stats[object.name+object.quality][0].req += Number(object.count); + this.stats[object.name+object.quality][0].given += Number(magicData[0]); } - this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] = object.count; + this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] = object.count; // 0. Number of items to give in that dose // 1. New dole for item X // 2. New carry for item X instance Y - this.dole[object.name] = magicData[1]; - this.carry[`${object.name} ${object.instanceId}`] = magicData[2]; - this.debt[`${object.name} ${object.instanceId}`] = magicData[3]; + this.dole[object.name+object.quality] = magicData[1]; + this.carry[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[2]; + this.debt[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[3]; // Remove item from DB and send it - this.items.removeItem(object.name, magicData[0]); + this.items.removeItem(object.name, magicData[0], object.quality); return magicData[0]; } @@ -106,13 +118,13 @@ export function doleDivider( logItemTransfers, logger, } :{ - object: { name:string, count:number, instanceId:number, instanceName:string }, + object: { name:string, quality:string, count:number, instanceId:number, instanceName:string }, items: lib.ItemDatabase, logItemTransfers: boolean, logger: lib.Logger, }, ) { - let itemCount = items.getItemCount(object.name); + let itemCount = items.getItemCount(object.name, object.quality); // lower rates will equal more dramatic swings const doleDivisionRetardation = 10; // a higher cap will divide the store more ways, but will take longer to @@ -120,13 +132,14 @@ export function doleDivider( const maxDoleDivision = 250; const originalCount = Number(object.count) || 0; - object.count /= ((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation; + const doleFactor = _doleDivisionFactor[object.name+object.quality] || 0; + object.count /= (doleFactor + doleDivisionRetardation) / doleDivisionRetardation; object.count = Math.round(object.count); if (logItemTransfers) { logger.verbose( `Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} `+ - `with dole division factor ${(_doleDivisionFactor[object.name]||0)} `+ - `(real=${((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation}), `+ + `with dole division factor ${doleFactor} `+ + `(real=${doleFactor + doleDivisionRetardation}), `+ `item is ${itemCount > object.count?"stocked":"short"}.` ); } @@ -134,18 +147,22 @@ export function doleDivider( // Update existing items if item name already exists if (itemCount > object.count) { // If successful, increase dole - _doleDivisionFactor[object.name] = Math.max(_doleDivisionFactor[object.name] || 1, 1) - 1; - items.removeItem(object.name, object.count); + const key = object.name + object.quality; + _doleDivisionFactor[key] = Math.max(_doleDivisionFactor[key] || 1, 1) - 1; + items.removeItem(object.name, object.count, object.quality); - prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); + prometheusDoleFactorGauge + .labels(object.name, object.quality) + .set(_doleDivisionFactor[key] || 0); return object.count; } // if we didn't have enough, attempt giving out a smaller amount next time - _doleDivisionFactor[object.name] = Math.min( - maxDoleDivision, Math.max(_doleDivisionFactor[object.name] || 1, 1) * 2 - ); - prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); + const key = object.name + object.quality; + _doleDivisionFactor[key] = Math.min(maxDoleDivision, Math.max(_doleDivisionFactor[key] || 1, 1) * 2); + prometheusDoleFactorGauge + .labels(object.name, object.quality) + .set(_doleDivisionFactor[key] || 0); return 0; // console.log(`failure out of ${object.name}/${object.count} from ${object.instanceID} (${object.instanceName})`); } diff --git a/plugins/subspace_storage/instance.ts b/plugins/subspace_storage/instance.ts index cc9be259..576c201e 100644 --- a/plugins/subspace_storage/instance.ts +++ b/plugins/subspace_storage/instance.ts @@ -9,7 +9,7 @@ import { Item, } from "./messages"; -type IpcItems = [string, number][]; +type IpcItems = [string, { [quality: string]: number }][]; export class InstancePlugin extends BaseInstancePlugin { pendingTasks!: Set; @@ -54,7 +54,7 @@ export class InstancePlugin extends BaseInstancePlugin { let items = await this.instance.sendTo("controller", new GetStorageRequest()); // TODO Diff with dump of invdata produce minimal command to sync let itemsJson = lib.escapeString(JSON.stringify(items)); - await this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true); + await this.sendRcon(`/c __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true); } async onStop() { @@ -78,8 +78,15 @@ export class InstancePlugin extends BaseInstancePlugin { return; } - - const fromIpcItems = items.map(item => new Item(item[0], item[1])); + const fromIpcItems: Item[] = []; + items.forEach(item => { + // Loop over quality + for (const quality in item[1]) { + if (Object.prototype.hasOwnProperty.call(item[1], quality)) { + fromIpcItems.push(new Item(item[0], item[1][quality], quality)); + } + } + }); this.instance.sendTo("controller", new PlaceEvent(fromIpcItems)); if (this.instance.config.get("subspace_storage.log_item_transfers")) { @@ -90,8 +97,17 @@ export class InstancePlugin extends BaseInstancePlugin { // request items -------------------------------------------------------------- async requestItems(requestItems: IpcItems) { + this.logger.info(`Requesting items: ${JSON.stringify(requestItems)}`); // Request the items all at once - let fromIpcItems = requestItems.map(item => new Item(item[0], item[1])); + const fromIpcItems: Item[] = []; + requestItems.forEach(item => { + // Loop over quality + for (const quality in item[1]) { + if (Object.prototype.hasOwnProperty.call(item[1], quality)) { + fromIpcItems.push(new Item(item[0], item[1][quality], quality)); + } + } + }); let items = await this.instance.sendTo("controller", new RemoveRequest(fromIpcItems)); if (!items.length) { @@ -104,7 +120,7 @@ export class InstancePlugin extends BaseInstancePlugin { } let itemsJson = lib.escapeString(JSON.stringify(items)); - await this.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`, true); + await this.sendRcon(`/c __subspace_storage__ Import("${itemsJson}")`, true); } // combinator signals --------------------------------------------------------- @@ -115,10 +131,10 @@ export class InstancePlugin extends BaseInstancePlugin { let items = event.items; // XXX this should be moved to instance/clusterio api - items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000))); + items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000), "normal")); let itemsJson = lib.escapeString(JSON.stringify(items)); - let task = this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true); + let task = this.sendRcon(`/c __subspace_storage__ UpdateInvData("${itemsJson}")`, true); this.pendingTasks.add(task); await task.finally(() => { this.pendingTasks.delete(task); }); } diff --git a/plugins/subspace_storage/messages.ts b/plugins/subspace_storage/messages.ts index f1acca9c..22914492 100644 --- a/plugins/subspace_storage/messages.ts +++ b/plugins/subspace_storage/messages.ts @@ -1,25 +1,26 @@ import { Type, Static } from "@sinclair/typebox"; import * as lib from "@clusterio/lib"; -import { type } from "os"; export class Item { constructor( public name: string, - public count: number + public count: number, + public quality: string = "normal" ) { } static jsonSchema = Type.Tuple([ Type.String(), Type.Number(), + Type.String(), ]); toJSON() { - return [this.name, this.count]; + return [this.name, this.count, this.quality]; } static fromJSON(json: Static): Item { - return new this(json[0], json[1]); + return new this(json[0], json[1], json[2]); } }