Merge pull request #733 from clusterio/feature/subspace-storage-space-age

Feature/subspace storage space age

plugins/subspace_storage/controller.ts

@@ -22,17 +22,17 @@ import {
 const exportCounter = new Counter(
 	"clusterio_subspace_storage_export_total",
 	"Resources exported by instance",
-	{ labels: ["instance_id", "resource"] }
+	{ labels: ["instance_id", "resource", "quality"] }
 );
 const importCounter = new Counter(
 	"clusterio_subspace_storage_import_total",
 	"Resources imported by instance",
-	{ labels: ["instance_id", "resource"] }
+	{ labels: ["instance_id", "resource", "quality"] }
 );
 const controllerInventoryGauge = new Gauge(
 	"clusterio_subspace_storage_controller_inventory",
 	"Amount of resources stored on controller",
-	{ labels: ["resource"] }
+	{ labels: ["resource", "quality"] }
 );
 
 
@@ -73,7 +73,7 @@ async function saveDatabase(
 export class ControllerPlugin extends BaseControllerPlugin {
 	items!: lib.ItemDatabase;
 	itemUpdateRateLimiter!: lib.RateLimiter;
-	itemsLastUpdate!: Map<string, number>;
+	itemsLastUpdate: Map<string, lib.ItemCountWithQuality> = new Map();
 	subscribedControlLinks!: Set<ControlConnection>;
 	doleMagicId!: ReturnType<typeof setInterval>;
 	neuralDole!: dole.NeuralDole;
@@ -91,7 +91,10 @@ export class ControllerPlugin extends BaseControllerPlugin {
 				}
 			},
 		});
-		this.itemsLastUpdate = new Map(this.items.getEntries());
+		this.itemsLastUpdate = new Map();
+		for (const [name, qualities] of this.items.getEntries()) {
+			this.itemsLastUpdate.set(name, { ...qualities });
+		}
 
 		this.neuralDole = new dole.NeuralDole({ items: this.items });
 		this.doleMagicId = setInterval(() => {
@@ -116,37 +119,48 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	}
 
 	broadcastStorage() {
-		let itemsToUpdateMap:Map<string, number> = new Map();
-		for (let [name, count] of this.items.getEntries()) {
-			if (this.itemsLastUpdate.get(name) === count) {
-				continue;
+		let itemsToUpdate: Item[] = [];
+		for (const [name, qualities] of this.items.getEntries()) {
+			const lastQualities = this.itemsLastUpdate.get(name);
+			for (const [quality, count] of Object.entries(qualities)) {
+				if (!lastQualities || lastQualities[quality] !== count) {
+					itemsToUpdate.push(new Item(name, count, quality));
+				}
 			}
-			itemsToUpdateMap.set(name, count);
 		}
 
-		if (!itemsToUpdateMap.size) {
+		if (!itemsToUpdate.length) {
 			return;
 		}
 
-		let itemsToUpdate = [...itemsToUpdateMap.entries()];
-		let update = UpdateStorageEvent.fromJSON({ items: itemsToUpdate });
+		let update = new UpdateStorageEvent(itemsToUpdate);
 		this.controller.sendTo("allInstances", update);
 		for (let link of this.subscribedControlLinks) {
 			link.send(update);
 		}
-		this.itemsLastUpdate = new Map(this.items.getEntries());
+
+		this.itemsLastUpdate = new Map();
+		for (const [name, qualities] of this.items.getEntries()) {
+			this.itemsLastUpdate.set(name, { ...qualities });
+		}
 	}
 
 	async handleGetStorageRequest() {
-		return [...this.items.getEntries()];
+		const result: Item[] = [];
+		for (const [name, qualities] of this.items.getEntries()) {
+			for (const [quality, count] of Object.entries(qualities)) {
+				result.push(new Item(name, count, quality));
+			}
+		}
+		return result;
 	}
 
 	async handlePlaceEvent(request: PlaceEvent, src: lib.Address) {
 		let instanceId = src.id;
 
 		for (let item of request.items) {
-			this.items.addItem(item.name, item.count);
-			exportCounter.labels(String(instanceId), item.name).inc(item.count);
+			this.items.addItem(item.name, item.count, item.quality || "normal");
+			exportCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count);
 		}
 
 		this.updateStorage();
@@ -164,55 +178,55 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 		let itemsRemoved = [];
 		if (method === "simple") {
-			// Give out as much items as possible until there are 0 left.  This
-			// might lead to one host getting all the items and the rest nothing.
 			for (let item of request.items) {
-				let count = this.items.getItemCount(item.name);
+				const quality = item.quality || "normal";
+				let count = this.items.getItemCount(item.name, quality);
 				let toRemove = Math.min(count, item.count);
 				if (toRemove > 0) {
-					this.items.removeItem(item.name, toRemove);
-					itemsRemoved.push(new Item(item.name, toRemove));
+					this.items.removeItem(item.name, toRemove, quality);
+					itemsRemoved.push(new Item(item.name, toRemove, quality));
 				}
 			}
-
 		} else {
 			let instance = this.controller.instances.get(instanceId);
-			let instanceName = instance ? instance.config.get("instance.name") : "unkonwn";
+			let instanceName = instance ? instance.config.get("instance.name") : "unknown";
 
 			// use fancy neural net to calculate a "fair" dole division rate.
 			if (method === "neural_dole") {
 				for (let item of request.items) {
+					const quality = item.quality || "normal";
 					let count = this.neuralDole.divider(
-						{ name: item.name, count: item.count, instanceId, instanceName }
+						{ name: item.name, quality, count: item.count, instanceId, instanceName }
 					);
 					if (count > 0) {
-						itemsRemoved.push(new Item(item.name, count));
+						itemsRemoved.push(new Item(item.name, count, quality));
 					}
 				}
 
 			// Use dole division. Makes it really slow to drain out the last little bit.
 			} else if (method === "dole") {
 				for (let item of request.items) {
+					const quality = item.quality || "normal";
 					let count = dole.doleDivider({
-						object: { name: item.name, count: item.count, instanceId, instanceName },
+						object: { name: item.name, quality, count: item.count, instanceId, instanceName },
 						items: this.items,
 						logItemTransfers: this.controller.config.get("subspace_storage.log_item_transfers"),
 						logger: this.logger,
 					});
 					if (count > 0) {
-						itemsRemoved.push(new Item(item.name, count));
+						itemsRemoved.push(new Item(item.name, count, quality));
 					}
 				}
 
 			// Should not be possible
 			} else {
-				throw Error(`Unkown division_method ${method}`);
+				throw Error(`Unknown division_method ${method}`);
 			}
 		}
 
 		if (itemsRemoved.length) {
 			for (let item of itemsRemoved) {
-				importCounter.labels(String(instanceId), item.name).inc(item.count);
+				importCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count);
 			}
 
 			this.updateStorage();
@@ -242,8 +256,10 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	async onMetrics() {
 		if (this.items) {
-			for (let [key, count] of this.items.getEntries()) {
-				controllerInventoryGauge.labels(key).set(Number(count) || 0);
+			for (const [name, qualities] of this.items.getEntries()) {
+				for (const [quality, count] of Object.entries(qualities)) {
+					controllerInventoryGauge.labels(name, quality).set(Number(count) || 0);
+				}
 			}
 		}
 	}

plugins/subspace_storage/dole.ts

@@ -8,23 +8,23 @@ const prometheusNNDoleGauge = new Gauge(
 	"clusterio_subspace_storage_nn_dole_gauge",
 	"Current demand being supplied by Neural Network ; 1 means all demand covered, "+
 	"0.5 means about half of each supply is covered, 0 means no items are given",
-	{ labels: ["resource"] }
+	{ labels: ["resource", "quality"] }
 );
 const prometheusDoleFactorGauge = new Gauge(
 	"clusterio_subspace_storage_dole_factor_gauge",
 	"The current dole division factor for this resource",
-	{ labels: ["resource"] }
+	{ labels: ["resource", "quality"] }
 );
 
 export class NeuralDole {
-	getRequestStats(itemname: string, samples: number): number {
+	getRequestStats(itemname: string, quality: string, samples: number): number {
 		let sum = 0;
-		samples = Math.min(samples, (this.stats[itemname] || []).length - 1);
+		samples = Math.min(samples, (this.stats[itemname+quality] || []).length - 1);
 		if (samples < 1) {
 			return 0.1;
 		}
 		for (let i = 1; i <= samples; i++) {
-			sum += this.stats[itemname][i].req;
+			sum += this.stats[itemname+quality][i].req;
 		}
 		if (sum === 0) {
 			return 0.1;
@@ -42,55 +42,67 @@ export class NeuralDole {
 	constructor({ items }: { items: lib.ItemDatabase }) {
 		// Set some storage variables for the dole divider
 		this.items = items;
-		this.itemsLastTick = new Map(items.getEntries());
+		this.itemsLastTick = new Map();
+		for (let [name, qualities] of items.getEntries()) {
+			for (let [quality, count] of Object.entries(qualities)) {
+				this.itemsLastTick.set(name+quality, count);
+			}
+		}
 	}
 
 	doMagic() {
-		for (let [name, count] of this.items.getEntries()) {
-			let magicData = doleNN.tick(
-				count,
-				this.dole[name],
-				this.itemsLastTick.get(name) || 0,
-				this.getRequestStats(name, 5)
-			);
-			this.stats[name] = this.stats[name] || [];
-			this.stats[name].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we currently collect
-			if (this.stats[name].length>10) {
-				this.stats[name].pop(); // remove if too many samples in stats
-			}
+		for (let [name, qualities] of this.items.getEntries()) {
+			for (let [quality, count] of Object.entries(qualities)) {
+				let magicData = doleNN.tick(
+					count,
+					this.dole[name+quality],
+					this.itemsLastTick.get(name+quality) || 0,
+					this.getRequestStats(name, quality, 5)
+				);
+				this.stats[name+quality] = this.stats[name+quality] || [];
+				this.stats[name+quality].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we are collecting
+				if (this.stats[name+quality].length>10) {
+					this.stats[name+quality].pop(); // remove if too many samples in stats
+				}
 
-			this.dole[name] = magicData[0];
-			// DONE handle magicData[1] for graphing for our users
-			prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
+				this.dole[name+quality] = magicData[0];
+				// DONE handle magicData[1] for graphing for our users
+				prometheusNNDoleGauge.labels(name+quality).set(magicData[1] || 0);
+			}
+		}
+		this.itemsLastTick = new Map();
+		for (let [name, qualities] of this.items.getEntries()) {
+			for (let [quality, count] of Object.entries(qualities)) {
+				this.itemsLastTick.set(name+quality, count);
+			}
 		}
-		this.itemsLastTick = new Map(this.items.getEntries());
 	}
 
-	divider(object: { name:string, count:number, instanceId:number, instanceName:string }): number {
+	divider(object: { name:string, quality:string, count:number, instanceId:number, instanceName:string }): number {
 		let magicData = doleNN.dose(
 			object.count, // numReq
-			this.items.getItemCount(object.name) || 0,
-			this.itemsLastTick.get(object.name) || 0,
-			this.dole[object.name],
-			this.carry[`${object.name} ${object.instanceId}`] || 0,
-			this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] || 0,
-			this.getRequestStats(object.name, 5),
-			this.debt[`${object.name} ${object.instanceId}`] || 0
+			this.items.getItemCount(object.name, object.quality) || 0,
+			this.itemsLastTick.get(object.name+object.quality) || 0,
+			this.dole[object.name+object.quality],
+			this.carry[`${object.name} ${object.quality} ${object.instanceId}`] || 0,
+			this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] || 0,
+			this.getRequestStats(object.name, object.quality, 5),
+			this.debt[`${object.name} ${object.quality} ${object.instanceId}`] || 0
 		);
-		if ((this.stats[object.name] || []).length > 0) {
-			this.stats[object.name][0].req += Number(object.count);
-			this.stats[object.name][0].given += Number(magicData[0]);
+		if ((this.stats[object.name+object.quality] || []).length > 0) {
+			this.stats[object.name+object.quality][0].req += Number(object.count);
+			this.stats[object.name+object.quality][0].given += Number(magicData[0]);
 		}
-		this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] = object.count;
+		this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] = object.count;
 		// 0. Number of items to give in that dose
 		// 1. New dole for item X
 		// 2. New carry for item X instance Y
-		this.dole[object.name] = magicData[1];
-		this.carry[`${object.name} ${object.instanceId}`] = magicData[2];
-		this.debt[`${object.name} ${object.instanceId}`] = magicData[3];
+		this.dole[object.name+object.quality] = magicData[1];
+		this.carry[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[2];
+		this.debt[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[3];
 
 		// Remove item from DB and send it
-		this.items.removeItem(object.name, magicData[0]);
+		this.items.removeItem(object.name, magicData[0], object.quality);
 
 		return magicData[0];
 	}
@@ -106,13 +118,13 @@ export function doleDivider(
 		logItemTransfers,
 		logger,
 	} :{
-		object: { name:string, count:number, instanceId:number, instanceName:string },
+		object: { name:string, quality:string, count:number, instanceId:number, instanceName:string },
 		items: lib.ItemDatabase,
 		logItemTransfers: boolean,
 		logger: lib.Logger,
 	},
 ) {
-	let itemCount = items.getItemCount(object.name);
+	let itemCount = items.getItemCount(object.name, object.quality);
 	// lower rates will equal more dramatic swings
 	const doleDivisionRetardation = 10;
 	// a higher cap will divide the store more ways, but will take longer to
@@ -120,13 +132,14 @@ export function doleDivider(
 	const maxDoleDivision = 250;
 
 	const originalCount = Number(object.count) || 0;
-	object.count /= ((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation;
+	const doleFactor = _doleDivisionFactor[object.name+object.quality] || 0;
+	object.count /= (doleFactor + doleDivisionRetardation) / doleDivisionRetardation;
 	object.count = Math.round(object.count);
 	if (logItemTransfers) {
 		logger.verbose(
 			`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} `+
-			`with dole division factor ${(_doleDivisionFactor[object.name]||0)} `+
-			`(real=${((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation}), `+
+			`with dole division factor ${doleFactor} `+
+			`(real=${doleFactor + doleDivisionRetardation}), `+
 			`item is ${itemCount > object.count?"stocked":"short"}.`
 		);
 	}
@@ -134,18 +147,22 @@ export function doleDivider(
 	// Update existing items if item name already exists
 	if (itemCount > object.count) {
 		// If successful, increase dole
-		_doleDivisionFactor[object.name] = Math.max(_doleDivisionFactor[object.name] || 1, 1) - 1;
-		items.removeItem(object.name, object.count);
+		const key = object.name + object.quality;
+		_doleDivisionFactor[key] = Math.max(_doleDivisionFactor[key] || 1, 1) - 1;
+		items.removeItem(object.name, object.count, object.quality);
 
-		prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+		prometheusDoleFactorGauge
+			.labels(object.name, object.quality)
+			.set(_doleDivisionFactor[key] || 0);
 		return object.count;
 	}
 
 	// if we didn't have enough, attempt giving out a smaller amount next time
-	_doleDivisionFactor[object.name] = Math.min(
-		maxDoleDivision, Math.max(_doleDivisionFactor[object.name] || 1, 1) * 2
-	);
-	prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+	const key = object.name + object.quality;
+	_doleDivisionFactor[key] = Math.min(maxDoleDivision, Math.max(_doleDivisionFactor[key] || 1, 1) * 2);
+	prometheusDoleFactorGauge
+		.labels(object.name, object.quality)
+		.set(_doleDivisionFactor[key] || 0);
 	return 0;
 	// console.log(`failure out of ${object.name}/${object.count} from ${object.instanceID} (${object.instanceName})`);
 }

plugins/subspace_storage/instance.ts

@@ -9,7 +9,7 @@ import {
 	Item,
 } from "./messages";
 
-type IpcItems = [string, number][];
+type IpcItems = [name: string, count: number, quality: string][];
 
 export class InstancePlugin extends BaseInstancePlugin {
 	pendingTasks!: Set<any>;
@@ -26,6 +26,8 @@ export class InstancePlugin extends BaseInstancePlugin {
 
 		this.pendingTasks = new Set();
 		this.instance.server.on("ipc-subspace_storage:output", (output: IpcItems) => {
+			this.logger.info("Received output items:");
+			this.logger.info(JSON.stringify(output));
 			this.provideItems(output).catch(err => this.unexpectedError(err));
 		});
 		this.instance.server.on("ipc-subspace_storage:orders", (orders: IpcItems) => {
@@ -78,8 +80,7 @@ export class InstancePlugin extends BaseInstancePlugin {
 			return;
 		}
 
-
-		const fromIpcItems = items.map(item => new Item(item[0], item[1]));
+		const fromIpcItems = items.map(item => new Item(item[0], item[1], item[2]));
 		this.instance.sendTo("controller", new PlaceEvent(fromIpcItems));
 
 		if (this.instance.config.get("subspace_storage.log_item_transfers")) {
@@ -90,8 +91,9 @@ export class InstancePlugin extends BaseInstancePlugin {
 
 	// request items --------------------------------------------------------------
 	async requestItems(requestItems: IpcItems) {
+		this.logger.info(`Requesting items: ${JSON.stringify(requestItems)}`);
 		// Request the items all at once
-		let fromIpcItems = requestItems.map(item => new Item(item[0], item[1]));
+		const fromIpcItems = requestItems.map(item => new Item(item[0], item[1], item[2]));
 		let items = await this.instance.sendTo("controller", new RemoveRequest(fromIpcItems));
 
 		if (!items.length) {
@@ -115,7 +117,7 @@ export class InstancePlugin extends BaseInstancePlugin {
 		let items = event.items;
 
 		// XXX this should be moved to instance/clusterio api
-		items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000)));
+		items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000), "normal"));
 
 		let itemsJson = lib.escapeString(JSON.stringify(items));
 		let task = this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true);

plugins/subspace_storage/messages.ts

@@ -1,25 +1,26 @@
 import { Type, Static } from "@sinclair/typebox";
 import * as lib from "@clusterio/lib";
-import { type } from "os";
 
 export class Item {
 	constructor(
 		public name: string,
-		public count: number
+		public count: number,
+		public quality: string = "normal"
 	) {
 	}
 
 	static jsonSchema = Type.Tuple([
 		Type.String(),
 		Type.Number(),
+		Type.String(),
 	]);
 
 	toJSON() {
-		return [this.name, this.count];
+		return [this.name, this.count, this.quality];
 	}
 
 	static fromJSON(json: Static<typeof Item.jsonSchema>): Item {
-		return new this(json[0], json[1]);
+		return new this(json[0], json[1], json[2]);
 	}
 }
 

plugins/subspace_storage/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-subspace_storage",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"description": "Clusterio plugin for sharing storage between Factorio servers",
 	"author": "Hornwitser <github@hornwitser.no>",
 	"homepage": "https://github.com/clusterio/clusterio#readme",

plugins/subspace_storage/web/index.tsx

@@ -35,7 +35,7 @@ function StoragePage() {
 	let locale = useLocale();
 	let itemMetadata = useItemMetadata();
 	let storage = useStorage(control);
-	type ItemFilter = ([name, count]: [string, number]) => boolean;
+	type ItemFilter = ([name, item]: [string, Item]) => boolean;
 	let [filter, setFilter] = useState<null | ItemFilter>(null);
 
 	function getLocaleName(itemName: string) {
@@ -73,9 +73,9 @@ function StoragePage() {
 					search = search.replace(/(^| )(\w)/g, "$1\\b$2");
 					search = search.replace(/ +/g, ".*");
 					let filterExpr = new RegExp(search, "i");
-					setFilter(() => ((item: [string, number]) => {
-						let name = getLocaleName(item[0]);
-						return filterExpr.test(name) || filterExpr.test(item[0]);
+					setFilter(() => ((item: [string, Item]) => {
+						let name = getLocaleName(item[1].name);
+						return filterExpr.test(name) || filterExpr.test(item[1].name);
 					}));
 				}}
 			/>
@@ -86,29 +86,34 @@ function StoragePage() {
 					title: "Resource",
 					key: "resource",
 					sorter: (a, b) => {
-						let aName = getLocaleName(a[0]);
-						let bName = getLocaleName(b[0]);
+						let aName = getLocaleName(a[1].name);
+						let bName = getLocaleName(b[1].name);
 						if (aName < bName) { return -1; }
 						if (aName > bName) { return 1; }
 						return 0;
 					},
 					render: (_, item) => {
-						let localeName = getLocaleName(item[0]);
-						let hasMeta = itemMetadata.get(item[0]);
+						let localeName = getLocaleName(item[1].name);
+						let hasMeta = itemMetadata.get(item[1].name);
 
 						return <>
-							<span className={`factorio-icon item-${hasMeta ? item[0] : "unknown-item"}`}/>
+							<span className={`factorio-icon item-${hasMeta ? item[1].name : "unknown-item"}`}/>
 							{localeName}
 						</>;
 					},
 				},
+				{
+					title: "Quality",
+					key: "quality",
+					render: (_, item) => item[1].quality,
+				},
 				{
 					title: "Quantity",
 					key: "quantity",
 					align: "right",
 					defaultSortOrder: "descend",
-					sorter: (a, b) => a[1] - b[1],
-					render: (_, item) => numberFormat.format(item[1]),
+					sorter: (a, b) => a[1].count - b[1].count,
+					render: (_, item) => numberFormat.format(item[1].count),
 				},
 			]}
 			dataSource={filter ? storage.filter(filter) : storage}
@@ -119,7 +124,7 @@ function StoragePage() {
 }
 
 export class WebPlugin extends BaseWebPlugin {
-	storage = new Map<string, number>();
+	storage = new Map<string, Item>();
 	callbacks: (() => void)[] = [];
 
 	async init() {
@@ -165,7 +170,7 @@ export class WebPlugin extends BaseWebPlugin {
 
 	updateStorage(items: Item[]) {
 		for (let item of items) {
-			this.storage.set(item.name, item.count);
+			this.storage.set(`${item.name}:${item.quality}`, item);
 		}
 		for (let callback of this.callbacks) {
 			callback();