Merge pull request #733 from clusterio/feature/subspace-storage-space-age

Feature/subspace storage space age

.github/workflows/node.js.yml

@@ -55,3 +55,23 @@ jobs:
         node-version: 18.x
     - run: pnpm i --no-frozen-lockfile
     - run: pnpm run lint
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: pnpm/action-setup@v4
+      with:
+        version: 8
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18.x
+    - run: pnpm i --no-frozen-lockfile
+    - run: pnpm run build-mod
+    # Upload build to artifacts
+    - name: Upload build to artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: clusterio_lib
+        path: ./dist
+        if-no-files-found: error

package.json

@@ -5,6 +5,7 @@
 	"scripts": {
 		"build": "tsc --build",
 		"watch": "tsc --build --watch",
+		"build-mod": "node ./packages/lib/build_mod.js --source-dir ./packages/host/lua/clusterio_lib",
 		"test": "pnpm run build && mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
 		"silent-test": "SILENT_TEST=y pnpm test",
 		"fast-test": "FAST_TEST=y pnpm test",

packages/host/lua/clusterio_lib/info.json

@@ -2,19 +2,19 @@
 	"name": "clusterio_lib",
 	"variants": [
 		{
-			"version": "0.1.0",
+			"version": "2.0.17",
 			"factorio_version": "0.17"
 		},
 		{
-			"version": "0.1.1",
+			"version": "2.0.18",
 			"factorio_version": "0.18"
 		},
 		{
-			"version": "0.1.2",
+			"version": "2.0.11",
 			"factorio_version": "1.1"
 		},
 		{
-			"version": "0.1.3",
+			"version": "2.0.20",
 			"factorio_version": "2.0"
 		}
 	],

packages/host/src/Instance.ts

@@ -416,7 +416,11 @@ local players = {}
 for _, p in pairs(game.players) do
 	players[p.name] = p.online_time
 end
-rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
+if helpers ~= nil then
+	rcon.print(helpers.table_to_json(players))
+else
+	rcon.print(game.table_to_json(players))
+end`.replace(/\r?\n/g, " ");
 		let playerTimes: Record<string, number> = JSON.parse(await this.sendRcon(exportPlayerTimes));
 		let count = 0;
 

packages/lib/src/database.ts

@@ -9,6 +9,7 @@ import fs from "fs-extra";
 import * as libFileOps from "./file_ops";
 import { basicType } from "./helpers";
 
+export type ItemCountWithQuality = { [quality: string]: number };
 
 /**
  * Converts a Map with only string keys to a JavaScript object.
@@ -154,6 +155,11 @@ function checkCount(count: unknown): asserts count is number {
 	}
 }
 
+function checkQuality(quality: unknown): asserts quality is string {
+	if (typeof quality !== "string") {
+		throw new Error("quality must be a string");
+	}
+}
 
 /**
  * Item database
@@ -164,7 +170,7 @@ function checkCount(count: unknown): asserts count is number {
  * deserialized the content is verified.
  */
 export class ItemDatabase {
-	private _items: Map<string, number> = new Map();
+	private _items: Map<string, ItemCountWithQuality> = new Map();
 
 	/**
 	 * Create a new item database
@@ -177,11 +183,17 @@ export class ItemDatabase {
 	constructor(serialized?: object) {
 		// Verify the content of the serialized database
 		if (serialized !== undefined) {
-			for (let [name, count] of Object.entries(serialized)) {
+			for (let [name, qualities] of Object.entries(serialized)) {
 				checkName(name);
-				checkCount(count);
+				// Migration from pre-quality database format
+				if (typeof qualities === "number") {
+					qualities = { normal: qualities };
+				}
+				for (const count of Object.values(qualities)) {
+					checkCount(count);
+				}
 
-				this._items.set(name, count);
+				this._items.set(name, qualities);
 			}
 		}
 	}
@@ -195,10 +207,15 @@ export class ItemDatabase {
 	 * @returns Serialized representation of the database
 	 */
 	serialize() {
-		let obj: Record<string, number> = {};
-		for (let [name, count] of this._items) {
-			if (count !== 0) {
-				obj[name] = count;
+		let obj: Record<string, ItemCountWithQuality> = {};
+		for (let [name, qualities] of this._items) {
+			for (const quality of Object.getOwnPropertyNames(qualities)) {
+				if (qualities[quality] !== 0) {
+					if (!obj[name]) {
+						obj[name] = {};
+					}
+					obj[name][quality] = qualities[quality];
+				}
 			}
 		}
 		return obj;
@@ -222,16 +239,18 @@ export class ItemDatabase {
 	 * then 0 is returned.
 	 *
 	 * @param name - The name of the item to get the count of.
+	 * @param quality - The quality of the item to get the count of.
 	 * @returns The count of the item stored.
 	 */
-	getItemCount(name: string): number {
+	getItemCount(name: string, quality: string): number {
 		checkName(name);
+		checkQuality(quality);
 
-		if (!this._items.has(name)) {
+		if (!this._items.get(name)?.[quality]) {
 			return 0;
 		}
 
-		return this._items.get(name)!;
+		return this._items.get(name)![quality];
 	}
 
 	/**
@@ -241,16 +260,21 @@ export class ItemDatabase {
 	 *
 	 * @param name - The name of the item to add.
 	 * @param count - The count of item to remove.
+	 * @param quality - The quality of the item to add.
 	 */
-	addItem(name: string, count: number) {
+	addItem(name: string, count: number, quality: string) {
 		checkName(name);
 		checkCount(count);
+		checkQuality(quality);
 
-		if (this._items.has(name)) {
-			count += this._items.get(name)!;
+		if (!this._items.has(name)) {
+			this._items.set(name, { [quality]: count });
+			return;
 		}
 
-		this._items.set(name, count);
+		// Nullish coalescing operator is used to avoid NaN when the quality is not present
+		let currentCount = this._items.get(name)![quality] ?? 0;
+		this._items.get(name)![quality] = currentCount + count;
 	};
 
 	/**
@@ -262,24 +286,17 @@ export class ItemDatabase {
 	 *
 	 * @param name - The name of the item to remove.
 	 * @param count - The count of items to remove.
+	 * @param quality - The quality of the item to remove.
 	 */
-	removeItem(name: string, count: number) {
-		checkName(name);
-		checkCount(count);
-
-		count = -count;
-		if (this._items.has(name)) {
-			count += this._items.get(name)!;
-		}
-
-		this._items.set(name, count);
+	removeItem(name: string, count: number, quality: string) {
+		this.addItem(name, -count, quality);
 	};
 
 	/**
 	 * Allow to iterate through the items.
 	 * @returns array of items.
 	 */
-	getEntries(): IterableIterator<[string, number]> {
+	getEntries(): IterableIterator<[string, ItemCountWithQuality]> {
 		return this._items.entries();
 	}
 }

plugins/subspace_storage/controller.ts

@@ -22,17 +22,17 @@ import {
 const exportCounter = new Counter(
 	"clusterio_subspace_storage_export_total",
 	"Resources exported by instance",
-	{ labels: ["instance_id", "resource"] }
+	{ labels: ["instance_id", "resource", "quality"] }
 );
 const importCounter = new Counter(
 	"clusterio_subspace_storage_import_total",
 	"Resources imported by instance",
-	{ labels: ["instance_id", "resource"] }
+	{ labels: ["instance_id", "resource", "quality"] }
 );
 const controllerInventoryGauge = new Gauge(
 	"clusterio_subspace_storage_controller_inventory",
 	"Amount of resources stored on controller",
-	{ labels: ["resource"] }
+	{ labels: ["resource", "quality"] }
 );
 
 
@@ -73,7 +73,7 @@ async function saveDatabase(
 export class ControllerPlugin extends BaseControllerPlugin {
 	items!: lib.ItemDatabase;
 	itemUpdateRateLimiter!: lib.RateLimiter;
-	itemsLastUpdate!: Map<string, number>;
+	itemsLastUpdate: Map<string, lib.ItemCountWithQuality> = new Map();
 	subscribedControlLinks!: Set<ControlConnection>;
 	doleMagicId!: ReturnType<typeof setInterval>;
 	neuralDole!: dole.NeuralDole;
@@ -91,7 +91,10 @@ export class ControllerPlugin extends BaseControllerPlugin {
 				}
 			},
 		});
-		this.itemsLastUpdate = new Map(this.items.getEntries());
+		this.itemsLastUpdate = new Map();
+		for (const [name, qualities] of this.items.getEntries()) {
+			this.itemsLastUpdate.set(name, { ...qualities });
+		}
 
 		this.neuralDole = new dole.NeuralDole({ items: this.items });
 		this.doleMagicId = setInterval(() => {
@@ -116,37 +119,48 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	}
 
 	broadcastStorage() {
-		let itemsToUpdateMap:Map<string, number> = new Map();
-		for (let [name, count] of this.items.getEntries()) {
-			if (this.itemsLastUpdate.get(name) === count) {
-				continue;
+		let itemsToUpdate: Item[] = [];
+		for (const [name, qualities] of this.items.getEntries()) {
+			const lastQualities = this.itemsLastUpdate.get(name);
+			for (const [quality, count] of Object.entries(qualities)) {
+				if (!lastQualities || lastQualities[quality] !== count) {
+					itemsToUpdate.push(new Item(name, count, quality));
+				}
 			}
-			itemsToUpdateMap.set(name, count);
 		}
 
-		if (!itemsToUpdateMap.size) {
+		if (!itemsToUpdate.length) {
 			return;
 		}
 
-		let itemsToUpdate = [...itemsToUpdateMap.entries()];
-		let update = UpdateStorageEvent.fromJSON({ items: itemsToUpdate });
+		let update = new UpdateStorageEvent(itemsToUpdate);
 		this.controller.sendTo("allInstances", update);
 		for (let link of this.subscribedControlLinks) {
 			link.send(update);
 		}
-		this.itemsLastUpdate = new Map(this.items.getEntries());
+
+		this.itemsLastUpdate = new Map();
+		for (const [name, qualities] of this.items.getEntries()) {
+			this.itemsLastUpdate.set(name, { ...qualities });
+		}
 	}
 
 	async handleGetStorageRequest() {
-		return [...this.items.getEntries()];
+		const result: Item[] = [];
+		for (const [name, qualities] of this.items.getEntries()) {
+			for (const [quality, count] of Object.entries(qualities)) {
+				result.push(new Item(name, count, quality));
+			}
+		}
+		return result;
 	}
 
 	async handlePlaceEvent(request: PlaceEvent, src: lib.Address) {
 		let instanceId = src.id;
 
 		for (let item of request.items) {
-			this.items.addItem(item.name, item.count);
-			exportCounter.labels(String(instanceId), item.name).inc(item.count);
+			this.items.addItem(item.name, item.count, item.quality || "normal");
+			exportCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count);
 		}
 
 		this.updateStorage();
@@ -164,55 +178,55 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 		let itemsRemoved = [];
 		if (method === "simple") {
-			// Give out as much items as possible until there are 0 left.  This
-			// might lead to one host getting all the items and the rest nothing.
 			for (let item of request.items) {
-				let count = this.items.getItemCount(item.name);
+				const quality = item.quality || "normal";
+				let count = this.items.getItemCount(item.name, quality);
 				let toRemove = Math.min(count, item.count);
 				if (toRemove > 0) {
-					this.items.removeItem(item.name, toRemove);
-					itemsRemoved.push(new Item(item.name, toRemove));
+					this.items.removeItem(item.name, toRemove, quality);
+					itemsRemoved.push(new Item(item.name, toRemove, quality));
 				}
 			}
-
 		} else {
 			let instance = this.controller.instances.get(instanceId);
-			let instanceName = instance ? instance.config.get("instance.name") : "unkonwn";
+			let instanceName = instance ? instance.config.get("instance.name") : "unknown";
 
 			// use fancy neural net to calculate a "fair" dole division rate.
 			if (method === "neural_dole") {
 				for (let item of request.items) {
+					const quality = item.quality || "normal";
 					let count = this.neuralDole.divider(
-						{ name: item.name, count: item.count, instanceId, instanceName }
+						{ name: item.name, quality, count: item.count, instanceId, instanceName }
 					);
 					if (count > 0) {
-						itemsRemoved.push(new Item(item.name, count));
+						itemsRemoved.push(new Item(item.name, count, quality));
 					}
 				}
 
 			// Use dole division. Makes it really slow to drain out the last little bit.
 			} else if (method === "dole") {
 				for (let item of request.items) {
+					const quality = item.quality || "normal";
 					let count = dole.doleDivider({
-						object: { name: item.name, count: item.count, instanceId, instanceName },
+						object: { name: item.name, quality, count: item.count, instanceId, instanceName },
 						items: this.items,
 						logItemTransfers: this.controller.config.get("subspace_storage.log_item_transfers"),
 						logger: this.logger,
 					});
 					if (count > 0) {
-						itemsRemoved.push(new Item(item.name, count));
+						itemsRemoved.push(new Item(item.name, count, quality));
 					}
 				}
 
 			// Should not be possible
 			} else {
-				throw Error(`Unkown division_method ${method}`);
+				throw Error(`Unknown division_method ${method}`);
 			}
 		}
 
 		if (itemsRemoved.length) {
 			for (let item of itemsRemoved) {
-				importCounter.labels(String(instanceId), item.name).inc(item.count);
+				importCounter.labels(String(instanceId), item.name, item.quality || "normal").inc(item.count);
 			}
 
 			this.updateStorage();
@@ -242,8 +256,10 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	async onMetrics() {
 		if (this.items) {
-			for (let [key, count] of this.items.getEntries()) {
-				controllerInventoryGauge.labels(key).set(Number(count) || 0);
+			for (const [name, qualities] of this.items.getEntries()) {
+				for (const [quality, count] of Object.entries(qualities)) {
+					controllerInventoryGauge.labels(name, quality).set(Number(count) || 0);
+				}
 			}
 		}
 	}

plugins/subspace_storage/dole.ts

@@ -8,23 +8,23 @@ const prometheusNNDoleGauge = new Gauge(
 	"clusterio_subspace_storage_nn_dole_gauge",
 	"Current demand being supplied by Neural Network ; 1 means all demand covered, "+
 	"0.5 means about half of each supply is covered, 0 means no items are given",
-	{ labels: ["resource"] }
+	{ labels: ["resource", "quality"] }
 );
 const prometheusDoleFactorGauge = new Gauge(
 	"clusterio_subspace_storage_dole_factor_gauge",
 	"The current dole division factor for this resource",
-	{ labels: ["resource"] }
+	{ labels: ["resource", "quality"] }
 );
 
 export class NeuralDole {
-	getRequestStats(itemname: string, samples: number): number {
+	getRequestStats(itemname: string, quality: string, samples: number): number {
 		let sum = 0;
-		samples = Math.min(samples, (this.stats[itemname] || []).length - 1);
+		samples = Math.min(samples, (this.stats[itemname+quality] || []).length - 1);
 		if (samples < 1) {
 			return 0.1;
 		}
 		for (let i = 1; i <= samples; i++) {
-			sum += this.stats[itemname][i].req;
+			sum += this.stats[itemname+quality][i].req;
 		}
 		if (sum === 0) {
 			return 0.1;
@@ -42,55 +42,67 @@ export class NeuralDole {
 	constructor({ items }: { items: lib.ItemDatabase }) {
 		// Set some storage variables for the dole divider
 		this.items = items;
-		this.itemsLastTick = new Map(items.getEntries());
+		this.itemsLastTick = new Map();
+		for (let [name, qualities] of items.getEntries()) {
+			for (let [quality, count] of Object.entries(qualities)) {
+				this.itemsLastTick.set(name+quality, count);
+			}
+		}
 	}
 
 	doMagic() {
-		for (let [name, count] of this.items.getEntries()) {
-			let magicData = doleNN.tick(
-				count,
-				this.dole[name],
-				this.itemsLastTick.get(name) || 0,
-				this.getRequestStats(name, 5)
-			);
-			this.stats[name] = this.stats[name] || [];
-			this.stats[name].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we currently collect
-			if (this.stats[name].length>10) {
-				this.stats[name].pop(); // remove if too many samples in stats
-			}
+		for (let [name, qualities] of this.items.getEntries()) {
+			for (let [quality, count] of Object.entries(qualities)) {
+				let magicData = doleNN.tick(
+					count,
+					this.dole[name+quality],
+					this.itemsLastTick.get(name+quality) || 0,
+					this.getRequestStats(name, quality, 5)
+				);
+				this.stats[name+quality] = this.stats[name+quality] || [];
+				this.stats[name+quality].unshift({ req: 0, given: 0 }); // stats[name][0] is the one we are collecting
+				if (this.stats[name+quality].length>10) {
+					this.stats[name+quality].pop(); // remove if too many samples in stats
+				}
 
-			this.dole[name] = magicData[0];
-			// DONE handle magicData[1] for graphing for our users
-			prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
+				this.dole[name+quality] = magicData[0];
+				// DONE handle magicData[1] for graphing for our users
+				prometheusNNDoleGauge.labels(name+quality).set(magicData[1] || 0);
+			}
+		}
+		this.itemsLastTick = new Map();
+		for (let [name, qualities] of this.items.getEntries()) {
+			for (let [quality, count] of Object.entries(qualities)) {
+				this.itemsLastTick.set(name+quality, count);
+			}
 		}
-		this.itemsLastTick = new Map(this.items.getEntries());
 	}
 
-	divider(object: { name:string, count:number, instanceId:number, instanceName:string }): number {
+	divider(object: { name:string, quality:string, count:number, instanceId:number, instanceName:string }): number {
 		let magicData = doleNN.dose(
 			object.count, // numReq
-			this.items.getItemCount(object.name) || 0,
-			this.itemsLastTick.get(object.name) || 0,
-			this.dole[object.name],
-			this.carry[`${object.name} ${object.instanceId}`] || 0,
-			this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] || 0,
-			this.getRequestStats(object.name, 5),
-			this.debt[`${object.name} ${object.instanceId}`] || 0
+			this.items.getItemCount(object.name, object.quality) || 0,
+			this.itemsLastTick.get(object.name+object.quality) || 0,
+			this.dole[object.name+object.quality],
+			this.carry[`${object.name} ${object.quality} ${object.instanceId}`] || 0,
+			this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] || 0,
+			this.getRequestStats(object.name, object.quality, 5),
+			this.debt[`${object.name} ${object.quality} ${object.instanceId}`] || 0
 		);
-		if ((this.stats[object.name] || []).length > 0) {
-			this.stats[object.name][0].req += Number(object.count);
-			this.stats[object.name][0].given += Number(magicData[0]);
+		if ((this.stats[object.name+object.quality] || []).length > 0) {
+			this.stats[object.name+object.quality][0].req += Number(object.count);
+			this.stats[object.name+object.quality][0].given += Number(magicData[0]);
 		}
-		this.lastRequest[`${object.name}_${object.instanceId}_${object.instanceName}`] = object.count;
+		this.lastRequest[`${object.name}_${object.quality}_${object.instanceId}_${object.instanceName}`] = object.count;
 		// 0. Number of items to give in that dose
 		// 1. New dole for item X
 		// 2. New carry for item X instance Y
-		this.dole[object.name] = magicData[1];
-		this.carry[`${object.name} ${object.instanceId}`] = magicData[2];
-		this.debt[`${object.name} ${object.instanceId}`] = magicData[3];
+		this.dole[object.name+object.quality] = magicData[1];
+		this.carry[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[2];
+		this.debt[`${object.name} ${object.quality} ${object.instanceId}`] = magicData[3];
 
 		// Remove item from DB and send it
-		this.items.removeItem(object.name, magicData[0]);
+		this.items.removeItem(object.name, magicData[0], object.quality);
 
 		return magicData[0];
 	}
@@ -106,13 +118,13 @@ export function doleDivider(
 		logItemTransfers,
 		logger,
 	} :{
-		object: { name:string, count:number, instanceId:number, instanceName:string },
+		object: { name:string, quality:string, count:number, instanceId:number, instanceName:string },
 		items: lib.ItemDatabase,
 		logItemTransfers: boolean,
 		logger: lib.Logger,
 	},
 ) {
-	let itemCount = items.getItemCount(object.name);
+	let itemCount = items.getItemCount(object.name, object.quality);
 	// lower rates will equal more dramatic swings
 	const doleDivisionRetardation = 10;
 	// a higher cap will divide the store more ways, but will take longer to
@@ -120,13 +132,14 @@ export function doleDivider(
 	const maxDoleDivision = 250;
 
 	const originalCount = Number(object.count) || 0;
-	object.count /= ((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation;
+	const doleFactor = _doleDivisionFactor[object.name+object.quality] || 0;
+	object.count /= (doleFactor + doleDivisionRetardation) / doleDivisionRetardation;
 	object.count = Math.round(object.count);
 	if (logItemTransfers) {
 		logger.verbose(
 			`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} `+
-			`with dole division factor ${(_doleDivisionFactor[object.name]||0)} `+
-			`(real=${((_doleDivisionFactor[object.name] || 0) + doleDivisionRetardation) / doleDivisionRetardation}), `+
+			`with dole division factor ${doleFactor} `+
+			`(real=${doleFactor + doleDivisionRetardation}), `+
 			`item is ${itemCount > object.count?"stocked":"short"}.`
 		);
 	}
@@ -134,18 +147,22 @@ export function doleDivider(
 	// Update existing items if item name already exists
 	if (itemCount > object.count) {
 		// If successful, increase dole
-		_doleDivisionFactor[object.name] = Math.max(_doleDivisionFactor[object.name] || 1, 1) - 1;
-		items.removeItem(object.name, object.count);
+		const key = object.name + object.quality;
+		_doleDivisionFactor[key] = Math.max(_doleDivisionFactor[key] || 1, 1) - 1;
+		items.removeItem(object.name, object.count, object.quality);
 
-		prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+		prometheusDoleFactorGauge
+			.labels(object.name, object.quality)
+			.set(_doleDivisionFactor[key] || 0);
 		return object.count;
 	}
 
 	// if we didn't have enough, attempt giving out a smaller amount next time
-	_doleDivisionFactor[object.name] = Math.min(
-		maxDoleDivision, Math.max(_doleDivisionFactor[object.name] || 1, 1) * 2
-	);
-	prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+	const key = object.name + object.quality;
+	_doleDivisionFactor[key] = Math.min(maxDoleDivision, Math.max(_doleDivisionFactor[key] || 1, 1) * 2);
+	prometheusDoleFactorGauge
+		.labels(object.name, object.quality)
+		.set(_doleDivisionFactor[key] || 0);
 	return 0;
 	// console.log(`failure out of ${object.name}/${object.count} from ${object.instanceID} (${object.instanceName})`);
 }

plugins/subspace_storage/instance.ts

@@ -9,7 +9,7 @@ import {
 	Item,
 } from "./messages";
 
-type IpcItems = [string, number][];
+type IpcItems = [name: string, count: number, quality: string][];
 
 export class InstancePlugin extends BaseInstancePlugin {
 	pendingTasks!: Set<any>;
@@ -26,6 +26,8 @@ export class InstancePlugin extends BaseInstancePlugin {
 
 		this.pendingTasks = new Set();
 		this.instance.server.on("ipc-subspace_storage:output", (output: IpcItems) => {
+			this.logger.info("Received output items:");
+			this.logger.info(JSON.stringify(output));
 			this.provideItems(output).catch(err => this.unexpectedError(err));
 		});
 		this.instance.server.on("ipc-subspace_storage:orders", (orders: IpcItems) => {
@@ -78,8 +80,7 @@ export class InstancePlugin extends BaseInstancePlugin {
 			return;
 		}
 
-
-		const fromIpcItems = items.map(item => new Item(item[0], item[1]));
+		const fromIpcItems = items.map(item => new Item(item[0], item[1], item[2]));
 		this.instance.sendTo("controller", new PlaceEvent(fromIpcItems));
 
 		if (this.instance.config.get("subspace_storage.log_item_transfers")) {
@@ -90,8 +91,9 @@ export class InstancePlugin extends BaseInstancePlugin {
 
 	// request items --------------------------------------------------------------
 	async requestItems(requestItems: IpcItems) {
+		this.logger.info(`Requesting items: ${JSON.stringify(requestItems)}`);
 		// Request the items all at once
-		let fromIpcItems = requestItems.map(item => new Item(item[0], item[1]));
+		const fromIpcItems = requestItems.map(item => new Item(item[0], item[1], item[2]));
 		let items = await this.instance.sendTo("controller", new RemoveRequest(fromIpcItems));
 
 		if (!items.length) {
@@ -115,7 +117,7 @@ export class InstancePlugin extends BaseInstancePlugin {
 		let items = event.items;
 
 		// XXX this should be moved to instance/clusterio api
-		items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000)));
+		items.push(new Item("signal-unixtime", Math.floor(Date.now()/1000), "normal"));
 
 		let itemsJson = lib.escapeString(JSON.stringify(items));
 		let task = this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true);

plugins/subspace_storage/messages.ts

@@ -1,25 +1,26 @@
 import { Type, Static } from "@sinclair/typebox";
 import * as lib from "@clusterio/lib";
-import { type } from "os";
 
 export class Item {
 	constructor(
 		public name: string,
-		public count: number
+		public count: number,
+		public quality: string = "normal"
 	) {
 	}
 
 	static jsonSchema = Type.Tuple([
 		Type.String(),
 		Type.Number(),
+		Type.String(),
 	]);
 
 	toJSON() {
-		return [this.name, this.count];
+		return [this.name, this.count, this.quality];
 	}
 
 	static fromJSON(json: Static<typeof Item.jsonSchema>): Item {
-		return new this(json[0], json[1]);
+		return new this(json[0], json[1], json[2]);
 	}
 }
 

plugins/subspace_storage/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@clusterio/plugin-subspace_storage",
-	"version": "2.0.0-alpha.18",
+	"version": "2.0.0-alpha.19",
 	"description": "Clusterio plugin for sharing storage between Factorio servers",
 	"author": "Hornwitser <github@hornwitser.no>",
 	"homepage": "https://github.com/clusterio/clusterio#readme",

plugins/subspace_storage/web/index.tsx

@@ -35,7 +35,7 @@ function StoragePage() {
 	let locale = useLocale();
 	let itemMetadata = useItemMetadata();
 	let storage = useStorage(control);
-	type ItemFilter = ([name, count]: [string, number]) => boolean;
+	type ItemFilter = ([name, item]: [string, Item]) => boolean;
 	let [filter, setFilter] = useState<null | ItemFilter>(null);
 
 	function getLocaleName(itemName: string) {
@@ -73,9 +73,9 @@ function StoragePage() {
 					search = search.replace(/(^| )(\w)/g, "$1\\b$2");
 					search = search.replace(/ +/g, ".*");
 					let filterExpr = new RegExp(search, "i");
-					setFilter(() => ((item: [string, number]) => {
-						let name = getLocaleName(item[0]);
-						return filterExpr.test(name) || filterExpr.test(item[0]);
+					setFilter(() => ((item: [string, Item]) => {
+						let name = getLocaleName(item[1].name);
+						return filterExpr.test(name) || filterExpr.test(item[1].name);
 					}));
 				}}
 			/>
@@ -86,29 +86,34 @@ function StoragePage() {
 					title: "Resource",
 					key: "resource",
 					sorter: (a, b) => {
-						let aName = getLocaleName(a[0]);
-						let bName = getLocaleName(b[0]);
+						let aName = getLocaleName(a[1].name);
+						let bName = getLocaleName(b[1].name);
 						if (aName < bName) { return -1; }
 						if (aName > bName) { return 1; }
 						return 0;
 					},
 					render: (_, item) => {
-						let localeName = getLocaleName(item[0]);
-						let hasMeta = itemMetadata.get(item[0]);
+						let localeName = getLocaleName(item[1].name);
+						let hasMeta = itemMetadata.get(item[1].name);
 
 						return <>
-							<span className={`factorio-icon item-${hasMeta ? item[0] : "unknown-item"}`}/>
+							<span className={`factorio-icon item-${hasMeta ? item[1].name : "unknown-item"}`}/>
 							{localeName}
 						</>;
 					},
 				},
+				{
+					title: "Quality",
+					key: "quality",
+					render: (_, item) => item[1].quality,
+				},
 				{
 					title: "Quantity",
 					key: "quantity",
 					align: "right",
 					defaultSortOrder: "descend",
-					sorter: (a, b) => a[1] - b[1],
-					render: (_, item) => numberFormat.format(item[1]),
+					sorter: (a, b) => a[1].count - b[1].count,
+					render: (_, item) => numberFormat.format(item[1].count),
 				},
 			]}
 			dataSource={filter ? storage.filter(filter) : storage}
@@ -119,7 +124,7 @@ function StoragePage() {
 }
 
 export class WebPlugin extends BaseWebPlugin {
-	storage = new Map<string, number>();
+	storage = new Map<string, Item>();
 	callbacks: (() => void)[] = [];
 
 	async init() {
@@ -165,7 +170,7 @@ export class WebPlugin extends BaseWebPlugin {
 
 	updateStorage(items: Item[]) {
 		for (let item of items) {
-			this.storage.set(item.name, item.count);
+			this.storage.set(`${item.name}:${item.quality}`, item);
 		}
 		for (let callback of this.callbacks) {
 			callback();

test/integration/clusterio.js

@@ -199,12 +199,16 @@ describe("Integration of Clusterio", function() {
 		it("should download mods from controller", async function() {
 			slowTest(this);
 			await runWithAltHost(async () => {
-				const clusterioLib = path.join("temp", "test", "alt-mods", "clusterio_lib_0.1.2.zip");
-				assert(!await fs.pathExists(clusterioLib), "mod was downloaded before the test");
+				// Use the latest vesion in /dist instead of a hardcoded version
+				async function checkModDownloaded() {
+					let files = await fs.readdir(path.join("temp", "test", "alt-mods"));
+					return files.find(name => name.startsWith("clusterio_lib_"));
+				}
+				assert(!await checkModDownloaded(), "mod was downloaded before the test");
 				await execCtl("instance create alt-mod --id 98");
 				await execCtl("instance assign alt-mod 5");
 				await execCtl("instance save create alt-mod");
-				assert(await fs.pathExists(clusterioLib), "mod was not downloaded by the test");
+				assert(await checkModDownloaded(), "mod was not downloaded by the test");
 			});
 		});
 		it("should auto start instances with auto_start enabled", async function() {

test/integration/index.js

@@ -18,9 +18,6 @@ require("../../plugins/research_sync/dist/node/index");
 require("../../plugins/statistics_exporter/dist/node/index");
 require("../../plugins/subspace_storage/dist/node/index");
 
-const subspaceStorageVersion = "1.99.8";
-const subspaceStorageGitHubTag = "1.99.6"; // Tag subspaceStorageVersion was released under on GitHub
-
 class TestControl extends lib.Link {
 	constructor(connector, subscribe = true) {
 		super(connector);
@@ -266,24 +263,6 @@ before(async function() {
 		sourceDir: "packages/host/lua/clusterio_lib",
 		outputDir: modsDir,
 	});
-	const ssv = subspaceStorageVersion;
-	const ssTag = subspaceStorageGitHubTag;
-	const ssZip = `subspace_storage_${ssv}.zip`;
-	if (!await fs.pathExists(`mods/${ssZip}`)) {
-		console.log(`Downloading ${ssZip}`);
-		const ssUrl = `https://github.com/clusterio/subspace_storage/releases/download/${ssTag}/${ssZip}`;
-		let response = await phin({
-			url: ssUrl,
-			method: "GET",
-			stream: false, // phin leaks open sockets when stream and followRedirects is combined
-			followRedirects: true,
-		});
-		if (response.statusCode !== 200) {
-			throw new Error(`${ssUrl} returned ${response.statusCode} ${response.statusMessage}`);
-		}
-		await fs.outputFile(`mods/${ssZip}`, response.body);
-	}
-	await fs.copyFile(`mods/${ssZip}`, `${modsDir}/${ssZip}`);
 
 	console.log("Setting Controller Config");
 	await exec("node ../../packages/controller config set controller.auth_secret TestSecretDoNotUse");
@@ -325,9 +304,8 @@ before(async function() {
 	const testPack = lib.ModPack.fromJSON({});
 	testPack.id = 12;
 	testPack.name = "subspace_storage-pack";
-	testPack.factorioVersion = "1.1.0";
-	testPack.mods.set("clusterio_lib", { name: "clusterio_lib", enabled: true, version: "0.1.2" });
-	testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: subspaceStorageVersion });
+	testPack.factorioVersion = "2.0.0";
+	testPack.mods.set("clusterio_lib", { name: "clusterio_lib", enabled: true, version: "2.0.20" });
 	await control.sendTo("controller", new lib.ModPackCreateRequest(testPack));
 	await control.sendTo(
 		"controller",

test/lib/data/ModPack.js

@@ -27,8 +27,7 @@ describe("lib/data/ModPack", function() {
 			check(ModPack.fromJSON({ description: "My Description" }));
 			check(ModPack.fromJSON({ factorio_version: "2.0" }));
 			check(ModPack.fromJSON({ mods: [
-				{ name: "subspace_storage", enabled: true, version: "1.99.8" },
-				{ name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" },
+				{ name: "clusterio_lib", enabled: true, version: "2.0.20", sha1: "012345abcd" },
 			]}));
 			check(ModPack.fromJSON({ settings: {
 				"startup": {
@@ -49,8 +48,7 @@ describe("lib/data/ModPack", function() {
 				description: "Every option at once.",
 				factorio_version: "2.0",
 				mods: [
-					{ name: "subspace_storage", enabled: true, version: "1.99.8" },
-					{ name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" },
+					{ name: "clusterio_lib", enabled: true, version: "2.0.20", sha1: "012345abcd" },
 				],
 				settings: {
 					"startup": {

test/lib/database.js

@@ -169,7 +169,7 @@ describe("lib/database", function() {
 
 			it("should restore the passed serialized database", function() {
 				let items = new lib.ItemDatabase({ "a": 1, "b": 2 });
-				assert.deepEqual(items._items, new Map([["a", 1], ["b", 2]]));
+				assert.deepEqual(items._items, new Map([["a", { normal: 1 }], ["b", { normal: 2 }]]));
 			});
 
 			it("should throw on invalid serialized database", function() {
@@ -188,7 +188,7 @@ describe("lib/database", function() {
 		describe(".serialize()", function() {
 			it("should return a serialized database", function() {
 				let items = new lib.ItemDatabase({ "a": 10 });
-				assert.deepEqual(items.serialize(), { "a": 10 });
+				assert.deepEqual(items.serialize(), { "a": { normal: 10 } });
 			});
 
 			it("should remove zero count entries", function() {
@@ -207,18 +207,18 @@ describe("lib/database", function() {
 		describe(".getItemCount()", function() {
 			it("should return the count of the given item", function() {
 				let items = new lib.ItemDatabase({ "a": 10 });
-				assert.equal(items.getItemCount("a"), 10);
+				assert.equal(items.getItemCount("a", "normal"), 10);
 			});
 
 			it("should return zero if item does not exist", function() {
 				let items = new lib.ItemDatabase();
-				assert.equal(items.getItemCount("b"), 0);
+				assert.equal(items.getItemCount("b", "normal"), 0);
 			});
 
 			it("should throw on invalid name", function() {
 				let items = new lib.ItemDatabase();
 				assert.throws(
-					() => items.getItemCount(2),
+					() => items.getItemCount(2, "normal"),
 					new Error("name must be a string")
 				);
 			});
@@ -227,20 +227,20 @@ describe("lib/database", function() {
 		describe(".addItem()", function() {
 			it("should add a new item", function() {
 				let items = new lib.ItemDatabase();
-				items.addItem("a", 10);
-				assert.deepEqual(items._items, new Map([["a", 10]]));
+				items.addItem("a", 10, "normal");
+				assert.deepEqual(items._items, new Map([["a", { normal: 10 }]]));
 			});
 
 			it("should add an existing item", function() {
 				let items = new lib.ItemDatabase({ "a": 10 });
-				items.addItem("a", 10);
-				assert.deepEqual(items._items, new Map([["a", 20]]));
+				items.addItem("a", 10, "normal");
+				assert.deepEqual(items._items, new Map([["a", { normal: 20 }]]));
 			});
 
 			it("should throw on invalid name", function() {
 				let items = new lib.ItemDatabase();
 				assert.throws(
-					() => items.addItem(2, 10),
+					() => items.addItem(2, 10, "normal"),
 					new Error("name must be a string")
 				);
 			});
@@ -248,11 +248,11 @@ describe("lib/database", function() {
 			it("should throw on invalid count", function() {
 				let items = new lib.ItemDatabase();
 				assert.throws(
-					() => items.addItem("a", NaN),
+					() => items.addItem("a", NaN, "normal"),
 					new Error("count must be a number")
 				);
 				assert.throws(
-					() => items.addItem("a", "1"),
+					() => items.addItem("a", "1", "normal"),
 					new Error("count must be a number")
 				);
 			});
@@ -261,20 +261,20 @@ describe("lib/database", function() {
 		describe(".removeItem()", function() {
 			it("should remove an existing item", function() {
 				let items = new lib.ItemDatabase({ "a": 20 });
-				items.removeItem("a", 10);
-				assert.deepEqual(items._items, new Map([["a", 10]]));
+				items.removeItem("a", 10, "normal");
+				assert.deepEqual(items._items, new Map([["a", { normal: 10 }]]));
 			});
 
 			it("should turn a non-existing item negative", function() {
 				let items = new lib.ItemDatabase();
-				items.removeItem("a", 10);
-				assert.deepEqual(items._items, new Map([["a", -10]]));
+				items.removeItem("a", 10, "normal");
+				assert.deepEqual(items._items, new Map([["a", { normal: -10 }]]));
 			});
 
 			it("should throw on invalid name", function() {
 				let items = new lib.ItemDatabase();
 				assert.throws(
-					() => items.removeItem(2, 10),
+					() => items.removeItem(2, 10, "normal"),
 					new Error("name must be a string")
 				);
 			});
@@ -282,11 +282,11 @@ describe("lib/database", function() {
 			it("should throw on invalid count", function() {
 				let items = new lib.ItemDatabase();
 				assert.throws(
-					() => items.removeItem("a", "b"),
+					() => items.removeItem("a", "b", "normal"),
 					new Error("count must be a number")
 				);
 				assert.throws(
-					() => items.removeItem("a", "1"),
+					() => items.removeItem("a", "1", "normal"),
 					new Error("count must be a number")
 				);
 			});