Send only recipe notification changes on inventory download

Lua includes its current cleared recipe list in the download request. The instance plugin diffs it against the stored list and sends back the names to add and remove instead of the full list. Lua rebuilds the full list from the snapshot kept in the download record and runs the existing deserializer, so the end state is unchanged.

Also stop the serializer from appending to the stored failed recipe list in place, which grew the list on every upload without a download in between.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

plugins/inventory_sync/README.md

@@ -32,8 +32,8 @@ The greatly simplified data flow is as follows:
     3. If the player inventory was previously uploaded but is not the same then the player inventory is deleted and the player is turned into a spectator.
     4. Otherwise the player inventory is kept.
 
-5. Scenario script asks for the player inventory from the controller.
-6. Controller sends it to the scenario in a stream of chunks.
+5. Scenario script asks for the player inventory from the controller, including its current recipe notification state.
+6. Controller sends it to the scenario in a stream of chunks, with only the recipe notifications that differ from the current state.
 7. Scenario displays and updates a progress bar as each chunk is received.
 8. Once all chunks have loaded the player's synced inventory is recreated from the data and the player can start playing.
 

plugins/inventory_sync/instance.ts

@@ -1,3 +1,5 @@
+import util from "util";
+import zlib from "zlib";
 import * as lib from "@clusterio/lib";
 import { BaseInstancePlugin } from "@clusterio/host";
 import {
@@ -8,6 +10,11 @@ type IpcPlayerName = {
 	player_name: string
 }
 
+type IpcDownloadRequest = {
+	player_name: string,
+	recipe_notifications?: string,
+}
+
 type IpcAcquireResponse = {
 	player_name: string,
 	status: string,
@@ -26,6 +33,49 @@ function chunkify(chunkSize: number, string: string): string[] {
 	return string.match(new RegExp(`.{1,${chunkSize}}`, "g")) || [];
 }
 
+const inflate = util.promisify(zlib.inflate);
+const deflate = util.promisify(zlib.deflate);
+
+/**
+ * Decode a string produced by helpers.encode_string in Factorio
+ */
+async function decodeLuaString(encoded: string): Promise<string> {
+	return (await inflate(Buffer.from(encoded, "base64"))).toString("utf8");
+}
+
+/**
+ * Encode a string so that helpers.decode_string in Factorio can read it
+ */
+async function encodeLuaString(text: string): Promise<string> {
+	return (await deflate(Buffer.from(text, "utf8"))).toString("base64");
+}
+
+export type RecipeNotificationDelta = {
+	add?: string[],
+	remove?: string[],
+}
+
+/**
+ * Compute the changes needed to turn the current cleared recipe list into the stored one
+ * @param stored - Cleared recipe names stored on the controller
+ * @param current - Cleared recipe names the instance currently has
+ * @returns names to add to and remove from the current list
+ */
+export function recipeNotificationDelta(stored: string[], current: string[]): RecipeNotificationDelta {
+	const storedSet = new Set(stored);
+	const currentSet = new Set(current);
+	const delta: RecipeNotificationDelta = {};
+	const add = stored.filter(name => !currentSet.has(name));
+	const remove = current.filter(name => !storedSet.has(name));
+	if (add.length) {
+		delta.add = add;
+	}
+	if (remove.length) {
+		delta.remove = remove;
+	}
+	return delta;
+}
+
 export class InstancePlugin extends BaseInstancePlugin {
 	playersToRelease!: Set<string>;
 	disconnecting!: boolean;
@@ -55,7 +105,7 @@ export class InstancePlugin extends BaseInstancePlugin {
 		);
 		this.instance.server.on(
 			"ipc-inventory_sync_download",
-			(request: IpcPlayerName) => this.handleDownload(request).catch(
+			(request: IpcDownloadRequest) => this.handleDownload(request).catch(
 				err => this.logger.error(`Error handling ipc-inventory_sync_download:\n${err.stack}`)
 			),
 		);
@@ -163,7 +213,27 @@ export class InstancePlugin extends BaseInstancePlugin {
 		);
 	}
 
-	async handleDownload(request: IpcPlayerName) {
+	/**
+	 * Replace the stored recipe notifications with the difference from the
+	 * current state on the instance to reduce the amount of data sent to Lua.
+	 */
+	async applyRecipeNotificationDelta(playerData: IpcPlayerData, current: string) {
+		if (!playerData.recipe_notifications) {
+			return;
+		}
+		try {
+			const delta = recipeNotificationDelta(
+				JSON.parse(await decodeLuaString(playerData.recipe_notifications)),
+				JSON.parse(await decodeLuaString(current)),
+			);
+			playerData.recipe_notifications_delta = await encodeLuaString(JSON.stringify(delta));
+			delete playerData.recipe_notifications;
+		} catch (err: any) {
+			this.logger.warn(`Sending full recipe notifications for ${playerData.name}:\n${err.stack}`);
+		}
+	}
+
+	async handleDownload(request: IpcDownloadRequest) {
 		const playerName = request.player_name;
 		this.logger.verbose(`Downloading ${playerName}`);
 
@@ -177,6 +247,10 @@ export class InstancePlugin extends BaseInstancePlugin {
 			return;
 		}
 
+		if (request.recipe_notifications) {
+			await this.applyRecipeNotificationDelta(response.playerData, request.recipe_notifications);
+		}
+
 		const chunkSize = this.instance.config.get("inventory_sync.rcon_chunk_size");
 		const chunks = chunkify(chunkSize, JSON.stringify(response.playerData));
 		this.logger.verbose(`Sending inventory for ${playerName} in ${chunks.length} chunks`);

plugins/inventory_sync/messages.ts

@@ -21,6 +21,7 @@ export type IpcPlayerData = {
 	personal_logistic_slots?: {name:string, min:number, max:number}[],
 	crafting_queue?: any,
 	recipe_notifications?: string,
+	recipe_notifications_delta?: string,
 }
 
 const jsonPlayerData = Type.Object({

plugins/inventory_sync/module/inventory_sync.lua

@@ -111,6 +111,13 @@ function inventory_sync.deserialize_player(player, finished_record)
 	-- Deserialize downloaded player data
 	local serialized_player = compat.json_to_table(finished_record.data)
 	assert(type(serialized_player) == "table", "wrong type for serialized_player")
+	if serialized_player.recipe_notifications_delta then
+		serialized_player.recipe_notifications = serialize.apply_crafting_notification_delta(
+			assert(finished_record.recipe_notifications, "recipe notification delta without snapshot"),
+			serialized_player.recipe_notifications_delta
+		)
+		serialized_player.recipe_notifications_delta = nil
+	end
 	script_data.failed_deserialization[player.name] = serialize.deserialize_player(player, serialized_player)
 
 	-- Restore player position and driving state
@@ -441,8 +448,14 @@ function inventory_sync.initiate_inventory_download(player, player_record, gener
 	local script_data = get_script_data()
 	script_data.active_downloads[player.name] = record
 
+	-- The plugin only sends back what differs from the current notification state
+	record.recipe_notifications = serialize.crafting_notification_snapshot(
+		player, script_data.failed_deserialization[player.name]
+	)
+
 	clusterio_api.send_json("inventory_sync_download", {
-		player_name = player.name
+		player_name = player.name,
+		recipe_notifications = record.recipe_notifications,
 	})
 
 	-- If this is a synced player turn them into a spectator while the

plugins/inventory_sync/module/serialize.lua

@@ -533,17 +533,61 @@ function serialize.serialize_crafting_notifications(player, failed)
 		recipes[recipe.name] = nil
 	end
 
-	-- Convert to a list of names
-	local recipe_names = failed or {} -- Includes previous recipes if any
-	local index = #recipe_names + 1
+	-- Convert to a list of names, including previously failed recipes if any
+	local recipe_names = {}
+	for _, name in ipairs(failed or {}) do
+		recipe_names[#recipe_names + 1] = name
+	end
 	for name in pairs(recipes) do
-		recipe_names[index] = name
-		index = index + 1
+		recipe_names[#recipe_names + 1] = name
 	end
 
 	return helpers.encode_string(helpers.table_to_json(recipe_names))
 end
 
+--- Current notification state, sent with download requests so the instance plugin only returns the difference
+--- @param player LuaPlayer
+--- @param failed FailedDeserializationPlayerData?
+--- @return string?
+function serialize.crafting_notification_snapshot(player, failed)
+	if not recipe_notifications_api then
+		return nil
+	end
+	return serialize.serialize_crafting_notifications(player, failed and failed.recipe_notifications)
+end
+
+--- @class CraftingNotificationDelta
+--- @field add string[]? Recipe names to add to the cleared list
+--- @field remove string[]? Recipe names to remove from the cleared list
+
+--- Rebuild the full serialized form from the snapshot the delta was computed against
+--- @param snapshot string
+--- @param serialized_delta string
+--- @return string?
+function serialize.apply_crafting_notification_delta(snapshot, serialized_delta)
+	local recipe_names = helpers.json_to_table(assert(helpers.decode_string(snapshot)))
+	assert(type(recipe_names) == "table", "wrong type decoded from json_to_table")
+	--- @type CraftingNotificationDelta
+	local delta = helpers.json_to_table(assert(helpers.decode_string(serialized_delta)))
+	assert(type(delta) == "table", "wrong type decoded from json_to_table")
+	local cleared = {}
+	for _, name in pairs(recipe_names) do
+		cleared[name] = true
+	end
+	for _, name in pairs(delta.remove or {}) do
+		cleared[name] = nil
+	end
+	for _, name in pairs(delta.add or {}) do
+		cleared[name] = true
+	end
+
+	local merged = {}
+	for name in pairs(cleared) do
+		merged[#merged + 1] = name
+	end
+	return helpers.encode_string(helpers.table_to_json(merged))
+end
+
 --- @param player LuaPlayer
 --- @param serialized string
 --- @return string[]? failed Recipe names that do not exist

plugins/inventory_sync/test/instance.js

@@ -0,0 +1,68 @@
+"use strict";
+const assert = require("assert").strict;
+const util = require("util");
+const zlib = require("zlib");
+
+const { recipeNotificationDelta, InstancePlugin } = require("../dist/node/instance");
+
+const inflate = util.promisify(zlib.inflate);
+const deflate = util.promisify(zlib.deflate);
+
+// Matches helpers.encode_string in Factorio
+async function encode(value) {
+	return (await deflate(Buffer.from(JSON.stringify(value)))).toString("base64");
+}
+async function decode(encoded) {
+	return JSON.parse((await inflate(Buffer.from(encoded, "base64"))).toString());
+}
+
+describe("inventory_sync", function() {
+	describe("recipeNotificationDelta()", function() {
+		it("should be empty when the lists match", function() {
+			assert.deepEqual(recipeNotificationDelta(["a", "b"], ["b", "a"]), {});
+		});
+		it("should add stored names missing from current", function() {
+			assert.deepEqual(recipeNotificationDelta(["a", "b", "c"], ["a"]), { add: ["b", "c"] });
+		});
+		it("should remove current names missing from stored", function() {
+			assert.deepEqual(recipeNotificationDelta(["a"], ["a", "b"]), { remove: ["b"] });
+		});
+		it("should handle both directions at once", function() {
+			assert.deepEqual(recipeNotificationDelta(["a", "b"], ["b", "c"]), { add: ["a"], remove: ["c"] });
+		});
+		it("should keep duplicates in stored out of the current list", function() {
+			assert.deepEqual(recipeNotificationDelta(["a", "a"], ["a"]), {});
+		});
+	});
+
+	describe("InstancePlugin.applyRecipeNotificationDelta()", function() {
+		const warnings = [];
+		const plugin = { logger: { warn: msg => warnings.push(msg) } };
+		const apply = InstancePlugin.prototype.applyRecipeNotificationDelta.bind(plugin);
+
+		it("should replace the stored list with an encoded delta", async function() {
+			const playerData = { name: "test", recipe_notifications: await encode(["a", "b"]) };
+			await apply(playerData, await encode(["b", "c"]));
+			assert.equal(playerData.recipe_notifications, undefined);
+			assert.deepEqual(await decode(playerData.recipe_notifications_delta), { add: ["a"], remove: ["c"] });
+		});
+		it("should send an empty delta when nothing changed", async function() {
+			const playerData = { name: "test", recipe_notifications: await encode(["a"]) };
+			await apply(playerData, await encode(["a"]));
+			assert.deepEqual(await decode(playerData.recipe_notifications_delta), {});
+		});
+		it("should do nothing without stored notifications", async function() {
+			const playerData = { name: "test" };
+			await apply(playerData, await encode(["a"]));
+			assert.deepEqual(playerData, { name: "test" });
+		});
+		it("should fall back to the full list on invalid input", async function() {
+			const stored = await encode(["a"]);
+			const playerData = { name: "test", recipe_notifications: stored };
+			await apply(playerData, "not base64 zlib");
+			assert.equal(playerData.recipe_notifications, stored);
+			assert.equal(playerData.recipe_notifications_delta, undefined);
+			assert.equal(warnings.length, 1);
+		});
+	});
+});