diff --git a/plugins/inventory_sync/README.md b/plugins/inventory_sync/README.md index c39a8283..ecf5d615 100644 --- a/plugins/inventory_sync/README.md +++ b/plugins/inventory_sync/README.md @@ -32,8 +32,8 @@ The greatly simplified data flow is as follows: 3. If the player inventory was previously uploaded but is not the same then the player inventory is deleted and the player is turned into a spectator. 4. Otherwise the player inventory is kept. -5. Scenario script asks for the player inventory from the controller. -6. Controller sends it to the scenario in a stream of chunks. +5. Scenario script asks for the player inventory from the controller, including its current recipe notification state. +6. Controller sends it to the scenario in a stream of chunks, with only the recipe notifications that differ from the current state. 7. Scenario displays and updates a progress bar as each chunk is received. 8. Once all chunks have loaded the player's synced inventory is recreated from the data and the player can start playing. diff --git a/plugins/inventory_sync/instance.ts b/plugins/inventory_sync/instance.ts index c831e8fd..39ddea17 100644 --- a/plugins/inventory_sync/instance.ts +++ b/plugins/inventory_sync/instance.ts @@ -1,3 +1,5 @@ +import util from "util"; +import zlib from "zlib"; import * as lib from "@clusterio/lib"; import { BaseInstancePlugin } from "@clusterio/host"; import { @@ -8,6 +10,11 @@ type IpcPlayerName = { player_name: string } +type IpcDownloadRequest = { + player_name: string, + recipe_notifications?: string, +} + type IpcAcquireResponse = { player_name: string, status: string, @@ -26,6 +33,49 @@ function chunkify(chunkSize: number, string: string): string[] { return string.match(new RegExp(`.{1,${chunkSize}}`, "g")) || []; } +const inflate = util.promisify(zlib.inflate); +const deflate = util.promisify(zlib.deflate); + +/** + * Decode a string produced by helpers.encode_string in Factorio + */ +async function decodeLuaString(encoded: string): Promise { + return (await inflate(Buffer.from(encoded, "base64"))).toString("utf8"); +} + +/** + * Encode a string so that helpers.decode_string in Factorio can read it + */ +async function encodeLuaString(text: string): Promise { + return (await deflate(Buffer.from(text, "utf8"))).toString("base64"); +} + +export type RecipeNotificationDelta = { + add?: string[], + remove?: string[], +} + +/** + * Compute the changes needed to turn the current cleared recipe list into the stored one + * @param stored - Cleared recipe names stored on the controller + * @param current - Cleared recipe names the instance currently has + * @returns names to add to and remove from the current list + */ +export function recipeNotificationDelta(stored: string[], current: string[]): RecipeNotificationDelta { + const storedSet = new Set(stored); + const currentSet = new Set(current); + const delta: RecipeNotificationDelta = {}; + const add = stored.filter(name => !currentSet.has(name)); + const remove = current.filter(name => !storedSet.has(name)); + if (add.length) { + delta.add = add; + } + if (remove.length) { + delta.remove = remove; + } + return delta; +} + export class InstancePlugin extends BaseInstancePlugin { playersToRelease!: Set; disconnecting!: boolean; @@ -55,7 +105,7 @@ export class InstancePlugin extends BaseInstancePlugin { ); this.instance.server.on( "ipc-inventory_sync_download", - (request: IpcPlayerName) => this.handleDownload(request).catch( + (request: IpcDownloadRequest) => this.handleDownload(request).catch( err => this.logger.error(`Error handling ipc-inventory_sync_download:\n${err.stack}`) ), ); @@ -163,7 +213,27 @@ export class InstancePlugin extends BaseInstancePlugin { ); } - async handleDownload(request: IpcPlayerName) { + /** + * Replace the stored recipe notifications with the difference from the + * current state on the instance to reduce the amount of data sent to Lua. + */ + async applyRecipeNotificationDelta(playerData: IpcPlayerData, current: string) { + if (!playerData.recipe_notifications) { + return; + } + try { + const delta = recipeNotificationDelta( + JSON.parse(await decodeLuaString(playerData.recipe_notifications)), + JSON.parse(await decodeLuaString(current)), + ); + playerData.recipe_notifications_delta = await encodeLuaString(JSON.stringify(delta)); + delete playerData.recipe_notifications; + } catch (err: any) { + this.logger.warn(`Sending full recipe notifications for ${playerData.name}:\n${err.stack}`); + } + } + + async handleDownload(request: IpcDownloadRequest) { const playerName = request.player_name; this.logger.verbose(`Downloading ${playerName}`); @@ -177,6 +247,10 @@ export class InstancePlugin extends BaseInstancePlugin { return; } + if (request.recipe_notifications) { + await this.applyRecipeNotificationDelta(response.playerData, request.recipe_notifications); + } + const chunkSize = this.instance.config.get("inventory_sync.rcon_chunk_size"); const chunks = chunkify(chunkSize, JSON.stringify(response.playerData)); this.logger.verbose(`Sending inventory for ${playerName} in ${chunks.length} chunks`); diff --git a/plugins/inventory_sync/messages.ts b/plugins/inventory_sync/messages.ts index 028c39e0..5585fe55 100644 --- a/plugins/inventory_sync/messages.ts +++ b/plugins/inventory_sync/messages.ts @@ -21,6 +21,7 @@ export type IpcPlayerData = { personal_logistic_slots?: {name:string, min:number, max:number}[], crafting_queue?: any, recipe_notifications?: string, + recipe_notifications_delta?: string, } const jsonPlayerData = Type.Object({ diff --git a/plugins/inventory_sync/module/inventory_sync.lua b/plugins/inventory_sync/module/inventory_sync.lua index bf56a57e..6df45f77 100644 --- a/plugins/inventory_sync/module/inventory_sync.lua +++ b/plugins/inventory_sync/module/inventory_sync.lua @@ -111,6 +111,13 @@ function inventory_sync.deserialize_player(player, finished_record) -- Deserialize downloaded player data local serialized_player = compat.json_to_table(finished_record.data) assert(type(serialized_player) == "table", "wrong type for serialized_player") + if serialized_player.recipe_notifications_delta then + serialized_player.recipe_notifications = serialize.apply_crafting_notification_delta( + assert(finished_record.recipe_notifications, "recipe notification delta without snapshot"), + serialized_player.recipe_notifications_delta + ) + serialized_player.recipe_notifications_delta = nil + end script_data.failed_deserialization[player.name] = serialize.deserialize_player(player, serialized_player) -- Restore player position and driving state @@ -441,8 +448,14 @@ function inventory_sync.initiate_inventory_download(player, player_record, gener local script_data = get_script_data() script_data.active_downloads[player.name] = record + -- The plugin only sends back what differs from the current notification state + record.recipe_notifications = serialize.crafting_notification_snapshot( + player, script_data.failed_deserialization[player.name] + ) + clusterio_api.send_json("inventory_sync_download", { - player_name = player.name + player_name = player.name, + recipe_notifications = record.recipe_notifications, }) -- If this is a synced player turn them into a spectator while the diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index 6d62a6aa..78430de3 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -533,17 +533,61 @@ function serialize.serialize_crafting_notifications(player, failed) recipes[recipe.name] = nil end - -- Convert to a list of names - local recipe_names = failed or {} -- Includes previous recipes if any - local index = #recipe_names + 1 + -- Convert to a list of names, including previously failed recipes if any + local recipe_names = {} + for _, name in ipairs(failed or {}) do + recipe_names[#recipe_names + 1] = name + end for name in pairs(recipes) do - recipe_names[index] = name - index = index + 1 + recipe_names[#recipe_names + 1] = name end return helpers.encode_string(helpers.table_to_json(recipe_names)) end +--- Current notification state, sent with download requests so the instance plugin only returns the difference +--- @param player LuaPlayer +--- @param failed FailedDeserializationPlayerData? +--- @return string? +function serialize.crafting_notification_snapshot(player, failed) + if not recipe_notifications_api then + return nil + end + return serialize.serialize_crafting_notifications(player, failed and failed.recipe_notifications) +end + +--- @class CraftingNotificationDelta +--- @field add string[]? Recipe names to add to the cleared list +--- @field remove string[]? Recipe names to remove from the cleared list + +--- Rebuild the full serialized form from the snapshot the delta was computed against +--- @param snapshot string +--- @param serialized_delta string +--- @return string? +function serialize.apply_crafting_notification_delta(snapshot, serialized_delta) + local recipe_names = helpers.json_to_table(assert(helpers.decode_string(snapshot))) + assert(type(recipe_names) == "table", "wrong type decoded from json_to_table") + --- @type CraftingNotificationDelta + local delta = helpers.json_to_table(assert(helpers.decode_string(serialized_delta))) + assert(type(delta) == "table", "wrong type decoded from json_to_table") + local cleared = {} + for _, name in pairs(recipe_names) do + cleared[name] = true + end + for _, name in pairs(delta.remove or {}) do + cleared[name] = nil + end + for _, name in pairs(delta.add or {}) do + cleared[name] = true + end + + local merged = {} + for name in pairs(cleared) do + merged[#merged + 1] = name + end + return helpers.encode_string(helpers.table_to_json(merged)) +end + --- @param player LuaPlayer --- @param serialized string --- @return string[]? failed Recipe names that do not exist diff --git a/plugins/inventory_sync/test/instance.js b/plugins/inventory_sync/test/instance.js new file mode 100644 index 00000000..3354142a --- /dev/null +++ b/plugins/inventory_sync/test/instance.js @@ -0,0 +1,68 @@ +"use strict"; +const assert = require("assert").strict; +const util = require("util"); +const zlib = require("zlib"); + +const { recipeNotificationDelta, InstancePlugin } = require("../dist/node/instance"); + +const inflate = util.promisify(zlib.inflate); +const deflate = util.promisify(zlib.deflate); + +// Matches helpers.encode_string in Factorio +async function encode(value) { + return (await deflate(Buffer.from(JSON.stringify(value)))).toString("base64"); +} +async function decode(encoded) { + return JSON.parse((await inflate(Buffer.from(encoded, "base64"))).toString()); +} + +describe("inventory_sync", function() { + describe("recipeNotificationDelta()", function() { + it("should be empty when the lists match", function() { + assert.deepEqual(recipeNotificationDelta(["a", "b"], ["b", "a"]), {}); + }); + it("should add stored names missing from current", function() { + assert.deepEqual(recipeNotificationDelta(["a", "b", "c"], ["a"]), { add: ["b", "c"] }); + }); + it("should remove current names missing from stored", function() { + assert.deepEqual(recipeNotificationDelta(["a"], ["a", "b"]), { remove: ["b"] }); + }); + it("should handle both directions at once", function() { + assert.deepEqual(recipeNotificationDelta(["a", "b"], ["b", "c"]), { add: ["a"], remove: ["c"] }); + }); + it("should keep duplicates in stored out of the current list", function() { + assert.deepEqual(recipeNotificationDelta(["a", "a"], ["a"]), {}); + }); + }); + + describe("InstancePlugin.applyRecipeNotificationDelta()", function() { + const warnings = []; + const plugin = { logger: { warn: msg => warnings.push(msg) } }; + const apply = InstancePlugin.prototype.applyRecipeNotificationDelta.bind(plugin); + + it("should replace the stored list with an encoded delta", async function() { + const playerData = { name: "test", recipe_notifications: await encode(["a", "b"]) }; + await apply(playerData, await encode(["b", "c"])); + assert.equal(playerData.recipe_notifications, undefined); + assert.deepEqual(await decode(playerData.recipe_notifications_delta), { add: ["a"], remove: ["c"] }); + }); + it("should send an empty delta when nothing changed", async function() { + const playerData = { name: "test", recipe_notifications: await encode(["a"]) }; + await apply(playerData, await encode(["a"])); + assert.deepEqual(await decode(playerData.recipe_notifications_delta), {}); + }); + it("should do nothing without stored notifications", async function() { + const playerData = { name: "test" }; + await apply(playerData, await encode(["a"])); + assert.deepEqual(playerData, { name: "test" }); + }); + it("should fall back to the full list on invalid input", async function() { + const stored = await encode(["a"]); + const playerData = { name: "test", recipe_notifications: stored }; + await apply(playerData, "not base64 zlib"); + assert.equal(playerData.recipe_notifications, stored); + assert.equal(playerData.recipe_notifications_delta, undefined); + assert.equal(warnings.length, 1); + }); + }); +});