From 7b8127fef8732a1885bfe0b3940a95cf4dfb8d4e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 19 Jul 2022 21:24:12 +0200 Subject: [PATCH] Refactor inventory_sync serialisation (#470) Fix broken support for retrying uploads and background downloads by making crafting queue serialisation non-destructive and only performing the serialisation and deserialisation while the player is online. This removes the /save-crafts and /restore-crafts commands because the existing implementation would voids the non-saved crafting queue items if /save-crafts is performed and then new items are added to the crafting queue before switching server. Fixes #419. --- .../module/download_inventory.lua | 102 +----------- .../inventory_sync/module/inventory_sync.lua | 154 +++++++++++++++++- .../module/load_crafting_queue.lua | 27 --- .../inventory_sync/module/restore_crafts.lua | 26 --- plugins/inventory_sync/module/save_crafts.lua | 103 ------------ plugins/inventory_sync/module/serialize.lua | 131 +++++++++++++++ .../module/upload_inventory.lua | 31 ---- 7 files changed, 283 insertions(+), 291 deletions(-) delete mode 100644 plugins/inventory_sync/module/load_crafting_queue.lua delete mode 100644 plugins/inventory_sync/module/restore_crafts.lua delete mode 100644 plugins/inventory_sync/module/save_crafts.lua delete mode 100644 plugins/inventory_sync/module/upload_inventory.lua diff --git a/plugins/inventory_sync/module/download_inventory.lua b/plugins/inventory_sync/module/download_inventory.lua index 30b0edf4..446fc934 100644 --- a/plugins/inventory_sync/module/download_inventory.lua +++ b/plugins/inventory_sync/module/download_inventory.lua @@ -1,34 +1,8 @@ local progress_dialog = require("modules/inventory_sync/gui/progress_dialog") local ensure_character = require("modules/inventory_sync/ensure_character") local serialize = require("modules/inventory_sync/serialize") -local load_crafting_queue = require("modules/inventory_sync/load_crafting_queue") -local can_enter_vehicle = { - [defines.controllers.character] = true, - [defines.controllers.god] = true, - [defines.controllers.editor] = true, -} - -local function restore_position(record, player) - if record.vehicle and record.vehicle.valid and can_enter_vehicle[player.controller_type] then - player.teleport(record.vehicle.position, record.vehicle.surface) - player.driving = true - - -- Teleport to safe location if unable to enter vehicle - if not player.driving and player.controller_type == defines.controllers.character then - local safe_position = record.vehicle.surface.find_non_colliding_position( - player.character.name, player.position, 32, 1/8 - ) - if safe_position then - player.teleport(safe_position, player.surface) - end - end - elseif record.surface and record.position then - player.teleport(record.position, record.surface) - end -end - local function download_inventory(player_name, data, number, total) local player = game.get_player(player_name) if player == nil then @@ -82,82 +56,20 @@ local function download_inventory(player_name, data, number, total) return end + -- Download is complete progress_dialog.remove(player) - -- Editor mode is not supported. - if player.controller_type == defines.controllers.editor then - player.toggle_map_editor() - end - - if player.controller_type == defines.controllers.editor then - log("ERROR: Inventory sync failed, unable to switch " .. player.name .. " out of editor mode") - player.print("ERROR: Inventory sync failed, unable to switch out of editor mode") - return - end - - -- Stash temporary inventory if it exists - local stashed_corpse - if player_record.dirty and player.character then - local character = player.character - local surface = character.surface - local position = character.position - local corpse_name = character.prototype.character_corpse.name - player.character = nil - character.die() - stashed_corpse = surface.find_entity(corpse_name, position) - end - - -- Deserialize downloaded player data - local serialized_player = game.json_to_table(record.data) - serialize.deserialize_player(player, serialized_player) - - -- Restore player position and driving state - restore_position(record, player) - - -- Load crafting queue - if player.character then - load_crafting_queue(serialized_player.crafting_queue, player) - end - - -- Transfer items from stashed inventory - if stashed_corpse then - local main = player.get_main_inventory() - local stash = stashed_corpse.get_inventory(defines.inventory.character_corpse) - if main then - for i = 1, #stash do - -- Try transfering a stack - local source = stash[i] - if source and source.valid_for_read then - local target = main.find_empty_stack() - if not target then - break - end - target.transfer_stack(source) - end - end - end - - if stash.is_empty() then - stashed_corpse.destroy() - else - player.print( - "Your temprorary inventory did not fit into your synced one and the remaining items have " .. - "been placed in a corpse below you." - ) - end - end - - -- Player may have left before the download completed - player_record.dirty = player.connected - player_record.sync = true - player_record.generation = serialized_player.generation - - -- Remove download session and data + -- Remove download session and store finished download global.inventory_sync.active_downloads[player_name] = nil + global.inventory_sync.finished_downloads[player_name] = record local ticks = game.ticks_played - record.started player.print("Imported player data in " .. ticks .. " ticks") log("Imported player data for " .. player_name .. " in " .. ticks .. " ticks") + + if player.connected then + inventory_sync.finish_download(player, record) + end end return download_inventory diff --git a/plugins/inventory_sync/module/inventory_sync.lua b/plugins/inventory_sync/module/inventory_sync.lua index 5adca874..3aff39b1 100644 --- a/plugins/inventory_sync/module/inventory_sync.lua +++ b/plugins/inventory_sync/module/inventory_sync.lua @@ -8,8 +8,8 @@ When a player joins the game, request the player data from master and deserializ ]] local clusterio_api = require("modules/clusterio/api") -local upload_inventory = require("modules/inventory_sync/upload_inventory") local download_inventory = require("modules/inventory_sync/download_inventory") +local serialize = require("modules/inventory_sync/serialize") local progress_dialog = require("modules/inventory_sync/gui/progress_dialog") local dialog_failed_download = require("modules/inventory_sync/gui/dialog_failed_download") @@ -30,10 +30,10 @@ end -- Remove all stored player data by this module local function remove_player(player) - global.inventory_sync.saved_crafting_queue[player.name] = nil global.inventory_sync.players_waiting_for_acquire[player.name] = nil global.inventory_sync.players_in_cutscene_to_sync[player.name] = nil global.inventory_sync.active_downloads[player.name] = nil + global.inventory_sync.finished_downloads[player.name] = nil global.inventory_sync.active_uploads[player.name] = nil global.inventory_sync.players[player.name] = nil end @@ -42,8 +42,110 @@ end inventory_sync = {} -- This function is called by instance.js as /sc inventory_sync.download_inventory("danielv", Escaped JSON string, package_number, total_packages_count) with data from the master inventory_sync.download_inventory = download_inventory --- This function is called internally when a player leaves the game for their inventory to upload to the master -inventory_sync.upload_inventory = upload_inventory + +-- This function is called internally when a player leaves the game to +-- serialize the player for upload. +function inventory_sync.serialize_player(player, player_record) + -- Editor mode is not supported. + if player.controller_type == defines.controllers.editor then + player.toggle_map_editor() + end + + if player.controller_type == defines.controllers.editor then + error("Unable to switch out of editor mode") + return + end + + local serialized_player = serialize.serialize_player(player) + serialized_player.generation = player_record.generation + + return serialized_player +end + +local can_enter_vehicle = { + [defines.controllers.character] = true, + [defines.controllers.god] = true, + [defines.controllers.editor] = true, +} + +local function restore_position(player, record) + if record.vehicle and record.vehicle.valid and can_enter_vehicle[player.controller_type] then + player.teleport(record.vehicle.position, record.vehicle.surface) + player.driving = true + + -- Teleport to safe location if unable to enter vehicle + if not player.driving and player.controller_type == defines.controllers.character then + local safe_position = record.vehicle.surface.find_non_colliding_position( + player.character.name, player.position, 32, 1/8 + ) + if safe_position then + player.teleport(safe_position, player.surface) + end + end + elseif record.surface and record.position then + player.teleport(record.position, record.surface) + end +end + +function inventory_sync.deserialize_player(player, finished_record) + -- Editor mode is not supported. + if player.controller_type == defines.controllers.editor then + player.toggle_map_editor() + end + + if player.controller_type == defines.controllers.editor then + error("Unable to switch out of editor mode") + return + end + + -- Stash temporary inventory if it exists + local stashed_corpse + local player_record = global.inventory_sync.players[player.name] + if player_record.dirty and player.character then + local character = player.character + local surface = character.surface + local position = character.position + local corpse_name = character.prototype.character_corpse.name + player.character = nil + character.die() + stashed_corpse = surface.find_entity(corpse_name, position) + end + + -- Deserialize downloaded player data + local serialized_player = game.json_to_table(finished_record.data) + serialize.deserialize_player(player, serialized_player) + + -- Restore player position and driving state + restore_position(player, finished_record) + + -- Transfer items from stashed inventory + if stashed_corpse then + local main = player.get_main_inventory() + local stash = stashed_corpse.get_inventory(defines.inventory.character_corpse) + if main then + for i = 1, #stash do + -- Try transfering a stack + local source = stash[i] + if source and source.valid_for_read then + local target = main.find_empty_stack() + if not target then + break + end + target.transfer_stack(source) + end + end + end + + if stash.is_empty() then + stashed_corpse.destroy() + else + player.print( + "Your temprorary inventory did not fit into your synced one and the remaining items have " .. + "been placed in a corpse below you." + ) + end + end +end inventory_sync.events = {} inventory_sync.events[clusterio_api.events.on_server_startup] = function(event) @@ -51,9 +153,6 @@ inventory_sync.events[clusterio_api.events.on_server_startup] = function(event) if not global.inventory_sync then global.inventory_sync = {} end - if not global.inventory_sync.saved_crafting_queue then - global.inventory_sync.saved_crafting_queue = {} - end if not global.inventory_sync.players then global.inventory_sync.players = {} @@ -65,6 +164,9 @@ inventory_sync.events[clusterio_api.events.on_server_startup] = function(event) if not global.inventory_sync.active_downloads then global.inventory_sync.active_downloads = {} end + if not global.inventory_sync.finished_downloads then + global.inventory_sync.finished_downloads = {} + end if not global.inventory_sync.active_uploads then global.inventory_sync.active_uploads = {} end @@ -146,6 +248,18 @@ function inventory_sync.sync_player(acquire_response) return end + local finished_record = global.inventory_sync.finished_downloads[acquire_response.player_name] + if finished_record then + if acquire_response.status == "acquired" and finished_record.generation == acquire_response.generation then + inventory_sync.finish_download(player, finished_record) + return + end + + -- The acquisition either failed or the current finished download is + -- stale, delete it and try downloading it again. + global.inventory_sync.finished_downloads[acquire_response.player_name] = nil + end + local download_record = global.inventory_sync.active_downloads[acquire_response.player_name] if download_record then -- Ignore the response if it affirms the current active download @@ -214,6 +328,20 @@ function inventory_sync.sync_player(acquire_response) end end +function inventory_sync.finish_download(player, finished_record) + local status, result = pcall(inventory_sync.deserialize_player, player, finished_record) + if not status then + log("ERROR: Deserializing player " .. player.name .. " failed: " .. result) + player.print("ERROR: Deserializing player data failed: " .. result) + end + global.inventory_sync.finished_downloads[player.name] = nil + + local player_record = global.inventory_sync.players[player.name] + player_record.dirty = true + player_record.sync = true + player_record.generation = finished_record.generation +end + -- Download inventory from master inventory_sync.events[defines.events.on_player_joined_game] = function(event) -- Send acquire request even if an active download is currently in plogress @@ -263,11 +391,19 @@ inventory_sync.events[defines.events.on_pre_player_left_game] = function(event) end player_record.generation = player_record.generation + 1 + local status, result = pcall(inventory_sync.serialize_player, player, player_record) + if not status then + log("ERROR: Serializing player " .. player.name .. " failed: " .. result) + player.print("ERROR: Serializing player data failed: " .. result) + return + end global.inventory_sync.active_uploads[player.name] = { + serialized = result, last_attempt = game.ticks_played, timeout = math.random(600, 1200), -- Start with 10-20 seconds for the timeout } - inventory_sync.upload_inventory(player, player_record) + + clusterio_api.send_json("inventory_sync_upload", result) end -- Invoked by the instance code to confirm the master has received the uploaded data @@ -362,7 +498,7 @@ function inventory_sync.check_active_uploads() log("Retrying upload for " .. player_name) record.last_attempt = game.ticks_played record.timeout = math.random(record.timeout, record.timeout * 2) - inventory_sync.upload_inventory(player, player_record) + clusterio_api.send_json("inventory_sync_upload", record.serialized) end end end diff --git a/plugins/inventory_sync/module/load_crafting_queue.lua b/plugins/inventory_sync/module/load_crafting_queue.lua deleted file mode 100644 index d2429fe1..00000000 --- a/plugins/inventory_sync/module/load_crafting_queue.lua +++ /dev/null @@ -1,27 +0,0 @@ --- Load crafting queue from a table -local function load_crafting_queue(crafting_queue, player) - local inventory = player.get_main_inventory() - - -- Give player some more inventory space to avoid duplicating items - player.character_inventory_slots_bonus = player.character_inventory_slots_bonus + 1000 - - -- Add items to inventory - for _, item in pairs(crafting_queue.ingredients) do - inventory.insert(item) - end - - -- Load crafting queue - for _, queueItem in pairs(crafting_queue.crafting_queue) do - -- Start crafting (consume items) - player.begin_crafting { - count = queueItem.count, - recipe = queueItem.recipe, - -- silent = true, -- Fail silently if items are missing - } - end - - -- Remove extra inventory slots - player.character_inventory_slots_bonus = player.character_inventory_slots_bonus - 1000 -end - -return load_crafting_queue diff --git a/plugins/inventory_sync/module/restore_crafts.lua b/plugins/inventory_sync/module/restore_crafts.lua deleted file mode 100644 index 99c422a2..00000000 --- a/plugins/inventory_sync/module/restore_crafts.lua +++ /dev/null @@ -1,26 +0,0 @@ -local load_crafting_queue = require("modules/inventory_sync/load_crafting_queue") - -local restore_crafts = {} - -function restore_crafts.command(event) - if event.player_index then - local player = game.get_player(event.player_index) - local crafting_queue = global.inventory_sync.saved_crafting_queue[player.name] - - if crafting_queue == nil then - player.print("No saved crafting queue") - return - end - - load_crafting_queue(crafting_queue, player) - - global.inventory_sync.saved_crafting_queue[player.name] = nil - end -end - -function restore_crafts.add_commands() - commands.add_command("restore-crafts", "Load crafting queue from global", restore_crafts.command) - commands.add_command("crc", "Shorthand for /restore-crafts", restore_crafts.command) -end - -return restore_crafts diff --git a/plugins/inventory_sync/module/save_crafts.lua b/plugins/inventory_sync/module/save_crafts.lua deleted file mode 100644 index 3b03b5f9..00000000 --- a/plugins/inventory_sync/module/save_crafts.lua +++ /dev/null @@ -1,103 +0,0 @@ -local save_crafts = {} - -function save_crafts.command(event) - if event.player_index then - local player = game.get_player(event.player_index) - local crafting_queue = {} - - -- Avoid overwriting old save - if global.inventory_sync.saved_crafting_queue[player.name] ~= nil then - player.print("You already have a saved crafting queue, restore it with /restore-crafts") - return - end - - -- Give player some more inventory space to avoid duplicating items - player.character_inventory_slots_bonus = player.character_inventory_slots_bonus + 1000 - - -- Save current items - local inventory = player.get_main_inventory() - local old_items = inventory.get_contents() - - -- Cancel old crafts to get the items back - while player.crafting_queue_size > 0 do - local old_queue = player.crafting_queue - local queueItem = player.crafting_queue[1] - - -- Cancel craft - player.cancel_crafting { - index = 1, - count = 1, - } - - local rightmost_right_index = 0 -- 0 indexed since it is subtractive in a 1 indexed language - local new_queue = player.crafting_queue - while - new_queue ~= nil and - new_queue[#new_queue - rightmost_right_index] ~= nil and - new_queue[#new_queue - rightmost_right_index].count >= old_queue[#old_queue - rightmost_right_index].count - do - rightmost_right_index = rightmost_right_index + 1 - end - local oldItem = old_queue[#old_queue - rightmost_right_index] - local newItem = nil - if new_queue ~= nil then - newItem = new_queue[#new_queue - rightmost_right_index] - end - - -- Figure out how many items to add to queue - local added = oldItem.count - if newItem ~= nil then - added = oldItem.count - newItem.count - if oldItem.recipe ~= newItem.recipe then - log("ERROR: Old item "..oldItem.recipe.." is not equal "..newItem.recipe) - end - end - - -- If the last item we added was of the same type, merge them in the queue - if #crafting_queue > 0 and crafting_queue[#crafting_queue].recipe == oldItem.recipe then - crafting_queue[#crafting_queue].count = crafting_queue[#crafting_queue].count + added - else - -- If the last item was of a different type, add a new item to the queue - table.insert(crafting_queue, { - recipe = oldItem.recipe, - count = added, - }) - end - -- game.print("Saved craft "..oldItem.recipe) - end - - -- Find amount of items added and remove from inventory - local new_items = inventory.get_contents() - local difference = {} - for k,v in pairs(new_items) do - local old_count = old_items[k] - local diff = v - if old_count ~= nil then - diff = diff - old_count - end - if diff > 0 then - local ingredient = { - name = k, - count = diff, - } - inventory.remove(ingredient) - table.insert(difference, ingredient) - end - end - - -- Remove extra inventory slots - player.character_inventory_slots_bonus = player.character_inventory_slots_bonus - 1000 - - global.inventory_sync.saved_crafting_queue[player.name] = { - crafting_queue = crafting_queue, - ingredients = difference, - } - end -end - -function save_crafts.add_commands() - commands.add_command("save-crafts", "Save crafting queue for transport to a different server", save_crafts.command) - commands.add_command("csc", "Shorthand for /save-crafts", save_crafts.command) -end - -return save_crafts diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index e0e264a8..ef176672 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -111,6 +111,125 @@ function serialize.deserialize_personal_logistic_slots(player, serialized) end end +-- Crafting queue is a table with the following fields: +-- crafting_queue +-- ingredients +function serialize.serialize_crafting_queue(player) + local crafting_queue = {} + + -- Give player some more inventory space to avoid duplicating items + player.character_inventory_slots_bonus = player.character_inventory_slots_bonus + 1000 + + -- Save current items + local inventory = player.get_main_inventory() + local old_items = inventory.get_contents() + + -- Cancel old crafts to get the items back + while player.crafting_queue_size > 0 do + local old_queue = player.crafting_queue + local queueItem = player.crafting_queue[1] + + -- Cancel craft + player.cancel_crafting { + index = 1, + count = 1, + } + + local rightmost_right_index = 0 -- 0 indexed since it is subtractive in a 1 indexed language + local new_queue = player.crafting_queue + while + new_queue ~= nil and + new_queue[#new_queue - rightmost_right_index] ~= nil and + new_queue[#new_queue - rightmost_right_index].count >= old_queue[#old_queue - rightmost_right_index].count + do + rightmost_right_index = rightmost_right_index + 1 + end + local oldItem = old_queue[#old_queue - rightmost_right_index] + local newItem = nil + if new_queue ~= nil then + newItem = new_queue[#new_queue - rightmost_right_index] + end + + -- Figure out how many items to add to queue + local added = oldItem.count + if newItem ~= nil then + added = oldItem.count - newItem.count + if oldItem.recipe ~= newItem.recipe then + log("ERROR: Old item "..oldItem.recipe.." is not equal "..newItem.recipe) + end + end + + -- If the last item we added was of the same type, merge them in the queue + if #crafting_queue > 0 and crafting_queue[#crafting_queue].recipe == oldItem.recipe then + crafting_queue[#crafting_queue].count = crafting_queue[#crafting_queue].count + added + else + -- If the last item was of a different type, add a new item to the queue + table.insert(crafting_queue, { + recipe = oldItem.recipe, + count = added, + }) + end + -- game.print("Saved craft "..oldItem.recipe) + end + + -- Find amount of items added and remove from inventory + local new_items = inventory.get_contents() + local difference = {} + for k,v in pairs(new_items) do + local old_count = old_items[k] + local diff = v + if old_count ~= nil then + diff = diff - old_count + end + if diff > 0 then + local ingredient = { + name = k, + count = diff, + } + inventory.remove(ingredient) + table.insert(difference, ingredient) + end + end + + -- Remove extra inventory slots + player.character_inventory_slots_bonus = player.character_inventory_slots_bonus - 1000 + + local serialized = { + crafting_queue = crafting_queue, + ingredients = difference, + } + + -- Restore the crafting queue that was just destructively serialized + serialize.deserialize_crafting_queue(player, serialized) + + return serialized +end + +function serialize.deserialize_crafting_queue(player, serialized) + local inventory = player.get_main_inventory() + + -- Give player some more inventory space to avoid duplicating items + player.character_inventory_slots_bonus = player.character_inventory_slots_bonus + 1000 + + -- Add items to inventory + for _, item in pairs(serialized.ingredients) do + inventory.insert(item) + end + + -- Load crafting queue + for _, queueItem in pairs(serialized.crafting_queue) do + -- Start crafting (consume items) + player.begin_crafting { + count = queueItem.count, + recipe = queueItem.recipe, + -- silent = true, -- Fail silently if items are missing + } + end + + -- Remove extra inventory slots + player.character_inventory_slots_bonus = player.character_inventory_slots_bonus - 1000 +end + local controller_to_name = {} for name, value in pairs(defines.controllers) do controller_to_name[value] = name @@ -130,6 +249,7 @@ end -- inventories: non-character inventories -- hotbar: table of string indexes to names (optional) -- personal_logistic_slots (optional) +-- crafting_queue (optional) function serialize.serialize_player(player) local serialized = { controller = controller_to_name[player.controller_type], @@ -178,6 +298,11 @@ function serialize.serialize_player(player) end end + -- Serialize crafting queue + if player.character then + serialized.crafting_queue = serialize.serialize_crafting_queue(player) + end + return serialized end @@ -253,6 +378,12 @@ function serialize.deserialize_player(player, serialized) end end end + + -- Deserialize crafting queue + if player.character and serialized.crafting_queue then + serialize.deserialize_crafting_queue(player, serialized.crafting_queue) + end + end return serialize diff --git a/plugins/inventory_sync/module/upload_inventory.lua b/plugins/inventory_sync/module/upload_inventory.lua deleted file mode 100644 index 5ab6051e..00000000 --- a/plugins/inventory_sync/module/upload_inventory.lua +++ /dev/null @@ -1,31 +0,0 @@ -local clusterio_api = require("modules/clusterio/api") -local serialize = require("modules/inventory_sync/serialize") -local save_crafts = require("modules/inventory_sync/save_crafts") - -local function upload_inventory(player, player_record) - -- Editor mode is not supported. - if player.controller_type == defines.controllers.editor then - player.toggle_map_editor() - end - - if player.controller_type == defines.controllers.editor then - log("ERROR: Inventory sync failed to upload, unable to switch " .. player.name .. " out of editor mode") - player.print("ERROR: Inventory sync failed to upload, unable to switch out of editor mode") - global.inventory_sync.active_uploads[player.name] = nil - return - end - - local serialized_player = serialize.serialize_player(player) - serialized_player.generation = player_record.generation - - -- Serialize crafting queue saved by /csc or /save-crafts OR automatically - if player.character then - save_crafts.command({player_index = player.name}) - end - serialized_player.crafting_queue = global.inventory_sync.saved_crafting_queue[player.name] - global.inventory_sync.saved_crafting_queue[player.name] = nil - - clusterio_api.send_json("inventory_sync_upload", serialized_player) -end - -return upload_inventory