factorio-scenario-ExpCluster/expcore/datastore.lua

--[[-- Core Module - Datastore
- A module used to store data in the global table with the option to have it sync to an external source.
@core Datastore
@alias DatastoreManager

@usage-- Types of Datastore
-- This datastore will not save data externally and can be used to watch for updates on values within it
-- A common use might be to store data for a gui and only update the gui when a value changes
local LocalDatastore = Datastore.connect('LocalDatastore')

-- This datastore will allow you to use the save and request method, this allows you to have persistent data
-- Should be used over auto save as it creates less save requests, but this means you need to tell the data to be saved
-- We use this type for player data as we know the data only needs to be saved when the player leaves
local PersistentDatastore = Datastore.connect('PersistentDatastore', true) -- save_to_disk

-- This datastore is the same as above but the save method will be called automatically when ever you change a value
-- An auto save datastore should be used if the data does not change often, this can be global settings and things of that sort
-- If it is at all possible to setup events to unload and/or save the data then this is preferable
local AutosaveDatastore = Datastore.connect('AutosaveDatastore', true, true) -- save_to_disk, auto_save

-- Finally you can have a datastore that propagates its changes to all other connected servers, this means request does not need to be used
-- This should be used when you might have data conflicts while saving, this is done by pushing the saved value to all active servers
-- The request method has little use after server start as any external changes to the value will be pushed automatically
-- Auto save can also be used with this type and you should follow the same guidelines above for when this should be avoided
local PropagateDatastore = Datastore.connect('PropagateDatastore', true, false, true) -- save_to_disk, propagate_changes

@usage-- Using Datastores Locally
-- Once you have your datastore connection setup, any further requests with connect will return the same datastore
-- This is important to know because the settings passed as parameters you have an effect when it is first created

-- One useful thing that you might want to set up before runtime is a serializer, this will convert non string keys into strings
-- This serializer will allow use to pass a player object and still have it serialized to the players name
local ExampleData = Datastore.connect('ExampleData')
ExampleData:set_serializer(function(rawKey)
    return rawKey.name
end)

-- If we want to get data from the datastore we can use get or get_all
local value = ExampleData:get(player, defaultValue)
local values = ExampleData:get_all()

-- If we want to set data then we can use set, increment, update, or update_all
ExampleData:set(player, 10)
ExampleData:increment(player)
ExampleData:update(player, function(player_name, value)
    return value * 2
end)
ExampleData:update_all(function(player_name, value)
    return value * 2
end)

-- If we want to remove data then we use remove
ExampleData:remove(player)

-- We can also listen for updates to a value done by any of the above methods with on_update
ExampleData:on_update(function(player_name, value)
    game.print(player_name..' has had their example data updated to '..tostring(value))
end)

@usage-- Using Datastore Externally
-- If save_to_disk is used then this opens up the option for persistent data which you can request, save, and remove
-- All of the local methods are still usable put now there is the option for extra events
-- In order for this to work there must be an external script to read datastore.pipe and inject with Datastore.ingest

-- To request data you would use request and the on_load event, this event can be used to modify data before it is used
ExampleData:request(player)
ExampleData:on_load(function(player_name, value)
    game.print('Loaded example data for '..player_name)
    -- A value can be returned here to overwrite the received value
end)

-- To save data you would use save and the on_save event, this event can be used to modify data before it is saved
ExampleData:save(player)
ExampleData:on_save(function(player_name, value)
    game.print('Saved example data for '..player_name)
    -- A value can be returned here to overwrite the value which is saved
end)

-- To remove data locally but not externally, like if a player logs off, you would use unload and on_unload
ExampleData:unload(player)
ExampleData:on_unload(function(player_name, value)
    game.print('Unloaded example data for '..player_name)
    -- Any return is ignored, this is event is for cleaning up other data
end)

@usage-- Using Datastore Messaging
-- The message action can be used regardless of save_to_disk being set as no data is saved, but an external script is still required
-- These messages can be used to send data to other servers which doesnt need to be saved such as shouts or commands
-- Using messages is quite simple only using message and on_message
ExampleData:message(key, message)
ExampleData:on_message(function(key, message)
    game.print('Received message '..message)
end)

@usage-- Combined Datastores
-- A combined datastore is a datastore which stores its data inside of another datastore
-- This means that the data is stored more efficiently in the external database and less requests need to be made
-- To understand how combined datastores work think of each key in the parent as a table where the sub datastore is a key in that table
-- Player data is the most used version of the combined datastore, below is how the player data module is setup
local PlayerData = Datastore.connect('PlayerData', true) -- saveToDisk
PlayerData:set_serializer(Datastore.name_serializer) -- use player name as key
PlayerData:combine('Statistics')
PlayerData:combine('Settings')
PlayerData:combine('Required')

-- You can then further combine datastores to any depth, below we add some possible settings and statistics that we might use
-- Although we dont in this example, each of these functions returns the datastore object which you should use as a local value
PlayerData.Settings:combine('Color')
PlayerData.Settings:combine('Quickbar')
PlayerData.Settings:combine('JoinMessage')
PlayerData.Statistics:combine('Playtime')
PlayerData.Statistics:combine('JoinCount')

-- Because sub datastore work just like a normal datastore you dont need any special code, using get and set will still return as if it wasnt a sub datastore
-- Things like the serializer and the datastore settings are always the same as the parent so you dont need to worry about setting up the serializer each time
-- And because save, request, and unload methods all point to the root datastore you are able to request and save your data as normal

-- If you used get_all on PlayerData this is what you would get:
{
    Cooldude2606 = {
        Settings = {
            Color = 'ColorValue',
            Quickbar = 'QuickbarValue',
            JoinMessage = 'JoinMessageValue'
        },
        Statistics = {
            Playtime = 'PlaytimeValue',
            JoinCount = 'JoinCountValue'
        }
    }
}

-- If you used get_all on PlayerData.Settings this is what you would get:
{
    Cooldude2606 = {
        Color = 'ColorValue',
        Quickbar = 'QuickbarValue',
        JoinMessage = 'JoinMessageValue'
    }
}

-- If you used get_all on PlayerData.Settings.Color this is what you would get:
{
    Cooldude2606 = 'ColorValue'
}

]]

local Event = require 'utils.event' --- @dep utils.event

local DatastoreManager = {}
local Datastores = {}
local Datastore = {}
local Data = {}
local copy = table.deep_copy

--- Save datastores in the global table
global.datastores = Data
Event.on_load(function()
    Data = global.datastores
    for datastoreName, datastore in pairs(Datastores) do
        datastore.data = Data[datastoreName]
    end
end)

----- Datastore Manager -----
-- @section datastoreManager

--- Metatable used on datastores
DatastoreManager.metatable = {
    __index = function(self, key) return rawget(self.children, key) or rawget(Datastore, key) end,
    __newidnex = function(_, _, _) error('Datastore can not be modified', 2) end,
    __call = function(self, ...) return self:get(...) end
}

--[[-- Make a new datastore connection, if a connection already exists then it is returned
@tparam string datastoreName The name that you want the new datastore to have, this can not have any whitespace
@tparam[opt=false] boolean saveToDisk When set to true, using the save method with write the data to datastore.pipe
@tparam[opt=false] boolean autoSave When set to true, using any method which modifies data will cause the data to be saved
@tparam[opt=false] boolean propagateChanges When set to true, using the save method will send the data to all other connected servers
@treturn table The new datastore connection that can be used to access and modify data in the datastore

@usage-- Connecting to the test datastore which will allow saving to disk
local ExampleData = Datastore.connect('ExampleData', true) -- saveToDisk

]]
function DatastoreManager.connect(datastoreName, saveToDisk, autoSave, propagateChanges)
    if Datastores[datastoreName] then return Datastores[datastoreName] end
    if _LIFECYCLE ~= _STAGE.control then
        -- Only allow this function to be called during the control stage
        error('New datastore connection can not be created during runtime', 2)
    end

    local new_datastore = {
        name = datastoreName,
        value_name = datastoreName,
        auto_save = autoSave or false,
        save_to_disk = saveToDisk or false,
        propagate_changes = propagateChanges or false,
        serializer = false,
        parent = false,
        children = {},
        metadata = {},
        events = {},
        data = {}
    }

    Data[datastoreName] = new_datastore.data
    Datastores[datastoreName] = new_datastore
    return setmetatable(new_datastore, DatastoreManager.metatable)
end

--[[-- Make a new datastore that stores its data inside of another one
@tparam string datastoreName The name of the datastore that will contain the data for the new datastore
@tparam string subDatastoreName The name of the new datastore, this name will also be used as the key inside the parent datastore
@treturn table The new datastore connection that can be used to access and modify data in the datastore

@usage-- Setting up a datastore which stores its data inside of another datastore
local BarData = Datastore.combine('ExampleData', 'Bar')

]]
function DatastoreManager.combine(datastoreName, subDatastoreName)
    local datastore = assert(Datastores[datastoreName], 'Datastore not found '..tostring(datastoreName))
    return datastore:combine(subDatastoreName)
end

--[[-- Ingest the result from a request, this is used through a rcon interface to sync data
@tparam string action The action that should be done, can be: remove, message, propagate, or request
@tparam string datastoreName The name of the datastore that should have the action done to it
@tparam string key The key of that datastore that is having the action done to it
@tparam string valueJson The json string for the value being ingested, remove does not require a value

@usage-- Replying to a data request
Datastore.ingest('request', 'ExampleData', 'TestKey', 'Foo')

]]
local function ingest_error(err) print('Datastore ingest error, Unable to parse json:', err) end
function DatastoreManager.ingest(action, datastoreName, key, valueJson)
    local datastore = assert(Datastores[datastoreName], 'Datastore ingest error, Datastore not found '..tostring(datastoreName))
    assert(type(action) == 'string', 'Datastore ingest error, Action is not a string got: '..type(action))
    assert(type(key) == 'string', 'Datastore ingest error, Key is not a string got: '..type(key))

    if action == 'remove' then
        datastore:raw_set(key)

    elseif action == 'message' then
        local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
        if not success then return end
        if value == nil then value = valueJson end
        datastore:raise_event('on_message', key, value)

    elseif action == 'propagate' or action == 'request' then
        local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
        if not success then return end
        if value == nil then value = valueJson end
        value = datastore:raise_event('on_load', key, value)
        datastore:set(key, value)

    end

end

--[[-- Debug, Use to get all datastores, or return debug info on a datastore
@tparam[opt] string datastoreName The name of the datastore to get the debug info of

@usage-- Get all the datastores
local datastores = Datastore.debug()

@usage-- Getting the debug info for a datastore
local debug_info = Datastore.debug('ExampleData')

]]
function DatastoreManager.debug(datastoreName)
    if not datastoreName then return Datastores end
    local datastore = assert(Datastores[datastoreName], 'Datastore not found '..tostring(datastoreName))
    return datastore:debug()
end

--[[-- Commonly used serializer, returns the name of the object
@tparam any rawKey The raw key that will be serialized, this can be things like player, force, surface, etc
@treturn string The name of the object that was passed

@usage-- Using the name serializer for your datastore
local ExampleData = Datastore.connect('ExampleData')
ExampleData:set_serializer(Datastore.name_serializer)

]]
function DatastoreManager.name_serializer(rawKey)
    return rawKey.name
end

----- Datastore Internal -----
-- @section datastore-internal

--[[-- Debug, Get the debug info for this datastore
@treturn table The debug info for this datastore, contains stuff like parent, settings, children, etc

@usage-- Get the debug info for a datastore
local ExampleData = Datastore.connect('ExampleData')
local debug_info = ExampleData:debug()

]]
function Datastore:debug()
    local debug_info = {}

    if self.parent then
        debug_info.parent = self.parent.name
    else
        debug_info.settings = { auto_save = self.auto_save, save_to_disk = self.save_to_disk, propagate_changes = self.propagate_changes, serializer = not not self.serializer }
    end

    local children = {}
    for name in pairs(self.children) do children[#children+1] = name end
    if #children > 0 then debug_info.children = children end

    local events = {}
    for name, handlers in pairs(self.events) do events[name] = #handlers end
    if next(events) then debug_info.events = events end

    if next(self.metadata) then debug_info.metadata = self.metadata end
    debug_info.data = self:get_all()

    return debug_info
end

--[[-- Internal, Get data following combine logic
@tparam string key The key to get the value of from this datastore
@tparam[opt=false] boolean fromChild If the get request came from a child of this datastore
@treturn any The value that was stored at this key in this datastore

@usage-- Internal, Get the data from a datastore
local value = self:raw_get('TestKey')

]]
function Datastore:raw_get(key, fromChild)
    local data = self.data
    if self.parent then
        data = self.parent:raw_get(key, true)
        key  = self.value_name
    end
    local value = data[key]
    if value ~= nil then return value end
    if fromChild then value = {} end
    data[key] = value
    return value
end

--[[-- Internal, Set data following combine logic
@tparam string key The key to set the value of in this datastore
@tparam any value The value that will be set at this key

@usage-- Internal, Set the value in a datastore
self:raw_set('TestKey', 'Foo')

]]
function Datastore:raw_set(key, value)
    if self.parent then
        local data = self.parent:raw_get(key, true)
        data[self.value_name] = value
    else
        self.data[key] = value
    end
end

local function serialize_error(err) error('An error ocurred in a datastore serializer: '..err) end
--[[-- Internal, Return the serialized key
@tparam any rawKey The key that needs to be serialized, if it is already a string then it is returned
@treturn string The key after it has been serialized

@usage-- Internal, Ensure that the key is a string
key = self:serialize(key)

]]
function Datastore:serialize(rawKey)
    if type(rawKey) == 'string' then return rawKey end
    assert(self.serializer, 'Datastore does not have a serializer and received non string key')
    local success, key = xpcall(self.serializer, serialize_error, rawKey)
    return success and key or nil
end

--[[-- Internal, Writes an event to the output file to be saved and/or propagated
@tparam string action The action that should be wrote to datastore.pipe, can be request, remove, message, save, propagate
@tparam string key The key that the action is being preformed on
@tparam any value The value that should be used with the action

@usage-- Write a data request to datastore.pipe
self:write_action('request', 'TestKey')

@usage-- Write a data save to datastore.pipe
self:write_action('save', 'TestKey', 'Foo')

]]
function Datastore:write_action(action, key, value)
    local data = {action, self.name, '"'..key..'"'}
    if value ~= nil then
        data[4] = type(value) == 'table' and '"'..game.table_to_json(value)..'"' or '"'..tostring(value)..'"'
    end
    game.write_file('datastore.pipe', table.concat(data, ' ')..'\n', true, 0)
end

----- Datastore -----
-- @section datastore

--[[-- Create a new datastore which is stores its data inside of this datastore
@tparam string subDatastoreName The name of the datastore that will have its data stored in this datastore
@treturn table The new datastore that was created inside of this datastore

@usage-- Add a new sub datastore
local ExampleData = Datastore.connect('ExampleData')
local BarData = ExampleData:combine('Bar')

]]
function Datastore:combine(subDatastoreName)
    local new_datastore = DatastoreManager.connect(self.name..'.'..subDatastoreName)
    self.children[subDatastoreName] = new_datastore
    new_datastore.value_name = subDatastoreName
    new_datastore.serializer = self.serializer
    new_datastore.auto_save = self.auto_save
    new_datastore.parent = self
    Data[new_datastore.name] = nil
    new_datastore.data = nil
    return new_datastore
end

--[[-- Set a callback that will be used to serialize keys which aren't strings
@tparam function callback The function that will be used to serialize non string keys passed as an argument

@usage-- Set a custom serializer, this would be the same as Datastore.name_serializer
local ExampleData = Datastore.connect('ExampleData')
ExampleData:set_serializer(function(rawKey)
    return rawKey.name
end)

]]
function Datastore:set_serializer(callback)
    assert(type(callback) == 'function', 'Callback must be a function')
    self.serializer = callback
end

--[[-- Set metadata tags on this datastore which can be accessed by other scripts
@tparam table tags A table of tags that you want to set in the metadata for this datastore

@usage-- Adding metadata that could be used by a gui to help understand the stored data
local ExampleData = Datastore.connect('ExampleData')
ExampleData:set_metadata{
    caption = 'Test Data',
    tooltip = 'Data used for testing datastores',
    type = 'table'
}

]]
function Datastore:set_metadata(tags)
    local metadata = self.metadata
    for key, value in pairs(tags) do
        metadata[key] = value
    end
end

--[[-- Get a value from local storage, option to have a default value
@tparam any key The key that you want to get the value of, must be a string unless a serializer is set
@tparam[opt] any default The default value that will be returned if no value is found in the datastore

@usage-- Get a key from the datastore, the default will be deep copied if no value exists in the datastore
local ExampleData = Datastore.connect('ExampleData')
local value = ExampleData:get('TestKey')

]]
function Datastore:get(key, default)
    key = self:serialize(key)
    local value = self:raw_get(key)
    if value ~= nil then return value end
    return copy(default)
end

--[[-- Set a value in local storage, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
@tparam any key The key that you want to set the value of, must be a string unless a serializer is set
@tparam any value The value that you want to set for this key

@usage-- Set a value in the datastore, this will trigger on_update, if auto_save is true then will trigger save
local ExampleData = Datastore.connect('ExampleData')
ExampleData:set('TestKey', 'Foo')

]]
function Datastore:set(key, value)
    key = self:serialize(key)
    self:raw_set(key, value)
    self:raise_event('on_update', key, value)
    if self.auto_save then self:save(key) end
    return value
end

--[[-- Increment the value in local storage, only works for number values, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
@tparam any key The key that you want to increment the value of, must be a string unless a serializer is set
@tparam[opt=1] number delta The amount that you want to increment the value by, can be negative or a decimal

@usage-- Increment a value in a datastore, the value must be a number or nil, if nil 0 is used as the start value
local ExampleData = Datastore.connect('ExampleData')
ExampleData:increment('TestNumber')

]]
function Datastore:increment(key, delta)
    key = self:serialize(key)
    local value = self:raw_get(key) or 0
    return Datastore:set(key, value + (delta or 1))
end

local function update_error(err) error('An error ocurred in datastore update: '..err, 2) end
--[[-- Use a function to update the value locally, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
@tparam any key The key that you want to apply the update to, must be a string unless a serializer is set
@tparam function callback The function that will be used to update the value at this key

@usage-- Using a function to update a value, if a value is returned then this will be the new value
local ExampleData = Datastore.connect('ExampleData')
ExampleData:increment('TestKey', function(key, value)
    return value..value
end)

]]
function Datastore:update(key, callback)
    key = self:serialize(key)
    local value = self:raw_get(key)
    local success, new_value = xpcall(callback, update_error, key, value)
    if success and new_value ~= nil then
        self:set(key, new_value)
    else
        self:raise_event('on_update', key, value)
        if self.auto_save then self:save(key) end
    end
end

--[[-- Remove a value locally and on the external source, works regardless of propagateChanges
@tparam any key The key that you want to remove locally and externally, must be a string unless a serializer is set

@usage-- Remove a key locally and externally
local ExampleData = Datastore.connect('ExampleData')
ExampleData:remove('TestKey')

]]
function Datastore:remove(key)
    key = self:serialize(key)
    self:raw_set(key)
    self:write_action('remove', key)
    if self.parent and self.parent.auto_save then return self.parent:save(key) end
end

local function filter_error(err) print('An error ocurred in a datastore filter:', err) end
--[[-- Internal, Used to filter elements from a table
@tparam table tbl The table that will have the filter applied to it
@tparam[opt] function callback The function that will be used as a filter, if none giving then the provided table is returned
@treturn table The table which has only the key values pairs which passed the filter

@usage-- Internal, Filter a table by the values it contains, return true to keep the key value pair
local filtered_table = filter({5,3,4,1,2}, function(key, value)
    return value > 2
end)

]]
local function filter(tbl, callback)
    if not callback then return tbl end
    local rtn = {}
    for key, value in pairs(tbl) do
        local success, add = xpcall(callback, filter_error, key, value)
        if success and add then rtn[key] = value end
    end
    return rtn
end

--[[-- Get all keys in this datastore, optional filter callback
@tparam[opt] function callback The filter function that can be used to filter the results returned
@treturn table All the data that is in this datastore, filtered if a filter was provided

@usage-- Get all the data in this datastore
local ExampleData = Datastore.connect('ExampleData')
local data = ExampleData:get_all()

@usage-- Get all the data in this datastore, with a filter
local ExampleData = Datastore.connect('ExampleData')
local data = ExampleData:get_all(function(key, value)
    return type(value) == 'string'
end)

]]
function Datastore:get_all(callback)
    if not self.parent then
        return filter(self.data, callback)
    else
        local data, value_name = {}, self.value_name
        for key, value in pairs(self.parent:get_all()) do
            data[key] = value[value_name]
        end
        return filter(data, callback)
    end
end

--[[-- Update all keys in this datastore using the same update function
@tparam function callback The update function that will be applied to each key

@usage-- Get all the data in this datastore, with a filter
local ExampleData = Datastore.connect('ExampleData')
ExampleData:update_all(function(key, value)
    return value..value
end)

]]
function Datastore:update_all(callback)
    local data = self:get_all()
    for key, value in pairs(data) do
        local success, new_value = xpcall(callback, update_error, key, value)
        if success and new_value ~= nil then
            self:set(key, new_value)
        else
            self:raise_event('on_update', key, value)
            if self.auto_save then self:save(key) end
        end
    end
end

----- Datastore External -----
-- @section datastore-external

--[[-- Request a value from an external source, will trigger on_load when data is received
@tparam any key The key that you want to request from an external source, must be a string unless a serializer is set

@usage-- Request a key from an external source, on_load is triggered when data is received
local ExampleData = Datastore.connect('ExampleData')
ExampleData:request('TestKey')

]]
function Datastore:request(key)
    if self.parent then return self.parent:request(key) end
    key = self:serialize(key)
    self:write_action('request', key)
end

--[[-- Save a value to an external source, will trigger on_save before data is saved, save_to_disk must be set to true
@tparam any key The key that you want to save to an external source, must be a string unless a serializer is set

@usage-- Save a key to an external source, save_to_disk must be set to true for there to be any effect
local ExampleData = Datastore.connect('ExampleData')
ExampleData:save('TestKey')

]]
function Datastore:save(key)
    if self.parent then self.parent:save(key) end
    if not self.save_to_disk then return end
    key = self:serialize(key)
    local value  = self:raise_event('on_save', key, copy(self:raw_get(key)))
    local action = self.propagate_changes and 'propagate' or 'save'
    self:write_action(action, key, value)
end

--[[-- Save a value to an external source and remove locally, will trigger on_unload then on_save, save_to_disk is not required for on_unload
@tparam any key The key that you want to unload from the datastore, must be a string unless a serializer is set

@usage-- Unload a key from the datastore, get will now return nil and value will be saved externally if save_to_disk is set to true
local ExampleData = Datastore.connect('ExampleData')
ExampleData:unload('TestKey')

]]
function Datastore:unload(key)
    if self.parent then return self.parent:unload(key) end
    key = self:serialize(key)
    self:raise_event('on_unload', key, copy(self:raw_get(key)))
    self:save(key)
    self:raw_set(key)
end

--[[-- Use to send a message over the connection, works regardless of saveToDisk and propagateChanges
@tparam any key The key that you want to send a message over, must be a string unless a serializer is set
@tparam any message The message that you want to send to other connected servers, or external source

@usage-- Send a message to other servers on this key, can listen for messages with on_message
local ExampleData = Datastore.connect('ExampleData')
ExampleData:message('TestKey', 'Foo')

]]
function Datastore:message(key, message)
    key = self:serialize(key)
    self:write_action('message', key, message)
end

--[[-- Save all the keys in the datastore, optional filter callback
@tparam[opt] function callback The filter function that can be used to filter the keys saved

@usage-- Save all the data in this datastore
local ExampleData = Datastore.connect('ExampleData')
local data = ExampleData:save_all()

@usage-- Save all the data in this datastore, with a filter
local ExampleData = Datastore.connect('ExampleData')
ExampleData:save_all(function(key, value)
    return type(value) == 'string'
end)

]]
function Datastore:save_all(callback)
    local data = self:get_all(callback)
    for key in pairs(data) do self:save(key) end
end

--[[-- Unload all the keys in the datastore, optional filter callback
@tparam[opt] function callback The filter function that can be used to filter the keys unloaded

@usage-- Unload all the data in this datastore
local ExampleData = Datastore.connect('ExampleData')
ExampleData:unload_all()

@usage-- Unload all the data in this datastore, with a filter
local ExampleData = Datastore.connect('ExampleData')
ExampleData:unload_all(function(key, value)
    return type(value) == 'string'
end)

]]
function Datastore:unload_all(callback)
    local data = self:get_all(callback)
    for key in pairs(data) do self:unload(key) end
end

----- Events -----
-- @section events

local function event_error(err) print('An error ocurred in a datastore event handler:', err) end
--[[-- Internal, Raise an event on this datastore
@tparam string event_name The name of the event to raise for this datastore
@tparam string key The key that this event is being raised for
@tparam[opt] any value The current value that this key has, might be a deep copy of the value
@tparam[opt] string source Where this call came from, used to do event recursion so can be parent or child
@treturn any The value that is left after being passed through all the event handlers

@usage-- Internal, Getting the value that should be saved
value = self:raise_event('on_save', key, value)

]]
function Datastore:raise_event(event_name, key, value, source)
    -- Raise the event for the children of this datastore
    if source ~= 'child' then
        for value_name, child in pairs(self.children) do
            value[value_name] = child:raise_event(event_name, key, value[value_name], 'parent')
        end
    end

    -- Raise the event for this datastore
    local handlers = self.events[event_name]
    if handlers then
        for _, handler in ipairs(handlers) do
            local success, new_value = xpcall(handler, event_error, key, value)
            if success and new_value ~= nil then value = new_value end
        end
    end

    -- Raise the event for the parent of this datastore
    if source ~= 'parent' and self.parent then
        self.parent:raise_event(event_name, key, self.parent:raw_get(key), 'child')
    end
    return value
end

--[[-- Internal, Returns a function which will add a callback to an event
@tparam string event_name The name of the event that this should create a handler adder for
@treturn function The function that can be used to add handlers to this event

@usage-- Internal, Get the function to add handlers to on_load
Datastore.on_load = event_factory('on_load')

]]
local function event_factory(event_name)
    return function(self, callback)
        assert(type(callback) == 'function', 'Handler must be a function')
        local handlers = self.events[event_name]
        if not handlers then
            self.events[event_name] = { callback }
        else
            handlers[#handlers+1] = callback
        end
    end
end

--[[-- Register a callback that triggers when data is loaded from an external source, returned value is saved locally
@tparam function callback The handler that will be registered to the on_load event
@usage-- Adding a handler to on_load, returned value will be saved locally, can be used to deserialize the value beyond a normal json
local ExampleData = Datastore.connect('ExampleData')
ExampleData:on_load(function(key, value)
    game.print('Test data loaded for: '..key)
end)
]]
Datastore.on_load = event_factory('on_load')

--[[-- Register a callback that triggers before data is saved, returned value is saved externally
@tparam function callback The handler that will be registered to the on_load event
@usage-- Adding a handler to on_save, returned value will be saved externally, can be used to serialize the value beyond a normal json
local ExampleData = Datastore.connect('ExampleData')
ExampleData:on_save(function(key, value)
    game.print('Test data saved for: '..key)
end)
]]
Datastore.on_save = event_factory('on_save')

--[[-- Register a callback that triggers before data is unloaded, returned value is ignored
@tparam function callback The handler that will be registered to the on_load event
@usage-- Adding a handler to on_unload, returned value is ignored, can be used to clean up guis or local values related to this data
local ExampleData = Datastore.connect('ExampleData')
ExampleData:on_load(function(key, value)
    game.print('Test data unloaded for: '..key)
end)
]]
Datastore.on_unload = event_factory('on_unload')

--[[-- Register a callback that triggers when a message is received, returned value is ignored
@tparam function callback The handler that will be registered to the on_load event
@usage-- Adding a handler to on_message, returned value is ignored, can be used to receive messages from other connected servers without saving data
local ExampleData = Datastore.connect('ExampleData')
ExampleData:on_message(function(key, value)
    game.print('Test data message for: '..key)
end)
]]
Datastore.on_message = event_factory('on_message')

--[[-- Register a callback that triggers any time a value is changed, returned value is ignored
@tparam function callback The handler that will be registered to the on_load event
@usage-- Adding a handler to on_update, returned value is ignored, can be used to update guis or send messages when data is changed
local ExampleData = Datastore.connect('ExampleData')
ExampleData:on_update(function(key, value)
    game.print('Test data updated for: '..key)
end)
]]
Datastore.on_update = event_factory('on_update')

----- Module Return -----
return DatastoreManager