Merge pull request #159 from Cooldude2606/feature/datastore

Persistent Datastores Including Player Data

config/_file_loader.lua

@@ -6,7 +6,9 @@
 return {
     --'example.file_not_loaded',
     'modules.factorio-control', -- base factorio free play scenario
-    -- Game Commands
+    'expcore.player_data',
+
+    --- Game Commands
     'modules.commands.me',
     'modules.commands.kill',
     'modules.commands.admin-chat',
@@ -28,7 +30,8 @@ return {
     'modules.commands.bonus',
     'modules.commands.home',
     'modules.commands.quickbar',
-    -- QoL Addons
+
+    --- Addons
     'modules.addons.station-auto-name',
     'modules.addons.greetings',
     'modules.addons.chat-popups',
@@ -43,7 +46,8 @@ return {
     'modules.addons.discord-alerts',
     'modules.addons.chat-reply',
     'modules.addons.tree-decon',
-    -- GUI
+
+    --- GUI
     'modules.gui.readme',
     'modules.gui.rocket-info',
     'modules.gui.science-info',
@@ -52,7 +56,8 @@ return {
     'modules.gui.player-list',
     'modules.gui.server-ups',
     'modules.commands.debug',
-    -- Config Files
+
+    --- Config Files
     'config.expcore.command_auth_admin', -- commands tagged with admin_only are blocked for non admins
     'config.expcore.command_auth_roles', -- commands must be allowed via the role config
     'config.expcore.command_runtime_disable', -- allows commands to be enabled and disabled during runtime

config/expcore/command_general_parse.lua

@@ -39,12 +39,12 @@ end)
 Commands.add_parse('string-options',function(input, player, reject, options)
     if not input then return end -- nil check
     input = input:lower()
-    for option in options do
+    for _, option in ipairs(options) do
         if input == option:lower() then
-            return true
+            return option
         end
     end
-    return reject{'reject-string-options',options:concat(', ')}
+    return reject{'expcore-commands.reject-string-options', table.concat(options, ', ')}
 end)
 
 Commands.add_parse('string-max-length',function(input, player, reject, max_length)

config/expcore/roles.lua

@@ -220,6 +220,8 @@ local default = Roles.new_role('Guest','')
     'command/report',
     'command/ratio',
     'command/server-ups',
+    'command/data-preference',
+    'command/set-data-preference',
     'gui/player-list',
     'gui/rocket-info',
     'gui/science-info',

expcore/datastore.lua

@@ -0,0 +1,830 @@
+--[[-- Core Module - Datastore
+- A module used to store data in the global table with the option to have it sync to an external source.
+@core Datastore
+@alias DatastoreManager
+
+@usage-- Types of Datastore
+-- This datastore will not save data externally and can be used to watch for updates on values within it
+-- A common use might be to store data for a gui and only update the gui when a value changes
+local LocalDatastore = Datastore.connect('LocalDatastore')
+
+-- This datastore will allow you to use the save and request method, this allows you to have persistent data
+-- Should be used over auto save as it creates less save requests, but this means you need to tell the data to be saved
+-- We use this type for player data as we know the data only needs to be saved when the player leaves
+local PersistentDatastore = Datastore.connect('PersistentDatastore', true) -- save_to_disk
+
+-- This datastore is the same as above but the save method will be called automatically when ever you change a value
+-- An auto save datastore should be used if the data does not change often, this can be global settings and things of that sort
+-- If it is at all possible to setup events to unload and/or save the data then this is preferable
+local AutosaveDatastore = Datastore.connect('AutosaveDatastore', true, true) -- save_to_disk, auto_save
+
+-- Finally you can have a datastore that propagates its changes to all other connected servers, this means request does not need to be used
+-- This should be used when you might have data conflicts while saving, this is done by pushing the saved value to all active servers
+-- The request method has little use after server start as any external changes to the value will be pushed automatically
+-- Auto save can also be used with this type and you should follow the same guidelines above for when this should be avoided
+local PropagateDatastore = Datastore.connect('PropagateDatastore', true, false, true) -- save_to_disk, propagate_changes
+
+@usage-- Using Datastores Locally
+-- Once you have your datastore connection setup, any further requests with connect will return the same datastore
+-- This is important to know because the settings passed as parameters you have an effect when it is first created
+
+-- One useful thing that you might want to set up before runtime is a serializer, this will convert non string keys into strings
+-- This serializer will allow use to pass a player object and still have it serialized to the players name
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_serializer(function(rawKey)
+    return rawKey.name
+end)
+
+-- If we want to get data from the datastore we can use get or get_all
+local value = ExampleData:get(player, defaultValue)
+local values = ExampleData:get_all()
+
+-- If we want to set data then we can use set, increment, update, or update_all
+ExampleData:set(player, 10)
+ExampleData:increment(player)
+ExampleData:update(player, function(player_name, value)
+    return value * 2
+end)
+ExampleData:update_all(function(player_name, value)
+    return value * 2
+end)
+
+-- If we want to remove data then we use remove
+ExampleData:remove(player)
+
+-- We can also listen for updates to a value done by any of the above methods with on_update
+ExampleData:on_update(function(player_name, value)
+    game.print(player_name..' has had their example data updated to '..tostring(value))
+end)
+
+@usage-- Using Datastore Externally
+-- If save_to_disk is used then this opens up the option for persistent data which you can request, save, and remove
+-- All of the local methods are still usable put now there is the option for extra events
+-- In order for this to work there must be an external script to read datastore.pipe and inject with Datastore.ingest
+
+-- To request data you would use request and the on_load event, this event can be used to modify data before it is used
+ExampleData:request(player)
+ExampleData:on_load(function(player_name, value)
+    game.print('Loaded example data for '..player_name)
+    -- A value can be returned here to overwrite the received value
+end)
+
+-- To save data you would use save and the on_save event, this event can be used to modify data before it is saved
+ExampleData:save(player)
+ExampleData:on_save(function(player_name, value)
+    game.print('Saved example data for '..player_name)
+    -- A value can be returned here to overwrite the value which is saved
+end)
+
+-- To remove data locally but not externally, like if a player logs off, you would use unload and on_unload
+ExampleData:unload(player)
+ExampleData:on_unload(function(player_name, value)
+    game.print('Unloaded example data for '..player_name)
+    -- Any return is ignored, this is event is for cleaning up other data
+end)
+
+@usage-- Using Datastore Messaging
+-- The message action can be used regardless of save_to_disk being set as no data is saved, but an external script is still required
+-- These messages can be used to send data to other servers which doesnt need to be saved such as shouts or commands
+-- Using messages is quite simple only using message and on_message
+ExampleData:message(key, message)
+ExampleData:on_message(function(key, message)
+    game.print('Received message '..message)
+end)
+
+@usage-- Combined Datastores
+-- A combined datastore is a datastore which stores its data inside of another datastore
+-- This means that the data is stored more efficiently in the external database and less requests need to be made
+-- To understand how combined datastores work think of each key in the parent as a table where the sub datastore is a key in that table
+-- Player data is the most used version of the combined datastore, below is how the player data module is setup
+local PlayerData = Datastore.connect('PlayerData', true) -- saveToDisk
+PlayerData:set_serializer(Datastore.name_serializer) -- use player name as key
+PlayerData:combine('Statistics')
+PlayerData:combine('Settings')
+PlayerData:combine('Required')
+
+-- You can then further combine datastores to any depth, below we add some possible settings and statistics that we might use
+-- Although we dont in this example, each of these functions returns the datastore object which you should use as a local value
+PlayerData.Settings:combine('Color')
+PlayerData.Settings:combine('Quickbar')
+PlayerData.Settings:combine('JoinMessage')
+PlayerData.Statistics:combine('Playtime')
+PlayerData.Statistics:combine('JoinCount')
+
+-- Because sub datastore work just like a normal datastore you dont need any special code, using get and set will still return as if it wasnt a sub datastore
+-- Things like the serializer and the datastore settings are always the same as the parent so you dont need to worry about setting up the serializer each time
+-- And because save, request, and unload methods all point to the root datastore you are able to request and save your data as normal
+
+-- If you used get_all on PlayerData this is what you would get:
+{
+    Cooldude2606 = {
+        Settings = {
+            Color = 'ColorValue',
+            Quickbar = 'QuickbarValue',
+            JoinMessage = 'JoinMessageValue'
+        },
+        Statistics = {
+            Playtime = 'PlaytimeValue',
+            JoinCount = 'JoinCountValue'
+        }
+    }
+}
+
+-- If you used get_all on PlayerData.Settings this is what you would get:
+{
+    Cooldude2606 = {
+        Color = 'ColorValue',
+        Quickbar = 'QuickbarValue',
+        JoinMessage = 'JoinMessageValue'
+    }
+}
+
+-- If you used get_all on PlayerData.Settings.Color this is what you would get:
+{
+    Cooldude2606 = 'ColorValue'
+}
+
+]]
+
+local Event = require 'utils.event' --- @dep utils.event
+
+local DatastoreManager = {}
+local Datastores = {}
+local Datastore = {}
+local Data = {}
+local copy = table.deep_copy
+
+--- Save datastores in the global table
+global.datastores = Data
+Event.on_load(function()
+    Data = global.datastores
+    for datastoreName, datastore in pairs(Datastores) do
+        datastore.data = Data[datastoreName]
+    end
+end)
+
+----- Datastore Manager
+-- @section datastoreManager
+
+--- Metatable used on datastores
+DatastoreManager.metatable = {
+    __index = function(self, key) return rawget(self.children, key) or rawget(Datastore, key) end,
+    __newidnex = function(_, _, _) error('Datastore can not be modified', 2) end,
+    __call = function(self, ...) return self:get(...) end
+}
+
+--[[-- Make a new datastore connection, if a connection already exists then it is returned
+@tparam string datastoreName The name that you want the new datastore to have, this can not have any whitespace
+@tparam[opt=false] boolean saveToDisk When set to true, using the save method with write the data to datastore.pipe
+@tparam[opt=false] boolean autoSave When set to true, using any method which modifies data will cause the data to be saved
+@tparam[opt=false] boolean propagateChanges When set to true, using the save method will send the data to all other connected servers
+@treturn table The new datastore connection that can be used to access and modify data in the datastore
+
+@usage-- Connecting to the test datastore which will allow saving to disk
+local ExampleData = Datastore.connect('ExampleData', true) -- saveToDisk
+
+]]
+function DatastoreManager.connect(datastoreName, saveToDisk, autoSave, propagateChanges)
+    if Datastores[datastoreName] then return Datastores[datastoreName] end
+    if _LIFECYCLE ~= _STAGE.control then
+        -- Only allow this function to be called during the control stage
+        error('New datastore connection can not be created during runtime', 2)
+    end
+
+    local new_datastore = {
+        name = datastoreName,
+        value_name = datastoreName,
+        auto_save = autoSave or false,
+        save_to_disk = saveToDisk or false,
+        propagate_changes = propagateChanges or false,
+        serializer = false,
+        parent = false,
+        children = {},
+        metadata = {},
+        events = {},
+        data = {}
+    }
+
+    Data[datastoreName] = new_datastore.data
+    Datastores[datastoreName] = new_datastore
+    return setmetatable(new_datastore, DatastoreManager.metatable)
+end
+
+--[[-- Make a new datastore that stores its data inside of another one
+@tparam string datastoreName The name of the datastore that will contain the data for the new datastore
+@tparam string subDatastoreName The name of the new datastore, this name will also be used as the key inside the parent datastore
+@treturn table The new datastore connection that can be used to access and modify data in the datastore
+
+@usage-- Setting up a datastore which stores its data inside of another datastore
+local BarData = Datastore.combine('ExampleData', 'Bar')
+
+]]
+function DatastoreManager.combine(datastoreName, subDatastoreName)
+    local datastore = assert(Datastores[datastoreName], 'Datastore not found '..tostring(datastoreName))
+    return datastore:combine(subDatastoreName)
+end
+
+local function ingest_error(err) print('Datastore ingest error, Unable to parse json:', err) end
+--[[-- Ingest the result from a request, this is used through a rcon interface to sync data
+@tparam string action The action that should be done, can be: remove, message, propagate, or request
+@tparam string datastoreName The name of the datastore that should have the action done to it
+@tparam string key The key of that datastore that is having the action done to it
+@tparam string valueJson The json string for the value being ingested, remove does not require a value
+
+@usage-- Replying to a data request
+Datastore.ingest('request', 'ExampleData', 'TestKey', 'Foo')
+
+]]
+function DatastoreManager.ingest(action, datastoreName, key, valueJson)
+    local datastore = assert(Datastores[datastoreName], 'Datastore ingest error, Datastore not found '..tostring(datastoreName))
+    assert(type(action) == 'string', 'Datastore ingest error, Action is not a string got: '..type(action))
+    assert(type(key) == 'string', 'Datastore ingest error, Key is not a string got: '..type(key))
+
+    if action == 'remove' then
+        datastore:raw_set(key)
+
+    elseif action == 'message' then
+        local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
+        if not success then return end
+        if value == nil then value = valueJson end
+        datastore:raise_event('on_message', key, value)
+
+    elseif action == 'propagate' or action == 'request' then
+        local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
+        if not success then return end
+        if value == nil then value = valueJson end
+        value = datastore:raise_event('on_load', key, value)
+        datastore:set(key, value)
+
+    end
+
+end
+
+--[[-- Debug, Use to get all datastores, or return debug info on a datastore
+@tparam[opt] string datastoreName The name of the datastore to get the debug info of
+
+@usage-- Get all the datastores
+local datastores = Datastore.debug()
+
+@usage-- Getting the debug info for a datastore
+local debug_info = Datastore.debug('ExampleData')
+
+]]
+function DatastoreManager.debug(datastoreName)
+    if not datastoreName then return Datastores end
+    local datastore = assert(Datastores[datastoreName], 'Datastore not found '..tostring(datastoreName))
+    return datastore:debug()
+end
+
+--[[-- Commonly used serializer, returns the name of the object
+@tparam any rawKey The raw key that will be serialized, this can be things like player, force, surface, etc
+@treturn string The name of the object that was passed
+
+@usage-- Using the name serializer for your datastore
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_serializer(Datastore.name_serializer)
+
+]]
+function DatastoreManager.name_serializer(rawKey)
+    return rawKey.name
+end
+
+----- Datastore Internal
+-- @section datastore-internal
+
+--[[-- Debug, Get the debug info for this datastore
+@treturn table The debug info for this datastore, contains stuff like parent, settings, children, etc
+
+@usage-- Get the debug info for a datastore
+local ExampleData = Datastore.connect('ExampleData')
+local debug_info = ExampleData:debug()
+
+]]
+function Datastore:debug()
+    local debug_info = {}
+
+    if self.parent then
+        debug_info.parent = self.parent.name
+    else
+        debug_info.settings = { auto_save = self.auto_save, save_to_disk = self.save_to_disk, propagate_changes = self.propagate_changes, serializer = not not self.serializer }
+    end
+
+    local children = {}
+    for name in pairs(self.children) do children[#children+1] = name end
+    if #children > 0 then debug_info.children = children end
+
+    local events = {}
+    for name, handlers in pairs(self.events) do events[name] = #handlers end
+    if next(events) then debug_info.events = events end
+
+    if next(self.metadata) then debug_info.metadata = self.metadata end
+    debug_info.data = self:get_all()
+
+    return debug_info
+end
+
+--[[-- Internal, Get data following combine logic
+@tparam string key The key to get the value of from this datastore
+@tparam[opt=false] boolean fromChild If the get request came from a child of this datastore
+@treturn any The value that was stored at this key in this datastore
+
+@usage-- Internal, Get the data from a datastore
+local value = self:raw_get('TestKey')
+
+]]
+function Datastore:raw_get(key, fromChild)
+    local data = self.data
+    if self.parent then
+        data = self.parent:raw_get(key, true)
+        key  = self.value_name
+    end
+    local value = data[key]
+    if value ~= nil then return value end
+    if fromChild then value = {} end
+    data[key] = value
+    return value
+end
+
+--[[-- Internal, Set data following combine logic
+@tparam string key The key to set the value of in this datastore
+@tparam any value The value that will be set at this key
+
+@usage-- Internal, Set the value in a datastore
+self:raw_set('TestKey', 'Foo')
+
+]]
+function Datastore:raw_set(key, value)
+    if self.parent then
+        local data = self.parent:raw_get(key, true)
+        data[self.value_name] = value
+    else
+        self.data[key] = value
+    end
+end
+
+local function serialize_error(err) error('An error ocurred in a datastore serializer: '..err) end
+--[[-- Internal, Return the serialized key
+@tparam any rawKey The key that needs to be serialized, if it is already a string then it is returned
+@treturn string The key after it has been serialized
+
+@usage-- Internal, Ensure that the key is a string
+key = self:serialize(key)
+
+]]
+function Datastore:serialize(rawKey)
+    if type(rawKey) == 'string' then return rawKey end
+    assert(self.serializer, 'Datastore does not have a serializer and received non string key')
+    local success, key = xpcall(self.serializer, serialize_error, rawKey)
+    return success and key or nil
+end
+
+--[[-- Internal, Writes an event to the output file to be saved and/or propagated
+@tparam string action The action that should be wrote to datastore.pipe, can be request, remove, message, save, propagate
+@tparam string key The key that the action is being preformed on
+@tparam any value The value that should be used with the action
+
+@usage-- Write a data request to datastore.pipe
+self:write_action('request', 'TestKey')
+
+@usage-- Write a data save to datastore.pipe
+self:write_action('save', 'TestKey', 'Foo')
+
+]]
+function Datastore:write_action(action, key, value)
+    local data = {action, self.name, '"'..key..'"'}
+    if value ~= nil then
+        data[4] = type(value) == 'table' and '"'..game.table_to_json(value)..'"' or '"'..tostring(value)..'"'
+    end
+    game.write_file('datastore.pipe', table.concat(data, ' ')..'\n', true, 0)
+end
+
+----- Datastore Local
+-- @section datastore-local
+
+--[[-- Create a new datastore which is stores its data inside of this datastore
+@tparam string subDatastoreName The name of the datastore that will have its data stored in this datastore
+@treturn table The new datastore that was created inside of this datastore
+
+@usage-- Add a new sub datastore
+local ExampleData = Datastore.connect('ExampleData')
+local BarData = ExampleData:combine('Bar')
+
+]]
+function Datastore:combine(subDatastoreName)
+    local new_datastore = DatastoreManager.connect(self.name..'.'..subDatastoreName)
+    self.children[subDatastoreName] = new_datastore
+    new_datastore.value_name = subDatastoreName
+    new_datastore.serializer = self.serializer
+    new_datastore.auto_save = self.auto_save
+    new_datastore.parent = self
+    Data[new_datastore.name] = nil
+    new_datastore.data = nil
+    return new_datastore
+end
+
+--[[-- Set a callback that will be used to serialize keys which aren't strings
+@tparam function callback The function that will be used to serialize non string keys passed as an argument
+
+@usage-- Set a custom serializer, this would be the same as Datastore.name_serializer
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_serializer(function(rawKey)
+    return rawKey.name
+end)
+
+]]
+function Datastore:set_serializer(callback)
+    assert(type(callback) == 'function', 'Callback must be a function')
+    self.serializer = callback
+end
+
+--[[-- Set metadata tags on this datastore which can be accessed by other scripts
+@tparam table tags A table of tags that you want to set in the metadata for this datastore
+
+@usage-- Adding metadata that could be used by a gui to help understand the stored data
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_metadata{
+    caption = 'Test Data',
+    tooltip = 'Data used for testing datastores',
+    type = 'table'
+}
+
+]]
+function Datastore:set_metadata(tags)
+    local metadata = self.metadata
+    for key, value in pairs(tags) do
+        metadata[key] = value
+    end
+end
+
+--[[-- Get a value from local storage, option to have a default value
+@tparam any key The key that you want to get the value of, must be a string unless a serializer is set
+@tparam[opt] any default The default value that will be returned if no value is found in the datastore
+
+@usage-- Get a key from the datastore, the default will be deep copied if no value exists in the datastore
+local ExampleData = Datastore.connect('ExampleData')
+local value = ExampleData:get('TestKey')
+
+]]
+function Datastore:get(key, default)
+    key = self:serialize(key)
+    local value = self:raw_get(key)
+    if value ~= nil then return value end
+    return copy(default)
+end
+
+--[[-- Set a value in local storage, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+@tparam any key The key that you want to set the value of, must be a string unless a serializer is set
+@tparam any value The value that you want to set for this key
+
+@usage-- Set a value in the datastore, this will trigger on_update, if auto_save is true then will trigger save
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set('TestKey', 'Foo')
+
+]]
+function Datastore:set(key, value)
+    key = self:serialize(key)
+    self:raw_set(key, value)
+    self:raise_event('on_update', key, value)
+    if self.auto_save then self:save(key) end
+    return value
+end
+
+--[[-- Increment the value in local storage, only works for number values, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+@tparam any key The key that you want to increment the value of, must be a string unless a serializer is set
+@tparam[opt=1] number delta The amount that you want to increment the value by, can be negative or a decimal
+
+@usage-- Increment a value in a datastore, the value must be a number or nil, if nil 0 is used as the start value
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:increment('TestNumber')
+
+]]
+function Datastore:increment(key, delta)
+    key = self:serialize(key)
+    local value = self:raw_get(key) or 0
+    return Datastore:set(key, value + (delta or 1))
+end
+
+local function update_error(err) error('An error ocurred in datastore update: '..err, 2) end
+--[[-- Use a function to update the value locally, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+@tparam any key The key that you want to apply the update to, must be a string unless a serializer is set
+@tparam function callback The function that will be used to update the value at this key
+
+@usage-- Using a function to update a value, if a value is returned then this will be the new value
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:increment('TestKey', function(key, value)
+    return value..value
+end)
+
+]]
+function Datastore:update(key, callback)
+    key = self:serialize(key)
+    local value = self:raw_get(key)
+    local success, new_value = xpcall(callback, update_error, key, value)
+    if success and new_value ~= nil then
+        self:set(key, new_value)
+    else
+        self:raise_event('on_update', key, value)
+        if self.auto_save then self:save(key) end
+    end
+end
+
+--[[-- Remove a value locally and on the external source, works regardless of propagateChanges
+@tparam any key The key that you want to remove locally and externally, must be a string unless a serializer is set
+
+@usage-- Remove a key locally and externally
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:remove('TestKey')
+
+]]
+function Datastore:remove(key)
+    key = self:serialize(key)
+    self:raw_set(key)
+    self:write_action('remove', key)
+    if self.parent and self.parent.auto_save then return self.parent:save(key) end
+end
+
+local function filter_error(err) print('An error ocurred in a datastore filter:', err) end
+--[[-- Internal, Used to filter elements from a table
+@tparam table tbl The table that will have the filter applied to it
+@tparam[opt] function callback The function that will be used as a filter, if none giving then the provided table is returned
+@treturn table The table which has only the key values pairs which passed the filter
+
+@usage-- Internal, Filter a table by the values it contains, return true to keep the key value pair
+local filtered_table = filter({5,3,4,1,2}, function(key, value)
+    return value > 2
+end)
+
+]]
+local function filter(tbl, callback)
+    if not callback then return tbl end
+    local rtn = {}
+    for key, value in pairs(tbl) do
+        local success, add = xpcall(callback, filter_error, key, value)
+        if success and add then rtn[key] = value end
+    end
+    return rtn
+end
+
+--[[-- Get all keys in this datastore, optional filter callback
+@tparam[opt] function callback The filter function that can be used to filter the results returned
+@treturn table All the data that is in this datastore, filtered if a filter was provided
+
+@usage-- Get all the data in this datastore
+local ExampleData = Datastore.connect('ExampleData')
+local data = ExampleData:get_all()
+
+@usage-- Get all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+local data = ExampleData:get_all(function(key, value)
+    return type(value) == 'string'
+end)
+
+]]
+function Datastore:get_all(callback)
+    if not self.parent then
+        return filter(self.data, callback)
+    else
+        local data, value_name = {}, self.value_name
+        for key, value in pairs(self.parent:get_all()) do
+            data[key] = value[value_name]
+        end
+        return filter(data, callback)
+    end
+end
+
+--[[-- Update all keys in this datastore using the same update function
+@tparam function callback The update function that will be applied to each key
+
+@usage-- Get all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:update_all(function(key, value)
+    return value..value
+end)
+
+]]
+function Datastore:update_all(callback)
+    local data = self:get_all()
+    for key, value in pairs(data) do
+        local success, new_value = xpcall(callback, update_error, key, value)
+        if success and new_value ~= nil then
+            self:set(key, new_value)
+        else
+            self:raise_event('on_update', key, value)
+            if self.auto_save then self:save(key) end
+        end
+    end
+end
+
+----- Datastore External
+-- @section datastore-external
+
+--[[-- Request a value from an external source, will trigger on_load when data is received
+@tparam any key The key that you want to request from an external source, must be a string unless a serializer is set
+
+@usage-- Request a key from an external source, on_load is triggered when data is received
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:request('TestKey')
+
+]]
+function Datastore:request(key)
+    if self.parent then return self.parent:request(key) end
+    key = self:serialize(key)
+    self:write_action('request', key)
+end
+
+--[[-- Save a value to an external source, will trigger on_save before data is saved, save_to_disk must be set to true
+@tparam any key The key that you want to save to an external source, must be a string unless a serializer is set
+
+@usage-- Save a key to an external source, save_to_disk must be set to true for there to be any effect
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:save('TestKey')
+
+]]
+function Datastore:save(key)
+    if self.parent then self.parent:save(key) end
+    if not self.save_to_disk then return end
+    key = self:serialize(key)
+    local value  = self:raise_event('on_save', key, copy(self:raw_get(key)))
+    local action = self.propagate_changes and 'propagate' or 'save'
+    self:write_action(action, key, value)
+end
+
+--[[-- Save a value to an external source and remove locally, will trigger on_unload then on_save, save_to_disk is not required for on_unload
+@tparam any key The key that you want to unload from the datastore, must be a string unless a serializer is set
+
+@usage-- Unload a key from the datastore, get will now return nil and value will be saved externally if save_to_disk is set to true
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:unload('TestKey')
+
+]]
+function Datastore:unload(key)
+    if self.parent then return self.parent:unload(key) end
+    key = self:serialize(key)
+    self:raise_event('on_unload', key, copy(self:raw_get(key)))
+    self:save(key)
+    self:raw_set(key)
+end
+
+--[[-- Use to send a message over the connection, works regardless of saveToDisk and propagateChanges
+@tparam any key The key that you want to send a message over, must be a string unless a serializer is set
+@tparam any message The message that you want to send to other connected servers, or external source
+
+@usage-- Send a message to other servers on this key, can listen for messages with on_message
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:message('TestKey', 'Foo')
+
+]]
+function Datastore:message(key, message)
+    key = self:serialize(key)
+    self:write_action('message', key, message)
+end
+
+--[[-- Save all the keys in the datastore, optional filter callback
+@tparam[opt] function callback The filter function that can be used to filter the keys saved
+
+@usage-- Save all the data in this datastore
+local ExampleData = Datastore.connect('ExampleData')
+local data = ExampleData:save_all()
+
+@usage-- Save all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:save_all(function(key, value)
+    return type(value) == 'string'
+end)
+
+]]
+function Datastore:save_all(callback)
+    local data = self:get_all(callback)
+    for key in pairs(data) do self:save(key) end
+end
+
+--[[-- Unload all the keys in the datastore, optional filter callback
+@tparam[opt] function callback The filter function that can be used to filter the keys unloaded
+
+@usage-- Unload all the data in this datastore
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:unload_all()
+
+@usage-- Unload all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:unload_all(function(key, value)
+    return type(value) == 'string'
+end)
+
+]]
+function Datastore:unload_all(callback)
+    local data = self:get_all(callback)
+    for key in pairs(data) do self:unload(key) end
+end
+
+----- Events
+-- @section events
+
+local function event_error(err) print('An error ocurred in a datastore event handler:', err) end
+--[[-- Internal, Raise an event on this datastore
+@tparam string event_name The name of the event to raise for this datastore
+@tparam string key The key that this event is being raised for
+@tparam[opt] any value The current value that this key has, might be a deep copy of the value
+@tparam[opt] string source Where this call came from, used to do event recursion so can be parent or child
+@treturn any The value that is left after being passed through all the event handlers
+
+@usage-- Internal, Getting the value that should be saved
+value = self:raise_event('on_save', key, value)
+
+]]
+function Datastore:raise_event(event_name, key, value, source)
+    -- Raise the event for the children of this datastore
+    if source ~= 'child' then
+        for value_name, child in pairs(self.children) do
+            value[value_name] = child:raise_event(event_name, key, value[value_name], 'parent')
+        end
+    end
+
+    -- Raise the event for this datastore
+    local handlers = self.events[event_name]
+    if handlers then
+        for _, handler in ipairs(handlers) do
+            local success, new_value = xpcall(handler, event_error, key, value)
+            if success and new_value ~= nil then value = new_value end
+        end
+    end
+
+    -- Raise the event for the parent of this datastore
+    if source ~= 'parent' and self.parent then
+        self.parent:raise_event(event_name, key, self.parent:raw_get(key), 'child')
+    end
+    return value
+end
+
+--[[-- Internal, Returns a function which will add a callback to an event
+@tparam string event_name The name of the event that this should create a handler adder for
+@treturn function The function that can be used to add handlers to this event
+
+@usage-- Internal, Get the function to add handlers to on_load
+Datastore.on_load = event_factory('on_load')
+
+]]
+local function event_factory(event_name)
+    return function(self, callback)
+        assert(type(callback) == 'function', 'Handler must be a function')
+        local handlers = self.events[event_name]
+        if not handlers then
+            self.events[event_name] = { callback }
+        else
+            handlers[#handlers+1] = callback
+        end
+    end
+end
+
+--[[-- Register a callback that triggers when data is loaded from an external source, returned value is saved locally
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_load, returned value will be saved locally, can be used to deserialize the value beyond a normal json
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_load(function(key, value)
+    game.print('Test data loaded for: '..key)
+end)
+]]
+Datastore.on_load = event_factory('on_load')
+
+--[[-- Register a callback that triggers before data is saved, returned value is saved externally
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_save, returned value will be saved externally, can be used to serialize the value beyond a normal json
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_save(function(key, value)
+    game.print('Test data saved for: '..key)
+end)
+]]
+Datastore.on_save = event_factory('on_save')
+
+--[[-- Register a callback that triggers before data is unloaded, returned value is ignored
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_unload, returned value is ignored, can be used to clean up guis or local values related to this data
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_load(function(key, value)
+    game.print('Test data unloaded for: '..key)
+end)
+]]
+Datastore.on_unload = event_factory('on_unload')
+
+--[[-- Register a callback that triggers when a message is received, returned value is ignored
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_message, returned value is ignored, can be used to receive messages from other connected servers without saving data
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_message(function(key, value)
+    game.print('Test data message for: '..key)
+end)
+]]
+Datastore.on_message = event_factory('on_message')
+
+--[[-- Register a callback that triggers any time a value is changed, returned value is ignored
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_update, returned value is ignored, can be used to update guis or send messages when data is changed
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_update(function(key, value)
+    game.print('Test data updated for: '..key)
+end)
+]]
+Datastore.on_update = event_factory('on_update')
+
+----- Module Return
+return DatastoreManager

expcore/player_data.lua

@@ -0,0 +1,110 @@
+--[[-- Core Module - PlayerData
+- A module used to store player data in a central datastore to minimize data requests and saves.
+@core PlayerData
+
+@usage-- Adding a colour setting for players
+local PlayerData = require 'expcore.player_data'
+local PlayerColors = PlayerData.Settings:combine('Color')
+
+-- Set the players color when their data is loaded
+PlayerColors:on_load(function(player_name, color)
+    local player = game.players[player_name]
+    player.color = color
+end)
+
+-- Overwrite the saved color with the players current color
+PlayerColors:on_save(function(player_name, _)
+    local player = game.players[player_name]
+    return player.color -- overwrite existing data with the current color
+end)
+
+@usage-- Add a playtime statistic for players
+local Event = require 'utils.event'
+local PlayerData = require 'expcore.player_data'
+local Playtime = PlayerData.Statistics:combine('Playtime')
+
+-- When playtime reaches an hour interval tell the player and say thanks
+Playtime:on_update(function(player_name, playtime)
+    if playtime % 60 == 0 then
+        local hours = playtime / 60
+        local player = game.players[player_name]
+        player.print('Thanks for playing on our servers, you have played for '..hours..' hours!')
+    end
+end)
+
+-- Update playtime for players, data is only loaded for online players so update_all can be used
+Event.add_on_nth_tick(3600, function()
+    Playtime:update_all(function(player_name, playtime)
+        return playtime + 1
+    end)
+end)
+
+]]
+
+local Event = require 'utils.event' --- @dep utils.event
+local Datastore = require 'expcore.datastore' --- @dep expcore.datastore
+local Commands = require 'expcore.commands' --- @dep expcore.commands
+require 'config.expcore.command_general_parse' --- @dep config.expcore.command_general_parse
+
+--- Common player data that acts as the root store for player data
+local PlayerData = Datastore.connect('PlayerData', true) -- saveToDisk
+PlayerData:set_serializer(Datastore.name_serializer) -- use player name
+
+--- Store and enum for the data saving preference
+local DataSavingPreference = PlayerData:combine('DataSavingPreference')
+local PreferenceEnum = { 'All', 'Statistics', 'Settings', 'Required' }
+for k,v in ipairs(PreferenceEnum) do PreferenceEnum[v] = k end
+
+--- Sets your data saving preference
+-- @command set-data-preference
+Commands.new_command('set-data-preference', 'Allows you to set your data saving preference')
+:add_param('option', false, 'string-options', PreferenceEnum)
+:register(function(player, option)
+    DataSavingPreference:set(player, option)
+    return {'expcore-data.set-preference', option}
+end)
+
+--- Gets your data saving preference
+-- @command data-preference
+Commands.new_command('data-preference', 'Shows you what your current data saving preference is')
+:register(function(player)
+    return {'expcore-data.get-preference', DataSavingPreference:get(player, 'All')}
+end)
+
+--- Remove data that the player doesnt want to have stored
+PlayerData:on_save(function(player_name, player_data)
+    local dataPreference = DataSavingPreference:get(player_name, 'All')
+    dataPreference = PreferenceEnum[dataPreference]
+    if dataPreference == PreferenceEnum.All then return player_data end
+
+    local saved_player_data = { PlayerRequired = player_data.PlayerRequired, DataSavingPreference = PreferenceEnum[dataPreference] }
+    if dataPreference <= PreferenceEnum.Settings then saved_player_data.PlayerSettings = player_data.PlayerSettings end
+    if dataPreference <= PreferenceEnum.Statistics then saved_player_data.PlayerStatistics = player_data.PlayerStatistics end
+
+    return saved_player_data
+end)
+
+--- Display your data preference when your data loads
+DataSavingPreference:on_load(function(player_name, dataPreference)
+    game.players[player_name].print{'expcore-data.get-preference', dataPreference or 'All'}
+end)
+
+--- Load player data when they join
+Event.add(defines.events.on_player_joined_game, function(event)
+    PlayerData:request(game.players[event.player_index])
+end)
+
+--- Unload player data when they leave
+Event.add(defines.events.on_player_left_game, function(event)
+    PlayerData:unload(game.players[event.player_index])
+end)
+
+----- Module Return -----
+return {
+    All = PlayerData, -- Root for all of a players data
+    Statistics = PlayerData:combine('Statistics'), -- Common place for stats
+    Settings = PlayerData:combine('Settings'), -- Common place for settings
+    Required = PlayerData:combine('Required'), -- Common place for required data
+    DataSavingPreference = DataSavingPreference, -- Stores what data groups will be saved
+    PreferenceEnum = PreferenceEnum -- Enum for the allowed options for data saving preference
+}

locale/en/expcore.cfg

@@ -36,3 +36,7 @@ button_tooltip=Shows/hides the toolbar.
 
 [expcore-gui]
 left-button-tooltip=Hide all open windows.
+
+[expcore-data]
+set-preference=You data saving preference has been set to __1__. Existing data will not be effected until you rejoin.
+get-preference=You data saving preference is __1__. Use /set-data-preference to change this. 

modules/commands/interface.lua

@@ -17,7 +17,8 @@ local interface_modules = {
     ['Roles']='expcore.roles',
     ['Store']='expcore.store',
     ['Gui']='expcore.gui',
-    ['Async']='expcore.async'
+    ['Async']='expcore.async',
+    ['Datastore']='expcore.datastore'
 }
 
 -- loads all the modules given in the above table

modules/gui/debug/expcore_datastore_view.lua

@@ -0,0 +1,130 @@
+local Gui = require 'utils.gui' --- @dep utils.gui
+local Datastore = require 'expcore.datastore' --- @dep expcore.datastore
+local Color = require 'utils.color_presets' --- @dep utils.color_presets
+local Model = require 'modules.gui.debug.model' --- @dep modules.gui.debug.model
+
+local dump = Model.dump
+local concat = table.concat
+
+local Public = {}
+
+local header_name = Gui.uid_name()
+local left_panel_name = Gui.uid_name()
+local right_panel_name = Gui.uid_name()
+local input_text_box_name = Gui.uid_name()
+local refresh_name = Gui.uid_name()
+
+Public.name = 'Datastore'
+
+function Public.show(container)
+    local main_flow = container.add {type = 'flow', direction = 'horizontal'}
+
+    local left_panel = main_flow.add {type = 'scroll-pane', name = left_panel_name}
+    local left_panel_style = left_panel.style
+    left_panel_style.width = 300
+
+    for name in pairs(table.keysort(Datastore.debug())) do
+        local header = left_panel.add({type = 'flow'}).add {type = 'label', name = header_name, caption = name}
+        Gui.set_data(header, name)
+    end
+
+    local right_flow = main_flow.add {type = 'flow', direction = 'vertical'}
+
+    local right_top_flow = right_flow.add {type = 'flow', direction = 'horizontal'}
+
+    local input_text_box = right_top_flow.add {type = 'text-box', name = input_text_box_name}
+    local input_text_box_style = input_text_box.style
+    input_text_box_style.horizontally_stretchable = true
+    input_text_box_style.height = 32
+    input_text_box_style.maximal_width = 1000
+
+    local refresh_button =
+        right_top_flow.add {type = 'sprite-button', name = refresh_name, sprite = 'utility/reset', tooltip = 'refresh'}
+    local refresh_button_style = refresh_button.style
+    refresh_button_style.width = 32
+    refresh_button_style.height = 32
+
+    local right_panel = right_flow.add {type = 'text-box', name = right_panel_name}
+    right_panel.read_only = true
+    right_panel.selectable = true
+
+    local right_panel_style = right_panel.style
+    right_panel_style.vertically_stretchable = true
+    right_panel_style.horizontally_stretchable = true
+    right_panel_style.maximal_width = 1000
+    right_panel_style.maximal_height = 1000
+
+    local data = {
+        right_panel = right_panel,
+        input_text_box = input_text_box,
+        selected_header = nil
+    }
+
+    Gui.set_data(input_text_box, data)
+    Gui.set_data(left_panel, data)
+    Gui.set_data(refresh_button, data)
+end
+
+Gui.on_click(
+    header_name,
+    function(event)
+        local element = event.element
+        local tableName = Gui.get_data(element)
+
+        local left_panel = element.parent.parent
+        local data = Gui.get_data(left_panel)
+        local right_panel = data.right_panel
+        local selected_header = data.selected_header
+        local input_text_box = data.input_text_box
+
+        if selected_header then
+            selected_header.style.font_color = Color.white
+        end
+
+        element.style.font_color = Color.orange
+        data.selected_header = element
+
+        input_text_box.text = tableName
+        input_text_box.style.font_color = Color.black
+
+        local content = Datastore.debug(tableName)
+        local content_string = {}
+        for key, value in pairs(content) do
+            content_string[#content_string+1] = key:gsub('^%l', string.upper)..' = '..dump(value)
+        end
+        right_panel.text = concat(content_string, '\n')
+    end
+)
+
+local function update_dump(text_input, data)
+    local content = Datastore.debug(text_input.text)
+    local content_string = {}
+    for key, value in pairs(content) do
+        content_string[#content_string+1] = key:gsub('^%l', string.upper)..' = '..dump(value)
+    end
+    data.right_panel.text = concat(content_string, '\n')
+end
+
+Gui.on_text_changed(
+    input_text_box_name,
+    function(event)
+        local element = event.element
+        local data = Gui.get_data(element)
+
+        update_dump(element, data)
+    end
+)
+
+Gui.on_click(
+    refresh_name,
+    function(event)
+        local element = event.element
+        local data = Gui.get_data(element)
+
+        local input_text_box = data.input_text_box
+
+        update_dump(input_text_box, data)
+    end
+)
+
+return Public

modules/gui/debug/global_view.lua

@@ -8,7 +8,7 @@ local concat = table.concat
 
 local Public = {}
 
-local ignore = {tokens = true, data_store = true}
+local ignore = {tokens = true, data_store = true, datastores = true}
 
 local header_name = Gui.uid_name()
 local left_panel_name = Gui.uid_name()

modules/gui/debug/main_view.lua

@@ -5,6 +5,7 @@ local Public = {}
 
 local pages = {
     require 'modules.gui.debug.redmew_global_view',
+    require 'modules.gui.debug.expcore_datastore_view',
     require 'modules.gui.debug.expcore_store_view',
     require 'modules.gui.debug.expcore_gui_view',
     require 'modules.gui.debug.global_view',