From d926100c2daca4ab31064a36f0fe1c1c5b631297 Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Thu, 21 May 2020 16:48:31 +0100
Subject: [PATCH 01/10] Module Layout

---
 expcore/datastore.lua | 131 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)
 create mode 100644 expcore/datastore.lua

diff --git a/expcore/datastore.lua b/expcore/datastore.lua
new file mode 100644
index 00000000..371e76f6
--- /dev/null
+++ b/expcore/datastore.lua
@@ -0,0 +1,131 @@
+
+local Event = require 'utils.event' --- @dep utils.event
+
+local DatastoreManager = {}
+local Datastores = {}
+local Datastore = {}
+
+--- Save datastores in the global table
+global.datastores = Datastores
+Event.on_load(function()
+    Datastores = global.datastores
+end)
+
+----- Datastore Manager -----
+
+--- Make a new datastore
+function DatastoreManager.connect(tableName, saveToDisk, propagateChanges)
+
+end
+
+--- Make a new datastore that is contained within another
+function DatastoreManager.combine(datastore, subTableName)
+
+end
+
+--- Ingest the result from a request
+function DatastoreManager.ingest(action, tableName, key, valueJson)
+
+end
+
+--- Commonly used serializer, returns the objects name
+function DatastoreManager.name_serializer(rawKey)
+
+end
+
+----- Datastore -----
+
+--- Request a value from an external source
+function Datastore:request(key)
+
+end
+
+--- Save a value to an external source
+function Datastore:save(key)
+
+end
+
+--- Save a value to an external source and remove locally
+function Datastore:unload(key)
+
+end
+
+--- Remove a value locally and on the external source
+function Datastore:remove(key)
+
+end
+
+--- Get a value from local storage
+function Datastore:get(key, default)
+
+end
+
+--- Set a value in local storage
+function Datastore:set(key, value)
+
+end
+
+--- Increment the value in local storage, only works for number values
+function Datastore:increment(key, delta)
+
+end
+
+--- Use a callback function to update the value locally
+function Datastore:update(key, callback)
+
+end
+
+--- Use to send a message over the connection, works regardless of saveToDisk and propagateChanges
+function Datastore:message(key, message)
+
+end
+
+--- Get all keys in the datastore, optional filter callback
+function Datastore:get_all(callback)
+
+end
+
+--- Save all the keys in the datastore, optional filter callback
+function Datastore:save_all(callback)
+
+end
+
+--- Unload all the keys in the datastore, optional filter callback
+function Datastore:unload_all(callback)
+
+end
+
+--- Set a callback that will be used to serialize keys which aren't strings
+function Datastore:set_serializer(callback)
+
+end
+
+----- Events -----
+
+--- Register a callback that triggers only when data is received
+function Datastore:on_received(callback)
+
+end
+
+--- Register a callback that triggers before data is saved
+function Datastore:on_save(callback)
+
+end
+
+--- Register a callback that triggers before data is unloaded
+function Datastore:on_unload(callback)
+
+end
+
+--- Register a callback that triggers when a message is received
+function Datastore:on_message(callback)
+
+end
+
+--- Register a callback that triggers any time a value is changed
+function Datastore:on_update(callback)
+
+end
+
+----- Module Return -----
+return DatastoreManager
\ No newline at end of file

From 07caa9c4a22a87a7cc456e8273863fbe3e98b13d Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Thu, 21 May 2020 20:47:56 +0100
Subject: [PATCH 02/10] Added datastore

---
 expcore/datastore.lua          | 267 +++++++++++++++++++++++++++------
 modules/commands/interface.lua |   3 +-
 2 files changed, 219 insertions(+), 51 deletions(-)

diff --git a/expcore/datastore.lua b/expcore/datastore.lua
index 371e76f6..0c6c22c9 100644
--- a/expcore/datastore.lua
+++ b/expcore/datastore.lua
@@ -9,123 +9,290 @@ local Datastore = {}
 global.datastores = Datastores
 Event.on_load(function()
     Datastores = global.datastores
+    for _, datastore in pairs(Datastores) do
+        setmetatable(datastore, DatastoreManager.metatable)
+    end
 end)
 
 ----- Datastore Manager -----
+-- @section datastoreManager
+
+--- Metatable used on datastores
+DatastoreManager.metatable = {
+    __newidnex = function(_, _, _) error('Datastore can not be modified', 2) end,
+    __call = function(self, ...) return self:get(...) end,
+    __index = Datastore
+}
 
 --- Make a new datastore
-function DatastoreManager.connect(tableName, saveToDisk, propagateChanges)
+function DatastoreManager.connect(tableName, saveToDisk, autoSave, propagateChanges)
+    if Datastores[tableName] then return Datastores[tableName] end
 
+    local new_datastore = {
+        name = tableName,
+        auto_save = autoSave or false,
+        save_to_disk = saveToDisk or false,
+        propagate_changes = propagateChanges or false,
+        serializer = false,
+        combined = false,
+        events = {},
+        data = {}
+    }
+
+    Datastores[tableName] = new_datastore
+    return setmetatable(new_datastore, DatastoreManager.metatable)
 end
 
 --- Make a new datastore that is contained within another
 function DatastoreManager.combine(datastore, subTableName)
-
+    local new_datastore = DatastoreManager.connect(subTableName)
+    new_datastore.serializer = datastore.serializer
+    new_datastore.auto_save = datastore.auto_save
+    new_datastore.combined = datastore
+    return new_datastore
 end
 
 --- Ingest the result from a request
+local function ingest_error(err) print('Datastore ingest error, Unable to parse json:', err) end
 function DatastoreManager.ingest(action, tableName, key, valueJson)
+    local datastore = assert(Datastores[tableName], 'Datastore ingest error, Datastore not found '..tostring(tableName))
+    assert(type(action) == 'string', 'Datastore ingest error, Action is not a string got: '..type(action))
+    assert(type(key) == 'string', 'Datastore ingest error, Key is not a string got: '..type(key))
+
+    if action == 'remove' then
+        datastore:raw_set(key)
+
+    elseif action == 'message' then
+        local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
+        if not success or value == nil then return end
+        datastore:raise_event('on_message', key, value)
+
+    elseif action == 'propagate' then
+        local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
+        if not success or value == nil then return end
+        value = datastore:raise_event('on_received', key, value)
+        datastore:set(key, value)
+
+    end
 
 end
 
 --- Commonly used serializer, returns the objects name
 function DatastoreManager.name_serializer(rawKey)
-
+    return rawKey.name
 end
 
 ----- Datastore -----
+-- @section datastore
+
+--- Internal, Get the data following combine logic
+function Datastore:raw_get(key, isTable)
+    if self.combined then
+        local data = self.combined:raw_get(key, true)
+        if data[self.name] == nil and isTable then
+            data[self.name] = {}
+        end
+        return data[self.name]
+    else
+        if self.data[key] == nil and isTable then
+            self.data[key] = {}
+        end
+        return self.data[key]
+    end
+end
+
+--- Internal, Set the data following combine logic
+function Datastore:raw_set(key, value)
+    if self.combined then
+        local data = self.combined:raw_get(key, true)
+        data[self.name] = value
+    else
+        self.data[key] = value
+    end
+end
+
+--- Internal, return the serialized key
+local function serialize_error(err) error('An error ocurred in a datastore serializer: '..err) end
+function Datastore:serialize(rawKey)
+    if type(rawKey) == 'string' then return rawKey end
+    assert(self.serializer, 'Datastore does not have a serializer and received non string key')
+    local success, key = xpcall(self.serializer, serialize_error, rawKey)
+    return success and key or nil
+end
+
+--- Internal, writes an event to the output file to be saved and/or propagated
+function Datastore:write_action(action, key, value)
+    local data = {action, self.name, '"'..key..'"'}
+    if value ~= nil then
+        data[4] = type(value) == 'table' and '"'..game.table_to_json(value)..'"' or '"'..tostring(value)..'"'
+    end
+    game.write_file('datastore.pipe', table.concat(data, ' ')..'\n', true, 0)
+end
 
 --- Request a value from an external source
 function Datastore:request(key)
-
+    if self.combined then return self.combined:request(key) end
+    key = self:serialize(key)
+    self:write_action('request', key)
 end
 
 --- Save a value to an external source
 function Datastore:save(key)
-
+    if self.combined then return self.combined:save(key) end
+    if not self.save_to_disk then return end
+    key = self:serialize(key)
+    local value = self:raw_get(key)
+    value = self:raise_event('on_save', key, value)
+    local action = self.propagateChanges and 'propagate' or 'save'
+    self:write_action(action, key, value)
 end
 
 --- Save a value to an external source and remove locally
 function Datastore:unload(key)
-
-end
-
---- Remove a value locally and on the external source
-function Datastore:remove(key)
-
-end
-
---- Get a value from local storage
-function Datastore:get(key, default)
-
-end
-
---- Set a value in local storage
-function Datastore:set(key, value)
-
-end
-
---- Increment the value in local storage, only works for number values
-function Datastore:increment(key, delta)
-
-end
-
---- Use a callback function to update the value locally
-function Datastore:update(key, callback)
-
+    if self.combined then return self.combined:unload(key) end
+    key = self:serialize(key)
+    self:save(key)
+    self:raw_set(key)
 end
 
 --- Use to send a message over the connection, works regardless of saveToDisk and propagateChanges
 function Datastore:message(key, message)
+    key = self:serialize(key)
+    self:write_action('message', key, message)
+end
 
+--- Remove a value locally and on the external source, works regardless of propagateChanges
+function Datastore:remove(key)
+    key = self:serialize(key)
+    self:raw_set(key)
+    self:write_action('remove', key)
+    if self.combined and self.combined.auto_save then return self.combined:save(key) end
+end
+
+--- Get a value from local storage
+function Datastore:get(key, default)
+    key = self:serialize(key)
+    local value = self:raw_get(key)
+    if value ~= nil then return value end
+    return table.deep_copy(default)
+end
+
+--- Set a value in local storage
+function Datastore:set(key, value)
+    key = self:serialize(key)
+    self:raw_set(key, value)
+    self:raise_event('on_update', key, value)
+    if self.auto_save then self:save(key) end
+    return value
+end
+
+--- Increment the value in local storage, only works for number values
+function Datastore:increment(key, delta)
+    key = self:serialize(key)
+    local value = self:raw_get(key) or 0
+    return Datastore:set(key, value + (delta or 1))
+end
+
+--- Use a callback function to update the value locally
+local function update_error(err) error('An error ocurred in datastore update: '..err, 2) end
+function Datastore:update(key, callback)
+    key = self:serialize(key)
+    local value = self:raw_get(key)
+    local success, new_value = xpcall(callback, update_error, key, value)
+    if success and new_value ~= nil then
+        self:set(key, new_value)
+    else
+        self:raise_event('on_update', key, value)
+        if self.auto_save then self:save(key) end
+    end
+end
+
+--- Used to filter elements from a table
+local function filter_error(err) print('An error ocurred in a datastore filter:', err) end
+local function filter(tbl, callback)
+    if not callback then return tbl end
+    local rtn = {}
+    for key, value in pairs(tbl) do
+        local success, add = xpcall(callback, filter_error, key, value)
+        if success and add then rtn[key] = value end
+    end
+    return rtn
 end
 
 --- Get all keys in the datastore, optional filter callback
 function Datastore:get_all(callback)
-
+    if not self.combined then
+        return filter(self.data, callback)
+    else
+        local name = self.name
+        local data = self.combined:get_all()
+        for key, value in pairs(data) do
+            data[key] = value[name]
+        end
+        return filter(data, callback)
+    end
 end
 
 --- Save all the keys in the datastore, optional filter callback
 function Datastore:save_all(callback)
-
+    local data = self:get_all(callback)
+    for key in pairs(data) do self:save(key) end
 end
 
 --- Unload all the keys in the datastore, optional filter callback
 function Datastore:unload_all(callback)
-
+    local data = self:get_all(callback)
+    for key in pairs(data) do self:unload(key) end
 end
 
 --- Set a callback that will be used to serialize keys which aren't strings
 function Datastore:set_serializer(callback)
-
+    assert(type(callback) == 'function', 'Callback must be a function')
+    self.serializer = callback
 end
 
 ----- Events -----
+-- @section events
+
+--- Raise a custom event on this datastore
+local function event_error(err) print('An error ocurred in a datastore event handler:', err) end
+function Datastore:raise_event(event_name, key, value)
+    local handlers = self.events[event_name]
+    if not handlers then return value end
+    for _, handler in ipairs(handlers) do
+        local success, new_value = xpcall(handler, event_error, key, value)
+        if success and new_value ~= nil then value = new_value end
+    end
+    return value
+end
+
+--- Returns a function which will add a callback to an event
+local function event_factory(event_name)
+    return function(self, callback)
+        assert(type(callback) == 'function', 'Handler must be a function')
+        local handlers = self.events[event_name]
+        if not handlers then
+            self.events[event_name] = { callback }
+        else
+            handlers[#handlers+1] = callback
+        end
+    end
+end
 
 --- Register a callback that triggers only when data is received
-function Datastore:on_received(callback)
-
-end
+Datastore.on_received = event_factory('on_received')
 
 --- Register a callback that triggers before data is saved
-function Datastore:on_save(callback)
-
-end
+Datastore.on_save = event_factory('on_save')
 
 --- Register a callback that triggers before data is unloaded
-function Datastore:on_unload(callback)
-
-end
+Datastore.on_unload = event_factory('on_unload')
 
 --- Register a callback that triggers when a message is received
-function Datastore:on_message(callback)
-
-end
+Datastore.on_message = event_factory('on_message')
 
 --- Register a callback that triggers any time a value is changed
-function Datastore:on_update(callback)
-
-end
+Datastore.on_update = event_factory('on_update')
 
 ----- Module Return -----
 return DatastoreManager
\ No newline at end of file
diff --git a/modules/commands/interface.lua b/modules/commands/interface.lua
index a826d286..f5c8b12f 100644
--- a/modules/commands/interface.lua
+++ b/modules/commands/interface.lua
@@ -17,7 +17,8 @@ local interface_modules = {
     ['Roles']='expcore.roles',
     ['Store']='expcore.store',
     ['Gui']='expcore.gui',
-    ['Async']='expcore.async'
+    ['Async']='expcore.async',
+    ['Datastore']='expcore.datastore'
 }
 
 -- loads all the modules given in the above table

From 29fee79feaa08f6043d4308527d2ac91c3f362cc Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Thu, 21 May 2020 22:17:02 +0100
Subject: [PATCH 03/10] Added player data

---
 config/_file_loader.lua                  | 13 +++--
 config/expcore/command_general_parse.lua |  6 +--
 config/expcore/roles.lua                 |  2 +
 expcore/datastore.lua                    | 23 +++++----
 expcore/playerdata.lua                   | 63 ++++++++++++++++++++++++
 locale/en/expcore.cfg                    |  4 ++
 6 files changed, 95 insertions(+), 16 deletions(-)
 create mode 100644 expcore/playerdata.lua

diff --git a/config/_file_loader.lua b/config/_file_loader.lua
index c6ea3ff9..bb45f8e7 100644
--- a/config/_file_loader.lua
+++ b/config/_file_loader.lua
@@ -6,7 +6,9 @@
 return {
     --'example.file_not_loaded',
     'modules.factorio-control', -- base factorio free play scenario
-    -- Game Commands
+    'expcore.playerdata',
+
+    --- Game Commands
     'modules.commands.me',
     'modules.commands.kill',
     'modules.commands.admin-chat',
@@ -28,7 +30,8 @@ return {
     'modules.commands.bonus',
     'modules.commands.home',
     'modules.commands.quickbar',
-    -- QoL Addons
+
+    --- Addons
     'modules.addons.station-auto-name',
     'modules.addons.greetings',
     'modules.addons.chat-popups',
@@ -43,7 +46,8 @@ return {
     'modules.addons.discord-alerts',
     'modules.addons.chat-reply',
     'modules.addons.tree-decon',
-    -- GUI
+
+    --- GUI
     'modules.gui.readme',
     'modules.gui.rocket-info',
     'modules.gui.science-info',
@@ -52,7 +56,8 @@ return {
     'modules.gui.player-list',
     'modules.gui.server-ups',
     'modules.commands.debug',
-    -- Config Files
+
+    --- Config Files
     'config.expcore.command_auth_admin', -- commands tagged with admin_only are blocked for non admins
     'config.expcore.command_auth_roles', -- commands must be allowed via the role config
     'config.expcore.command_runtime_disable', -- allows commands to be enabled and disabled during runtime
diff --git a/config/expcore/command_general_parse.lua b/config/expcore/command_general_parse.lua
index 7b9e282a..9cbf9a72 100644
--- a/config/expcore/command_general_parse.lua
+++ b/config/expcore/command_general_parse.lua
@@ -40,12 +40,12 @@ end)
 Commands.add_parse('string-options',function(input,player,reject,options)
     if not input then return end -- nil check
     input = input:lower()
-    for option in options do
+    for _, option in ipairs(options) do
         if input == option:lower() then
-            return true
+            return option
         end
     end
-    return reject{'reject-string-options',options:concat(', ')}
+    return reject{'expcore-commands.reject-string-options', table.concat(options, ', ')}
 end)
 
 Commands.add_parse('string-max-length',function(input,player,reject,max_length)
diff --git a/config/expcore/roles.lua b/config/expcore/roles.lua
index 7d1fd4e1..578e2a8e 100644
--- a/config/expcore/roles.lua
+++ b/config/expcore/roles.lua
@@ -220,6 +220,8 @@ local default = Roles.new_role('Guest','')
     'command/report',
     'command/ratio',
     'command/server-ups',
+    'command/data-policy',
+    'command/set-data-policy',
     'gui/player-list',
     'gui/rocket-info',
     'gui/science-info',
diff --git a/expcore/datastore.lua b/expcore/datastore.lua
index 0c6c22c9..863933a1 100644
--- a/expcore/datastore.lua
+++ b/expcore/datastore.lua
@@ -4,6 +4,7 @@ local Event = require 'utils.event' --- @dep utils.event
 local DatastoreManager = {}
 local Datastores = {}
 local Datastore = {}
+local copy = table.deep_copy
 
 --- Save datastores in the global table
 global.datastores = Datastores
@@ -129,6 +130,15 @@ function Datastore:write_action(action, key, value)
     game.write_file('datastore.pipe', table.concat(data, ' ')..'\n', true, 0)
 end
 
+--- Set a callback that will be used to serialize keys which aren't strings
+function Datastore:set_serializer(callback)
+    assert(type(callback) == 'function', 'Callback must be a function')
+    self.serializer = callback
+end
+
+--- Create a new datastore which is combined into this one
+Datastore.combine = DatastoreManager.combine
+
 --- Request a value from an external source
 function Datastore:request(key)
     if self.combined then return self.combined:request(key) end
@@ -138,11 +148,11 @@ end
 
 --- Save a value to an external source
 function Datastore:save(key)
-    if self.combined then return self.combined:save(key) end
+    if self.combined then self.combined:save(key) end
     if not self.save_to_disk then return end
     key = self:serialize(key)
     local value = self:raw_get(key)
-    value = self:raise_event('on_save', key, value)
+    value = self:raise_event('on_save', key, copy(value))
     local action = self.propagateChanges and 'propagate' or 'save'
     self:write_action(action, key, value)
 end
@@ -151,6 +161,7 @@ end
 function Datastore:unload(key)
     if self.combined then return self.combined:unload(key) end
     key = self:serialize(key)
+    self:raise_event('on_unload', key, copy(self:raw_get(key)))
     self:save(key)
     self:raw_set(key)
 end
@@ -174,7 +185,7 @@ function Datastore:get(key, default)
     key = self:serialize(key)
     local value = self:raw_get(key)
     if value ~= nil then return value end
-    return table.deep_copy(default)
+    return copy(default)
 end
 
 --- Set a value in local storage
@@ -245,12 +256,6 @@ function Datastore:unload_all(callback)
     for key in pairs(data) do self:unload(key) end
 end
 
---- Set a callback that will be used to serialize keys which aren't strings
-function Datastore:set_serializer(callback)
-    assert(type(callback) == 'function', 'Callback must be a function')
-    self.serializer = callback
-end
-
 ----- Events -----
 -- @section events
 
diff --git a/expcore/playerdata.lua b/expcore/playerdata.lua
new file mode 100644
index 00000000..f400f4ca
--- /dev/null
+++ b/expcore/playerdata.lua
@@ -0,0 +1,63 @@
+
+local Event = require 'utils.event' --- @dep utils.event
+local Datastore = require 'expcore.datastore' --- @dep expcore.datastore
+local Commands = require 'expcore.commands' --- @dep expcore.commands
+require 'config.expcore.command_general_parse' --- @dep config.expcore.command_general_parse
+
+--- Common player data that acts as the root store for player data
+local PlayerData = Datastore.connect('PlayerData', true) -- saveToDisk
+PlayerData:set_serializer(Datastore.name_serializer) -- use player name
+
+--- Store and enum for the data collection policy
+local DataCollectionPolicy = PlayerData:combine('DataCollectionPolicy')
+local PolicyEnum = { 'All', 'Tracking', 'Settings', 'Required' }
+for k,v in ipairs(PolicyEnum) do PolicyEnum[v] = k end
+
+--- Sets your data collection policy
+-- @command set-data-policy
+Commands.new_command('set-data-policy', 'Allows you to set your data collection policy')
+:add_param('option', false, 'string-options', PolicyEnum)
+:register(function(player, option)
+    DataCollectionPolicy:set(player, option)
+    return {'expcore-data.set-policy', option}
+end)
+
+--- Gets your data collection policy
+-- @command data-policy
+Commands.new_command('data-policy', 'Shows you what your current data collection policy is')
+:register(function(player)
+    return {'expcore-data.get-policy', DataCollectionPolicy:get(player, 'All')}
+end)
+
+--- Remove data that the player doesnt want to have stored
+PlayerData:on_save(function(player_name, player_data)
+    local collectData = DataCollectionPolicy:get(player_name, 'All')
+    collectData = PolicyEnum[collectData]
+    if collectData == PolicyEnum.All then return player_data end
+
+    local saved_player_data = { PlayerRequired = player_data.PlayerRequired, DataCollectionPolicy = PolicyEnum[collectData] }
+    if collectData <= PolicyEnum.Settings then saved_player_data.PlayerSettings = player_data.PlayerSettings end
+    if collectData <= PolicyEnum.Tracking then saved_player_data.PlayerTracking = player_data.PlayerTracking end
+
+    return saved_player_data
+end)
+
+--- Load player data when they join
+Event.add(defines.events.on_player_joined_game, function(event)
+    PlayerData:request(game.players[event.player_index])
+end)
+
+--- Unload player data when they leave
+Event.add(defines.events.on_player_left_game, function(event)
+    PlayerData:unload(game.players[event.player_index])
+end)
+
+----- Module Return -----
+return {
+    All = PlayerData, -- Root for all of a players data
+    Tracking = PlayerData:combine('PlayerTracking'), -- Common place for tracing stats
+    Settings = PlayerData:combine('PlayerSettings'), -- Common place for settings
+    Required = PlayerData:combine('PlayerRequired'), -- Common place for required data
+    DataCollectionPolicy = DataCollectionPolicy, -- Stores what data groups will be saved
+    PolicyEnum = PolicyEnum -- Enum for the allowed options for the data collection policy
+}
\ No newline at end of file
diff --git a/locale/en/expcore.cfg b/locale/en/expcore.cfg
index ce4d3d45..ea0a5d0c 100644
--- a/locale/en/expcore.cfg
+++ b/locale/en/expcore.cfg
@@ -36,3 +36,7 @@ button_tooltip=Shows/hides the toolbar.
 
 [expcore-gui]
 left-button-tooltip=Hide all open windows.
+
+[expcore-data]
+set-policy=You data collection policy has been set to __1__. Existing data will not be effected until you rejoin.
+get-policy=You data collection policy is __1__. Use /set-data-policy to change this. 
\ No newline at end of file

From df2a79780914a2631884b3e0a7809cc12d005def Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Thu, 21 May 2020 22:31:34 +0100
Subject: [PATCH 04/10] Renamed to player_data

---
 expcore/{playerdata.lua => player_data.lua} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename expcore/{playerdata.lua => player_data.lua} (100%)

diff --git a/expcore/playerdata.lua b/expcore/player_data.lua
similarity index 100%
rename from expcore/playerdata.lua
rename to expcore/player_data.lua

From d283acca3bfc62d4cf1cd0d9f6469a6eafff8eb9 Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Thu, 21 May 2020 22:32:10 +0100
Subject: [PATCH 05/10] Removed player data from file loader

---
 config/_file_loader.lua | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config/_file_loader.lua b/config/_file_loader.lua
index bb45f8e7..acab613a 100644
--- a/config/_file_loader.lua
+++ b/config/_file_loader.lua
@@ -6,7 +6,6 @@
 return {
     --'example.file_not_loaded',
     'modules.factorio-control', -- base factorio free play scenario
-    'expcore.playerdata',
 
     --- Game Commands
     'modules.commands.me',

From 537980cda430babfeca7011c94215e7571ec041e Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Fri, 22 May 2020 20:18:00 +0100
Subject: [PATCH 06/10] Few bug fixes for datastore

---
 config/_file_loader.lua  |   1 +
 config/expcore/roles.lua |   4 +-
 expcore/datastore.lua    | 164 +++++++++++++++++++++++----------------
 expcore/player_data.lua  |  51 ++++++------
 locale/en/expcore.cfg    |   4 +-
 5 files changed, 131 insertions(+), 93 deletions(-)

diff --git a/config/_file_loader.lua b/config/_file_loader.lua
index acab613a..ce035ac0 100644
--- a/config/_file_loader.lua
+++ b/config/_file_loader.lua
@@ -6,6 +6,7 @@
 return {
     --'example.file_not_loaded',
     'modules.factorio-control', -- base factorio free play scenario
+    'expcore.player_data',
 
     --- Game Commands
     'modules.commands.me',
diff --git a/config/expcore/roles.lua b/config/expcore/roles.lua
index 578e2a8e..4d9f9dad 100644
--- a/config/expcore/roles.lua
+++ b/config/expcore/roles.lua
@@ -220,8 +220,8 @@ local default = Roles.new_role('Guest','')
     'command/report',
     'command/ratio',
     'command/server-ups',
-    'command/data-policy',
-    'command/set-data-policy',
+    'command/data-preference',
+    'command/set-data-preference',
     'gui/player-list',
     'gui/rocket-info',
     'gui/science-info',
diff --git a/expcore/datastore.lua b/expcore/datastore.lua
index 863933a1..31f65189 100644
--- a/expcore/datastore.lua
+++ b/expcore/datastore.lua
@@ -4,14 +4,15 @@ local Event = require 'utils.event' --- @dep utils.event
 local DatastoreManager = {}
 local Datastores = {}
 local Datastore = {}
+local Data = {}
 local copy = table.deep_copy
 
 --- Save datastores in the global table
-global.datastores = Datastores
+global.datastores = Data
 Event.on_load(function()
-    Datastores = global.datastores
-    for _, datastore in pairs(Datastores) do
-        setmetatable(datastore, DatastoreManager.metatable)
+    Data = global.datastores
+    for tableName, datastore in pairs(Datastores) do
+        datastore.data = Data[tableName]
     end
 end)
 
@@ -20,40 +21,52 @@ end)
 
 --- Metatable used on datastores
 DatastoreManager.metatable = {
+    __index = function(self, key) return rawget(self.children, key) or rawget(Datastore, key) end,
     __newidnex = function(_, _, _) error('Datastore can not be modified', 2) end,
-    __call = function(self, ...) return self:get(...) end,
-    __index = Datastore
+    __call = function(self, ...) return self:get(...) end
 }
 
---- Make a new datastore
+--- Make a new datastore connection, if a connection already exists then it is returned
 function DatastoreManager.connect(tableName, saveToDisk, autoSave, propagateChanges)
     if Datastores[tableName] then return Datastores[tableName] end
+    if _LIFECYCLE ~= _STAGE.control then
+        -- Only allow this function to be called during the control stage
+        error('New datastore connection can not be created during runtime', 2)
+    end
 
     local new_datastore = {
         name = tableName,
+        table_name = tableName,
         auto_save = autoSave or false,
         save_to_disk = saveToDisk or false,
         propagate_changes = propagateChanges or false,
         serializer = false,
-        combined = false,
+        parent = false,
+        children = {},
+        metadata = {},
         events = {},
         data = {}
     }
 
+    Data[tableName] = new_datastore.data
     Datastores[tableName] = new_datastore
     return setmetatable(new_datastore, DatastoreManager.metatable)
 end
 
---- Make a new datastore that is contained within another
+--- Make a new datastore that stores its data inside of another one
 function DatastoreManager.combine(datastore, subTableName)
-    local new_datastore = DatastoreManager.connect(subTableName)
+    local new_datastore = DatastoreManager.connect(datastore.name..'.'..subTableName)
+    datastore.children[subTableName] = new_datastore
     new_datastore.serializer = datastore.serializer
     new_datastore.auto_save = datastore.auto_save
-    new_datastore.combined = datastore
+    new_datastore.table_name = subTableName
+    new_datastore.parent = datastore
+    Data[new_datastore.name] = nil
+    new_datastore.data = nil
     return new_datastore
 end
 
---- Ingest the result from a request
+--- Ingest the result from a request, this is used through a rcon interface to sync data
 local function ingest_error(err) print('Datastore ingest error, Unable to parse json:', err) end
 function DatastoreManager.ingest(action, tableName, key, valueJson)
     local datastore = assert(Datastores[tableName], 'Datastore ingest error, Datastore not found '..tostring(tableName))
@@ -71,14 +84,14 @@ function DatastoreManager.ingest(action, tableName, key, valueJson)
     elseif action == 'propagate' then
         local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
         if not success or value == nil then return end
-        value = datastore:raise_event('on_received', key, value)
+        value = datastore:raise_event('on_load', key, value)
         datastore:set(key, value)
 
     end
 
 end
 
---- Commonly used serializer, returns the objects name
+--- Commonly used serializer, returns the name of the object
 function DatastoreManager.name_serializer(rawKey)
     return rawKey.name
 end
@@ -86,33 +99,31 @@ end
 ----- Datastore -----
 -- @section datastore
 
---- Internal, Get the data following combine logic
-function Datastore:raw_get(key, isTable)
-    if self.combined then
-        local data = self.combined:raw_get(key, true)
-        if data[self.name] == nil and isTable then
-            data[self.name] = {}
-        end
-        return data[self.name]
-    else
-        if self.data[key] == nil and isTable then
-            self.data[key] = {}
-        end
-        return self.data[key]
+--- Internal, Get data following combine logic
+function Datastore:raw_get(key, fromChild)
+    local data = self.data
+    if self.parent then
+        data = self.parent:raw_get(key, true)
+        key  = self.table_name
     end
+    local value = data[key]
+    if value ~= nil then return value end
+    if fromChild then value = {} end
+    data[key] = value
+    return value
 end
 
---- Internal, Set the data following combine logic
+--- Internal, Set data following combine logic
 function Datastore:raw_set(key, value)
-    if self.combined then
-        local data = self.combined:raw_get(key, true)
-        data[self.name] = value
+    if self.parent then
+        local data = self.parent:raw_get(key, true)
+        data[self.table_name] = value
     else
         self.data[key] = value
     end
 end
 
---- Internal, return the serialized key
+--- Internal, Return the serialized key
 local function serialize_error(err) error('An error ocurred in a datastore serializer: '..err) end
 function Datastore:serialize(rawKey)
     if type(rawKey) == 'string' then return rawKey end
@@ -121,7 +132,7 @@ function Datastore:serialize(rawKey)
     return success and key or nil
 end
 
---- Internal, writes an event to the output file to be saved and/or propagated
+--- Internal, Writes an event to the output file to be saved and/or propagated
 function Datastore:write_action(action, key, value)
     local data = {action, self.name, '"'..key..'"'}
     if value ~= nil then
@@ -136,30 +147,37 @@ function Datastore:set_serializer(callback)
     self.serializer = callback
 end
 
---- Create a new datastore which is combined into this one
+--- Set metadata tags on this datastore which can be accessed by other scripts
+function Datastore:set_metadata(tags)
+    local metadata = self.metadata
+    for key, value in pairs(tags) do
+        metadata[key] = value
+    end
+end
+
+--- Create a new datastore which is stores its data inside of this datastore
 Datastore.combine = DatastoreManager.combine
 
---- Request a value from an external source
+--- Request a value from an external source, will trigger on_load when data is received
 function Datastore:request(key)
-    if self.combined then return self.combined:request(key) end
+    if self.parent then return self.parent:request(key) end
     key = self:serialize(key)
     self:write_action('request', key)
 end
 
---- Save a value to an external source
+--- Save a value to an external source, will trigger on_save before data is saved, save_to_disk must be set to true
 function Datastore:save(key)
-    if self.combined then self.combined:save(key) end
+    if self.parent then self.parent:save(key) end
     if not self.save_to_disk then return end
     key = self:serialize(key)
-    local value = self:raw_get(key)
-    value = self:raise_event('on_save', key, copy(value))
+    local value  = self:raise_event('on_save', key, copy(self:raw_get(key)))
     local action = self.propagateChanges and 'propagate' or 'save'
     self:write_action(action, key, value)
 end
 
---- Save a value to an external source and remove locally
+--- Save a value to an external source and remove locally, will trigger on_unload then on_save, save_to_disk is not required for on_unload
 function Datastore:unload(key)
-    if self.combined then return self.combined:unload(key) end
+    if self.parent then return self.parent:unload(key) end
     key = self:serialize(key)
     self:raise_event('on_unload', key, copy(self:raw_get(key)))
     self:save(key)
@@ -177,10 +195,10 @@ function Datastore:remove(key)
     key = self:serialize(key)
     self:raw_set(key)
     self:write_action('remove', key)
-    if self.combined and self.combined.auto_save then return self.combined:save(key) end
+    if self.parent and self.parent.auto_save then return self.parent:save(key) end
 end
 
---- Get a value from local storage
+--- Get a value from local storage, option to have a default value
 function Datastore:get(key, default)
     key = self:serialize(key)
     local value = self:raw_get(key)
@@ -188,7 +206,7 @@ function Datastore:get(key, default)
     return copy(default)
 end
 
---- Set a value in local storage
+--- Set a value in local storage, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
 function Datastore:set(key, value)
     key = self:serialize(key)
     self:raw_set(key, value)
@@ -197,14 +215,14 @@ function Datastore:set(key, value)
     return value
 end
 
---- Increment the value in local storage, only works for number values
+--- Increment the value in local storage, only works for number values, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
 function Datastore:increment(key, delta)
     key = self:serialize(key)
     local value = self:raw_get(key) or 0
     return Datastore:set(key, value + (delta or 1))
 end
 
---- Use a callback function to update the value locally
+--- Use a function to update the value locally, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
 local function update_error(err) error('An error ocurred in datastore update: '..err, 2) end
 function Datastore:update(key, callback)
     key = self:serialize(key)
@@ -218,7 +236,7 @@ function Datastore:update(key, callback)
     end
 end
 
---- Used to filter elements from a table
+--- Internal, Used to filter elements from a table
 local function filter_error(err) print('An error ocurred in a datastore filter:', err) end
 local function filter(tbl, callback)
     if not callback then return tbl end
@@ -230,15 +248,15 @@ local function filter(tbl, callback)
     return rtn
 end
 
---- Get all keys in the datastore, optional filter callback
+--- Get all keys in this datastore, optional filter callback
 function Datastore:get_all(callback)
-    if not self.combined then
+    if not self.parent then
         return filter(self.data, callback)
     else
-        local name = self.name
-        local data = self.combined:get_all()
+        local table_name = self.table_name
+        local data = self.parent:get_all()
         for key, value in pairs(data) do
-            data[key] = value[name]
+            data[key] = value[table_name]
         end
         return filter(data, callback)
     end
@@ -259,19 +277,33 @@ end
 ----- Events -----
 -- @section events
 
---- Raise a custom event on this datastore
+--- Internal, Raise an event on this datastore
 local function event_error(err) print('An error ocurred in a datastore event handler:', err) end
-function Datastore:raise_event(event_name, key, value)
+function Datastore:raise_event(event_name, key, value, source)
+    -- Raise the event for the children of this datastore
+    if source ~= 'child' then
+        for table_name, child in pairs(self.children) do
+            value[table_name] = child:raise_event(event_name, key, value[table_name], 'parent')
+        end
+    end
+
+    -- Raise the event for this datastore
     local handlers = self.events[event_name]
-    if not handlers then return value end
-    for _, handler in ipairs(handlers) do
-        local success, new_value = xpcall(handler, event_error, key, value)
-        if success and new_value ~= nil then value = new_value end
+    if handlers then
+        for _, handler in ipairs(handlers) do
+            local success, new_value = xpcall(handler, event_error, key, value)
+            if success and new_value ~= nil then value = new_value end
+        end
+    end
+
+    -- Raise the event for the parent of this datastore
+    if source ~= 'parent' and self.parent then
+        self.parent:raise_event(event_name, key, self.parent:raw_get(key), 'child')
     end
     return value
 end
 
---- Returns a function which will add a callback to an event
+--- Internal, Returns a function which will add a callback to an event
 local function event_factory(event_name)
     return function(self, callback)
         assert(type(callback) == 'function', 'Handler must be a function')
@@ -284,19 +316,19 @@ local function event_factory(event_name)
     end
 end
 
---- Register a callback that triggers only when data is received
-Datastore.on_received = event_factory('on_received')
+--- Register a callback that triggers when data is loaded from an external source, returned value is saved locally
+Datastore.on_load = event_factory('on_load')
 
---- Register a callback that triggers before data is saved
+--- Register a callback that triggers before data is saved, returned value is saved externally
 Datastore.on_save = event_factory('on_save')
 
---- Register a callback that triggers before data is unloaded
+--- Register a callback that triggers before data is unloaded, returned value is ignored
 Datastore.on_unload = event_factory('on_unload')
 
---- Register a callback that triggers when a message is received
+--- Register a callback that triggers when a message is received, returned value is ignored
 Datastore.on_message = event_factory('on_message')
 
---- Register a callback that triggers any time a value is changed
+--- Register a callback that triggers any time a value is changed, returned value is ignored
 Datastore.on_update = event_factory('on_update')
 
 ----- Module Return -----
diff --git a/expcore/player_data.lua b/expcore/player_data.lua
index f400f4ca..827abf1e 100644
--- a/expcore/player_data.lua
+++ b/expcore/player_data.lua
@@ -8,40 +8,45 @@ require 'config.expcore.command_general_parse' --- @dep config.expcore.command_g
 local PlayerData = Datastore.connect('PlayerData', true) -- saveToDisk
 PlayerData:set_serializer(Datastore.name_serializer) -- use player name
 
---- Store and enum for the data collection policy
-local DataCollectionPolicy = PlayerData:combine('DataCollectionPolicy')
-local PolicyEnum = { 'All', 'Tracking', 'Settings', 'Required' }
-for k,v in ipairs(PolicyEnum) do PolicyEnum[v] = k end
+--- Store and enum for the data saving preference
+local DataSavingPreference = PlayerData:combine('DataSavingPreference')
+local PreferenceEnum = { 'All', 'Statistics', 'Settings', 'Required' }
+for k,v in ipairs(PreferenceEnum) do PreferenceEnum[v] = k end
 
---- Sets your data collection policy
--- @command set-data-policy
-Commands.new_command('set-data-policy', 'Allows you to set your data collection policy')
-:add_param('option', false, 'string-options', PolicyEnum)
+--- Sets your data saving preference
+-- @command set-data-preference
+Commands.new_command('set-data-preference', 'Allows you to set your data saving preference')
+:add_param('option', false, 'string-options', PreferenceEnum)
 :register(function(player, option)
-    DataCollectionPolicy:set(player, option)
-    return {'expcore-data.set-policy', option}
+    DataSavingPreference:set(player, option)
+    return {'expcore-data.set-preference', option}
 end)
 
---- Gets your data collection policy
--- @command data-policy
-Commands.new_command('data-policy', 'Shows you what your current data collection policy is')
+--- Gets your data saving preference
+-- @command data-preference
+Commands.new_command('data-preference', 'Shows you what your current data saving preference is')
 :register(function(player)
-    return {'expcore-data.get-policy', DataCollectionPolicy:get(player, 'All')}
+    return {'expcore-data.get-preference', DataSavingPreference:get(player, 'All')}
 end)
 
 --- Remove data that the player doesnt want to have stored
 PlayerData:on_save(function(player_name, player_data)
-    local collectData = DataCollectionPolicy:get(player_name, 'All')
-    collectData = PolicyEnum[collectData]
-    if collectData == PolicyEnum.All then return player_data end
+    local dataPreference = DataSavingPreference:get(player_name, 'All')
+    dataPreference = PreferenceEnum[dataPreference]
+    if dataPreference == PreferenceEnum.All then return player_data end
 
-    local saved_player_data = { PlayerRequired = player_data.PlayerRequired, DataCollectionPolicy = PolicyEnum[collectData] }
-    if collectData <= PolicyEnum.Settings then saved_player_data.PlayerSettings = player_data.PlayerSettings end
-    if collectData <= PolicyEnum.Tracking then saved_player_data.PlayerTracking = player_data.PlayerTracking end
+    local saved_player_data = { PlayerRequired = player_data.PlayerRequired, DataSavingPreference = PreferenceEnum[dataPreference] }
+    if dataPreference <= PreferenceEnum.Settings then saved_player_data.PlayerSettings = player_data.PlayerSettings end
+    if dataPreference <= PreferenceEnum.Statistics then saved_player_data.PlayerStatistics = player_data.PlayerStatistics end
 
     return saved_player_data
 end)
 
+--- Display your data preference when your data loads
+DataSavingPreference:on_load(function(player_name, dataPreference)
+    game.players[player_name].print{'expcore-data.get-preference', dataPreference or 'All'}
+end)
+
 --- Load player data when they join
 Event.add(defines.events.on_player_joined_game, function(event)
     PlayerData:request(game.players[event.player_index])
@@ -55,9 +60,9 @@ end)
 ----- Module Return -----
 return {
     All = PlayerData, -- Root for all of a players data
-    Tracking = PlayerData:combine('PlayerTracking'), -- Common place for tracing stats
+    Statistics = PlayerData:combine('PlayerStatistics'), -- Common place for stats
     Settings = PlayerData:combine('PlayerSettings'), -- Common place for settings
     Required = PlayerData:combine('PlayerRequired'), -- Common place for required data
-    DataCollectionPolicy = DataCollectionPolicy, -- Stores what data groups will be saved
-    PolicyEnum = PolicyEnum -- Enum for the allowed options for the data collection policy
+    DataSavingPreference = DataSavingPreference, -- Stores what data groups will be saved
+    PreferenceEnum = PreferenceEnum -- Enum for the allowed options for data saving preference
 }
\ No newline at end of file
diff --git a/locale/en/expcore.cfg b/locale/en/expcore.cfg
index ea0a5d0c..5fd4fefe 100644
--- a/locale/en/expcore.cfg
+++ b/locale/en/expcore.cfg
@@ -38,5 +38,5 @@ button_tooltip=Shows/hides the toolbar.
 left-button-tooltip=Hide all open windows.
 
 [expcore-data]
-set-policy=You data collection policy has been set to __1__. Existing data will not be effected until you rejoin.
-get-policy=You data collection policy is __1__. Use /set-data-policy to change this. 
\ No newline at end of file
+set-preference=You data saving preference has been set to __1__. Existing data will not be effected until you rejoin.
+get-preference=You data saving preference is __1__. Use /set-data-preference to change this. 
\ No newline at end of file

From b6699df3aae7bb4f13ddad0b2b71ebfc4521aa2d Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Sat, 23 May 2020 22:50:34 +0100
Subject: [PATCH 07/10] Added datastore to debug

---
 expcore/datastore.lua                        |  38 +++++-
 modules/gui/debug/expcore_datastore_view.lua | 131 +++++++++++++++++++
 modules/gui/debug/global_view.lua            |   2 +-
 modules/gui/debug/main_view.lua              |   1 +
 4 files changed, 167 insertions(+), 5 deletions(-)
 create mode 100644 modules/gui/debug/expcore_datastore_view.lua

diff --git a/expcore/datastore.lua b/expcore/datastore.lua
index 31f65189..f9f81b59 100644
--- a/expcore/datastore.lua
+++ b/expcore/datastore.lua
@@ -91,6 +91,13 @@ function DatastoreManager.ingest(action, tableName, key, valueJson)
 
 end
 
+--- Debug, Use to get all datastores, or return debug info on a datastore
+function DatastoreManager.debug(tableName)
+    if not tableName then return Datastores end
+    local datastore = assert(Datastores[tableName], 'Datastore not found '..tostring(tableName))
+    return datastore:debug()
+end
+
 --- Commonly used serializer, returns the name of the object
 function DatastoreManager.name_serializer(rawKey)
     return rawKey.name
@@ -99,6 +106,30 @@ end
 ----- Datastore -----
 -- @section datastore
 
+--- Debug, Get the debug info for this datastore
+function Datastore:debug()
+    local debug_info = {}
+
+    if self.parent then
+        debug_info.parent = self.parent.name
+    else
+        debug_info.settings = { auto_save = self.auto_save, save_to_disk = self.save_to_disk, propagate_changes = self.propagate_changes, serializer = not not self.serializer }
+    end
+
+    local children = {}
+    for name in pairs(self.children) do children[#children+1] = name end
+    if #children > 0 then debug_info.children = children end
+
+    local events = {}
+    for name, handlers in pairs(self.events) do events[name] = #handlers end
+    if next(events) then debug_info.events = events end
+
+    if next(self.metadata) then debug_info.metadata = self.metadata end
+    debug_info.data = self:get_all()
+
+    return debug_info
+end
+
 --- Internal, Get data following combine logic
 function Datastore:raw_get(key, fromChild)
     local data = self.data
@@ -171,7 +202,7 @@ function Datastore:save(key)
     if not self.save_to_disk then return end
     key = self:serialize(key)
     local value  = self:raise_event('on_save', key, copy(self:raw_get(key)))
-    local action = self.propagateChanges and 'propagate' or 'save'
+    local action = self.propagate_changes and 'propagate' or 'save'
     self:write_action(action, key, value)
 end
 
@@ -253,9 +284,8 @@ function Datastore:get_all(callback)
     if not self.parent then
         return filter(self.data, callback)
     else
-        local table_name = self.table_name
-        local data = self.parent:get_all()
-        for key, value in pairs(data) do
+        local data, table_name = {}, self.table_name
+        for key, value in pairs(self.parent:get_all()) do
             data[key] = value[table_name]
         end
         return filter(data, callback)
diff --git a/modules/gui/debug/expcore_datastore_view.lua b/modules/gui/debug/expcore_datastore_view.lua
new file mode 100644
index 00000000..f7a9912f
--- /dev/null
+++ b/modules/gui/debug/expcore_datastore_view.lua
@@ -0,0 +1,131 @@
+local Gui = require 'utils.gui' --- @dep utils.gui
+local Datastore = require 'expcore.datastore' --- @dep expcore.datastore
+local Color = require 'utils.color_presets' --- @dep utils.color_presets
+local Model = require 'modules.gui.debug.model' --- @dep modules.gui.debug.model
+
+local dump = Model.dump
+local dump_text = Model.dump_text
+local concat = table.concat
+
+local Public = {}
+
+local header_name = Gui.uid_name()
+local left_panel_name = Gui.uid_name()
+local right_panel_name = Gui.uid_name()
+local input_text_box_name = Gui.uid_name()
+local refresh_name = Gui.uid_name()
+
+Public.name = 'Datastore'
+
+function Public.show(container)
+    local main_flow = container.add {type = 'flow', direction = 'horizontal'}
+
+    local left_panel = main_flow.add {type = 'scroll-pane', name = left_panel_name}
+    local left_panel_style = left_panel.style
+    left_panel_style.width = 300
+
+    for name in pairs(table.keysort(Datastore.debug())) do
+        local header = left_panel.add({type = 'flow'}).add {type = 'label', name = header_name, caption = name}
+        Gui.set_data(header, name)
+    end
+
+    local right_flow = main_flow.add {type = 'flow', direction = 'vertical'}
+
+    local right_top_flow = right_flow.add {type = 'flow', direction = 'horizontal'}
+
+    local input_text_box = right_top_flow.add {type = 'text-box', name = input_text_box_name}
+    local input_text_box_style = input_text_box.style
+    input_text_box_style.horizontally_stretchable = true
+    input_text_box_style.height = 32
+    input_text_box_style.maximal_width = 1000
+
+    local refresh_button =
+        right_top_flow.add {type = 'sprite-button', name = refresh_name, sprite = 'utility/reset', tooltip = 'refresh'}
+    local refresh_button_style = refresh_button.style
+    refresh_button_style.width = 32
+    refresh_button_style.height = 32
+
+    local right_panel = right_flow.add {type = 'text-box', name = right_panel_name}
+    right_panel.read_only = true
+    right_panel.selectable = true
+
+    local right_panel_style = right_panel.style
+    right_panel_style.vertically_stretchable = true
+    right_panel_style.horizontally_stretchable = true
+    right_panel_style.maximal_width = 1000
+    right_panel_style.maximal_height = 1000
+
+    local data = {
+        right_panel = right_panel,
+        input_text_box = input_text_box,
+        selected_header = nil
+    }
+
+    Gui.set_data(input_text_box, data)
+    Gui.set_data(left_panel, data)
+    Gui.set_data(refresh_button, data)
+end
+
+Gui.on_click(
+    header_name,
+    function(event)
+        local element = event.element
+        local tableName = Gui.get_data(element)
+
+        local left_panel = element.parent.parent
+        local data = Gui.get_data(left_panel)
+        local right_panel = data.right_panel
+        local selected_header = data.selected_header
+        local input_text_box = data.input_text_box
+
+        if selected_header then
+            selected_header.style.font_color = Color.white
+        end
+
+        element.style.font_color = Color.orange
+        data.selected_header = element
+
+        input_text_box.text = tableName
+        input_text_box.style.font_color = Color.black
+
+        local content = Datastore.debug(tableName)
+        local content_string = {}
+        for key, value in pairs(content) do
+            content_string[#content_string+1] = key:gsub('^%l', string.upper)..' = '..dump(value)
+        end
+        right_panel.text = concat(content_string, '\n')
+    end
+)
+
+local function update_dump(text_input, data)
+    local content = Datastore.debug(text_input.text)
+    local content_string = {}
+    for key, value in pairs(content) do
+        content_string[#content_string+1] = key:gsub('^%l', string.upper)..' = '..dump(value)
+    end
+    data.right_panel.text = concat(content_string, '\n')
+end
+
+Gui.on_text_changed(
+    input_text_box_name,
+    function(event)
+        local element = event.element
+        local data = Gui.get_data(element)
+
+        update_dump(element, data)
+    end
+)
+
+Gui.on_click(
+    refresh_name,
+    function(event)
+        local element = event.element
+        local data = Gui.get_data(element)
+
+        local input_text_box = data.input_text_box
+
+        update_dump(input_text_box, data)
+    end
+)
+
+return Public
diff --git a/modules/gui/debug/global_view.lua b/modules/gui/debug/global_view.lua
index 3ab51d8d..aff9a6fd 100644
--- a/modules/gui/debug/global_view.lua
+++ b/modules/gui/debug/global_view.lua
@@ -8,7 +8,7 @@ local concat = table.concat
 
 local Public = {}
 
-local ignore = {tokens = true, data_store = true}
+local ignore = {tokens = true, data_store = true, datastores = true}
 
 local header_name = Gui.uid_name()
 local left_panel_name = Gui.uid_name()
diff --git a/modules/gui/debug/main_view.lua b/modules/gui/debug/main_view.lua
index 0771bdb1..bdc93f43 100644
--- a/modules/gui/debug/main_view.lua
+++ b/modules/gui/debug/main_view.lua
@@ -5,6 +5,7 @@ local Public = {}
 
 local pages = {
     require 'modules.gui.debug.redmew_global_view',
+    require 'modules.gui.debug.expcore_datastore_view',
     require 'modules.gui.debug.expcore_store_view',
     require 'modules.gui.debug.expcore_gui_view',
     require 'modules.gui.debug.global_view',

From 08a86aac09e299a6a0ca18b5b6a217efc32c7eec Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Mon, 25 May 2020 02:11:45 +0100
Subject: [PATCH 08/10] Added doc comments

---
 expcore/datastore.lua   | 669 ++++++++++++++++++++++++++++++++++------
 expcore/player_data.lua |  48 ++-
 2 files changed, 612 insertions(+), 105 deletions(-)

diff --git a/expcore/datastore.lua b/expcore/datastore.lua
index f9f81b59..4189aaef 100644
--- a/expcore/datastore.lua
+++ b/expcore/datastore.lua
@@ -1,3 +1,150 @@
+--[[-- Core Module - Datastore
+- A module used to store data in the global table with the option to have it sync to an external source.
+@core Datastore
+@alias DatastoreManager
+
+@usage-- Types of Datastore
+-- This datastore will not save data externally and can be used to watch for updates on values within it
+-- A common use might be to store data for a gui and only update the gui when a value changes
+local LocalDatastore = Datastore.connect('LocalDatastore')
+
+-- This datastore will allow you to use the save and request method, this allows you to have persistent data
+-- Should be used over auto save as it creates less save requests, but this means you need to tell the data to be saved
+-- We use this type for player data as we know the data only needs to be saved when the player leaves
+local PersistentDatastore = Datastore.connect('PersistentDatastore', true) -- save_to_disk
+
+-- This datastore is the same as above but the save method will be called automatically when ever you change a value
+-- An auto save datastore should be used if the data does not change often, this can be global settings and things of that sort
+-- If it is at all possible to setup events to unload and/or save the data then this is preferable
+local AutosaveDatastore = Datastore.connect('AutosaveDatastore', true, true) -- save_to_disk, auto_save
+
+-- Finally you can have a datastore that propagates its changes to all other connected servers, this means request does not need to be used
+-- This should be used when you might have data conflicts while saving, this is done by pushing the saved value to all active servers
+-- The request method has little use after server start as any external changes to the value will be pushed automatically
+-- Auto save can also be used with this type and you should follow the same guidelines above for when this should be avoided
+local PropagateDatastore = Datastore.connect('PropagateDatastore', true, false, true) -- save_to_disk, propagate_changes
+
+@usage-- Using Datastores Locally
+-- Once you have your datastore connection setup, any further requests with connect will return the same datastore
+-- This is important to know because the settings passed as parameters you have an effect when it is first created
+
+-- One useful thing that you might want to set up before runtime is a serializer, this will convert non string keys into strings
+-- This serializer will allow use to pass a player object and still have it serialized to the players name
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_serializer(function(rawKey)
+    return rawKey.name
+end)
+
+-- If we want to get data from the datastore we can use get or get_all
+local value = ExampleData:get(player, defaultValue)
+local values = ExampleData:get_all()
+
+-- If we want to set data then we can use set, increment, update, or update_all
+ExampleData:set(player, 10)
+ExampleData:increment(player)
+ExampleData:update(player, function(player_name, value)
+    return value * 2
+end)
+ExampleData:update_all(function(player_name, value)
+    return value * 2
+end)
+
+-- If we want to remove data then we use remove
+ExampleData:remove(player)
+
+-- We can also listen for updates to a value done by any of the above methods with on_update
+ExampleData:on_update(function(player_name, value)
+    game.print(player_name..' has had their example data updated to '..tostring(value))
+end)
+
+@usage-- Using Datastore Externally
+-- If save_to_disk is used then this opens up the option for persistent data which you can request, save, and remove
+-- All of the local methods are still usable put now there is the option for extra events
+-- In order for this to work there must be an external script to read datastore.pipe and inject with Datastore.ingest
+
+-- To request data you would use request and the on_load event, this event can be used to modify data before it is used
+ExampleData:request(player)
+ExampleData:on_load(function(player_name, value)
+    game.print('Loaded example data for '..player_name)
+    -- A value can be returned here to overwrite the received value
+end)
+
+-- To save data you would use save and the on_save event, this event can be used to modify data before it is saved
+ExampleData:save(player)
+ExampleData:on_save(function(player_name, value)
+    game.print('Saved example data for '..player_name)
+    -- A value can be returned here to overwrite the value which is saved
+end)
+
+-- To remove data locally but not externally, like if a player logs off, you would use unload and on_unload
+ExampleData:unload(player)
+ExampleData:on_unload(function(player_name, value)
+    game.print('Unloaded example data for '..player_name)
+    -- Any return is ignored, this is event is for cleaning up other data
+end)
+
+@usage-- Using Datastore Messaging
+-- The message action can be used regardless of save_to_disk being set as no data is saved, but an external script is still required
+-- These messages can be used to send data to other servers which doesnt need to be saved such as shouts or commands
+-- Using messages is quite simple only using message and on_message
+ExampleData:message(key, message)
+ExampleData:on_message(function(key, message)
+    game.print('Received message '..message)
+end)
+
+@usage-- Combined Datastores
+-- A combined datastore is a datastore which stores its data inside of another datastore
+-- This means that the data is stored more efficiently in the external database and less requests need to be made
+-- To understand how combined datastores work think of each key in the parent as a table where the sub datastore is a key in that table
+-- Player data is the most used version of the combined datastore, below is how the player data module is setup
+local PlayerData = Datastore.connect('PlayerData', true) -- saveToDisk
+PlayerData:set_serializer(Datastore.name_serializer) -- use player name as key
+PlayerData:combine('Statistics')
+PlayerData:combine('Settings')
+PlayerData:combine('Required')
+
+-- You can then further combine datastores to any depth, below we add some possible settings and statistics that we might use
+-- Although we dont in this example, each of these functions returns the datastore object which you should use as a local value
+PlayerData.Settings:combine('Color')
+PlayerData.Settings:combine('Quickbar')
+PlayerData.Settings:combine('JoinMessage')
+PlayerData.Statistics:combine('Playtime')
+PlayerData.Statistics:combine('JoinCount')
+
+-- Because sub datastore work just like a normal datastore you dont need any special code, using get and set will still return as if it wasnt a sub datastore
+-- Things like the serializer and the datastore settings are always the same as the parent so you dont need to worry about setting up the serializer each time
+-- And because save, request, and unload methods all point to the root datastore you are able to request and save your data as normal
+
+-- If you used get_all on PlayerData this is what you would get:
+{
+    Cooldude2606 = {
+        Settings = {
+            Color = 'ColorValue',
+            Quickbar = 'QuickbarValue',
+            JoinMessage = 'JoinMessageValue'
+        },
+        Statistics = {
+            Playtime = 'PlaytimeValue',
+            JoinCount = 'JoinCountValue'
+        }
+    }
+}
+
+-- If you used get_all on PlayerData.Settings this is what you would get:
+{
+    Cooldude2606 = {
+        Color = 'ColorValue',
+        Quickbar = 'QuickbarValue',
+        JoinMessage = 'JoinMessageValue'
+    }
+}
+
+-- If you used get_all on PlayerData.Settings.Color this is what you would get:
+{
+    Cooldude2606 = 'ColorValue'
+}
+
+]]
 
 local Event = require 'utils.event' --- @dep utils.event
 
@@ -11,8 +158,8 @@ local copy = table.deep_copy
 global.datastores = Data
 Event.on_load(function()
     Data = global.datastores
-    for tableName, datastore in pairs(Datastores) do
-        datastore.data = Data[tableName]
+    for datastoreName, datastore in pairs(Datastores) do
+        datastore.data = Data[datastoreName]
     end
 end)
 
@@ -26,17 +173,27 @@ DatastoreManager.metatable = {
     __call = function(self, ...) return self:get(...) end
 }
 
---- Make a new datastore connection, if a connection already exists then it is returned
-function DatastoreManager.connect(tableName, saveToDisk, autoSave, propagateChanges)
-    if Datastores[tableName] then return Datastores[tableName] end
+--[[-- Make a new datastore connection, if a connection already exists then it is returned
+@tparam string datastoreName The name that you want the new datastore to have, this can not have any whitespace
+@tparam[opt=false] boolean saveToDisk When set to true, using the save method with write the data to datastore.pipe
+@tparam[opt=false] boolean autoSave When set to true, using any method which modifies data will cause the data to be saved
+@tparam[opt=false] boolean propagateChanges When set to true, using the save method will send the data to all other connected servers
+@treturn table The new datastore connection that can be used to access and modify data in the datastore
+
+@usage-- Connecting to the test datastore which will allow saving to disk
+local ExampleData = Datastore.connect('ExampleData', true) -- saveToDisk
+
+]]
+function DatastoreManager.connect(datastoreName, saveToDisk, autoSave, propagateChanges)
+    if Datastores[datastoreName] then return Datastores[datastoreName] end
     if _LIFECYCLE ~= _STAGE.control then
         -- Only allow this function to be called during the control stage
         error('New datastore connection can not be created during runtime', 2)
     end
 
     local new_datastore = {
-        name = tableName,
-        table_name = tableName,
+        name = datastoreName,
+        value_name = datastoreName,
         auto_save = autoSave or false,
         save_to_disk = saveToDisk or false,
         propagate_changes = propagateChanges or false,
@@ -48,28 +205,38 @@ function DatastoreManager.connect(tableName, saveToDisk, autoSave, propagateChan
         data = {}
     }
 
-    Data[tableName] = new_datastore.data
-    Datastores[tableName] = new_datastore
+    Data[datastoreName] = new_datastore.data
+    Datastores[datastoreName] = new_datastore
     return setmetatable(new_datastore, DatastoreManager.metatable)
 end
 
---- Make a new datastore that stores its data inside of another one
-function DatastoreManager.combine(datastore, subTableName)
-    local new_datastore = DatastoreManager.connect(datastore.name..'.'..subTableName)
-    datastore.children[subTableName] = new_datastore
-    new_datastore.serializer = datastore.serializer
-    new_datastore.auto_save = datastore.auto_save
-    new_datastore.table_name = subTableName
-    new_datastore.parent = datastore
-    Data[new_datastore.name] = nil
-    new_datastore.data = nil
-    return new_datastore
+--[[-- Make a new datastore that stores its data inside of another one
+@tparam string datastoreName The name of the datastore that will contain the data for the new datastore
+@tparam string subDatastoreName The name of the new datastore, this name will also be used as the key inside the parent datastore
+@treturn table The new datastore connection that can be used to access and modify data in the datastore
+
+@usage-- Setting up a datastore which stores its data inside of another datastore
+local BarData = Datastore.combine('ExampleData', 'Bar')
+
+]]
+function DatastoreManager.combine(datastoreName, subDatastoreName)
+    local datastore = assert(Datastores[datastoreName], 'Datastore not found '..tostring(datastoreName))
+    return datastore:combine(subDatastoreName)
 end
 
---- Ingest the result from a request, this is used through a rcon interface to sync data
+--[[-- Ingest the result from a request, this is used through a rcon interface to sync data
+@tparam string action The action that should be done, can be: remove, message, propagate, or request
+@tparam string datastoreName The name of the datastore that should have the action done to it
+@tparam string key The key of that datastore that is having the action done to it
+@tparam string valueJson The json string for the value being ingested, remove does not require a value
+
+@usage-- Replying to a data request
+Datastore.ingest('request', 'ExampleData', 'TestKey', 'Foo')
+
+]]
 local function ingest_error(err) print('Datastore ingest error, Unable to parse json:', err) end
-function DatastoreManager.ingest(action, tableName, key, valueJson)
-    local datastore = assert(Datastores[tableName], 'Datastore ingest error, Datastore not found '..tostring(tableName))
+function DatastoreManager.ingest(action, datastoreName, key, valueJson)
+    local datastore = assert(Datastores[datastoreName], 'Datastore ingest error, Datastore not found '..tostring(datastoreName))
     assert(type(action) == 'string', 'Datastore ingest error, Action is not a string got: '..type(action))
     assert(type(key) == 'string', 'Datastore ingest error, Key is not a string got: '..type(key))
 
@@ -78,12 +245,14 @@ function DatastoreManager.ingest(action, tableName, key, valueJson)
 
     elseif action == 'message' then
         local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
-        if not success or value == nil then return end
+        if not success then return end
+        if value == nil then value = valueJson end
         datastore:raise_event('on_message', key, value)
 
-    elseif action == 'propagate' then
+    elseif action == 'propagate' or action == 'request' then
         local success, value = xpcall(game.json_to_table, ingest_error, valueJson)
-        if not success or value == nil then return end
+        if not success then return end
+        if value == nil then value = valueJson end
         value = datastore:raise_event('on_load', key, value)
         datastore:set(key, value)
 
@@ -91,22 +260,46 @@ function DatastoreManager.ingest(action, tableName, key, valueJson)
 
 end
 
---- Debug, Use to get all datastores, or return debug info on a datastore
-function DatastoreManager.debug(tableName)
-    if not tableName then return Datastores end
-    local datastore = assert(Datastores[tableName], 'Datastore not found '..tostring(tableName))
+--[[-- Debug, Use to get all datastores, or return debug info on a datastore
+@tparam[opt] string datastoreName The name of the datastore to get the debug info of
+
+@usage-- Get all the datastores
+local datastores = Datastore.debug()
+
+@usage-- Getting the debug info for a datastore
+local debug_info = Datastore.debug('ExampleData')
+
+]]
+function DatastoreManager.debug(datastoreName)
+    if not datastoreName then return Datastores end
+    local datastore = assert(Datastores[datastoreName], 'Datastore not found '..tostring(datastoreName))
     return datastore:debug()
 end
 
---- Commonly used serializer, returns the name of the object
+--[[-- Commonly used serializer, returns the name of the object
+@tparam any rawKey The raw key that will be serialized, this can be things like player, force, surface, etc
+@treturn string The name of the object that was passed
+
+@usage-- Using the name serializer for your datastore
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_serializer(Datastore.name_serializer)
+
+]]
 function DatastoreManager.name_serializer(rawKey)
     return rawKey.name
 end
 
------ Datastore -----
--- @section datastore
+----- Datastore Internal -----
+-- @section datastore-internal
 
---- Debug, Get the debug info for this datastore
+--[[-- Debug, Get the debug info for this datastore
+@treturn table The debug info for this datastore, contains stuff like parent, settings, children, etc
+
+@usage-- Get the debug info for a datastore
+local ExampleData = Datastore.connect('ExampleData')
+local debug_info = ExampleData:debug()
+
+]]
 function Datastore:debug()
     local debug_info = {}
 
@@ -130,12 +323,20 @@ function Datastore:debug()
     return debug_info
 end
 
---- Internal, Get data following combine logic
+--[[-- Internal, Get data following combine logic
+@tparam string key The key to get the value of from this datastore
+@tparam[opt=false] boolean fromChild If the get request came from a child of this datastore
+@treturn any The value that was stored at this key in this datastore
+
+@usage-- Internal, Get the data from a datastore
+local value = self:raw_get('TestKey')
+
+]]
 function Datastore:raw_get(key, fromChild)
     local data = self.data
     if self.parent then
         data = self.parent:raw_get(key, true)
-        key  = self.table_name
+        key  = self.value_name
     end
     local value = data[key]
     if value ~= nil then return value end
@@ -144,18 +345,32 @@ function Datastore:raw_get(key, fromChild)
     return value
 end
 
---- Internal, Set data following combine logic
+--[[-- Internal, Set data following combine logic
+@tparam string key The key to set the value of in this datastore
+@tparam any value The value that will be set at this key
+
+@usage-- Internal, Set the value in a datastore
+self:raw_set('TestKey', 'Foo')
+
+]]
 function Datastore:raw_set(key, value)
     if self.parent then
         local data = self.parent:raw_get(key, true)
-        data[self.table_name] = value
+        data[self.value_name] = value
     else
         self.data[key] = value
     end
 end
 
---- Internal, Return the serialized key
 local function serialize_error(err) error('An error ocurred in a datastore serializer: '..err) end
+--[[-- Internal, Return the serialized key
+@tparam any rawKey The key that needs to be serialized, if it is already a string then it is returned
+@treturn string The key after it has been serialized
+
+@usage-- Internal, Ensure that the key is a string
+key = self:serialize(key)
+
+]]
 function Datastore:serialize(rawKey)
     if type(rawKey) == 'string' then return rawKey end
     assert(self.serializer, 'Datastore does not have a serializer and received non string key')
@@ -163,7 +378,18 @@ function Datastore:serialize(rawKey)
     return success and key or nil
 end
 
---- Internal, Writes an event to the output file to be saved and/or propagated
+--[[-- Internal, Writes an event to the output file to be saved and/or propagated
+@tparam string action The action that should be wrote to datastore.pipe, can be request, remove, message, save, propagate
+@tparam string key The key that the action is being preformed on
+@tparam any value The value that should be used with the action
+
+@usage-- Write a data request to datastore.pipe
+self:write_action('request', 'TestKey')
+
+@usage-- Write a data save to datastore.pipe
+self:write_action('save', 'TestKey', 'Foo')
+
+]]
 function Datastore:write_action(action, key, value)
     local data = {action, self.name, '"'..key..'"'}
     if value ~= nil then
@@ -172,13 +398,57 @@ function Datastore:write_action(action, key, value)
     game.write_file('datastore.pipe', table.concat(data, ' ')..'\n', true, 0)
 end
 
---- Set a callback that will be used to serialize keys which aren't strings
+----- Datastore -----
+-- @section datastore
+
+--[[-- Create a new datastore which is stores its data inside of this datastore
+@tparam string subDatastoreName The name of the datastore that will have its data stored in this datastore
+@treturn table The new datastore that was created inside of this datastore
+
+@usage-- Add a new sub datastore
+local ExampleData = Datastore.connect('ExampleData')
+local BarData = ExampleData:combine('Bar')
+
+]]
+function Datastore:combine(subDatastoreName)
+    local new_datastore = DatastoreManager.connect(self.name..'.'..subDatastoreName)
+    self.children[subDatastoreName] = new_datastore
+    new_datastore.value_name = subDatastoreName
+    new_datastore.serializer = self.serializer
+    new_datastore.auto_save = self.auto_save
+    new_datastore.parent = self
+    Data[new_datastore.name] = nil
+    new_datastore.data = nil
+    return new_datastore
+end
+
+--[[-- Set a callback that will be used to serialize keys which aren't strings
+@tparam function callback The function that will be used to serialize non string keys passed as an argument
+
+@usage-- Set a custom serializer, this would be the same as Datastore.name_serializer
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_serializer(function(rawKey)
+    return rawKey.name
+end)
+
+]]
 function Datastore:set_serializer(callback)
     assert(type(callback) == 'function', 'Callback must be a function')
     self.serializer = callback
 end
 
---- Set metadata tags on this datastore which can be accessed by other scripts
+--[[-- Set metadata tags on this datastore which can be accessed by other scripts
+@tparam table tags A table of tags that you want to set in the metadata for this datastore
+
+@usage-- Adding metadata that could be used by a gui to help understand the stored data
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set_metadata{
+    caption = 'Test Data',
+    tooltip = 'Data used for testing datastores',
+    type = 'table'
+}
+
+]]
 function Datastore:set_metadata(tags)
     local metadata = self.metadata
     for key, value in pairs(tags) do
@@ -186,50 +456,15 @@ function Datastore:set_metadata(tags)
     end
 end
 
---- Create a new datastore which is stores its data inside of this datastore
-Datastore.combine = DatastoreManager.combine
+--[[-- Get a value from local storage, option to have a default value
+@tparam any key The key that you want to get the value of, must be a string unless a serializer is set
+@tparam[opt] any default The default value that will be returned if no value is found in the datastore
 
---- Request a value from an external source, will trigger on_load when data is received
-function Datastore:request(key)
-    if self.parent then return self.parent:request(key) end
-    key = self:serialize(key)
-    self:write_action('request', key)
-end
+@usage-- Get a key from the datastore, the default will be deep copied if no value exists in the datastore
+local ExampleData = Datastore.connect('ExampleData')
+local value = ExampleData:get('TestKey')
 
---- Save a value to an external source, will trigger on_save before data is saved, save_to_disk must be set to true
-function Datastore:save(key)
-    if self.parent then self.parent:save(key) end
-    if not self.save_to_disk then return end
-    key = self:serialize(key)
-    local value  = self:raise_event('on_save', key, copy(self:raw_get(key)))
-    local action = self.propagate_changes and 'propagate' or 'save'
-    self:write_action(action, key, value)
-end
-
---- Save a value to an external source and remove locally, will trigger on_unload then on_save, save_to_disk is not required for on_unload
-function Datastore:unload(key)
-    if self.parent then return self.parent:unload(key) end
-    key = self:serialize(key)
-    self:raise_event('on_unload', key, copy(self:raw_get(key)))
-    self:save(key)
-    self:raw_set(key)
-end
-
---- Use to send a message over the connection, works regardless of saveToDisk and propagateChanges
-function Datastore:message(key, message)
-    key = self:serialize(key)
-    self:write_action('message', key, message)
-end
-
---- Remove a value locally and on the external source, works regardless of propagateChanges
-function Datastore:remove(key)
-    key = self:serialize(key)
-    self:raw_set(key)
-    self:write_action('remove', key)
-    if self.parent and self.parent.auto_save then return self.parent:save(key) end
-end
-
---- Get a value from local storage, option to have a default value
+]]
 function Datastore:get(key, default)
     key = self:serialize(key)
     local value = self:raw_get(key)
@@ -237,7 +472,15 @@ function Datastore:get(key, default)
     return copy(default)
 end
 
---- Set a value in local storage, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+--[[-- Set a value in local storage, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+@tparam any key The key that you want to set the value of, must be a string unless a serializer is set
+@tparam any value The value that you want to set for this key
+
+@usage-- Set a value in the datastore, this will trigger on_update, if auto_save is true then will trigger save
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:set('TestKey', 'Foo')
+
+]]
 function Datastore:set(key, value)
     key = self:serialize(key)
     self:raw_set(key, value)
@@ -246,15 +489,33 @@ function Datastore:set(key, value)
     return value
 end
 
---- Increment the value in local storage, only works for number values, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+--[[-- Increment the value in local storage, only works for number values, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+@tparam any key The key that you want to increment the value of, must be a string unless a serializer is set
+@tparam[opt=1] number delta The amount that you want to increment the value by, can be negative or a decimal
+
+@usage-- Increment a value in a datastore, the value must be a number or nil, if nil 0 is used as the start value
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:increment('TestNumber')
+
+]]
 function Datastore:increment(key, delta)
     key = self:serialize(key)
     local value = self:raw_get(key) or 0
     return Datastore:set(key, value + (delta or 1))
 end
 
---- Use a function to update the value locally, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
 local function update_error(err) error('An error ocurred in datastore update: '..err, 2) end
+--[[-- Use a function to update the value locally, will trigger on_update then on_save, save_to_disk and auto_save is required for on_save
+@tparam any key The key that you want to apply the update to, must be a string unless a serializer is set
+@tparam function callback The function that will be used to update the value at this key
+
+@usage-- Using a function to update a value, if a value is returned then this will be the new value
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:increment('TestKey', function(key, value)
+    return value..value
+end)
+
+]]
 function Datastore:update(key, callback)
     key = self:serialize(key)
     local value = self:raw_get(key)
@@ -267,8 +528,33 @@ function Datastore:update(key, callback)
     end
 end
 
---- Internal, Used to filter elements from a table
+--[[-- Remove a value locally and on the external source, works regardless of propagateChanges
+@tparam any key The key that you want to remove locally and externally, must be a string unless a serializer is set
+
+@usage-- Remove a key locally and externally
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:remove('TestKey')
+
+]]
+function Datastore:remove(key)
+    key = self:serialize(key)
+    self:raw_set(key)
+    self:write_action('remove', key)
+    if self.parent and self.parent.auto_save then return self.parent:save(key) end
+end
+
 local function filter_error(err) print('An error ocurred in a datastore filter:', err) end
+--[[-- Internal, Used to filter elements from a table
+@tparam table tbl The table that will have the filter applied to it
+@tparam[opt] function callback The function that will be used as a filter, if none giving then the provided table is returned
+@treturn table The table which has only the key values pairs which passed the filter
+
+@usage-- Internal, Filter a table by the values it contains, return true to keep the key value pair
+local filtered_table = filter({5,3,4,1,2}, function(key, value)
+    return value > 2
+end)
+
+]]
 local function filter(tbl, callback)
     if not callback then return tbl end
     local rtn = {}
@@ -279,26 +565,153 @@ local function filter(tbl, callback)
     return rtn
 end
 
---- Get all keys in this datastore, optional filter callback
+--[[-- Get all keys in this datastore, optional filter callback
+@tparam[opt] function callback The filter function that can be used to filter the results returned
+@treturn table All the data that is in this datastore, filtered if a filter was provided
+
+@usage-- Get all the data in this datastore
+local ExampleData = Datastore.connect('ExampleData')
+local data = ExampleData:get_all()
+
+@usage-- Get all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+local data = ExampleData:get_all(function(key, value)
+    return type(value) == 'string'
+end)
+
+]]
 function Datastore:get_all(callback)
     if not self.parent then
         return filter(self.data, callback)
     else
-        local data, table_name = {}, self.table_name
+        local data, value_name = {}, self.value_name
         for key, value in pairs(self.parent:get_all()) do
-            data[key] = value[table_name]
+            data[key] = value[value_name]
         end
         return filter(data, callback)
     end
 end
 
---- Save all the keys in the datastore, optional filter callback
+--[[-- Update all keys in this datastore using the same update function
+@tparam function callback The update function that will be applied to each key
+
+@usage-- Get all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:update_all(function(key, value)
+    return value..value
+end)
+
+]]
+function Datastore:update_all(callback)
+    local data = self:get_all()
+    for key, value in pairs(data) do
+        local success, new_value = xpcall(callback, update_error, key, value)
+        if success and new_value ~= nil then
+            self:set(key, new_value)
+        else
+            self:raise_event('on_update', key, value)
+            if self.auto_save then self:save(key) end
+        end
+    end
+end
+
+----- Datastore External -----
+-- @section datastore-external
+
+--[[-- Request a value from an external source, will trigger on_load when data is received
+@tparam any key The key that you want to request from an external source, must be a string unless a serializer is set
+
+@usage-- Request a key from an external source, on_load is triggered when data is received
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:request('TestKey')
+
+]]
+function Datastore:request(key)
+    if self.parent then return self.parent:request(key) end
+    key = self:serialize(key)
+    self:write_action('request', key)
+end
+
+--[[-- Save a value to an external source, will trigger on_save before data is saved, save_to_disk must be set to true
+@tparam any key The key that you want to save to an external source, must be a string unless a serializer is set
+
+@usage-- Save a key to an external source, save_to_disk must be set to true for there to be any effect
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:save('TestKey')
+
+]]
+function Datastore:save(key)
+    if self.parent then self.parent:save(key) end
+    if not self.save_to_disk then return end
+    key = self:serialize(key)
+    local value  = self:raise_event('on_save', key, copy(self:raw_get(key)))
+    local action = self.propagate_changes and 'propagate' or 'save'
+    self:write_action(action, key, value)
+end
+
+--[[-- Save a value to an external source and remove locally, will trigger on_unload then on_save, save_to_disk is not required for on_unload
+@tparam any key The key that you want to unload from the datastore, must be a string unless a serializer is set
+
+@usage-- Unload a key from the datastore, get will now return nil and value will be saved externally if save_to_disk is set to true
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:unload('TestKey')
+
+]]
+function Datastore:unload(key)
+    if self.parent then return self.parent:unload(key) end
+    key = self:serialize(key)
+    self:raise_event('on_unload', key, copy(self:raw_get(key)))
+    self:save(key)
+    self:raw_set(key)
+end
+
+--[[-- Use to send a message over the connection, works regardless of saveToDisk and propagateChanges
+@tparam any key The key that you want to send a message over, must be a string unless a serializer is set
+@tparam any message The message that you want to send to other connected servers, or external source
+
+@usage-- Send a message to other servers on this key, can listen for messages with on_message
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:message('TestKey', 'Foo')
+
+]]
+function Datastore:message(key, message)
+    key = self:serialize(key)
+    self:write_action('message', key, message)
+end
+
+--[[-- Save all the keys in the datastore, optional filter callback
+@tparam[opt] function callback The filter function that can be used to filter the keys saved
+
+@usage-- Save all the data in this datastore
+local ExampleData = Datastore.connect('ExampleData')
+local data = ExampleData:save_all()
+
+@usage-- Save all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:save_all(function(key, value)
+    return type(value) == 'string'
+end)
+
+]]
 function Datastore:save_all(callback)
     local data = self:get_all(callback)
     for key in pairs(data) do self:save(key) end
 end
 
---- Unload all the keys in the datastore, optional filter callback
+--[[-- Unload all the keys in the datastore, optional filter callback
+@tparam[opt] function callback The filter function that can be used to filter the keys unloaded
+
+@usage-- Unload all the data in this datastore
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:unload_all()
+
+@usage-- Unload all the data in this datastore, with a filter
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:unload_all(function(key, value)
+    return type(value) == 'string'
+end)
+
+]]
 function Datastore:unload_all(callback)
     local data = self:get_all(callback)
     for key in pairs(data) do self:unload(key) end
@@ -307,13 +720,23 @@ end
 ----- Events -----
 -- @section events
 
---- Internal, Raise an event on this datastore
 local function event_error(err) print('An error ocurred in a datastore event handler:', err) end
+--[[-- Internal, Raise an event on this datastore
+@tparam string event_name The name of the event to raise for this datastore
+@tparam string key The key that this event is being raised for
+@tparam[opt] any value The current value that this key has, might be a deep copy of the value
+@tparam[opt] string source Where this call came from, used to do event recursion so can be parent or child
+@treturn any The value that is left after being passed through all the event handlers
+
+@usage-- Internal, Getting the value that should be saved
+value = self:raise_event('on_save', key, value)
+
+]]
 function Datastore:raise_event(event_name, key, value, source)
     -- Raise the event for the children of this datastore
     if source ~= 'child' then
-        for table_name, child in pairs(self.children) do
-            value[table_name] = child:raise_event(event_name, key, value[table_name], 'parent')
+        for value_name, child in pairs(self.children) do
+            value[value_name] = child:raise_event(event_name, key, value[value_name], 'parent')
         end
     end
 
@@ -333,7 +756,14 @@ function Datastore:raise_event(event_name, key, value, source)
     return value
 end
 
---- Internal, Returns a function which will add a callback to an event
+--[[-- Internal, Returns a function which will add a callback to an event
+@tparam string event_name The name of the event that this should create a handler adder for
+@treturn function The function that can be used to add handlers to this event
+
+@usage-- Internal, Get the function to add handlers to on_load
+Datastore.on_load = event_factory('on_load')
+
+]]
 local function event_factory(event_name)
     return function(self, callback)
         assert(type(callback) == 'function', 'Handler must be a function')
@@ -346,19 +776,54 @@ local function event_factory(event_name)
     end
 end
 
---- Register a callback that triggers when data is loaded from an external source, returned value is saved locally
+--[[-- Register a callback that triggers when data is loaded from an external source, returned value is saved locally
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_load, returned value will be saved locally, can be used to deserialize the value beyond a normal json
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_load(function(key, value)
+    game.print('Test data loaded for: '..key)
+end)
+]]
 Datastore.on_load = event_factory('on_load')
 
---- Register a callback that triggers before data is saved, returned value is saved externally
+--[[-- Register a callback that triggers before data is saved, returned value is saved externally
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_save, returned value will be saved externally, can be used to serialize the value beyond a normal json
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_save(function(key, value)
+    game.print('Test data saved for: '..key)
+end)
+]]
 Datastore.on_save = event_factory('on_save')
 
---- Register a callback that triggers before data is unloaded, returned value is ignored
+--[[-- Register a callback that triggers before data is unloaded, returned value is ignored
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_unload, returned value is ignored, can be used to clean up guis or local values related to this data
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_load(function(key, value)
+    game.print('Test data unloaded for: '..key)
+end)
+]]
 Datastore.on_unload = event_factory('on_unload')
 
---- Register a callback that triggers when a message is received, returned value is ignored
+--[[-- Register a callback that triggers when a message is received, returned value is ignored
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_message, returned value is ignored, can be used to receive messages from other connected servers without saving data
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_message(function(key, value)
+    game.print('Test data message for: '..key)
+end)
+]]
 Datastore.on_message = event_factory('on_message')
 
---- Register a callback that triggers any time a value is changed, returned value is ignored
+--[[-- Register a callback that triggers any time a value is changed, returned value is ignored
+@tparam function callback The handler that will be registered to the on_load event
+@usage-- Adding a handler to on_update, returned value is ignored, can be used to update guis or send messages when data is changed
+local ExampleData = Datastore.connect('ExampleData')
+ExampleData:on_update(function(key, value)
+    game.print('Test data updated for: '..key)
+end)
+]]
 Datastore.on_update = event_factory('on_update')
 
 ----- Module Return -----
diff --git a/expcore/player_data.lua b/expcore/player_data.lua
index 827abf1e..268cdaca 100644
--- a/expcore/player_data.lua
+++ b/expcore/player_data.lua
@@ -1,3 +1,45 @@
+--[[-- Core Module - PlayerData
+- A module used to store player data in a central datastore to minimize data requests and saves.
+@core PlayerData
+
+@usage-- Adding a colour setting for players
+local PlayerData = require 'expcore.player_data'
+local PlayerColors = PlayerData.Settings:combine('Color')
+
+-- Set the players color when their data is loaded
+PlayerColors:on_load(function(player_name, color)
+    local player = game.players[player_name]
+    player.color = color
+end)
+
+-- Overwrite the saved color with the players current color
+PlayerColors:on_save(function(player_name, _)
+    local player = game.players[player_name]
+    return player.color -- overwrite existing data with the current color
+end)
+
+@usage-- Add a playtime statistic for players
+local Event = require 'utils.event'
+local PlayerData = require 'expcore.player_data'
+local Playtime = PlayerData.Statistics:combine('Playtime')
+
+-- When playtime reaches an hour interval tell the player and say thanks
+Playtime:on_update(function(player_name, playtime)
+    if playtime % 60 == 0 then
+        local hours = playtime / 60
+        local player = game.players[player_name]
+        player.print('Thanks for playing on our servers, you have played for '..hours..' hours!')
+    end
+end)
+
+-- Update playtime for players, data is only loaded for online players so update_all can be used
+Event.add_on_nth_tick(3600, function()
+    Playtime:update_all(function(player_name, playtime)
+        return playtime + 1
+    end)
+end)
+
+]]
 
 local Event = require 'utils.event' --- @dep utils.event
 local Datastore = require 'expcore.datastore' --- @dep expcore.datastore
@@ -60,9 +102,9 @@ end)
 ----- Module Return -----
 return {
     All = PlayerData, -- Root for all of a players data
-    Statistics = PlayerData:combine('PlayerStatistics'), -- Common place for stats
-    Settings = PlayerData:combine('PlayerSettings'), -- Common place for settings
-    Required = PlayerData:combine('PlayerRequired'), -- Common place for required data
+    Statistics = PlayerData:combine('Statistics'), -- Common place for stats
+    Settings = PlayerData:combine('Settings'), -- Common place for settings
+    Required = PlayerData:combine('Required'), -- Common place for required data
     DataSavingPreference = DataSavingPreference, -- Stores what data groups will be saved
     PreferenceEnum = PreferenceEnum -- Enum for the allowed options for data saving preference
 }
\ No newline at end of file

From b17d74fff7c97da4ade239cb37c79118ae82f822 Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Tue, 26 May 2020 18:33:15 +0100
Subject: [PATCH 09/10] Removed unused variable

---
 modules/gui/debug/expcore_datastore_view.lua | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/gui/debug/expcore_datastore_view.lua b/modules/gui/debug/expcore_datastore_view.lua
index f7a9912f..cecd92c6 100644
--- a/modules/gui/debug/expcore_datastore_view.lua
+++ b/modules/gui/debug/expcore_datastore_view.lua
@@ -4,7 +4,6 @@ local Color = require 'utils.color_presets' --- @dep utils.color_presets
 local Model = require 'modules.gui.debug.model' --- @dep modules.gui.debug.model
 
 local dump = Model.dump
-local dump_text = Model.dump_text
 local concat = table.concat
 
 local Public = {}

From ddcda05ab9b868a3e64c8636c8ddfd20c158791e Mon Sep 17 00:00:00 2001
From: Cooldude2606 <cooldude260607@gmail.com>
Date: Tue, 26 May 2020 19:06:48 +0100
Subject: [PATCH 10/10] Fixed Datastore Docs

---
 expcore/datastore.lua | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/expcore/datastore.lua b/expcore/datastore.lua
index 4189aaef..f2f526ad 100644
--- a/expcore/datastore.lua
+++ b/expcore/datastore.lua
@@ -163,7 +163,7 @@ Event.on_load(function()
     end
 end)
 
------ Datastore Manager -----
+----- Datastore Manager
 -- @section datastoreManager
 
 --- Metatable used on datastores
@@ -224,6 +224,7 @@ function DatastoreManager.combine(datastoreName, subDatastoreName)
     return datastore:combine(subDatastoreName)
 end
 
+local function ingest_error(err) print('Datastore ingest error, Unable to parse json:', err) end
 --[[-- Ingest the result from a request, this is used through a rcon interface to sync data
 @tparam string action The action that should be done, can be: remove, message, propagate, or request
 @tparam string datastoreName The name of the datastore that should have the action done to it
@@ -234,7 +235,6 @@ end
 Datastore.ingest('request', 'ExampleData', 'TestKey', 'Foo')
 
 ]]
-local function ingest_error(err) print('Datastore ingest error, Unable to parse json:', err) end
 function DatastoreManager.ingest(action, datastoreName, key, valueJson)
     local datastore = assert(Datastores[datastoreName], 'Datastore ingest error, Datastore not found '..tostring(datastoreName))
     assert(type(action) == 'string', 'Datastore ingest error, Action is not a string got: '..type(action))
@@ -289,7 +289,7 @@ function DatastoreManager.name_serializer(rawKey)
     return rawKey.name
 end
 
------ Datastore Internal -----
+----- Datastore Internal
 -- @section datastore-internal
 
 --[[-- Debug, Get the debug info for this datastore
@@ -398,8 +398,8 @@ function Datastore:write_action(action, key, value)
     game.write_file('datastore.pipe', table.concat(data, ' ')..'\n', true, 0)
 end
 
------ Datastore -----
--- @section datastore
+----- Datastore Local
+-- @section datastore-local
 
 --[[-- Create a new datastore which is stores its data inside of this datastore
 @tparam string subDatastoreName The name of the datastore that will have its data stored in this datastore
@@ -615,7 +615,7 @@ function Datastore:update_all(callback)
     end
 end
 
------ Datastore External -----
+----- Datastore External
 -- @section datastore-external
 
 --[[-- Request a value from an external source, will trigger on_load when data is received
@@ -717,7 +717,7 @@ function Datastore:unload_all(callback)
     for key in pairs(data) do self:unload(key) end
 end
 
------ Events -----
+----- Events
 -- @section events
 
 local function event_error(err) print('An error ocurred in a datastore event handler:', err) end
@@ -826,5 +826,5 @@ end)
 ]]
 Datastore.on_update = event_factory('on_update')
 
------ Module Return -----
+----- Module Return
 return DatastoreManager
\ No newline at end of file