Fixed remoteMap stuff

Neural Dole v2

.gitignore

@@ -22,6 +22,7 @@ coverage
 
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
+dist/
 
 node_modules/*
 !node_modules/_app

.snyk

@@ -1,8 +1,45 @@
 # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.12.0
+version: v1.13.3
 ignore: {}
 # patches apply the minimum changes required to fix a vulnerability
 patch:
   'npm:lodash:20180130':
     - html5-validator > caporal > cli-table2 > lodash:
         patched: '2018-07-03T06:07:55.532Z'
+  'npm:hawk:20160119':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > hawk:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:http-signature:20150122':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > http-signature:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:mime:20170907':
+    - node-factorio-api > documentation > highlight.js > gear-lib > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > form-data > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:minimatch:20160620':
+    - node-factorio-api > documentation > highlight.js > gear > liftoff > findup-sync > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > gear > liftoff > findup-sync > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jslint > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jshint > cli > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jshint > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:request:20160119':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:tunnel-agent:20170305':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > tunnel-agent:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:uglify-js:20151024':
+    - node-factorio-api > documentation > highlight.js > gear-lib > handlebars > uglify-js:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > uglify-js:
+        patched: '2019-02-25T06:48:21.845Z'

.vs/ProjectSettings.json

@@ -0,0 +1,3 @@
+{
+  "CurrentProjectSetting": null
+}

.vs/VSWorkspaceState.json

@@ -0,0 +1,8 @@
+{
+  "ExpandedNodes": [
+    "",
+    "\\sharedPlugins"
+  ],
+  "SelectedNode": "\\.gitignore",
+  "PreviewInSolutionExplorer": false
+}

README.md

@@ -78,7 +78,7 @@ Clusterio exploits one of the games logging features, game.write_file and RCON t
 
 7. client.js on server2: We were allowed to import x of item y, run command /c remote.call("clusterio", "importMany", "{'copper-plate':120}")
 
-This process works the same for both items and liquids, independant on what mods are used. Yes, modded items are fully supported.
+This process works the same for both items and liquids, independent on what mods are used. Yes, modded items are fully supported.
 
 Clusterio can also do a few other neat things, such as giving you access to epoch time, syncing player inventories between servers, keeping track of playtime (playerManager plugin), teleporting trains between servers (trainTeleports) and exporting tons of factorio related statistics to Prometheus for graphing in grafana.
 

client.js

@@ -640,7 +640,9 @@ async function instanceManagement(instanceconfig) {
 									if(stuff[o] && !stuff[o].includes('\u0000\u0000')) {
 										try{
 											plugins[i].scriptOutput(stuff[o]);
-										}catch(e){console.error(e)}
+										}catch(e){
+											throw new Error(`Plugin ${pluginConfig.name} has a fileOutputSubscription but plugin.scriptOutput is not a function`)
+										}
 									}
 								}
 							},

config.json.dist

@@ -47,7 +47,9 @@
 		"entriesPerSecond": 0.01666666667
 	},
 	"logItemTransfers": false,
+	"__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.",
 	"disableFairItemDistribution": false,
+	"useNeuralNetDoleDivider": false,
 	"uploadModsToMaster": true,
 	"msBetweenCommands": 10,
 	"allowRemoteCommandExecution": true,

lib/dole_nn_base.js

@@ -0,0 +1,118 @@
+//Sigmoids...
+
+function sigmoid0(x)
+{
+	return x/(1+Math.abs(x));
+}
+function sigmoid1(x)
+{
+	if (x>=0.0) {
+		return 1.0
+	} else {
+		return 0.0
+	}
+}
+function sigmoid2(x)
+{
+	if (x>=0.0){
+		return x
+	} else {
+		return 0.0
+	}
+}
+function sigmoid3(x)
+{
+	return x;
+}
+
+//DOSE - Doses items for item X slave Y
+//Numreq - number of items that are requested for item X for slave Y
+//Instore - number of items for item X in master gauge (currently)
+//store_last_tick - Number of items for item X in master gauge at time of last tick
+//dole - Current dole for item X
+//carry - carry for item X slave Y
+//prev_req - Numreq for previous request for item X slave Y
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+
+//Returns: Array of 3 elements
+//0.Number of items to give in that dose
+//1.New dole for item X
+//2.New carry for item X slave Y
+function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj)
+{
+	numreq=Number(numreq);
+	instore=Number(instore);
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	
+	var b=Math.trunc(prev_req/instore_adj*numreq);
+	var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
+	var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b];//5..9
+	var inputs=inp1.concat(inp2);
+	var internal=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0];
+	var outputs=[0,0,0];
+	//NN: Please don't touch :), code is generated
+	
+outputs[0]=inputs[2]*3.101320028;
+outputs[0]=outputs[0]+inputs[7]*0.5471593142;
+outputs[0]=outputs[0]+inputs[3]*2.691268921;
+outputs[0]=outputs[0]+outputs[1]*-1.64255929;
+outputs[0]=outputs[0]+inputs[6]*0.7746122479;
+outputs[0]=sigmoid0(outputs[0]);
+outputs[1]=inputs[2]*-0.9946503043;
+outputs[1]=outputs[1]+inputs[1]*0.2128598392;
+outputs[1]=sigmoid0(outputs[1]);
+outputs[2]=sigmoid2(outputs[2]);
+
+	outputs[0]=Number(outputs[0]);
+	if (outputs[0]<0) outputs[0]=0;
+	outputs[0]=outputs[0]*numreq;
+	var res1=Math.round(outputs[0]);
+	if (res1>numreq) res1=numreq;
+	if (res1>instore) res1=instore;
+	if (res1<0) res1=0;
+	carry=sigmoid0(outputs[2]);
+	return [res1,sigmoid0(outputs[1]),carry];
+}
+
+//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
+//Instore - number of items for item X in master gauge (currently)
+//dole - Current dole for item X
+//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+
+
+//Returns: Array of 2 elements
+//0.New dole for item X
+//1.Guessed percentage average of items supplied in last 10 ticks
+function Tick(instore,dole,store_last_tick,numreq_total_adj)
+{
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj];
+	var internal=[0,0,0,0,0];
+	var outputs=[0,0];
+	//NN: Please don't touch :), code is generated
+	
+internal[2]=sigmoid0(internal[2]);
+outputs[0]=internal[2]*0.4049594998;
+outputs[0]=outputs[0]+1.0*-5.026565075;
+outputs[0]=outputs[0]+inputs[0]*0.9100194573;
+outputs[0]=outputs[0]+inputs[1]*-1.063795686;
+outputs[0]=sigmoid1(outputs[0]);
+outputs[1]=outputs[0]*1.049500823;
+outputs[1]=outputs[1]*inputs[0]*3.437163115;
+outputs[1]=sigmoid0(outputs[1]);
+
+
+
+	avg=outputs[1];
+	if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
+	if (avg>1) avg=1;
+	if (avg<0) avg=0;
+	
+	return [sigmoid0(outputs[0]),avg];
+}
+
+module.exports = {Dose,Tick}

lib/manager/instanceManager.js

@@ -0,0 +1,2 @@
+const fs = require("fs-extra");
+const needle = require("needle");

lib/scenarios/Hotpatch/control.lua

@@ -36,53 +36,97 @@ require 'hotpatch.gui'
 -- Any other use would require licensing/permission from Wube
 -- If you don't want to run this, either comment this out before creating your scenario, OR uninstall freeplay at runtime
 hotpatch_tools.static_mod('freeplay', '1.0.0', [===[
+local util = require("util")
 local silo_script = require("silo-script")
-local version = 1
+
+local created_items = function()
+  return
+  {
+    ["iron-plate"] = 8,
+    ["wood"] = 1,
+    ["pistol"] = 1,
+    ["firearm-magazine"] = 10,
+    ["burner-mining-drill"] = 1,
+    ["stone-furnace"] = 1
+  }
+end
+
+local respawn_items = function()
+  return
+  {
+    ["pistol"] = 1,
+    ["firearm-magazine"] = 10
+  }
+end
+
+for k,v in pairs(silo_script.get_events()) do
+  script.on_event(k, v)
+end
 
 script.on_event(defines.events.on_player_created, function(event)
   local player = game.players[event.player_index]
-  player.insert{name="iron-plate", count=8}
+  util.insert_safe(player, global.created_items)
-  player.insert{name="pistol", count=1}
+
-  player.insert{name="firearm-magazine", count=10}
+  local r = global.chart_distance or 200
-  player.insert{name="burner-mining-drill", count = 1}
+  player.force.chart(player.surface, {{player.position.x - r, player.position.y - r}, {player.position.x + r, player.position.y + r}})
-  player.insert{name="stone-furnace", count = 1}
+
-  player.force.chart(player.surface, {{player.position.x - 200, player.position.y - 200}, {player.position.x + 200, player.position.y + 200}})
+  if not global.skip_intro then
-  if (#game.players <= 1) then
+    if game.is_multiplayer() then
-    game.show_message_dialog{text = {"msg-intro"}}
+      player.print({"msg-intro"})
-  else
+    else
-    player.print({"msg-intro"})
+      game.show_message_dialog{text = {"msg-intro"}}
+    end
   end
-  silo_script.on_player_created(event)
+
+  silo_script.on_event(event)
 end)
 
 script.on_event(defines.events.on_player_respawned, function(event)
   local player = game.players[event.player_index]
-  player.insert{name="pistol", count=1}
+  util.insert_safe(player, global.respawn_items)
-  player.insert{name="firearm-magazine", count=10}
+  silo_script.on_event(event)
-end)
-
-script.on_event(defines.events.on_gui_click, function(event)
-  silo_script.on_gui_click(event)
-end)
-
-script.on_init(function()
-  global.version = version
-  silo_script.on_init()
-end)
-
-script.on_event(defines.events.on_rocket_launched, function(event)
-  silo_script.on_rocket_launched(event)
 end)
 
 script.on_configuration_changed(function(event)
-  if global.version ~= version then
+  global.created_items = global.created_items or created_items()
-    global.version = version
+  global.respawn_items = global.respawn_items or respawn_items()
-  end
   silo_script.on_configuration_changed(event)
 end)
 
+script.on_load(function()
+  silo_script.on_load()
+end)
+
+script.on_init(function()
+  global.created_items = created_items()
+  global.respawn_items = respawn_items()
+  silo_script.on_init()
+end)
+
 silo_script.add_remote_interface()
 silo_script.add_commands()
+
+remote.add_interface("freeplay",
+{
+  get_created_items = function()
+    return global.created_items
+  end,
+  set_created_items = function(map)
+    global.created_items = map
+  end,
+  get_respawn_items = function()
+    return global.respawn_items
+  end,
+  set_respawn_items = function(map)
+    global.respawn_items = map
+  end,
+  set_skip_intro = function(bool)
+    global.skip_intro = bool
+  end,
+  set_chart_distance = function(value)
+    global.chart_distance = tonumber(value)
+  end
+})
 ]===])
 
 -- Multi-file test

lib/scenarios/Hotpatch/hotpatch/commands.lua

@@ -61,7 +61,7 @@ sub_commands = {
         if (not param) or (param == '') then
             caller.print('Usage: /hotpatch [command] [parameters]')
             caller.print('Available commands:')
-            for k, v in pairs(sub_commands) do caller.print(k) end
+            for k, _ in pairs(sub_commands) do caller.print(k) end
             caller.print('Use /hotpatch help [command] for help on a particular command')
         else
             local text = help_commands[param]
@@ -130,6 +130,6 @@ _ENV.commands.add_command('hotpatch', 'Commands for hotpatch. Run /hotpatch help
     end
 end)
 
-]===])
+--]===])
 
 return true

lib/scenarios/Hotpatch/hotpatch/gui.lua

@@ -95,10 +95,10 @@ on_gui_click_handlers = {
             menu = left.add{type = 'frame', name = 'hotpatch-menu', direction = 'vertical'}
             menu.add{type = 'button', name = 'hotpatch-menu.IDE', caption = 'Mod IDE', tooltip = 'Open debugging GUI'}
             menu.add{type = 'button', name = 'hotpatch-menu.console', caption = 'Mod Console', tooltip = 'Open debugging console'}
-            menu.style.visible = true
+            menu.visible = true
             return
         end
-        menu.style.visible = not menu.style.visible
+        menu.visible = not menu.visible
     end,
     ['hotpatch-menu'] = function(e)
         local player = game.players[e.player_index]
@@ -107,7 +107,7 @@ on_gui_click_handlers = {
         local center = player.gui.center
 
         local menu = left['hotpatch-menu']
-        menu.style.visible = not menu.style.visible
+        menu.visible = not menu.visible
 
         on_gui_click_handlers[e.element.name](e)
     end,
@@ -132,22 +132,22 @@ on_gui_click_handlers = {
             local code = IDE_table.add{type = 'text-box', name = 'hotpatch-IDE-code'}
             code.word_wrap = true
             files.style.width = 400
-            files.style.cell_spacing = 0
+            files.style.cell_padding = 0
             files.style.top_padding = 0
             files.style.bottom_padding = 0
             code.style.width = 600
             code.style.height = 600
-            IDE.style.visible = false
+            IDE.visible = false
         end
 
         local IDE_dropdown = IDE['hotpatch-IDE-top']['hotpatch-IDE-mod-selector']
         IDE_dropdown.clear_items()
-        for k, v in ipairs(installed_mods) do
+        for _, v in ipairs(installed_mods) do
             IDE_dropdown.add_item(v.name)
         end
 
-        IDE.style.visible = not IDE.style.visible
+        IDE.visible = not IDE.visible
-        if IDE.style.visible then
+        if IDE.visible then
             player.opened = IDE
         else
             player.opened = nil
@@ -159,14 +159,14 @@ on_gui_click_handlers = {
         local left = mod_gui.get_frame_flow(player)
         local center = player.gui.center
 
-        local console = center['hotpatch-console']
+        local hotpatch_console = center['hotpatch-console']
-        if not console then
+        if not hotpatch_console then
-            console = center.add{type = 'frame', name = 'hotpatch-console', direction = 'vertical', caption = 'Hotpatch Console'}
+            hotpatch_console = center.add{type = 'frame', name = 'hotpatch-console', direction = 'vertical', caption = 'Hotpatch Console'}
-            local top_flow = console.add{type = 'flow', name = 'hotpatch-console-top', direction = 'horizontal'}
+            local top_flow = hotpatch_console.add{type = 'flow', name = 'hotpatch-console-top', direction = 'horizontal'}
             top_flow.add{type = 'label', name = 'hotpatch-console-mod-label', caption = 'Mod: '}
             local console_dropdown = top_flow.add{type = 'drop-down', name = 'hotpatch-console-mod-selector'}
             top_flow.add{type = 'label', name = 'hotpatch-console-mod-version', caption = 'Version: No mod selected'}
-            local bottom_flow = console.add{type = 'flow', name = 'hotpatch-console-bottom', direction = 'vertical'}
+            local bottom_flow = hotpatch_console.add{type = 'flow', name = 'hotpatch-console-bottom', direction = 'vertical'}
             local output = bottom_flow.add{type = 'text-box', name = 'hotpatch-console-output'}
             local bottom_input_flow = bottom_flow.add{type = 'flow', name = 'hotpatch-console-bottom-input', direction = 'horizontal'}
             local input = bottom_input_flow.add{type = 'textfield', name = 'hotpatch-console-input'}
@@ -174,17 +174,18 @@ on_gui_click_handlers = {
             output.style.width = 800
             output.style.height = 600
             input.style.width = 600
+			hotpatch_console.visible = false
         end
 
-        local console_dropdown = console['hotpatch-console-top']['hotpatch-console-mod-selector']
+        local console_dropdown = hotpatch_console['hotpatch-console-top']['hotpatch-console-mod-selector']
         console_dropdown.clear_items()
-        for k, v in ipairs(loaded_mods) do
+        for _, v in ipairs(loaded_mods) do
             console_dropdown.add_item(v.name)
         end
 
-        console.style.visible = not console.style.visible
+        hotpatch_console.visible = not hotpatch_console.visible
-        if console.style.visible then
+        if hotpatch_console.visible then
-            player.opened = console
+            player.opened = hotpatch_console
         else
             player.opened = nil
         end
@@ -196,10 +197,10 @@ on_gui_click_handlers = {
         local left = mod_gui.get_frame_flow(player)
         local center = player.gui.center
 
-        local console = center['hotpatch-console']
+        local hotpatch_console = center['hotpatch-console']
-        local output = console['hotpatch-console-bottom']['hotpatch-console-output']
+        local output = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-output']
-        local input = console['hotpatch-console-bottom']['hotpatch-console-bottom-input']['hotpatch-console-input']
+        local input = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-bottom-input']['hotpatch-console-input']
-        local console_dropdown = console['hotpatch-console-top']['hotpatch-console-mod-selector']
+        local console_dropdown = hotpatch_console['hotpatch-console-top']['hotpatch-console-mod-selector']
 
         if not player.admin then
             output.text = table.concat{output.text, '\n', 'Only admins!'}
@@ -235,9 +236,10 @@ on_gui_click_handlers = {
             output.text = table.concat{output.text, '\n', text}
         end
 
+        local success
         local code, err = load(input.text, input.text, 't', env)
         if code then
-            local success, err = pcall(code)
+            success, err = pcall(code)
             if not success then
                 output.text = table.concat{output.text, '\n', err}
             end
@@ -256,7 +258,7 @@ on_gui_click_handlers = {
 
         local element = e.element
         local previous
-        for k, v in pairs(center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-files']['hotpatch-IDE-files-table'].children) do
+        for _, v in pairs(center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-files']['hotpatch-IDE-files-table'].children) do
             if table.compare(v.style.font_color or {}, {r=1.0, g=1.0, b=0.0, a=1.0}) then
                 previous = v
                 break
@@ -306,8 +308,8 @@ on_gui_selection_state_changed_handlers = {
                 local mod = loaded_mods[index]
                 local version_label = center['hotpatch-console']['hotpatch-console-top']['hotpatch-console-mod-version']
                 version_label.caption = 'Version: ' .. mod.version
-                local console = center['hotpatch-console']
+                local hotpatch_console = center['hotpatch-console']
-                local output = console['hotpatch-console-bottom']['hotpatch-console-output']
+                local output = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-output']
                 output.text = ''
             end
         end
@@ -332,7 +334,7 @@ on_gui_selection_state_changed_handlers = {
                 local file = list.add{type = 'label', style = 'hoverable_bold_label', caption = 'control', name = 'hotpatch-IDE-file.control'}
                 file.style.bottom_padding = 0
                 file.style.top_padding = 0
-                for k, v in pairs(mod.files) do
+                for k, _ in pairs(mod.files) do
                     file = list.add{type = 'label', style = 'hoverable_bold_label', caption = k, name = 'hotpatch-IDE-file.' .. k}
                     file.style.bottom_padding = 0
                     file.style.top_padding = 0
@@ -356,20 +358,20 @@ script.on_event(defines.events.on_gui_closed, function(e)
     if e.gui_type == defines.gui_type.custom then
         if element and element.valid then
             if element.name == 'hotpatch-IDE' then
-                if element.style.visible then
+                if element.visible then
                     player.play_sound{path = "utility/gui_click"}
                 end
-                element.style.visible = false
+                element.visible = false
             elseif element.name == 'hotpatch-console' then
-                if element.style.visible then
+                if element.visible then
                     player.play_sound{path = "utility/gui_click"}
                 end
-                element.style.visible = false
+                element.visible = false
             end
         end
     end
 end)
 
-]===])
+--]===])
 
 return true

lib/scenarios/Hotpatch/hotpatch/mod-tools.lua

@@ -101,6 +101,8 @@ local mod_on_configuration_changed
 local on_event
 local on_nth_tick
 
+local register_all_events
+
 -- core handlers
 local on_init -- installs static mods, loads, runs, and calls mod_on_init
 local on_load -- loads, runs, and calls mod_on_load
@@ -363,7 +365,7 @@ debug_log({'hotpatch-info.logging-enabled'})
 
 -- track _ENV accesses
 setmetatable(_ENV, {
-	__index = function(t, k)
+	__index = function(_, k)
 		debug_log({'hotpatch-trace.nil-var-access', k}, nil, 3)
 		return nil
 	end,
@@ -382,7 +384,6 @@ debug_log({'hotpatch-info.loading-libs'})
 local libraries = {
     'camera',
     'flying_tags',
-    'inspect',
     'math3d',
     'mod-gui',
     'noise',
@@ -541,7 +542,7 @@ load_mod = function(installed_index)
             mod_script.on_nth_tick = function(tick, f)
                 if tick then
                     if type(tick) == 'table' then
-                        for k, v in pairs(tick) do
+                        for _, v in pairs(tick) do
                             mod_script.on_nth_tick(v, f)
                         end
                         return
@@ -556,7 +557,7 @@ load_mod = function(installed_index)
                 if event == defines.events.on_tick then
                     mod_obj.on_tick = f
                     if mod_obj.running then
-                        register_on_tick(mod_name, event)
+                        register_on_tick(mod_name)
                     end
                 else
                     mod_obj.on_event[event] = f
@@ -568,7 +569,7 @@ load_mod = function(installed_index)
             mod_script.on_nth_tick = function(tick, f)
                 if tick then
                     if type(tick) == 'table' then
-                        for k, v in pairs(tick) do
+                        for _, v in pairs(tick) do
                             mod_script.on_nth_tick(v, f)
                         end
                         return
@@ -578,11 +579,11 @@ load_mod = function(installed_index)
                         register_nth_tick(mod_name, tick)
                     end
                 else
-                    local on_nth_tick = mod_obj.on_nth_tick
+                    local mod_on_nth_tick = mod_obj.on_nth_tick
                     mod_obj.on_nth_tick = {}
                     if mod_obj.running then
-                        for k, v in pairs(on_nth_tick) do
+                        for _, v in pairs(mod_on_nth_tick) do
-                            register_nth_tick(mod_name, tick)
+                            register_nth_tick(mod_name, v)
                         end
                     end
                 end
@@ -650,7 +651,7 @@ load_mod = function(installed_index)
                 return remote.call(...)
             end,
             interfaces = setmetatable({}, {
-                __index = function(t, k) return remote.interfaces[k] end,
+                __index = function(_, k) return remote.interfaces[k] end,
                 __pairs = function(t) local function iter(t, k) local v; k, v = next(remote.interfaces, k); if v then return k, t[k] end; end; return iter, t, nil end
             })
         }
@@ -666,11 +667,11 @@ load_mod = function(installed_index)
                 return commands.remove_command(name)
             end,
             commands = setmetatable({}, {
-                __index = function(t, k) return commands.commands[k] end,
+                __index = function(_, k) return commands.commands[k] end,
                 __pairs = function(t) local function iter(t, k) local v; k, v = next(commands.commands, k); if v then return k, t[k] end; end; return iter, t, nil end
             }),
             game_commands = setmetatable({}, {
-                __index = function(t, k) return commands.game_commands[k] end,
+                __index = function(_, k) return commands.game_commands[k] end,
                 __pairs = function(t) local function iter(t, k) local v; k, v = next(commands.game_commands, k); if v then return k, t[k] end; end; return iter, t, nil end
             })
         }
@@ -710,7 +711,7 @@ load_mod = function(installed_index)
                         error(err)
                     end
                 end
-				local file = global.mods[installed_index].files[alt_path]
+				file = global.mods[installed_index].files[alt_path]
                 if file then
                     debug_log({'hotpatch-trace.load-require', alt_path}, mod_name)
                     local code, err = load(file, '[' .. mod_name .. '] ' .. alt_path .. '.lua', 'bt', env)
@@ -729,7 +730,7 @@ load_mod = function(installed_index)
                 env.package._current_path_in_package = oldbase
 				local lib = package.loaded[path]
 				if not lib then
-					debug_log(path .. ' not found', nil, 3)
+					debug_log({'hotpatch-error.path-not-found', path}, mod_name, 3)
                     error(path .. ' not found')
 				end
                 return lib
@@ -743,12 +744,12 @@ load_mod = function(installed_index)
 
 
         env['game'] = setmetatable({}, {
-            __index = function(t, k) return game[k] end,
+            __index = function(_, k) return game[k] end,
             __pairs = function(t) local function iter(t, k) local v; k, v = next(game, k); if v then return k, t[k] end; end; return iter, t, nil end
         })
 
         local mt = {}
-        mt.__index = function(t, k)
+        mt.__index = function(_, k)
             debug_log({'hotpatch-trace.nil-var-access', k}, nil, 3)
             return nil
         end
@@ -816,12 +817,12 @@ reset_mod = function(loaded_index)
 end
 
 reset_mod_events = function(loaded_index)
-    local loaded_index = find_loaded_mod(mod_name)
+    --local loaded_index = find_loaded_mod(mod_name)
+    local loaded_mod = loaded_mods[loaded_index]
 
-    if not loaded_index then
+    if not loaded_mod then
-        debug_log({'hotpatch-warning.reset-events-not-running'}, mod_name)
+        --debug_log({'hotpatch-warning.reset-events-not-running'}, mod_name)
     else
-        local loaded_mod = loaded_mods[loaded_index]
         loaded_mod.on_event = {}
         loaded_mod.on_nth_tick = {}
         loaded_mod.on_init = nil
@@ -866,7 +867,7 @@ run_mod = function(loaded_index)
 		-- if a mod is being loaded that already has global data, it must be restored after
 		-- running control.lua, discarding any changes made
 		if not table.compare(old_global, {}) then
-			for k,v in pairs(mod.env.global) do
+			for k,_ in pairs(mod.env.global) do
 				mod.env.global[k] = nil
 			end
 			for k,v in pairs(old_global) do
@@ -884,7 +885,7 @@ run_mod = function(loaded_index)
             mod.env.script.on_event = function(event, f)
                 if event == defines.events.on_tick then
                     mod.on_tick = f
-                    register_on_tick(mod_name, event)
+                    register_on_tick(mod_name)
                 else
                     mod.on_event[event] = f
                     register_event(mod_name, event)
@@ -893,18 +894,18 @@ run_mod = function(loaded_index)
             mod.env.script.on_nth_tick = function(tick, f)
                 if tick then
                     if type(tick) == 'table' then
-                        for k, v in pairs(tick) do
+                        for _, v in pairs(tick) do
-                            mod_script.on_nth_tick(v, f)
+                            mod.env.script.on_nth_tick(v, f)
                         end
                         return
                     end
                     mod.on_nth_tick[tick] = f
                     register_nth_tick(mod_name, tick)
                 else
-                    local on_nth_tick = mod.on_nth_tick
+                    local mod_on_nth_tick = mod.on_nth_tick
                     mod.on_nth_tick = {}
-                    for k, v in pairs(on_nth_tick) do
+                    for _, v in pairs(mod_on_nth_tick) do
-                        register_nth_tick(mod_name, tick)
+                        register_nth_tick(mod_name, v)
                     end
                 end
             end
@@ -962,7 +963,7 @@ else
     end
 end
 
-local register_all_events = function()
+register_all_events = function()
     --unregister all events
     script.on_event(defines.events, nil)
     script.on_nth_tick(nil, nil)
@@ -981,12 +982,12 @@ register_mod_events = function(loaded_index)
         debug_log({'hotpatch-trace.on-tick-event-registered'}, mod_name)
         script.on_event(defines.events.on_tick, on_tick)
     end
-    for k,v in pairs(mod.on_event) do 
+    for k,_ in pairs(mod.on_event) do
         local event_name = (event_names[k] or k)
         debug_log({'hotpatch-trace.on-event-registered', event_name}, mod_name)
         script.on_event(k, on_event)
     end
-    for k,v in pairs(mod.on_nth_tick) do 
+    for k,_ in pairs(mod.on_nth_tick) do
         debug_log({'hotpatch-trace.on-nth-tick-event-registered', k}, mod_name)
         script.on_nth_tick(k, on_nth_tick)
     end
@@ -1000,8 +1001,8 @@ mod_on_init = function(loaded_index)
         if mod.on_init then
             local success, result = xpcall(mod.on_init, debug.traceback)
             if not success then
-                debug_log('on_init failed for ' .. mod.name)
+				debug_log({'hotpatch-error.on-init-failed'}, mod_name)
-                debug_log(result)
+                debug_log(result, mod_name)
                 return false
             end
         end
@@ -1020,8 +1021,8 @@ mod_on_load = function(loaded_index)
 			--local old_global = table.deepcopy(mod.env.global)
             local success, result = xpcall(mod.on_load, debug.traceback)
             if not success then
-                debug_log('on_load failed for ' .. mod.name)
+                debug_log({'hotpatch-error.on-load-failed'}, mod_name)
-                debug_log(result)
+                debug_log(result, mod_name)
                 return false
             end
         end
@@ -1039,8 +1040,8 @@ mod_on_configuration_changed = function(loaded_index, config)
         if mod.on_configuration_changed then
             local success, result = xpcall(mod.on_configuration_changed, debug.traceback, config)
             if not success then
-                debug_log('on_configuration_changed failed for ' .. mod.name)
+				debug_log({'hotpatch-error.on-configuration-changed-failed'}, mod_name)
-                debug_log(result)
+                debug_log(result, mod_name)
                 return false
             end
         end
@@ -1212,7 +1213,7 @@ on_configuration_changed = function(config)
     debug_log({'hotpatch-info.on-configuration-changed'})
 	local failed_mods = {}
     for i = 1, #loaded_mods do
-        if not mod_on_configuration_changed(i) then
+        if not mod_on_configuration_changed(i, config) then
 			table.insert(failed_mods, i)
 		end
     end
@@ -1293,24 +1294,24 @@ local mod_tools_internal = setmetatable({
     loaded_mods = loaded_mods,
     loaded_libraries = loaded_libraries,
     installed_mods = setmetatable({}, {
-        __index = function(t, k)
+        __index = function(_, k)
             return global.mods[k]
         end,
-        __newindex = function(t, k, v)
+        __newindex = function(_, __, ___)
         -- do nothing
         end,
-        __len = function(t)
+        __len = function(_)
             return #global.mods
         end,
-        __pairs = function(t) local function iter(t, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
+        __pairs = function(t) local function iter(_, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
-        __ipairs = function(t) local function iter(t, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
+        __ipairs = function(t) local function iter(_, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
         __metatable = false,
     })
 },{
-    __index = function(t, k)
+    __index = function(_, k)
         debug_log({'hotpatch-error.invalid-API-access', k}, nil, 3)
     end,
-    __newindex = function(t, k, v)
+    __newindex = function(_, __, ___)
         -- do nothing
     end,
     -- Don't let mods muck around
@@ -1322,11 +1323,12 @@ local mod_tools = setmetatable({
     -- most code should use static_mod
     static_mod = static_mod, -- (name, version, code, files)
     set_debug_level = function(level)
-        debug_level = tonumber(_ENV.debug_settings.level) or debug_levels[_ENV.debug_settings.level]
+        --debug_level = tonumber(_ENV.debug_settings.level) or debug_levels[_ENV.debug_settings.level]
+		debug_level = tonumber(level) or debug_levels[level]
     end,
 },{
     __index = mod_tools_internal,
-    __newindex = function(t, k, v)
+    __newindex = function(_, __, ___)
         -- do nothing, read-only table
     end,
     -- Don't let mods muck around

lib/scenarios/Hotpatch/hotpatch/remote-interface.lua

@@ -65,7 +65,7 @@ remote_interface['install'] = function(mod_name, mod_version, mod_code, mod_file
     local caller = game.player or console
     if caller.admin then
 		if mod_files then
-			for k,v in pairs(mod_files) do
+			for k,_ in pairs(mod_files) do
 				if k == 'control' then
 					caller.print('ERROR: control.lua loaded twice for mod ' .. mod_name)
 					return
@@ -77,8 +77,8 @@ remote_interface['install'] = function(mod_name, mod_version, mod_code, mod_file
         if installed_index then
             local old_version = installed_mods[installed_index].version
             if old_version then
-                debug_log('WARNING: mod already exists: ' .. mod_name .. ' ' .. old_version)
+                debug_log({'hotpatch-warning.already-exists', mod_name, old_version})
-                debug_log('WARNING: reinstalling mod in-place: ' .. mod_name .. ' ' .. mod_version)
+                debug_log({'hotpatch-warning.reinstalling', mod_name, old_version})
             end
             if loaded_index then
                 unload_mod(loaded_index)
@@ -131,6 +131,8 @@ remote_interface['run'] = function(mod_name)
     if caller.admin then
         local installed_index = find_installed_mod(mod_name)
         local loaded_index = find_loaded_mod(mod_name)
+		local mod
+        local version
         if installed_index and not loaded_index then
             if not load_mod(installed_index) then
                 caller.print('compilation failed for mod ' .. mod_name)
@@ -139,10 +141,10 @@ remote_interface['run'] = function(mod_name)
             loaded_index = find_loaded_mod(mod_name)
         else
             if loaded_index then
-                local mod = loaded_mods[loaded_index]
+                mod = loaded_mods[loaded_index]
-                local version = mod.version
+                version = mod.version
-                debug_log('WARNING: mod already loaded: ' .. mod_name .. ' ' .. version)
+                debug_log({'hotpatch-warning.already-loaded', mod_name, version})
-                debug_log('WARNING: reinitializing: ' .. mod_name .. ' ' .. version)
+                debug_log({'hotpatch-warning.reinitializing', mod_name, version})
             end
         end
 
@@ -178,7 +180,7 @@ remote_interface['update'] = function(mod_name, mod_version, mod_code, mod_files
     local caller = game.player or console
     if caller.admin then
 		if mod_files then
-			for k,v in pairs(mod_files) do
+			for k,_ in pairs(mod_files) do
 				if k == 'control' then
 					caller.print('ERROR: control.lua loaded twice for mod ' .. mod_name)
 					return
@@ -186,6 +188,9 @@ remote_interface['update'] = function(mod_name, mod_version, mod_code, mod_files
 			end
 		end
         local installed_index = find_installed_mod(mod_name)
+		if installed_index then
+			;
+		end
         local loaded_index = find_loaded_mod(mod_name)
         local old_version
         if loaded_index then
@@ -245,16 +250,40 @@ remote_interface['install_mod_file'] = function(mod_name, mod_file, mod_file_cod
     end
 end
 
+remote_interface['exec'] = function(mod_name, exec_code)
+    local caller = game.player or console
+    if caller.admin then
+		local loaded_index = find_loaded_mod(mod_name)
+		local env = loaded_mods[loaded_index].env
+		
+		local success
+		local code, err = load(exec_code, exec_code, 't', env)
+		if code then
+			success, err = pcall(code)
+			if not success then
+				debug_log(err)
+			end
+		else
+			debug_log(err)
+		end
+	else
+		caller.print('You must be an admin to run this command.')
+    end
+end
+
 --TODO: most of this function
 remote_interface['clean'] = function(mod_name)
     -- Removes ALL mod data and reinitializes
     -- doesn't remove things a mod may add like surfaces, etc
     local caller = game.player or console
     if caller.admin then
-        mod_reset(mod_name)
+        --mod_reset(mod_name)
-        run_mod(mod_name)
+        --run_mod(mod_name)
-        mod_on_init(mod_name)
+        --mod_on_init(mod_name)
-        mod_on_configuration_changed(mod_name, {mod_changes = {mod_name={new_version=mod_version}}})
+		local loaded_index = find_loaded_mod(mod_name)
+		local mod = loaded_mods[loaded_index]
+        local version = mod.version
+        mod_on_configuration_changed(loaded_index, {mod_changes = {mod_name={new_version=version}}})
     else
         caller.print('You must be an admin to run this command.')
     end
@@ -274,7 +303,7 @@ remote.add_interface('hotpatch', remote_interface)
 
 debug_log({'hotpatch-info.complete', {'hotpatch-info.remote-installing'}})
 
-]===])
+--]===])
 
 --[[
 

lib/scenarios/Hotpatch/locale/en/hotpatch.cfg

@@ -65,13 +65,17 @@ contains-comments-console=comments from console will comment out the entire code
 reset-events-not-running=tried to reset events for mod that isn't running!
 remote-interface-exists=remote interface __1__ already exists, removing...
 command-exists=command __1__ already exists, removing...
-already-loaded=mod already loaded
+already-exists=mod already exists: __1__ __2__
-reinitializing=reinitializing...
+reinstalling=reinstalling mod in-place: __1__ __2__
-already-exists=mod already exists
+already-loaded=mod already loaded: __1__ __2__
-reinstalling=reinstalling mod in-place
+reinitializing=reinitializing: __1__ __2__
 
 [hotpatch-error]
 invalid-API-access=Invalid API access: __1__
 not-installed=mod not installed, cannot install file for mod that does not exist
 compilation-failed=compilation failed for mod
 execution-failed=execution failed for mod
+path-not-found=path '__1__' not found
+on-init-failed=on_init failed
+on-load-failed=on_load failed
+on-configuration-changed-failed=on_configuration_changed failed

master.js

@@ -131,7 +131,7 @@ if(!config.disablePrometheusPushgateway){
 	setInterval(() => {
 		registerMoreMetrics();
 		pushgateway.push({ jobName: 'clusterio', groupings: {instance: config.publicIP + ":" + config.masterPort, owner: config.username}}, function(err, resp, body) {})
-	}, 5000)
+	}, 15000)
 }
 
 // collect express request durations ms
@@ -258,7 +258,7 @@ db.items.addItem = function(object) {
 		if(this[object.name] && Number(this[object.name]) != NaN){
 			this[object.name] = Number(this[object.name]) + Number(object.count);
 		} else {
-			this[object.name] = object.count;
+			this[object.name] = Number(object.count);
 		}
 		return true;
 	}
@@ -283,29 +283,35 @@ process.on('SIGHUP', shutdown); // terminal closed
 async function shutdown() {
 	console.log('Ctrl-C...');
 	let exitStartTime = Date.now();
-	// set insane limit to slave length, if its longer than this we are probably being ddosed or something
+	try {
-	if(slaves && Object.keys(slaves).length < 2000000){
+		// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-		console.log("saving to slaves.json");
+		if(slaves && Object.keys(slaves).length < 2000000){
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
+			console.log("saving to slaves.json");
-	} else if(slaves) {
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
-		console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
+		} else if(slaves) {
-	}
+			console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
-	if(db.items && Object.keys(db.items).length < 50000){
-		console.log("saving to items.json");
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
-	} else if(slaves) {
-		console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
-	}
-	for(let i in masterPlugins){
-		let plugin = masterPlugins[i];
-		if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
-			let startTime = Date.now();
-			await plugin.main.onExit();
-			console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
 		}
+		if(db.items && Object.keys(db.items).length < 50000){
+			console.log("saving to items.json");
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
+		} else if(slaves) {
+			console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
+		}
+		for(let i in masterPlugins){
+			let plugin = masterPlugins[i];
+			if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){
+				let startTime = Date.now();
+				await plugin.main.onExit();
+				console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms");
+			}
+		}
+		console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
+		process.exit(0);
+	} catch(e) {
+		console.log(e)
+		console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms")
+		process.exit(1)
 	}
-	console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
-	process.exit(0);
 }
 
 /**
@@ -528,6 +534,13 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 		res.send("failure");
 	}
 });
+const routes_api_remove = require("./routes/api/remove.js")
+var neuralDole = null
+
+if(config.useNeuralNetDoleDivider)//Only initialize neural network when it's enabled, otherwise it might override gauge
+	neuralDole=new routes_api_remove.neuralDole({
+		items: db.items, gaugePrefix: prometheusPrefix
+	})
 /**
 POST endpoint to remove items from DB when client orders items.
 
@@ -539,11 +552,8 @@ POST endpoint to remove items from DB when client orders items.
 @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name]
 @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
 */
-_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 app.post("/api/remove", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
-	const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// save items we get
@@ -557,7 +567,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	if(slaves[object.instanceID]) object.instanceName = slaves[object.instanceID].instanceName;
 	let item = db.items[object.name]
 		// console.dir(doc);
-	if(!item
+	if((item === undefined)
 	||((
 		config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
 		&& object.name != "electricity")
@@ -567,6 +577,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		}
 		res.send({name:object.name, count:0});
 	} else if(config.disableFairItemDistribution){
+		// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
 		let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item));
 		db.items.removeItem({count: numberToRemove, name: object.name});
 		res.send({count: numberToRemove, name: object.name});
@@ -587,45 +598,27 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		});
 		//console.log(sentItemStatistics.data)
 		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+	} else if(config.useNeuralNetDoleDivider){
+		// use fancy neural net to calculate a "fair" dole division rate.
+		neuralDole.divider({
+			res,
+			object,
+			config,
+			sentItemStatisticsBySlaveID,
+			prometheusImportGauge
+		})
 	} else {
-		const originalCount = Number(object.count) || 0;
+		// Use dole division. Makes it really slow to drain out the last little bit.
-		object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+		routes_api_remove.doleDivider({
-		object.count = Math.round(object.count);
+			item,
-		if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
+			object,
-		
+			db,
-		// Update existing items if item name already exists
+			sentItemStatisticsBySlaveID,
-		if(Number(item) > Number(object.count)) {
+			config,
-			//If successful, increase dole
+			prometheusDoleFactorGauge,
-			_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+			prometheusImportGauge,
-			if(config.logItemTransfers){
+			req,res,
-				console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
+		})
-			}
-			if(db.items.removeItem({count: object.count, name: object.name})){
-				let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-				if(sentItemStatistics === undefined){
-					sentItemStatistics = new averagedTimeSeries({
-						maxEntries: config.itemStats.maxEntries,
-						entriesPerSecond: config.itemStats.entriesPerSecond,
-						mergeMode: "add",
-					}, console.log);
-				}
-				sentItemStatistics.add({
-					key:object.name,
-					value:object.count,
-				});
-				//console.log(sentItemStatistics.data)
-				sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-			}
-			
-			prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
-			prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
-			res.send({count: object.count, name: object.name});
-		} else {
-			// if we didn't have enough, attempt giving out a smaller amount next time
-			_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
-			res.send({name:object.name, count:0});
-			//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
-		}
 	}
 });
 

package.json

@@ -11,7 +11,9 @@
     "docs": "documentation build index.js --config documentation.yml -f html -o docs",
     "livedocs": "documentation serve --watch *.js lib/** --config documentation.yml",
     "install": "node lib/npmPostinstall.js",
-    "bundle-dependencies": "bundle-dependencies"
+    "bundle-dependencies": "bundle-dependencies",
+    "snyk-protect": "snyk protect",
+    "prepublish": "npm run snyk-protect"
   },
   "keywords": [
     "factorio"
@@ -65,7 +67,8 @@
     "socket.io-client": "^2.0.4",
     "socket.io-prometheus": "^0.2.1",
     "sync-request": "^4.0.1",
-    "underscore": "^1.8.3"
+    "underscore": "^1.8.3",
+    "snyk": "^1.134.2"
   },
   "devDependencies": {
     "documentation": "^5.4.0",

routes/api/remove.js

@@ -0,0 +1,180 @@
+const averagedTimeSeries = require("averaged-timeseries")
+
+const doleNN = require("./../../lib/dole_nn_base.js")
+
+const Prometheus = require('prom-client');
+
+class neuralDole {
+    
+    getRequestStats(itemname,samples)
+    {
+        var sum=0;
+        samples=Math.min(samples,(this.stats[itemname] || []).length-1);
+        if (samples<1) return 0.1;
+        for (var i = 1; i <= samples; i++) {
+            sum+=this.stats[itemname][i].req;
+        }
+        if (sum==0) return 0.1;
+        return sum/samples;
+    }
+    
+    constructor({
+        items, gaugePrefix
+    }){
+        // Set some storage variables for the dole divider
+        this.prometheusNNDoleGauge = new Prometheus.Gauge({
+            name: gaugePrefix+'nn_dole_gauge',
+            help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
+            labelNames: ["itemName"],
+        });
+        this.items = items
+        this.itemsLastTick = JSON.parse(JSON.stringify(items))
+        this.dole = {}
+        this.carry = {}
+        this.lastRequest = {}
+        this.stats=[]
+
+        setInterval(()=>{
+            for(let name in this.items){
+                // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties.
+                if(typeof this.items[name] !== "number") continue
+
+                let count = this.items[name]
+                let magicData = doleNN.Tick(
+                    count,
+                    this.dole[name],
+                    this.itemsLastTick[name],
+                    this.getRequestStats(name,5)
+                )
+                this.stats[name]=this.stats[name] || []
+                this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
+                if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
+                
+                this.dole[name] = magicData[0]
+                // DONE handle magicData[1] for graphing for our users
+                this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
+            }
+            this.itemsLastTick = JSON.parse(JSON.stringify(this.items))
+        }, 1000)
+    }
+    divider({
+        res,
+        object,
+        config,
+        sentItemStatisticsBySlaveID,
+        prometheusImportGauge
+    }){
+        let magicData = doleNN.Dose(
+            object.count, // numReq
+            this.items[object.name],
+            this.itemsLastTick[object.name] || 0,
+            this.dole[object.name],
+            this.carry[object.name+" "+object.instanceID] || 0,
+            this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
+            this.getRequestStats(object.name,5)
+        )
+        if ((this.stats[object.name] || []).length>0)
+        {
+            this.stats[object.name][0].req+=Number(object.count);
+            this.stats[object.name][0].given+=Number(magicData[0]);
+        }
+        this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
+        //0.Number of items to give in that dose
+        //1.New dole for item X
+        //2.New carry for item X slave Y
+        this.dole[object.name] = magicData[1]
+        this.carry[object.name+" "+object.instanceID] = magicData[2]
+        
+        // Remove item from DB and send it
+        if(this.items.removeItem({count: magicData[0], name: object.name})){
+            if(config.logItemTransfers){
+                console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName);
+            }
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:magicData[0],
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+            prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
+
+            res.send({
+                name: object.name,
+                count: magicData[0],
+            })
+        } else {
+            res.send({
+                name: object.name,
+                count: 0,
+            })
+        }        
+    }
+}
+
+_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
+function doleDivider({
+    item,
+    object,
+    db,
+    sentItemStatisticsBySlaveID,
+    config,
+    prometheusDoleFactorGauge,
+    prometheusImportGauge,
+    req,res,
+}){
+    const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
+    const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
+    
+    const originalCount = Number(object.count) || 0;
+    object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+    object.count = Math.round(object.count);
+    if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
+    
+    // Update existing items if item name already exists
+    if(Number(item) > Number(object.count)) {
+        //If successful, increase dole
+        _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+        if(config.logItemTransfers){
+            console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
+        }
+        if(db.items.removeItem({count: object.count, name: object.name})){
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:object.count,
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+        }
+        
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
+        res.send({count: object.count, name: object.name});
+    } else {
+        // if we didn't have enough, attempt giving out a smaller amount next time
+        _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        res.send({name:object.name, count:0});
+        //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
+    }
+}
+
+module.exports = {
+    doleDivider,
+    neuralDole
+}

sharedPlugins/remoteMap/chunkStore.js

@@ -1,100 +0,0 @@
-var mkdirp = require("mkdirp-promise");
-var mkdirpSync = require("mkdirp").sync;
-var fs = require("fs");
-var events = require("events");
-var path = require("path");
-
-var objectOps = require("./../../lib/objectOps");
-
-var LinvoDB = require("linvodb3");
-// LinvoDB.dbPath = "./database/linvodb/";
-
-
-let allowLogging = true;
-let log = string => (allowLogging)? console.log(string):""
-
-module.exports = class chunkStore {
-	constructor(name, chunkSize = 32, path = "./chunkStore/"){
-		if(!name || (typeof name != "string" && typeof name != "number")) throw new Error("chunkstore requires a name string as first param");
-		name = name.toString();
-		this.name = name;
-		this.chunkSize = chunkSize;
-		this.path = (path[path.length] == "/") ? path : path+"/";
-		mkdirpSync(path+name);
-		LinvoDB.dbPath = this.path;
-		
-		this.db = new LinvoDB(name, {}, {});
-		
-		this.eventEmitter = new events.EventEmitter();
-	}
-	setEntity(x/**{number}*/, y/**{number}*/, newEntity/**{object}*/){
-		return new Promise((resolve, reject) => {
-			x = Math.floor(x);
-			y = Math.floor(y);
-			this.db.find({x, y}, (err, docs) => {
-				if(typeof newEntity == "object"){
-					if(err) reject(err);
-					let entityDoc = {x, y, entity: newEntity};
-					if(docs && docs[0] && docs[0]._id){
-						entityDoc._id = docs[0]._id;
-					}
-					this.db.save(entityDoc, (err,docs) => {
-						if(err) reject(err);
-						resolve(docs);
-						this.eventEmitter.emit("change", docs)
-					});
-				} else {
-					docs.forEach(doc => {
-						doc.remove(()=>{
-							// removed :)
-						});
-					});
-					resolve();
-					this.eventEmitter.emit("change", {x,y});
-				}
-			});
-		});
-	}
-	getEntity(x, y){
-		return new Promise((resolve, reject) => {
-			this.db.find({x:Math.floor(x), y:Math.floor(y)}, (err, docs) => {
-				if(err) reject(err);
-				resolve(docs);
-			});
-		});
-	}
-	onEntityChange(eventhandler/**function*/){
-		if(typeof eventhandler == "function") {
-			this.eventEmitter.on("change", eventhandler);
-		}
-	}
-	getChunk(x, y){
-		return new Promise((resolve, reject) => {
-			
-			/*
-			// make new chunk in case we can't find an existing one
-			let chunk = {
-				dataObject: {},
-				position:{x,y}, // fancy ES6 feature that generates to {x:x, y:y}
-				chunkSize:this.chunkSize
-			};
-			fs.readFile(this.path+this.name+"/"+x+"/"+y, (err, chunkFile) => {
-				
-				if(err){
-					log(err)
-					resolve(chunk);
-				} else {
-					if(objectOps.isJSON(chunkFile)) {
-						log("returned from file")
-						let chunk = JSON.parse(chunkFile);
-						resolve(chunk);
-					} else {
-						log("file failed")
-						resolve(chunk);
-					}
-				}
-			}); // add x and y values as well and we are done?
-			*/
-		});
-	}
-}

sharedPlugins/remoteMap/chunkStore.spe_c.js

@@ -1,64 +0,0 @@
-var assert = require("assert");
-
-var chunkStore = require("./chunkStore.js");
-
-describe("remoteMap/chunkStore.js", () => {
-	// define some values for our test
-	const inserterX = 14.5; // rounds to 15
-	const inserterY = -17.5; // rounds to -17, +64 would be 47
-	const inserterName = "inserter";
-	
-	let store // make our store elevated
-	it("is a constructor", ()=>{
-		assert.equal(typeof chunkStore, "function");
-		// create a new store and elevate it
-		store = new chunkStore("910239"/* instanceID, use defaults for the rest */, undefined, "./database/unitTests/chunkStore/");
-		assert.equal(store.name, "910239");
-	});
-	it("allows you to store the locations of entities", done => {
-		store.setEntity(inserterX, inserterY, {name:inserterName, otherData:"can be arbitrary"}).then(() => {
-			done();
-			// nothing to do here really
-		});
-	});
-	it("returns entities at positions you query for", done => {
-		setTimeout(()=>{
-			store.getEntity(inserterX, inserterY).then(entities => {
-				entity = entities[0]
-				// console.log(entity)
-				assert.equal(entity.x, 14);
-				assert.equal(entity.y, -18);
-				assert.equal(entity.entity.name, "inserter");
-				assert.equal(entity.entity.otherData, "can be arbitrary");
-				done();
-			});
-		},300);
-	});
-	it("deletes entities when not provided with an object as 3rd arg", done => {
-		store.setEntity(inserterX, inserterY).then(entity => {
-			assert(entity === undefined);
-			setTimeout(()=>{
-				store.getEntity(inserterX, inserterY).then(entities => {
-					// console.log(entities)
-					assert.equal(entities.length, 0, "Why are we getting results, it should be empty?");
-					done();
-				});
-			},300);
-		});
-	});
-	describe("validates input by throwing when it is invalid", ()=>{
-		it("throws if called without a string as the first parameter", ()=>{
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore(undefined, undefined, "asdasda");
-			});
-			assert.throws(()=>{
-				let throwingChunkStore = new chunkStore({}, undefined, "asdasdas");
-			});
-		});
-		it("converts numbers to strings automatically if they are provided  as the first parameter", ()=>{
-			assert.ok(()=>{
-				let nonThrowingChunkStore = new chunkStore(42, undefined, "./database/unitTests/chunkStore/");
-			});
-		});
-	});
-});

sharedPlugins/remoteMap/index.js

@@ -8,7 +8,9 @@ module.exports = class remoteMap {
 		this.socket = extras.socket;
 		this.instances = {};
 		
-        this.socket.on("hello", () => this.socket.emit("registerMapServer"));
+        this.socket.on("hello", () => this.socket.emit("registerMapServer", {
+			instanceID: this.config.unique, // instanceID
+		}));
 
         this.socket.on("createEntity", req => {
 			// req = {name:"", position:{x,y}, ?direction:4}
@@ -29,7 +31,9 @@ module.exports = class remoteMap {
 			this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
         })
 		this.socket.on("getChunk", async (data, callback) => {
-			callback(await this.messageInterface(`/silent-command remote.call("remoteMap", "exportChunk", ${data.x}, ${data.y})`));
+			let chunkData = await this.messageInterface(`/silent-command remote.call("remoteMap", "exportChunk", ${data.x}, ${data.y})`)
+			console.log(`Sending chunk data: ${chunkData}`)
+			callback(chunkData);
 		})
 		
 		// detect new entities created/deleted on the server and update the master

sharedPlugins/remoteMap/masterPlugin.js

@@ -4,6 +4,11 @@ const sanitizer = require('sanitizer');
 
 const editorSockets = [];
 
+const blacklistedEntities = [
+	"fish",
+	"player",
+]
+
 class masterPlugin {
 	constructor({ config, pluginConfig, pluginPath, socketio, express }) {
 		this.config = config;
@@ -24,7 +29,8 @@ class masterPlugin {
 		// handle websocket connections
 		this.io.on("connection", socket => {
 			let instanceID = "unknown";
-			socket.on("registerSlave", data => {
+			socket.on("registerMapServer", data => {
+				console.log("Registered mapServer socket")
 				if (data.instanceID && !isNaN(Number(data.instanceID))) {
 					instanceID = data.instanceID;
 					this.slaves[instanceID] = socket;
@@ -66,12 +72,15 @@ class masterPlugin {
 				}}),10000)
 				
 				socket.on("createEntity", data => {
+					console.log(`Creating entity: ${JSON.stringify(data)}`)
 					this.slaves[Object.keys(this.slaves)[0]].emit("createEntity", data.entity)
 				})
 				socket.on("deleteEntity", data => {
+					console.log(`Deleting entity: ${JSON.stringify(data)}`)
 					this.slaves[Object.keys(this.slaves)[0]].emit("deleteEntity", data.entity)
 				})
 				socket.on("getChunk", (data, callback) => {
+					console.log(`Getting chunk x:${data.x} y:${data.y} from slave`)
 					this.slaves[Object.keys(this.slaves)[0]].emit("getChunk", data, resp => {
 						// parse response
 						// |name:transport-belt,direction:2,x:99,y:1231|name:assembly-machine,direction:6,x:939,y:88
@@ -87,7 +96,7 @@ class masterPlugin {
 							!str.includes("crude-oil") &&
 							!str.includes("remnants") &&
 							!str.includes("cliff") &&
-							!str.includes("player") &&
+							!blacklistedEntities.includes(str) &&
 							!str.includes("ore"))
 							.map(ent => ent.split(",")													// split entity into properties
 								.map(prop => [prop.split(":")[0], prop.split(":")[1]])					// split properties into KV pairs
@@ -102,6 +111,11 @@ class masterPlugin {
 								return entity
 							})
 							.filter(entity => Object.keys(entity).length > 1)
+							.filter(entity => !blacklistedEntities.includes(entity.name))
+							/*.map(entity => {
+								if(entity.name.includes("tree")) entity.name = "tree"
+								return entity
+							})*/
 						)
 					})
 				})

sharedPlugins/scenario_loader/index.js

@@ -12,14 +12,6 @@ module.exports = class remoteCommands {
 			messageInterface("Hotpach installation status: "+hotpatchInstallStatus);
 			if(hotpatchInstallStatus){
 				await this.loadScenariosFromFolder("scenarios") // located in project root folder (with sharedMods, sharedPlugins etc)
-				let plugins = new AsyncArray((await fs.readdir(path.join(__dirname, "../"))))
-				plugins.filterAsync(async plugin => {
-					// check if plugin is a folder and if plugin has a scenarios folder
-					
-				})
-				for(let g = 0; g < plugins.length; g++){
-					
-				}
 			}
 		})();
 	}
@@ -27,7 +19,7 @@ module.exports = class remoteCommands {
 		await fs.ensureDir(scenarioDir)
 		let scenarios = await fs.readdir(scenarioDir);
 		for(let i = 0; i < scenarios.length; i++){
-			messageInterface(`Loading scenario ${scenarios[i]}`)
+			this.messageInterface(`Loading scenario ${scenarios[i]}`)
 			let startTime = Date.now();
 			let scenPath = path.join(scenarioDir, scenarios[i]);
 			let stat = await fs.stat(scenPath);
@@ -68,9 +60,9 @@ module.exports = class remoteCommands {
 					if(name != "control") fileImportString += `["${name}"] = '${files[k]}', `;
 				}
 				fileImportString += `\}`;
-				if(files.control) var returnValue = await messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
+				if(files.control) var returnValue = await this.messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
-				if(returnValue) messageInterface(returnValue);
+				if(returnValue) this.messageInterface(returnValue);
-				messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
+				this.messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
 			}
 		}
 	}
@@ -108,20 +100,17 @@ function readDirectorySynchronously(directory) {
 }
 class AsyncArray extends Array {
 	constructor(arr) {
-		// this.data = arr; // In place of Array subclassing
+		super(arr)
+		this.data = arr; // In place of Array subclassing
 	}
-
+	async filterAsync(predicate) {
-	filterAsync(predicate) {
 		// Take a copy of the array, it might mutate by the time we've finished
 		const data = Array.from(this.data);
 		// Transform all the elements into an array of promises using the predicate
 		// as the promise
-		return Promise.all(data.map((element, index) => predicate(element, index, data)))
+		const result = await Promise.all(data.map((element, index) => predicate(element, index, data)));
-		// Use the result of the promises to call the underlying sync filter function
+		return data.filter((element, index) => {
-		.then(result => {
+			return result[index];
-			return data.filter((element, index) => {
-				return result[index];
-			});
 		});
 	}
 }