Fixed remoteMap stuff

Neural Dole v2

.gitignore

@@ -22,6 +22,7 @@ coverage
 
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
+dist/
 
 node_modules/*
 !node_modules/_app

.snyk

@@ -1,8 +1,45 @@
 # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.12.0
+version: v1.13.3
 ignore: {}
 # patches apply the minimum changes required to fix a vulnerability
 patch:
   'npm:lodash:20180130':
     - html5-validator > caporal > cli-table2 > lodash:
         patched: '2018-07-03T06:07:55.532Z'
+  'npm:hawk:20160119':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > hawk:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:http-signature:20150122':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > http-signature:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:mime:20170907':
+    - node-factorio-api > documentation > highlight.js > gear-lib > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > form-data > mime:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:minimatch:20160620':
+    - node-factorio-api > documentation > highlight.js > gear > liftoff > findup-sync > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > gear > liftoff > findup-sync > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jslint > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jshint > cli > glob > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > jshint > minimatch:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:request:20160119':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:tunnel-agent:20170305':
+    - node-factorio-api > documentation > highlight.js > gear-lib > less > request > tunnel-agent:
+        patched: '2019-02-25T06:48:21.845Z'
+  'npm:uglify-js:20151024':
+    - node-factorio-api > documentation > highlight.js > gear-lib > handlebars > uglify-js:
+        patched: '2019-02-25T06:48:21.845Z'
+    - node-factorio-api > documentation > highlight.js > gear-lib > uglify-js:
+        patched: '2019-02-25T06:48:21.845Z'

.vs/ProjectSettings.json

@@ -0,0 +1,3 @@
+{
+  "CurrentProjectSetting": null
+}

.vs/VSWorkspaceState.json

@@ -0,0 +1,8 @@
+{
+  "ExpandedNodes": [
+    "",
+    "\\sharedPlugins"
+  ],
+  "SelectedNode": "\\.gitignore",
+  "PreviewInSolutionExplorer": false
+}

Dockerfile

@@ -2,22 +2,20 @@ FROM node:9.11.2
 RUN apt-get update && apt install git curl tar -y
 RUN mkdir factorioClusterio
 
-#RUN git clone -b master https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install
+RUN git clone -b master https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install
-COPY . /factorioClusterio
+RUN cd factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz
-RUN cd /factorioClusterio && npm install
-RUN cd /factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
 
-WORKDIR /factorioClusterio
+WORKDIR factorioClusterio
 RUN mkdir instances sharedMods
 RUN cp config.json.dist config.json
 
 RUN node client.js download
 
-LABEL maintainer "danielv@live.no"
+LABEL maintainer "Sir3lit@gmail.com"
 
 EXPOSE 8080 34167
 VOLUME /factorioClusterio/instances
 VOLUME /factorioClusterio/sharedMods
 VOLUME /factorioClusterio/sharedPlugins
 
-CMD RCONPORT="$RCONPORT" FACTORIOPORT="34167" MODE="$MODE" node $MODE\.js start $INSTANCE
+CMD RCONPORT="$RCONPORT" FACTORIOPORT="$FACTORIOPORT" MODE="$MODE" node $MODE\.js start $INSTANCE

README.md

@@ -78,7 +78,7 @@ Clusterio exploits one of the games logging features, game.write_file and RCON t
 
 7. client.js on server2: We were allowed to import x of item y, run command /c remote.call("clusterio", "importMany", "{'copper-plate':120}")
 
-This process works the same for both items and liquids, independant on what mods are used. Yes, modded items are fully supported.
+This process works the same for both items and liquids, independent on what mods are used. Yes, modded items are fully supported.
 
 Clusterio can also do a few other neat things, such as giving you access to epoch time, syncing player inventories between servers, keeping track of playtime (playerManager plugin), teleporting trains between servers (trainTeleports) and exporting tons of factorio related statistics to Prometheus for graphing in grafana.
 

client.js

@@ -640,7 +640,9 @@ async function instanceManagement(instanceconfig) {
 									if(stuff[o] && !stuff[o].includes('\u0000\u0000')) {
 										try{
 											plugins[i].scriptOutput(stuff[o]);
-										}catch(e){console.error(e)}
+										}catch(e){
+											throw new Error(`Plugin ${pluginConfig.name} has a fileOutputSubscription but plugin.scriptOutput is not a function`)
+										}
 									}
 								}
 							},

config.json.dist

@@ -47,7 +47,9 @@
 		"entriesPerSecond": 0.01666666667
 	},
 	"logItemTransfers": false,
+	"__comment6.5": "When the master is undersupplied, we have 3 different modes to 'fairly' distribute items. First to ask gets all, dole division using weird formula or experimental neural net tries to make stuff as fair as possible. Feel free to play around.",
 	"disableFairItemDistribution": false,
+	"useNeuralNetDoleDivider": false,
 	"uploadModsToMaster": true,
 	"msBetweenCommands": 10,
 	"allowRemoteCommandExecution": true,

lib/dole_nn_base.js

@@ -0,0 +1,118 @@
+//Sigmoids...
+
+function sigmoid0(x)
+{
+	return x/(1+Math.abs(x));
+}
+function sigmoid1(x)
+{
+	if (x>=0.0) {
+		return 1.0
+	} else {
+		return 0.0
+	}
+}
+function sigmoid2(x)
+{
+	if (x>=0.0){
+		return x
+	} else {
+		return 0.0
+	}
+}
+function sigmoid3(x)
+{
+	return x;
+}
+
+//DOSE - Doses items for item X slave Y
+//Numreq - number of items that are requested for item X for slave Y
+//Instore - number of items for item X in master gauge (currently)
+//store_last_tick - Number of items for item X in master gauge at time of last tick
+//dole - Current dole for item X
+//carry - carry for item X slave Y
+//prev_req - Numreq for previous request for item X slave Y
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+
+//Returns: Array of 3 elements
+//0.Number of items to give in that dose
+//1.New dole for item X
+//2.New carry for item X slave Y
+function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj)
+{
+	numreq=Number(numreq);
+	instore=Number(instore);
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	
+	var b=Math.trunc(prev_req/instore_adj*numreq);
+	var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
+	var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b];//5..9
+	var inputs=inp1.concat(inp2);
+	var internal=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0];
+	var outputs=[0,0,0];
+	//NN: Please don't touch :), code is generated
+	
+outputs[0]=inputs[2]*3.101320028;
+outputs[0]=outputs[0]+inputs[7]*0.5471593142;
+outputs[0]=outputs[0]+inputs[3]*2.691268921;
+outputs[0]=outputs[0]+outputs[1]*-1.64255929;
+outputs[0]=outputs[0]+inputs[6]*0.7746122479;
+outputs[0]=sigmoid0(outputs[0]);
+outputs[1]=inputs[2]*-0.9946503043;
+outputs[1]=outputs[1]+inputs[1]*0.2128598392;
+outputs[1]=sigmoid0(outputs[1]);
+outputs[2]=sigmoid2(outputs[2]);
+
+	outputs[0]=Number(outputs[0]);
+	if (outputs[0]<0) outputs[0]=0;
+	outputs[0]=outputs[0]*numreq;
+	var res1=Math.round(outputs[0]);
+	if (res1>numreq) res1=numreq;
+	if (res1>instore) res1=instore;
+	if (res1<0) res1=0;
+	carry=sigmoid0(outputs[2]);
+	return [res1,sigmoid0(outputs[1]),carry];
+}
+
+//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
+//Instore - number of items for item X in master gauge (currently)
+//dole - Current dole for item X
+//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+
+
+//Returns: Array of 2 elements
+//0.New dole for item X
+//1.Guessed percentage average of items supplied in last 10 ticks
+function Tick(instore,dole,store_last_tick,numreq_total_adj)
+{
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj];
+	var internal=[0,0,0,0,0];
+	var outputs=[0,0];
+	//NN: Please don't touch :), code is generated
+	
+internal[2]=sigmoid0(internal[2]);
+outputs[0]=internal[2]*0.4049594998;
+outputs[0]=outputs[0]+1.0*-5.026565075;
+outputs[0]=outputs[0]+inputs[0]*0.9100194573;
+outputs[0]=outputs[0]+inputs[1]*-1.063795686;
+outputs[0]=sigmoid1(outputs[0]);
+outputs[1]=outputs[0]*1.049500823;
+outputs[1]=outputs[1]*inputs[0]*3.437163115;
+outputs[1]=sigmoid0(outputs[1]);
+
+
+
+	avg=outputs[1];
+	if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
+	if (avg>1) avg=1;
+	if (avg<0) avg=0;
+	
+	return [sigmoid0(outputs[0]),avg];
+}
+
+module.exports = {Dose,Tick}

lib/getDatabase.ts

@@ -1,15 +0,0 @@
-import { getItemDatabase } from "./getItemDatabase";
-import { getSlaveDatabase } from "./getSlaveDatabase";
-
-export type db = {
-    items: {[name:string]: any},
-    slaves: Slaves,
-}
-
-export function getDatabase(config: ClusterioConfig) {
-    let db: db = {
-        items: getItemDatabase(config),
-        slaves: getSlaveDatabase(config),
-    };
-    return db
-}

lib/getFactorioLocale.d.ts

@@ -1,6 +0,0 @@
-declare module "getFactorioLocale" {
-    interface callback {
-        (error: Error, factorioLocale?: any): void
-    }
-    export function asObject(factorioDirectory: string, languageCode: string, callback: callback): void
-}

lib/getItemDatabase.ts

@@ -1,40 +0,0 @@
-import * as fs from "fs"
-import * as path from "path"
-
-export function getItemDatabase(config: ClusterioConfig): {[name:string]: number | Function} {
-	var items: {[name: string]: number | Function} = {}
-	try{
-		let x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
-		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
-		items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")).toString());
-	} catch (e){
-		items = {};
-	}
-	items.addItem = function addItem(object: item): boolean {
-		if(object.name == "addItem" || object.name == "removeItem") {
-			console.error("Fuck you, that would screw everything up if you named your item that.");
-			return false;
-		} else {
-			if(this[object.name] && Number(this[object.name]) != NaN){
-				this[object.name] = Number(this[object.name]) + Number(object.count);
-			} else {
-				this[object.name] = object.count;
-			}
-			return true;
-		}
-	},
-	items.removeItem = function removeItem(object: item): boolean {
-		if(object.name == "addItem" || object.name == "removeItem") {
-			console.error("Fuck you, that would screw everything up if you named your item that.");
-			return false;
-		} else {
-			if(this[object.name] && Number(this[object.name]) != NaN){
-				this[object.name] = Number(this[object.name]) - Number(object.count);
-			} else {
-				this[object.name] = 0;
-			}
-			return true;
-		}
-	}
-	return items
-}

lib/getSlaveDatabase.ts

@@ -1,14 +0,0 @@
-import * as fs from "fs"
-import * as path from "path"
-
-export function getSlaveDatabase(config: ClusterioConfig): Slaves {
-    var slaves: Slaves = {};
-    try{
-		let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json"));
-		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
-		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json"), "utf-8"));
-	} catch (e){
-		slaves = {};
-    }
-    return slaves
-}

lib/manager/instanceManager.js

@@ -0,0 +1,2 @@
+const fs = require("fs-extra");
+const needle = require("needle");

lib/scenarios/Hotpatch/control.lua

@@ -36,53 +36,97 @@ require 'hotpatch.gui'
 -- Any other use would require licensing/permission from Wube
 -- If you don't want to run this, either comment this out before creating your scenario, OR uninstall freeplay at runtime
 hotpatch_tools.static_mod('freeplay', '1.0.0', [===[
+local util = require("util")
 local silo_script = require("silo-script")
-local version = 1
+
+local created_items = function()
+  return
+  {
+    ["iron-plate"] = 8,
+    ["wood"] = 1,
+    ["pistol"] = 1,
+    ["firearm-magazine"] = 10,
+    ["burner-mining-drill"] = 1,
+    ["stone-furnace"] = 1
+  }
+end
+
+local respawn_items = function()
+  return
+  {
+    ["pistol"] = 1,
+    ["firearm-magazine"] = 10
+  }
+end
+
+for k,v in pairs(silo_script.get_events()) do
+  script.on_event(k, v)
+end
 
 script.on_event(defines.events.on_player_created, function(event)
   local player = game.players[event.player_index]
-  player.insert{name="iron-plate", count=8}
+  util.insert_safe(player, global.created_items)
-  player.insert{name="pistol", count=1}
+
-  player.insert{name="firearm-magazine", count=10}
+  local r = global.chart_distance or 200
-  player.insert{name="burner-mining-drill", count = 1}
+  player.force.chart(player.surface, {{player.position.x - r, player.position.y - r}, {player.position.x + r, player.position.y + r}})
-  player.insert{name="stone-furnace", count = 1}
+
-  player.force.chart(player.surface, {{player.position.x - 200, player.position.y - 200}, {player.position.x + 200, player.position.y + 200}})
+  if not global.skip_intro then
-  if (#game.players <= 1) then
+    if game.is_multiplayer() then
-    game.show_message_dialog{text = {"msg-intro"}}
-  else
       player.print({"msg-intro"})
+    else
+      game.show_message_dialog{text = {"msg-intro"}}
     end
-  silo_script.on_player_created(event)
+  end
+
+  silo_script.on_event(event)
 end)
 
 script.on_event(defines.events.on_player_respawned, function(event)
   local player = game.players[event.player_index]
-  player.insert{name="pistol", count=1}
+  util.insert_safe(player, global.respawn_items)
-  player.insert{name="firearm-magazine", count=10}
+  silo_script.on_event(event)
-end)
-
-script.on_event(defines.events.on_gui_click, function(event)
-  silo_script.on_gui_click(event)
-end)
-
-script.on_init(function()
-  global.version = version
-  silo_script.on_init()
-end)
-
-script.on_event(defines.events.on_rocket_launched, function(event)
-  silo_script.on_rocket_launched(event)
 end)
 
 script.on_configuration_changed(function(event)
-  if global.version ~= version then
+  global.created_items = global.created_items or created_items()
-    global.version = version
+  global.respawn_items = global.respawn_items or respawn_items()
-  end
   silo_script.on_configuration_changed(event)
 end)
 
+script.on_load(function()
+  silo_script.on_load()
+end)
+
+script.on_init(function()
+  global.created_items = created_items()
+  global.respawn_items = respawn_items()
+  silo_script.on_init()
+end)
+
 silo_script.add_remote_interface()
 silo_script.add_commands()
+
+remote.add_interface("freeplay",
+{
+  get_created_items = function()
+    return global.created_items
+  end,
+  set_created_items = function(map)
+    global.created_items = map
+  end,
+  get_respawn_items = function()
+    return global.respawn_items
+  end,
+  set_respawn_items = function(map)
+    global.respawn_items = map
+  end,
+  set_skip_intro = function(bool)
+    global.skip_intro = bool
+  end,
+  set_chart_distance = function(value)
+    global.chart_distance = tonumber(value)
+  end
+})
 ]===])
 
 -- Multi-file test

lib/scenarios/Hotpatch/hotpatch/commands.lua

@@ -61,7 +61,7 @@ sub_commands = {
         if (not param) or (param == '') then
             caller.print('Usage: /hotpatch [command] [parameters]')
             caller.print('Available commands:')
-            for k, v in pairs(sub_commands) do caller.print(k) end
+            for k, _ in pairs(sub_commands) do caller.print(k) end
             caller.print('Use /hotpatch help [command] for help on a particular command')
         else
             local text = help_commands[param]
@@ -130,6 +130,6 @@ _ENV.commands.add_command('hotpatch', 'Commands for hotpatch. Run /hotpatch help
     end
 end)
 
-]===])
+--]===])
 
 return true

lib/scenarios/Hotpatch/hotpatch/gui.lua

@@ -95,10 +95,10 @@ on_gui_click_handlers = {
             menu = left.add{type = 'frame', name = 'hotpatch-menu', direction = 'vertical'}
             menu.add{type = 'button', name = 'hotpatch-menu.IDE', caption = 'Mod IDE', tooltip = 'Open debugging GUI'}
             menu.add{type = 'button', name = 'hotpatch-menu.console', caption = 'Mod Console', tooltip = 'Open debugging console'}
-            menu.style.visible = true
+            menu.visible = true
             return
         end
-        menu.style.visible = not menu.style.visible
+        menu.visible = not menu.visible
     end,
     ['hotpatch-menu'] = function(e)
         local player = game.players[e.player_index]
@@ -107,7 +107,7 @@ on_gui_click_handlers = {
         local center = player.gui.center
 
         local menu = left['hotpatch-menu']
-        menu.style.visible = not menu.style.visible
+        menu.visible = not menu.visible
 
         on_gui_click_handlers[e.element.name](e)
     end,
@@ -132,22 +132,22 @@ on_gui_click_handlers = {
             local code = IDE_table.add{type = 'text-box', name = 'hotpatch-IDE-code'}
             code.word_wrap = true
             files.style.width = 400
-            files.style.cell_spacing = 0
+            files.style.cell_padding = 0
             files.style.top_padding = 0
             files.style.bottom_padding = 0
             code.style.width = 600
             code.style.height = 600
-            IDE.style.visible = false
+            IDE.visible = false
         end
 
         local IDE_dropdown = IDE['hotpatch-IDE-top']['hotpatch-IDE-mod-selector']
         IDE_dropdown.clear_items()
-        for k, v in ipairs(installed_mods) do
+        for _, v in ipairs(installed_mods) do
             IDE_dropdown.add_item(v.name)
         end
 
-        IDE.style.visible = not IDE.style.visible
+        IDE.visible = not IDE.visible
-        if IDE.style.visible then
+        if IDE.visible then
             player.opened = IDE
         else
             player.opened = nil
@@ -159,14 +159,14 @@ on_gui_click_handlers = {
         local left = mod_gui.get_frame_flow(player)
         local center = player.gui.center
 
-        local console = center['hotpatch-console']
+        local hotpatch_console = center['hotpatch-console']
-        if not console then
+        if not hotpatch_console then
-            console = center.add{type = 'frame', name = 'hotpatch-console', direction = 'vertical', caption = 'Hotpatch Console'}
+            hotpatch_console = center.add{type = 'frame', name = 'hotpatch-console', direction = 'vertical', caption = 'Hotpatch Console'}
-            local top_flow = console.add{type = 'flow', name = 'hotpatch-console-top', direction = 'horizontal'}
+            local top_flow = hotpatch_console.add{type = 'flow', name = 'hotpatch-console-top', direction = 'horizontal'}
             top_flow.add{type = 'label', name = 'hotpatch-console-mod-label', caption = 'Mod: '}
             local console_dropdown = top_flow.add{type = 'drop-down', name = 'hotpatch-console-mod-selector'}
             top_flow.add{type = 'label', name = 'hotpatch-console-mod-version', caption = 'Version: No mod selected'}
-            local bottom_flow = console.add{type = 'flow', name = 'hotpatch-console-bottom', direction = 'vertical'}
+            local bottom_flow = hotpatch_console.add{type = 'flow', name = 'hotpatch-console-bottom', direction = 'vertical'}
             local output = bottom_flow.add{type = 'text-box', name = 'hotpatch-console-output'}
             local bottom_input_flow = bottom_flow.add{type = 'flow', name = 'hotpatch-console-bottom-input', direction = 'horizontal'}
             local input = bottom_input_flow.add{type = 'textfield', name = 'hotpatch-console-input'}
@@ -174,17 +174,18 @@ on_gui_click_handlers = {
             output.style.width = 800
             output.style.height = 600
             input.style.width = 600
+			hotpatch_console.visible = false
         end
 
-        local console_dropdown = console['hotpatch-console-top']['hotpatch-console-mod-selector']
+        local console_dropdown = hotpatch_console['hotpatch-console-top']['hotpatch-console-mod-selector']
         console_dropdown.clear_items()
-        for k, v in ipairs(loaded_mods) do
+        for _, v in ipairs(loaded_mods) do
             console_dropdown.add_item(v.name)
         end
 
-        console.style.visible = not console.style.visible
+        hotpatch_console.visible = not hotpatch_console.visible
-        if console.style.visible then
+        if hotpatch_console.visible then
-            player.opened = console
+            player.opened = hotpatch_console
         else
             player.opened = nil
         end
@@ -196,10 +197,10 @@ on_gui_click_handlers = {
         local left = mod_gui.get_frame_flow(player)
         local center = player.gui.center
 
-        local console = center['hotpatch-console']
+        local hotpatch_console = center['hotpatch-console']
-        local output = console['hotpatch-console-bottom']['hotpatch-console-output']
+        local output = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-output']
-        local input = console['hotpatch-console-bottom']['hotpatch-console-bottom-input']['hotpatch-console-input']
+        local input = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-bottom-input']['hotpatch-console-input']
-        local console_dropdown = console['hotpatch-console-top']['hotpatch-console-mod-selector']
+        local console_dropdown = hotpatch_console['hotpatch-console-top']['hotpatch-console-mod-selector']
 
         if not player.admin then
             output.text = table.concat{output.text, '\n', 'Only admins!'}
@@ -235,9 +236,10 @@ on_gui_click_handlers = {
             output.text = table.concat{output.text, '\n', text}
         end
 
+        local success
         local code, err = load(input.text, input.text, 't', env)
         if code then
-            local success, err = pcall(code)
+            success, err = pcall(code)
             if not success then
                 output.text = table.concat{output.text, '\n', err}
             end
@@ -256,7 +258,7 @@ on_gui_click_handlers = {
 
         local element = e.element
         local previous
-        for k, v in pairs(center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-files']['hotpatch-IDE-files-table'].children) do
+        for _, v in pairs(center['hotpatch-IDE']['hotpatch-IDE-table']['hotpatch-IDE-files']['hotpatch-IDE-files-table'].children) do
             if table.compare(v.style.font_color or {}, {r=1.0, g=1.0, b=0.0, a=1.0}) then
                 previous = v
                 break
@@ -306,8 +308,8 @@ on_gui_selection_state_changed_handlers = {
                 local mod = loaded_mods[index]
                 local version_label = center['hotpatch-console']['hotpatch-console-top']['hotpatch-console-mod-version']
                 version_label.caption = 'Version: ' .. mod.version
-                local console = center['hotpatch-console']
+                local hotpatch_console = center['hotpatch-console']
-                local output = console['hotpatch-console-bottom']['hotpatch-console-output']
+                local output = hotpatch_console['hotpatch-console-bottom']['hotpatch-console-output']
                 output.text = ''
             end
         end
@@ -332,7 +334,7 @@ on_gui_selection_state_changed_handlers = {
                 local file = list.add{type = 'label', style = 'hoverable_bold_label', caption = 'control', name = 'hotpatch-IDE-file.control'}
                 file.style.bottom_padding = 0
                 file.style.top_padding = 0
-                for k, v in pairs(mod.files) do
+                for k, _ in pairs(mod.files) do
                     file = list.add{type = 'label', style = 'hoverable_bold_label', caption = k, name = 'hotpatch-IDE-file.' .. k}
                     file.style.bottom_padding = 0
                     file.style.top_padding = 0
@@ -356,20 +358,20 @@ script.on_event(defines.events.on_gui_closed, function(e)
     if e.gui_type == defines.gui_type.custom then
         if element and element.valid then
             if element.name == 'hotpatch-IDE' then
-                if element.style.visible then
+                if element.visible then
                     player.play_sound{path = "utility/gui_click"}
                 end
-                element.style.visible = false
+                element.visible = false
             elseif element.name == 'hotpatch-console' then
-                if element.style.visible then
+                if element.visible then
                     player.play_sound{path = "utility/gui_click"}
                 end
-                element.style.visible = false
+                element.visible = false
             end
         end
     end
 end)
 
-]===])
+--]===])
 
 return true

lib/scenarios/Hotpatch/hotpatch/mod-tools.lua

@@ -101,6 +101,8 @@ local mod_on_configuration_changed
 local on_event
 local on_nth_tick
 
+local register_all_events
+
 -- core handlers
 local on_init -- installs static mods, loads, runs, and calls mod_on_init
 local on_load -- loads, runs, and calls mod_on_load
@@ -363,7 +365,7 @@ debug_log({'hotpatch-info.logging-enabled'})
 
 -- track _ENV accesses
 setmetatable(_ENV, {
-	__index = function(t, k)
+	__index = function(_, k)
 		debug_log({'hotpatch-trace.nil-var-access', k}, nil, 3)
 		return nil
 	end,
@@ -382,7 +384,6 @@ debug_log({'hotpatch-info.loading-libs'})
 local libraries = {
     'camera',
     'flying_tags',
-    'inspect',
     'math3d',
     'mod-gui',
     'noise',
@@ -541,7 +542,7 @@ load_mod = function(installed_index)
             mod_script.on_nth_tick = function(tick, f)
                 if tick then
                     if type(tick) == 'table' then
-                        for k, v in pairs(tick) do
+                        for _, v in pairs(tick) do
                             mod_script.on_nth_tick(v, f)
                         end
                         return
@@ -556,7 +557,7 @@ load_mod = function(installed_index)
                 if event == defines.events.on_tick then
                     mod_obj.on_tick = f
                     if mod_obj.running then
-                        register_on_tick(mod_name, event)
+                        register_on_tick(mod_name)
                     end
                 else
                     mod_obj.on_event[event] = f
@@ -568,7 +569,7 @@ load_mod = function(installed_index)
             mod_script.on_nth_tick = function(tick, f)
                 if tick then
                     if type(tick) == 'table' then
-                        for k, v in pairs(tick) do
+                        for _, v in pairs(tick) do
                             mod_script.on_nth_tick(v, f)
                         end
                         return
@@ -578,11 +579,11 @@ load_mod = function(installed_index)
                         register_nth_tick(mod_name, tick)
                     end
                 else
-                    local on_nth_tick = mod_obj.on_nth_tick
+                    local mod_on_nth_tick = mod_obj.on_nth_tick
                     mod_obj.on_nth_tick = {}
                     if mod_obj.running then
-                        for k, v in pairs(on_nth_tick) do
+                        for _, v in pairs(mod_on_nth_tick) do
-                            register_nth_tick(mod_name, tick)
+                            register_nth_tick(mod_name, v)
                         end
                     end
                 end
@@ -650,7 +651,7 @@ load_mod = function(installed_index)
                 return remote.call(...)
             end,
             interfaces = setmetatable({}, {
-                __index = function(t, k) return remote.interfaces[k] end,
+                __index = function(_, k) return remote.interfaces[k] end,
                 __pairs = function(t) local function iter(t, k) local v; k, v = next(remote.interfaces, k); if v then return k, t[k] end; end; return iter, t, nil end
             })
         }
@@ -666,11 +667,11 @@ load_mod = function(installed_index)
                 return commands.remove_command(name)
             end,
             commands = setmetatable({}, {
-                __index = function(t, k) return commands.commands[k] end,
+                __index = function(_, k) return commands.commands[k] end,
                 __pairs = function(t) local function iter(t, k) local v; k, v = next(commands.commands, k); if v then return k, t[k] end; end; return iter, t, nil end
             }),
             game_commands = setmetatable({}, {
-                __index = function(t, k) return commands.game_commands[k] end,
+                __index = function(_, k) return commands.game_commands[k] end,
                 __pairs = function(t) local function iter(t, k) local v; k, v = next(commands.game_commands, k); if v then return k, t[k] end; end; return iter, t, nil end
             })
         }
@@ -710,7 +711,7 @@ load_mod = function(installed_index)
                         error(err)
                     end
                 end
-				local file = global.mods[installed_index].files[alt_path]
+				file = global.mods[installed_index].files[alt_path]
                 if file then
                     debug_log({'hotpatch-trace.load-require', alt_path}, mod_name)
                     local code, err = load(file, '[' .. mod_name .. '] ' .. alt_path .. '.lua', 'bt', env)
@@ -729,7 +730,7 @@ load_mod = function(installed_index)
                 env.package._current_path_in_package = oldbase
 				local lib = package.loaded[path]
 				if not lib then
-					debug_log(path .. ' not found', nil, 3)
+					debug_log({'hotpatch-error.path-not-found', path}, mod_name, 3)
                     error(path .. ' not found')
 				end
                 return lib
@@ -743,12 +744,12 @@ load_mod = function(installed_index)
 
 
         env['game'] = setmetatable({}, {
-            __index = function(t, k) return game[k] end,
+            __index = function(_, k) return game[k] end,
             __pairs = function(t) local function iter(t, k) local v; k, v = next(game, k); if v then return k, t[k] end; end; return iter, t, nil end
         })
 
         local mt = {}
-        mt.__index = function(t, k)
+        mt.__index = function(_, k)
             debug_log({'hotpatch-trace.nil-var-access', k}, nil, 3)
             return nil
         end
@@ -816,12 +817,12 @@ reset_mod = function(loaded_index)
 end
 
 reset_mod_events = function(loaded_index)
-    local loaded_index = find_loaded_mod(mod_name)
+    --local loaded_index = find_loaded_mod(mod_name)
-    
-    if not loaded_index then
-        debug_log({'hotpatch-warning.reset-events-not-running'}, mod_name)
-    else 
     local loaded_mod = loaded_mods[loaded_index]
+
+    if not loaded_mod then
+        --debug_log({'hotpatch-warning.reset-events-not-running'}, mod_name)
+    else
         loaded_mod.on_event = {}
         loaded_mod.on_nth_tick = {}
         loaded_mod.on_init = nil
@@ -866,7 +867,7 @@ run_mod = function(loaded_index)
 		-- if a mod is being loaded that already has global data, it must be restored after
 		-- running control.lua, discarding any changes made
 		if not table.compare(old_global, {}) then
-			for k,v in pairs(mod.env.global) do
+			for k,_ in pairs(mod.env.global) do
 				mod.env.global[k] = nil
 			end
 			for k,v in pairs(old_global) do
@@ -884,7 +885,7 @@ run_mod = function(loaded_index)
             mod.env.script.on_event = function(event, f)
                 if event == defines.events.on_tick then
                     mod.on_tick = f
-                    register_on_tick(mod_name, event)
+                    register_on_tick(mod_name)
                 else
                     mod.on_event[event] = f
                     register_event(mod_name, event)
@@ -893,18 +894,18 @@ run_mod = function(loaded_index)
             mod.env.script.on_nth_tick = function(tick, f)
                 if tick then
                     if type(tick) == 'table' then
-                        for k, v in pairs(tick) do
+                        for _, v in pairs(tick) do
-                            mod_script.on_nth_tick(v, f)
+                            mod.env.script.on_nth_tick(v, f)
                         end
                         return
                     end
                     mod.on_nth_tick[tick] = f
                     register_nth_tick(mod_name, tick)
                 else
-                    local on_nth_tick = mod.on_nth_tick
+                    local mod_on_nth_tick = mod.on_nth_tick
                     mod.on_nth_tick = {}
-                    for k, v in pairs(on_nth_tick) do
+                    for _, v in pairs(mod_on_nth_tick) do
-                        register_nth_tick(mod_name, tick)
+                        register_nth_tick(mod_name, v)
                     end
                 end
             end
@@ -962,7 +963,7 @@ else
     end
 end
 
-local register_all_events = function()
+register_all_events = function()
     --unregister all events
     script.on_event(defines.events, nil)
     script.on_nth_tick(nil, nil)
@@ -981,12 +982,12 @@ register_mod_events = function(loaded_index)
         debug_log({'hotpatch-trace.on-tick-event-registered'}, mod_name)
         script.on_event(defines.events.on_tick, on_tick)
     end
-    for k,v in pairs(mod.on_event) do 
+    for k,_ in pairs(mod.on_event) do
         local event_name = (event_names[k] or k)
         debug_log({'hotpatch-trace.on-event-registered', event_name}, mod_name)
         script.on_event(k, on_event)
     end
-    for k,v in pairs(mod.on_nth_tick) do 
+    for k,_ in pairs(mod.on_nth_tick) do
         debug_log({'hotpatch-trace.on-nth-tick-event-registered', k}, mod_name)
         script.on_nth_tick(k, on_nth_tick)
     end
@@ -1000,8 +1001,8 @@ mod_on_init = function(loaded_index)
         if mod.on_init then
             local success, result = xpcall(mod.on_init, debug.traceback)
             if not success then
-                debug_log('on_init failed for ' .. mod.name)
+				debug_log({'hotpatch-error.on-init-failed'}, mod_name)
-                debug_log(result)
+                debug_log(result, mod_name)
                 return false
             end
         end
@@ -1020,8 +1021,8 @@ mod_on_load = function(loaded_index)
 			--local old_global = table.deepcopy(mod.env.global)
             local success, result = xpcall(mod.on_load, debug.traceback)
             if not success then
-                debug_log('on_load failed for ' .. mod.name)
+                debug_log({'hotpatch-error.on-load-failed'}, mod_name)
-                debug_log(result)
+                debug_log(result, mod_name)
                 return false
             end
         end
@@ -1039,8 +1040,8 @@ mod_on_configuration_changed = function(loaded_index, config)
         if mod.on_configuration_changed then
             local success, result = xpcall(mod.on_configuration_changed, debug.traceback, config)
             if not success then
-                debug_log('on_configuration_changed failed for ' .. mod.name)
+				debug_log({'hotpatch-error.on-configuration-changed-failed'}, mod_name)
-                debug_log(result)
+                debug_log(result, mod_name)
                 return false
             end
         end
@@ -1212,7 +1213,7 @@ on_configuration_changed = function(config)
     debug_log({'hotpatch-info.on-configuration-changed'})
 	local failed_mods = {}
     for i = 1, #loaded_mods do
-        if not mod_on_configuration_changed(i) then
+        if not mod_on_configuration_changed(i, config) then
 			table.insert(failed_mods, i)
 		end
     end
@@ -1293,24 +1294,24 @@ local mod_tools_internal = setmetatable({
     loaded_mods = loaded_mods,
     loaded_libraries = loaded_libraries,
     installed_mods = setmetatable({}, {
-        __index = function(t, k)
+        __index = function(_, k)
             return global.mods[k]
         end,
-        __newindex = function(t, k, v)
+        __newindex = function(_, __, ___)
         -- do nothing
         end,
-        __len = function(t)
+        __len = function(_)
             return #global.mods
         end,
-        __pairs = function(t) local function iter(t, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
+        __pairs = function(t) local function iter(_, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
-        __ipairs = function(t) local function iter(t, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
+        __ipairs = function(t) local function iter(_, k) local v; v = global.mods[k+1]; if v then return k+1, v end; end; return iter, t, 0 end,
         __metatable = false,
     })
 },{
-    __index = function(t, k)
+    __index = function(_, k)
         debug_log({'hotpatch-error.invalid-API-access', k}, nil, 3)
     end,
-    __newindex = function(t, k, v)
+    __newindex = function(_, __, ___)
         -- do nothing
     end,
     -- Don't let mods muck around
@@ -1322,11 +1323,12 @@ local mod_tools = setmetatable({
     -- most code should use static_mod
     static_mod = static_mod, -- (name, version, code, files)
     set_debug_level = function(level)
-        debug_level = tonumber(_ENV.debug_settings.level) or debug_levels[_ENV.debug_settings.level]
+        --debug_level = tonumber(_ENV.debug_settings.level) or debug_levels[_ENV.debug_settings.level]
+		debug_level = tonumber(level) or debug_levels[level]
     end,
 },{
     __index = mod_tools_internal,
-    __newindex = function(t, k, v)
+    __newindex = function(_, __, ___)
         -- do nothing, read-only table
     end,
     -- Don't let mods muck around

lib/scenarios/Hotpatch/hotpatch/remote-interface.lua

@@ -65,7 +65,7 @@ remote_interface['install'] = function(mod_name, mod_version, mod_code, mod_file
     local caller = game.player or console
     if caller.admin then
 		if mod_files then
-			for k,v in pairs(mod_files) do
+			for k,_ in pairs(mod_files) do
 				if k == 'control' then
 					caller.print('ERROR: control.lua loaded twice for mod ' .. mod_name)
 					return
@@ -77,8 +77,8 @@ remote_interface['install'] = function(mod_name, mod_version, mod_code, mod_file
         if installed_index then
             local old_version = installed_mods[installed_index].version
             if old_version then
-                debug_log('WARNING: mod already exists: ' .. mod_name .. ' ' .. old_version)
+                debug_log({'hotpatch-warning.already-exists', mod_name, old_version})
-                debug_log('WARNING: reinstalling mod in-place: ' .. mod_name .. ' ' .. mod_version)
+                debug_log({'hotpatch-warning.reinstalling', mod_name, old_version})
             end
             if loaded_index then
                 unload_mod(loaded_index)
@@ -131,6 +131,8 @@ remote_interface['run'] = function(mod_name)
     if caller.admin then
         local installed_index = find_installed_mod(mod_name)
         local loaded_index = find_loaded_mod(mod_name)
+		local mod
+        local version
         if installed_index and not loaded_index then
             if not load_mod(installed_index) then
                 caller.print('compilation failed for mod ' .. mod_name)
@@ -139,10 +141,10 @@ remote_interface['run'] = function(mod_name)
             loaded_index = find_loaded_mod(mod_name)
         else
             if loaded_index then
-                local mod = loaded_mods[loaded_index]
+                mod = loaded_mods[loaded_index]
-                local version = mod.version
+                version = mod.version
-                debug_log('WARNING: mod already loaded: ' .. mod_name .. ' ' .. version)
+                debug_log({'hotpatch-warning.already-loaded', mod_name, version})
-                debug_log('WARNING: reinitializing: ' .. mod_name .. ' ' .. version)
+                debug_log({'hotpatch-warning.reinitializing', mod_name, version})
             end
         end
 
@@ -178,7 +180,7 @@ remote_interface['update'] = function(mod_name, mod_version, mod_code, mod_files
     local caller = game.player or console
     if caller.admin then
 		if mod_files then
-			for k,v in pairs(mod_files) do
+			for k,_ in pairs(mod_files) do
 				if k == 'control' then
 					caller.print('ERROR: control.lua loaded twice for mod ' .. mod_name)
 					return
@@ -186,6 +188,9 @@ remote_interface['update'] = function(mod_name, mod_version, mod_code, mod_files
 			end
 		end
         local installed_index = find_installed_mod(mod_name)
+		if installed_index then
+			;
+		end
         local loaded_index = find_loaded_mod(mod_name)
         local old_version
         if loaded_index then
@@ -245,16 +250,40 @@ remote_interface['install_mod_file'] = function(mod_name, mod_file, mod_file_cod
     end
 end
 
+remote_interface['exec'] = function(mod_name, exec_code)
+    local caller = game.player or console
+    if caller.admin then
+		local loaded_index = find_loaded_mod(mod_name)
+		local env = loaded_mods[loaded_index].env
+		
+		local success
+		local code, err = load(exec_code, exec_code, 't', env)
+		if code then
+			success, err = pcall(code)
+			if not success then
+				debug_log(err)
+			end
+		else
+			debug_log(err)
+		end
+	else
+		caller.print('You must be an admin to run this command.')
+    end
+end
+
 --TODO: most of this function
 remote_interface['clean'] = function(mod_name)
     -- Removes ALL mod data and reinitializes
     -- doesn't remove things a mod may add like surfaces, etc
     local caller = game.player or console
     if caller.admin then
-        mod_reset(mod_name)
+        --mod_reset(mod_name)
-        run_mod(mod_name)
+        --run_mod(mod_name)
-        mod_on_init(mod_name)
+        --mod_on_init(mod_name)
-        mod_on_configuration_changed(mod_name, {mod_changes = {mod_name={new_version=mod_version}}})
+		local loaded_index = find_loaded_mod(mod_name)
+		local mod = loaded_mods[loaded_index]
+        local version = mod.version
+        mod_on_configuration_changed(loaded_index, {mod_changes = {mod_name={new_version=version}}})
     else
         caller.print('You must be an admin to run this command.')
     end
@@ -274,7 +303,7 @@ remote.add_interface('hotpatch', remote_interface)
 
 debug_log({'hotpatch-info.complete', {'hotpatch-info.remote-installing'}})
 
-]===])
+--]===])
 
 --[[
 

lib/scenarios/Hotpatch/locale/en/hotpatch.cfg

@@ -65,13 +65,17 @@ contains-comments-console=comments from console will comment out the entire code
 reset-events-not-running=tried to reset events for mod that isn't running!
 remote-interface-exists=remote interface __1__ already exists, removing...
 command-exists=command __1__ already exists, removing...
-already-loaded=mod already loaded
+already-exists=mod already exists: __1__ __2__
-reinitializing=reinitializing...
+reinstalling=reinstalling mod in-place: __1__ __2__
-already-exists=mod already exists
+already-loaded=mod already loaded: __1__ __2__
-reinstalling=reinstalling mod in-place
+reinitializing=reinitializing: __1__ __2__
 
 [hotpatch-error]
 invalid-API-access=Invalid API access: __1__
 not-installed=mod not installed, cannot install file for mod that does not exist
 compilation-failed=compilation failed for mod
 execution-failed=execution failed for mod
+path-not-found=path '__1__' not found
+on-init-failed=on_init failed
+on-load-failed=on_load failed
+on-configuration-changed-failed=on_configuration_changed failed

master.d.ts

@@ -1,16 +0,0 @@
-type MasterPluginInstance = {
-    config: object,
-    pluginConfig: object,
-    path: string,
-    socketio: any,
-    express: any,
-    Prometheus: any,
-    onExit?: Function,
-}
-type pluginConfig = {
-    name: string,
-}
-interface MasterPlugin {
-    main: MasterPluginInstance,
-    pluginConfig: pluginConfig,
-}

master.js

@@ -1,6 +1,3 @@
-import { getDatabase } from "./lib/getDatabase";
-import { db } from "./lib/getDatabase";
-
 /**
 Clusterio master server. Facilitates communication between slaves through
 a webserver, storing data related to slaves like production graphs and
@@ -16,9 +13,7 @@ node master.js
 // Attempt updating
 // const updater = require("./updater.js");
 // updater.update().then(console.log);
-namespace Master {
+
-// homebrew types
-//import {MasterPlugin} from "master"
 (async ()=>{
 
 // Set the process title, shows up as the title of the CMD window on windows
@@ -29,7 +24,7 @@ process.title = "clusterioMaster";
 process.on('unhandledRejection', r => console.log(r));
 
 // configgy stuff
-const debug = false;
+debug = false;
 
 // reduces loadtime from 1700 ms to 1200 ms on my i3 + SSD
 // tested by logging Date.now() here and when the webserver started listening
@@ -68,7 +63,7 @@ const fileOps = require("_app/fileOps");
 const authenticate = require("./lib/authenticate")(config);
 
 /** Sync */
-function randomStringAsBase64Url(size: Number) {
+function randomStringAsBase64Url(size) {
   return base64url(crypto.randomBytes(size));
 }
 if (!fs.existsSync("secret-api-token.txt")) {
@@ -110,11 +105,11 @@ app.set('view engine', 'html');
 app.engine('html', ejs.renderFile);
 
 // give ejs access to some interesting information
-app.use(function(req: Request, res: any, next: any){
+app.use(function(req, res, next){
 	res.locals.res = res;
 	res.locals.req = req;
 	res.locals.masterPlugins = masterPlugins;
-	res.locals.slaves = db.slaves;
+	res.locals.slaves = slaves;
 	res.locals.moment = moment;
 	next();
 });
@@ -136,7 +131,7 @@ if(!config.disablePrometheusPushgateway){
 	setInterval(() => {
 		registerMoreMetrics();
 		pushgateway.push({ jobName: 'clusterio', groupings: {instance: config.publicIP + ":" + config.masterPort, owner: config.username}}, function(err, resp, body) {})
-	}, 5000)
+	}, 15000)
 }
 
 // collect express request durations ms
@@ -210,15 +205,15 @@ app.get('/metrics', (req, res) => {
 });
 
 function registerMoreMetrics(){
-	for(let instanceID in db.slaves){
+	for(let instanceID in slaves){
 		// playercount
 		try{
-			prometheusPlayerCountGauge.labels(instanceID, db.slaves[instanceID].instanceName).set(Number(db.slaves[instanceID].playerCount) || 0);
+			prometheusPlayerCountGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].playerCount) || 0);
 		}catch(e){}
 		// UPS
 		try{
-			prometheusUPSGauge.labels(instanceID, db.slaves[instanceID].instanceName).set(Number(db.slaves[instanceID].meta.UPS) || 60);
+			prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.UPS) || 60);
-			if(db.slaves[instanceID].meta.tick && typeof db.slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, db.slaves[instanceID].instanceName).set(Number(db.slaves[instanceID].meta.tick) || 0);
+			if(slaves[instanceID].meta.tick && typeof slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.tick) || 0);
 		}catch(e){}
 	}
 	// inventory
@@ -229,15 +224,58 @@ function registerMoreMetrics(){
 	}
 	// Slave count
 	let numberOfActiveSlaves = 0;
-	for(let instance in db.slaves){
+	for(let instance in slaves){
-		if(Date.now() - Number(db.slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
+		if(Date.now() - Number(slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++;
 	}
 	prometheusConnectedInstancesCounter.set(numberOfActiveSlaves);
 }
 
 // set up database
-const db: db = getDatabase(config);
+const db = {};
 
+(function(){
+	try{
+		let x = fs.statSync(path.resolve(config.databaseDirectory, "/slaves.json"));
+		console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`);
+		slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json")));
+	} catch (e){
+		slaves = {};
+	}
+	try{
+		x = fs.statSync(path.resolve(config.databaseDirectory, "items.json"));
+		console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`);
+		db.items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json")));
+	} catch (e){
+		db.items = {};
+	}
+})()
+
+db.items.addItem = function(object) {
+	if(object.name == "addItem" || object.name == "removeItem") {
+		console.error("Fuck you, that would screw everything up if you named your item that.");
+		return false;
+	} else {
+		if(this[object.name] && Number(this[object.name]) != NaN){
+			this[object.name] = Number(this[object.name]) + Number(object.count);
+		} else {
+			this[object.name] = Number(object.count);
+		}
+		return true;
+	}
+}
+db.items.removeItem = function(object) {
+	if(object.name == "addItem" || object.name == "removeItem") {
+		console.error("Fuck you, that would screw everything up if you named your item that.");
+		return false;
+	} else {
+		if(this[object.name] && Number(this[object.name]) != NaN){
+			this[object.name] = Number(this[object.name]) - Number(object.count);
+		} else {
+			this[object.name] = 0;
+		}
+		return true;
+	}
+}
 
 // store slaves and inventory in a .json full of JSON data
 process.on('SIGINT', shutdown); // ctrl + c
@@ -245,18 +283,19 @@ process.on('SIGHUP', shutdown); // terminal closed
 async function shutdown() {
 	console.log('Ctrl-C...');
 	let exitStartTime = Date.now();
+	try {
 		// set insane limit to slave length, if its longer than this we are probably being ddosed or something
-	if(db.slaves && Object.keys(db.slaves).length < 2000000){
+		if(slaves && Object.keys(slaves).length < 2000000){
 			console.log("saving to slaves.json");
-		fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(db.slaves));
+			fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves));
-	} else if(db.slaves) {
+		} else if(slaves) {
-		console.log("Slave database too large, not saving ("+Object.keys(db.slaves).length+")");
+			console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")");
 		}
 		if(db.items && Object.keys(db.items).length < 50000){
 			console.log("saving to items.json");
 			fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items));
-	} else if(db.slaves) {
+		} else if(slaves) {
-		console.log("Item database too large, not saving ("+Object.keys(db.slaves).length+")");
+			console.log("Item database too large, not saving ("+Object.keys(slaves).length+")");
 		}
 		for(let i in masterPlugins){
 			let plugin = masterPlugins[i];
@@ -268,6 +307,11 @@ async function shutdown() {
 		}
 		console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
 		process.exit(0);
+	} catch(e) {
+		console.log(e)
+		console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms")
+		process.exit(1)
+	}
 }
 
 /**
@@ -276,7 +320,7 @@ POST World ID management. Slaves post here to tell the server they exist
 @instance
 @alias /api/getID
 */
-app.post("/api/getID", authenticate.middleware, function(req, res) {
+app.post("/api/getID", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	// request.body should be an object
 	// {rconPort, rconPassword, serverPort, mac, time}
@@ -286,12 +330,12 @@ app.post("/api/getID", authenticate.middleware, function(req, res) {
 		if(debug){
 			console.log(req.body)
 		}
-		if(!db.slaves[req.body.unique]){
+		if(!slaves[req.body.unique]){
-			db.slaves[req.body.unique] = {};
+			slaves[req.body.unique] = {};
 		}
-		for(let k in req.body){
+		for(k in req.body){
 			if(k != "meta" && req.body.hasOwnProperty(k)){
-				db.slaves[req.body.unique][k] = req.body[k];
+				slaves[req.body.unique][k] = req.body[k];
 			}
 		}
 		// console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName);
@@ -310,11 +354,11 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) {
 	
 	if(req.body && req.body.instanceID && req.body.password && req.body.meta){
 		// check for editing permissions
-		if(db.slaves[req.body.instanceID] && db.slaves[req.body.instanceID].rconPassword == req.body.password){
+		if(slaves[req.body.instanceID] && slaves[req.body.instanceID].rconPassword == req.body.password){
-			if(!db.slaves[req.body.instanceID].meta){
+			if(!slaves[req.body.instanceID].meta){
-				db.slaves[req.body.instanceID].meta = {};
+				slaves[req.body.instanceID].meta = {};
 			}
-			db.slaves[req.body.instanceID].meta = deepmerge(db.slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
+			slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true});
 			let metaPortion = JSON.stringify(req.body.meta);
 			if(metaPortion.length > 50) metaPortion = metaPortion.substring(0,20) + "...";
 			// console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion);
@@ -335,7 +379,7 @@ app.post("/api/getSlaveMeta", function (req, res) {
 	console.log("body", req.body);
     if(req.body && req.body.instanceID && req.body.password){
     	console.log("returning meta for ", req.body.instanceID);
-    	res.send(JSON.stringify(db.slaves[req.body.instanceID].meta))
+    	res.send(JSON.stringify(slaves[req.body.instanceID].meta))
 	} else {
     	res.status(400);
     	res.send('{"INVALID REQUEST":1}');
@@ -351,9 +395,9 @@ POST Check if a mod has been uploaded to the master before. Only checks against
 */
 app.post("/api/checkMod", function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	let files: string[] = fs.readdirSync(masterModFolder);
+	let files = fs.readdirSync(masterModFolder);
 	let found = false;
-	files.forEach((file) => {
+	files.forEach(file => {
 		if(file == req.body.modName) {
 			found = true;
 		}
@@ -376,7 +420,7 @@ app.post("/api/uploadMod", authenticate.middleware, function(req,res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	if (req.files && req.files.file) {
 		// console.log(req.files.file);
-		req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err: Error) {
+		req.files.file.mv(path.resolve(config.databaseDirectory, "masterMods", req.files.file.name), function(err) {
 			if (err) {
 				res.status(500).send(err);
 			} else {
@@ -395,7 +439,7 @@ GET endpoint for getting information about all our slaves
 @instance
 @alias /api/slaves
 */
-let slaveCache: {timestamp:number, cache?:Slaves} = {
+let slaveCache = {
 	timestamp: Date.now(),
 };
 app.get("/api/slaves", function(req, res) {
@@ -403,9 +447,9 @@ app.get("/api/slaves", function(req, res) {
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000){
-		let copyOfSlaves: Slaves = JSON.parse(JSON.stringify(db.slaves));
+		let copyOfSlaves = JSON.parse(JSON.stringify(slaves));
 		// filter out the rcon password because thats kindof not a safe thing to share
-		for(let key in copyOfSlaves){
+		for(key in copyOfSlaves){
 			copyOfSlaves[key].rconPassword = "hidden";
 		}
 		slaveCache.cache = copyOfSlaves;
@@ -414,8 +458,8 @@ app.get("/api/slaves", function(req, res) {
 	
 	res.send(slaveCache.cache);
 });
-var recievedItemStatisticsBySlaveID: any = {};
+var recievedItemStatisticsBySlaveID = {};
-var sentItemStatisticsBySlaveID: any = {};
+var sentItemStatisticsBySlaveID = {};
 /*
 var recievedItemStatistics = new averagedTimeSeries({
 	maxEntries: config.itemStats.maxEntries,
@@ -437,7 +481,7 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-	let x: item;
+	let x;
 	try {
 		x = JSON.parse(req.body);
 	} catch (e) {
@@ -448,8 +492,8 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 	}
 	if(!x.instanceID) {
 		x.instanceID = "unknown";
-		Object.keys(db.slaves).forEach(instanceID => {
+		Object.keys(slaves).forEach(instanceID => {
-			if(db.slaves[instanceID].instanceName == req.body.instanceName) {
+			if(slaves[instanceID].instanceName == req.body.instanceName) {
 				x.instanceID = instanceID;
 			}
 		});
@@ -490,6 +534,13 @@ app.post("/api/place", authenticate.middleware, function(req, res) {
 		res.send("failure");
 	}
 });
+const routes_api_remove = require("./routes/api/remove.js")
+var neuralDole = null
+
+if(config.useNeuralNetDoleDivider)//Only initialize neural network when it's enabled, otherwise it might override gauge
+	neuralDole=new routes_api_remove.neuralDole({
+		items: db.items, gaugePrefix: prometheusPrefix
+	})
 /**
 POST endpoint to remove items from DB when client orders items.
 
@@ -501,11 +552,8 @@ POST endpoint to remove items from DB when client orders items.
 @param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name]
 @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages.
 */
-var _doleDivisionFactor: {[name:string]: number} = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
 app.post("/api/remove", authenticate.middleware, function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
-	const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
-	const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
 	res.header("Access-Control-Allow-Origin", "*");
 	res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
 	// save items we get
@@ -516,10 +564,10 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 	if(!object.instanceName) {
 		object.instanceName = "unknown";
 	}
-	if(db.slaves[object.instanceID]) object.instanceName = db.slaves[object.instanceID].instanceName;
+	if(slaves[object.instanceID]) object.instanceName = slaves[object.instanceID].instanceName;
 	let item = db.items[object.name]
 		// console.dir(doc);
-	if(!item
+	if((item === undefined)
 	||((
 		config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
 		&& object.name != "electricity")
@@ -529,6 +577,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		}
 		res.send({name:object.name, count:0});
 	} else if(config.disableFairItemDistribution){
+		// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
 		let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item));
 		db.items.removeItem({count: numberToRemove, name: object.name});
 		res.send({count: numberToRemove, name: object.name});
@@ -549,45 +598,27 @@ app.post("/api/remove", authenticate.middleware, function(req, res) {
 		});
 		//console.log(sentItemStatistics.data)
 		sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+	} else if(config.useNeuralNetDoleDivider){
+		// use fancy neural net to calculate a "fair" dole division rate.
+		neuralDole.divider({
+			res,
+			object,
+			config,
+			sentItemStatisticsBySlaveID,
+			prometheusImportGauge
+		})
 	} else {
-		const originalCount = Number(object.count) || 0;
+		// Use dole division. Makes it really slow to drain out the last little bit.
-		object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+		routes_api_remove.doleDivider({
-		object.count = Math.round(object.count);
+			item,
-		if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
+			object,
-		
+			db,
-		// Update existing items if item name already exists
+			sentItemStatisticsBySlaveID,
-		if(Number(item) > Number(object.count)) {
+			config,
-			//If successful, increase dole
+			prometheusDoleFactorGauge,
-			_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+			prometheusImportGauge,
-			if(config.logItemTransfers){
+			req,res,
-				console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
+		})
-			}
-			if(db.items.removeItem({count: object.count, name: object.name})){
-				let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
-				if(sentItemStatistics === undefined){
-					sentItemStatistics = new averagedTimeSeries({
-						maxEntries: config.itemStats.maxEntries,
-						entriesPerSecond: config.itemStats.entriesPerSecond,
-						mergeMode: "add",
-					}, console.log);
-				}
-				sentItemStatistics.add({
-					key:object.name,
-					value:object.count,
-				});
-				//console.log(sentItemStatistics.data)
-				sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
-			}
-			
-			prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
-			prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
-			res.send({count: object.count, name: object.name});
-		} else {
-			// if we didn't have enough, attempt giving out a smaller amount next time
-			_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
-			res.send({name:object.name, count:0});
-			//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
-		}
 	}
 });
 
@@ -634,18 +665,18 @@ app.get("/api/inventoryAsObject", function(req, res) {
 
  @memberof clusterioMaster
  @instance
- @alias api/modmeta
+ @alias api/modData
  @returns {object} JSON
  */
 
 // wrap a request in an promise
-async function downloadPage(url: string): Promise<object> {
+    async function downloadPage(url) {
         return new Promise((resolve, reject) => {
-		request(url, (_error: any, _response: any, body: object) => {
+            request(url, (error, response, body) => {
                 resolve(body);
             });
         });
-}
+    }
 
 app.get("/api/modmeta", async function(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
@@ -671,7 +702,7 @@ app.post("/api/runCommand", (req,res) => {
 	var token = req.headers['x-access-token'];
 	if(!token) return res.status(401).send({ auth: false, message: 'No token provided.' });
 	
-	jwt.verify(token, config.masterAuthSecret, function(err: Error) {
+	jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
 		if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.' });
 		
 		// validate request.body
@@ -698,7 +729,7 @@ app.post("/api/runCommand", (req,res) => {
             && body.command[0] == "/")
         {
             // execute command
-            wsSlaves[body.instanceID].runCommand(body.command, (data: string) => {
+            wsSlaves[body.instanceID].runCommand(body.command, data => {
                 res.status(200).send({auth: true, message: "success", data});
             });
         } else {
@@ -716,7 +747,7 @@ GET endpoint. Returns factorio's base locale as a JSON object.
 */
 app.get("/api/getFactorioLocale", function(req,res){
 	endpointHitCounter.labels(req.route.path).inc();
-	getFactorioLocale.asObject(config.factorioDirectory, "en", (_err, factorioLocale) => {
+	getFactorioLocale.asObject(config.factorioDirectory, "en", (err, factorioLocale) => {
 		res.send(factorioLocale);
 	});
 });
@@ -725,7 +756,7 @@ if(args.sslPort || config.sslPort){
 		var certificate = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.crt'));
 		var privateKey = fs.readFileSync(path.resolve(config.databaseDirectory, 'certificates/cert.key'));
 
-		var httpsServer = require("https").createServer({
+		httpsServer = require("https").createServer({
 			key: privateKey,
 			cert: certificate
 		}, app).listen(args.sslPort || config.sslPort);
@@ -734,7 +765,7 @@ if(args.sslPort || config.sslPort){
 	}
 }
 if(args.masterPort || config.masterPort){
-	var server = require("http").Server(app);
+	server = require("http").Server(app);
 	server.listen(args.masterPort || config.masterPort || 8080, function () {
 		console.log("Listening on port %s...", server.address().port);
 	});
@@ -747,9 +778,6 @@ const ioMetrics = require("socket.io-prometheus");
 ioMetrics(io);
 var slaveMappers = {};
 class slaveMapper {
-	instanceID: string
-	socket: any
-	lastBeat: number
 	constructor(instanceID, socket) {
 		this.instanceID = instanceID;
 		this.socket = socket;
@@ -780,10 +808,6 @@ class slaveMapper {
 }
 var mapRequesters = {};
 class mapRequester {
-	requesterID: string
-	socket: any
-	instanceID: string
-	lastBeat: number
 	constructor(requesterID, socket, instanceID){
 		this.requesterID = requesterID;
 		this.socket = socket;
@@ -823,13 +847,7 @@ class mapRequester {
 }
 /* Websockets for send and recieve combinators */
 var wsSlaves = {};
-var wsChatRecievers = [];
-var wsAlertRecievers = [];
 class wsSlave {
-	instanceID: string
-	socket: any
-	lastBeat: number
-	commandsWaitingForReturn: object
 	constructor(instanceID, socket){
 		this.instanceID = instanceID;
 		this.socket = socket;
@@ -861,8 +879,8 @@ class wsSlave {
 			prometheusWsUsageCounter.labels('gameChat', this.instanceID).inc();
 			if(typeof data === "object"){
 				data.timestamp = Date.now();
-				if(!wsChatRecievers) wsChatRecievers = [];
+				if(!global.wsChatRecievers) global.wsChatRecievers = [];
-				wsChatRecievers.forEach(socket => {
+				global.wsChatRecievers.forEach(socket => {
 					socket.emit("gameChat", data);
 				});
 			}
@@ -872,8 +890,8 @@ class wsSlave {
 			prometheusWsUsageCounter.labels('alert', this.instanceID).inc();
 			if(typeof alert === "object"){
 				alert.timestamp = Date.now();
-				if(!wsAlertRecievers) wsAlertRecievers = [];
+				if(!global.wsAlertRecievers) global.wsAlertRecievers = [];
-				wsAlertRecievers.forEach(socket => {
+				global.wsAlertRecievers.forEach(socket => {
 					socket.emit("alert", alert);
 				});
 			}
@@ -928,26 +946,27 @@ io.on('connection', function (socket) {
 	});
 	socket.on("registerChatReciever", function(data){
 		prometheusWsUsageCounter.labels("registerChatReciever", "other").inc();
-		if(!wsChatRecievers) wsChatRecievers = [];
+		if(!global.wsChatRecievers) global.wsChatRecievers = [];
-		wsChatRecievers.push(socket);
+		global.wsChatRecievers.push(socket);
 	});
 	socket.on("registerAlertReciever", function(data){
 		prometheusWsUsageCounter.labels("registerAlertReciever", "other").inc();
-		if(!wsAlertRecievers) wsAlertRecievers = [];
+		if(!global.wsAlertRecievers) global.wsAlertRecievers = [];
-		wsAlertRecievers.push(socket);
+		global.wsAlertRecievers.push(socket);
 	});
 });
 
 // handle plugins on the master
 pluginManagement();
-var masterPlugins: MasterPlugin[] = [];
+masterPlugins = [];
 async function pluginManagement(){
 	let startPluginLoad = Date.now();
 	masterPlugins = await getPlugins();
 	console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms");
 	return;
 }
-async function getPlugins(): Promise<MasterPlugin[]>{
+
+async function getPlugins(){
 	const pluginManager = require("./lib/manager/pluginManager.js")(config);
 	let plugins = [];
 	let pluginsToLoad = await pluginManager.getPlugins();
@@ -982,4 +1001,3 @@ async function getPlugins(): Promise<MasterPlugin[]>{
 }
 
 })();
-}

package.json

@@ -5,17 +5,15 @@
   "repository": "https://github.com/Danielv123/factorioClusterio",
   "main": "master.js",
   "scripts": {
-    "build": "tsc",
-    "watch-ts": "tsc -w",
-    "watch-node": "nodemon dist/master.js",
-    "watch": "concurrently -k -p \"[{name}]\" -n \"TypeScript, Node\" -c \"yello.bold, cyan.bold\" \"npm run watch-ts\" \"npm run watch-node\"",
     "test": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{{,node_modules/_app/**/},!(node_modules)/**/}*.spec.js\" -R spec || echo .",
     "start": "echo Run node client.js start [instance] to start slaves && node master.js",
     "cover": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{{,node_modules/_app/**/},!(node_modules)/**/}*.spec.js\" -R spec || echo . && codecov -t f7b38085-36eb-4d72-84e6-64721910171e",
     "docs": "documentation build index.js --config documentation.yml -f html -o docs",
     "livedocs": "documentation serve --watch *.js lib/** --config documentation.yml",
     "install": "node lib/npmPostinstall.js",
-    "bundle-dependencies": "bundle-dependencies"
+    "bundle-dependencies": "bundle-dependencies",
+    "snyk-protect": "snyk protect",
+    "prepublish": "npm run snyk-protect"
   },
   "keywords": [
     "factorio"
@@ -23,17 +21,6 @@
   "author": "Danielv123",
   "license": "ISC",
   "dependencies": {
-    "@types/bcrypt": "^3.0.0",
-    "@types/body-parser": "^1.17.0",
-    "@types/deepmerge": "^2.2.0",
-    "@types/ejs": "^2.6.3",
-    "@types/express": "^4.16.1",
-    "@types/fs-extra": "^5.0.5",
-    "@types/mkdirp": "^0.5.2",
-    "@types/moment": "^2.13.0",
-    "@types/needle": "^2.0.4",
-    "@types/request": "^2.48.1",
-    "@types/socket.io": "^2.1.2",
     "app-root-path": "^2.0.1",
     "as-table": "^1.0.31",
     "averaged-timeseries": "^1.4.0",
@@ -80,8 +67,8 @@
     "socket.io-client": "^2.0.4",
     "socket.io-prometheus": "^0.2.1",
     "sync-request": "^4.0.1",
-    "typescript-require": "^0.2.10",
+    "underscore": "^1.8.3",
-    "underscore": "^1.8.3"
+    "snyk": "^1.134.2"
   },
   "devDependencies": {
     "documentation": "^5.4.0",

routes/api/remove.js

@@ -0,0 +1,180 @@
+const averagedTimeSeries = require("averaged-timeseries")
+
+const doleNN = require("./../../lib/dole_nn_base.js")
+
+const Prometheus = require('prom-client');
+
+class neuralDole {
+    
+    getRequestStats(itemname,samples)
+    {
+        var sum=0;
+        samples=Math.min(samples,(this.stats[itemname] || []).length-1);
+        if (samples<1) return 0.1;
+        for (var i = 1; i <= samples; i++) {
+            sum+=this.stats[itemname][i].req;
+        }
+        if (sum==0) return 0.1;
+        return sum/samples;
+    }
+    
+    constructor({
+        items, gaugePrefix
+    }){
+        // Set some storage variables for the dole divider
+        this.prometheusNNDoleGauge = new Prometheus.Gauge({
+            name: gaugePrefix+'nn_dole_gauge',
+            help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
+            labelNames: ["itemName"],
+        });
+        this.items = items
+        this.itemsLastTick = JSON.parse(JSON.stringify(items))
+        this.dole = {}
+        this.carry = {}
+        this.lastRequest = {}
+        this.stats=[]
+
+        setInterval(()=>{
+            for(let name in this.items){
+                // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties.
+                if(typeof this.items[name] !== "number") continue
+
+                let count = this.items[name]
+                let magicData = doleNN.Tick(
+                    count,
+                    this.dole[name],
+                    this.itemsLastTick[name],
+                    this.getRequestStats(name,5)
+                )
+                this.stats[name]=this.stats[name] || []
+                this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
+                if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
+                
+                this.dole[name] = magicData[0]
+                // DONE handle magicData[1] for graphing for our users
+                this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
+            }
+            this.itemsLastTick = JSON.parse(JSON.stringify(this.items))
+        }, 1000)
+    }
+    divider({
+        res,
+        object,
+        config,
+        sentItemStatisticsBySlaveID,
+        prometheusImportGauge
+    }){
+        let magicData = doleNN.Dose(
+            object.count, // numReq
+            this.items[object.name],
+            this.itemsLastTick[object.name] || 0,
+            this.dole[object.name],
+            this.carry[object.name+" "+object.instanceID] || 0,
+            this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
+            this.getRequestStats(object.name,5)
+        )
+        if ((this.stats[object.name] || []).length>0)
+        {
+            this.stats[object.name][0].req+=Number(object.count);
+            this.stats[object.name][0].given+=Number(magicData[0]);
+        }
+        this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
+        //0.Number of items to give in that dose
+        //1.New dole for item X
+        //2.New carry for item X slave Y
+        this.dole[object.name] = magicData[1]
+        this.carry[object.name+" "+object.instanceID] = magicData[2]
+        
+        // Remove item from DB and send it
+        if(this.items.removeItem({count: magicData[0], name: object.name})){
+            if(config.logItemTransfers){
+                console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName);
+            }
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:magicData[0],
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+            prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
+
+            res.send({
+                name: object.name,
+                count: magicData[0],
+            })
+        } else {
+            res.send({
+                name: object.name,
+                count: 0,
+            })
+        }        
+    }
+}
+
+_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
+function doleDivider({
+    item,
+    object,
+    db,
+    sentItemStatisticsBySlaveID,
+    config,
+    prometheusDoleFactorGauge,
+    prometheusImportGauge,
+    req,res,
+}){
+    const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
+    const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
+    
+    const originalCount = Number(object.count) || 0;
+    object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+    object.count = Math.round(object.count);
+    if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`);
+    
+    // Update existing items if item name already exists
+    if(Number(item) > Number(object.count)) {
+        //If successful, increase dole
+        _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+        if(config.logItemTransfers){
+            console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName);
+        }
+        if(db.items.removeItem({count: object.count, name: object.name})){
+            let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID];
+            if(sentItemStatistics === undefined){
+                sentItemStatistics = new averagedTimeSeries({
+                    maxEntries: config.itemStats.maxEntries,
+                    entriesPerSecond: config.itemStats.entriesPerSecond,
+                    mergeMode: "add",
+                }, console.log);
+            }
+            sentItemStatistics.add({
+                key:object.name,
+                value:object.count,
+            });
+            //console.log(sentItemStatistics.data)
+            sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics;
+        }
+        
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0);
+        res.send({count: object.count, name: object.name});
+    } else {
+        // if we didn't have enough, attempt giving out a smaller amount next time
+        _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        res.send({name:object.name, count:0});
+        //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
+    }
+}
+
+module.exports = {
+    doleDivider,
+    neuralDole
+}

sharedPlugins/remoteMap/index.js

@@ -8,7 +8,9 @@ module.exports = class remoteMap {
 		this.socket = extras.socket;
 		this.instances = {};
 		
-        this.socket.on("hello", () => this.socket.emit("registerMapServer"));
+        this.socket.on("hello", () => this.socket.emit("registerMapServer", {
+			instanceID: this.config.unique, // instanceID
+		}));
 
         this.socket.on("createEntity", req => {
 			// req = {name:"", position:{x,y}, ?direction:4}
@@ -29,7 +31,9 @@ module.exports = class remoteMap {
 			this.messageInterface('/silent-command local toDelete = game.surfaces[1].find_entities({{'+req.position.x+','+req.position.y+'},{'+(req.position.x+0.5)+','+(req.position.y+0.5)+'}}) for i, entity in pairs(toDelete) do entity.destroy() end');
         })
 		this.socket.on("getChunk", async (data, callback) => {
-			callback(await this.messageInterface(`/silent-command remote.call("remoteMap", "exportChunk", ${data.x}, ${data.y})`));
+			let chunkData = await this.messageInterface(`/silent-command remote.call("remoteMap", "exportChunk", ${data.x}, ${data.y})`)
+			console.log(`Sending chunk data: ${chunkData}`)
+			callback(chunkData);
 		})
 		
 		// detect new entities created/deleted on the server and update the master

sharedPlugins/remoteMap/masterPlugin.js

@@ -4,6 +4,11 @@ const sanitizer = require('sanitizer');
 
 const editorSockets = [];
 
+const blacklistedEntities = [
+	"fish",
+	"player",
+]
+
 class masterPlugin {
 	constructor({ config, pluginConfig, pluginPath, socketio, express }) {
 		this.config = config;
@@ -24,7 +29,8 @@ class masterPlugin {
 		// handle websocket connections
 		this.io.on("connection", socket => {
 			let instanceID = "unknown";
-			socket.on("registerSlave", data => {
+			socket.on("registerMapServer", data => {
+				console.log("Registered mapServer socket")
 				if (data.instanceID && !isNaN(Number(data.instanceID))) {
 					instanceID = data.instanceID;
 					this.slaves[instanceID] = socket;
@@ -66,12 +72,15 @@ class masterPlugin {
 				}}),10000)
 				
 				socket.on("createEntity", data => {
+					console.log(`Creating entity: ${JSON.stringify(data)}`)
 					this.slaves[Object.keys(this.slaves)[0]].emit("createEntity", data.entity)
 				})
 				socket.on("deleteEntity", data => {
+					console.log(`Deleting entity: ${JSON.stringify(data)}`)
 					this.slaves[Object.keys(this.slaves)[0]].emit("deleteEntity", data.entity)
 				})
 				socket.on("getChunk", (data, callback) => {
+					console.log(`Getting chunk x:${data.x} y:${data.y} from slave`)
 					this.slaves[Object.keys(this.slaves)[0]].emit("getChunk", data, resp => {
 						// parse response
 						// |name:transport-belt,direction:2,x:99,y:1231|name:assembly-machine,direction:6,x:939,y:88
@@ -87,7 +96,7 @@ class masterPlugin {
 							!str.includes("crude-oil") &&
 							!str.includes("remnants") &&
 							!str.includes("cliff") &&
-							!str.includes("player") &&
+							!blacklistedEntities.includes(str) &&
 							!str.includes("ore"))
 							.map(ent => ent.split(",")													// split entity into properties
 								.map(prop => [prop.split(":")[0], prop.split(":")[1]])					// split properties into KV pairs
@@ -102,6 +111,11 @@ class masterPlugin {
 								return entity
 							})
 							.filter(entity => Object.keys(entity).length > 1)
+							.filter(entity => !blacklistedEntities.includes(entity.name))
+							/*.map(entity => {
+								if(entity.name.includes("tree")) entity.name = "tree"
+								return entity
+							})*/
 						)
 					})
 				})

sharedPlugins/scenario_loader/index.js

@@ -12,14 +12,6 @@ module.exports = class remoteCommands {
 			messageInterface("Hotpach installation status: "+hotpatchInstallStatus);
 			if(hotpatchInstallStatus){
 				await this.loadScenariosFromFolder("scenarios") // located in project root folder (with sharedMods, sharedPlugins etc)
-				let plugins = new AsyncArray((await fs.readdir(path.join(__dirname, "../"))))
-				plugins.filterAsync(async plugin => {
-					// check if plugin is a folder and if plugin has a scenarios folder
-					
-				})
-				for(let g = 0; g < plugins.length; g++){
-					
-				}
 			}
 		})();
 	}
@@ -27,7 +19,7 @@ module.exports = class remoteCommands {
 		await fs.ensureDir(scenarioDir)
 		let scenarios = await fs.readdir(scenarioDir);
 		for(let i = 0; i < scenarios.length; i++){
-			messageInterface(`Loading scenario ${scenarios[i]}`)
+			this.messageInterface(`Loading scenario ${scenarios[i]}`)
 			let startTime = Date.now();
 			let scenPath = path.join(scenarioDir, scenarios[i]);
 			let stat = await fs.stat(scenPath);
@@ -68,9 +60,9 @@ module.exports = class remoteCommands {
 					if(name != "control") fileImportString += `["${name}"] = '${files[k]}', `;
 				}
 				fileImportString += `\}`;
-				if(files.control) var returnValue = await messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
+				if(files.control) var returnValue = await this.messageInterface(`/silent-command remote.call('hotpatch', 'update', '${scenarios[i]}', '1.0.0', '${files.control}', ${fileImportString})`);
-				if(returnValue) messageInterface(returnValue);
+				if(returnValue) this.messageInterface(returnValue);
-				messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
+				this.messageInterface(`Loaded scenario ${scenarios[i]} in ${Math.floor(Date.now()-startTime)}ms`);
 			}
 		}
 	}
@@ -108,20 +100,17 @@ function readDirectorySynchronously(directory) {
 }
 class AsyncArray extends Array {
 	constructor(arr) {
-		// this.data = arr; // In place of Array subclassing
+		super(arr)
+		this.data = arr; // In place of Array subclassing
 	}
-
+	async filterAsync(predicate) {
-	filterAsync(predicate) {
 		// Take a copy of the array, it might mutate by the time we've finished
 		const data = Array.from(this.data);
 		// Transform all the elements into an array of promises using the predicate
 		// as the promise
-		return Promise.all(data.map((element, index) => predicate(element, index, data)))
+		const result = await Promise.all(data.map((element, index) => predicate(element, index, data)));
-		// Use the result of the promises to call the underlying sync filter function
-		.then(result => {
 		return data.filter((element, index) => {
 			return result[index];
 		});
-		});
 	}
 }

tsconfig.json

@@ -1,23 +0,0 @@
-{
-    "compilerOptions": {
-        "module": "commonjs",
-        "esModuleInterop": false,
-        "target": "es6",
-        "noImplicitAny": false,
-        "moduleResolution": "node",
-        "sourceMap": true,
-        "outDir": "dist",
-        "baseUrl": ".",
-        "paths": {
-            "*": [
-                "node_modules/",
-                "src/types/*",
-                "master.d.ts",
-                "types/*"
-            ]
-        }
-    },
-    "include": [
-        "**/*"
-    ]
-}

types/ClusterioConfig.d.ts

@@ -1,39 +0,0 @@
-type ClusterioConfig = {
-    factorioDirectory: string,
-    instanceDirectory: string,
-    databaseDirectory: string,
-    publicIP: string,
-    masterIP: string,
-    masterPort: number,
-    sslPort: number,
-    sslCert: string,
-    sslPrivKey: string,
-    masterAuthToken: string,
-    masterAuthSecret: string,
-    username: string,
-    token: string,
-    factorio_version: string,
-    verify_user_identity: boolean,
-    auto_pause: boolean,
-    allow_commands: "admins-only" | "true" | "false",
-    game_password: string,
-    description: string,
-    visibility: {
-        public: boolean,
-        lan: boolean
-    },
-    itemStats: {
-        maxEntries: number,
-        entriesPerSecond: number
-    },
-    logItemTransfers: boolean,
-    disableFairItemDistribution: boolean,
-    uploadModsToMaster: boolean,
-    msBetweenCommands: 10,
-    allowRemoteCommandExecution: boolean,
-    enableCrossServerShout: boolean,
-    mirrorAllChat: boolean,
-    autosaveInterval: 600000,
-    disablePrometheusPushgateway: boolean,
-    disableImportsOfEverythingExceptElectricity: boolean
-}

types/Mod.d.ts

@@ -1,4 +0,0 @@
-type Mod = {
-    modName: string,
-    hash: string,
-}

types/Request.d.ts

@@ -1,9 +0,0 @@
-interface Request {
-       route?: any,
-       body?: requestBody,
-    }
-    interface requestBody extends Body {
-        unique: string,
-        
-    }
- 

types/Slave.d.ts

@@ -1,15 +0,0 @@
-interface Slave {
-    time?: string,
-    rconPort?: string,
-    rconPassword?: string,
-    serverPort?: string,
-    unique?: string, // this one isn't really optional, but I got tired of fighting with types and rewriting logic in master.ts
-    publicIP?: string,
-    mods?: Mod[],
-    instanceName?: string,
-    playerCount?: string | number,
-    mac?: string,
-    meta?: {
-        [name:string]:any,
-    },
-}

types/Slaves.d.ts

@@ -1,3 +0,0 @@
-type Slaves = {
-    [name: string]: Slave,
-}

types/item.d.ts

@@ -1,6 +0,0 @@
-type item = {
-    name: string,
-    count: number,
-    instanceName?: string,
-    instanceID?: string,
-}

types/master.d.ts

@@ -1,16 +0,0 @@
-type MasterPluginInstance = {
-    config: object,
-    pluginConfig: object,
-    path: string,
-    socketio: any,
-    express: any,
-    Prometheus: any,
-    onExit?: Function,
-}
-type pluginConfig = {
-    name: string,
-}
-interface MasterPlugin {
-    main: MasterPluginInstance,
-    pluginConfig: pluginConfig,
-}

types/slaveMapper.d.ts

@@ -1,3 +0,0 @@
-interface slaveMapper {
-    instanceID: string,
-}